Repository: tensorflow/examples Branch: master Commit: 5ae5bbf6b9bc Files: 1881 Total size: 9.8 MB Directory structure: gitextract_a1tly3__/ ├── .gitignore ├── AUTHORS ├── CODEOWNERS ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── __init__.py ├── courses/ │ ├── udacity_deep_learning/ │ │ ├── .gitignore │ │ ├── 1_notmnist.ipynb │ │ ├── 2_fullyconnected.ipynb │ │ ├── 3_regularization.ipynb │ │ ├── 4_convolutions.ipynb │ │ ├── 5_word2vec.ipynb │ │ ├── 6_lstm.ipynb │ │ ├── Dockerfile │ │ └── README.md │ ├── udacity_intro_to_tensorflow_for_deep_learning/ │ │ ├── l01c01_introduction_to_colab_and_python.ipynb │ │ ├── l02c01_celsius_to_fahrenheit.ipynb │ │ ├── l03c01_classifying_images_of_clothing.ipynb │ │ ├── l04c01_image_classification_with_cnns.ipynb │ │ ├── l05c01_dogs_vs_cats_without_augmentation.ipynb │ │ ├── l05c02_dogs_vs_cats_with_augmentation.ipynb │ │ ├── l05c03_exercise_flowers_with_data_augmentation.ipynb │ │ ├── l05c04_exercise_flowers_with_data_augmentation_solution.ipynb │ │ ├── l06c01_tensorflow_hub_and_transfer_learning.ipynb │ │ ├── l06c02_exercise_flowers_with_transfer_learning.ipynb │ │ ├── l06c03_exercise_flowers_with_transfer_learning_solution.ipynb │ │ ├── l07c01_saving_and_loading_models.ipynb │ │ ├── l08c01_common_patterns.ipynb │ │ ├── l08c02_naive_forecasting.ipynb │ │ ├── l08c03_moving_average.ipynb │ │ ├── l08c04_time_windows.ipynb │ │ ├── l08c05_forecasting_with_machine_learning.ipynb │ │ ├── l08c06_forecasting_with_rnn.ipynb │ │ ├── l08c07_forecasting_with_stateful_rnn.ipynb │ │ ├── l08c08_forecasting_with_lstm.ipynb │ │ ├── l08c09_forecasting_with_cnn.ipynb │ │ ├── l09c01_nlp_turn_words_into_tokens.ipynb │ │ ├── l09c02_nlp_padding.ipynb │ │ ├── l09c03_nlp_prepare_larger_text_corpus.ipynb │ │ ├── l09c04_nlp_embeddings_and_sentiment.ipynb │ │ ├── l09c05_nlp_tweaking_the_model.ipynb │ │ ├── l09c06_nlp_subwords.ipynb │ │ ├── l10c01_nlp_lstms_with_reviews_subwords_dataset.ipynb │ │ ├── l10c02_nlp_multiple_models_for_predicting_sentiment.ipynb │ │ ├── l10c03_nlp_constructing_text_generation_model.ipynb │ │ └── l10c04_nlp_optimizing_the_text_generation_model.ipynb │ └── udacity_intro_to_tensorflow_lite/ │ ├── tflite_c01_linear_regression.ipynb │ ├── tflite_c02_transfer_learning.ipynb │ ├── tflite_c03_exercise_convert_model_to_tflite.ipynb │ ├── tflite_c04_exercise_convert_model_to_tflite_solution.ipynb │ ├── tflite_c05_exercise_rock_paper_scissors.ipynb │ └── tflite_c06_exercise_rock_paper_scissors_solution.ipynb ├── lite/ │ ├── .gitignore │ ├── README.md │ ├── codelabs/ │ │ ├── digit_classifier/ │ │ │ ├── README.md │ │ │ ├── android/ │ │ │ │ ├── finish/ │ │ │ │ │ ├── app/ │ │ │ │ │ │ ├── build.gradle │ │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ │ └── src/ │ │ │ │ │ │ └── main/ │ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ │ ├── assets/ │ │ │ │ │ │ │ └── ADD_TFLITE_MODEL_HERE │ │ │ │ │ │ ├── java/ │ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ │ └── codelabs/ │ │ │ │ │ │ │ └── digitclassifier/ │ │ │ │ │ │ │ ├── DigitClassifier.kt │ │ │ │ │ │ │ └── MainActivity.kt │ │ │ │ │ │ └── res/ │ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ │ └── ic_launcher_background.xml │ │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ │ ├── layout/ │ │ │ │ │ │ │ └── activity_main.xml │ │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ │ └── values/ │ │ │ │ │ │ ├── colors.xml │ │ │ │ │ │ ├── strings.xml │ │ │ │ │ │ └── styles.xml │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── gradle/ │ │ │ │ │ │ └── wrapper/ │ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ │ ├── gradle.properties │ │ │ │ │ ├── gradlew │ │ │ │ │ ├── gradlew.bat │ │ │ │ │ └── settings.gradle │ │ │ │ └── start/ │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── assets/ │ │ │ │ │ │ └── ADD_TFLITE_MODEL_HERE │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── codelabs/ │ │ │ │ │ │ └── digitclassifier/ │ │ │ │ │ │ ├── DigitClassifier.kt │ │ │ │ │ │ └── MainActivity.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ └── ic_launcher_background.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ └── activity_main.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ └── ml/ │ │ │ ├── step2_train_ml_model.ipynb │ │ │ └── step7_improve_accuracy.ipynb │ │ └── flower_classification/ │ │ ├── README.md │ │ ├── android/ │ │ │ ├── finish/ │ │ │ │ ├── .gitignore │ │ │ │ ├── app/ │ │ │ │ │ ├── .gitignore │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── classification/ │ │ │ │ │ │ └── ClassifierTest.java │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── assets/ │ │ │ │ │ │ ├── ADD_TFLITE_MODEL_HERE │ │ │ │ │ │ └── labels.txt │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── classification/ │ │ │ │ │ │ ├── CameraActivity.java │ │ │ │ │ │ ├── CameraConnectionFragment.java │ │ │ │ │ │ ├── ClassifierActivity.java │ │ │ │ │ │ ├── LegacyCameraConnectionFragment.java │ │ │ │ │ │ ├── customview/ │ │ │ │ │ │ │ ├── AutoFitTextureView.java │ │ │ │ │ │ │ ├── OverlayView.java │ │ │ │ │ │ │ ├── RecognitionScoreView.java │ │ │ │ │ │ │ └── ResultsView.java │ │ │ │ │ │ ├── env/ │ │ │ │ │ │ │ ├── BorderedText.java │ │ │ │ │ │ │ ├── ImageUtils.java │ │ │ │ │ │ │ └── Logger.java │ │ │ │ │ │ └── tflite/ │ │ │ │ │ │ ├── Classifier.java │ │ │ │ │ │ └── ClassifierFloatMobileNet.java │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── bottom_sheet_bg.xml │ │ │ │ │ │ ├── ic_baseline_add.xml │ │ │ │ │ │ ├── ic_baseline_remove.xml │ │ │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ │ │ └── rectangle.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── tfe_ic_activity_camera.xml │ │ │ │ │ │ ├── tfe_ic_camera_connection_fragment.xml │ │ │ │ │ │ └── tfe_ic_layout_bottom_sheet.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ └── start/ │ │ │ ├── .gitignore │ │ │ ├── app/ │ │ │ │ ├── .gitignore │ │ │ │ ├── build.gradle │ │ │ │ ├── proguard-rules.pro │ │ │ │ └── src/ │ │ │ │ ├── androidTest/ │ │ │ │ │ └── java/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── classification/ │ │ │ │ │ └── ClassifierTest.java │ │ │ │ └── main/ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ ├── assets/ │ │ │ │ │ └── ADD_TFLITE_MODEL_HERE │ │ │ │ ├── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── classification/ │ │ │ │ │ ├── CameraActivity.java │ │ │ │ │ ├── CameraConnectionFragment.java │ │ │ │ │ ├── ClassifierActivity.java │ │ │ │ │ ├── LegacyCameraConnectionFragment.java │ │ │ │ │ ├── customview/ │ │ │ │ │ │ ├── AutoFitTextureView.java │ │ │ │ │ │ ├── OverlayView.java │ │ │ │ │ │ ├── RecognitionScoreView.java │ │ │ │ │ │ └── ResultsView.java │ │ │ │ │ ├── env/ │ │ │ │ │ │ ├── BorderedText.java │ │ │ │ │ │ ├── ImageUtils.java │ │ │ │ │ │ └── Logger.java │ │ │ │ │ └── tflite/ │ │ │ │ │ ├── Classifier.java │ │ │ │ │ └── ClassifierFloatMobileNet.java │ │ │ │ └── res/ │ │ │ │ ├── drawable/ │ │ │ │ │ ├── bottom_sheet_bg.xml │ │ │ │ │ ├── ic_baseline_add.xml │ │ │ │ │ ├── ic_baseline_remove.xml │ │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ │ └── rectangle.xml │ │ │ │ ├── drawable-v24/ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ ├── layout/ │ │ │ │ │ ├── tfe_ic_activity_camera.xml │ │ │ │ │ ├── tfe_ic_camera_connection_fragment.xml │ │ │ │ │ └── tfe_ic_layout_bottom_sheet.xml │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ └── values/ │ │ │ │ ├── colors.xml │ │ │ │ ├── dimens.xml │ │ │ │ ├── strings.xml │ │ │ │ └── styles.xml │ │ │ ├── build.gradle │ │ │ ├── gradle/ │ │ │ │ └── wrapper/ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ │ ├── gradle.properties │ │ │ ├── gradlew │ │ │ ├── gradlew.bat │ │ │ └── settings.gradle │ │ └── ml/ │ │ └── Flower_Classification_with_TFLite_Model_Maker.ipynb │ ├── examples/ │ │ ├── acceleration_service/ │ │ │ └── android_play_services/ │ │ │ ├── app/ │ │ │ │ ├── build.gradle │ │ │ │ ├── download.gradle │ │ │ │ ├── proguard-rules.pro │ │ │ │ └── src/ │ │ │ │ ├── androidTest/ │ │ │ │ │ └── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── accelerationservice/ │ │ │ │ │ ├── CustomValidationTest.java │ │ │ │ │ └── NoopLogger.java │ │ │ │ └── main/ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ ├── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── accelerationservice/ │ │ │ │ │ ├── MainActivity.java │ │ │ │ │ ├── logger/ │ │ │ │ │ │ ├── Logger.java │ │ │ │ │ │ └── TextViewLogger.java │ │ │ │ │ ├── model/ │ │ │ │ │ │ ├── AssetModel.java │ │ │ │ │ │ ├── AssetModelFactory.java │ │ │ │ │ │ ├── MobileNetV1.java │ │ │ │ │ │ └── PlainAddition.java │ │ │ │ │ └── validator/ │ │ │ │ │ └── MeanSquaredErrorValidator.java │ │ │ │ └── res/ │ │ │ │ ├── layout/ │ │ │ │ │ └── activity_main.xml │ │ │ │ └── values/ │ │ │ │ ├── colors.xml │ │ │ │ ├── strings.xml │ │ │ │ └── styles.xml │ │ │ ├── build.gradle │ │ │ ├── gradle/ │ │ │ │ └── wrapper/ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ │ ├── gradle.properties │ │ │ ├── gradlew │ │ │ ├── gradlew.bat │ │ │ ├── proguard.pgcfg │ │ │ └── settings.gradle │ │ ├── audio_classification/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_model.gradle │ │ │ │ │ └── src/ │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── audio/ │ │ │ │ │ │ ├── AudioClassificationHelper.kt │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ ├── fragments/ │ │ │ │ │ │ │ ├── AudioFragment.kt │ │ │ │ │ │ │ └── PermissionsFragment.kt │ │ │ │ │ │ └── ui/ │ │ │ │ │ │ └── ProbabilitiesAdapter.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── controls_bottom_sheet.xml │ │ │ │ │ │ ├── fragment_audio.xml │ │ │ │ │ │ └── item_probability.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ ├── ios/ │ │ │ │ ├── AudioClassification/ │ │ │ │ │ ├── AppDelegate.swift │ │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ │ ├── AccentColor.colorset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── Contents.json │ │ │ │ │ │ ├── icn_chevron_down.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── icn_chevron_up.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ └── tfl_logo.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── Base.lproj/ │ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ │ └── Main.storyboard │ │ │ │ │ ├── HomeViewController.swift │ │ │ │ │ ├── InferenceView.swift │ │ │ │ │ ├── Info.plist │ │ │ │ │ ├── ResultTableViewCell.swift │ │ │ │ │ └── TFLite/ │ │ │ │ │ └── AudioClassificationHelper.swift │ │ │ │ ├── AudioClassification.xcodeproj/ │ │ │ │ │ └── project.pbxproj │ │ │ │ ├── Podfile │ │ │ │ ├── README.md │ │ │ │ └── RunScripts/ │ │ │ │ └── download_models.sh │ │ │ └── raspberry_pi/ │ │ │ ├── README.md │ │ │ ├── classify.py │ │ │ ├── requirements.txt │ │ │ ├── setup.sh │ │ │ ├── test_data/ │ │ │ │ └── ground_truth.csv │ │ │ └── utils.py │ │ ├── bert_qa/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_models.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── bertqa/ │ │ │ │ │ │ └── BertQaHelperTest.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── bertqa/ │ │ │ │ │ │ ├── BertQaHelper.kt │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ ├── dataset/ │ │ │ │ │ │ │ ├── DataSet.kt │ │ │ │ │ │ │ └── LoadDataSetClient.kt │ │ │ │ │ │ └── fragments/ │ │ │ │ │ │ ├── DatasetAdapter.kt │ │ │ │ │ │ ├── DatasetFragment.kt │ │ │ │ │ │ ├── QaAdapter.kt │ │ │ │ │ │ └── QaFragment.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── bg_question_items.xml │ │ │ │ │ │ ├── ic_ask_active.xml │ │ │ │ │ │ ├── ic_ask_inactive.xml │ │ │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── fragment_dataset.xml │ │ │ │ │ │ ├── fragment_qa.xml │ │ │ │ │ │ ├── info_bottom_sheet.xml │ │ │ │ │ │ ├── item_dataset.xml │ │ │ │ │ │ └── item_question.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ └── ios/ │ │ │ ├── .gitignore │ │ │ ├── Assets.xcassets/ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ └── Contents.json │ │ │ │ └── Contents.json │ │ │ ├── BertQA.xcodeproj/ │ │ │ │ └── project.pbxproj │ │ │ ├── BertQACore/ │ │ │ │ ├── Constants.swift │ │ │ │ ├── Extensions/ │ │ │ │ │ ├── DataExtension.swift │ │ │ │ │ ├── StringExtension.swift │ │ │ │ │ └── UnicodeScalarExtension.swift │ │ │ │ └── Models/ │ │ │ │ ├── Dataset.swift │ │ │ │ ├── FileLoader.swift │ │ │ │ ├── ML/ │ │ │ │ │ ├── BertQAHandler.swift │ │ │ │ │ ├── ContentData.swift │ │ │ │ │ ├── InputFeatures.swift │ │ │ │ │ └── Result.swift │ │ │ │ └── Tokenizers/ │ │ │ │ ├── BasicTokenizer.swift │ │ │ │ ├── FullTokenizer.swift │ │ │ │ └── WordpieceTokenizer.swift │ │ │ ├── BertQATests/ │ │ │ │ ├── ExtensionTest/ │ │ │ │ │ ├── StringExtensionTest.swift │ │ │ │ │ └── UnicodeScalarExtensionTest.swift │ │ │ │ ├── Info.plist │ │ │ │ ├── MLTest/ │ │ │ │ │ └── BertQAHandlerTest.swift │ │ │ │ └── TokenizerTest/ │ │ │ │ ├── BasicTokenizerTest.swift │ │ │ │ ├── FullTokenizerTest.swift │ │ │ │ └── WordpieceTokenizerTest.swift │ │ │ ├── Podfile │ │ │ ├── README.md │ │ │ ├── RunScripts/ │ │ │ │ └── download_resources.sh │ │ │ ├── ViewInSwiftUI/ │ │ │ │ ├── AppDelegate.swift │ │ │ │ ├── Base.lproj/ │ │ │ │ │ └── LaunchScreen.storyboard │ │ │ │ ├── Info.plist │ │ │ │ ├── KeyboardHeightObserver.swift │ │ │ │ ├── SceneDelegate.swift │ │ │ │ └── Views/ │ │ │ │ ├── ContentView.swift │ │ │ │ ├── DatasetDetailView.swift │ │ │ │ ├── DatasetListView.swift │ │ │ │ ├── StatusView.swift │ │ │ │ └── SuggestedQuestionsView.swift │ │ │ └── ViewInUIKit/ │ │ │ ├── AppDelegate.swift │ │ │ ├── Base.lproj/ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ └── Main.storyboard │ │ │ ├── Controllers/ │ │ │ │ ├── DataSetDetailViewController.swift │ │ │ │ ├── DataSetsTableViewController.swift │ │ │ │ ├── OptionTableViewController.swift │ │ │ │ └── OptionViewController.swift │ │ │ ├── Info.plist │ │ │ └── Views/ │ │ │ ├── ControlPanelView.swift │ │ │ ├── DataSetTitleCell.swift │ │ │ └── SuggestedQuestionButton.swift │ │ ├── classification_by_retrieval/ │ │ │ ├── .bazelrc │ │ │ ├── .bazelversion │ │ │ ├── README.md │ │ │ ├── WORKSPACE │ │ │ ├── ios/ │ │ │ │ ├── BUILD │ │ │ │ ├── ImageClassifierBuilder/ │ │ │ │ │ ├── Album.swift │ │ │ │ │ ├── AlbumSelector.swift │ │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── CameraView.swift │ │ │ │ │ ├── CameraViewController.swift │ │ │ │ │ ├── Classifier.h │ │ │ │ │ ├── Classifier.mm │ │ │ │ │ ├── Field.swift │ │ │ │ │ ├── FileManager+Additions.swift │ │ │ │ │ ├── ImageClassifierBuilderApp.swift │ │ │ │ │ ├── Info.plist │ │ │ │ │ ├── Model.swift │ │ │ │ │ ├── ModelCreationView.swift │ │ │ │ │ ├── ModelData.swift │ │ │ │ │ ├── ModelInfoView.swift │ │ │ │ │ ├── ModelList.swift │ │ │ │ │ ├── ModelMetadata.swift │ │ │ │ │ ├── ModelTrainer.swift │ │ │ │ │ ├── ModelTrainerView.swift │ │ │ │ │ ├── ModelTrainingUtils.h │ │ │ │ │ ├── ModelTrainingUtils.mm │ │ │ │ │ ├── ModelVisualizer.swift │ │ │ │ │ ├── NSData+PixelBuffer.h │ │ │ │ │ ├── NSData+PixelBuffer.m │ │ │ │ │ ├── NSString+AbseilStringView.h │ │ │ │ │ ├── NSString+AbseilStringView.mm │ │ │ │ │ ├── OnBoardingView.swift │ │ │ │ │ ├── RuntimeError.swift │ │ │ │ │ ├── ShareSheet.swift │ │ │ │ │ ├── String+Identifiable.swift │ │ │ │ │ ├── UIImage+CoreVideo.h │ │ │ │ │ ├── UIImage+CoreVideo.mm │ │ │ │ │ ├── UINavigationController+Additions.swift │ │ │ │ │ └── VisualEffectView.swift │ │ │ │ └── README.md │ │ │ └── lib/ │ │ │ ├── BUILD │ │ │ ├── labeled_image_helper.cc │ │ │ ├── labeled_image_helper.h │ │ │ ├── model_builder.cc │ │ │ ├── model_builder.h │ │ │ ├── tflite_builder.cc │ │ │ ├── tflite_builder.h │ │ │ ├── tflite_cbr_builder.cc │ │ │ └── tflite_cbr_builder.h │ │ ├── digit_classifier/ │ │ │ ├── README.md │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_models.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── digitclassification/ │ │ │ │ │ │ └── DigitClassificationTest.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── digitclassification/ │ │ │ │ │ │ ├── DigitClassifierHelper.kt │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ └── fragments/ │ │ │ │ │ │ ├── ClassificationResultsAdapter.kt │ │ │ │ │ │ └── DigitCanvasFragment.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── fragment_digit_canvas.xml │ │ │ │ │ │ ├── info_bottom_sheet.xml │ │ │ │ │ │ └── item_classification_result.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ ├── ios/ │ │ │ │ ├── DigitClassifier/ │ │ │ │ │ ├── AppDelegate.swift │ │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── Base.lproj/ │ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ │ └── Main.storyboard │ │ │ │ │ ├── DigitClassifier.swift │ │ │ │ │ ├── Info.plist │ │ │ │ │ ├── TFLiteExtensions.swift │ │ │ │ │ └── ViewController.swift │ │ │ │ ├── DigitClassifier.xcodeproj/ │ │ │ │ │ └── project.pbxproj │ │ │ │ ├── Podfile │ │ │ │ └── README.md │ │ │ └── ml/ │ │ │ └── mnist_tflite.ipynb │ │ ├── generative_ai/ │ │ │ └── android/ │ │ │ ├── README.md │ │ │ ├── app/ │ │ │ │ ├── build.gradle.kts │ │ │ │ ├── download.gradle │ │ │ │ ├── libs/ │ │ │ │ │ ├── .gitignore │ │ │ │ │ └── build_aar/ │ │ │ │ │ ├── README.md │ │ │ │ │ ├── build_aar.sh │ │ │ │ │ └── tftext-2.12.patch │ │ │ │ ├── proguard-rules.pro │ │ │ │ └── src/ │ │ │ │ ├── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── com/ │ │ │ │ │ │ └── google/ │ │ │ │ │ │ └── tensorflowdemo/ │ │ │ │ │ │ ├── DemoApplication.kt │ │ │ │ │ │ ├── data/ │ │ │ │ │ │ │ └── autocomplete/ │ │ │ │ │ │ │ └── AutoCompleteService.kt │ │ │ │ │ │ ├── di/ │ │ │ │ │ │ │ ├── appModule.kt │ │ │ │ │ │ │ └── viewmodelModule.kt │ │ │ │ │ │ ├── ui/ │ │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ │ ├── components/ │ │ │ │ │ │ │ │ └── HeaderBar.kt │ │ │ │ │ │ │ ├── screens/ │ │ │ │ │ │ │ │ └── autocomplete/ │ │ │ │ │ │ │ │ ├── AutoCompleteScreen.kt │ │ │ │ │ │ │ │ ├── AutoCompleteViewModel.kt │ │ │ │ │ │ │ │ └── components/ │ │ │ │ │ │ │ │ ├── AutoCompleteInfo.kt │ │ │ │ │ │ │ │ ├── AutoCompleteTextField.kt │ │ │ │ │ │ │ │ ├── TextControlBar.kt │ │ │ │ │ │ │ │ └── WindowSizeSelection.kt │ │ │ │ │ │ │ └── theme/ │ │ │ │ │ │ │ ├── Color.kt │ │ │ │ │ │ │ ├── Shape.kt │ │ │ │ │ │ │ ├── Theme.kt │ │ │ │ │ │ │ └── Type.kt │ │ │ │ │ │ └── util/ │ │ │ │ │ │ └── StringExt.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── header_background.xml │ │ │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── values/ │ │ │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ │ │ ├── strings.xml │ │ │ │ │ │ └── themes.xml │ │ │ │ │ └── xml/ │ │ │ │ │ ├── backup_rules.xml │ │ │ │ │ └── data_extraction_rules.xml │ │ │ │ └── test/ │ │ │ │ └── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── tensorflowdemo/ │ │ │ │ └── util/ │ │ │ │ └── StringExtKtTest.kt │ │ │ ├── build.gradle.kts │ │ │ ├── gradle/ │ │ │ │ └── libs.versions.toml │ │ │ ├── gradle.properties │ │ │ ├── how-to-build.md │ │ │ ├── ml/ │ │ │ │ └── README.md │ │ │ └── settings.gradle.kts │ │ ├── gesture_classification/ │ │ │ ├── README.md │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── adding_metadata_to_tflite_model_colab_notebook.ipynb │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_models.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── gestureclassification/ │ │ │ │ │ │ └── GestureClassificationTest.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── gestureclassification/ │ │ │ │ │ │ ├── GestureClassifierHelper.kt │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ └── fragments/ │ │ │ │ │ │ ├── CameraFragment.kt │ │ │ │ │ │ ├── ClassificationResultsAdapter.kt │ │ │ │ │ │ └── PermissionsFragment.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── color/ │ │ │ │ │ │ └── selector_ic.xml │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── fragment_camera.xml │ │ │ │ │ │ ├── info_bottom_sheet.xml │ │ │ │ │ │ └── item_classification_result.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ ├── ios/ │ │ │ │ ├── .gitignore │ │ │ │ ├── GestureClassification/ │ │ │ │ │ ├── AppDelegate.swift │ │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── Contents.json │ │ │ │ │ │ ├── down.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── down_icon.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── left.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── left_click.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── right.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── right_click.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── scroll_down.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── scroll_up.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── selection_base.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── selection_base_default.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── tfl_logo.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── up.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ └── up_icon.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── Base.lproj/ │ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ │ └── Main.storyboard │ │ │ │ │ ├── Camera Feed/ │ │ │ │ │ │ ├── CameraFeedManager.swift │ │ │ │ │ │ └── PreviewView.swift │ │ │ │ │ ├── Cells/ │ │ │ │ │ │ ├── GestureCollectionViewCell.swift │ │ │ │ │ │ └── InfoCell.swift │ │ │ │ │ ├── Extension/ │ │ │ │ │ │ └── StringExtension.swift │ │ │ │ │ ├── Info.plist │ │ │ │ │ ├── Model/ │ │ │ │ │ │ └── display_labels.txt │ │ │ │ │ ├── ModelDataHandler/ │ │ │ │ │ │ └── ModelDataHandler.swift │ │ │ │ │ ├── ViewControllers/ │ │ │ │ │ │ ├── InferenceViewController.swift │ │ │ │ │ │ └── ViewController.swift │ │ │ │ │ └── Views/ │ │ │ │ │ └── CurvedView.swift │ │ │ │ ├── GestureClassification.xcodeproj/ │ │ │ │ │ └── project.pbxproj │ │ │ │ ├── Podfile │ │ │ │ ├── README.md │ │ │ │ └── RunScripts/ │ │ │ │ └── download_model.sh │ │ │ ├── ml/ │ │ │ │ ├── .gitignore │ │ │ │ ├── README.md │ │ │ │ └── tensorflowjs_to_tflite_colab_notebook.ipynb │ │ │ └── web/ │ │ │ ├── .gitignore │ │ │ ├── README.md │ │ │ ├── controller_dataset.js │ │ │ ├── index.html │ │ │ ├── index.js │ │ │ ├── style.css │ │ │ ├── ui.js │ │ │ └── webcam.js │ │ ├── image_classification/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_models.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── imageclassification/ │ │ │ │ │ │ └── ImageClassificationTest.kt │ │ │ │ │ ├── main/ │ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ │ ├── java/ │ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ │ └── imageclassification/ │ │ │ │ │ │ │ ├── ImageClassifierHelper.kt │ │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ │ └── fragments/ │ │ │ │ │ │ │ ├── CameraFragment.kt │ │ │ │ │ │ │ ├── ClassificationResultsAdapter.kt │ │ │ │ │ │ │ └── PermissionsFragment.kt │ │ │ │ │ │ └── res/ │ │ │ │ │ │ ├── color/ │ │ │ │ │ │ │ └── selector_ic.xml │ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ │ ├── layout/ │ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ │ ├── fragment_camera.xml │ │ │ │ │ │ │ ├── info_bottom_sheet.xml │ │ │ │ │ │ │ └── item_classification_result.xml │ │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ │ └── values/ │ │ │ │ │ │ ├── colors.xml │ │ │ │ │ │ ├── dimens.xml │ │ │ │ │ │ ├── strings.xml │ │ │ │ │ │ └── styles.xml │ │ │ │ │ └── test/ │ │ │ │ │ └── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── imageclassification/ │ │ │ │ │ └── ExampleUnitTest.kt │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ ├── android_java/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_models.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── imageclassification/ │ │ │ │ │ │ └── ImageClassificationTest.java │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── imageclassification/ │ │ │ │ │ │ ├── ImageClassifierHelper.java │ │ │ │ │ │ ├── MainActivity.java │ │ │ │ │ │ └── fragments/ │ │ │ │ │ │ ├── CameraFragment.java │ │ │ │ │ │ ├── ClassificationResultAdapter.java │ │ │ │ │ │ └── PermissionsFragment.java │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── fragment_camera.xml │ │ │ │ │ │ ├── info_bottom_sheet.xml │ │ │ │ │ │ └── item_classification_result.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ ├── android_play_services/ │ │ │ │ ├── .google/ │ │ │ │ │ └── packaging.yaml │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ │ └── kotlin/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── classification/ │ │ │ │ │ │ └── playservices/ │ │ │ │ │ │ └── InstrumentationTest.kt │ │ │ │ │ ├── androidTestJava/ │ │ │ │ │ │ └── kotlin/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── classification/ │ │ │ │ │ │ └── playservices/ │ │ │ │ │ │ └── TestUtil.kt │ │ │ │ │ ├── androidTestKotlin/ │ │ │ │ │ │ └── kotlin/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── classification/ │ │ │ │ │ │ └── playservices/ │ │ │ │ │ │ └── TestUtil.kt │ │ │ │ │ ├── java/ │ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── classification/ │ │ │ │ │ │ └── playservices/ │ │ │ │ │ │ ├── CameraActivity.java │ │ │ │ │ │ └── ImageClassificationHelper.java │ │ │ │ │ ├── kotlin/ │ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ │ └── kotlin/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── classification/ │ │ │ │ │ │ └── playservices/ │ │ │ │ │ │ ├── CameraActivity.kt │ │ │ │ │ │ └── ImageClassificationHelper.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── assets/ │ │ │ │ │ │ └── labels_without_background.txt │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ │ │ ├── ic_shutter.xml │ │ │ │ │ │ ├── ic_shutter_focused.xml │ │ │ │ │ │ ├── ic_shutter_normal.xml │ │ │ │ │ │ ├── ic_shutter_pressed.xml │ │ │ │ │ │ └── shape_rectangle.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ └── activity_camera.xml │ │ │ │ │ ├── layout-land/ │ │ │ │ │ │ └── activity_camera.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ ├── ios/ │ │ │ │ ├── ImageClassification/ │ │ │ │ │ ├── AppDelegate.swift │ │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── Contents.json │ │ │ │ │ │ ├── darkOrLight.colorset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── down_icon.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── tfl_logo.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ └── up_icon.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── Camera Feed/ │ │ │ │ │ │ ├── CameraFeedManager.swift │ │ │ │ │ │ └── PreviewView.swift │ │ │ │ │ ├── Info.plist │ │ │ │ │ ├── Storyboards/ │ │ │ │ │ │ └── Base.lproj/ │ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ │ └── Main.storyboard │ │ │ │ │ ├── TFLite/ │ │ │ │ │ │ └── ImageClassificationHelper.swift │ │ │ │ │ └── ViewControllers/ │ │ │ │ │ ├── InferenceViewController.swift │ │ │ │ │ └── ViewController.swift │ │ │ │ ├── ImageClassification.xcodeproj/ │ │ │ │ │ └── project.pbxproj │ │ │ │ ├── Podfile │ │ │ │ ├── README.md │ │ │ │ └── RunScripts/ │ │ │ │ └── download_models.sh │ │ │ ├── metadata/ │ │ │ │ ├── README.md │ │ │ │ ├── metadata_writer_for_image_classifier.py │ │ │ │ └── requirements.txt │ │ │ └── raspberry_pi/ │ │ │ ├── README.md │ │ │ ├── classify.py │ │ │ ├── requirements.txt │ │ │ ├── setup.sh │ │ │ └── test_data/ │ │ │ └── ground_truth.csv │ │ ├── image_segmentation/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_models.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── imagesegmentation/ │ │ │ │ │ │ └── ImageSegmentationHelperTest.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── imagesegmentation/ │ │ │ │ │ │ ├── ImageSegmentationHelper.kt │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ ├── OverlayView.kt │ │ │ │ │ │ └── fragments/ │ │ │ │ │ │ ├── CameraFragment.kt │ │ │ │ │ │ ├── ColorLabelsAdapter.kt │ │ │ │ │ │ └── PermissionsFragment.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── color/ │ │ │ │ │ │ └── selector_ic.xml │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── bg_color_labels.xml │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── fragment_camera.xml │ │ │ │ │ │ ├── info_bottom_sheet.xml │ │ │ │ │ │ └── item_color_labels.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ ├── ios/ │ │ │ │ ├── ImageSegmentation/ │ │ │ │ │ ├── AppDelegate.swift │ │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── Contents.json │ │ │ │ │ │ ├── photo_camera.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ └── photo_library.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── Base.lproj/ │ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ │ └── Main.storyboard │ │ │ │ │ ├── Info.plist │ │ │ │ │ ├── TFLite/ │ │ │ │ │ │ ├── ImageSegmentationHelper.swift │ │ │ │ │ │ └── TFLiteExtension.swift │ │ │ │ │ ├── UIKitExtension.swift │ │ │ │ │ └── ViewController.swift │ │ │ │ ├── ImageSegmentation.xcodeproj/ │ │ │ │ │ ├── project.pbxproj │ │ │ │ │ └── xcshareddata/ │ │ │ │ │ └── xcschemes/ │ │ │ │ │ └── ImageSegmentation.xcscheme │ │ │ │ ├── Podfile │ │ │ │ └── README.md │ │ │ └── raspberry_pi/ │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ ├── segment.py │ │ │ ├── setup.sh │ │ │ ├── test_data/ │ │ │ │ └── ground_truth_label.txt │ │ │ └── utils.py │ │ ├── model_personalization/ │ │ │ ├── README.md │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_models.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── modelpersonalization/ │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ ├── MainViewModel.kt │ │ │ │ │ │ ├── TransferLearningHelper.kt │ │ │ │ │ │ └── fragments/ │ │ │ │ │ │ ├── CameraFragment.kt │ │ │ │ │ │ ├── HelperDialog.kt │ │ │ │ │ │ ├── PermissionsFragment.kt │ │ │ │ │ │ └── SettingFragment.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── color/ │ │ │ │ │ │ └── selector_ic.xml │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── btn_big_gray.xml │ │ │ │ │ │ ├── btn_big_green.xml │ │ │ │ │ │ ├── btn_big_yellow.xml │ │ │ │ │ │ ├── btn_default.xml │ │ │ │ │ │ ├── btn_default_highlight.xml │ │ │ │ │ │ ├── ic_baseline_settings.xml │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ ├── ic_plus.xml │ │ │ │ │ │ ├── tf_out_line.xml │ │ │ │ │ │ └── toggle_widget_background.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── fragment_camera.xml │ │ │ │ │ │ └── fragment_setting.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ └── transfer_learning/ │ │ │ ├── generate_training_model.py │ │ │ └── requirements.txt │ │ ├── native_api/ │ │ │ └── android_play_services/ │ │ │ ├── c_api/ │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── com/ │ │ │ │ │ │ └── google/ │ │ │ │ │ │ └── samples/ │ │ │ │ │ │ └── gms/ │ │ │ │ │ │ └── tflite/ │ │ │ │ │ │ └── c/ │ │ │ │ │ │ └── instrumentation/ │ │ │ │ │ │ ├── BasicScenarioTest.java │ │ │ │ │ │ └── TfLiteNativeGPUAccelerationTest.java │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── cpp/ │ │ │ │ │ │ ├── CMakeLists.txt │ │ │ │ │ │ ├── com_google_samples_gms_tflite_c_TfLiteJni.cc │ │ │ │ │ │ ├── java_interop.h │ │ │ │ │ │ └── logging_assert.h │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── com/ │ │ │ │ │ │ └── google/ │ │ │ │ │ │ └── samples/ │ │ │ │ │ │ └── gms/ │ │ │ │ │ │ └── tflite/ │ │ │ │ │ │ └── c/ │ │ │ │ │ │ ├── MainActivity.java │ │ │ │ │ │ └── TfLiteJni.java │ │ │ │ │ └── res/ │ │ │ │ │ ├── layout/ │ │ │ │ │ │ └── activity_main.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ └── cc_api/ │ │ │ ├── app/ │ │ │ │ ├── build.gradle │ │ │ │ ├── proguard-rules.pro │ │ │ │ ├── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── com/ │ │ │ │ │ │ └── google/ │ │ │ │ │ │ └── samples/ │ │ │ │ │ │ └── gms/ │ │ │ │ │ │ └── tflite/ │ │ │ │ │ │ └── c/ │ │ │ │ │ │ └── instrumentation/ │ │ │ │ │ │ ├── BasicScenarioTest.java │ │ │ │ │ │ └── TfLiteNativeGPUAccelerationTest.java │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── cpp/ │ │ │ │ │ │ ├── CMakeLists.txt │ │ │ │ │ │ ├── Modules/ │ │ │ │ │ │ │ └── Findtflite_cc_api.cmake │ │ │ │ │ │ ├── com_google_samples_gms_tflite_cc_TfLiteJni.cc │ │ │ │ │ │ ├── java_interop.h │ │ │ │ │ │ └── logging_assert.h │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── com/ │ │ │ │ │ │ └── google/ │ │ │ │ │ │ └── samples/ │ │ │ │ │ │ └── gms/ │ │ │ │ │ │ └── tflite/ │ │ │ │ │ │ └── cc/ │ │ │ │ │ │ ├── MainActivity.java │ │ │ │ │ │ └── TfLiteJni.java │ │ │ │ │ └── res/ │ │ │ │ │ ├── layout/ │ │ │ │ │ │ └── activity_main.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ └── tflite-java-extract-cpp-sdk.gradle │ │ │ ├── build.gradle │ │ │ ├── gradle/ │ │ │ │ └── wrapper/ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ │ ├── gradle.properties │ │ │ ├── gradlew │ │ │ ├── gradlew.bat │ │ │ └── settings.gradle │ │ ├── object_detection/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_models.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── objectdetection/ │ │ │ │ │ │ └── TFObjectDetectionTest.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── objectdetection/ │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ ├── ObjectDetectorHelper.kt │ │ │ │ │ │ ├── OverlayView.kt │ │ │ │ │ │ └── fragments/ │ │ │ │ │ │ ├── CameraFragment.kt │ │ │ │ │ │ └── PermissionsFragment.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── color/ │ │ │ │ │ │ └── selector_ic.xml │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── fragment_camera.xml │ │ │ │ │ │ └── info_bottom_sheet.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ ├── android_play_services/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_models.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── objectdetection/ │ │ │ │ │ │ └── TFObjectDetectionTest.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── objectdetection/ │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ ├── ObjectDetectorHelper.kt │ │ │ │ │ │ ├── OverlayView.kt │ │ │ │ │ │ └── fragments/ │ │ │ │ │ │ ├── CameraFragment.kt │ │ │ │ │ │ └── PermissionsFragment.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── color/ │ │ │ │ │ │ └── selector_ic.xml │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── fragment_camera.xml │ │ │ │ │ │ └── info_bottom_sheet.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ ├── ios/ │ │ │ │ ├── ObjectDetection/ │ │ │ │ │ ├── AppDelegate.swift │ │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── Contents.json │ │ │ │ │ │ ├── down_icon.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── tfl_logo.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ └── up_icon.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── Base.lproj/ │ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ │ └── Main.storyboard │ │ │ │ │ ├── Camera Feed/ │ │ │ │ │ │ ├── CameraFeedManager.swift │ │ │ │ │ │ └── PreviewView.swift │ │ │ │ │ ├── Info.plist │ │ │ │ │ ├── TFLite/ │ │ │ │ │ │ └── ObjectDetectionHelper.swift │ │ │ │ │ ├── ViewControllers/ │ │ │ │ │ │ ├── InferenceViewController.swift │ │ │ │ │ │ └── ViewController.swift │ │ │ │ │ └── Views/ │ │ │ │ │ └── OverlayView.swift │ │ │ │ ├── ObjectDetection.xcodeproj/ │ │ │ │ │ └── project.pbxproj │ │ │ │ ├── Podfile │ │ │ │ ├── README.md │ │ │ │ └── RunScripts/ │ │ │ │ └── download_models.sh │ │ │ └── raspberry_pi/ │ │ │ ├── README.md │ │ │ ├── detect.py │ │ │ ├── requirements.txt │ │ │ ├── setup.sh │ │ │ ├── test_data/ │ │ │ │ └── table_results.csv │ │ │ └── utils.py │ │ ├── optical_character_recognition/ │ │ │ └── android/ │ │ │ ├── README.md │ │ │ ├── app/ │ │ │ │ ├── build.gradle │ │ │ │ ├── download.gradle │ │ │ │ ├── proguard-rules.pro │ │ │ │ ├── settings.gradle │ │ │ │ └── src/ │ │ │ │ └── main/ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ ├── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── ocr/ │ │ │ │ │ ├── ImageUtils.kt │ │ │ │ │ ├── MLExecutionViewModel.kt │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ ├── ModelExecutionResult.kt │ │ │ │ │ └── OCRModelExecutor.kt │ │ │ │ └── res/ │ │ │ │ ├── drawable/ │ │ │ │ │ ├── bottom_sheet_bg.xml │ │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ │ └── rounded_edge.xml │ │ │ │ ├── drawable-v24/ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ ├── layout/ │ │ │ │ │ ├── tfe_is_activity_main.xml │ │ │ │ │ └── tfe_is_bottom_sheet_layout.xml │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ └── values/ │ │ │ │ ├── colors.xml │ │ │ │ ├── dimens.xml │ │ │ │ ├── strings.xml │ │ │ │ └── styles.xml │ │ │ ├── build.gradle │ │ │ ├── gradle/ │ │ │ │ └── wrapper/ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ │ ├── gradle.properties │ │ │ ├── gradlew │ │ │ ├── gradlew.bat │ │ │ └── settings.gradle │ │ ├── pose_estimation/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── .gitignore │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ ├── assets/ │ │ │ │ │ │ │ ├── image_credits.txt │ │ │ │ │ │ │ └── pose_landmark_truth.csv │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── poseestimation/ │ │ │ │ │ │ ├── ml/ │ │ │ │ │ │ │ ├── EvaluationUtils.kt │ │ │ │ │ │ │ ├── MovenetLightningTest.kt │ │ │ │ │ │ │ ├── MovenetMultiPoseTest.kt │ │ │ │ │ │ │ ├── MovenetThunderTest.kt │ │ │ │ │ │ │ ├── PoseClassifierTest.kt │ │ │ │ │ │ │ ├── PosenetTest.kt │ │ │ │ │ │ │ └── VisualizationTest.kt │ │ │ │ │ │ └── tracker/ │ │ │ │ │ │ ├── BoundingBoxTrackerTest.kt │ │ │ │ │ │ └── KeyPointsTrackerTest.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── assets/ │ │ │ │ │ │ └── labels.txt │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── poseestimation/ │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ ├── VisualizationUtils.kt │ │ │ │ │ │ ├── YuvToRgbConverter.kt │ │ │ │ │ │ ├── camera/ │ │ │ │ │ │ │ └── CameraSource.kt │ │ │ │ │ │ ├── data/ │ │ │ │ │ │ │ ├── BodyPart.kt │ │ │ │ │ │ │ ├── Device.kt │ │ │ │ │ │ │ ├── KeyPoint.kt │ │ │ │ │ │ │ ├── Person.kt │ │ │ │ │ │ │ └── TorsoAndBodyDistance.kt │ │ │ │ │ │ ├── ml/ │ │ │ │ │ │ │ ├── MoveNet.kt │ │ │ │ │ │ │ ├── MoveNetMultiPose.kt │ │ │ │ │ │ │ ├── PoseClassifier.kt │ │ │ │ │ │ │ ├── PoseDetector.kt │ │ │ │ │ │ │ └── PoseNet.kt │ │ │ │ │ │ └── tracker/ │ │ │ │ │ │ ├── AbstractTracker.kt │ │ │ │ │ │ ├── BoundingBoxTracker.kt │ │ │ │ │ │ ├── KeyPointsTracker.kt │ │ │ │ │ │ ├── Track.kt │ │ │ │ │ │ └── TrackerConfig.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ └── rounded_edge.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ └── bottom_sheet_layout.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── themes.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ ├── ios/ │ │ │ │ ├── Podfile │ │ │ │ ├── PoseEstimation/ │ │ │ │ │ ├── AppDelegate.swift │ │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── Contents.json │ │ │ │ │ │ └── tfl_logo.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── Camera/ │ │ │ │ │ │ └── CameraFeedManager.swift │ │ │ │ │ ├── Info.plist │ │ │ │ │ ├── ML/ │ │ │ │ │ │ ├── Extensions/ │ │ │ │ │ │ │ ├── CGSize+TFLite.swift │ │ │ │ │ │ │ ├── CVPixelBuffer+TFLite.swift │ │ │ │ │ │ │ └── Data+TFLite.swift │ │ │ │ │ │ └── Models/ │ │ │ │ │ │ ├── MoveNet.swift │ │ │ │ │ │ ├── PoseConfig.swift │ │ │ │ │ │ ├── PoseData.swift │ │ │ │ │ │ ├── PoseEstimator.swift │ │ │ │ │ │ └── PoseNet.swift │ │ │ │ │ └── UI/ │ │ │ │ │ ├── Storyboards/ │ │ │ │ │ │ └── Base.lproj/ │ │ │ │ │ │ ├── Launch Screen.storyboard │ │ │ │ │ │ └── Main.storyboard │ │ │ │ │ ├── ViewControllers/ │ │ │ │ │ │ └── ViewController.swift │ │ │ │ │ └── Views/ │ │ │ │ │ └── OverlayView.swift │ │ │ │ ├── PoseEstimation.xcodeproj/ │ │ │ │ │ ├── project.pbxproj │ │ │ │ │ └── xcshareddata/ │ │ │ │ │ └── xcschemes/ │ │ │ │ │ └── PoseEstimation.xcscheme │ │ │ │ ├── README.md │ │ │ │ └── RunScripts/ │ │ │ │ └── download_models.sh │ │ │ └── raspberry_pi/ │ │ │ ├── README.md │ │ │ ├── data.py │ │ │ ├── labels.txt │ │ │ ├── ml/ │ │ │ │ ├── __init__.py │ │ │ │ ├── classifier.py │ │ │ │ ├── classifier_test.py │ │ │ │ ├── movenet.py │ │ │ │ ├── movenet_multipose.py │ │ │ │ ├── movenet_multipose_test.py │ │ │ │ ├── movenet_test.py │ │ │ │ ├── posenet.py │ │ │ │ └── posenet_test.py │ │ │ ├── pose_estimation.py │ │ │ ├── requirements.txt │ │ │ ├── setup.sh │ │ │ ├── test_data/ │ │ │ │ ├── image_credits.txt │ │ │ │ └── pose_landmark_truth.csv │ │ │ ├── tracker/ │ │ │ │ ├── __init__.py │ │ │ │ ├── bounding_box_tracker.py │ │ │ │ ├── bounding_box_tracker_test.py │ │ │ │ ├── config.py │ │ │ │ ├── keypoint_tracker.py │ │ │ │ ├── keypoint_tracker_test.py │ │ │ │ └── tracker.py │ │ │ ├── utils.py │ │ │ └── visualizer.py │ │ ├── posenet/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── posenet/ │ │ │ │ │ │ └── ExampleInstrumentedTest.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── posenet/ │ │ │ │ │ │ ├── CameraActivity.kt │ │ │ │ │ │ ├── ConfirmationDialog.kt │ │ │ │ │ │ ├── Constants.kt │ │ │ │ │ │ ├── ImageUtils.kt │ │ │ │ │ │ ├── PosenetActivity.kt │ │ │ │ │ │ └── TestActivity.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ └── ic_launcher_background.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── tfe_pn_activity_camera.xml │ │ │ │ │ │ ├── tfe_pn_activity_posenet.xml │ │ │ │ │ │ └── tfe_pn_activity_test.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ ├── posenet/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── posenet/ │ │ │ │ │ │ └── ExampleInstrumentedTest.java │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ └── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── posenet/ │ │ │ │ │ └── lib/ │ │ │ │ │ └── Posenet.kt │ │ │ │ └── settings.gradle │ │ │ └── ios/ │ │ │ ├── .gitignore │ │ │ ├── Podfile │ │ │ ├── PoseNet/ │ │ │ │ ├── AppDelegate.swift │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ └── Contents.json │ │ │ │ ├── Camera Feed/ │ │ │ │ │ ├── CameraFeedManager.swift │ │ │ │ │ └── PreviewView.swift │ │ │ │ ├── Cells/ │ │ │ │ │ └── InfoCell.swift │ │ │ │ ├── Constants.swift │ │ │ │ ├── Extensions/ │ │ │ │ │ ├── CGSizeExtension.swift │ │ │ │ │ ├── CVPixelBufferExtension.swift │ │ │ │ │ └── TFLiteExtension.swift │ │ │ │ ├── Info.plist │ │ │ │ ├── ModelDataHandler/ │ │ │ │ │ └── ModelDataHandler.swift │ │ │ │ ├── Storyboards/ │ │ │ │ │ └── Base.lproj/ │ │ │ │ │ ├── Launch Screen.storyboard │ │ │ │ │ └── Main.storyboard │ │ │ │ ├── ViewControllers/ │ │ │ │ │ └── ViewController.swift │ │ │ │ └── Views/ │ │ │ │ └── OverlayView.swift │ │ │ ├── PoseNet.xcodeproj/ │ │ │ │ └── project.pbxproj │ │ │ ├── README.md │ │ │ └── RunScripts/ │ │ │ └── download_models.sh │ │ ├── recommendation/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── assets/ │ │ │ │ │ │ ├── config.json │ │ │ │ │ │ ├── config_cnn_i10o100.json │ │ │ │ │ │ ├── config_rnn_i10o100.json │ │ │ │ │ │ ├── movie_genre_vocab.txt │ │ │ │ │ │ └── sorted_movie_vocab.json │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── recommendation/ │ │ │ │ │ │ ├── Config.java │ │ │ │ │ │ ├── MainActivity.java │ │ │ │ │ │ ├── MovieFragment.java │ │ │ │ │ │ ├── MovieRecyclerViewAdapter.java │ │ │ │ │ │ ├── RecommendationClient.java │ │ │ │ │ │ ├── RecommendationFragment.java │ │ │ │ │ │ ├── RecommendationRecyclerViewAdapter.java │ │ │ │ │ │ └── data/ │ │ │ │ │ │ ├── FileUtil.java │ │ │ │ │ │ └── MovieItem.java │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ └── ic_launcher_background.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── tfe_re_activity_main.xml │ │ │ │ │ │ ├── tfe_re_fragment_recommendation.xml │ │ │ │ │ │ ├── tfe_re_fragment_recommendation_list.xml │ │ │ │ │ │ ├── tfe_re_fragment_selection.xml │ │ │ │ │ │ └── tfe_re_fragment_selection_list.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ └── ml/ │ │ │ ├── configs/ │ │ │ │ ├── __init__.py │ │ │ │ ├── input-config.proto │ │ │ │ ├── input_config_generated_pb2.py │ │ │ │ ├── model_config.py │ │ │ │ └── sample_input_config.pbtxt │ │ │ ├── data/ │ │ │ │ ├── __init__.py │ │ │ │ ├── example_generation_movielens.py │ │ │ │ └── example_generation_movielens_test.py │ │ │ ├── model/ │ │ │ │ ├── __init__.py │ │ │ │ ├── context_encoder.py │ │ │ │ ├── context_encoder_test.py │ │ │ │ ├── dotproduct_similarity.py │ │ │ │ ├── dotproduct_similarity_test.py │ │ │ │ ├── input_pipeline.py │ │ │ │ ├── input_pipeline_test.py │ │ │ │ ├── label_encoder.py │ │ │ │ ├── label_encoder_test.py │ │ │ │ ├── losses.py │ │ │ │ ├── losses_test.py │ │ │ │ ├── metrics.py │ │ │ │ ├── metrics_test.py │ │ │ │ ├── recommendation_model.py │ │ │ │ ├── recommendation_model_launcher.py │ │ │ │ ├── recommendation_model_launcher_test.py │ │ │ │ ├── recommendation_model_test.py │ │ │ │ └── utils.py │ │ │ ├── ondevice_recommendation.ipynb │ │ │ └── requirements.txt │ │ ├── reinforcement_learning/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── reinforcementlearning/ │ │ │ │ │ │ ├── BoardCellAdapter.java │ │ │ │ │ │ ├── BoardCellStatus.java │ │ │ │ │ │ ├── Constants.java │ │ │ │ │ │ ├── HiddenBoardCellStatus.java │ │ │ │ │ │ ├── MainActivity.java │ │ │ │ │ │ ├── PlaneStrikeAgent.java │ │ │ │ │ │ ├── RLAgent.java │ │ │ │ │ │ └── RLAgentFromTFAgents.java │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ └── ic_launcher_background.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ └── activity_main.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ └── ml/ │ │ │ ├── README.md │ │ │ ├── common.py │ │ │ ├── tf_agents/ │ │ │ │ ├── planestrike_py_environment.py │ │ │ │ ├── requirements.txt │ │ │ │ └── training_tf_agents.py │ │ │ └── tf_and_jax/ │ │ │ ├── gym_planestrike/ │ │ │ │ ├── gym_planestrike/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── envs/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── planestrike.py │ │ │ │ └── setup.py │ │ │ ├── requirements.txt │ │ │ ├── training_jax.py │ │ │ └── training_tf.py │ │ ├── smart_reply/ │ │ │ └── android/ │ │ │ ├── README.md │ │ │ ├── app/ │ │ │ │ ├── build.gradle │ │ │ │ ├── download.gradle │ │ │ │ ├── libs/ │ │ │ │ │ ├── .bazelrc │ │ │ │ │ ├── .bazelversion │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── LICENSE │ │ │ │ │ ├── WORKSPACE │ │ │ │ │ └── cc/ │ │ │ │ │ ├── BUILD │ │ │ │ │ ├── DummyManifest.xml │ │ │ │ │ ├── misc/ │ │ │ │ │ │ └── BUILD │ │ │ │ │ ├── ops/ │ │ │ │ │ │ ├── extract_feature.cc │ │ │ │ │ │ ├── extract_feature_test.cc │ │ │ │ │ │ ├── normalize.cc │ │ │ │ │ │ ├── normalize_test.cc │ │ │ │ │ │ ├── predict.cc │ │ │ │ │ │ └── predict_test.cc │ │ │ │ │ ├── predictor.cc │ │ │ │ │ ├── predictor.h │ │ │ │ │ ├── predictor_test.cc │ │ │ │ │ ├── smartreply_jni.cc │ │ │ │ │ └── testdata/ │ │ │ │ │ ├── BUILD │ │ │ │ │ └── smartreply_samples.tsv │ │ │ │ └── src/ │ │ │ │ └── main/ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ ├── assets/ │ │ │ │ │ └── backoff_response.txt │ │ │ │ ├── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── smartreply/ │ │ │ │ │ ├── AssetsUtil.java │ │ │ │ │ ├── MainActivity.java │ │ │ │ │ ├── SmartReply.java │ │ │ │ │ └── SmartReplyClient.java │ │ │ │ └── res/ │ │ │ │ ├── drawable/ │ │ │ │ │ └── ic_launcher_background.xml │ │ │ │ ├── drawable-v24/ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ ├── layout/ │ │ │ │ │ └── tfe_sr_main_activity.xml │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ └── values/ │ │ │ │ ├── colors.xml │ │ │ │ ├── dimens.xml │ │ │ │ ├── strings.xml │ │ │ │ └── styles.xml │ │ │ ├── build.gradle │ │ │ ├── gradle/ │ │ │ │ └── wrapper/ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ │ ├── gradle.properties │ │ │ ├── gradlew │ │ │ ├── gradlew.bat │ │ │ ├── how-to-build.md │ │ │ └── settings.gradle │ │ ├── sound_classification/ │ │ │ ├── README.md │ │ │ ├── ios/ │ │ │ │ ├── Podfile │ │ │ │ ├── README.md │ │ │ │ ├── RunScripts/ │ │ │ │ │ └── download_models.sh │ │ │ │ ├── SoundClassification/ │ │ │ │ │ ├── AppDelegate.swift │ │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── Contents.json │ │ │ │ │ │ └── tfl2_logo_dark.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── AudioInputManager.swift │ │ │ │ │ ├── Base.lproj/ │ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ │ └── Main.storyboard │ │ │ │ │ ├── Info.plist │ │ │ │ │ ├── Model/ │ │ │ │ │ │ └── labels.txt │ │ │ │ │ ├── ProbabilityTableViewCell.swift │ │ │ │ │ ├── SoundClassifier.swift │ │ │ │ │ └── ViewController.swift │ │ │ │ └── SoundClassification.xcodeproj/ │ │ │ │ └── project.pbxproj │ │ │ └── raspberry_pi/ │ │ │ ├── README.md │ │ │ ├── classify.py │ │ │ ├── requirements.txt │ │ │ ├── setup.sh │ │ │ ├── test_data/ │ │ │ │ └── ground_truth.csv │ │ │ └── utils.py │ │ ├── speech_commands/ │ │ │ ├── README.md │ │ │ ├── ios/ │ │ │ │ ├── Podfile │ │ │ │ ├── README.md │ │ │ │ ├── RunScripts/ │ │ │ │ │ └── download_models.sh │ │ │ │ ├── SpeechCommands/ │ │ │ │ │ ├── AppDelegate.swift │ │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── Contents.json │ │ │ │ │ │ ├── base.imageset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── border_color.colorset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ ├── inner_shadow_color.colorset/ │ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ │ └── tfl_logo.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── AudioInputManager/ │ │ │ │ │ │ └── AudioInputManager.swift │ │ │ │ │ ├── Base.lproj/ │ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ │ └── Main.storyboard │ │ │ │ │ ├── Cells/ │ │ │ │ │ │ ├── InfoCell.swift │ │ │ │ │ │ └── WordCell.swift │ │ │ │ │ ├── Info.plist │ │ │ │ │ ├── Model/ │ │ │ │ │ │ └── .gitignore │ │ │ │ │ ├── ModelDataHandler/ │ │ │ │ │ │ ├── ModelDataHandler.swift │ │ │ │ │ │ └── RecognizeCommands.swift │ │ │ │ │ └── ViewControllers/ │ │ │ │ │ ├── InferenceViewController.swift │ │ │ │ │ └── ViewController.swift │ │ │ │ └── SpeechCommands.xcodeproj/ │ │ │ │ └── project.pbxproj │ │ │ └── ml/ │ │ │ ├── .gitignore │ │ │ ├── README.md │ │ │ ├── callbacks.py │ │ │ ├── classes.py │ │ │ ├── download.py │ │ │ ├── export/ │ │ │ │ ├── __init__.py │ │ │ │ ├── convert_keras_lite.py │ │ │ │ ├── convert_keras_to_quantized.py │ │ │ │ └── convert_tensorflow_lite.sh │ │ │ ├── generator.py │ │ │ ├── model.py │ │ │ ├── requirements.txt │ │ │ ├── train.py │ │ │ └── utils.py │ │ ├── speech_recognition/ │ │ │ └── android/ │ │ │ ├── app/ │ │ │ │ └── src/ │ │ │ │ └── main/ │ │ │ │ └── res/ │ │ │ │ ├── drawable/ │ │ │ │ │ └── ic_launcher_background.xml │ │ │ │ ├── drawable-v24/ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ └── mipmap-anydpi-v26/ │ │ │ │ ├── ic_launcher.xml │ │ │ │ └── ic_launcher_round.xml │ │ │ ├── gradle/ │ │ │ │ └── wrapper/ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ │ ├── gradlew │ │ │ └── gradlew.bat │ │ ├── style_transfer/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_model.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── styletransfer/ │ │ │ │ │ │ └── StyleTransferHelperTest.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── styletransfer/ │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ ├── MainViewModel.kt │ │ │ │ │ │ ├── SingleLiveEvent.kt │ │ │ │ │ │ ├── StyleTransferHelper.kt │ │ │ │ │ │ └── fragments/ │ │ │ │ │ │ ├── CameraFragment.kt │ │ │ │ │ │ ├── PermissionsFragment.kt │ │ │ │ │ │ ├── StyleAdapter.kt │ │ │ │ │ │ └── TransformationFragment.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── bg_item_style.xml │ │ │ │ │ │ ├── bg_item_style_selected.xml │ │ │ │ │ │ ├── decoration_divider.xml │ │ │ │ │ │ ├── ic_baseline_camera_enhance.xml │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── fragment_camera.xml │ │ │ │ │ │ ├── fragment_transformation.xml │ │ │ │ │ │ └── item_style.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── navigation/ │ │ │ │ │ │ └── nav_graph.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── dimens.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── style.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ └── ios/ │ │ │ ├── Podfile │ │ │ ├── README.md │ │ │ ├── StyleTransfer/ │ │ │ │ ├── AppDelegate.swift │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── Contents.json │ │ │ │ │ ├── photo_camera.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── photo_library.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style0.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style1.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style10.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style11.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style12.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style13.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style14.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style15.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style16.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style17.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style18.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style19.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style2.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style20.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style21.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style22.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style23.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style24.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style25.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style3.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style4.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style5.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style6.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style7.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── style8.imageset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ └── style9.imageset/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── Base.lproj/ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ └── Main.storyboard │ │ │ │ ├── Info.plist │ │ │ │ ├── StylePickerViewController.swift │ │ │ │ ├── StyleTransferer.swift │ │ │ │ ├── TFLiteExtension.swift │ │ │ │ ├── UIKitExtension.swift │ │ │ │ └── ViewController.swift │ │ │ ├── StyleTransfer.xcodeproj/ │ │ │ │ ├── project.pbxproj │ │ │ │ └── xcshareddata/ │ │ │ │ └── xcschemes/ │ │ │ │ └── StyleTransfer.xcscheme │ │ │ └── download_tflite_models.sh │ │ ├── super_resolution/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── cc/ │ │ │ │ │ │ ├── CMakeLists.txt │ │ │ │ │ │ ├── SuperResolution.cpp │ │ │ │ │ │ ├── SuperResolution.h │ │ │ │ │ │ └── SuperResolution_jni.cpp │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── superresolution/ │ │ │ │ │ │ ├── AssetsUtil.java │ │ │ │ │ │ └── MainActivity.java │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ │ │ └── rounded_edge.xml │ │ │ │ │ ├── drawable-v24/ │ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ └── bottom_sheet_layout.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ └── values/ │ │ │ │ │ ├── colors.xml │ │ │ │ │ ├── strings.xml │ │ │ │ │ └── styles.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ └── ml/ │ │ │ └── super_resolution.ipynb │ │ ├── text_classification/ │ │ │ ├── android/ │ │ │ │ ├── README.md │ │ │ │ ├── app/ │ │ │ │ │ ├── build.gradle │ │ │ │ │ ├── download_model.gradle │ │ │ │ │ ├── proguard-rules.pro │ │ │ │ │ └── src/ │ │ │ │ │ ├── androidTest/ │ │ │ │ │ │ └── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── textclassification/ │ │ │ │ │ │ └── TextClassifierInstrumentationTest.kt │ │ │ │ │ └── main/ │ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ │ ├── java/ │ │ │ │ │ │ └── org/ │ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ │ └── lite/ │ │ │ │ │ │ └── examples/ │ │ │ │ │ │ └── textclassification/ │ │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ │ ├── ResultsAdapter.kt │ │ │ │ │ │ └── TextClassificationHelper.kt │ │ │ │ │ └── res/ │ │ │ │ │ ├── drawable/ │ │ │ │ │ │ ├── ic_minus.xml │ │ │ │ │ │ └── ic_plus.xml │ │ │ │ │ ├── layout/ │ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ │ ├── info_bottom_sheet.xml │ │ │ │ │ │ └── item_classification.xml │ │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ │ ├── values/ │ │ │ │ │ │ ├── colors.xml │ │ │ │ │ │ ├── dimens.xml │ │ │ │ │ │ ├── strings.xml │ │ │ │ │ │ └── themes.xml │ │ │ │ │ └── values-night/ │ │ │ │ │ └── themes.xml │ │ │ │ ├── build.gradle │ │ │ │ ├── gradle/ │ │ │ │ │ └── wrapper/ │ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ │ └── gradle-wrapper.properties │ │ │ │ ├── gradle.properties │ │ │ │ ├── gradlew │ │ │ │ ├── gradlew.bat │ │ │ │ └── settings.gradle │ │ │ └── ios/ │ │ │ ├── Podfile │ │ │ ├── README.md │ │ │ ├── TextClassification/ │ │ │ │ ├── AppDelegate.swift │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ └── Contents.json │ │ │ │ ├── Base.lproj/ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ └── Main.storyboard │ │ │ │ ├── Info.plist │ │ │ │ └── ViewController.swift │ │ │ └── TextClassification.xcodeproj/ │ │ │ └── project.pbxproj │ │ ├── text_searcher/ │ │ │ └── android/ │ │ │ ├── README.md │ │ │ ├── app/ │ │ │ │ ├── build.gradle │ │ │ │ ├── download.gradle │ │ │ │ ├── proguard-rules.pro │ │ │ │ └── src/ │ │ │ │ ├── androidTest/ │ │ │ │ │ └── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── textsearcher/ │ │ │ │ │ └── TextSearcherClientTest.kt │ │ │ │ └── main/ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ ├── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── textsearcher/ │ │ │ │ │ ├── tflite/ │ │ │ │ │ │ └── TextSearcherClient.kt │ │ │ │ │ └── ui/ │ │ │ │ │ ├── TextSearcherActivity.kt │ │ │ │ │ ├── TextSearcherResultActivity.kt │ │ │ │ │ └── WebviewActivity.kt │ │ │ │ └── res/ │ │ │ │ ├── drawable/ │ │ │ │ │ ├── ic_baseline_close_48.xml │ │ │ │ │ ├── ic_baseline_search_48.xml │ │ │ │ │ ├── ic_baseline_web_48.xml │ │ │ │ │ └── ic_launcher_background.xml │ │ │ │ ├── drawable-v24/ │ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ │ ├── layout/ │ │ │ │ │ ├── activity_text_searcher.xml │ │ │ │ │ ├── activity_text_searcher_result.xml │ │ │ │ │ ├── activity_webview.xml │ │ │ │ │ ├── preset_query_item_layout.xml │ │ │ │ │ └── searcher_result_item_layout.xml │ │ │ │ ├── menu/ │ │ │ │ │ └── action_bar_menu.xml │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ └── values/ │ │ │ │ ├── colors.xml │ │ │ │ ├── dimens.xml │ │ │ │ ├── strings.xml │ │ │ │ └── themes.xml │ │ │ ├── build.gradle │ │ │ ├── gradle/ │ │ │ │ └── wrapper/ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ │ ├── gradle.properties │ │ │ ├── gradlew │ │ │ ├── gradlew.bat │ │ │ └── settings.gradle │ │ └── video_classification/ │ │ ├── android/ │ │ │ ├── README.md │ │ │ ├── app/ │ │ │ │ ├── build.gradle │ │ │ │ ├── download.gradle │ │ │ │ ├── proguard-rules.pro │ │ │ │ └── src/ │ │ │ │ ├── androidTest/ │ │ │ │ │ └── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── videoclassification/ │ │ │ │ │ └── VideoClassifierTest.kt │ │ │ │ └── main/ │ │ │ │ ├── AndroidManifest.xml │ │ │ │ ├── assets/ │ │ │ │ │ └── kinetics600_label_map.txt │ │ │ │ ├── java/ │ │ │ │ │ └── org/ │ │ │ │ │ └── tensorflow/ │ │ │ │ │ └── lite/ │ │ │ │ │ └── examples/ │ │ │ │ │ └── videoclassification/ │ │ │ │ │ ├── CalculateUtils.kt │ │ │ │ │ ├── MainActivity.kt │ │ │ │ │ └── ml/ │ │ │ │ │ └── VideoClassifier.kt │ │ │ │ └── res/ │ │ │ │ ├── drawable/ │ │ │ │ │ ├── bg_bottom_sheet.xml │ │ │ │ │ ├── bg_rectangle.xml │ │ │ │ │ ├── ic_baseline_add.xml │ │ │ │ │ └── ic_baseline_remove.xml │ │ │ │ ├── layout/ │ │ │ │ │ ├── activity_main.xml │ │ │ │ │ └── layout_bottom_sheet.xml │ │ │ │ ├── mipmap-anydpi-v26/ │ │ │ │ │ ├── ic_launcher.xml │ │ │ │ │ └── ic_launcher_round.xml │ │ │ │ └── values/ │ │ │ │ ├── colors.xml │ │ │ │ ├── dimens.xml │ │ │ │ ├── strings.xml │ │ │ │ └── themes.xml │ │ │ ├── build.gradle │ │ │ ├── gradle.properties │ │ │ └── settings.gradle │ │ └── raspberry_pi/ │ │ ├── README.md │ │ ├── classify.py │ │ ├── kinetics600_label_map.txt │ │ ├── requirements.txt │ │ ├── setup.sh │ │ ├── video_classifier.py │ │ └── video_classifier_test.py │ └── tools/ │ ├── build_all_android.sh │ ├── build_all_ios.sh │ ├── build_android_app.sh │ ├── build_ios_app.sh │ ├── build_model_maker_api_docs.py │ ├── shared/ │ │ └── android/ │ │ ├── app/ │ │ │ └── src/ │ │ │ └── main/ │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ └── ic_launcher_background.xml │ │ │ ├── drawable-v24/ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ └── mipmap-anydpi-v26/ │ │ │ ├── ic_launcher.xml │ │ │ └── ic_launcher_round.xml │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ └── gradlew.bat │ └── test_pip_setup.sh ├── setup.py ├── templates/ │ └── notebook.ipynb └── tensorflow_examples/ ├── __init__.py ├── lite/ │ ├── __init__.py │ └── model_maker/ │ ├── README.md │ ├── RELEASE.md │ ├── __init__.py │ ├── cli/ │ │ ├── __init__.py │ │ ├── cli.py │ │ └── cli_test.py │ ├── core/ │ │ ├── __init__.py │ │ ├── api/ │ │ │ ├── __init__.py │ │ │ ├── api_gen.py │ │ │ ├── api_gen_test.py │ │ │ ├── api_util.py │ │ │ ├── api_util_test.py │ │ │ ├── deprecated_api.py │ │ │ ├── golden_api.json │ │ │ ├── golden_api_doc.py │ │ │ └── include.py │ │ ├── compat.py │ │ ├── data_util/ │ │ │ ├── __init__.py │ │ │ ├── audio_dataloader.py │ │ │ ├── audio_dataloader_test.py │ │ │ ├── data_util.py │ │ │ ├── dataloader.py │ │ │ ├── dataloader_test.py │ │ │ ├── image_dataloader.py │ │ │ ├── image_dataloader_test.py │ │ │ ├── image_searcher_dataloader.py │ │ │ ├── metadata_loader.py │ │ │ ├── metadata_loader_test.py │ │ │ ├── object_detector_dataloader.py │ │ │ ├── object_detector_dataloader_test.py │ │ │ ├── object_detector_dataloader_util.py │ │ │ ├── object_detector_dataloader_util_test.py │ │ │ ├── recommendation_config.py │ │ │ ├── recommendation_dataloader.py │ │ │ ├── recommendation_dataloader_test.py │ │ │ ├── recommendation_testutil.py │ │ │ ├── searcher_dataloader.py │ │ │ ├── searcher_dataloader_test.py │ │ │ ├── testdata/ │ │ │ │ ├── annotations/ │ │ │ │ │ └── 2012_12.xml │ │ │ │ ├── annotations.json │ │ │ │ ├── mobilenet_v2_035_96_embedder_with_metadata.tflite │ │ │ │ ├── movies.csv │ │ │ │ ├── object_detection_csv/ │ │ │ │ │ ├── json_files/ │ │ │ │ │ │ ├── test_annotations.json │ │ │ │ │ │ └── train_annotations.json │ │ │ │ │ └── salads_ml_use.csv │ │ │ │ ├── regex_one_embedding_with_metadata.tflite │ │ │ │ ├── squad_testdata/ │ │ │ │ │ ├── dev-v1.1.json │ │ │ │ │ ├── dev-v2.0.json │ │ │ │ │ ├── train-v1.1.json │ │ │ │ │ └── train-v2.0.json │ │ │ │ └── trips.csv │ │ │ ├── text_dataloader.py │ │ │ ├── text_dataloader_test.py │ │ │ ├── text_searcher_dataloader.py │ │ │ └── text_searcher_dataloader_test.py │ │ ├── export_format.py │ │ ├── file_util.py │ │ ├── optimization/ │ │ │ ├── __init__.py │ │ │ └── warmup.py │ │ ├── task/ │ │ │ ├── __init__.py │ │ │ ├── audio_classifier.py │ │ │ ├── audio_classifier_test.py │ │ │ ├── classification_model.py │ │ │ ├── classification_model_test.py │ │ │ ├── configs.py │ │ │ ├── custom_model.py │ │ │ ├── custom_model_test.py │ │ │ ├── hub_loader.py │ │ │ ├── hub_loader_test.py │ │ │ ├── image_classifier.py │ │ │ ├── image_classifier_test.py │ │ │ ├── image_classifier_v1_test.py │ │ │ ├── image_preprocessing.py │ │ │ ├── image_preprocessing_test.py │ │ │ ├── make_image_classifier.py │ │ │ ├── metadata_writer_for_image_classifier.py │ │ │ ├── metadata_writers/ │ │ │ │ ├── __init__.py │ │ │ │ ├── bert/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── metadata_writer_for_bert.py │ │ │ │ │ ├── question_answerer/ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── metadata_writer_for_bert_question_answerer.py │ │ │ │ │ └── text_classifier/ │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── metadata_writer_for_bert_text_classifier.py │ │ │ │ ├── metadata_writer.py │ │ │ │ └── text_classifier/ │ │ │ │ ├── __init__.py │ │ │ │ └── metadata_writer_for_text_classifier.py │ │ │ ├── model_spec/ │ │ │ │ ├── __init__.py │ │ │ │ ├── audio_spec.py │ │ │ │ ├── audio_spec_test.py │ │ │ │ ├── image_spec.py │ │ │ │ ├── model_spec_test.py │ │ │ │ ├── object_detector_spec.py │ │ │ │ ├── object_detector_spec_test.py │ │ │ │ ├── recommendation_spec.py │ │ │ │ ├── recommendation_spec_test.py │ │ │ │ ├── testdata/ │ │ │ │ │ └── fake_effdet_lite0_hub/ │ │ │ │ │ ├── keras_metadata.pb │ │ │ │ │ ├── saved_model.pb │ │ │ │ │ └── variables/ │ │ │ │ │ ├── variables.data-00000-of-00001 │ │ │ │ │ └── variables.index │ │ │ │ ├── text_spec.py │ │ │ │ ├── text_spec_test.py │ │ │ │ └── util.py │ │ │ ├── model_util.py │ │ │ ├── model_util_test.py │ │ │ ├── model_util_v1_test.py │ │ │ ├── object_detector.py │ │ │ ├── object_detector_test.py │ │ │ ├── question_answer.py │ │ │ ├── question_answer_test.py │ │ │ ├── question_answer_v1_test.py │ │ │ ├── recommendation.py │ │ │ ├── recommendation_test.py │ │ │ ├── searcher.py │ │ │ ├── searcher_test.py │ │ │ ├── testdata/ │ │ │ │ ├── average_word_vec_metadata.json │ │ │ │ ├── bert_classifier_metadata.json │ │ │ │ ├── bert_qa_metadata.json │ │ │ │ ├── dummy_sp_text_embedder.tflite │ │ │ │ ├── efficientdet_lite0_metadata.json │ │ │ │ ├── efficientnet_lite0_metadata.json │ │ │ │ ├── hub_module_v1_mini/ │ │ │ │ │ ├── saved_model.pb │ │ │ │ │ └── tfhub_module.pb │ │ │ │ ├── hub_module_v1_mini_train/ │ │ │ │ │ ├── saved_model.pb │ │ │ │ │ ├── tfhub_module.pb │ │ │ │ │ └── variables/ │ │ │ │ │ ├── variables.data-00000-of-00001 │ │ │ │ │ └── variables.index │ │ │ │ ├── mobilenet_v3_small_100_224_embedder_scann.json │ │ │ │ └── saved_model_v2_mini/ │ │ │ │ ├── saved_model.pb │ │ │ │ └── variables/ │ │ │ │ ├── variables.data-00000-of-00001 │ │ │ │ └── variables.index │ │ │ ├── text_classifier.py │ │ │ ├── text_classifier_test.py │ │ │ ├── text_classifier_v1_test.py │ │ │ └── train_image_classifier_lib.py │ │ ├── test_util.py │ │ └── utils/ │ │ ├── __init__.py │ │ ├── ondevice_scann_builder.py │ │ ├── ondevice_scann_builder_test.py │ │ ├── scann_converter.py │ │ └── scann_converter_test.py │ ├── demo/ │ │ ├── __init__.py │ │ ├── audio_classification_demo.py │ │ ├── audio_classification_demo_test.py │ │ ├── custom_model_demo.py │ │ ├── custom_model_demo_test.py │ │ ├── image_classification.ipynb │ │ ├── image_classification_demo.py │ │ ├── image_classification_demo_test.py │ │ ├── question_answer_demo.py │ │ ├── recommendation_demo.py │ │ ├── recommendation_demo_test.py │ │ ├── text_classification.ipynb │ │ ├── text_classification_demo.py │ │ └── text_classification_demo_test.py │ ├── pip_package/ │ │ ├── create_venv.sh │ │ ├── golden_api_test.py │ │ ├── setup.py │ │ ├── setup_util.py │ │ └── test_pip_package.sh │ ├── public/ │ │ ├── __init__.py │ │ ├── audio_classifier/ │ │ │ └── __init__.py │ │ ├── config/ │ │ │ └── __init__.py │ │ ├── image_classifier/ │ │ │ └── __init__.py │ │ ├── model_spec/ │ │ │ └── __init__.py │ │ ├── object_detector/ │ │ │ └── __init__.py │ │ ├── question_answer/ │ │ │ └── __init__.py │ │ ├── recommendation/ │ │ │ ├── __init__.py │ │ │ └── spec/ │ │ │ └── __init__.py │ │ ├── searcher/ │ │ │ └── __init__.py │ │ └── text_classifier/ │ │ └── __init__.py │ ├── requirements.txt │ ├── requirements_nightly.txt │ └── third_party/ │ ├── efficientdet/ │ │ ├── Det-AdvProp.md │ │ ├── README.md │ │ ├── __init__.py │ │ ├── aug/ │ │ │ ├── __init__.py │ │ │ ├── autoaugment.py │ │ │ ├── gridmask.py │ │ │ └── mosaic.py │ │ ├── backbone/ │ │ │ ├── __init__.py │ │ │ ├── autoaugment.py │ │ │ ├── backbone_factory.py │ │ │ ├── efficientnet_builder.py │ │ │ ├── efficientnet_lite_builder.py │ │ │ ├── efficientnet_model.py │ │ │ ├── preprocessing.py │ │ │ └── train_backbone.py │ │ ├── coco_metric.py │ │ ├── dataloader.py │ │ ├── dataset/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── create_coco_tfrecord.py │ │ │ ├── create_pascal_tfrecord.py │ │ │ ├── inspect_tfrecords.py │ │ │ ├── label_map_util.py │ │ │ └── tfrecord_util.py │ │ ├── det_advprop_tutorial.ipynb │ │ ├── det_model_fn.py │ │ ├── efficientdet_arch.py │ │ ├── hparams_config.py │ │ ├── inference.py │ │ ├── install_deps.sh │ │ ├── iou_utils.py │ │ ├── keras/ │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── anchors.py │ │ │ ├── efficientdet_keras.py │ │ │ ├── eval.py │ │ │ ├── eval_tflite.py │ │ │ ├── fpn_configs.py │ │ │ ├── infer.py │ │ │ ├── infer_lib.py │ │ │ ├── inspector.py │ │ │ ├── label_util.py │ │ │ ├── postprocess.py │ │ │ ├── segmentation.py │ │ │ ├── tfmot.py │ │ │ ├── train.py │ │ │ ├── train_lib.py │ │ │ ├── util_keras.py │ │ │ └── wbf.py │ │ ├── main.py │ │ ├── model_inspect.py │ │ ├── nms_np.py │ │ ├── object_detection/ │ │ │ ├── __init__.py │ │ │ ├── argmax_matcher.py │ │ │ ├── box_coder.py │ │ │ ├── box_list.py │ │ │ ├── faster_rcnn_box_coder.py │ │ │ ├── matcher.py │ │ │ ├── preprocessor.py │ │ │ ├── region_similarity_calculator.py │ │ │ ├── shape_utils.py │ │ │ ├── target_assigner.py │ │ │ └── tf_example_decoder.py │ │ ├── requirements.txt │ │ ├── run_tflite.py │ │ ├── tensorrt.py │ │ ├── tutorial.ipynb │ │ ├── utils.py │ │ └── visualize/ │ │ ├── __init__.py │ │ ├── shape_utils.py │ │ ├── standard_fields.py │ │ ├── static_shape.py │ │ └── vis_utils.py │ └── recommendation/ │ └── ml/ │ ├── configs/ │ │ ├── __init__.py │ │ ├── input-config.proto │ │ ├── input_config_pb2.py │ │ ├── model_config.py │ │ └── sample_input_config.pbtxt │ ├── data/ │ │ ├── __init__.py │ │ ├── example_generation_movielens.py │ │ └── example_generation_movielens_test.py │ ├── model/ │ │ ├── __init__.py │ │ ├── context_encoder.py │ │ ├── context_encoder_test.py │ │ ├── dotproduct_similarity.py │ │ ├── dotproduct_similarity_test.py │ │ ├── input_pipeline.py │ │ ├── input_pipeline_test.py │ │ ├── label_encoder.py │ │ ├── label_encoder_test.py │ │ ├── losses.py │ │ ├── losses_test.py │ │ ├── metrics.py │ │ ├── metrics_test.py │ │ ├── recommendation_model.py │ │ ├── recommendation_model_launcher.py │ │ ├── recommendation_model_launcher_test.py │ │ ├── recommendation_model_test.py │ │ └── utils.py │ ├── ondevice_recommendation.ipynb │ └── requirements.txt ├── models/ │ ├── __init__.py │ ├── dcgan/ │ │ ├── __init__.py │ │ ├── dcgan.py │ │ └── dcgan_test.py │ ├── densenet/ │ │ ├── README.md │ │ ├── __init__.py │ │ ├── densenet.py │ │ ├── densenet_distributed_test.py │ │ ├── densenet_test.py │ │ ├── distributed_train.py │ │ ├── train.py │ │ └── utils.py │ ├── nmt_with_attention/ │ │ ├── __init__.py │ │ ├── distributed_test.py │ │ ├── distributed_train.py │ │ ├── nmt.py │ │ ├── nmt_test.py │ │ ├── train.py │ │ └── utils.py │ └── pix2pix/ │ ├── __init__.py │ ├── data_download.py │ ├── pix2pix.py │ └── pix2pix_test.py └── profiling/ ├── imagenet_preprocessing_ineffecient_input_pipeline.py └── resnet_model.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ **/*.pyc **/.DS_Store **/.idea **/.ipynb_checkpoints ================================================ FILE: AUTHORS ================================================ # This is the official list of TensorFlow authors for copyright purposes. # This file is distinct from the CONTRIBUTORS files. # See the latter for an explanation. # Names should be added to this file as: # Name or Organization # The email address is not required for organizations. Google Inc. ================================================ FILE: CODEOWNERS ================================================ # https://help.github.com/articles/about-codeowners/ # Last matching pattern takes preecedence. # Default owners for everything in repo. * @tensorflow/docs-team # Courses /courses/ @josbecker @MarkDaoust # TF Lite /lite/ @khanhlvg @lu-wang-g @miaout17 @terryheo /tensorflow_examples/lite/ @ziyeqinghan @khanhlvg @lu-wang-g @miaout17 @terryheo ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing You don't need to be a developer to make a significant impact on TensorFlow documentation and examples-—just a [GitHub account](https://github.com/). Questions about TensorFlow usage are best addressed on [StackOverflow](https://stackoverflow.com/questions/tagged/tensorflow) or the [discuss@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/discuss) mailing list. To contribute to the TensorFlow code repositories, see the [Contributing to TensorFlow](https://www.tensorflow.org/community/contribute) guide and the [TensorFlow contribution guidelines](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md). ## Contributor License Agreements We love patches! To publish your changes, you must sign either the individual or corporate Contributor License Agreement (CLA): * If you are an individual writing original documentation or source code and you're sure you own the intellectual property, sign an [individual CLA](http://code.google.com/legal/individual-cla-v1.0.html). * If you work for a company that wants to allow you to contribute your work, sign a [corporate CLA](http://code.google.com/legal/corporate-cla-v1.0.html). We can accept your pull requests after you sign the CLA. We can only receive original documentation and source code from you and other people that have signed the CLA. # Pull requests To contribute documentation or code, please send us a pull request. If you are new to pull requests, read GitHub's [Creating a pull request from a fork](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) guide. Notebooks can be viewed, edited, and run in [Colab](https://colab.research.google.com/notebooks/welcome.ipynb) by passing the GitHub path as a URL parameter. For example, open the notebook at https://github.com/tensorflow/docs/blob/r1.11/site/en/tutorials/keras/basic_classification.ipynb in Colab here: https://colab.research.google.com/github/tensorflow/docs/blob/r1.11/site/en/tutorials/keras/basic_classification.ipynb The [Open in Colab](https://chrome.google.com/webstore/detail/open-in-colab/iogfkhleblhcpcekbiedikdehleodpjo) Chrome extension will automatically perform the URL substitution. ================================================ FILE: LICENSE ================================================ Copyright 2018 The TensorFlow Authors. All rights reserved. Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2017, The TensorFlow Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # TensorFlow Examples


Most important links!

* [Community examples](./community) * [Course materials](./courses/udacity_deep_learning) for the [Deep Learning](https://www.udacity.com/course/deep-learning--ud730) class on Udacity If you are looking to learn TensorFlow, don't miss the [core TensorFlow documentation](http://github.com/tensorflow/docs) which is largely runnable code. Those notebooks can be opened in Colab from [tensorflow.org](https://tensorflow.org).

What is this repo?

This is the TensorFlow example repo. It has several classes of material: * Showcase examples and documentation for our fantastic [TensorFlow Community](https://tensorflow.org/community) * Provide examples mentioned on TensorFlow.org * Publish material supporting official TensorFlow courses * Publish supporting material for the [TensorFlow Blog](https://blog.tensorflow.org) and [TensorFlow YouTube Channel](https://youtube.com/tensorflow) We welcome community contributions, see [CONTRIBUTING.md](CONTRIBUTING.md) and, for style help, [Writing TensorFlow documentation](https://www.tensorflow.org/community/contribute/docs_style) guide. To file an issue, use the tracker in the [tensorflow/tensorflow](https://github.com/tensorflow/tensorflow/issues/new?template=20-documentation-issue.md) repo. ## License [Apache License 2.0](LICENSE) ================================================ FILE: __init__.py ================================================ """tensorflow_examples is a package for examples of TensorFlow.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function ================================================ FILE: courses/udacity_deep_learning/.gitignore ================================================ notMNIST_large* notMNIST_small* ================================================ FILE: courses/udacity_deep_learning/1_notmnist.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "5hIbr52I7Z7U" }, "source": [ "Deep Learning\n", "=============\n", "\n", "Assignment 1\n", "------------\n", "\n", "The objective of this assignment is to learn about simple data curation practices, and familiarize you with some of the data we'll be reusing later.\n", "\n", "This notebook uses the [notMNIST](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html) dataset to be used with python experiments. This dataset is designed to look like the classic [MNIST](http://yann.lecun.com/exdb/mnist/) dataset, while looking a little more like real data: it's a harder task, and the data is a lot less 'clean' than MNIST." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "apJbCsBHl-2A" }, "outputs": [], "source": [ "# These are all the modules we'll be using later. Make sure you can import them\n", "# before proceeding further.\n", "from __future__ import print_function\n", "import imageio\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import os\n", "import sys\n", "import tarfile\n", "from IPython.display import display, Image\n", "from sklearn.linear_model import LogisticRegression\n", "from six.moves.urllib.request import urlretrieve\n", "from six.moves import cPickle as pickle\n", "\n", "# Config the matplotlib backend as plotting inline in IPython\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": { "id": "jNWGtZaXn-5j" }, "source": [ "First, we'll download the dataset to our local machine. The data consists of characters rendered in a variety of fonts on a 28x28 image. The labels are limited to 'A' through 'J' (10 classes). The training set has about 500k and the testset 19000 labeled examples. Given these sizes, it should be possible to train models quickly on any machine." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "EYRJ4ICW6-da" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found and verified notMNIST_large.tar.gz\n", "Found and verified notMNIST_small.tar.gz\n" ] } ], "source": [ "url = 'https://commondatastorage.googleapis.com/books1000/'\n", "last_percent_reported = None\n", "data_root = '.' # Change me to store data elsewhere\n", "\n", "def download_progress_hook(count, blockSize, totalSize):\n", " \"\"\"A hook to report the progress of a download. This is mostly intended for users with\n", " slow internet connections. Reports every 5% change in download progress.\n", " \"\"\"\n", " global last_percent_reported\n", " percent = int(count * blockSize * 100 / totalSize)\n", "\n", " if last_percent_reported != percent:\n", " if percent % 5 == 0:\n", " sys.stdout.write(\"%s%%\" % percent)\n", " sys.stdout.flush()\n", " else:\n", " sys.stdout.write(\".\")\n", " sys.stdout.flush()\n", " \n", " last_percent_reported = percent\n", " \n", "def maybe_download(filename, expected_bytes, force=False):\n", " \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n", " dest_filename = os.path.join(data_root, filename)\n", " if force or not os.path.exists(dest_filename):\n", " print('Attempting to download:', filename) \n", " filename, _ = urlretrieve(url + filename, dest_filename, reporthook=download_progress_hook)\n", " print('\\nDownload Complete!')\n", " statinfo = os.stat(dest_filename)\n", " if statinfo.st_size == expected_bytes:\n", " print('Found and verified', dest_filename)\n", " else:\n", " raise Exception(\n", " 'Failed to verify ' + dest_filename + '. Can you get to it with a browser?')\n", " return dest_filename\n", "\n", "train_filename = maybe_download('notMNIST_large.tar.gz', 247336696)\n", "test_filename = maybe_download('notMNIST_small.tar.gz', 8458043)" ] }, { "cell_type": "markdown", "metadata": { "id": "cC3p0oEyF8QT" }, "source": [ "Extract the dataset from the compressed .tar.gz file.\n", "This should give you a set of directories, labeled A through J." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "H8CBE-WZ8nmj" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['notMNIST_large/A', 'notMNIST_large/B', 'notMNIST_large/C', 'notMNIST_large/D', 'notMNIST_large/E', 'notMNIST_large/F', 'notMNIST_large/G', 'notMNIST_large/H', 'notMNIST_large/I', 'notMNIST_large/J']\n", "['notMNIST_small/A', 'notMNIST_small/B', 'notMNIST_small/C', 'notMNIST_small/D', 'notMNIST_small/E', 'notMNIST_small/F', 'notMNIST_small/G', 'notMNIST_small/H', 'notMNIST_small/I', 'notMNIST_small/J']\n" ] } ], "source": [ "num_classes = 10\n", "np.random.seed(133)\n", "\n", "def maybe_extract(filename, force=False):\n", " root = os.path.splitext(os.path.splitext(filename)[0])[0] # remove .tar.gz\n", " if os.path.isdir(root) and not force:\n", " # You may override by setting force=True.\n", " print('%s already present - Skipping extraction of %s.' % (root, filename))\n", " else:\n", " print('Extracting data for %s. This may take a while. Please wait.' % root)\n", " tar = tarfile.open(filename)\n", " sys.stdout.flush()\n", " tar.extractall(data_root)\n", " tar.close()\n", " data_folders = [\n", " os.path.join(root, d) for d in sorted(os.listdir(root))\n", " if os.path.isdir(os.path.join(root, d))]\n", " if len(data_folders) != num_classes:\n", " raise Exception(\n", " 'Expected %d folders, one per class. Found %d instead.' % (\n", " num_classes, len(data_folders)))\n", " print(data_folders)\n", " return data_folders\n", " \n", "train_folders = maybe_extract(train_filename)\n", "test_folders = maybe_extract(test_filename)" ] }, { "cell_type": "markdown", "metadata": { "id": "4riXK3IoHgx6" }, "source": [ "---\n", "Problem 1\n", "---------\n", "\n", "Let's take a peek at some of the data to make sure it looks sensible. Each exemplar should be an image of a character A through J rendered in a different font. Display a sample of the images that we just downloaded. Hint: you can use the package IPython.display.\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "PBdkjESPK8tw" }, "source": [ "Now let's load the data in a more manageable format. Since, depending on your computer setup you might not be able to fit it all in memory, we'll load each class into a separate dataset, store them on disk and curate them independently. Later we'll merge them into a single dataset of manageable size.\n", "\n", "We'll convert the entire dataset into a 3D array (image index, x, y) of floating point values, normalized to have approximately zero mean and standard deviation ~0.5 to make training easier down the road. \n", "\n", "A few images might not be readable, we'll just skip them." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "h7q0XhG3MJdf" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "notMNIST_large/A\n", "Could not read: notMNIST_large/A/Um9tYW5hIEJvbGQucGZi.png : cannot identify image file - it's ok, skipping.\n", "Could not read: notMNIST_large/A/RnJlaWdodERpc3BCb29rSXRhbGljLnR0Zg==.png : cannot identify image file - it's ok, skipping.\n", "Could not read: notMNIST_large/A/SG90IE11c3RhcmQgQlROIFBvc3Rlci50dGY=.png : cannot identify image file - it's ok, skipping.\n", "Full dataset tensor: (52909, 28, 28)\n", "Mean: -0.12848\n", "Standard deviation: 0.425576\n", "notMNIST_large/B\n", "Could not read: notMNIST_large/B/TmlraXNFRi1TZW1pQm9sZEl0YWxpYy5vdGY=.png : cannot identify image file - it's ok, skipping.\n", "Full dataset tensor: (52911, 28, 28)\n", "Mean: -0.00755947\n", "Standard deviation: 0.417272\n", "notMNIST_large/C\n", "Full dataset tensor: (52912, 28, 28)\n", "Mean: -0.142321\n", "Standard deviation: 0.421305\n", "notMNIST_large/D\n", "Could not read: notMNIST_large/D/VHJhbnNpdCBCb2xkLnR0Zg==.png : cannot identify image file - it's ok, skipping.\n", "Full dataset tensor: (52911, 28, 28)\n", "Mean: -0.0574553\n", "Standard deviation: 0.434072\n", "notMNIST_large/E\n", "Full dataset tensor: (52912, 28, 28)\n", "Mean: -0.0701406\n", "Standard deviation: 0.42882\n", "notMNIST_large/F\n", "Full dataset tensor: (52912, 28, 28)\n", "Mean: -0.125914\n", "Standard deviation: 0.429645\n", "notMNIST_large/G\n", "Full dataset tensor: (52912, 28, 28)\n", "Mean: -0.0947771\n", "Standard deviation: 0.421674\n", "notMNIST_large/H\n", "Full dataset tensor: (52912, 28, 28)\n", "Mean: -0.0687667\n", "Standard deviation: 0.430344\n", "notMNIST_large/I\n", "Full dataset tensor: (52912, 28, 28)\n", "Mean: 0.0307405\n", "Standard deviation: 0.449686\n", "notMNIST_large/J\n", "Full dataset tensor: (52911, 28, 28)\n", "Mean: -0.153479\n", "Standard deviation: 0.397169\n", "notMNIST_small/A\n", "Could not read: notMNIST_small/A/RGVtb2NyYXRpY2FCb2xkT2xkc3R5bGUgQm9sZC50dGY=.png : cannot identify image file - it's ok, skipping.\n", "Full dataset tensor: (1872, 28, 28)\n", "Mean: -0.132588\n", "Standard deviation: 0.445923\n", "notMNIST_small/B\n", "Full dataset tensor: (1873, 28, 28)\n", "Mean: 0.00535619\n", "Standard deviation: 0.457054\n", "notMNIST_small/C\n", "Full dataset tensor: (1873, 28, 28)\n", "Mean: -0.141489\n", "Standard deviation: 0.441056\n", "notMNIST_small/D\n", "Full dataset tensor: (1873, 28, 28)\n", "Mean: -0.0492094\n", "Standard deviation: 0.460477\n", "notMNIST_small/E\n", "Full dataset tensor: (1873, 28, 28)\n", "Mean: -0.0598952\n", "Standard deviation: 0.456146\n", "notMNIST_small/F\n", "Could not read: notMNIST_small/F/Q3Jvc3NvdmVyIEJvbGRPYmxpcXVlLnR0Zg==.png : cannot identify image file - it's ok, skipping.\n", "Full dataset tensor: (1872, 28, 28)\n", "Mean: -0.118148\n", "Standard deviation: 0.451134\n", "notMNIST_small/G\n", "Full dataset tensor: (1872, 28, 28)\n", "Mean: -0.092519\n", "Standard deviation: 0.448468\n", "notMNIST_small/H\n", "Full dataset tensor: (1872, 28, 28)\n", "Mean: -0.0586729\n", "Standard deviation: 0.457387\n", "notMNIST_small/I\n", "Full dataset tensor: (1872, 28, 28)\n", "Mean: 0.0526481\n", "Standard deviation: 0.472657\n", "notMNIST_small/J\n", "Full dataset tensor: (1872, 28, 28)\n", "Mean: -0.15167\n", "Standard deviation: 0.449521\n" ] } ], "source": [ "image_size = 28 # Pixel width and height.\n", "pixel_depth = 255.0 # Number of levels per pixel.\n", "\n", "def load_letter(folder, min_num_images):\n", " \"\"\"Load the data for a single letter label.\"\"\"\n", " image_files = os.listdir(folder)\n", " dataset = np.ndarray(shape=(len(image_files), image_size, image_size),\n", " dtype=np.float32)\n", " print(folder)\n", " num_images = 0\n", " for image in image_files:\n", " image_file = os.path.join(folder, image)\n", " try:\n", " image_data = (imageio.imread(image_file).astype(float) - \n", " pixel_depth / 2) / pixel_depth\n", " if image_data.shape != (image_size, image_size):\n", " raise Exception('Unexpected image shape: %s' % str(image_data.shape))\n", " dataset[num_images, :, :] = image_data\n", " num_images = num_images + 1\n", " except (IOError, ValueError) as e:\n", " print('Could not read:', image_file, ':', e, '- it\\'s ok, skipping.')\n", " \n", " dataset = dataset[0:num_images, :, :]\n", " if num_images < min_num_images:\n", " raise Exception('Many fewer images than expected: %d < %d' %\n", " (num_images, min_num_images))\n", " \n", " print('Full dataset tensor:', dataset.shape)\n", " print('Mean:', np.mean(dataset))\n", " print('Standard deviation:', np.std(dataset))\n", " return dataset\n", " \n", "def maybe_pickle(data_folders, min_num_images_per_class, force=False):\n", " dataset_names = []\n", " for folder in data_folders:\n", " set_filename = folder + '.pickle'\n", " dataset_names.append(set_filename)\n", " if os.path.exists(set_filename) and not force:\n", " # You may override by setting force=True.\n", " print('%s already present - Skipping pickling.' % set_filename)\n", " else:\n", " print('Pickling %s.' % set_filename)\n", " dataset = load_letter(folder, min_num_images_per_class)\n", " try:\n", " with open(set_filename, 'wb') as f:\n", " pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)\n", " except Exception as e:\n", " print('Unable to save data to', set_filename, ':', e)\n", " \n", " return dataset_names\n", "\n", "train_datasets = maybe_pickle(train_folders, 45000)\n", "test_datasets = maybe_pickle(test_folders, 1800)" ] }, { "cell_type": "markdown", "metadata": { "id": "vUdbskYE2d87" }, "source": [ "---\n", "Problem 2\n", "---------\n", "\n", "Let's verify that the data still looks good. Displaying a sample of the labels and images from the ndarray. Hint: you can use matplotlib.pyplot.\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "cYznx5jUwzoO" }, "source": [ "---\n", "Problem 3\n", "---------\n", "Another check: we expect the data to be balanced across classes. Verify that.\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "LA7M7K22ynCt" }, "source": [ "Merge and prune the training data as needed. Depending on your computer setup, you might not be able to fit it all in memory, and you can tune `train_size` as needed. The labels will be stored into a separate array of integers 0 through 9.\n", "\n", "Also create a validation dataset for hyperparameter tuning." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "s3mWgZLpyuzq" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training (200000, 28, 28) (200000,)\n", "Validation (10000, 28, 28) (10000,)\n", "Testing (10000, 28, 28) (10000,)\n" ] } ], "source": [ "def make_arrays(nb_rows, img_size):\n", " if nb_rows:\n", " dataset = np.ndarray((nb_rows, img_size, img_size), dtype=np.float32)\n", " labels = np.ndarray(nb_rows, dtype=np.int32)\n", " else:\n", " dataset, labels = None, None\n", " return dataset, labels\n", "\n", "def merge_datasets(pickle_files, train_size, valid_size=0):\n", " num_classes = len(pickle_files)\n", " valid_dataset, valid_labels = make_arrays(valid_size, image_size)\n", " train_dataset, train_labels = make_arrays(train_size, image_size)\n", " vsize_per_class = valid_size // num_classes\n", " tsize_per_class = train_size // num_classes\n", " \n", " start_v, start_t = 0, 0\n", " end_v, end_t = vsize_per_class, tsize_per_class\n", " end_l = vsize_per_class+tsize_per_class\n", " for label, pickle_file in enumerate(pickle_files): \n", " try:\n", " with open(pickle_file, 'rb') as f:\n", " letter_set = pickle.load(f)\n", " # let's shuffle the letters to have random validation and training set\n", " np.random.shuffle(letter_set)\n", " if valid_dataset is not None:\n", " valid_letter = letter_set[:vsize_per_class, :, :]\n", " valid_dataset[start_v:end_v, :, :] = valid_letter\n", " valid_labels[start_v:end_v] = label\n", " start_v += vsize_per_class\n", " end_v += vsize_per_class\n", " \n", " train_letter = letter_set[vsize_per_class:end_l, :, :]\n", " train_dataset[start_t:end_t, :, :] = train_letter\n", " train_labels[start_t:end_t] = label\n", " start_t += tsize_per_class\n", " end_t += tsize_per_class\n", " except Exception as e:\n", " print('Unable to process data from', pickle_file, ':', e)\n", " raise\n", " \n", " return valid_dataset, valid_labels, train_dataset, train_labels\n", " \n", " \n", "train_size = 200000\n", "valid_size = 10000\n", "test_size = 10000\n", "\n", "valid_dataset, valid_labels, train_dataset, train_labels = merge_datasets(\n", " train_datasets, train_size, valid_size)\n", "_, _, test_dataset, test_labels = merge_datasets(test_datasets, test_size)\n", "\n", "print('Training:', train_dataset.shape, train_labels.shape)\n", "print('Validation:', valid_dataset.shape, valid_labels.shape)\n", "print('Testing:', test_dataset.shape, test_labels.shape)" ] }, { "cell_type": "markdown", "metadata": { "id": "GPTCnjIcyuKN" }, "source": [ "Next, we'll randomize the data. It's important to have the labels well shuffled for the training and test distributions to match." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "6WZ2l2tN2zOL" }, "outputs": [], "source": [ "def randomize(dataset, labels):\n", " permutation = np.random.permutation(labels.shape[0])\n", " shuffled_dataset = dataset[permutation,:,:]\n", " shuffled_labels = labels[permutation]\n", " return shuffled_dataset, shuffled_labels\n", "train_dataset, train_labels = randomize(train_dataset, train_labels)\n", "test_dataset, test_labels = randomize(test_dataset, test_labels)\n", "valid_dataset, valid_labels = randomize(valid_dataset, valid_labels)" ] }, { "cell_type": "markdown", "metadata": { "id": "puDUTe6t6USl" }, "source": [ "---\n", "Problem 4\n", "---------\n", "Convince yourself that the data is still good after shuffling!\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "tIQJaJuwg5Hw" }, "source": [ "Finally, let's save the data for later reuse:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "QiR_rETzem6C" }, "outputs": [], "source": [ "pickle_file = os.path.join(data_root, 'notMNIST.pickle')\n", "\n", "try:\n", " f = open(pickle_file, 'wb')\n", " save = {\n", " 'train_dataset': train_dataset,\n", " 'train_labels': train_labels,\n", " 'valid_dataset': valid_dataset,\n", " 'valid_labels': valid_labels,\n", " 'test_dataset': test_dataset,\n", " 'test_labels': test_labels,\n", " }\n", " pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)\n", " f.close()\n", "except Exception as e:\n", " print('Unable to save data to', pickle_file, ':', e)\n", " raise" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "hQbLjrW_iT39" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Compressed pickle size: 718193801\n" ] } ], "source": [ "statinfo = os.stat(pickle_file)\n", "print('Compressed pickle size:', statinfo.st_size)" ] }, { "cell_type": "markdown", "metadata": { "id": "gE_cRAQB33lk" }, "source": [ "---\n", "Problem 5\n", "---------\n", "\n", "By construction, this dataset might contain a lot of overlapping samples, including training data that's also contained in the validation and test set! Overlap between training and test can skew the results if you expect to use your model in an environment where there is never an overlap, but are actually ok if you expect to see training samples recur when you use it.\n", "Measure how much overlap there is between training, validation and test samples.\n", "\n", "Optional questions:\n", "- What about near duplicates between datasets? (images that are almost identical)\n", "- Create a sanitized validation and test set, and compare your accuracy on those in subsequent assignments.\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "L8oww1s4JMQx" }, "source": [ "---\n", "Problem 6\n", "---------\n", "\n", "Let's get an idea of what an off-the-shelf classifier can give you on this data. It's always good to check that there is something to learn, and that it's a problem that is not so trivial that a canned solution solves it.\n", "\n", "Train a simple model on this data using 50, 100, 1000 and 5000 training samples. Hint: you can use the LogisticRegression model from sklearn.linear_model.\n", "\n", "Optional question: train an off-the-shelf model on all the data!\n", "\n", "---" ] } ], "metadata": { "colab": { "name": "1_notmnist.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_deep_learning/2_fullyconnected.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "kR-4eNdK6lYS" }, "source": [ "Deep Learning\n", "=============\n", "\n", "Assignment 2\n", "------------\n", "\n", "Previously in `1_notmnist.ipynb`, we created a pickle with formatted datasets for training, development and testing on the [notMNIST dataset](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html).\n", "\n", "The goal of this assignment is to progressively train deeper and more accurate models using TensorFlow." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "JLpLa8Jt7Vu4" }, "outputs": [], "source": [ "# These are all the modules we'll be using later. Make sure you can import them\n", "# before proceeding further.\n", "from __future__ import print_function\n", "import numpy as np\n", "import tensorflow as tf\n", "from six.moves import cPickle as pickle\n", "from six.moves import range" ] }, { "cell_type": "markdown", "metadata": { "id": "1HrCK6e17WzV" }, "source": [ "First reload the data we generated in `1_notmnist.ipynb`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "y3-cj1bpmuxc" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training set (200000, 28, 28) (200000,)\n", "Validation set (10000, 28, 28) (10000,)\n", "Test set (18724, 28, 28) (18724,)\n" ] } ], "source": [ "pickle_file = 'notMNIST.pickle'\n", "\n", "with open(pickle_file, 'rb') as f:\n", " save = pickle.load(f)\n", " train_dataset = save['train_dataset']\n", " train_labels = save['train_labels']\n", " valid_dataset = save['valid_dataset']\n", " valid_labels = save['valid_labels']\n", " test_dataset = save['test_dataset']\n", " test_labels = save['test_labels']\n", " del save # hint to help gc free up memory\n", " print('Training set', train_dataset.shape, train_labels.shape)\n", " print('Validation set', valid_dataset.shape, valid_labels.shape)\n", " print('Test set', test_dataset.shape, test_labels.shape)" ] }, { "cell_type": "markdown", "metadata": { "id": "L7aHrm6nGDMB" }, "source": [ "Reformat into a shape that's more adapted to the models we're going to train:\n", "- data as a flat matrix,\n", "- labels as float 1-hot encodings." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "IRSyYiIIGIzS" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training set (200000, 784) (200000, 10)\n", "Validation set (10000, 784) (10000, 10)\n", "Test set (18724, 784) (18724, 10)\n" ] } ], "source": [ "image_size = 28\n", "num_labels = 10\n", "\n", "def reformat(dataset, labels):\n", " dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)\n", " # Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...]\n", " labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n", " return dataset, labels\n", "train_dataset, train_labels = reformat(train_dataset, train_labels)\n", "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n", "test_dataset, test_labels = reformat(test_dataset, test_labels)\n", "print('Training set', train_dataset.shape, train_labels.shape)\n", "print('Validation set', valid_dataset.shape, valid_labels.shape)\n", "print('Test set', test_dataset.shape, test_labels.shape)" ] }, { "cell_type": "markdown", "metadata": { "id": "nCLVqyQ5vPPH" }, "source": [ "We're first going to train a multinomial logistic regression using simple gradient descent.\n", "\n", "TensorFlow works like this:\n", "* First you describe the computation that you want to see performed: what the inputs, the variables, and the operations look like. These get created as nodes over a computation graph. This description is all contained within the block below:\n", "\n", " with graph.as_default():\n", " ...\n", "\n", "* Then you can run the operations on this graph as many times as you want by calling `session.run()`, providing it outputs to fetch from the graph that get returned. This runtime operation is all contained in the block below:\n", "\n", " with tf.Session(graph=graph) as session:\n", " ...\n", "\n", "Let's load all the data into TensorFlow and build the computation graph corresponding to our training:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "Nfv39qvtvOl_" }, "outputs": [], "source": [ "# With gradient descent training, even this much data is prohibitive.\n", "# Subset the training data for faster turnaround.\n", "train_subset = 10000\n", "\n", "graph = tf.Graph()\n", "with graph.as_default():\n", "\n", " # Input data.\n", " # Load the training, validation and test data into constants that are\n", " # attached to the graph.\n", " tf_train_dataset = tf.constant(train_dataset[:train_subset, :])\n", " tf_train_labels = tf.constant(train_labels[:train_subset])\n", " tf_valid_dataset = tf.constant(valid_dataset)\n", " tf_test_dataset = tf.constant(test_dataset)\n", " \n", " # Variables.\n", " # These are the parameters that we are going to be training. The weight\n", " # matrix will be initialized using random values following a (truncated)\n", " # normal distribution. The biases get initialized to zero.\n", " weights = tf.Variable(\n", " tf.truncated_normal([image_size * image_size, num_labels]))\n", " biases = tf.Variable(tf.zeros([num_labels]))\n", " \n", " # Training computation.\n", " # We multiply the inputs with the weight matrix, and add biases. We compute\n", " # the softmax and cross-entropy (it's one operation in TensorFlow, because\n", " # it's very common, and it can be optimized). We take the average of this\n", " # cross-entropy across all training examples: that's our loss.\n", " logits = tf.matmul(tf_train_dataset, weights) + biases\n", " loss = tf.reduce_mean(\n", " tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n", " \n", " # Optimizer.\n", " # We are going to find the minimum of this loss using gradient descent.\n", " optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n", " \n", " # Predictions for the training, validation, and test data.\n", " # These are not part of training, but merely here so that we can report\n", " # accuracy figures as we train.\n", " train_prediction = tf.nn.softmax(logits)\n", " valid_prediction = tf.nn.softmax(\n", " tf.matmul(tf_valid_dataset, weights) + biases)\n", " test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)" ] }, { "cell_type": "markdown", "metadata": { "id": "KQcL4uqISHjP" }, "source": [ "Let's run this computation and iterate:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "z2cjdenH869W" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initialized\n", "Loss at step 0 : 17.2939\n", "Training accuracy: 10.8%\n", "Validation accuracy: 13.8%\n", "Loss at step 100 : 2.26903\n", "Training accuracy: 72.3%\n", "Validation accuracy: 71.6%\n", "Loss at step 200 : 1.84895\n", "Training accuracy: 74.9%\n", "Validation accuracy: 73.9%\n", "Loss at step 300 : 1.60701\n", "Training accuracy: 76.0%\n", "Validation accuracy: 74.5%\n", "Loss at step 400 : 1.43912\n", "Training accuracy: 76.8%\n", "Validation accuracy: 74.8%\n", "Loss at step 500 : 1.31349\n", "Training accuracy: 77.5%\n", "Validation accuracy: 75.0%\n", "Loss at step 600 : 1.21501\n", "Training accuracy: 78.1%\n", "Validation accuracy: 75.4%\n", "Loss at step 700 : 1.13515\n", "Training accuracy: 78.6%\n", "Validation accuracy: 75.4%\n", "Loss at step 800 : 1.0687\n", "Training accuracy: 79.2%\n", "Validation accuracy: 75.6%\n", "Test accuracy: 82.9%\n" ] } ], "source": [ "num_steps = 801\n", "\n", "def accuracy(predictions, labels):\n", " return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n", " / predictions.shape[0])\n", "\n", "with tf.Session(graph=graph) as session:\n", " # This is a one-time operation which ensures the parameters get initialized as\n", " # we described in the graph: random weights for the matrix, zeros for the\n", " # biases. \n", " tf.global_variables_initializer().run()\n", " print('Initialized')\n", " for step in range(num_steps):\n", " # Run the computations. We tell .run() that we want to run the optimizer,\n", " # and get the loss value and the training predictions returned as numpy\n", " # arrays.\n", " _, l, predictions = session.run([optimizer, loss, train_prediction])\n", " if (step % 100 == 0):\n", " print('Loss at step %d: %f' % (step, l))\n", " print('Training accuracy: %.1f%%' % accuracy(\n", " predictions, train_labels[:train_subset, :]))\n", " # Calling .eval() on valid_prediction is basically like calling run(), but\n", " # just to get that one numpy array. Note that it recomputes all its graph\n", " # dependencies.\n", " print('Validation accuracy: %.1f%%' % accuracy(\n", " valid_prediction.eval(), valid_labels))\n", " print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))" ] }, { "cell_type": "markdown", "metadata": { "id": "x68f-hxRGm3H" }, "source": [ "Let's now switch to stochastic gradient descent training instead, which is much faster.\n", "\n", "The graph will be similar, except that instead of holding all the training data into a constant node, we create a `Placeholder` node which will be fed actual data at every call of `session.run()`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "qhPMzWYRGrzM" }, "outputs": [], "source": [ "batch_size = 128\n", "\n", "graph = tf.Graph()\n", "with graph.as_default():\n", "\n", " # Input data. For the training data, we use a placeholder that will be fed\n", " # at run time with a training minibatch.\n", " tf_train_dataset = tf.placeholder(tf.float32,\n", " shape=(batch_size, image_size * image_size))\n", " tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n", " tf_valid_dataset = tf.constant(valid_dataset)\n", " tf_test_dataset = tf.constant(test_dataset)\n", " \n", " # Variables.\n", " weights = tf.Variable(\n", " tf.truncated_normal([image_size * image_size, num_labels]))\n", " biases = tf.Variable(tf.zeros([num_labels]))\n", " \n", " # Training computation.\n", " logits = tf.matmul(tf_train_dataset, weights) + biases\n", " loss = tf.reduce_mean(\n", " tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n", " \n", " # Optimizer.\n", " optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n", " \n", " # Predictions for the training, validation, and test data.\n", " train_prediction = tf.nn.softmax(logits)\n", " valid_prediction = tf.nn.softmax(\n", " tf.matmul(tf_valid_dataset, weights) + biases)\n", " test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)" ] }, { "cell_type": "markdown", "metadata": { "id": "XmVZESmtG4JH" }, "source": [ "Let's run it:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "FoF91pknG_YW" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initialized\n", "Minibatch loss at step 0 : 16.8091\n", "Minibatch accuracy: 12.5%\n", "Validation accuracy: 14.0%\n", "Minibatch loss at step 500 : 1.75256\n", "Minibatch accuracy: 77.3%\n", "Validation accuracy: 75.0%\n", "Minibatch loss at step 1000 : 1.32283\n", "Minibatch accuracy: 77.3%\n", "Validation accuracy: 76.6%\n", "Minibatch loss at step 1500 : 0.944533\n", "Minibatch accuracy: 83.6%\n", "Validation accuracy: 76.5%\n", "Minibatch loss at step 2000 : 1.03795\n", "Minibatch accuracy: 78.9%\n", "Validation accuracy: 77.8%\n", "Minibatch loss at step 2500 : 1.10219\n", "Minibatch accuracy: 80.5%\n", "Validation accuracy: 78.0%\n", "Minibatch loss at step 3000 : 0.758874\n", "Minibatch accuracy: 82.8%\n", "Validation accuracy: 78.8%\n", "Test accuracy: 86.1%\n" ] } ], "source": [ "num_steps = 3001\n", "\n", "with tf.Session(graph=graph) as session:\n", " tf.global_variables_initializer().run()\n", " print(\"Initialized\")\n", " for step in range(num_steps):\n", " # Pick an offset within the training data, which has been randomized.\n", " # Note: we could use better randomization across epochs.\n", " offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n", " # Generate a minibatch.\n", " batch_data = train_dataset[offset:(offset + batch_size), :]\n", " batch_labels = train_labels[offset:(offset + batch_size), :]\n", " # Prepare a dictionary telling the session where to feed the minibatch.\n", " # The key of the dictionary is the placeholder node of the graph to be fed,\n", " # and the value is the numpy array to feed to it.\n", " feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n", " _, l, predictions = session.run(\n", " [optimizer, loss, train_prediction], feed_dict=feed_dict)\n", " if (step % 500 == 0):\n", " print(\"Minibatch loss at step %d: %f\" % (step, l))\n", " print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n", " print(\"Validation accuracy: %.1f%%\" % accuracy(\n", " valid_prediction.eval(), valid_labels))\n", " print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))" ] }, { "cell_type": "markdown", "metadata": { "id": "7omWxtvLLxik" }, "source": [ "---\n", "Problem\n", "-------\n", "\n", "Turn the logistic regression example with SGD into a 1-hidden layer neural network with rectified linear units [nn.relu()](https://www.tensorflow.org/versions/r0.7/api_docs/python/nn.html#relu) and 1024 hidden nodes. This model should improve your validation / test accuracy.\n", "\n", "---" ] } ], "metadata": { "colab": { "name": "2_fullyconnected.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_deep_learning/3_regularization.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "kR-4eNdK6lYS" }, "source": [ "Deep Learning\n", "=============\n", "\n", "Assignment 3\n", "------------\n", "\n", "Previously in `2_fullyconnected.ipynb`, you trained a logistic regression and a neural network model.\n", "\n", "The goal of this assignment is to explore regularization techniques." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "JLpLa8Jt7Vu4" }, "outputs": [], "source": [ "# These are all the modules we'll be using later. Make sure you can import them\n", "# before proceeding further.\n", "from __future__ import print_function\n", "import numpy as np\n", "import tensorflow as tf\n", "from six.moves import cPickle as pickle" ] }, { "cell_type": "markdown", "metadata": { "id": "1HrCK6e17WzV" }, "source": [ "First reload the data we generated in `1_notmnist.ipynb`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "y3-cj1bpmuxc" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training set (200000, 28, 28) (200000,)\n", "Validation set (10000, 28, 28) (10000,)\n", "Test set (18724, 28, 28) (18724,)\n" ] } ], "source": [ "pickle_file = 'notMNIST.pickle'\n", "\n", "with open(pickle_file, 'rb') as f:\n", " save = pickle.load(f)\n", " train_dataset = save['train_dataset']\n", " train_labels = save['train_labels']\n", " valid_dataset = save['valid_dataset']\n", " valid_labels = save['valid_labels']\n", " test_dataset = save['test_dataset']\n", " test_labels = save['test_labels']\n", " del save # hint to help gc free up memory\n", " print('Training set', train_dataset.shape, train_labels.shape)\n", " print('Validation set', valid_dataset.shape, valid_labels.shape)\n", " print('Test set', test_dataset.shape, test_labels.shape)" ] }, { "cell_type": "markdown", "metadata": { "id": "L7aHrm6nGDMB" }, "source": [ "Reformat into a shape that's more adapted to the models we're going to train:\n", "- data as a flat matrix,\n", "- labels as float 1-hot encodings." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "IRSyYiIIGIzS" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training set (200000, 784) (200000, 10)\n", "Validation set (10000, 784) (10000, 10)\n", "Test set (18724, 784) (18724, 10)\n" ] } ], "source": [ "image_size = 28\n", "num_labels = 10\n", "\n", "def reformat(dataset, labels):\n", " dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)\n", " # Map 1 to [0.0, 1.0, 0.0 ...], 2 to [0.0, 0.0, 1.0 ...]\n", " labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n", " return dataset, labels\n", "train_dataset, train_labels = reformat(train_dataset, train_labels)\n", "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n", "test_dataset, test_labels = reformat(test_dataset, test_labels)\n", "print('Training set', train_dataset.shape, train_labels.shape)\n", "print('Validation set', valid_dataset.shape, valid_labels.shape)\n", "print('Test set', test_dataset.shape, test_labels.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "RajPLaL_ZW6w" }, "outputs": [], "source": [ "def accuracy(predictions, labels):\n", " return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n", " / predictions.shape[0])" ] }, { "cell_type": "markdown", "metadata": { "id": "sgLbUAQ1CW-1" }, "source": [ "---\n", "Problem 1\n", "---------\n", "\n", "Introduce and tune L2 regularization for both logistic and neural network models. Remember that L2 amounts to adding a penalty on the norm of the weights to the loss. In TensorFlow, you can compute the L2 loss for a tensor `t` using `nn.l2_loss(t)`. The right amount of regularization should improve your validation / test accuracy.\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "na8xX2yHZzNF" }, "source": [ "---\n", "Problem 2\n", "---------\n", "Let's demonstrate an extreme case of overfitting. Restrict your training data to just a few batches. What happens?\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "ww3SCBUdlkRc" }, "source": [ "---\n", "Problem 3\n", "---------\n", "Introduce Dropout on the hidden layer of the neural network. Remember: Dropout should only be introduced during training, not evaluation, otherwise your evaluation results would be stochastic as well. TensorFlow provides `nn.dropout()` for that, but you have to make sure it's only inserted during training.\n", "\n", "What happens to our extreme overfitting case?\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "-b1hTz3VWZjw" }, "source": [ "---\n", "Problem 4\n", "---------\n", "\n", "Try to get the best performance you can using a multi-layer model! The best reported test accuracy using a deep network is [97.1%](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html?showComment=1391023266211#c8758720086795711595).\n", "\n", "One avenue you can explore is to add multiple layers.\n", "\n", "Another one is to use learning rate decay:\n", "\n", " global_step = tf.Variable(0) # count the number of steps taken.\n", " learning_rate = tf.train.exponential_decay(0.5, global_step, ...)\n", " optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)\n", " \n", " ---\n" ] } ], "metadata": { "colab": { "name": "3_regularization.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_deep_learning/4_convolutions.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "4embtkV0pNxM" }, "source": [ "Deep Learning\n", "=============\n", "\n", "Assignment 4\n", "------------\n", "\n", "Previously in `2_fullyconnected.ipynb` and `3_regularization.ipynb`, we trained fully connected networks to classify [notMNIST](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html) characters.\n", "\n", "The goal of this assignment is make the neural network convolutional." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "tm2CQN_Cpwj0" }, "outputs": [], "source": [ "# These are all the modules we'll be using later. Make sure you can import them\n", "# before proceeding further.\n", "from __future__ import print_function\n", "import numpy as np\n", "import tensorflow as tf\n", "from six.moves import cPickle as pickle\n", "from six.moves import range" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "y3-cj1bpmuxc" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training set (200000, 28, 28) (200000,)\n", "Validation set (10000, 28, 28) (10000,)\n", "Test set (18724, 28, 28) (18724,)\n" ] } ], "source": [ "pickle_file = 'notMNIST.pickle'\n", "\n", "with open(pickle_file, 'rb') as f:\n", " save = pickle.load(f)\n", " train_dataset = save['train_dataset']\n", " train_labels = save['train_labels']\n", " valid_dataset = save['valid_dataset']\n", " valid_labels = save['valid_labels']\n", " test_dataset = save['test_dataset']\n", " test_labels = save['test_labels']\n", " del save # hint to help gc free up memory\n", " print('Training set', train_dataset.shape, train_labels.shape)\n", " print('Validation set', valid_dataset.shape, valid_labels.shape)\n", " print('Test set', test_dataset.shape, test_labels.shape)" ] }, { "cell_type": "markdown", "metadata": { "id": "L7aHrm6nGDMB" }, "source": [ "Reformat into a TensorFlow-friendly shape:\n", "- convolutions need the image data formatted as a cube (width by height by #channels)\n", "- labels as float 1-hot encodings." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "IRSyYiIIGIzS" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training set (200000, 28, 28, 1) (200000, 10)\n", "Validation set (10000, 28, 28, 1) (10000, 10)\n", "Test set (18724, 28, 28, 1) (18724, 10)\n" ] } ], "source": [ "image_size = 28\n", "num_labels = 10\n", "num_channels = 1 # grayscale\n", "\n", "import numpy as np\n", "\n", "def reformat(dataset, labels):\n", " dataset = dataset.reshape(\n", " (-1, image_size, image_size, num_channels)).astype(np.float32)\n", " labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n", " return dataset, labels\n", "train_dataset, train_labels = reformat(train_dataset, train_labels)\n", "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n", "test_dataset, test_labels = reformat(test_dataset, test_labels)\n", "print('Training set', train_dataset.shape, train_labels.shape)\n", "print('Validation set', valid_dataset.shape, valid_labels.shape)\n", "print('Test set', test_dataset.shape, test_labels.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "AgQDIREv02p1" }, "outputs": [], "source": [ "def accuracy(predictions, labels):\n", " return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n", " / predictions.shape[0])" ] }, { "cell_type": "markdown", "metadata": { "id": "5rhgjmROXu2O" }, "source": [ "Let's build a small network with two convolutional layers, followed by one fully connected layer. Convolutional networks are more expensive computationally, so we'll limit its depth and number of fully connected nodes." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "IZYv70SvvOan" }, "outputs": [], "source": [ "batch_size = 16\n", "patch_size = 5\n", "depth = 16\n", "num_hidden = 64\n", "\n", "graph = tf.Graph()\n", "\n", "with graph.as_default():\n", "\n", " # Input data.\n", " tf_train_dataset = tf.placeholder(\n", " tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n", " tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n", " tf_valid_dataset = tf.constant(valid_dataset)\n", " tf_test_dataset = tf.constant(test_dataset)\n", " \n", " # Variables.\n", " layer1_weights = tf.Variable(tf.truncated_normal(\n", " [patch_size, patch_size, num_channels, depth], stddev=0.1))\n", " layer1_biases = tf.Variable(tf.zeros([depth]))\n", " layer2_weights = tf.Variable(tf.truncated_normal(\n", " [patch_size, patch_size, depth, depth], stddev=0.1))\n", " layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))\n", " layer3_weights = tf.Variable(tf.truncated_normal(\n", " [image_size // 4 * image_size // 4 * depth, num_hidden], stddev=0.1))\n", " layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))\n", " layer4_weights = tf.Variable(tf.truncated_normal(\n", " [num_hidden, num_labels], stddev=0.1))\n", " layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))\n", " \n", " # Model.\n", " def model(data):\n", " conv = tf.nn.conv2d(data, layer1_weights, [1, 2, 2, 1], padding='SAME')\n", " hidden = tf.nn.relu(conv + layer1_biases)\n", " conv = tf.nn.conv2d(hidden, layer2_weights, [1, 2, 2, 1], padding='SAME')\n", " hidden = tf.nn.relu(conv + layer2_biases)\n", " shape = hidden.get_shape().as_list()\n", " reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])\n", " hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_biases)\n", " return tf.matmul(hidden, layer4_weights) + layer4_biases\n", " \n", " # Training computation.\n", " logits = model(tf_train_dataset)\n", " loss = tf.reduce_mean(\n", " tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n", " \n", " # Optimizer.\n", " optimizer = tf.train.GradientDescentOptimizer(0.05).minimize(loss)\n", " \n", " # Predictions for the training, validation, and test data.\n", " train_prediction = tf.nn.softmax(logits)\n", " valid_prediction = tf.nn.softmax(model(tf_valid_dataset))\n", " test_prediction = tf.nn.softmax(model(tf_test_dataset))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "noKFb2UovVFR" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initialized\n", "Minibatch loss at step 0 : 3.51275\n", "Minibatch accuracy: 6.2%\n", "Validation accuracy: 12.8%\n", "Minibatch loss at step 50 : 1.48703\n", "Minibatch accuracy: 43.8%\n", "Validation accuracy: 50.4%\n", "Minibatch loss at step 100 : 1.04377\n", "Minibatch accuracy: 68.8%\n", "Validation accuracy: 67.4%\n", "Minibatch loss at step 150 : 0.601682\n", "Minibatch accuracy: 68.8%\n", "Validation accuracy: 73.0%\n", "Minibatch loss at step 200 : 0.898649\n", "Minibatch accuracy: 75.0%\n", "Validation accuracy: 77.8%\n", "Minibatch loss at step 250 : 1.3637\n", "Minibatch accuracy: 56.2%\n", "Validation accuracy: 75.4%\n", "Minibatch loss at step 300 : 1.41968\n", "Minibatch accuracy: 62.5%\n", "Validation accuracy: 76.0%\n", "Minibatch loss at step 350 : 0.300648\n", "Minibatch accuracy: 81.2%\n", "Validation accuracy: 80.2%\n", "Minibatch loss at step 400 : 1.32092\n", "Minibatch accuracy: 56.2%\n", "Validation accuracy: 80.4%\n", "Minibatch loss at step 450 : 0.556701\n", "Minibatch accuracy: 81.2%\n", "Validation accuracy: 79.4%\n", "Minibatch loss at step 500 : 1.65595\n", "Minibatch accuracy: 43.8%\n", "Validation accuracy: 79.6%\n", "Minibatch loss at step 550 : 1.06995\n", "Minibatch accuracy: 75.0%\n", "Validation accuracy: 81.2%\n", "Minibatch loss at step 600 : 0.223684\n", "Minibatch accuracy: 100.0%\n", "Validation accuracy: 82.3%\n", "Minibatch loss at step 650 : 0.619602\n", "Minibatch accuracy: 87.5%\n", "Validation accuracy: 81.8%\n", "Minibatch loss at step 700 : 0.812091\n", "Minibatch accuracy: 75.0%\n", "Validation accuracy: 82.4%\n", "Minibatch loss at step 750 : 0.276302\n", "Minibatch accuracy: 87.5%\n", "Validation accuracy: 82.3%\n", "Minibatch loss at step 800 : 0.450241\n", "Minibatch accuracy: 81.2%\n", "Validation accuracy: 82.3%\n", "Minibatch loss at step 850 : 0.137139\n", "Minibatch accuracy: 93.8%\n", "Validation accuracy: 82.3%\n", "Minibatch loss at step 900 : 0.52664\n", "Minibatch accuracy: 75.0%\n", "Validation accuracy: 82.2%\n", "Minibatch loss at step 950 : 0.623835\n", "Minibatch accuracy: 87.5%\n", "Validation accuracy: 82.1%\n", "Minibatch loss at step 1000 : 0.243114\n", "Minibatch accuracy: 93.8%\n", "Validation accuracy: 82.9%\n", "Test accuracy: 90.0%\n" ] } ], "source": [ "num_steps = 1001\n", "\n", "with tf.Session(graph=graph) as session:\n", " tf.global_variables_initializer().run()\n", " print('Initialized')\n", " for step in range(num_steps):\n", " offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n", " batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n", " batch_labels = train_labels[offset:(offset + batch_size), :]\n", " feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n", " _, l, predictions = session.run(\n", " [optimizer, loss, train_prediction], feed_dict=feed_dict)\n", " if (step % 50 == 0):\n", " print('Minibatch loss at step %d: %f' % (step, l))\n", " print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n", " print('Validation accuracy: %.1f%%' % accuracy(\n", " valid_prediction.eval(), valid_labels))\n", " print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))" ] }, { "cell_type": "markdown", "metadata": { "id": "KedKkn4EutIK" }, "source": [ "---\n", "Problem 1\n", "---------\n", "\n", "The convolutional model above uses convolutions with stride 2 to reduce the dimensionality. Replace the strides by a max pooling operation (`nn.max_pool()`) of stride 2 and kernel size 2.\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "klf21gpbAgb-" }, "source": [ "---\n", "Problem 2\n", "---------\n", "\n", "Try to get the best performance you can using a convolutional net. Look for example at the classic [LeNet5](http://yann.lecun.com/exdb/lenet/) architecture, adding Dropout, and/or adding learning rate decay.\n", "\n", "---" ] } ], "metadata": { "colab": { "name": "4_convolutions.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_deep_learning/5_word2vec.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "D7tqLMoKF6uq" }, "source": [ "Deep Learning\n", "=============\n", "\n", "Assignment 5\n", "------------\n", "\n", "The goal of this assignment is to train a Word2Vec skip-gram model over [Text8](http://mattmahoney.net/dc/textdata) data." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "0K1ZyLn04QZf" }, "outputs": [], "source": [ "# These are all the modules we'll be using later. Make sure you can import them\n", "# before proceeding further.\n", "%matplotlib inline\n", "from __future__ import print_function\n", "import collections\n", "import math\n", "import numpy as np\n", "import os\n", "import random\n", "import tensorflow as tf\n", "import zipfile\n", "from matplotlib import pylab\n", "from six.moves import range\n", "from six.moves.urllib.request import urlretrieve\n", "from sklearn.manifold import TSNE" ] }, { "cell_type": "markdown", "metadata": { "id": "aCjPJE944bkV" }, "source": [ "Download the data from the source website if necessary." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "RJ-o3UBUFtCw" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found and verified text8.zip\n" ] } ], "source": [ "url = 'http://mattmahoney.net/dc/'\n", "\n", "def maybe_download(filename, expected_bytes):\n", " \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n", " if not os.path.exists(filename):\n", " filename, _ = urlretrieve(url + filename, filename)\n", " statinfo = os.stat(filename)\n", " if statinfo.st_size == expected_bytes:\n", " print('Found and verified %s' % filename)\n", " else:\n", " print(statinfo.st_size)\n", " raise Exception(\n", " 'Failed to verify ' + filename + '. Can you get to it with a browser?')\n", " return filename\n", "\n", "filename = maybe_download('text8.zip', 31344016)" ] }, { "cell_type": "markdown", "metadata": { "id": "Zqz3XiqI4mZT" }, "source": [ "Read the data into a string." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "Mvf09fjugFU_" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Data size 17005207\n" ] } ], "source": [ "def read_data(filename):\n", " \"\"\"Extract the first file enclosed in a zip file as a list of words\"\"\"\n", " with zipfile.ZipFile(filename) as f:\n", " data = tf.compat.as_str(f.read(f.namelist()[0])).split()\n", " return data\n", " \n", "words = read_data(filename)\n", "print('Data size %d' % len(words))" ] }, { "cell_type": "markdown", "metadata": { "id": "Zdw6i4F8glpp" }, "source": [ "Build the dictionary and replace rare words with UNK token." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "gAL1EECXeZsD" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Most common words (+UNK) [['UNK', 418391], ('the', 1061396), ('of', 593677), ('and', 416629), ('one', 411764)]\n", "Sample data [5243, 3083, 12, 6, 195, 2, 3136, 46, 59, 156]\n" ] } ], "source": [ "vocabulary_size = 50000\n", "\n", "def build_dataset(words):\n", " count = [['UNK', -1]]\n", " count.extend(collections.Counter(words).most_common(vocabulary_size - 1))\n", " dictionary = dict()\n", " for word, _ in count:\n", " dictionary[word] = len(dictionary)\n", " data = list()\n", " unk_count = 0\n", " for word in words:\n", " if word in dictionary:\n", " index = dictionary[word]\n", " else:\n", " index = 0 # dictionary['UNK']\n", " unk_count = unk_count + 1\n", " data.append(index)\n", " count[0][1] = unk_count\n", " reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) \n", " return data, count, dictionary, reverse_dictionary\n", "\n", "data, count, dictionary, reverse_dictionary = build_dataset(words)\n", "print('Most common words (+UNK)', count[:5])\n", "print('Sample data', data[:10])\n", "del words # Hint to reduce memory." ] }, { "cell_type": "markdown", "metadata": { "id": "lFwoyygOmWsL" }, "source": [ "Function to generate a training batch for the skip-gram model." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "w9APjA-zmfjV" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "data: ['anarchism', 'originated', 'as', 'a', 'term', 'of', 'abuse', 'first']\n", "\n", "with num_skips = 2 and skip_window = 1:\n", " batch: ['originated', 'originated', 'as', 'as', 'a', 'a', 'term', 'term']\n", " labels: ['as', 'anarchism', 'a', 'originated', 'term', 'as', 'a', 'of']\n", "\n", "with num_skips = 4 and skip_window = 2:\n", " batch: ['as', 'as', 'as', 'as', 'a', 'a', 'a', 'a']\n", " labels: ['anarchism', 'originated', 'term', 'a', 'as', 'of', 'originated', 'term']\n" ] } ], "source": [ "data_index = 0\n", "\n", "def generate_batch(batch_size, num_skips, skip_window):\n", " global data_index\n", " assert batch_size % num_skips == 0\n", " assert num_skips <= 2 * skip_window\n", " batch = np.ndarray(shape=(batch_size), dtype=np.int32)\n", " labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n", " span = 2 * skip_window + 1 # [ skip_window target skip_window ]\n", " buffer = collections.deque(maxlen=span)\n", " for _ in range(span):\n", " buffer.append(data[data_index])\n", " data_index = (data_index + 1) % len(data)\n", " for i in range(batch_size // num_skips):\n", " target = skip_window # target label at the center of the buffer\n", " targets_to_avoid = [ skip_window ]\n", " for j in range(num_skips):\n", " while target in targets_to_avoid:\n", " target = random.randint(0, span - 1)\n", " targets_to_avoid.append(target)\n", " batch[i * num_skips + j] = buffer[skip_window]\n", " labels[i * num_skips + j, 0] = buffer[target]\n", " buffer.append(data[data_index])\n", " data_index = (data_index + 1) % len(data)\n", " return batch, labels\n", "\n", "print('data:', [reverse_dictionary[di] for di in data[:8]])\n", "\n", "for num_skips, skip_window in [(2, 1), (4, 2)]:\n", " data_index = 0\n", " batch, labels = generate_batch(batch_size=8, num_skips=num_skips, skip_window=skip_window)\n", " print('\\nwith num_skips = %d and skip_window = %d:' % (num_skips, skip_window))\n", " print(' batch:', [reverse_dictionary[bi] for bi in batch])\n", " print(' labels:', [reverse_dictionary[li] for li in labels.reshape(8)])" ] }, { "cell_type": "markdown", "metadata": { "id": "Ofd1MbBuwiva" }, "source": [ "Train a skip-gram model." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "8pQKsV4Vwlzy" }, "outputs": [], "source": [ "batch_size = 128\n", "embedding_size = 128 # Dimension of the embedding vector.\n", "skip_window = 1 # How many words to consider left and right.\n", "num_skips = 2 # How many times to reuse an input to generate a label.\n", "# We pick a random validation set to sample nearest neighbors. here we limit the\n", "# validation samples to the words that have a low numeric ID, which by\n", "# construction are also the most frequent. \n", "valid_size = 16 # Random set of words to evaluate similarity on.\n", "valid_window = 100 # Only pick dev samples in the head of the distribution.\n", "valid_examples = np.array(random.sample(range(valid_window), valid_size))\n", "num_sampled = 64 # Number of negative examples to sample.\n", "\n", "graph = tf.Graph()\n", "\n", "with graph.as_default(), tf.device('/cpu:0'):\n", "\n", " # Input data.\n", " train_dataset = tf.placeholder(tf.int32, shape=[batch_size])\n", " train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n", " valid_dataset = tf.constant(valid_examples, dtype=tf.int32)\n", " \n", " # Variables.\n", " embeddings = tf.Variable(\n", " tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n", " softmax_weights = tf.Variable(\n", " tf.truncated_normal([vocabulary_size, embedding_size],\n", " stddev=1.0 / math.sqrt(embedding_size)))\n", " softmax_biases = tf.Variable(tf.zeros([vocabulary_size]))\n", " \n", " # Model.\n", " # Look up embeddings for inputs.\n", " embed = tf.nn.embedding_lookup(embeddings, train_dataset)\n", " # Compute the softmax loss, using a sample of the negative labels each time.\n", " loss = tf.reduce_mean(\n", " tf.nn.sampled_softmax_loss(weights=softmax_weights, biases=softmax_biases, inputs=embed,\n", " labels=train_labels, num_sampled=num_sampled, num_classes=vocabulary_size))\n", "\n", " # Optimizer.\n", " # Note: The optimizer will optimize the softmax_weights AND the embeddings.\n", " # This is because the embeddings are defined as a variable quantity and the\n", " # optimizer's `minimize` method will by default modify all variable quantities \n", " # that contribute to the tensor it is passed.\n", " # See docs on `tf.train.Optimizer.minimize()` for more details.\n", " optimizer = tf.train.AdagradOptimizer(1.0).minimize(loss)\n", " \n", " # Compute the similarity between minibatch examples and all embeddings.\n", " # We use the cosine distance:\n", " norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True))\n", " normalized_embeddings = embeddings / norm\n", " valid_embeddings = tf.nn.embedding_lookup(\n", " normalized_embeddings, valid_dataset)\n", " similarity = tf.matmul(valid_embeddings, tf.transpose(normalized_embeddings))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "1bQFGceBxrWW" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initialized\n", "Average loss at step 0 : 8.58149623871\n", "Nearest to been: unfavourably, marmara, ancestral, legal, bogart, glossaries, worst, rooms,\n", "Nearest to time: conformist, strawberries, sindhi, waterfall, xia, nominates, psp, sensitivity,\n", "Nearest to over: overlord, panda, golden, semigroup, rawlings, involved, shreveport, handling,\n", "Nearest to not: hymenoptera, reintroducing, lamiaceae, because, davao, omnipotent, combustion, debilitating,\n", "Nearest to three: catalog, koza, gn, braque, holstein, postgresql, luddite, justine,\n", "Nearest to if: chilled, vince, fiddler, represented, sandinistas, happiness, lya, glands,\n", "Nearest to there: coast, photosynthetic, kimmei, legally, inner, illyricum, formats, fullmetal,\n", "Nearest to between: chuvash, prinz, suitability, wolfe, guideline, computability, diminutive, paulo,\n", "Nearest to from: tanganyika, workshop, elphinstone, spearhead, resurrected, kevlar, shangri, loves,\n", "Nearest to state: sextus, wuppertal, glaring, inches, unrounded, courageous, adler, connie,\n", "Nearest to on: gino, phocas, rhine, jg, macrocosm, jackass, jays, theorie,\n", "Nearest to and: standings, towed, reyes, willard, equality, juggling, wladislaus, faked,\n", "Nearest to eight: gresham, dogg, moko, tennis, superseded, telegraphy, scramble, vinod,\n", "Nearest to they: prisons, divisor, coder, ribeira, willingness, factional, nne, lotta,\n", "Nearest to more: blues, fur, sterling, tangier, khwarizmi, discouraged, cal, deicide,\n", "Nearest to other: enemies, bogged, brassicaceae, lascaux, dispense, alexandrians, crimea, dou,\n", "Average loss at step 2000 : 4.39983723116\n", "Average loss at step 4000 : 3.86921076906\n", "Average loss at step 6000 : 3.72542127335\n", "Average loss at step 8000 : 3.57835536212\n", "Average loss at step 10000 : 3.61056993055\n", "Nearest to been: glossaries, legal, unfavourably, be, hadad, wore, scarcity, were,\n", "Nearest to time: strawberries, conformist, gleichschaltung, waterfall, molality, nominates, baal, dole,\n", "Nearest to over: golden, semigroup, catus, motorways, brick, shehri, mussolini, overlord,\n", "Nearest to not: hinayana, it, often, they, boots, also, noaa, lindsey,\n", "Nearest to three: four, seven, six, five, nine, eight, two, zero,\n", "Nearest to if: glands, euros, wallpaper, redefine, toho, confuse, unsound, shepherd,\n", "Nearest to there: it, they, fullmetal, pace, legally, harpsichord, mma, bug,\n", "Nearest to between: chuvash, wandering, from, kirsch, pursuant, eurocents, suitability, jackie,\n", "Nearest to from: into, in, workshop, to, at, misogynist, elphinstone, spearhead,\n", "Nearest to state: sextus, glaring, connie, adler, esoteric, didactic, handedness, presidents,\n", "Nearest to on: in, at, for, ruminants, wakefulness, torrey, foley, gino,\n", "Nearest to and: or, who, but, zelda, of, for, thirst, chisel,\n", "Nearest to eight: nine, six, seven, five, four, three, zero, two,\n", "Nearest to they: he, prisons, there, we, hydrate, it, not, cumbersome,\n", "Nearest to more: skye, blues, trypomastigotes, deicide, most, readable, used, sterling,\n", "Nearest to other: trochaic, hush, surveyors, joachim, differentiation, attackers, reverence, attestation,\n", "Average loss at step 12000 : 3.66169466591\n", "Average loss at step 14000 : 3.60342905837\n", "Average loss at step 16000 : 3.57761328053\n", "Average loss at step 18000 : 3.57667332476\n", "Average loss at step 20000 : 3.53310145146\n", "Nearest to been: be, become, was, hadad, unfavourably, were, wore, partido,\n", "Nearest to time: gleichschaltung, strawberries, year, nominates, conformist, etch, admittedly, treasuries,\n", "Nearest to over: golden, semigroup, motorways, rawlings, triangle, trey, ustawa, mattingly,\n", "Nearest to not: they, boots, often, dieppe, still, hinayana, nearly, be,\n", "Nearest to three: two, four, five, seven, eight, six, nine, one,\n", "Nearest to if: wallpaper, euros, before, toho, unsound, so, bg, pfc,\n", "Nearest to there: they, it, he, usually, which, we, not, transactions,\n", "Nearest to between: from, with, about, near, reactance, eurocents, wandering, voltaire,\n", "Nearest to from: into, workshop, by, between, in, on, elphinstone, under,\n", "Nearest to state: glaring, esoteric, succeeding, sextus, vorarlberg, presidents, depends, connie,\n", "Nearest to on: in, at, upon, during, from, janis, foley, nubian,\n", "Nearest to and: or, thirst, but, where, s, who, pfaff, including,\n", "Nearest to eight: nine, seven, six, five, four, three, zero, one,\n", "Nearest to they: there, he, we, not, it, you, prisons, who,\n", "Nearest to more: less, most, deicide, skye, trypomastigotes, interventionism, toed, drummond,\n", "Nearest to other: such, joachim, hush, attackers, surveyors, trochaic, differentiation, reverence,\n", "Average loss at step 22000 : 3.59519316927\n", "Average loss at step 24000 : 3.55378576797\n", "Average loss at step 26000 : 3.56455037558\n", "Average loss at step 28000 : 3.5040882225\n", "Average loss at step 30000 : 3.39208897972\n", "Nearest to been: become, be, were, was, spotless, hadad, by, hausdorff,\n", "Nearest to time: gleichschaltung, year, day, nominates, jesus, strawberries, way, admittedly,\n", "Nearest to over: golden, semigroup, motorways, rawlings, interventionism, counternarcotics, adaption, brick,\n", "Nearest to not: often, they, it, never, still, nor, boots, pki,\n", "Nearest to three: four, six, two, eight, five, seven, nine, zero,\n", "Nearest to if: when, before, so, should, toho, where, bg, wallpaper,\n", "Nearest to there: they, it, which, usually, he, that, also, now,\n", "Nearest to between: with, from, in, panasonic, presupposes, churchmen, hijacking, where,\n", "Nearest to from: into, elphinstone, workshop, between, through, speculates, sosa, in,\n", "Nearest to state: esoteric, glaring, presidents, vorarlberg, atmosphere, succeeding, lute, connie,\n", "Nearest to on: upon, in, janis, during, torrey, against, infield, catalans,\n", "Nearest to and: or, thirst, in, but, of, sobib, cleaves, including,\n", "Nearest to eight: nine, six, four, seven, three, zero, five, one,\n", "Nearest to they: we, there, he, you, it, these, who, i,\n", "Nearest to more: less, most, deicide, faster, toed, very, skye, tonic,\n", "Nearest to other: different, attackers, joachim, various, such, many, differentiation, these,\n", "Average loss at step 32000 : 3.49501452419\n", "Average loss at step 34000 : 3.48593705952\n", "Average loss at step 36000 : 3.50112806576\n", "Average loss at step" ] }, { "name": "stdout", "output_type": "stream", "text": [ " 38000 : 3.49244426501\n", "Average loss at step 40000 : 3.3890105716\n", "Nearest to been: become, be, were, was, jolie, hausdorff, spotless, had,\n", "Nearest to time: year, way, gleichschaltung, period, day, stanislav, stage, outcome,\n", "Nearest to over: through, semigroup, rawlings, golden, about, brick, on, motorways,\n", "Nearest to not: they, radiated, never, pki, still, omnipotent, hinayana, really,\n", "Nearest to three: four, six, five, two, seven, eight, one, nine,\n", "Nearest to if: when, before, where, then, bg, because, can, should,\n", "Nearest to there: they, it, he, usually, this, typically, still, often,\n", "Nearest to between: with, in, from, about, against, churchmen, johansen, presupposes,\n", "Nearest to from: into, through, elphinstone, in, workshop, between, suing, under,\n", "Nearest to state: esoteric, presidents, atmosphere, vorarlberg, lute, succeeding, glaring, didactic,\n", "Nearest to on: upon, at, in, during, unitarians, under, catalans, batavians,\n", "Nearest to and: or, but, s, incapacitation, including, while, of, which,\n", "Nearest to eight: nine, six, seven, four, five, three, one, two,\n", "Nearest to they: we, he, there, you, she, i, not, it,\n", "Nearest to more: less, most, deicide, toed, greater, faster, quite, longer,\n", "Nearest to other: various, different, attackers, joachim, clutter, nz, trochaic, apulia,\n", "Average loss at step 42000 : 3.45294014364\n", "Average loss at step 44000 : 3.47660055941\n", "Average loss at step 46000 : 3.47458503014\n", "Average loss at step 48000 : 3.47261548793\n", "Average loss at step 50000 : 3.45390708435\n", "Nearest to been: become, be, had, was, were, hausdorff, prem, remained,\n", "Nearest to time: way, year, period, stv, day, gleichschaltung, stage, outcome,\n", "Nearest to over: through, golden, semigroup, about, brick, counternarcotics, theremin, mattingly,\n", "Nearest to not: they, still, never, really, sometimes, it, kiwifruit, nearly,\n", "Nearest to three: five, four, six, seven, two, eight, one, nine,\n", "Nearest to if: when, before, where, because, connexion, though, so, whether,\n", "Nearest to there: they, it, he, this, now, often, usually, still,\n", "Nearest to between: with, from, fashioned, churchmen, panasonic, explores, within, racial,\n", "Nearest to from: into, through, under, elphinstone, between, workshop, circumpolar, idiom,\n", "Nearest to state: atmosphere, vorarlberg, esoteric, presidents, madhya, majority, moulin, bowmen,\n", "Nearest to on: upon, in, catalans, tezuka, minotaurs, wakefulness, batavians, guglielmo,\n", "Nearest to and: or, but, thirst, signifier, which, however, including, unattractive,\n", "Nearest to eight: six, nine, seven, five, four, three, zero, two,\n", "Nearest to they: we, there, he, you, it, she, these, not,\n", "Nearest to more: less, most, quite, very, further, faster, toed, deicide,\n", "Nearest to other: various, different, many, attackers, are, joachim, nihilo, reject,\n", "Average loss at step 52000 : 3.43597227755\n", "Average loss at step 54000 : 3.25126817495\n", "Average loss at step 56000 : 3.35102432287\n", "Average loss at step 58000 : 3.44654818082\n", "Average loss at step 60000 : 3.4287913968\n", "Nearest to been: become, be, was, prem, had, remained, hadad, stanislavsky,\n", "Nearest to time: year, way, period, stv, barely, name, stage, restoring,\n", "Nearest to over: about, through, golden, adaption, counternarcotics, up, mattingly, brick,\n", "Nearest to not: still, never, nor, kiwifruit, they, nearly, therefore, rarely,\n", "Nearest to three: two, five, four, six, seven, eight, one, nine,\n", "Nearest to if: when, though, before, where, although, because, can, could,\n", "Nearest to there: they, it, he, still, she, we, this, often,\n", "Nearest to between: with, from, churchmen, among, ethical, within, vma, panasonic,\n", "Nearest to from: through, into, under, during, between, in, suing, across,\n", "Nearest to state: atmosphere, infringe, madhya, vorarlberg, government, bowmen, vargas, republic,\n", "Nearest to on: upon, through, within, ridiculous, janis, in, under, over,\n", "Nearest to and: or, while, including, but, of, like, whose, bannister,\n", "Nearest to eight: nine, six, five, four, seven, zero, three, two,\n", "Nearest to they: we, there, you, he, it, these, she, prisons,\n", "Nearest to more: less, most, quite, further, toed, very, faster, rather,\n", "Nearest to other: different, various, many, nihilo, these, amour, including, screenplays,\n", "Average loss at step 62000 : 3.38358767056\n", "Average loss at step 64000 : 3.41693099326\n", "Average loss at step 66000 : 3.39588000977\n", "Average loss at step 68000 : 3.35567189544\n", "Average loss at step 70000 : 3.38878934443\n", "Nearest to been: become, be, was, prem, remained, were, being, discounts,\n", "Nearest to time: year, way, day, period, barely, ethos, stage, reason,\n", "Nearest to over: about, through, fortunately, semigroup, theremin, off, loudest, up,\n", "Nearest to not: still, nor, never, they, actually, nearly, unelected, therefore,\n", "Nearest to three: five, two, four, six, seven, eight, nine, zero,\n", "Nearest to if: when, though, before, where, because, then, after, since,\n", "Nearest to there: they, it, he, often, she, we, usually, still,\n", "Nearest to between: among, with, within, from, ethical, churchmen, racial, prentice,\n", "Nearest to from: through, into, within, during, under, until, between, across,\n", "Nearest to state: city, atmosphere, desks, surrounding, preservation, bohr, principal, republic,\n", "Nearest to on: upon, tezuka, through, within, wakefulness, catalans, at, ingeborg,\n", "Nearest to and: or, but, while, including, thirst, jerzy, massing, abadan,\n", "Nearest to eight: seven, six, nine, five, four, three, two, zero,\n", "Nearest to they: we, you, he, there, she, it, prisons, who,\n", "Nearest to more: less, most, quite, very, faster, smaller, further, larger,\n", "Nearest to other: various, different, some, screenplays, lab, many, including, debugging,\n", "Average loss at step 72000 : 3.41103189731\n", "Average loss at step 74000 : 3.44926435578\n", "Average loss at step 76000 : 3.4423020488\n", "Average loss at step 78000 : 3.41976813722\n", "Average loss at step 80000 : 3.39511853886\n", "Nearest to been: become, be, remained, was, grown, were, prem, already," ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Nearest to time: year, way, period, reason, barely, distance, stage, day,\n", "Nearest to over: about, fortunately, through, semigroup, further, mattingly, rawlings, golden,\n", "Nearest to not: still, they, nor, never, we, kiwifruit, noaa, really,\n", "Nearest to three: five, two, seven, four, eight, six, nine, zero,\n", "Nearest to if: when, where, though, before, since, because, although, follows,\n", "Nearest to there: they, it, he, we, she, still, typically, actually,\n", "Nearest to between: with, among, within, in, racial, around, from, serapeum,\n", "Nearest to from: into, through, in, within, under, using, during, towards,\n", "Nearest to state: city, atmosphere, ferro, vorarlberg, surrounding, republic, madhya, national,\n", "Nearest to on: upon, poll, in, from, tezuka, janis, through, within,\n", "Nearest to and: or, but, including, while, s, which, thirst, although,\n", "Nearest to eight: nine, seven, six, five, four, three, zero, two,\n", "Nearest to they: we, you, there, he, she, it, these, not,\n", "Nearest to more: less, most, smaller, very, faster, quite, rather, larger,\n", "Nearest to other: various, different, joachim, including, theos, smaller, individual, screenplays,\n", "Average loss at step 82000 : 3.40933967865\n", "Average loss at step 84000 : 3.41618054378\n", "Average loss at step 86000 : 3.31485116804\n", "Average loss at step 88000 : 3.37068593091\n", "Average loss at step 90000 : 3.2785516749\n", "Nearest to been: become, be, was, prem, remained, grown, recently, already,\n", "Nearest to time: year, way, period, day, barely, battle, buds, name,\n", "Nearest to over: through, about, fortunately, off, theremin, semigroup, extraterrestrial, mattingly,\n", "Nearest to not: nor, still, never, otherwise, generally, separately, gown, hydrate,\n", "Nearest to three: four, five, six, two, eight, seven, nine, zero,\n", "Nearest to if: when, where, before, though, because, since, then, while,\n", "Nearest to there: they, it, he, we, she, still, typically, fiorello,\n", "Nearest to between: with, among, within, from, churchmen, prentice, racial, panasonic,\n", "Nearest to from: through, into, across, during, towards, until, at, within,\n", "Nearest to state: bohr, city, atmosphere, ferro, bowmen, republic, retaliation, vorarlberg,\n", "Nearest to on: upon, in, tezuka, at, during, within, via, catalans,\n", "Nearest to and: or, including, but, while, like, thirst, with, schuman,\n", "Nearest to eight: seven, nine, six, five, four, three, zero, two,\n", "Nearest to they: we, there, he, you, she, it, prisons, these,\n", "Nearest to more: less, most, very, faster, larger, quite, smaller, better,\n", "Nearest to other: different, various, tamara, prosthetic, including, individual, failing, restaurants,\n", "Average loss at step 92000 : 3.40355363208\n", "Average loss at step 94000 : 3.35647508007\n", "Average loss at step 96000 : 3.34374570692\n", "Average loss at step 98000 : 3.4230104093\n", "Average loss at step 100000 : 3.36909827\n", "Nearest to been: become, be, grown, was, being, already, remained, prem,\n", "Nearest to time: way, year, day, period, years, days, mothersbaugh, separators,\n", "Nearest to over: through, about, semigroup, further, fortunately, off, into, theremin,\n", "Nearest to not: never, nor, still, dieppe, really, unelected, actually, now,\n", "Nearest to three: four, two, five, seven, six, eight, nine, zero,\n", "Nearest to if: when, though, where, before, is, abe, then, follows,\n", "Nearest to there: they, it, he, we, still, she, typically, often,\n", "Nearest to between: within, with, among, churchmen, around, explores, from, reactance,\n", "Nearest to from: into, through, within, across, in, between, using, workshop,\n", "Nearest to state: atmosphere, bohr, national, ferro, germ, desks, city, unpaid,\n", "Nearest to on: upon, in, within, tezuka, janis, batavians, about, macrocosm,\n", "Nearest to and: or, but, purview, thirst, sukkot, epr, including, honesty,\n", "Nearest to eight: seven, nine, six, four, five, three, zero, one,\n", "Nearest to they: we, there, you, he, she, prisons, it, these,\n", "Nearest to more: less, most, very, quite, faster, larger, rather, smaller,\n", "Nearest to other: various, different, tamara, theos, some, cope, many, others,\n" ] } ], "source": [ "num_steps = 100001\n", "\n", "with tf.Session(graph=graph) as session:\n", " tf.global_variables_initializer().run()\n", " print('Initialized')\n", " average_loss = 0\n", " for step in range(num_steps):\n", " batch_data, batch_labels = generate_batch(\n", " batch_size, num_skips, skip_window)\n", " feed_dict = {train_dataset : batch_data, train_labels : batch_labels}\n", " _, l = session.run([optimizer, loss], feed_dict=feed_dict)\n", " average_loss += l\n", " if step % 2000 == 0:\n", " if step > 0:\n", " average_loss = average_loss / 2000\n", " # The average loss is an estimate of the loss over the last 2000 batches.\n", " print('Average loss at step %d: %f' % (step, average_loss))\n", " average_loss = 0\n", " # note that this is expensive (~20% slowdown if computed every 500 steps)\n", " if step % 10000 == 0:\n", " sim = similarity.eval()\n", " for i in range(valid_size):\n", " valid_word = reverse_dictionary[valid_examples[i]]\n", " top_k = 8 # number of nearest neighbors\n", " nearest = (-sim[i, :]).argsort()[1:top_k+1]\n", " log = 'Nearest to %s:' % valid_word\n", " for k in range(top_k):\n", " close_word = reverse_dictionary[nearest[k]]\n", " log = '%s %s,' % (log, close_word)\n", " print(log)\n", " final_embeddings = normalized_embeddings.eval()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "jjJXYA_XzV79" }, "outputs": [], "source": [ "num_points = 400\n", "\n", "tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')\n", "two_d_embeddings = tsne.fit_transform(final_embeddings[1:num_points+1, :])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "o_e0D_UezcDe" }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3MAAANpCAYAAAChBGCHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XdAldUfx/H3BdlbQEVzoyDukZaae5aZ5tbcIzUz9x5Z\njhwNNXMVztTExFHqT9Ny50hFc+ZKEVBwAbLh/v4gSXILChc/r3+69/Lc53yfewL8cM5zjsFoNBoR\nERERERERk2KW0QWIiIiIiIjI01OYExERERERMUEKcyIiIiIiIiZIYU5ERERERMQEKcyJiIiIiIiY\nIIU5ERERERERE5TmMBceHk7fvn1p2LAhb775JgEBAdy6dYvOnTtTv359unTpQnh4eHrUKiIiIiIi\nIv8wpHWfuaFDh/Lqq6/SvHlzEhISiI6OZvbs2bi4uNC9e3fmzZtHeHg4gwYNSq+aRUREREREXnpp\nGpmLiIjg4MGDNG/eHIBs2bLh4ODAtm3baNq0KQBNmzbll19+SXulIiIiIiIikiJbWt4cGBhI9uzZ\nGT58OKdOnaJ48eKMGDGC69ev4+bmBoCbmxvXr19Pl2JFREREREQkWZpG5hISEjhx4gRt2rTB398f\nGxsb5s2bl+oYg8GAwWBIU5EiIiIiIiKSWprCXK5cuciZMyelSpUCoH79+pw4cQI3NzdCQ0MBuHbt\nGtmzZ3/kedJ4256IiIiIiMhLJ03TLN3d3fHw8ODChQsULFiQvXv34unpiaenJ/7+/vTo0YM1a9ZQ\np06dR57HYDAQGhqRllIkE3N3d1D/ZmHq36xLfZu1qX+zLvVt1qb+zbrc3R2e+j1pCnMAo0ePZtCg\nQcTHx5MvXz4mTZpEYmIi/fr148cffyRPnjx89dVXaW1GRERERERE7pHmMOft7c2PP/543+sLFy5M\n66lFRERERETkIdK8abiIiIiIiIi8eApzIiIiIiIiJkhhTkRERERExAQpzImIiIiIiJgghTkRERER\nERETpDAnIiIiIiJighTmRERERERETJDCnIiIiIiIiAlSmBMRERERETFBCnMiIiIiIiImSGFORERE\nRETEBCnMiYiIiIiImCCFOREREREREROkMCciIiIiImKCFOZERERERERMkMKciIiIiIiICVKYExER\nERERMUEKcyIiIiIiIiZIYU5ERERERMQEKcyJiIiIiIiYIIU5ERERERERE6QwJyIiIiIiYoIU5kRE\nREREREyQwpyIiIiIiIgJUpgTERERERExQQpzIiIiIiIiJkhhTkRERERExAQpzImIiIiIiJgghTkR\nERERERETpDAnIiIiIiJighTmRERERERETJDCnIiIiIiIiAlSmBMRERERETFBCnMiIiIiIiImSGFO\nRERERETEBCnMiYiIiIiImCCFOREREREREROkMCciIiIiImKCFOZERERERERMkMKciIiIiIiICVKY\nExERERERMUEKcyIiIiIiIiZIYU5ERERERMQEKcyJiIiIiIiYIIU5ERERERERE6QwJyIiIiIiYoIU\n5kREREREREyQwpyIiIiIiIgJUpgTERERERExQQpzIiIiIiIiJkhhTkRERERExAQpzImIiIiIiJgg\nhTkRERERERETpDAnIiIiIiJighTmRERERERETJDCnIiIiIiIiAlSmBMRERERETFBCnMiIiIiIiIm\nSGFORERERETEBCnMiYiIiIiImCCFOREREREREROkMCciIiIiImKCFOZERERERERMkMKciIiIiIiI\nCVKYExERERERMUEKcyIiIiIiIiZIYU5ERERERMQEKcyJiIiIiIiYIIU5ERERERERE6QwJyIiIiIi\nYoIU5kREREREREyQwpyIiIiIiIgJUpgTERERERExQQpzIiIiIiIiJkhhTkRERERExAQpzImIiIiI\niJgghTkRERERERETpDAnIiIiIiJighTmRERERERETJDCnIiIiIiIiAlSmBMRERERETFBCnMiIiIi\nIiImSGFORERERETEBCnMiYiIiIiImCCFOREREREREROULaMLEBERkcxnxYqlbNiwHoBGjZpQrVoN\nBg78kFKlyvLnnwG4u+dg0qTPsbKy4sqVQL74Ygq3bt3E2tqaoUNHki9fgYy9ABGRl4BG5kRERCSV\nU6dOsnHjT8yfv4i5cxeyfr0/ERHhBAZeplmzlixZshJ7ewe2b98GwJQpE+jffzDffbeE3r0/4vPP\nJ2fwFYiIvBw0MiciIiKpHD16hGrVamJlZQ1A9eq1CAg4jIdHHjw9iwDg5eVNcHAQ0dHRHDt2lNGj\nh6a8Pz4+IUPqFhF52SjMiYiISCoGg+GBr1taWqQ8NjMzJykpDqMxCQcHBxYsWPaiyhMRkX9omqWI\niIikUrp0GXbs+I3Y2Biio6PZseNXSpcue99xRqMRW1s7cufOza+//pLy2tmzf73okkVEXkoamRMR\nEZFUihb15s03G9G9e0cA3n67KQ4OjveN2N19PmbMeKZN+4xFi3xJSEigTp16KdMxRUTk+TEYjUZj\nRhcBEBoakdElyHPi7u6g/s3C1L9Zl/o2a1P/Zl3q26xN/Zt1ubs7PPV7NM1SREREntnx4+eYP38j\nf/xxMqNLERF56SjMiYiIyDNZt24/LVpEM3JkC1q1smLhwu0ZXZKIyEtFYU5ERESeyeLFNwgLex0w\nEB5emqVLYzK6JBGRl4rCnIiIiDwTo9HwyOciIvJ8KcyJiIjIM2nd2gEXl0MA2NmdpkULLZItIvIi\n6aeuiIiIPJMWLSpToMAJ9u3zo2TJXFSvXiujSxIReakozImIiMgze/VVH1591SejyxAReSlpmqWI\niIiIiIgJUpgTERERERExQQpzIiIiIiIiJkhhTkRERERExAQpzImIiGSAyMhI/P1XZXQZIiJiwhTm\nREREMkBERDj+/n4ZXYaIiJgwbU0gIiKSAebMmcmVK4F07tyWIkW8qFatJlWrVmP48EE4OjoyfPgY\nfvppLUFBV+jRozcrVixlw4b1ADRq1ISWLdtk8BWIiEhG08iciIhIBujVqy958rzCggXLqFTpdY4e\nPQxAWNg1/v77IgBHjx6hbNlynDp1ko0bf2L+/EXMnbuQ9ev9+euv0xlYvYiIZAYKcyIiIhnAaDSm\nPC5VqgwBAUe4ePECBQsWxsUlO9evh3H8+DFKlCjN0aNHqFatJlZW1tjY2FC9ei0CAg5nYPUiIpIZ\naJqliIhIBnN3z0FkZAT79u2hdOmyhIeHs3XrFmxtbbGxscFgMKQ63mg03veaiIi8fDQyJyIikgFs\nbW2JiopKeV68eElWrlxOmTLlKF26DCtWLKVUqbIAlC5dhh07fiM2Nobo6Gh27vwt5WsiIvLy0sic\niIhIBnBycqZkydJ06NCK116rTKlSZThwYB958rxCzpy5iIgIp3Tp5MBWtKg3b77ZiO7dOwLw9ttN\nKVKkaEaWLyIimYDBeO+k/QwUGhqR0SXIc+Lu7qD+zcLUv1mX+jbzuHr1KsHBoXh7e2JtbZ0u51T/\nZl3q26xN/Zt1ubs7PPV7NM1SREQkE/P13U61apepV8+DRo02cflySEaXJCIimYTCnIiISCYVHx/P\nrFnx3LxZEyjA0aPt+eKLgxldloiIZBIKcyIiIplUXFwcUVH2qV6LibHMoGpERCSzUZgTERHJpOzs\n7Kha9W8gGgBHx8M0bOiUsUWJiEimodUsRUREMrHZs5tRvPhPhIVBzZq5qF27UkaXJCIimYTCnIiI\nSCaWLVs2+vVrkNFliIhIJqRpliIiIiIiIiZIYU5ERERERMQEKcyJiIjIA02ePJ6LFy9kdBkiIvIQ\numdOREREHmjo0FEZXYKIiDyCwpyIiMhLIDo6mjFjhhEaGkpSUiIfftgHBwc3vv76S6Kjo3FwcMBo\nNBIaGsrVq8GMGPEx/v5+tG79Ht98M4OkpCQGDRrGwoXfcvr0SfLnL8DkyV/i6upGnz49KF68JIcO\nHSQyMoJhw8ZQunSZjL5kEZEsT9MsRUREXgL79u3BzS0HCxcuY/HiH6hWrRrTp09lwoQpfPfdEgoX\n9uTatWssW7aKfPnyU6BAAQwGAwcO7KNBgzdp2/Y9xowZTrly5alf/03atGnPvHnfAGAwGEhKSmL+\n/EX07TuQBQvmZfDVioi8HBTmRETkpRcZGYm//yoADh06yJAh/TO4ovRXuHARDh7cx+zZMwkIOEJQ\nUBDnz5+jX7/edO7clj17dnPtWgizZ8+kWLES7N27G4CDB/dRu3Y9SpQoxZ07kSxZspBjx46yeLEv\noaGhKeevXr0mAF5e3oSEBGfINYqIvGw0zVJERF56ERHh+Pv70bRp84wuBUgOl1u2bKJp0+YcOnSQ\nFSu+Z8qUL9N0zrx58+Hr+z179+5i/vxveOONKhQsWJg5c3xTjomIiGDv3l34+a1g9+4d5MiREzCQ\nJ88rnDhxnGzZsuHunoMvvpiJq6tbqvNbWFgCYGZmTmJiYppqFRGRJ6OROREReenNmTOTK1cC6dy5\nLbNnzyA6OopRo4bSrl1zPvlk9Auv5264fBpJSUmP/HpYWBiWlpbUq9eQNm3ac/ToUW7dusWffx4D\nICQkhKCgK9Sr15BOnboRGxtLSEgwFSu+BsDGjeuwsrKmXr03mTx5AgkJCVy4cP7ZLlBERNKFRuZE\nROSl16tXXy5cOM+CBcs4fPgPhg8fyNKlfri6utGrV1eOHj1CqVIvbkGPe8NltmzZsLa2YdSooVy4\ncA4vr2KMGfMpAM2bv03t2vU4cGAf7dp1wMHBEV/fecTFxZEnzyuMGDEWGxsbTp06ycSJ47hy5TLm\n5ubkyZOXyZMncft2DNOnTyMyMpLIyAgSExNxdnYmWzYLGjZshJ/fcl59tRKHD//BX3/9xfTp3zB9\n+uf89dcZWrR4m65d36dgwUIPuALDC/usREReZgpzIiLy0jMajakeFytWHDc3dwA8PYsSEhL8QsPc\n48LlsWMBlCxZGoPBgJOTM76+S7l16xajRg1h+vRvsLKyZunShYwePYzg4CvcunWLZctW4eTkzNat\nm9m//3eKFy9OaGgEX3/98MVKPvpoIJA86jdt2nTs7R0eePzMmXNTHjs7O+Pntzb9PxQREbmPwpyI\niMh/3L3/C8Dc3OyF3wP2uHAZHBxMyZKlAahduy4Ax48f4+LF8/Ts2QWA+PgEwsKuMWHCVEaMGES/\nfr2B5GDm6ur+xLVs3nyETz4JJCzMHR+fv5k3rw5ubtn/aSOedet2YzQaady4CpaWlo85m4iIpCeF\nOREReaiFC79l8+aNODu7kCNHTry8itGmzXsZXVa6s7W1JSoqKqPLeKj7w2VCynMbG5uUxxUqVOLj\njycAMHXqRDZsWM/UqZNwcHAkVy4PgoKCuHPnDtHR0VSsWJEiRbyoVKlySp+2b9+SqVNnYDQmMWBA\nH4oXL8mmTfs5f96PhAQPdu0yMn7893z11TvEx8fToYMfW7e2BQysWLGM779/FysrqxfzoYiISPos\ngJKYmEiTJk3o2bMnALdu3aJz587Ur1+fLl26EB4enh7NiIgIEBwcRLt2zZk8eQLt27dkwIA+xMbG\npns7R48eZfv2bSxatIJp02Zw6tRJDFn0VignJ2dKlixNhw6tmD17RoZf57OESx+fEhw7FsCVK4EA\n9OnTH2dnF775Zj4RERE4O2dn0aLlJCUlYWZmxv79+++bOmq458KvXAnknXfe5fbtYSQkeNw9gtu3\nbQHw89vO1q0dAAfAnh07OrFs2fZnvWQREXkG6TIyt3jxYgoXLsydO3cAmDdvHpUrV6Z79+7MmzeP\nefPmMWjQoPRoSkREgMDAy4wbN4mhQ0cyZsxwtm/fRr16DdO1jUOHDvHGGzWwsLDAwsKCKlXe4J7Z\nf1nO2LHjH/h6//5DXnAlqcOllZUV2bO7PvY9Li4ujBz5MR9/PIK4uHgAEhISMDc3x9XVjbNnz/Dm\nm7UJD7+NmZkZc+fOZe/e3VStWo3mzd9m1ar1AMTGxtC3b09y5sxF9uyu5Mo1HisrX5KSbLh+/UMq\nVkwOfPHxRlL/M8Kc+PhHr6gpIiLpK81hLiQkhO3bt9OzZ08WLlwIwLZt21i6dCkATZs2pX379gpz\nIiLpyMMjD56eRYDkTZqDg4PSvQ2DwZDq3i3IwknuH7Nnb+P77+NJTDTn3XcTGDy4QYbV8iTh0s9v\nXaqvlStXgfnzF6c8b9GiMVFRUZiZGRgz5lPy5s1HixaNSUhIwMnJCTMzAxYWFhQpUpRDhw4SFxfH\nwYP7KVOmHBcunGPKlAnMmjWeRYv+4tKlq0RHf0rPnsv/OXcV/PwWs39/J8BAuXILadv2zXT/HERE\n5OHSPM1y4sSJDBkyBDOzf091/fp13NySNxN1c3Pj+vXraW1GRETuYWlpkfL4eW3SXK5cOXbv3klc\nXBxRUVHs2bMrw6cfPk8HDhxn6tTCnDnTnHPnmjJjxqts3Lgvo8t6qMGDP+LOnchHHnPt2jUaNVrP\n0aNedOz4FZcvhxAbG4u1tTUXLlzA1taO06dPUatWXVavXklwcBC7du2gcuU3SEpK4tixo0yaNI6g\noGVky7YVa+uklKmYtra2/PDDW3z88Y+MHbsKP7+G2Nvbv4hLFxGRf6RpZO7XX3/F1dUVHx8f9u17\n8C88g8GQag7+w7i7O6SlFMnk1L9Zm/r3xYqNtSNbNvOUz93e3gozs8R07wd395LUr1+XLl3a4ubm\nho9PMXLlcsuy/X3hQiiRkZVSnsfGFiAo6Gimvd6FC30f+XWj0UhiIoSFVSE21gczsxF07dqNhIQI\nmjdvxrFjx/D0LMTly5dZunQBwcHB5MuXj7//Pk/16q+zfPkinJwc+emn9Q9tw93dgbFjW6b3pUk6\nyKz/30r6UP/KXWkKc4cPH2bbtm1s376duLg4IiMjGTx4MK6uroSGhuLu7s61a9fInj37Y88VGhqR\nllIkE3N3d1D/ZmHq3xfvxo07JCYmpXzukZGxxMTEpXs/uLs70LhxS1q16khMTAx9+vSgdeuCWba/\ny5cvjIfHbwQH1wTA1XU/ZcrkzhTX+7//bWDVqh9ISIjHx6cEAwYMpVWrJvj6LsXR0emBq462aNGa\nhIT82Nntxs3tC8zNw8mevTEJCetZvdqf6OgoTpw4ycCBw6hVqw6jRw/D0tICOzt7rK2dWbhwBb16\ndWHlSn9q1qyD0Wjk3LmzKdN7JfPSz+WsTf2bdT1LSE9TmBswYAADBgwAYP/+/fj6+jJ16lSmTJmC\nv78/PXr0YM2aNdSpUyctzYiICBAXF8fhwydwdXVk0aIVKa+n91YBcXFx/PzzXlxd7dm2bTV//32B\nuLg4GjZsRJEiXunaVmZSuHBepk+/yXff+WE0GmjdOjvly7+a0WVx8eIFtm3bwpw5vpibm/P555PZ\nvHljyqyXkyePp6w6Gh8fT5cu7+HtXYxs2bLh6BhHeHgcly/74ez8A0lJSzAYzOjQoTO//76L3Lnz\nUqtW8u/omjVrM2bMCF55pRtjxqxn5Mh6jBkznmnTPmPRIl8SEhKoU6eewpyISCbyXPaZ69GjB/36\n9ePHH38kT548fPXVV8+jGRGRl0ZERAStWn1HaOgerl79mM6d/Tl/fjmffPIZBQsWSrd2YmJiaNvW\nn1272gLxNGhwmgULPsHc3Dzd2sjMatQoRY0apTK6jFT++GM/p0+folu39kDyfXDR0cnbFhiNRo4d\nC7hv1dG7ihZ1p2xZC+LjV1GqlBmbNsWwYsVaNmxYT6FChejZs1/Ksb/9FsOZM39w5owd27bFcePG\nMr7+uhmffz7jxV6wiIg8sXQLcxUrVqRixYoAODs7p6xsKSIiaTdjxg4OHhyJq+sM7O1/Y+3aS3Tv\nXjVdgxzAokW/sWtXZyB5gZVNm1qxfv0umjSpnq7tyNNp2LAR77//QarXWrRo/M+jh686ajAY6NKl\nBl5e3ty6dYuff/7moW0cOWIN2P3zzJJDh+wYMuQn4uKy8e67r1CtWon0uBQREUlH6bJpuIiIPF8x\nMeaAGdevf4Cd3W6yZbtMzZr10r2d+/cOsyImJiHd25HHu7s5/OHDh1i+fCkffvg+sbGxjB07An//\nVQB07tyOCxfOsWzZYjp0aMWZMyfZs2cX8fHxTJw4jtOnTzJu3Eh27Uq9mbednV3K3rB3ubqm3qQ8\nOPgKCxe2YdmyFvTuncDBg6ef7wWLiMhTU5gTETEB775bmNy5t2BufhODIQp7+zDy5Xsl3dtp1+51\nSpVaTPLoTgKvv76UJk2qpHs7mcXKlcuIjY3J6DIeKjDwMp06dWP06E/4668ztG37LgcP7vtnS4Lk\n1aI9PYvSrl1Hbt68yaBB/Shc2JOAgCNUqFARL69iDB48glmzpv9zncn32ZUtW4GzZ8/SuXNbtm37\nBYCxY1/l9dcX4e6+gcKF53DnzuspdVy79gb/+9+FDPgERETkUZ7LPXMiIpK+ypYtwqJF8PHH3cmf\n/zVKlnRm/vxvUm0gnR5cXJzx86vFkiWrcHKyokWLxlhbW6drG5mJn98K6td/EyurzHmNdzeH9/Qs\nQkhIEAkJCVy5EkjevPnw81tLixaNqV69FnZ2dlSqVJk5c2YSEhJCdHQ0S5cuxNzcnBkzPic+Pp7Y\n2Bj8/NYC4OjoyKpVq1KtiFegQG7Wrn2XuLg4LlzITf36iUSlDNZF4eqqv/+KiGQ2CnMiIiYiKOg0\nxYvnYfz4oSQlJdGzZxcOHTpIuXIV0rUdFxdn+vZtkOWWv46OjmbMmGGEhoaSlJRIzZp1CAsLpW/f\nnjg7uzB9+uyMLvE+928OH/vAY6ZMmcCpUye4du0anTt349dft/LxxxPImzcfAElJSQwe7M/Oneew\ns4ulf393unat9ZA2LfHyKkyfPv9jwYJQYmIcqFnzBN26NX8+FykiIs9MYU5ExEQ0bNiIhg0bAWBm\nZsa8eQsztiATs2/fHtzccjB16nQA7tyJZMOG9cycORdHR6cMri5txo4dz6lTJ5g1azrvvdeJO3fu\nsGrVipSR2wkTFrNkSUfAEYAxY36mWbNbwMNXKR00qD49e0YQGxtH9uwlU7ZCEBGRzENzJkREMrmp\nUzdRpcoWqlX7H99++1tGl2OyChcuwsGD+5g9eyYBAUews7PP6JIe6+kClCHl+E6dupGQkEDHjq1p\n374lBw78wt0gBxAUVIRLl4Iee0Z7ewdcXV0V5EREMimDMfV6xhkmK03lkdSy2lQtSU39+3z9/PM+\nevb0JDa2AAAODkdYtSqesmW9n3vbaenbjRt/YsWK7zEYDBQu7Mno0Z88cx11677Bli07n/n994qI\niGDv3l2sW+dP+fKvsmHDer77bonJj8w9zurVe+nXz5OYmOStLHx8fuDgwcZERmql0qxIP5ezNvVv\n1uXu7vDU79E0SxGRTOzMmZspQQ4gIqIkR4+ufiFh7lmdP3+OxYt9mTt3AY6OToSHh6fxjOkzKhQW\nFoaDgwP16jXEzs6en35ai61t8hL9WSXMHT78FzNmnCEmxoIGDazo2DF5f8B3332d69e3sm3bYWxt\n4xg4sDg2NjZERuofhCIipkxhTkQkE6tcOR8uLge5eTN5kZPcuX/L9Js3Hzp0gFq16qYEJEdHx8e8\n48U4f/4ss2ZNx8zMQLZsFgwaNJw//wxg4MAPcXfPkSkXQHmQNWt+ZO3aHwGIjIzEwyM37dt3Yt68\nbzhx4haRkaUICZnE3r2XWbmyHo0avc2BA/to164DBQoksXTpUj77zMiBA7Xo2PH9DL4aERFJC4U5\nEZFMrFIlHz777Hf8/FZhZmaka1cPChZM//3l0pPBYCCTzOBPpWLF16hY8TWMRmPKKJ2XlzfNmrXK\n6NKeSpMmzWjSpBkJCQl89FEv3nqrMYsW+dK2bS9atSqBi8tPuLgs4MaND4iNTcLJyRlf36WEhYXy\n/vud8fVdir29A0OHfsTOnb/xxhs1MvqSRETkGWkBFBGRTK5p09dYtqw+S5c2oGbN0mk6V3BwEB06\nPN/wUq7cq/z66y+Eh98GSPlvZhAREUHLlj9QqVIIlSvv5vvvd2d0Sc/sq6+mUb78qzg4OHLx4nl8\nfb+kUKH2ODquxcIiGIPhOtmyGahduy4AJ08ep1y5Cjg5OWNubs7bb7/NkSOHM/gqREQkLRTmREQk\nXRUsWIgOHbrQp08POnVqy9dff5XRJaWYMmU727d3JTKyMoGBTZg2LYbo6OiMLuupbdiwnmvXrtKl\nSw+MRiMVKlRiyZKVDBw4DHv7jtjbV6Br1w04ONhgY2MD3D9imhlHT0VE5OlomqWIyEvqypVARo8e\nypAho/D2Lpau5753TzyAFSuWsmHDegAaNWpCy5Zt0rW9J3X7tgX3/h3z5s1cREREpAQeU3Dq1ElW\nrFjKrFnfAuDjU4IvvpjMlSuBtGjxGo0aRRMWFkrevPlo0cI35X3e3sX56qtp3L59C3t7BzZs2EDj\nxtoIXETElCnMiYi8hC5dusjHH49k5MhxFC7smebzrVq1h3XrIrGwSKBPnyKULVsk5WunTp1k48af\nmD9/EUlJRnr06EjZsuUoUsTric6dnnuc1azpxLp1J4iK8gGSKF/+GO7upnXP3OrVK4mIiKBv3+TF\nS7y9fRg58mM+/ngEcXHxAPTo0Zu8efOlep+bmxs9e/ahb9+eGI1G6tSpTdWq1V54/SIikn60z5w8\nd9oPJWtT/5oGP78VrF37I/ny5ePYsWM4OjoyceI08ucv8ND3PGnf/vbbUbp1syM8vAwAhQqtYePG\ncri4uACwcuVyIiLC6do1OXx8++0cnJ2dad68ddov7Bn4+//Otm23cXSMY+jQ6plmtc0XTd+7WZf6\nNmtT/2Zd2mdOROQldvdvcw8ayVqzZhXTp88mPj6eAQP6kDOnBwEBhx8Z5p7U7t1BhIe3SHl+/nx1\nfv99Hw0bVnlgPUaj8YE1nj79N7NnHyc+PhvNmuWkVq20LfbyME2bvkbTps/l1JnW5s2HmTs3hPh4\ncxo3tqBbt5oZXZKIiKQDLYAiImLCgoODaNPmXcaPH0uHDq24du3qfcdMnTqRoKArDBz4IT//vBYL\nCwsmTpzXjaYmAAAgAElEQVTKpk0/s2XLpjTXkC+fNWZmoSnPnZ1PUKxY3pTnpUuXYceO34iNTV5s\nZOfO3yhVqmyqc9y6dYuuXU+ybFkr/Pya8eGHBg4ePJ3m2gQCA4MZPDiOnTtb8vvvzZgwwZv//e+P\njC5LRETSgUbmRERMXPJCJp/g4/PgzcQHDx7B/v2/M3PmXO7cucPOnduxtrZmypSv6N+/N7a2dlSp\n8sYzt//ee9U5eXINmzc7YmWVQI8eNhQoUCrl60WLevPmm43o3r0jAG+/3ZQiRYqmOseOHcc4c6ZB\nyvPQ0Cps3epHhQpPdl+dJNu1awcXL57nvfc68d13c7G1tcPK6hWSkrZjb59IZGR97O0Xs3OnB++9\nVyOjyxURkTRSmBMRMXE5c3o8NMjd6/btW5iZmbNgwTIA7O3tmT9/cZrbNxgMTJzYlAkTHjx9EqBV\nq3a0atXuoecoVCgXtrbniIoq8885b5Ejh0Waa3vZVK1aLWVRE4PBgMEA5cp5Ym29mjt3kvvm9u1u\nvPba2YwsU0RE0onCnIiIibOxsX7sMeHh0dSvf5bIyLxUqfIDCxY0xdr68e97Gg+6N27t2l1cu3aH\nd96pQM6cbg99b4kSRejXbwsLFvxNfLwN9epdoWPHd9O1vsfp1asLs2f7PvTrdeu+wZYtO19gRakF\nBwcxcOCHlChRimPHAvD29qFhw0YsWDCPmzdvMXbsp1y4cJ7Tp0/Sv/8QAIxGKFDgFcqVS+L8+d0Y\njZE4OMzD0/MTALZs2cTSpQsxGo28/npVevX6MOVaW7Row549u7CysuKzzz7HxSV7hl27iIg8mO6Z\nExHJ4kJCgrl924wbN2oQE1OerVs7M336tufaptFopF+/H+nZsyKjRjWnWbODnD8f+Mj39OtXlwMH\nqnLgQCm++qo5ZmYv9lfUo4JcsvTbIuFZXbkSSOvW77Fs2Y9cuvQ3W7duZvZsX/r0+YjFixc8dGQ0\nf353Ro0qxy+/1OWVV7JjMBi4evUqc+Z8zYwZc1iwYBmnTp1g587fAIiJiaFEiVIsXLiM0qXLsm6d\n/wu8ShEReVIKcyIiJu5x+7CFhd0iKeneiRgWREQ83x//gYGB+PuXIinJDTBw5kwLvvsu4LHvs7S0\nxM7O7rnW9jB16ybfNxgWFsYHH3Snc+e2dOjQiqNHj6QcM3PmF7Rv35KPPurNrVu3AOjTpwezZ8+k\ne/eOtGnzLgEBRx54/vTg4ZGHQoUKYzAYKFiwEBUqVASgYMHChIQEPfF5jEYjx44do2zZ8jg5OWNu\nbk7dug04cuQwABYWFlSuXBUAL69ihIQEp//FiIhIminMiYiYmIiIcE6dOk1UVBQeHrlZtGjFA4+7\ndu0aoaGhFC1amBw5OpKU5ASAq+teGjTI+8D3pJcHbWFqNGb8yNajJde3ZcsmKlV6nQULlrFw4XI8\nPZMXa4mJicbb24clS1ZStmw5FiyYl/wug4GkpCTmz19E374DU15/Hiwt/72P0MzMDAsLi5THiYmJ\n91/RIz7y+/8I8O89j+bm/4Z/MzPDA88tIiIZT2FORMSEbN58hJo1/6BaNUfq19/B4cN/3XeM0Wik\nf/9VVKp0lddeC2LUqJ9ZtKgO77+/gg4dVjF7dhJVqxZ/rnXmzZuXd94JwGC4ARjx9PyRrl1LPtc2\n04uPT3E2bFiPr+88zp07i62tLZAcmGrXrgdAvXoNU43YVa+evG+bl5d3phnFMhqNPCBTA8lBrlSp\nUhw5cojbt2+RmJjIL79spkyZci+2SBERSRMtgCIiYkK+/PIKly61BuD06aJMm7aC778vkuqY1at3\nsHx5Y5KSXAFYssSTGjUC+PTTRg88Z3BwEEOH9mfx4h/SrU6DwcCMGc2pXn0H169H8/bb5cidO0e6\nnf95Kl26LLNmzWfPnl1MnPgxrVq1o0GDt1Id89+Nzy0sLAEwMzN/rqNY/x1Ne9AU27uv3V3N8mHc\n3d3p2bMPffv2xGg0UrnyG6lWwnxUGyIikjkozImImJA7d6xSPY+KsrzvmKtXo1KCHEBiYg6CgyOe\ne23/ZTAYaN68+gtvN61CQkJwd3fn7bebEBcXy19/naZBg7dISkri119/oXbtemzZsum+jc+ft/9O\nqR0xYmyqr90N4w0bJof2Ll16PPDYmTPnpjyuU6c+derUT9XO7du3+PbbxSQmJmJubk6NGrWpUaN2\n+l6MiIikC02zFBExIVWrRmIw3ATA0jKQGjXun0fXqFEZChZcl/Lc03Mtb7316OlzSUlJTJ48gfbt\nWzJgQB9iY2PTt3ATcHcE6vDhg3Tu3JYuXdrx669badGiDQDW1jacOHGcDh1acfjwITp37vawMz1V\nu8HBQXTo0CotpaebefN+o3LlE1SunEiLFquIiHjxfwQQEZEnZzA+6C71DBAaql8YWZW7u4P6NwtT\n/75YRqORefO2cvFiEqVK2dKmTdUHHnfixAUWLjwFGOnWrQRFi+Z76DmDg4No3bop3323FE/PIowZ\nM5yqVavRrl1L9e0L8DymuT6J/37vhoff5vXX/yQ0tME/ryTRq9cPjBv34Om5knnp53LWpv7Nutzd\nHZ76PZpmKSJiQgwGA++/X+exx/n4FGTKlIJPfF4Pjzx4eibfe+fl5U1w8JMvc/8y+vvvYEaO3E9Q\nkB1Fitzm88/rY29vn+bzXrkSyOjRQ6lTpwHHjh0hJiaGwMDLtG7djtjYOH75ZRMWFpZMnTodR0fH\ndLiSf0VERBAefu99jWZERlo89HgREcl4mmYpIiL/WfL++S7ikRUMGbKPzZvf488/m+Lv34FRo35J\n8zkvXbrI6NFDGTlyHM7Ozly4cJ6JE6cxf/5i5s37Bjs7O3x9v6dEiZJs2vRzOlxFah4eualU6Q8g\nue+dnf+gfn33dG9HXi6DB3/EnTuRjzxm8WLfF1SNSNajMCciIvKULl26dyqMGZcvp21U7ubNmwwf\nPoixYydQuLAnAGXLVsDGxgZnZ2fs7R2oUiV5pclChTyfaoPwJ2VmZsbChY3o08ePjh1/ZNasKOrV\n01YFkjZTp07Hzu7R3x9Llix8McWIZEGaZikiIk+05L38q0CBcM6dM5K82EkCBQs+euThcezt7cmZ\n04OAgMPkz18Ag8Fw3wbhd58/bIPw9GBvb8+YMW89/kB5KURHRzNmzDBCQ0NJSkqkY8duODk58c03\n00lMTMTb24dBg4bzxx8H+PnndXz66WcAHDp0kBUrvmfKlC9p3vxtfH2X4ujoxP/+t4FVq34gISEe\nH58SDBw4jLlzZxEXF0vnzm0pVKgwo0d/msFXLWJaFOZERF5y/13yvk2b915Y2w/6x52ZWeafNPLF\nF1UZOXIpwcH2FCkSzqefNkzT+SwsLJg4cSoDBvTBxsbmkcdmknXL5CWwb98e3NxyMHXqdBYu/Jb5\n87/h6tUQXn21EmXLVuDYsQA6dWqDlZU1Fy6c4+zZ03h6ejF16iRy5MhBjx6diIgI54svpmA0JrF/\n/+/Y2zswfPgYpk6dyLvvvknFiq9jaWnFggXLmDbtM7p160BsbAw1atSma9f3AWje/G0aNmzE7t07\nSUxM4NNPPyNfvgIZ++GIZBKZ/zemiIg8F3fu3GH9+p0cOHAsQ9q/ePEC27ZtYc4cXxYsWIbBYMbm\nzRszpJan5eHhjq9vEzZurMOMGe8+NoA9jsFgwNramilTvmLlymXcuRP5n9HR1Jt4a+RUXoTChYtw\n8OA+xo8fy6ZNPzN27AS8vX24dOkSAMHBweTMmQtf36W89loVxo0bTUJCAqGh17CwsGDu3AU4OjoB\nEBh4GSsrawA++qg3CQkJNG78LufOncVoTAKgR4/efPvtYhYuXM6RI4c4f/4skPz/vLOzC76+S2nS\npDnLly/NgE9DJHPSyJyIyEvo2rXrtGu3nYCAd7G0DKFTp3WMH9/4hdbwxx/7OX36FN26tQcgNjYW\nV1fXx7wr67l3ZNTe3p758xffd4yf39qUxw0bNkrZGFzkecqbNx++vt/z5ZdTSEhI4Pffd2Nubk6V\nKm8QFxfLxYvnCAqyonPntkRFRXHz5g0OHz6Ik5MTderUT/VHh0KFPKlY8XUaNXqHgQP7smLFagCC\ngq5w4cJ5ALZt28y6dWtITEzk+vUwLly4QKFCyfeQVq9eC4CiRb3Zvn3bC/4kRDIvhTkRkZfQrFl7\nCQjoABiIi3NgyZLr9O59hdy587zQOho2bMT773/wQts0Fdu2HWXSpEvcvGlD+fI3mTHjbaysrDK6\nLHmJhIWF4eDggLe3D0lJifz55zFCQoLJk+cVHBwcMDMzo3v3njRv3prExERat27KunVryJ07D9bW\n1qnOVaRIUVavXkX16rWwtLQgPPw2UVHRmJmZYW5uxuXLl1ix4nu+/XYJ9vb2TJw4jri42JT3371n\n1Nz8+d0zKmKKNM1SRMTEzJnzNatX+6U8/+67uU897Sg+3px7p+7Fx9sRHR2TXiU+kfLlK/Lrr1u5\nefMmkLxpdUhIyAutIbNKSEhg1KhAAgLacOlSE/z92zFlypaMLuup1K37RkaXIGl0/vxZevTohL+/\nH7t27aBz5+707z+EzZs34u+/CltbW5ydXYDkhXl8fEqwb99ecuTIec9Zkn/O5MiRk+7dezF+/BgC\nAy/Tv38fbtwIA6BChUoMHPght2/fws7Ojhs3rvP773te9OWKmCSFORERE1O7dl22bfv3H/a//rqV\nOnXqPdU52rQpQt68d+9Pi6Zu3T0ULPjkm4ynhwIFCtK9ey8GDPiAjh3bpPrH3cvu9u3bXL36yj2v\nWBISYplh9Twb3ddn6ipWfI1Fi5azfPlq2rbtwIQJY/nss08pWtSL7t17Mm/eIjZu/JlOndrSvn0r\nChYsxObN2zE3N0+ZYunntxZLS0sMBgO1a9dl6tTp5M2bj+++W4KPTwkAGjR4k5Ur1/LGGzVo27YZ\n48aNplSp0g+pSveMitzLYMwky2KFhkZkdAnynLi7O6h/szD1b8Z4770WfPXVbG7evMEXX0xm9uzv\nnvocp0//zdq1J3F2NqNLl1pky5Z65n16922vXl2YPfvBmwPfu5S5JK9Y2ajRjxw40BkAc/MQPvlk\nH92710q3Np73927dutXYsmUHUVFRDB8+iIiIcBITE+jevRdVq1YnODiIQYP6UqpUWf78MwB39xxM\nmvQ5VlZWnDx5nM8++xQzMzMqVKjEvn17WLz4BzZsWM/p0yfp338IAEOG9KNNm/aULVueadM+49Sp\nE/ethLh37y6+/vorrK1tKFmyFEFBQUyZ8iXR0dF8+eUULlw4T2JiAl269KBq1eqcP3+OSZM+ISEh\nnqQkIxMmTOGVV/I+t8/peXjSvk1MTMTc3PyJzhkdHY2NjQ2ffjqGY8eOMmHCZIoU8Xqm+vbsOcGm\nTZdwdjbywQe1NH34Ken3btbl7u7w+IP+Q/fMiYiYoJo16/Dbb79w/fr1px6Vu8vLKz9DhuRP58oe\n7m6QMxqNLFmyg2PHYihUyIyePeu8sBpMhcFgYPbsKkyc+D3h4da89hp061Y3o8t6JlZWVkyaNBVb\nWztu3bpFz56dqVq1OpC8wuG4cZMYOnQkY8YMZ/v2bdSr15CJE8cxbNgYihcvwZw5Xz9iJObfUZoe\nPXrj6OhIYmIi/fr15ty5s7zySl6mTp3EN998S65cHnz88UjunmrxYl8qVKjIiBFjiYiIoEePjlSo\nUIl161bTokUb6tVrQEJCQqa8P+vu3+EfN0K1cOG3bN68EWdnF3LkyImXVzH27NlJkSJFOXo0gLp1\n61O6dDm+/jo53Do5OTNy5FhcXd24ciWQL76Ywq1bN7G2tsbOzo7Q0GsEBQVRpcobFCnixfz5swkN\nvcawYaOfeEuR3347Su/eBsLCWgBxHDq0gCVL2jzwWuLi4pgyZTOBgZYUL26gT586GpUT+Q+FORER\nE1SrVl0mTx7P7du3mDVrfkaX80Tq1n2DLVt20rlzf06cuILRaMHq1R0IDFxLs2avEB0dxahRQ7lw\n4RxeXsUYMyZ58+CXdY+pfPk8mDPnxa4w+jwYjUbmzPmagIAjmJkZCAsL5ebNGwB4eOTB07MIAF5e\n3gQHBxEZGUl0dDTFiydPwatbtwF79ux8bDv/XQnx4sXzJCUlkjt3HnLl8gCgTp36rFvnD8D+/b+z\ne/cOli9fAkB8fDxXr4ZQvHhJFi/2JTT0KtWr18o0o3LBwUEMGNCH4sVLcvr0SYoVK86pUycwGAx0\n6NCV2rXrcujQQXx95+Hq6kJAwFESExPp2fMDVq/2Y/v2bSmfw+XLlzAzM2PTpp/x9Z3PvHkLyZ+/\nACNHDuGDD7rj7p6D48f/pG3b9+jWrRfHj//JvHmzWLBgGRMnjqNy5arMmjWd6OhoRowY+1TXsX59\nMGFhzf95ZsnOnWW5ejUkpbZ7DRiwjpUr2wDW+Ptf586djQwb9mYaP0mRrEVhTkTEBBUsWIjo6Chy\n5MhJ9uymspy/ge3bt3HxYggXL27E3PwG+fI1Z9euzjRrBn/9dZqlS/1wdXWjV6+uHDsWQMmSpVPt\nMeXvv4rly5cydOiojL4YeUKbN2/k9u1b+PouxdzcnBYtGhMbGwf8u0IhgJmZOYmJsfe9/967QczN\nzUlK+vf53dUOg4KuPGAlxDjuv28v9Z0lEyZMJW/efKley5+/AMWLl2TPnp0MGvQRQ4aMoFy5Cs9y\n6enuypVARo/+hNDQa6xZ8yOLFq3g1q2bdOvWgTJlygJw9uxfzJq1iWXLVrJgwXxCQkL47rulfPjh\n+xw7dgQzM3Pefbclr79ehfPnz9KtW8d/Apw7YWFhxMfHM3/+Yt55pz6LFy9g166dGAwQH58AJPfH\nwoXf4eNTnCFDRj71NVhaxpPcD8l9Y2d3E1vb3A889vBhZ8D6n3ZdOXDA1O4bFXn+tACKiIiJWrRo\nBdOnz87oMh4rODiIDh1aAXD06BGcnEoABhITXYmOfhVr6wsYDAaKFSuOm5s7BoMBT8+iBAcHp5zj\n3j2mgoODMuIy5BnduXMHF5fsmJubc+jQQUJCgh95vL29Pba2tpw48ScAW7duTvlarly5OXv2NEaj\nkatXQzh58jgAUVFRWFvb3LcSYr58+QkKupLS5tatW1KmWVas+BqrVq1IOfeZM6eA5GCYO3cemjdv\nzRtvVOfcubPp80Gkg5w5PfDxKUFAwGHq1m2AwWDAxSU7ZcqU4+TJE/98H/ng5uaGuXk2HBwcqVTp\ndQCcnJwJD0++zyoqKor+/T9gxIjBAHh7F2PBgmU0bdqcdu06YGZmwMHBkXz58jNt2nQWLFjG0qUr\nAVLaOH36FOHh4U99DQMGvE758guBSzg47OH99++kbCz+Xy4uqVfYdXKKfur2RLI6jcyJiJgIo9HI\n6tU7CQmJon794nh6Zo7pX0/HQP36Obl504/Tp0tgb3+ZJk2qAmBh8e9f3ZP3kkpIea49pkzP3Xub\n6tVrwNChA+jYsTVeXsXIn7/gfcf89/mwYaOZPHkCZmYGypQpj52dPQClS5fBwyMP773Xgvz5C+Ll\nVQwAT88iFC3qRdu2zciRI1fKSohWVlYMHDiMgQM/xNrahmLFfFLa6NSpGzNmfE7Hjq1JSkoid+48\nTJ78Jdu2beF//9tAtmzZcHV1o0OHLs/3g3oKNjbJo1QGg4H/rl9397rufh+VKlWa+fNnAwaioqI4\nceJPbGxsAfj++0V07fo+FSu+RsuW73DjxnUAkpKSCA8Px87Onty5c3P16lUSEhIxGo2cO3c2ZUps\npUqvU7HiawwZ0o8vvvgaW1vbJ74Gd3dX1qx5m+PHz5AzZ3by5Cn50GNHjCjM8OHLCArKQ5EiFxk5\nsuITtyPyslCYExExEYMHr2bp0rdISnLH13cj8+dHU65c0Ywu64kkJiYSFxfLtm1bSEhIYM2aFVy8\neIEJE4Jo3boRFy6cz+gSJZ1t3rwdSB4RGjt2PAMHfghAYmICc+d+TcOGjbC1taV163cZO/ZTmjRp\nxpdfTqF7947Ex8fRvXtPqlatzqhRQ7lx4zoDB/blypVAqlWrkXI/5b0edu9WuXIV+P77VQB8/vlk\nvL19gOSgN3jwiJTjbt68we7df/DWW415771O6flRpLtSpcqydu1qGjZsxO3btwkIOEyfPv1SfR95\ne/vg5OTE2LHDyZXLAw+P3ERGRmIwGIiJicbNzR0LCwu8vLw5cuQQnTq1JSwslLJlywMwZsx4OnRo\nxaBBHwIG6tSplxLmDAYDNWrUJioqimHDBjBt2gwsLZ98CqSVlRXlyj08xN1VuXIxfv3Vi/Dw2zg5\nldXiJyIPoDAnImICwsNvs25dXpKS3AG4fLkhixevxMPDnoEDP8Tb24czZ05RoEAhRo8eh5WVdQZX\nnNqlS39jZWWFv/8GOnRoRadObXBxceGDD/rh4pKdixcv8GT/TtMeU8/Kz28Fa9f+iJeXN6NH3x+G\nnrcrVwIZP34Kw4ePoVu3DmzdupnZs33ZtWs7ixcvoECBgimrS/700zpGjx5GnjyvkC2bBYmJiXz6\n6SSyZbPgnXca0Lhx04cuTDJ58nhatWpHgQLJI4Dr1/uzceNPxMcn4OXlxTvvvHvfe7ZtC2DQoNsE\nBpYlb97DTJ3qRK1aD9vnLOPc/X+/evWaHD9+lE6dkleB7N37owd+H+XIkZOPPhpE/vwF6NKlHS4u\n2ZkxYw67dm1n9OihODg4Ur58BaKiopgxYw6+vvNSRtk8PHLj4ZGbKVOmkytXrpRz3hua33qrMW+9\n9eSL9AQHBzF0aH8WL/6BU6dOsGnTBvr1G/TAY+/druTuxuQicj+FORERE2AwGDAzS0r12t2VwC9f\nvsSIEWMpUaIUkyZ9wurVq2jT5r0MqPLh3NzcU/az6tdvMH5+K5g0aVrK18uWLZ8yIgCk7CMG4Oe3\nLuWxt3cxZsyY8wIqznrWrFnF9OmzcXNzT3ktISHhvv0FnxcPjzwUKlQYSF7Ap0KFiv88LkxISBCh\noddSrS7p6urGhAlTOXHiT44eDcDW1g5IXvTk8uW/HxjmkpKS7lscp2XLtrRs2faRtc2YcYXAwOT7\nOi9fzs3MmT9kujDn4ZGbRYv+vcevd++P6N37o1TH/Pf7KGfOXHz22SfExcXx5ptvp4w4Vq1aPWV7\niHt16dIj1fPFi39IeZyYmMjnn2/mr7+ykS9fLMOG1cfCwuK/p3hi3t4+KaOkIvLsFOZEREyAg4Mj\nLVtew9f3EnFxr1Co0Dq6dfMGkv/6XqJEKQDq138TP78VmSrM3bhxnRs3rtO370Ag+d6/Jxld+/HH\nPaxZE4GlZSK9exemfPln26BYYOrUiQQFXWHgwA+5ejWEKlWqERoagqtrDt5//wM+/XQM0dHJi0sM\nGDCEEiVKpSxz7+zsct92ESdPHmfGjM+Jjo7BwsKCGTPmYGlpyZw5X3PkyB9cv34do9GIk5Mznp5F\naNy4KWFhoXTs2AZnZxccHR2xsLBgwoSP8fEp/s/m1dm4ciWQrVt3p7Q9f/5sjh37N8j5+a0gLi6O\nL7+cyooV3zN9+mzq1n2Dd95pxsGD+xkwYAjz5n1Dnz798fYuxv79v+PrO4+4uDjy5HmFESPGYmNj\nw+zZM9m9eyfm5uZUrPgaMTGpQ0VMzLOHlMxk7NjxT3zsnj0n+PXXS3h4WNCpU8379o3r1WsSf/zx\nFwZDEnv3Fuf69RiOH59OixZt2LNnF1ZWVnz22ee4uGTnypVAxo0bRWxsDFWqVMPPbwVbtuxIdb57\nR94OH/6DGTM+B5L/cPX118nbrTxsuxIR+ZfCnIiIifjkk8ZUq7afS5f28+ab5ciVy53g4KBUwehJ\ng9Kz6tWrS8rm308qe3ZXkpKSUhar2LJlE6VLl3nke3bsOMawYTm4fbs+AH/+uZYNG9xwdTWVbRgy\nl8GDR7B//+/MnDmXVat+YM+eXfj5/cDt27HExsbw5ZezsLS05PLlS4wbN4pvv10MwNmzZ+7bLsLb\n24exY0fwySef4e1djKioKCwtLfnpp7XY29szfPhYRowYhI2NLRMnTsXOzp5Ro4bi4ODAokXL+fnn\ndfj6zqN27bqp/l+tWPE1/vrrTMrz06dPsnz5avbt28vcubM4diyAFi1aM2fOTD76aCBVqlQDICYm\nhuLFS9CnTz8gOQwYDAZu3brF4sX/Z+8+A5o6uwCO/zNI2MsFooKigoLgrnsWt7Yqjrq1asW6xf1q\nnbgHWndFcSuu2rp3XXWhOHHjYIlMIRAgyfshEkGw1bo6nt8ncnPHc29Sm3Ofc88JwN9/CUqlMevX\nr2HLlg20adOOkyePs3HjdgBSUpJJTT3F9evhpKc7oFA85csv/1tFdvbvv8SQIabExbUDErh6dQcL\nFngb3g8Le8idO7d58mQ7IKNgwUlcvnwfrTYNd3cP+vbtz5IlC9m9eyfdu3+Lv/8cOnToRMOGjdi1\na/ufHn/z5vUMHz4ad3cP0tLSDDN+r7cruXr1Ch4ef/xvhyD814hgThAE4R/kyy9zV3OLjo7i+vVr\nuLuXe6tA6X28ayAH+h/XxYo5snPnVmbMmIyTUwm+/tr7D7c5dSqcxMR2htcPH9bj7NkztGhR652P\nL7ySVQGxVq06LwtWqMnIyGT+/Jncu3cXqVTK06dPDOtntYsAXraLiMDU1Ix8+fLj6qoPzrOesbpw\n4Xfu37/Hrl3b0Wgy0Wq1PH36hCpVvuDu3VAKFCgE6GeP586dkSOQk0gk9OjRmw0b1tK9e0dUKhVK\npZL8+QsglUqxtrYhMjKScuWyUh9fbSuVSqlXr2Gu87xx4xphYQ/o109fjTIjI5Ny5TwwMzNHoVAy\nffpkatSoTc2atfH1bULRoqe4ceMM7u4WtG/f5MNd9H+AnTtjiYur9/KVNYcP26FWq1EqlQBcunQe\nne4pxYq1BUAiUWNsXJSMDCNq1ND/N+niUoaLF88BcOPGNWbMmAeAl1djFi/2/8PjlyvnycKF82jU\nqKibZbUAACAASURBVAl16zagQIGCQO7vX1RUpAjmBOE1IpgTBEH4h3vXQOl9eHnV5tChk++0jZ2d\nvaGa4NtydDRBKo0xFHyxsrpJmTJF3mkfwptlL5CzZcsG8uXLz/jxU9BoNDRoUMPwXu52EZo/LFQz\nbNhIHj9+RGxsLH379s+2rYyAgPWG16amptSt24Dffz+DlZUNgYGb0Wq1SKUSAgM3G1LwALy8mhAa\netPQqsLWNh/lynkY9qVQKN84G1258hdMnDgt1/KVKwO5ePE8x48fYceOrfj7L6VDh9w3CqZNm0jN\nmrVzBYvva9Wq5Ziamv1t0qGNjDJyvFYq03I9S9mkiRcnT5bhwYPCFC0aycyZbowYccHwvlQq+ctt\nQ7p06UGNGrU5e/YUPj7fMm/eopfjyv39EwQhJ9E0XBAE4R9OJpMxfvwU1q8PYurUmYa76R/H26dw\nqtVqZs3ay6hRB9m79+I7HaVTpzr07r2fYsV2UapUEOPGxeHs7PSOY/37yN44HWDjxnUEBKz4jCN6\n5dixw8TFxQGwf/8etFp9oZ3ExARu374F6J9vOn1aH8QXK+ZEbOxzQkNvAtC2bQvi4+OoWrU6O3Zs\nw9OzIseOHebmzeukpaWRlJSIu7uHofn3wYP78PSsAOgD/axjnDr1G5mZr3oLJieraN58J+XLn+bX\nXx8QHa0fo6mpKSkpKX94ThKJBDe3cly7FkJ4+FMAUlNTefLkMampqSQnv6B69ZoMHDiMe/fu/OF+\n3tW0aRM5fvwIgOFavmm/Pj5/3MNu7dqcM+F/tv5fNWCAG6VLBwHxWFhcoHdvqaFgEUClSlUJCbnI\n+vV1uXjRjW3b6mFnZ/7G/bm5lePYMf01OHz44BvXyxIe/pQSJZzp3Lk7rq5lefz4kahaKwhvSczM\nCYIg/AMdP36V48cjMDdP5F0CrE/pu+92sHdvD0BBUNB15sw5S5s21d9qW4lEwtSpXzNlysd9BvBz\n+TznlD2t8dVSR8fiXL58iR49OvHFF9UNjaWtrKwNqZTZyeVyJk+ezvz5s1Gr1cTFxZKRkUnLll8T\nGRnBlCnjSU1NZeDAfjg4OODqWpYhQ0YyffokNm5ch42NjaG8fatWrRk9eniuYwPcufOCq1e7AaDT\nXWbnzkf06KHfZvjwgRQoUBB//6VvvJbW1taMGzeRiRPHkp6un3nq27c/pqamjB49nPT0dEDHwIHD\nDNvs2/crmzdvQCKR4OxcEplMxpUrl9myZQOxsbH07z+IevUa5ijeATBv3kzKlHGjadMWHD9+hOTk\nZNauXU3nzt0wMzNnxYolaLVarK2tWbBA/3dY2APkciPat/+K9u2/wdu7Y65zWLduTY6m5X8lzflt\nuLo6sXevDWfOnMfZ2Z5SpRrkeN/JqTh9+vjw7bedSUhIxMTEhNmz/XOlymYZNGg4kyePZ9261VSt\nWg1zc/M818v6MyhoE8HBF4mJeUbx4s5Uq1aTa9dC3rJdiSD8t0l0WQn0n1lMzIvPPQThIylQwEJ8\nvv9i4vP99H7++Ry+vvlITKwAJNGhwzYWLWr3p9u9q7w+Wy+vOrmq0uUlOTmZypWvExfnZVjWtu12\nli5t9MHH+U+Qvb8WwKZN60lNVeUqBf8pZAUsRkYyHB1LIJPJMDU14/btmzkCluxjzh68JCYmMHHi\nOJ4/j8Hd3YMLF84RELCe6OgoRo8eTrlynty/f5fZsxdy9OhBjh07THp6BnXq1OPbb78jMjICX99B\neHhU4Pr1EAoUKMj06XNzzSh7eR0mJKS14XXFijvYv9/r9dP5YB48uM+4cSNYvnw1lpZWJCUl8eOP\n80lLS2Py5OmEhT1k9OhhbN68M8f12LfvV378cT5KpTEVKlTit9+OUbKkC6AlJiYGlUpFQMB6IiLC\nWb58MTY2Nly7FkKxYo7cvXuX7dt/oWPH1hQv7kxqqgqNRsPw4WM4c+Ykmzevp0QJZ0qUcGb8+CmG\nNGeVSsWYMb68eJGERpNJnz4+1KpV13Btv/iiKhcuXHzjtf2rOnf2fqv2Fmp1miGV9/DhAxw5cihH\nK5I38fObRI0atT54Wuu/jfj/7r9XgQIW77yNmJkTBEH4h/nllwQSE798+cqS48cLkZ6e/rKgxd+D\nsbEx5uYveJm9B+gwNVV/ziF9VjKZDK321b1TtTrts4zjwYP7rF0bwPLlq3F2LsL9++H8+ON84uJi\nWbo0wBCw/NGP6dWrV+LpWYEePXpz9uwpfv31Z44evcbUqc9QKp8RFVUcf/9uPH4cxtOnT2jduh2h\noTe5fTuUkJDLFCxYiKdPnzBp0nRGjRrHhAljOHHiKI0aNc1xHHf3JEJC1IASSMPD449TK9/GihVH\n2b1bg1yuoU+f/DRvXtnwXnDwBRo08MLS0goAS0tLAGrX1vdjc3IqbkhHff161qlTDw+PCtSsWYff\nfjuGhYUFs2bNZ8eOIJYtW4SdnT0REeE8eHCPdeu2snfvLxgZGXHv3j2srKyRy40oV84TH5+BaLVa\n0tLS8PQsz44dQaxevTHbEfVTVUqlkunTZ2NqakZCQgL9+vU09I17+vQJCxf6M2jQyDde278ir/YW\nERHh2NnZM3iwL3Pm+BEdHQVAs2at+PnnHcTGPkcikWBnZ59rBjL7LGjJkqX43/8mAXDlSjBz5/5I\nYmIKtrZfMnlyGzw8Sr73+AXh30oEc4IgCP8wSmXOYgXGxrmLFXwsb5seKJfLGTzYhBkzDhAbW5SK\nFS8wYkTtjzy6vy9b23wkJMSRlJSIsbEJZ86colq1Gn++4Qf2VwKW14WEXMbPTz/LUr16LSwsLFm6\nNJKIiEYUKbKWsLChzJ+/merV73LhwjnOnTtLWloaFhaWPH36hIIFC2Fv70DJkqUAcHFxJTIyItdx\nZs5sgaXlDsLCFDg7pzNmTPP3OvcDBy7i51cWlUp/3Hv3juHuHo6jowOg/27nlayUvTF21vsymRyd\nTmu4nrGxz4FX17NWLX3bhEKF9FUhs5Qp44adnT0Acvmr/RobG3PkyEGUSiW1a9ejVKnSf3guOp2O\nZct+JCTkClKphOfPY4iP139u9vYOuLq6EhPz4o3X9q/Iq73FkiU/oVAomDhxHO3bd8LDozxRUVH4\n+g5k/fogVq1azsWL51m0aDkpKcl06tSW1q3b8ehRmOGmgqWlFS9evDCc1/nztzh/fgtGRgmkpfkw\nZEgZDh50+mT/xgnCP434L0MQBOEfZsgQD65f38KtW7WxsrpHv37KXA1+35dOp0OlUuVYlpiYYPix\n+ja6dq1FixZxPH8ei6Nji7/VzOGnJpfL6dGjN336dKdAgYI4ORX/LM/N5RWwqFQqFi9ewNmzp7lz\nJxS1Og21Oo379+8RERHOt992RSKRYGGhT/9Rq9WMGeOLTqfDwaHIy++KEnv7YchkyRQr9jVPn8YR\nF1eBLl16oFAoCA29ydChI4mPj2fq1AnExETTp083Bg0ajlQqQ6PJPWurUCiYNKnFBzv3K1eeo1LV\nN7x+9qwa588fNARzFStWYexYXzp27PwyzTLxjfuys7MjLOwhVap8gVqt5tKli4aiLoAh8Chb1h2t\nVmsIqN4UkJiYmDBmzETu3buNn99EOnToTJMmbw5eDx7cR2JiAgEB65HJZLRr1wq1Oh0AheJVkPim\na/s+cre3gIsXz/Po0UPDOiqVitTUVCQSCTVq1EIul2NlZY2NjS1xcbG5bipkfbckEglSqSs6XT7S\n0/Mhkz3n3r1SPH8eYwiCBUHISQRzgiAI/zClShVjzx5bQkJu4+hoR5EiFf58o3cQHHyXUaNuEhFh\nR/Hi4fj7V8XKSsnAgd/xzTdd32lfNja22NjYftDx/VN5e3fMs8jFp5Q9YClQwMIQsDx//pw2bdrh\n7u5B3bpfsH37Vo4cOUihQnasWrWOn35axt69vwD6oL5OnfqMGvU/Jk36Hy9eJFGrlopDhzIBHVFR\ni+jVaytXr/7C06dPadasJQAxMc+YP382LVp8RUzMM6ZMmYWv70CaN//qk5y7h0c+jI3vk5bmDEDB\nguepUuXVDFjx4iXo1q0XAwb0RSqVUbq0C/B6wQ7934UK2VG//pds2bKJxMR4KlWqApArALSxsUGh\nUDBu3AiSk1NQqZKz7evVepmZGVhaWtKy5dekp6u5e/c2TZo0Ry6X5/lMWkpKCjY2tshkMoKDLxIV\nFfkBrtC7yd7eAnSsWBGYYxYzS/YZSKk0q71F3rOgAHZ2MuAFYIFEosPR8QH58n28ZyUF4Z9OBHOC\nIAj/QObm5tSsWemj7HvKlFBCQvT9r2JiYMqUDaxZ04pNm3Z8lOP9W92794S5c0NQqRR4eRnTpUud\nzz2kHAGLQmFEiRKlkEj0lSvd3fW92+RyI86d+53Hjx+h0Wjo2bMTKSnJpKenk5KSjEKhJDo6iq5d\n21OihDNyuRFjxjQhNHQLyclS5sy5TseOfWnbdjd16tQnMHAVarWaO3dCCQsL4/HjMCIiwhkzZhgq\nlYrMzIxPMkvZtGkVRo06xC+/hGBkpKF3b1ucnIq8tk4LmjZ982zgwYMnDH/37z+I/v0HsW/fr2za\ntI4tWzZy+fIl6tf/EjMzM8N6MpmcgIANBAdfZMsWfe+86tXrk5qaZgjounbtxZgxw5DL5Ziamhme\nH2vVqjU9enyDi4sr48dPMVynRo2aMGrUMLp374iLSxkcHYsbjvf6tfwU17ZKlWoEBW2mUyf9zZ67\nd+/8QaqoJI9Z0CTDrH+zZp7IZDs5f96CjIxMZs4snmeQKAiCngjmBEEQhBxiY01yvI6LM3nDmsKb\npKWl8d13l7l2rTMAx4/fwcLiHF999cVnHtmrgCWrIl5kZAQDB35neH/WrPls374VZ+dSLFuWsxR+\ncnIyUqmUefN+BPT9wZ48eYKVlTVFixakZ89xVKz4qqhImzbe2Nracvv2LYYOHUnz5l8yfPgEbGws\ncXJy/DQnnM3333vx/fcfdp9vGwBWrFiZChUq4eu7nY0bK5OZaU3Dhv1IT09/4z58fAbi4zMw176s\nrKxzfTZZAgM3G/7+8E3J825vMWSIL/PmzaR792/QaDSUL18RX9/RudbLktcsaFa7CplMysyZ+iqm\njRpNoUaN3O0xBEF4RQRzgiAIQg6enomEhmZVEUyiQoXPU3nxn+zhw0dcu/YqqElNLc3Zs1f56tNk\nFL6z6Ogorl+/hrt7OQ4d2o+bmzu//LLLsCwzM5MnTx5TvHgJLCwsCQm5gqdnefbv30OFCvoZYp1O\nx9Gjh6hYsTIhIVcwN7fA1PTVDFVaWhrp6fZ07HiS1NSG9Oz5K126lKJUKZfPddqf3Jkzl9m4sS6Z\nmfqZtCNHerBq1W58fJq8975TU1NZvvw46eng41MTC4u3f771bQUF/QyQq6WGlZU1kyZNz7X+6+tl\nteaAvIPg778fTGamxvA6+0yoIAh5E8GcIAiCkMPcuS3Jn38Hz56Z4+SUyrBhzT73kP5x7O0LUqjQ\nTaKjswIVFfZ/4/oNxYo5snPnVmbMmIyTUwm8vTtStWp1/P3nkJycjEaTSYcOnShevATjxk1kzpzp\npKWl4eBQxDCjIpFIUCgU9OrVGY1Gw5gxEwzLJRIJS5Yc49q1lRQs6Iel5WT27UtBo3Fl6tQZn/PU\nP6m4uBQyM/NnW6JEpXr/NMiMjAw6d/6ZU6d6AnJ27/6ZdetcKF7c4b33/alMnvwrGzYUIDNTQbNm\nR/H3b/vBCzsJwr+RaBoufHSiueW/m/h8/51WrVpOwYK2tGz54ZuR/1cEBf2Ov38cycnG1KwZy8KF\nbZDJZJ97WAbZ0yyzNzT/qwYO/I4BA4bi4uKa5/tTpux/rbl9JDt33v5oz37+HalUKtq23cOlSz0A\nKSVK7GLTJtf3DrqOHz9P+/buQCHDsqFDtzJmzPv1l/Px6cXSpXmnc35IZ85coX17O9LTS71cksDc\nuSfo2rXBRz/2P5H4/+6/l2gaLgiCIHwQn6Ns/r9Nu3bV8PbWodVq/1ZBXF4+9uedmJiIVhuDtfXP\nJCR8BWipWnUvlSt/uLzTXbu2Y2xs/Icl/T83U1NTNm1qxJIlW8nMlPLNN27vHMidPHmcokUdcXJ6\nVfTEzMwYmewFGk1WMKdBofjr9+qzKmh+ikAO4NGj56SnV822xJqYmPRPcmxB+KcTwZwgCIIAQGDg\nKvbv34ONjS0FCxaiQAGbzz2kfzyJRPK3D+Ts7QvnKJrxVy1atDzP5bGxcXTocIyrV/sBV3F0nEKr\nViUYPLgJSqXyvY8LoNFoOHBgzycLPt6HtbUVY8f+9YDzt9+OU7NmbZYvX8yzZ9Gkp6vx9u5Ihw53\nOH++LQkJbSlY8ABhYY5cv16UZcsWER0dxeDBvtSqVQeNRsOyZT9y5col0tMzaNOmHV991Ybg4Iv8\n9NMyLC0tefz4ERs3bsfLqzaHDp0EYP36NRw6tB+JREr16jX57rvv2b17J7/8spOMjEyKFCnC+PGT\nUSqNmTZtImZm5ty+fZPY2Fj69x9EvXoN33hOjRtXwsXlZ27fbg9AkSKHaNYs7xleQRByEsGcIAiC\nQGjoLY4ePcSaNZvQaDLp1asLlSt/2P51wn9TQMDvXL3aHX0lxPI8elSI2rVv5tmAPjU1lQkTRhMT\nE4NWq6F79944OBThxx/nk5qaipWVNePG/UC+fPkZMKAvpUu7EBJyBS+vxlSpUo1Nm9bzzTddCA9/\nyrx5s0hIiMfY2JhRo8ZRrJgTR48eZs2alUilMszNzfnxxxWf5Bps3LgWhUKBt3dHFi6cy/379/D3\nX8qlSxfYs2c3TZs2Z9WqFaSnpxueQzQxMWHp0kWcPn0SmUxG1arVqFu3PqdPn+TKlcuYmJgwY8Zc\n1Oo0+vbtgYNDUaTSNLp31zJt2lG8vBoxfvwoChQogIdHeaZNm0jjxk05c+Y0L14kMWrUOGrUqE3/\n/r2pWrUaAHfv3mbduq3ZGnTrZ2zPnj3N6dO/sWJFIEqlkqSkJADq1WtAq1b6ypMrVy7l119/pm3b\nDgDExcWydGkAYWEPGT162B8Gc7a2NgQGlmPp0s1oNDI6dXLC1dXp43wYgvAvI4I5QRAEgatXL1On\nTv2XMyVKatas88amvoLwrgoXHoBcHolEkk5CQkugJF5etWnd2puzZ0+TL19+evf2YcaMyTx79owJ\nE6ZQq1YdkpIS6datA7a2+dBotBQqVIgVK5bQuHEz7t69Q3R0FEZGRnTs2IV69arRr98AAIYNG4BM\nJkOhUGJv78DcuTNp2LAR/v5zsLd3oFixIgwbNuqTnb+nZ0U2b16Pt3dHQkNvkZmZSWZmJiEhl3F2\nLklgYAALFizB2NiY9evXsGXLBtq0acfJk8fZuHE7ACkpyZiZmVOrVh1q1qzNvXt3GTt2BOHhT5BI\npIwYMZbvv+9NRMQ9lEolFhYWZGRksHz5GnQ6HQ0a1CAuLo7SpUsTGnqLSZP+h5NTcVJSUnj69Aky\nmYwyZdyyBXKvXLx4nubNWxlmUrMC8fv377Fy5VJSUpJRqVL54ovqgH5GunbtugA4ORUnLi7uT69R\niRJFmD27yJ+uJwhCTqJMkCAIgkD2/lF6IpATPoxvv61OvnzuPH68jcePAyladC3ly5ciLS2NSpWq\nsm7dVkxNzVi1ahnTps3C1NSUmTOnEhJyhaCgTSQmJr68saDj4MF9PH36BIDUVBU+PoMMwU7Wd/jE\niWNERIRjZGSERAJ37twiNjaWevUavOyvVwC1Op19+379ZNfAxcWV27dvoVKloFAocHcvR2joLa5e\nvYJSqSQs7AE+Pr3o2bMT+/fvJTo6CjMzcxQKJdOnT+bEiWMolcaG/V25cplNm9axYMFitFotOp2W\nKVPGo9FoiI2N1V8NiQSpVEpw8EWkUikajYbq1WsAMGrUOLRaLT/+uIKtW3+mShV9/0Nj41c9Jb29\nWxpu6EgkEvK6t+PnN4nhw0cTGLiZXr36kJ6uNryXvdG3uDEkCB+PmJkTBEEQKF++AtOmTaJLlx5o\nNJmcPn2K4sU7fe5hCf8CtrY2tG0bh6lpfaRSCWq1mvBwfbCVNZPj7FwShUKBo2NxAgM307Ztc1au\nXEJCQryhOItUKiVfvvz06NEbAFNTMxwccs/kBAdfwNTULNdzgJcvX+LBg/vExj4nMTGRkJBgWrb8\nGktLq498BUAul2Nv78Devb9Qrpwnzs4lCQ6+QHj4U+ztHahc+QsmTpyWa7uVKwO5ePE8x48fYceO\nrfj7LwUgIyMdqVSKkZERJiamqFQqRo36HyNHDmX9+q0A6HTg4lKGSpWqvHytQ6fTUbVqdXbs2IZC\nocTMzJzHjx9RsGChXMfOXhSnSpUvWLNmJY0aNUGpNCYpKQlLS0tSU1XY2uYjMzOTAwf25rkfQRA+\nLhHMCYIgCJQu7UrDhl706PENNja2lC3r9rmH9LcTGRnB8OEDcXf34Nq1EFxdy9K0aQtWr15BfHwC\nP/wwhTJlxHV7XXDwRa5fv8q2bdtRKpUMHPgd6elqZLJXP0EkEglyuRHPnz/HwsICiUTKN990ZebM\nqVhaWjF8+Jgczcvj4+Py7EGm04GRkQJrayuOHTtM/fpfotPpuH//Hn5+kxg2bCTVq9di375f+fHH\n+Tx79uyTBHMAnp7l2bRpPWPH/kCJEs4sXDiPMmXK4uZWjnnzZhIe/hQHhyKkpqby/HkM+fMXIC0t\nlerVa1KunCcdOugrf5qampI/fwG0Wh1t2jQnNTUVY2MT0tPTSU1V4ec3iUePHpCc/IKrV0M4fvwI\nz5/HALBq1QocHIrg5laOkyeP07mzNzY2tigUCsLDnxIfH8fRo4dp0OBLQF/VslevLmg0mVSp8gXf\nftsNIyM51avXom/f/vTu3Y++fXtgbW2Nm5s7KpXKcL7Zg0FRHVcQPh4RzAmCIAgAdOvWi27dehle\ni15GuYWHP2Xq1FmMGTOB3r27ceTIQZYuDeDUqROsXbua6dPnfO4h/u2oVClYWFi8TCd8yI0b19+4\n7oMH91i82J+0tFTWrPmJFi2+5vr1EJYuXUhKSgppaSo6dOiMo2PxPLeXSPSzSJcvX2T37p0EBgag\nVqtp3LgpqakqduwIYunSRURGRlCokB0lS5bKcz8fg6dnBdatW427ezmUSmOUSiWenhWwtrZm3LiJ\nTJw4lvT0DAD69u2Pqakpo0cPJz09HdAxcOAwABo2bMS0aRNJS0tl6tSZuLiUoX//3vj5TUIul/P8\n+XN27NjB0KG+REVFIpFI8PbuyPLli+ndux9Nm7YA9NUply5dRXDwRc6d+515834E9M/mZRk4cCht\n27Zn585t3LkTapj1y/L11958/bV3rnPNaiSf5eDBEx/sOgqCkJMI5gRBEP7Drl9/wPjx14mONqNs\n2Xj8/ZtiZmb2uYf1t2Vv70CJEs4AFC9egsqVq77825moqIjPObS/rS++qMGuXdvp0qUdRYs64u5e\nDsg9WyORQNWq1ahatRqNGtVl5cpAdDodK1Ys4cyZk+h0OgoVsqdRo6bcuXObcuU8cjQoVygUdOzY\nBdBXZdy/fy9GRnLq129Ijx69sba2ZsOGdVhbW9O8eascs0ifQqVKVTh27Kzh9aZNOwx/V6xYmZUr\n1+baZuXKwFzLypXzZP78xQwY0NdQIXL8+MkEBW3i3r27jBo1DtAHVH5+kwzbWVvbULNmbcPrrEIn\nzs6lWLzYn6VLF1GjRm08Pcsb1qlbV9+0u3RpV06cOPqn5xgWFs66dVeRy7X061cTGxvrP91GEIT3\nI4I5QRCE/7DRo69x/rz+B/C9exqsrTcxZ86Ha+T8b6NQvCrqkPXMUtbfGo3mcw3rb83IyIg5cxbm\nWp59tqZXr755vieRSPjuu+/57rvvc7xfoUIlKlSo9Mb9denSgy5degDw/HkcPj47iYqywtW1F5Mm\nNUWhULzXOf0dZA+GdTodEok+7dTExCTXujduPCAhQUVg4DF8fFogl7/6+Ve0aDECAjZw9uwpVq5c\nQuXKVQ3PJWZ932WyP/9+P3kSRefON7h7tz2g48SJNWzfLm4OCcLHJoI5QRCE/yidTkd4uHm2JTIi\nInL/EBT+fsaM8TU0jG7X7htatWrNr7/uYsOGtZibW1CyZCkUCgVDh44kPj6euXOnEx0dBcCgQcMp\nV87zM5/Bp9O6dSC3b3sC6Zw+XQ2p9ADTprX83MN6b9HRUVy/fg1393IcOrQfDw9P7t69nWu9+/cj\nGDLEFLm8INOnV+fy5SBWrepoeD/rOcVGjZpiZmbOnj27/9J4tm27wt277V6+khAc3JF9+w7h7V3/\nL+1PEIS3I4I5QRCE/yiJREKpUgmEh+vQl3VX4eKS/rmH9beWOzXw8xR5GDNmApaWlqjVafTp050a\nNWoRGBhAQMAGTExMGDzYh1KlSgPg7z+H9u074eFRnqioKHx9B7J+fdAnG+vrIiMjGDVqKGvXbnnv\nfQUHX2Tz5g3MmjU/z/c3bz7N7dudAeeXSzYRGip77+N+bhKJhGLFHNm5cyszZkzGyakErVt7s337\n1lzrnj4dTXi4D9bW0RQpMoCrV02JiHiVbpn1nKJUKkEul+PrOzavI/7p99vMTAKkAfoWClJpHDY2\npu9xloIgvA0RzAmCIPyHLVpUlwkTNhATY4qbWyrjxjX73EP627K3L5yj3H32Ig+vv/exBQVt4uRJ\nfVrhs2fR7N+/hwoVKmFhYQFA/foNefLkMaBv+Pzo0UPDtiqVirS0NMDik433c7lwIZlXgRxAWayt\nj3yu4Xwwdnb2rF27BZksZ2AaFJRzVm3s2B8YN+5XQEdCQhcSErpgaXkChUJpWDfrOcXXZd+Xq2sZ\nFi5c9odj6tmzASdOBHLoUCPk8jS8vc/SoEHu4iiCIHxYIpgTBEH4DytUKD/Ll4tn5N6WTqdjzZrj\nPH6cTrVq+WncuNKfb/SBBQdf5NKlCyxfvtpQ6t/R0YlHj8KyjTP7TKGOFSsCczRx/qsOHNjL1Vgz\nsQAAIABJREFUtm1byMzMoGxZd4YNG0WTJvVo1+4bzpw5hVKpZMaMudjY2BIe/pRJk/6HWp1GzZp1\nCArazKFDv+XYX2RkBFOn/kBqaioAw4aNxN3dg+DgiwQErMDa2oaHD+/j4lKGCROmAPD772dYtGge\nSqUxHh7lc40xO3t7LZAK6NOHTUxuMHVqi/e+Dh/Sm67poUMnATh27DBnz55m7NgfmDZtIgqFgrt3\n7+DhUZ7GjZsye/Z01Go1Dg5FGDNmAhYWFnTt2hVHR2euXLmEWp2Ou/sTrl//DmPj+1SsuISxYzPQ\naDLp1asvtWrVzfE5xMa+4MWLlsjlxahX7wkREefy/BxeZ2RkxNq1Hbh8+QbGxgrc3LxFSwJB+ARy\nN2kRBEEQBCFPY8bsYvToWixe7E2/foVYv/63P98om8jICLp16/BeY8he6v/RozBu3LhOamoaV64E\n8+LFCzIzM3NUHqxSpRpBQa9mDfN6rupthIU95OjRQyxbFsDq1RuRSmUcPLiPtLQ03N09WLNmI56e\nFdi9eyegT+/s0KETgYGb39hM2tbWlvnzFxMQsJ5Jk/xYsOBVa4d79+4wZIgv69cHERERzrVrIajV\nambNmsasWQsICFhPXFwsfxQvDB78Jd7emyhadBdly27mxx/tsbe3/0vn/2d8fHr96Tpbt25ErU4z\nvH7TNdWnPeu9HhA9fx7D8uWrGTBgCFOn/sD33w8mMHATzs4lWb16BaCffX38+BGrV29k1KhxWFv/\nSt++k/n224307t2WlSsD8fdf9rINRJrhc+jVawQ3b44lKekUV660Zf36QoSGhub4HK5evWIYi5dX\n7Rxjk8lkVK7sgbu7qwjkBOETETNzgiAIgvCWjh61QKezBSAlpQz79t2kS5e32zYzM/ODjCGvUv8F\nCxaka9ee9OnTHUtLSxwdnTA11VcRHDLEl3nzZtK9+zdoNBrKl6+Ir+/odz7upUvnuX07lN69uwKQ\nnp6OjY0NRkZG1KhRCwAXlzJcvHgOgBs3rjFjxjwAvLwas3ixf659ZmRkMn/+TO7du4tUKuXp0yeG\n98qUcSN//gIAlCxZmsjICIyNjSlc2AEHhyIANGrU1BA85sXIyIglS9q9rPb4cYOLpUsD/nSdoKDN\nNG7cDKVS/1zZm65pdlqtzvC3RCKhfv0vkUgkJCcnk5ycjKdnBQCaNGnO+PH6z1WlUmFrmx/Q97eT\nSCSMHu3L4ME+3L9/k02b1gGQkZHBs2dR2NrmZ/78mfz+ezBWVrYoFI8ASEkpgY2NfY7PISoqMtuM\nqAjYBOFzE8GcIAiCILwlE5MM8uVbiEZjRUJCd5TKDJYvX4ytbT6ePYvm3LkzSCQSunX7loYNvQgO\nvshPPy3D0tKSx48fGRozg74B+fjxoxg58n+4upZ56zG8qdS/i0sZWrVqTWZmJuPGjaBOnXoAWFlZ\nM2nS9Pc+d4CmTVvkahOwadN6w99SqeSdWjRs2bKBfPnyM378FDQaDQ0a1DC8Z2T0qn3Aq9L4rwcP\nOt7Gp5gl8vKqzaFDJ9+YIhoUtJnnz2MYNKgf1tY2+PsvJSws7OX4pDg4FGHs2B8wMTFh9eqVLF26\niAsXzuHuXo5jxw5TqJAdp079xqVLFyhb1g1b2/xkZGTQr18v0tPVAKSnZ5CRkcHTp0+Jjn5Gz56d\n6NKlJ6mpqSxZog+mhwwZwZo1P5GYmIiDQ1GUSmO2bNnAnTu38fT05PDhh0gkKszND2BmFoWNjTmD\nB/fnxYskoqIiMTKS06hR049+PQVBeDsizVIQBEEQ3pKPjxUyWWEsLbfh7LyTAQNKcvToIQoWLMi9\ne3cIDNzMggVLWLLEn9jY54A+rXHIkBFs3LgdnU4ffDx+HMb48aMYN27SOwVyfyQgYAU9e3aie/eO\nFC5cBDc3T+bN28f8+ftITEx87/1XqlSVY8eOEB8fD0BSUiJRUZFvXN/NrRzHjumLjRw+fDDPdVSq\nFGxt8wGwf/8etFrtH47B0dGJyMgIwsOfAnDo0IF3Po+P51XAmFeKaLt2HcmfvwCLFi3H338pCQkJ\n3Lp1HaVSydy5i3BxcSUwcBVRUZFIpVK0Wi0//bTW8D2ytrahVq06VK1ajU2b1mNubo61tQ3fffc9\nAQEbKF7cmczMDIyMjHBwcKBQoUKsXr2R/PkLYGxsjJGREVWrVsPPbxLNmrUkMHATHh6eLFgwB5Uq\nBaXSGLlcR7NmVZBIoGjRSfTunYyVlQXTp88mIGA9derUe+NnKQjC5yFm5gRBEAThLXXsWIN69aIY\nPVrG4ME2qNWxlCrlwtWrV/DyaoJEIsHGxpby5Sty69ZNzMzMKFPGDTu7V89pxcfHM2aML35+c3B0\ndPpgY/v++8GGvxMTE2nb9jBXr3YHdBw4sJpt25phbm7+5h38CSen4vTp48OwYd+j1eowMjJi6NCR\nb2zPMGjQcCZPHs+6daupWrVajmNnrde6dTvGjRvJ/v17+eKL6piYmGZbJ/cYFAoFI0eOY+TIISiV\nxnh6ViAi4mme49VoNLmqPX4quVNEI3P19rtx4xpRUZEYG5vg7d0CrVaHqakptWvXw8LCkmPHDnP1\n6hVDsF+3bgNu375F4cJFuHTpPADffz+IsWNHoFanYWRkRL58+tRKiUSCVCqlV6/OaDQamjdvRVJS\nIj169GbLlg1s3LiODRsCsbcvzM2b1/n++8Hs3fsr8fFxfPll45cpumnUrl2WzZuDWbbsR0JCrvD8\neQwqVQrx8XHY2Nh+ugsqCMIbiWBOEARBEN6BnZ0dXbt248SJY8THx9K8eSsuXjxnmHXLkhWwGBvn\nbMRubm5OoUL2hIRc/qDBXHabN5/h6tVu6GeLJAQHdyco6Gd69mz0Xvtt2NCLhg29ciw7ePCE4e96\n9RpSr15DAAoUKMCKFWsAOHz4gKFVQvY2DkWKFCUwcJNhex+fgQBUrFiZihUrA/pqjzdv3uDq1Stc\nv36VYcNG8exZdI5qj35+k96p2uOAAX0pVcqFK1cuodFoGDNmAmXKuJGamsr8+bN4+PBBjmqP7yor\nRTQoaDNHjhzk3r07NGrUJNd6lSt/wcSJ03ItNzExYdWqdVhaWgFw5swpFAojxo79gdDQm5w/f/bl\ndT1I797f0bZtB6KiIhk48DvDPooVc2LyZH167b59v5KUlIhSqcTU1JSAgPXI5XIyMzP5+usmFClS\nlNq161KjRi3q1WuIj89AvLzqULFiZaKiIjl37gwBAeuRyWS0a9cKtVr0oxSEvwuRZikIgiAI76hu\n3fqcO3eG0NBbVKtWAw+PChw5cgitVkt8fDwhIZcpW9YtV4AH+mfe/Pxms3//Hg4d2v9RxmdsLAey\n/+BOw8Tk085ShYaG0qNHJ7p3/4Zdu7YzYMCQd95H9mqPP/20jjt3IvD1nZXjuv6Vao8SiQS1Oo3V\nqzcyfPhopk+fDMDatQFUrlw1V7XHv2rXrm3Url2Xr75qA4CpqSkpKSkAlC3rzrVrIYaU0eTkF4aA\n922lpKQYZgD37HnVF04mk+UYd/br5e7uwZEj+lTJgwf3GQqovC4jI5PmzQ+ycOEFXrzQz3IGB1/8\nw9RaQRA+PTEzJwiCIPyrZRWmeF/BwRfZvHkDs2bNRy6XU6lSFSwsLJFIJNStW58bN67So8c3SCQS\n+vcfjI2NLWFhD3OlC0okEoyNjZk1awFDh/bH1NSMmjVr53nMrVs38tVXbQzVD99Wp071OHAgkMOH\nvQEtTZrsoF2792uJ8K48PcuzZs3G99pH9mqPT54kkJhoyosXVciXT8eNGw9wcyuRY/23rfYI8OWX\njV+OswIpKSkkJydz/vzvnD79W65qj8WKOf3pWHOmm8Ls2X5ERIQTHx9Perqa3347RmJiIp07e1Oy\nZClWrAikQoVK9OnTnfR0NXK5EUOHjmDJkoXExDxjwIC+jBz5P9zdy6FSpTJkyPfodFoKFy5iOE6n\nTt2YNu0HAgNXUb16LbKe29uwYQPdu/cwFECRSCSG8Q0ZMpLp0yexceM6bGxsGDv2h1znsGvXWdLT\n5dy82RaptAGJiR3p3NkbN7dyODoWz/OcBUH4PEQwJwiCIOQQGnqL/fv3MG3aJFatWo6pqRnffPOW\n9fffko9PL5YuDSAqKpJr10Lw8sqdgvbhfPgfnFqtlhs3rjF16izDsv79B9O//+Ac61WoUIkKFV41\nFs+eYmhubs7KlWv/8Divl7J/fQxSad4JNvoGzu05fPg8UqmEhg07fLbnx95X06YtaNq0JTVrpqNW\n6wMzG5sANmy4jZ9fCdRqdY71jY3fLfDNkhWXTJs2m6JFixmWBwVtZuzYEbi4uDJ+fN4Ns+FVumn2\nFNHz539n1ap1rFq1HBsbW6ZPn0tw8EUWLdK3bLC3L0zhwg4sWfITCoWCCRPGULFiZaZPn4NOp0Ol\nSiEs7CHu7u74+c1BJpMxZ84MatfWp366u5dj06YdhjH06eMDgJWVVa7vVtOm+mbpdnZ2+PsvzTX+\n7EHdtWuJ3Lt3GQCt1ob79zcyadIlGjWqnuc5C4Lw+YhgThAEQcjB1bWMoejC+9x5z8zMRC7P+38z\nWT25IiLCOXTowEcO5vR0Oh1LlizMs31AXqXkAX7//QyLFs1DqTQ29NZ6+PABI0YMRiaTMXbsCNLS\nUpFIpBgbG6PT6ShWzJELF86RlpaGvb09CxYsoVAhO/r2/Z6wsOJIpR589ZWMHTsmvnMpey+v2nz1\nVVsuXjxPvXoNuH07lOnT9Y22L1z4nZ07t+PnNxsAuVxOkyY18r4Y/xCVKlVl9Ojh1KvXEJlMg1Sa\ngFSagkaTn9TUaLRaLb/9dgwzs9yFXczNzbGwsCQk5AqenuXZv3+PIbDW6XQcPXqIihUrExJyBXNz\nC8zMzKlatRrbtm1m6NCRANy5E8quXdvw919qSGf8I3l953U6HdeuhTBtmv5zqVixMomJiahUKUgk\nEmrVqoNCoX/GLjj4ouG7J5FIMDMzZ//+PTl60anVavLly/cXr+jbcXOzRKl8jFqtD2rz5TvPyZPh\nPHmSQs+eDd54E0EQhE9PBHOCIAj/cpGREYwaNZS1a7cAsHHjOtLSUrl8+RJly7oTHHyR5OQXjB49\nAU/P8kyfPpkjRw7h5laWGzdu0rVrD8LDnzJv3iyCgy9QurQrAwYMZcuW9URHRwH6yoXlynmyatVy\nIiKeEhERgZ2dPV279mT69ElkZmai1erw85uNg0MRQ+rjsmU/8vhxGD17dqJp0xb89ttxBg/2pVSp\n0gD4+HyLr+8YnJ1Lvvd1OHHiqKF9QEJCPL17d6N8ef1Mz717d1i/Poh8+fLj4/Mt166FULq0K7Nm\nTWPRouU4OBRhwoQxSCRQvHgJatasjY2NLXXq1Gf48IFYWFiyZs1GlixZyC+/7GLgwKHUqlUHb++W\nzJ8/m169BnL5spzY2BokJzcmNPQuJUq8KsOf1/HbtevI1q0bWbRouaEQRlpaGm5u7obnzzp39iYx\nMQErK2v27PmFFi2+eu/r9HeSVUFzxozJlC6dQHy8FdHRP2BsXJXHj7fj43MCV9cypKamGrbJfgNi\n3LiJzJkznbS0NEMft6x1FAqFodrjmDETAOjRozcLF86le/eOaLVaVCoVcXGxDB8+kKZNWxAScpmI\nCH3z8pEjx+HsXDLXd37QoGHMmuVHZGQEMTHPCA29CcCJE8c4cuQgmZkZJCUlodVq0Wq1nDlzkqNH\nDyGRSEhLS8vzOcu8+vt9TG3a1CAs7AAHDlwiNTWeiAhYvrw3kERwcBCLF7f/ZGMRBOGPiWBOEATh\nPyb7j12tVsvKlYGcPXua1atX0K/fQC5fvkT58hVZvHghdevW5cGDe8yadYmvv26LVquhd28fRo8e\nxrRps/DwKE9UVBS+vgNZvz4IgEePHhnSxhYsmE27dp1o1KgJmZmZ2RpK68fg4zOQTZvWM2vWfAAs\nLCzZt+8XSpUazuPHj8jIyPgggRzwp+0DcpaS1/9gL1zYAQcH/TNKjRo1ZffunQCGmZbTp3+jWbOW\n7Nv3KypVCsbGxmRkpNOkSXNkMhkFCxbi6tXLnD9/h7S0VzM7KlUptNpXP9rfppQ9gFQqNVSLBGjc\nuBkHDuyladOW3Lhx3TCr80/wtq0DslfQPH78ApGRT2natB/W1qNyrZs9VRCgVKnSLF++Os/9Nm7c\nnEGDhhteJye/ICzsCT4+g3K0UWjXrhWLFi1n1arluLiUMaRKTp06gdWr9c8EZv/OZ6VKtmvXEW/v\nlhQr5oiTUwl2797BunVbCQm5zPjxozl16jdiY2NJSUkxpEqOGzeCnTu30b79N2g0GtLSUg2zk+3b\nd8LGxoakpERUqlTs7Oz+9Nq9j2HDGjNsGAwatJ/Q0HYvl1py8GAJww2E1w0Y0JeBA4fh4uLKiBGD\nmThxGjodHDq0n9atvQF9gZoFC+YwderMdx7TtGkTqVmzdo7/BgThv04Ec4IgCP9hdevWB8DFxZWo\nqEiuXr2Mh0d5kpKSMDc3p1q1GoSGhhITE839+3dQKJTMmeNHYmIC8+e/el5MpVKRmpqaK23Mza0c\na9cGEBMTTd26DShSpGiO478+C1G//pcEBq6if//B7Nmzm2bNWn6wc5VIJG9sH5BVSh5AJpO+DDpf\nTzHNua1Op8tzn1nvAUil+mClenVXjI33kZKin40zM7uFTvdqZi738TPzPAeFQpkjGG/WrBWjRg1F\noVDQoMGXnyX9bc2anzh4cB/W1jYULFgIF5cy1KlTj3nzZpGQEI+xsTGjRo2jWDGnHK0DypXz5MWL\nJMPr+Pg4Ro8ez969vxAaepOyZd0NwdmcOTMIDb2JWp1GvXoNsbb+EgBv75Y0bdqC06dPotFkMmXK\nDIoUKUanTt4sWxaAtbU1Wq2WTp3asnz56jwDEIDjx68xcmQMYWHuODufY84ce2rWLGt4P69UybCw\nh2+RKinFzMwcZ+eSHD9+BC+vOkil+psJkZERWFpakpSUyIIFs6levRbDh49m9mw/9uz5GalUiq/v\nWNzc3HP095PL5QwfPuqjB3NZjIw0OV4rFKnI5UZ5rpv9uzl7tj+gzwzYuTPIEMzlz1/gLwVyWfsX\nRVcEIScRzAmCIPzLyWSyHLNA6emvCkZk/SiTSmV5BjD58uUnKSkBMzNzlErjl72vLGnR4ktWrAjE\nyCj3j7rsxTq8vJrg5laOM2dO4us7mJEjxxqKQ+TF2NiYypW/4OTJ4xw7dpiAgA1/9bRz8fCowM8/\n76Bp0xYkJiYSEnKZAQOG8PDhgzzXd3R0IjIygvDwpzg4FOHQoQM59nXw4D7q1m3AsGEDsLKywtTU\njLS0NOzsCnPkyEEaN25GcvILypRxw9m5GF9+ac2FC9tRKlVUrHib48c1eR43u6xS9llplq/Lnz8/\n+fPnJzAwAH//JX/twryHW7ducOLEUQIDN5ORkUGvXl1wcSnDrFl+jBgxhiJFinLjxnXmzp1pKLqR\n1TpAIpHg5zeJ5ORkli9fzalTJxg9ejjLlgVQvHgJevfuxt27dyhVqjR9+/bH0tISjUbDkCH9efDg\nHiVKlEQikWBtbUNAwHp27tzGpk3rGTXqfzRu3JSDB/fRvv03XLx4npIlS+cI5BYtWp7jPObPf0xY\nWEcA7t93ZsGCzTmCuSzZA/fsTbOzvvOZmZk51gsK+hnQf687dOicZ6pkz559OHfuDLt2bcfS0pLp\n0+fmeF+tVlOyZCmWLFmFiYlJru3/iqwiR0OG+L5xneDgiwQGriIiIoLSpbejVr9AoylEhw5NuHXr\nBkuW+KPRaHB1LYuv75hc/xZ4e7dk1ap1LFu2iPDwp/Ts2YkqVarRpk07RowYzLp1W9FoNCxduojz\n588ikUhp1ao1bdu2Z/XqlZw5cxK1Wo27uwcjR44z7DevmyeC8F8mgjlBEIR/OVvbfCQkxJGUlIix\nsQlnzpziiy+q57lu+fIV2LZtM8WKOZGcnMzp06dwdi7Fw4f3sbCwwNLSEp1Oh6urG0FBm+nUSV+U\nIetH9+siIsIpXNgBb++OREdHc//+vRzBnKmpGSpVSo5tWrb8mpEjh1C+fMUc6W5/Vdad/HdpHwCg\nUCgYOXIcI0cOQak0xtOzAhER+p5gvXr1Zfr0yRw7dgSFQkFaWio9enRCItEfZ+/eX9i4cR3JyS/o\n1asvACNHDmT06OGo1WsoXLg6Jiam2caY99hbtWrN8OEDKVCgIP7+S/OclfDyakJiYuJblc//0K5d\nC6F27XoYGRlhZGREzZq1SU9Xc/16COPHv0qDzMjQBznZWwdkyWrLULy4M7a2+ShRwvnl6xJERUVQ\nqlRpjh49yO7du9BoNMTGPufhw4eUKKFPv61btwEApUu7cuLEUQCaNWvJmDG+tG//DXv2/IydnT19\n+nQnMzODsmXdGTZsFE2a1KN1a2/Onj1NfLwUY2NX8uefg1weRWKifsZ6795fiIuLZfTo4UREhDNj\nxlQWLlxKcPBFYmKeYWpqRkREOCEhlwkJCebx40dUqlSF4cMHkpqqQq1Op0WLr6hatVqeqZImJsbI\n5XLq1m1A0aLFmDJlQo7re/nyXYYMucPdu2VwcjrJjBkO1Knj9t6fW/YiR38mKiqCuXNHExOj4bff\ndlGoUBR+fktZuHAZRYoUZerUHwypodllzaL5+Azi4cMHhpTUyMgIw+e/e/dOoqOjWLNmE1KplKSk\nJADatu1Az559AJgyZQKnT598Y/sOQfivE8GcIAjCv5xcLqdHj9706dOdAgUK4ujoBOSVsiShdGlX\nKlaszOHDB+nbty9ly7phbW3D5cuXyJcvPz16dCIzM5Patetw+/ZNunfXP9tTvnxFfH1Hv9zvqz0e\nPXqIAwf2IpfLyZcvP9269TIcG6BkyVLIZDJ69OhEs2Ytad/+G1xcXDE3N6d581Yf5Pyzl09/m/YB\nWZUMAb74ojobNmzLtU/9DMqcHMtUKhXPnkVjb18YpVKZaxsbG9scz2/5+AwEcpayf/34bdt2oG3b\nV/3hsp+LRqMhKiqS4OCLtGz5dR5n/inkTjPV6XSYm1sYfry/7vXWAVkzOlKpFIXi1eyOVCpFq9US\nERHO5s0b+OmndZibm+PnNynH7HLWNq/SY6FQITtsbW25dOkCV6+GULq0C8uWBSCTyZg7dyYHD+4j\nLS2NSpWq0r//YNq27UJy8hyePg3E1PQs+fOPAwYB+l5z//vfJJRKJZ06edOxYxusra1RKPSfsUQi\nISbmGYsWLcfOzp6NG9exZ8/PyOVypFIJP/+8ndq16+aZKqlQKPHzm2RIue3Xb2COazN79m1u3dIH\nSffueTJnzmbq1HEjNTWVCRNGExMTg1aroXv33lhZWRlmy8qX92TAAF+MjIy4desGCxfOJTU1DSMj\nI/z9lxIaetPQM/HmzessXDiP9HQ1SqWSMWN+oFgxR8MYChYsZLj5U7iwMWvW/EThwg6GlOmmTVuw\nY8fWXMFc9u/Dm1y6dJ6vv/Y2pAdbWloCEBx8gY0b16FWp5GUlESJEs4imBOENxDBnCAIwn+At3dH\nvL07vvF9a2trQ0rY6NHjGT16PAUKWBAT84L09HR8fAbmmVL5uqxZqCxduvSgS5ceudbLCkrkcnmO\nnlcZGRnEx8eh1WqpWrXa25za38KxY1cZOzaaR49KUrr0YRYudMHD48MUbslLRMQzevf+jefPNyOT\nSXF2bvDRjvVHPDw8mTXLj65de5KZmcmZMydp1aoNhQsX5tixw9Sv/yU6nY779+9RsmSpd96/vtea\nCmNjE8zMzIiLi+X338/kCL7fpGXLr5k8eTwlSjjnKO2fnp6OjY0NRkZGhiCladNaXL8ejonJTsqU\nUbBx46vZ4saNmxmK4HTo0AkLC0vat/8GL686L7dtQXR0FHZ29gDcvHktRw/AjIwMnj59gpubO506\ndcvVhiMgYP0bz+HFC2Wer8+dO0P+/AUNz6UlJyfTrVsHw2zZnDlT2blzG61be/PDD2OZPHkGrq5l\nUKlUuW40ODkVZ/HilchkMi5cOMeKFYtz9E/MfsMnK1BPSkrMsex9vL69Wq1m3rxZrFq1jgIFChIQ\nsIL09PT3OoYg/JuJYE74P3vnHRbF1cXhd5elSK8qikRABBVFECtW7L0XiNIixoLGFmvsBaMkdgVR\nEMUSNcZesNdYsUQFKwQQLKj0ust+f6xsqJaoMeab93l8dGbuvXPv7IL3zDnndwQEBARKRS6XM336\nHn77TQ+xWIaraw6TJnX8JPfKz89n9OidnD37J+XKHaBVq8/lafp7+PvH8vChwjNx504dFi7cQljY\npzPmFiy4wJUrnoAXAIsXb6VXL/k7iUMUFIIfNWrYB8/D1rYmTZs2x8NjAIaGRlhZVUNHR5vp0+fi\n77+A0NBgpFIpbdq0UxpzxedY+Li0a9WqWVO9ug1ubr0pX74ideqUVPl83bpY+GZz5s+fRc2adtja\n1iySryaTydiy5S8jSiwW07ChDa6u7QHYuLGoKmYBcrkcsbjkM9bQKJrLNnbsBOrXL/oyIiLiirKm\nYlLSS/z8zpGWpk6LFuX4+uvSvU5Nm+Zx5cozZLLyQDJNmiiMTCsra1auXMrq1ctp0qQZmpqaRbxl\nPXr0ICQkFCen+hgZGStDKjU1NUvcIy0tjTlzZvD4cRwikUiZ91fA06dPuHXrD+zsanPkyCFsbWuw\ne/dOZS7p4cMH3mhca2pqkpmZWeo1J6eG7N69E0dHJ1RUVEhNTVV+hrq6emRmZnLixFFcXNqWOb6A\nwP87gjEnICAgIFAq27efZu1aF6RShcdh1aqHNG58lRYt3u4VeV/Wrj3G1q29AV2MjIzYu/cpQ4c+\nJjQ06IuQIk9LKxo6mJFRMszy49/vL6MiJUUbqVT6Tt7Tj60G6Oo6CG/vIWRnZ+PrOwQbmxqYmlbi\np5+WlWhbvHRA4WNT00q0atUGN7feSmXMhIQE7t+/S0xMNGpq6mhpaTFx4jRevnyBj4/FVRY3AAAg\nAElEQVQH27fvARR5WH5+swgN3cqdO7fw85tHRkYaYrGY+vUbsmDBXK5du0LNmnZcuxZB48bO5ORk\ns3r1ciIirvD4cTytW7cjKiqSgIDl5ObmMGbMCO7du4tUKsXUNIjTp48TFxfHokVLAJDL8xk3bhQJ\nCfGkpCQTGxuDuXlVkpKes2iRH4aGhrx8+ZJ+/Vzp0qVHkZqKjx+bcf36KkDEoUMPUFU9T79+JQu8\nT5jQESOjE9y5k4elpZjhwxXqrlWqmBMcvInffz9LUNAq6tWr/7c/v7VrA3Byqo+fnz9PniQycuS3\nRa6bm3/Fb79tY8GC2VStakn//l9Tq1Ztpk2biEwmo0aNWvTo0afM8fX09Kld2x539/40auRMr159\nld/Brl17EBcXi4eHKxKJhG7detKrV1+6du2Bu3t/DA2NqFnTrsh4gpqlgEBRBGNOQEBAQKBUoqNT\nlIYcQHa2BQ8eRNCixce/V2KiDFDky8jlIrKyTIiOTvxipMgbN04jKioV0EUieULTpm9XqvwQWrZU\n59ixR2RnWwK5NGjw5I2GXGjoOg4d2o+BgaGyfMD27dvZtGkzeXlSzMzMmDZtNjKZDA8PN7Zs+RWJ\nREJGRjqenl+zZcuv/PbbDnbv3omKigpVq1owa9Z8ABYunEdMzCNyc3Pp2LEL1tY2f2tNZSljzp07\nk7FjJ2Bv78C6dYGEhKxh1KhxSKV5JCYmYGqqUA9t3bod2dlZjBw5mceP26CnF06lSnU4cGCvsvB4\nXFwspqaVcXZuTljYemWdxVmzfuDy5QuMGzeRdu06cvXqZebP92f//j0sX/4zR48eIj9fjrW1NQ8e\n3MfR0Ync3FzGjPmeZ8+esnZtgFKx08zMnLS0dLKyshCLxaxevZyOHbsoayqOGzeJhg1fUGCMZ2dX\n4+zZ6/QrpQ63SCRi8OCSIbRJSUno6OjQrl1HtLS02blzO0+eJCq9Zbt378bBoR7m5lV58SKJqKg7\n2NrWJDMzo4jaLEBGRoayxuH+/XtK3EtFRYVp04rWL6xXr36pSrOFVUILDG2AGTPmFmkXGrpVOfbI\nkWMYOXJMkes+PsPw8SnpOS7+MkBAQEAw5gQEBAQEyqBzZzuWLj1BYqJC2c/c/CDt2tX92+MdPLiP\nrVs3KUPnBg8eyvz5s0hJSUEul2BoqM/Ll4pwqgoVHuHg0JczZ/Yrc2qioiJZsWIxWVlZ6OnpM3Xq\nDIyMjImMvM2CBXMQi8U4OTXk4sXzbNjwCzKZjICAFVy/fpXc3Dx69epL9+69PvzBlIKfX3fMzI4Q\nEyOndm0NPDzaf5L7FODh0QJ19XNcuBBB+fJSxo0rOyw1KiqS48ePsH79FmQyKd7eA7G1rUHbtm1p\n2VKRvxUUtJp9+3bTu3d/HBwc+f33szRr1pKjR8Np2dIFiUTCpk2h7NixV2nkFVB8o/53KU0ZMzs7\ni/T0NOztHQDo0KEz06YphHZcXNpy7Fg4Awd6cvz4UebMWcCKFdvJzExDW/sKMpkh8fFxJCdfYcqU\nGezatYOqVS0wMDAkIGA5xsYm/P77Wc6ePcWIEd8RGXmbiIgrbNu2hS5depCXl8uBA3vIz8+nXLly\nTJkyk6ioO+zevZO7dyORy+UMGtQPfX199PUNSE1VhCdKpVIGDPhaKUrTrl0LtLS0ld9jXV1djIxu\n8VfkoRQDg7z3elaPHj1g5cqliMUiJBJVxo+fTHp6mtJb5uBQlx49+iCRSJg924/FixeRk5ODhoYG\nixevfP2SRDGWm5s78+bNIDR0HY0bN6VoeZLP8zIlJSWFn38+Q1aWKl26VKZ5c7u3dxIQ+D9FMOYE\nBAQEBErFzs6SlSufEha2HbFYzuDBVlSp8vcKFT969JANG4IJDAxBV1eP1NRU5s6dQadOXenQoTP7\n9+9BLA5CVTWVly//oEWLr9DW1gFQ5vEsWbKIH3/8GT09fY4dC2fNmlVMnjyd+fNnMWnSdGrVsiMg\nYIVy87lv3260tbUJCtpAbm4uw4cPpkGDRpiaVvpoz6gAsVjMyJHtPvq4b2LAAGcGlK1po+TmzWs0\nb97qtfCFOs7OzZHL4d69eyxa9BMZGelkZmYpxUC6du3B5s0baNasJQcP7mPixB8ARZ7WzJlTad68\nJc2atfwEKyq9AHtZuLi0Zdq0SbRo4YJIJKJyZTOysmTk5lYjLu4XACSSaJycPJR9ChdnB5g2bQ4v\nX75g06ZQpRImKBQy160LpGLFSsTHx+PpOZi5c6fTr58bcnk+8fFxGBgYsnHjL7i59Wbt2o2IxWLG\njfuVkydT+O23RCIi9jFjRpcSaypXrhxTp+qwcOF2UlL0qFfvTyZOfD/l1gYNGpUqEFTgLSsQLwJF\nXmNhFVUoquBqZ1ebLVt2Kq8VeMSKq6z+U+Tl5TFo0GEuXPACxOzbd46goDul1v0TEBAA8eeegICA\ngIDAv5emTe0ICOjAqlUdcXQsWUfuXYmIuIyLS1tl8WtdXV3u3PlDqezXvn0nkpL+JCioHZ07V8fE\nxEDZVy6XExsbQ3T0Q0aPHo6XlxsbNgTz/Plz0tMV4Wy1aine3Ldt20G5eb58+QKHDu3Hy8uNb7/1\nJDU1hfj4uL+9hi+X0j0rkydPZty4SYSGbsXb20cp91+7tj2JiYqSBzKZDAsLSwAWLVpCr159uXs3\nCh8f9yLGz8egTh17zp07Q25uLpmZmZw/fwYNjXLo6Ohy48Z1AA4d2q80QipXNkNFRcz69Wtp3Vph\nSHt5daRcuVg0NK4B+dSqtRMNjb/CTwsbVvr6iiLiNja2PH36tMR8/vjjBiNHjkFPTw97ewdSUlLI\nzs4GRDRt2pzKlSsTEXEFAwNDXr58werVW9m0qSu5uZVIS7MnKKgRJ09eUY5XuKZir14NOH++HVeu\n1CEsbECpwiT/JHK5nLCwU8yZc4jDh69+1rncvfuACxdaULBFTUpy5uDB2M86JwGBfzOCZ05AQEBA\n4JMjEpXudSnLE1NaZJeFhRUBAcFFzqWlpb1xvNJUBf/fqFvXgXnzZjFwoCcymZRz587QvXsvMjIy\nMDQ0QiqVcvjwAcqXr6Ds06FDJ2bPnoan52BA8VyfPn2Co6MTderU5dixcLKzs9DS+vCi7gWUpYw5\ndepM/P39yM7OpnJlsyJ5Uy4u7Vi9ehk+PsMBMDOriL//HObPn4hUmoOOjoScnGxl+8JKjQUeXLFY\nhfx8GWKxSolriu9TUbVNkQgkElWlYmdCwmNGjhyKnp41+fkFpTlE5Oaa8+efV99YU1FLS+ujPb8P\nYdasfQQGtkEmK4+WVhSzZp3G3b35Z5mLsbE+OjqJpKUVqMFK0db+tDmoAgJfMoIxJyAgICDwyXF0\nrM+UKeMZMODr12GWKdjZ1eHYsXDat+9EePhBZV6UXC6nsE0mEokwN69KcvIrpUS6VColLi4WCwtL\nNDU1uXPnFjVr2nHsWLiyX4MGjdm5cwcODk5IJBJiY/+kfPkKJYpWv4n09HSOHDlEz55lq/X9E6xb\nF6isbwYQGLgSQ0Mj8vJyOXHiKLm5eTRv3pJvvlEoEU6ePJ5nz56Sm5tD376utG7dFk9PVx4/jqdK\nFXO2bt1Ez549cXXtSW5uHuXKafDixQvl/dq27UBQ0GratlXk/slkMubMmU5GRjpyuZy+fQd8VEOu\ngNKUMa2tq5cIE/yr/UBcXQcWOdeoUUP27PkVUBhvPXp0IDU1hZ9+Ws7Ikd8qw0m//34qNja2JCcn\nIxaL2b59NxERVyhfvgKjR3/PkiX+hIcfVJ7X1zegR4/evHiRBKBU7HR378/ChUtJTc3l8uVw4uP9\nALCy2kWHDvXw8Ci9puK/ifBwzdflDyAjw5YDB+7g7v555lKxoikjR95i9epjZGQY4+x8ke+++7JK\nlQgI/JMIxpyAgICAwCfHwsISd3dvfH2HIBarUL26DaNHT8DPbxabN2/EwMBA6XEpLM5QgEQiYc6c\nH1m61J/09HRkMin9+7thYWHJpEnT+PHHeYjFIurWrac0Mrp27UFiYgLffDMQuVyOgYEh8+cveuc5\nS6VS0tJS+e237Z/dmOvcuRtTpnxPv36u5Ofnc/z4EYYMGcHVq5cICtpAfn4+kyaN48aNa9jbOzB5\n8nR0dXXJycnGx8eDFSuCcHf3plmz+gwePJRWrdogkUg5evQYmzcrDJ/CoiY3b16nVas2ymcpkUhY\ntWrtJ1/nx1LGLEAikeDpORgfHw9MTMrz1VdVAUpRSS3sfVP87e09BD+/2Xh4uFKuXDl++GFmob4l\n71W9ujlr1mSxceM2VFTk+PjUQE1Nwvff7yUlRYOmTdU/m7frbWhoFBVgUVOTltHyn2H06La4u78g\nPT0dM7P+ygLsAgICJRHJ3yfb+BNSkKgr8N+jcCK2wH8P4fP97/KlfLZZWVmUK6co2jxt2kQiIq5i\nYlK+hGKmvr4BU6ZMp0KFisybN7NI/bq2bZtx5MgZIiKusHZtALq6uvz5ZwzVq9ty9uwpzM2/on79\nRgwfPuqzrXPMmBEMHz6KFy9esG/fbkxNK3Hy5DG0tbVfP4dsBg3ypHPnbqxbF8iZMwqP0JMnCfz8\n8wpq1rSjRYuGnDx5AZFIhIFBObp374mNjS1NmjTD2bkZEomExYsXcvHiBfz9l2JmVoWwsDOcOpWF\nnl42kyc3xcjI8LM9gy8JuVxO797bOHv2G0CEmlo08+ff+UcMuvf92d227XdmzlQlKckOC4vzLF9u\nSoMGNT7hDAU+hC/ld7PA+2NiovPefQTPnICAgIDAF83582cJCwshKyuLpKTnTJmykGXLnrN/v5gz\nZ0YzbFg/evTozf79e1iyxB8/P/9S5Nb/Or5//y4bN26jYkVTnjxJJDr6ISEhm//ZRZVCly492L9/\nL69evaBz525cvXqZgQM9S5RbiIi4wtWrlwkMDEFdXZ2RI78lNzcXADU1deXaJRIJQUGhXLlyiZMn\nj7Fz5zaWLl3NmDETlGNt3XqOKVNsyM62AuQ8eBDMb7/1/SJq/30KQkNPcfZsDrq6CsPW2LhswzY5\n+RV//GFNwXcrN9eCCxciPlv44pvo168xTZsmEhl5A0fHuhgYCAa7gMCXguC3FhAQEBD4omndui0h\nIZvp06c//ft/zeLFzzh/fiAPH7qRkvKSkycVm+n27Tvxxx/X3zpejRq1qFhRUSz9nw5e8fUdQlRU\nZInzBw7s5fr1q1y8eJ6oqEgaNWpCw4aN2L9/D1lZWQA8f/6MV69ekZmZgY6ODurq6vz5Zwy3b98q\n9V6ZmZmkp6fRuLEzI0eO5cGDeyXanD2b8dqQAxBx40ZNkpKSPtp6vyQ2bTrDDz/UYvfu3mzc6Mbg\nwcfe+P3Q1tbB0PD566N8QIaBQc4/Mte/Q6VKprRu3VAw5AQEvjAEz5yAgICAwH+CAsXMhISiYSqJ\nieVKtFVRUSE/X7ERz8/PRyr9K2dIQ6Nk+38CmUxWSi7XX4jFYurVq4+Oji4ikYj69RsRExPD0KFe\nAGhqajJt2hwaNmzCrl2/MnBgX6pU+Qo7u9rKMQqPnZGRwYQJY1577eSMHDm2xD2NjHIBKQXbBWPj\nRHR1rT/amr8kfv89i5wci9dHIv74w5YXL15gbGxcQnCmW7eedOrkQpMmzkgky0lNdUVffwUmJr0Z\nNGgjRkbGDB48jICA5Tx79pRRo8bRtGlzfH2H8N1347G2VpQBGTbsG8aPn4yVVbWyJyYgIPB/jWDM\nCQgICAj8JyhQzLSy0iUuTo5YnEJ2tj36+hFAxyKKmRUrmnL3biQuLm04e/Z0Ecn6wmhqapKZmfnW\ne2/evAE1NTX69BnAsmU/8fDhA5YuXc3Vq5fZv38PjRs7Exa2HrlcTuPGTRk2bCSgyNXr3r03V65c\nYuzYCUXG3L9/D2Fh69HW1qFateqoqkq4ffsP5s5dqGzTt+8A+vYtWTnc339ZqfMMDz+l/LeJiQlB\nQaFvXNfEiS48eBBCRMRX6OklM3Gi0evi4/9/GBrmUNiwNTJKRFdXIdBSXHCmZUsXsrOz6datDT/9\nNJ+srCw6dFhC48bOjB49nilTvmfdugCWLl1NdPQj5s2bQdOmzencuRsHD+7F2nocsbF/kpeXJxhy\nAgICb0Qw5gQEBAQE/hMUKGZu3LieOnW2AGY4OjYjO/ssHh6uRRQzu3XryaRJ4/D0dKNhw8aUK/dX\n0ebCjjE9PX1q17bH3b0/jRo5lymAYm/vyNatYfTpM4CoqEikUilSqZQbN65RpYo5AQErCA4OQ1tb\nh7FjfTlz5iTNmrUkOzubWrXs8PUdXWS8pKQkgoPXEBwchpaWNj4+HiQkxNOtWy8qVzb7oOckl8vZ\nvv0UmZlyWra0oWrVSmW21dTUZNOmAWRmZqKhofHZVAWLC9Z8DiZNas2jRyFcu2aOnl4ykyaZoKam\nBsD27VuUgjPPnj0jLi4OsVhMy5atEYlEaGlpoaqqqiyLYGVVDTU1NVRUVLC0tCIxMRGAVq3aEBq6\njuHDv2P//j106tT18yxWQEDgi0Ew5gQEBAQE/jN07NiFjh27FDs7sEQ7AwPDIrXLCjxljo5OODo6\nFWk7Y8bct97XxsaWu3cjyczMQE1NDVvbGkRFRXLz5nWcnZvj6OiEnp4+oKjhdv36NZo1a6nc8BdG\nLpdz584tHBzqKft07tyVuLhYRoz47q1zeRNyuZzRo39l69buyOWGWFjsJTg4m1q1LN/YT1NT843X\nPyVvCz/9p9DU1CQsbABZWVloaGgo51O64ExOEbEZABWVv7ZcIpEIiUQVUITPymSKotgaGho4OTXk\nzJmTnDhxlODgTf/gCgUEBL5EBAEUAQEBAQGBQty8+YAffjjIrFn7ePny1Tv1kUgkmJpW5sCBvdSu\nbU+dOnWJiLjM48fxmJqaFhPKkCs3+cU3/AUUP/WxdFiePXvGnj22yOUKkYvo6K5s2FBS+ORTcPjw\nAXx8PPDycmPRovnk5+fj7+/H4MHuDBrUj3XrApVt+/TpyurVy/H2HsjJk8cAhSEaEXGFyZPHK9td\nvnyBKVO+/0fmX0C5cuWKfGaFBWdiYqLLFJx5V7p27cGSJf7UqFFLWXZCQEBAoCwEY05AQEBAQOA1\nkZExeHk9Zc2afqxcOQA3t2PvlDMHYG9fly1bwqhb1xF7ewd27fqV6tVtqFGjFtevR5CSkoxMJuPo\n0XDq1nUscxyRSETNmnZcvx5BamoKUqmUEyeOfpT1icVixOL8Yvf79IqdMTHRHD9+hICAYEJCNiMS\niQkPP8iQISNYu3YD69dv4fr1CB49evB6TiL09PQJDg6jdet2ynOOjk7ExsaQkpIMwP79e+nSpfsn\nn/+baNiwCTKZjIED+xIYuFIpOFPcSC95XPo1GxtbtLW16dy526ebtICAwH8GIcxSQEBA4F/I/fv3\nSEp6TuPGzp97Kv9X7N4dRVxc39dHIiIiunP27BXatWv81r729g5s3BiCnV1t1NU1UFdXx97eASMj\nY4YO9WXUqKHI5XKaNGlG06aKwtFlhQ4aGRnj7T2Eb7/1Qltbh+rVbT4ozDA/Px+xWIyJiQl9+pxh\n40YLpNKKWFvvZPBgu7897rty9eol7t6NYvDgQQDk5uZiZGTE8ePh7NmzC5lMxosXSURHR2NpqRD8\naN26baljtW/ficOHD9CxY1du377F9OlzPvn834SqqmqpgjOFxWaKH3t7Dylx7eHDP4mMjMPaujz5\n+fk0aNDo00xYQEDgP4VgzAkICAj8C7l//y5370a+lzEnlUqRSIRf6x+Cjg5ADqBQbFRTe4qJid47\n9a1Xrz4nTvyuPN6yZafy323atKdNm/Yl+hTf8C9fHsi6dYHcuHGNfv1c6dSpK4GBKzE0NCIvLxcf\nH3dyc/No3rwl33zzLUCpsvhQVClz3LiJ1K5tD8CCBT1p2fIC6ekRNG/uRIUKRu/6eD6Ijh278O23\nI5THCQmPGTvWl7VrN6Ktrc38+bPIzf2rDlu5ckVLRBSEqnbq1I2JE8egpqaGi0ubzybK8jHZsOEM\nc+fqI5MlU7HiTLy8/oWVxQUEBP6VCP/rCwgICJTCwYP72Lp1EyKRiGrVrBk8eCjz588iJSUFfX0D\npkyZToUKFZk3bybq6hrcv3+XV69eMmnSNA4c2EtU1B1q1rRTqie2bduMbt16cunSBQwNjZk1az76\n+vr4+g7B13cMtrY1SE5OxsfHnS1bdrJ2bQC5ubncvHmdQYO8adzYmcWLFxId/QiZTIq39xCaNm3B\ngQN7OXXqONnZ2eTn57N8eeBbVibwJnx8XDh/PoTjx5ujppaGh8cDHBz+2XC3jh27MGzYcB4+NKB1\na3OOHz/CkCEjuHr1EkFBG8jPz2fSpHHcuHENe3uHUmTxW6Orq1umUqZIJKJjx8aYmOjw/HnaP7Km\nevUaMGnSOPr1c8PAwIDU1BSePn2ChkY5tLS0ePnyBRcunMfBod5bxzI2NsbY2JjQ0GCWLl31D8z+\n03L27GnWrDlEcvJyjIyukpT0LWfPaiISBVK3riP16tVn27bNdO/eC3V1jc89XQEBgX8ZgjEnICAg\nUIxHjx6yYUMwgYEh6OrqkZqayty5M+jUqSsdOnRm//49LFnij5+fPwDp6WkEBoZw9uwpJk0aR0BA\nMBYWlgwe7M6DB/epVs2a7OxsbG1rMnLkWNavX0tIyBrGjJlQqkqfRCLBx2cYd+9GMnq0QtwhMHAl\nTk4NmDJlBmlpaQwZ4oGTU0NAEZIZGroVHZ2ixbIF3h81NTU2bnTlwYOHlCunQ5Uq/3ze0oIFvxMb\na0ZEhB1bt56mUaMKREXd4fLli3h5uQGQlZVNfHwc9vYObNoUSnj4QfT09Hn27Cnx8bHUrGlXqlLm\n56JqVQt8fIYxduwI8vPlqKqqMmbMBKpXt8HNrTfly1ekTh37N45R+OekbdsOpKSkYG5e9RPP/NPT\ntGlz8vJyAZDLFWvMy5MoPa8A27dvpX37Tu9lzBWE1goICPy3EYw5AQEBgWJERFzGxaUturqK8Dpd\nXV3u3PlDaby1b9+J1asVOTIikQhn52YAWFhYYWhohKWl1etjS548SaBaNWvEYrFSyKFdu45Mnfpm\nBT65XF5EAfHSpQucO3eaLVs2ApCXl8fTp08QiUQ4OTUQDLmPiFgspnp1689y7/T0dMLDzcjNHYCu\n7q/I5S/IzKyFXJ7PwIGedO/eq0j7All8LS0t1q/f/FoWX2EYlKWUWUB+fn6Z1z4FrVu3LZEHV6tW\n6fl627fvKXJc4OEu4ObN63Tt2uPjTvADWLcuEE1NLVxdi5bBuHHjGqNHD6dt2w5cvXoJNTUNxoz5\nnpCQNbx6lcyMGXOIjn6EhcUBYmObAKCunkD37pWUtfWSkp6TlPScUaOGoq9vwNKlq/H39yMqKpKc\nnGxatmytNPz69OlK69btuH79Cs7OLTh58jjBwWEAxMXFMmPGFOWxgIDAfwPBmBMQEBAohkgkKiYl\nr6C0c6AQQACFEaCmpqo8X7h+VPFxCjbZKioqyOWKTXXhfKHSmDdvEVWqmBc5d+fOrRK5RQJfFoVD\nei0sLFFXb4S6+jm0tBT5dKqqX9OwYQPmz5/NjRvXeP78GQkJj+nevReWllYkJT0jLS2Nr7/uQ3x8\nHPfv32Xz5o3K8X/++Udq1KhFx45dlJv9y5cv0rlzR/bvP/jFbPZlMhlz5x7k5Mk1qKlJGDjQ+4PG\nK/h5/hj16940Rl5eHgMGDGTy5OkMHuzOsWPhrF4dzNmzp9iwIYTmzVvSoIEFPXpc4fDh21hZGdOv\nnzPz5x9FJBLRp88AfvllM8uXBypfMA0ZMgJdXV1kMhmjRw/n0aMHWFpWU6qA7ty5k+fP07hy5RL3\n79/D2ro6Bw7sFRQyBQT+gwj+dwEBAYFiODrW58SJo6SmpgCQmpqCnV0djh0LByA8/CD29g7vNWZ+\nfr5SXv7IkUPUqaPob2paiaioOwDKeloAWlpaRSTxGzRoxI4dW5XH9+5FAWUbmAJfBgUhvcuXB7B+\n/WbGjJlAzZq/kZ3tTGpqF8TiuqSnH6F+/UZYWlpx+vQJUlKSMTQ0YsuWjTg5NaRKla/Iz8/H3Lwq\n9vYOypp0BQZGYUOjsOT/0KFD0dbW5v59RZ2599nsr1sXyJYtH8/oe5fx5s8/yMqV3bh9+wTXroUz\nevTp975PYmICrq69mDt3Bv36dWfhwnn4+Ljj4eGKp6crJ08eIzExATe33owf/x29enXmhx8mkpOT\nDSg8XwW/F6Ki7jBy5F+hkA8e3GPoUG8GDOjF3r27lOclElUsLa24du0qL1++wMmpAZmZmRw8uJ+L\nF88TFLSahIR4BgxoTrNm1bCyqvTWdRw/Ho6390C8vQcSHf2I6Oho5bXC3s8uXXpw4MBe8vPzOX78\nCG3bdnjvZyYgIPDvRjDmBAQEBIphYWGJu7s3vr5D8PR0Y8WKJYwePYEDB/bi4eFKePhBvvvur8LF\nxTfLpaGhUY47d27j7t6fa9ci8PIaDICr60B+++1XvL2/JiUlBVD0d3BwIibmEV5ebhw/fhRPz8FI\npVI8PAYUKbBcWs7d52LdukCuXLn0uafxRVFaSG9aWiwODsEYG++lSpWnSKV5ZGVlUatWbTw8vmHD\nhl8ICgrFyMiY9PQ0pkyZQZUq5vj5+bNsWQDVqilCRIsrZRbwMTb7H/s79y7j3bmjBmgV9ODePf2/\nda/Hj+Pp1asvtWvb8/DhA4KCNhASsonU1FRiYhRGUVxcLNWr29CsWQu0tLTYuXPHG+cpl8t5+PAB\ny5YFEBgYTEhIEC9eJJXoIxKJUFVVZf36tWhr62BmVgUfn2GYmJQv1ObN809IeMzWrZtYtiyA0NAt\nNGnStEwV0JYtXbhw4Rznz5/B1rYGurq67/ewBAQE/vUIYZYCAgICpdCxYxc6duxS5NzSpatLtCuc\ny2NqWonQ0K2lXgMYOXJMif7m5lUJDd2iPPbxGQYoNvVBQRuKtP3++ynvNM9PidoUXuUAACAASURB\nVEwmQ0VFpdRrhQUbBN6N0kJ6ZTIp+fky+vYdwIgR3xW5JpEUDeOVSouG8W7deo6tW2+TkfGU06dv\n0by5HTk5RcN3i2/2Q0LWUK+e01s3+6Gh6zh0aD8GBoaUL18BG5saPH4cz88/LyQ5+RUaGhpMnDgV\nQ0NjPD1d2bFjLwBZWVl8/XUftm/fw5MniSXaFxcxuX//LosW+ZGTk0PlymZMnjwdHR0dXr0KwMQk\nknLlriAS5aGpmY2n5zqSkp7z1VdVSUtLIzY2BiMjYzQ1tRCJwMSkAjExjwgJ2URCQgLz5s1ALpez\natUyIiNvk5eXR7Nm9dHW1iYjI4ONG9dz+PABxGIxu3f/ikgkRktLi0ePHpTIhyv+OTZr1gI1NTXU\n1NRwdHTizp1bSiO9OFevXsbXdzSRkbcAhfAOFOTKlmyvqalJRkYGurp6ZGRkvLMKqJqaGg0bNsbf\nfwGTJ08vc/4CAgJfLoJnTkBAQOAf4GN6MtauPUn79ofp0OEQmzad/VtjZGVl8f333+Hp6Ya7e3+O\nHTtCVFQkvr5D+OabQYwdO5Lnz58D4Os7hGXLfmLwYHc2bAimT5+uSgMkKyuLXr06I5VKmTdvpjJU\nNDLyNsOGeePp6YaPjwdZWVnIZDJWrlyqDGvbvXtnmfP7J0hMTMDdvf87tz94cB9JSUnK423bNivD\n76BoCN674uBQr0RIb6NGzvTo0UdpyBWEQRZQPCRRU1OTzMxMTp/+gx9+MOXSJTdSUzMYPfoF9+8/\n5OrVK2Xev/Bmv1OnskMso6IiOX78COvXb8Hff6kyNHjhwvmMGfM969ZtZPjw7/jppx/R1tbG2ro6\nERGK+54/f4aGDZugoqLCwoXzSrQvoOBHZO7cGYwY8R2hoVuwsqpGSMgaACwtDTE3v4lY7ImeXmXU\n1V+xfv1mevbsg0wmIy0tlXnzFpGc/ApDQyO6dOmBuro6ubk5SKVSlixZxMCBnqirq9OzZx/KldOk\nefNW1K3rSJcuPWjfvhO2tjXQ1zfAwMAQsViFNm3aMWHCVExMKgCKHNf8fMV3Pycn942frUgkLrKu\nv84rThQ24guHxJb2q6Jbt56MGzeS774bhrV1daUK6KxZ096qAtqmTQfEYrFQhFxA4D+K4JkTEBAQ\n+AcoK+TtTWRlZTF9+iSeP39Ofr4MD4/BLF7sT1xcc9TUbpOfr8HcuV7UqnWX9PSnbNgQjFSah66u\nHjNmzMXAwJDMzEyWLFnE3buRgAhvbx9atHAhLGw9UVGRmJiUp0oVc+ztHZg2bQILFvyMnp4+x46F\ns3jxYsaMmYxIJEIqlbJ2rcJTeO9eFNeuXcXR0Um5UZdIJMqQz7y8PGbMmMLs2Qt49OgBt2/fRE1N\njX37dqOtrU1Q0AZyc3MZPnwwDRo0wtT07TlCoDCWgoPDyvR2fGoOHNiLhYUVxsbGAISErCUnJ4dB\ng7xYtuwnXr58ASi8Lvv370FTU4uoqDtlKg5evnyRr7/2UIb0isUqVK9uw+jR4/n55x/x8HBFJpNR\nt64j48dPAhSGQfEXA3p6+tSubY+f3yTU1LqSmvo9aWkd0dZeyowZGtjY2LxxXW3adOD06ZNv3Ozf\nvHmN5s1boa6uDqjj7Nyc3Nwcbt26wbRpE5Xt8vKkALi4tOX48SM4Ojpx9Gg4vXv3IzMzkz/+uFlq\n+wIyMtJJT09X5qR26NCZadMUa1dRUWHePB8cHZ2Ii7Nm4MC+LF36ExkZGdjY1EAikdC4sTNyuRx3\ndy927tyGlVU1rl+/yuPH8URHP2Tt2gBycnLYsCEYFRUVIiNvU6FCRVq0aMW2bZtp0qQZv/22nRcv\nkl6LE8k5cuQQ9vZ1AahY0ZSoqDs0atSEU6f+ynGVy+WcPXuKQYO8yMrK5Nq1qwwbNpLc3FzMzKoo\n21WrVp0WLVyIjLzzWgDlFwCaNm0BgLf3EGXbEyeOKr37vXv3p3fvv148FPf6F1BcBRQUdexyc3P/\nNeHYpSGUURAQ+PsIxpyAgIDAv5SLF89jbFyeRYuWAoqNbl6ejOxsCxITF6Cjswsdnd1cvtyaAQOa\nsmbNegD27t3Fpk0b8PUdzfr1a9HR0VGGf6alpZGcnMylS7+jrq5O/foNSU9PY8OGYB49esjo0cMB\nxebK1LSici4FZRWg9I16AXK5nNjYPzEyMsbWtgbR0Q+RSFRRUVHh8uULPHz4QOm9y8jIID4+7p2N\nuU+xGZXJZMyePY1796KoWtWSadNmsXnzRs6fP0NOTg52dnWYMGEqJ04cJSoqktmzf0BdXZ1OnbqR\nlZVJWFgoV65cIi8vD7lcjlQq5caNa6iqqnLnzi3k8nxq17bn2rWrJRQHC6tGFg+VnTXLr8hxaOg6\njhw5VCTEsW9fV6ZNm0BenhQzMzO8vL5n3DgbqlZtTUzMYaTSZkycCHPnTlGOV9pm/+bN63Tu3O0t\nz7fkNblcjra2DiEhm0tcc3Zuzpo1q0hNTeXevSjq1atPZmYGOjqlt39fqlQxx8jIGEtLS0JDgzE3\n/wpQhJ6qqEiUirEFf2SyfCwsrOjXz43582cRGrqV4OA13Lt3lytXLjF37gwyMtKxsamBTCbD3Pwr\nYmNjCQ8/RL169enRow8AXl5DWLBgNmvXauPgUK+IR83KyppRo4aSnJyMl9dgjIyMSUxMKJYzp/jb\nw+Mbfv75R9zd+yMWq+DtPYTmzVu+9Zm/D1FR0cyaNZ3s7GT09P5efuHHYvLk8Tx79pTc3Bz69nWl\nW7eetG3bjO7de3PlyiXGjp1AYmICO3b8glSaR82adowbNwmxWIy//4JSX4oICAgoEIw5AQEBgX8p\nVlbWrFy5lNWrl9OkSTPs7euipiYBLAFIS+tMhQpzaNjwW549e8r06ZN4+fIFeXl5VKpUGVB4iWbP\n/ssw0NHR4dy5MyQmJmBoaMTRo4dJTk7mq6+qYmFhRUBAsLKtiYkOz5+nAQoBlwKcnZsTGLiSMWNG\ncO3aVeLjY/H09CE5+RWBgasASEp6rlTjTEp6zrhxo7h58xqNGjkzZ84CQKHquWLFYuRyOY0bN2XY\nsJHK82Fh60uc/xTExv7J5MnTsbOrg5/fbHbu3EHv3v3x8vIBYM6c6Zw7d4ZWrdqwc+d2fH3HYGNj\nC8Avv2wCwM/PnylTvkdNTY379+9x8eLvqKmp0aVLN/bt28PJk8eRSvOIjo7G0rIaQIl6a2+icIij\nTCbF23sgtrY1aNGilbLWWlDQalRUnjJ8eAb791egRo35fPONEzExz2nZ0qVEnqNMJmP58gPs2LEG\niSSTNWuCS7u1krp1HZg3bxYDB3oik0k5d+4M3bv3olKlSpw4cZRWrdogl8t58OA+1tbV0dTUxNa2\nJkuXLsLZuRkikQgtLe0S7R8+fKAUbJHLQUtLGx0dXW7cuI69fV0OHdqvzAeTy+XKlwinT59EW1uH\nrl17cvXqFe7diyI3N5fHj+MBOHz4AHXrOpKamoqOji6XLv3O8+fPOHXqGDVq1GTECB8qVjTF2ro6\nGRnp+PqO4ddff0Ff3wBQeAFVVSU0bdqcqVNnKp+DvX1dtmwpGR5c2KNWmMJ5tI6OTjg6OgGKvMXC\n476JzMxMJk8eT1paKjKZFB+fYTRt2oLExATGjx9FnToO3Lp1AxOT8vj5/YS6ujoBAdvx9w8gL0+b\nvLxGmJmdf6d7fSomT56Orq4uOTnZ+Ph40LKlC9nZ2dSqZYev72hiYqLZtCmUgACFx9TffwHh4Qfp\n0KEzQ4YML1KG4eHDB1hZVfus6xEQ+DchGHMCAgIC/1KqVDEnOHgTv/9+lqCgVdSrVx8NDTVGj37K\nwYM7AClZWVCnjjW+vkNwdR2Es3Mzrl27SnDwGuU4pZUvqF27LjNnzkNdXZ1z586wa9cO4uLiuHXr\nD+zsaiOVSnnw4AF6ehVK9NXU1MTY2JiEhAS6devJ2LETychIZ8GCOXh6DqZ/fzdcXXsRHf0IuVzO\n3btRhIRs4siRw6xcuYQnTxKRSCSsWLGENWvWY2xswtixvpw5c5IaNWoRELCC4OAwtLV1lOebNWv5\nSZ5x+fIVsLOrAyiKwW/fvhVTU1M2bdpAbm4OqampWFpaKQvDF89zKl++IgcO7KV2bXsePLjPjRvX\niIuLRUVFheXLF2Nu/hX6+vqoq2uUqTj4NkoLcZTLea3EuJqMjHQyM7No2LAxkydPoksXMzZv3oC7\n+xiGDvVm4sQfiownl8sZOnQ7u3e7AZ2oVOkI9+49wcmp7PDV6tVtad26LZ6erhgYGFKzZi1EIpg+\nfS7+/gsIDQ1GKpXSpk07rK2rAwqDdfr0ySxfHqgc5969e+zbt6dI+wJjTiRSFE1v3NiZVauWkp2d\nTeXKZsqQQpFIhJqaGt7eX5OWloZYrIKXlxuvXr3CxaUtzs7NmDZtItnZWaioqNCjRx82bAjGyakh\nO3b8gpaWFjdv3iA9PQ25XI6lZTVOnz5JQkI89+7dVd4DFN48FRWJUlF20CBvXFzavPNnVhY7d/7O\nlSspmJmJGTq0zTuFFqqrq+PntwhNTS2Sk5MZOtRLGZYZHx/HrFl+TJw4lenTJ3Pq1HHatevIypUr\nePz4J7KznTA2Xkhy8gdP/YPYvn0LZ84oQs2fPXtGXFwcYrGYli1bA3D16iXu3o1i8OBBAOTk5GBk\nZAQoyjDs2bMLmUzGixdJxMQ8Eow5AYFCCMacgICAwL+UpKQkdHR0aNeuI1pa2uzbtxsAHZ1X7Nnj\nyeHDBzhxwhGAzMwMjI1NAIVQRwH16zdk585tjBo1DlCEWdaqVRs/v9l4eX2NuroaKioquLt7Y2pa\nmaVL/UlPT0cmk/LNN960bFm6VH2bNh1YsmQRaWm1uHHjOtra2mhoqGNmZoZEImHOnAUsXryI58+f\nkZeXh6qqGj179mHr1jBGjRqKTCZ7HaanjYqKCm3bduD69WuIRCIcHOopw8IKzn8qY65wCFxBaN7P\nPy9k3bqNmJiUJzh4Dbm5uaW2B7Czs2PLljCmTJnB/v17OHBgL+XLl8fWtiZRUZGEhGzi1auXeHq6\nfcgsSz07f/5sFiz4CSurahw8uI9r164CULu2PYmJiUREXEEmk2FhYVmk39OnTwgPt6dA5j8hoS1b\nt27HyenNuXXu7t64u5cs1P3TT8tKbd+yZWtOny5aqkIsFpfavsCzlZiYwLlzp5W5ZAUkJiZw585t\ntLV1ycvLe12K4SdiY2NYtMiPq1cv8+RJIkuXBqCjo4Ov7xBWrVrKzZs3aN68JRKJBIlEFR0dbVas\nWMOCBXPQ1NRETU0NY2MTzM2/omvX7oAi5PVThPSuXXuC2bNrkZ1tBaQQHb2LRYt6vbWfXC4nIGAF\nN25cRywWkZT0nFevXgJgalpZaQzb2NiSmJhAeno6+fnZZGcrvICpqd0xMNhX5vifmoiIK1y9epnA\nwBDU1dUZOfJbcnNzUFNTL/KcO3bswrffjijSt6AMw9q1G9HW1mb+/FlFfh4FBAQENUsBAQGBfy2P\nHj1gyBBPvLzcWL9+LR4e3wAKg8zDw5UdO35h5MixgGIzPG3aRL75ZhD6+vrKTZKHxzekpaXh7t4f\nT083rl27ir6+PrNn+6GlpUl+vpy8PCkqKhKsrauzYsUa1q/fzMaN2+jbty8Ay5cHKkMLC+jTpz8H\nD56gcWNngoJWcerUcczMzGnRwgUAW9uaBAaG4OMzDBeXNmhoaCASiaha1ZLJk6czZsz3NGjQCC0t\n7dcjllX8XP5JhRuePn3CrVt/AAXF3BXKgLq6emRmZioLvUOBPHx6kWMrq+q8fPkCO7vaqKiooKam\nRqNGTbh+/RpffVUVN7feTJs2merV32wovYm6dR04ffokOTk5ZGZmcO7cGQCysjIwNDRCKpVy+PCB\nIn06dOjE7NnTSi0CrpDPzyh0Ro6qqrREu09JZmYm3303HG/vgXh4DODsWYXXJiBgOY8fx+Pl5caq\nVQqjb/PmDUyaNI6cnGzEYhEbN25DW1uHU6eOM3fuzFKVLwuL9ri7e9O0aXN8fb8jOHgTlSubAYq8\n0KCgUEaMGM20afMZNOggTZv+jKPjcZo1O8D27Rc+6pqPHs17bcgB6HHmzLsJ+YSHHyQlJZng4DBC\nQjZjYGCoVNJUUytcqkIFmUxRqkJbWwUTk98BUFePQ+/zaAYBKHMl1dXViYmJ5vbtWyXa1KvXgBMn\njvHq1StAoer65MkTMjMzS5RhEBAQKIrgmRMQEPi/4ODBfdSv30ipRPgl0KBBo1IVBr/+2r1EHlnT\npi2UoVeFKSs3x9HRqUQdu/ehuNdw164dvHz5gqioO9ja1iQzMwN1dQ1lWGJQ0AmOH5fy8uVTmjR5\nQrNmDVmyxJ+UlGS0tXU4ejScPn0GUKNGzVLPfwpEIhHm5l/x22/bWLBgNlWrWtKzZx+l8WtoaETN\nmnbK9p06dcXf3w8NDQ1Wrw6mW7eerF8fRJ06dVFX1wBg7doN6OrqYW1tS1hYCGpq6mRnZzFixCjl\nWKWJkLyJskIcBw8eypAhnujr61Orlp0yRxEUHs2goNW0bdu+xHiGhka4u//OmjVR5ORUplatPfj6\nNvw7j/BvU1bo4LBho4iOfqQUSbl06QLx8XEsWPATo0ePIC8vjxs3rmFjY8vjx/Gkp6eVqnwJRUV7\noGS4cYsWrQDYuTOOJ0/yuHBBExgHKF4wzJ59iDZtXmJgYPhR1qypmVfkWEvr3TxMGRkZGBgYoqKi\nQkTEFZ48SXxje21tbSpUMOabb2JISIgjNvYMiYnab+zzKWnYsAm7dv3KwIF9qVLlK+zsagNFvdxV\nq1rg4zOMsWNHkJ8vRyKRMG7cRGrWtFOWYShfvuJbyzAICPw/IhhzAgIC/xcUl5X/cvlwL9Xu3eeI\niUnDxaUatWv/vdyTR48esHLlUsRiERKJKuPHT0Yuz2fx4kXk5OSgoaHB4sUrEYlEPHz4hN27a5OT\nY0GlSuEsXhxN+/YuDB3qy6hRQ5HL5TRp0oymTZsDlHn+Y6y9MBUrmrJp044S5318himLtxemRQsX\npecRSsrFFzbSWrduW0Lk5NSpCB48eEbr1nWoWvXdFDwLKCvEsUBlsTg3b16nVas2hTyfRZk2rQtf\nfx3D7dtnaNmyFTo6Ou81nw+lrNDB4gbXpUsXuHz5IjdvXuf586eIRCLi4+MQi1VIT0974z0Ki/ZA\nyRBZVVVFoe6oKG1EonwgnwJDDuDpUysSEp5+NGNu7FhbHj7cRmRkPUxN7/Ldd2/+XVQw33btOjBx\n4lg8PAZgY1ODr76yKHNNBcd+fn5MmDAJkQjq12/EkyePPsoa/g6qqqr4+5cMrS1erqW0nxkouwyD\ngICAgg8y5hITE5kwYQIvX75EJBLRr18/3N3dSU5OZsyYMSQkJFC5cmWWLFmCrq7ux5qzgIDA/yGl\n1Vw7evQwfn7+AFy+fIHffvuVuXN/xM9vNnfvRiISiejcuRvly1dQysoXeFWiox+xYsVisrKy0NPT\nZ+rUGRgZGePrOwQbG1tu3LhOVlYmP/wwiw0bQoiOfkTr1m3x8RlW6lzeR53wQ9i+ffcH9Z8xYy9B\nQa2QSk1Zu/YMy5bdoFWr93/bXZbXMDAwpMhxx45dOHlSlZwcxQY0ISGAhIS7xMbG0aZNe9q0Kek5\natOmPS1auJCYmKDMA4QPX/s/wfbtv7NxYyoAbm7aDBjgDMDChQdZscKB7OxmVKkSzurVKTRoUOOj\n3lsul7NmzTHCw3eRlfWAwMA1b2zfsGFtLC2rftQ5vCuFQwdVVFTo27dbmUW4Bw70pEGDRkycOEaZ\nS7dlSxhaWtro6paufAlFPXGKENm/QktzcnKYPn0Sv/yyCyOjLF68ADAG7gOKHLRata5gadn6o63Z\nzs6KgwdNefgwhipV7JTKmWVRYOzo6ekXUZktTIFSJoCr60Dlv2vVqsX69X+VgBg+fNSHTP2zcPz4\nTVaufExOjoQOHcDX95/5HSsg8KXxQcacRCJhypQp1KhRg4yMDHr16oWzszO//vorTZo0wcfHhzVr\n1rBmzRrGjx//seYsICDwf0hpNdeCgwNJSVHUUNq/fy9dunTn/v17JCU9V276MjLS0dLS5tdftyll\n5aVSKUuWLOLHH/8qkL1mzSomT56OSCRCVVWNtWs3sH37ViZNGkdIyCZ0dHTp378H/fu7ERFxpcRc\nvgSkUim7d2shlZoC8PRpMzZt2v63jLn3wcxMBKQBCu9P5cr3MDV1KLP93bt/MmLETSIja2Fmdp45\nc4xp167uJ53jx+Datbv88IMRr14pwvuioq5hZXUHR0cbtm5VIztbYSTExXVg7dptH92YmzlzLwEB\nHZDLe6Ki8oywsONMnmz+3uMkJiYUMZxAUR7h0KH9jB79cf4vLyt0UFNTs0i4aMOGjQgKCqBOnbqI\nRCKeP3+GRKLIExOJREyZMhN/f78SypcF1wto3bodP/44jx07flGWxijg++/t8fVdS4UKSWhoHKdK\nlaqULy9hzJja76U6+i5oampSu3bNjzpmYY4cucrNm89p1coSR8fqn+w+n5qkpBeMH59KfLyihuXN\nm9FUqXKB7t3LLmwvIPD/ygcJoJiYmFCjhuI/Iy0tLaysrHj69CnHjx+nZ8+eAPTs2ZOjR4++aRgB\nAQGBt2JlZc2VKxdZvXo5N25cR0tLm/btO3H48AHS0tK4ffsWjRo1wdS0EgkJj1myZBEXL/6OpqaW\ncoyCN/WxsTFERysKZHt5ubFhQzDPnz9XtisI67O0tMLS0gpDQyNUVVWpVKkyz549K3UuXwIikQix\nOL/IueLHn4IRI9owcOCvWFjsws7uF2bN0kJXt2xFhh9//IObN93Iy7MnOron/v7xn3yOH4OLFx/x\n6tVfnqHkZAcuX/4TuVxOfrHHnJ//8UVdzp3TRC5XyLnLZOU5c0bto41ta1vjoxhyhUMHo6Ii8fAY\nwKFD+5Whg3p6+tSubY+7e39WrVpG/fqNaNu2AzNnTgFg+vRJZGVl4uo6EC8vH6ytqxMYGEJo6Bbm\nz1+EtrbiZ7G4aE/t2vaEhW0jODiMypXNmDVrPqqqqvz44zwWLZqBk5MtFy82ZPPmDhgb/0ZWVhih\noStIS1OEcvr6DiEqKhKA5ORk+vZVCMs8evQQHx8PvLzc8PBwLVLnruD8okXzyS/+BfgErF59DB+f\nSvz4Y1/69NEmOPjkJ7/np6Jv367Ex9dHReUppqajyM624ObNlM89LQGBfyUfLWcuPj6eyMhI6tSp\nw4sXL5R5KcbGxrxQxC8ICAgI/G2K11xzcmpAly49mDhxDGpqari4KGo26erqEhq6lYsXz7Nr168c\nP36EyZOnA39tJOVyShTILkxBLk2Bl64AkUiETCYrdS6enoM/8RP4cFRUVPj6axnLlt0lK8sac/ND\n+PhYvr3jByIWi/n5597v3D49vagRkpb2cb0jnwpHR3P09G6QklKgiPkHDg6KUg09emQSFPSYvLzK\nVKx4mkGDKn/0++vo5BQ51tb+cAn3x4/jmTZtIm3adOD69QgWLlzMunWBPH36hMTEBJ4+fUK/fq5K\nkZr169cSHn4QfX0DypevgI1NjSLhf+8SOjhjxlwA0tJSefXqJX37DqBv348jgpOdnc2aNSd58eIV\ncXGxzJw5X1mj7ezZU2zatIGxYydgb+/AunWBhISsYdSocYhEolJVVXfv/pW+fV1p164DUqkUmUxG\nTEw0x48fKbUA9qdk1y4ZmZkKb1x6ug07d97Cu2Sa5Wfj7NnTxMQ8YuBAz7e2VVFRoWLFazx50pbE\nxGWoqcVRo8aX8dJMQOCf5qMYcxkZGYwaNYqpU6cq34oVUNYvQAEBAYH3obh64v79ezA2NsbY2JjQ\n0GCWLl0FQEpKMhKJhBYtXKhSxZy5cxVhV4Vl5c3NvyI5+VWRAtlxcbEl6nGVhlwuL7P+25fAuHHt\nadz4Bvfv36J16zqYmVX83FMqQbNmIs6dUxg+kEHDhq8+95TeiQYNajFt2mk2b76HXC7C1VWDxo0V\nCqMzZ3bF0fEcf/55DhcXG2rV+vhG9LhxFjx5spPoaBuqVYtk/Hirt3d6A7GxMcycOZWpU2eRmprC\n9esRymtxcbEsXx5IRkY6bm696dmzL/fuRXHq1HFCQ7eSl5eHt/dAbG3/XijpvHn7CQszJC9Pnfbt\nj/6PvfMOqKn/4/jrdttLQ0h2KDREZkY/hOxRZBXx8JgP2VtWZh57RzYRHnvzGI+RyCoy00BG2rfu\n7f7+uE+XFIoQz3n9wznne77ne86593Y+5/P5vt8sXuySK4PtTyGVSunRYzdnzvRCVfU5Zcv6I5Mp\nXhzkRh0zJ6ysbP7N7D+nYcNGlChR8pMG2N8SVdWs2T+xWPbNj5kX6tVr8J6Y0acRiUTMmqXG4sUr\nSEzcQps2g9DWljJu3EgkEglRUZE0aOConAt4+fJFpSdkZrltfpfICggUVL46mEtPT2fIkCG0adOG\nJk2aAGBsbExsbCwmJia8ePECI6PPK0GZmHxfJS2B74twf39tvsf9vXs3hNGj56CiooKqqire3t6Y\nmOjRsWN7Nm7cSPXqCrnrV6+iGDVqnLKsadSokZiY6OHm1glf39loaWmxbds2li5dwvTp00lISEAm\nk+Hh4UHNmraoqYkxNNTGxEQPQ0MdNDRUleenpibGyEiHV6+ilGNRU1NjypQpP9VnvG3berlu+yPO\na+rUDpQseZIrV4IpVUrO2LHuiMXi7z6OL2H48JYMH57ztt69czZgz4lt27ahqalJu3btctU+MjIS\nP7/p3LixnaioaEqUaJmrh9mc7q9EosPbt3FMmDCKJUuWYG5uzqVLl5TfBV1dTZycGmNqaggYUrhw\nYUQiCQ8fhtG8eTOKF1f8zXdyaoyOjkaeP0MXLoSwcqUdqakKb76AADsaN75Av365v345cf58MGfO\nNAcUc+7S0ozYt+8B9epZoa+vzfPnCYjFKsrxpqTooKYmxsREDy0tDQoVEIHtwgAAIABJREFU0sTE\nRA+ZLAkVFREmJnp07epK/fq1OX36NGPGDMPb2xtdXU06duyAl5fXV403r/zxhynDhl0kNTUGE5NV\nqKtLWbz4FlOmTOHcuXP8+eefyGQyDA0NWb9+PXFxcYwbN47IyEi0tLSYOnUqFhYWLF68mOjoaCIj\nI4mJicHDw4MePRSB6bp16wgMDATAxcUFDw8PIiMj6dOnD3Z2dgQHB2NlZUX79u1ZsmQJb968Ye7c\nudjY2BAYGMjt27eZOHEiL1++ZPLkyURGKspSp0yZgp3du3m0IhF4eDjSuHF5+vc/wJw5nQgMDOTR\no/vs2bMHdXV1mjdvzu+/90FNTY2tW/3ZvHkjmpqarFq1in37Ahg4cGD2i/QL8TP9zRH4tnxVMCeX\nyxk/fjzm5ub07NlTub5Ro0bs3r2bvn37smfPHmWQ9yliYz8tMSzw82Jioifc31+Y73V/LSxsWbt2\nc5Z1sbEJnDv3D82bt1aOwdjYjJUr/bO1s7Orw8aNAQDEx6dhbGzGggXLs7Xz9V2m/H/ZspWYNm2u\nsu/MbUWKlMpxLL8aP/K7265dDTLjmNevkz/d+BdDJpPRuLGiJC+31//16ySkUhlJSTIMDIqSmCj9\nrHT/x+7v69dJaGvrULhwUU6fPo++fhHi4pKRSKTExiaQlCRBS0us3Fcuhxcv3pKUlEZiYqpyfXKy\nhMRESZ4/Q9evPyY19X2POH0ePkz46s+iVCpHVTUeqdIfXY5UmkJsbAKJiRJUVNTR0dHl+PGz2NpW\nZcuWHVhZVSU2NgFj4yJcvBhEsWJl2LVrLxkZcmJjE4iKisTMrATNm7fjwYMnBAffpEaNWqxbN5xW\nrVwwNDQkPv4tyckpFCv2bbPgTZvasmTJWdasWc+ffy7BxKQI8+fPZtOm7axevZxly9ZQrJgpCQmK\na7lgwXzKlq2At/dsgoODGD58BOvWbSEpSUJ4+IMsmVcnp9aEh98jIGAnq1atJyNDTt++HlSoUAVd\nXT0iIiLw9p7FsGFj6dPHncDAvSxevJpz586waNFSfHzmkZCQSkpKGrGxCUyaNAVr66pMmTKLjIwM\nUlKSs9xfuVzx2c/8XMfGJpCQkErVqvakpMhJSZFQsmRpbt26R0JCAuHh4bi4uAKQni7F2trml/xN\nzkR4rvp1+ZIg/auCuatXr/LXX39hYWGhfHvo5eVF3759GTp0KLt27VJaEwgICAjkN56e3dHW1mbI\nkI+kQvIZuVzO/v0XiI6Ox9m5KqVKmX6X434JP6NJ+q9ETEw0w4cPxtKyMvfuhVGmTDkmTvTm0aNH\nH7XEqFjRghs3QmjSpCnJycloaWnTpUt3wsPvMneuDxKJBDOzEowdOwk9PT3CwkLx8ZmKSCSiZs38\nNf1WU1Nj5sy5eHkNQktLC2Pjd5+jD73gFIiwsbFlzpyZ9OjRC6lUyoUL52jbtkOej92kSTUsLfcR\nFqZQMixe/CTNm1f40lNRUrlyRbp1C2TLFg3kcglaWnEMHPiu7O9T6phdunRn4sSx/PXXburUqUem\n5+HJk8c5evQgqqqqGBsXxt3dEz09vRwNsL91MAfw+nU0SUmvGDduOFKpDIlEwp07t7Czq0axYorf\nq0xPwZs3Q5gxYy4A1arZ8/btW5KTkxCJRNStWw9VVVUKFTLA0NCI169fcePGdRo0+B8aGpqAwncx\nJOQa9eo1xNTUjHLlFGW9ZcuWw96+5r//N+fZs+hs4wwODmLSpGmAYk5tbkWk1NXVlP9XUREjkylK\nSe3tazFlyow8Xy8BgV+Brwrm7O3tCQsLy3Hb+vXrv6ZrAQEBgc/i57fpux5v3Li9+Ps7IZUWZe3a\nA6xZk4SNzZeZbn9rfh2T9J+Xp08jGDduMlZWNvj4TGXXrh2cPXsaHx9fDAyyW2JIpVLWrNkAgJ/f\nKjKnm0+fPhkvr9HZRDl8fLzx8hqDrW1Vli1b+Nnx7NmzC01NzWxCHDlZEYhEIjQ1NZkz50+GDRuA\nh0cf5XgUc+Hf7f/2bRwSiQRLy8rUq9cADw83jIyMMTcvn20efW4wNDRg/Xprli3bjkymQufOpbCy\n+rr5f5nMnduBzp1vEReXSL16u9HUVAQm74u0fOiVCFCqVBn8/bcqlzNN5Xv06EmPHj2ztf+YAfb3\nwNm5FRMmjFFmbs6fP8uJE0dzbJtzYI7S/gEUwZZMJsumfyCXy5XrsgZZivLz9/fNy7HzgkgkokoV\na3x9ZyuzpCkpKbx8GUvJknm35RAQ+BnJNzVLAQEBgV+Z+Pi3BAaaIpUq3q4/ftyK9et34Ov7bYK5\ngmCS3rp1S7p16/1Nzu+/QJEiRbGysgGgWbMW+Pv78fDhA4YNGwBARkYGxsbvTNEbN26arY+kpEQS\nExOziXIkJmaur/pv/y25ePHCJ8fTrl3uFEVNTYsrzah1dXVZvVoRYGaKV3h69s3SXl1dAwMDhdVE\nly498PTsS2pq6r+frS8TQClXrgTz5pX4on0/h7291TfpFxSCcNu3n0NVVYSbmyPq6vlnD5Ebqlev\nyZgxwxkwoC+gRnz8W8zNyzN//ixiYqIxNS1OfPxb9PULYWNjx9Gjh+jZsw/BwUEYGBiira2DXC4n\nIeEt7u6d3wvwRdjaVmXGDG+6d/cgI0POqVPHcXZupQzKMr0IczfOGuzevZNOnbogk8lITU3Jkp17\nP3DM/P/HBPUMDAwYP34KU6aMIy0tHYC+fQcIwZzAfwYhmBMQEBAogBQEk/QuXdrTurUr+vr6P/JS\n/LS8/+Apl8vR0dH5pCWGpuY7wZLExAQCAwO4c+cWL1/GMmHCaCZO9GbgwN/IyJAxcOBvpKamKlX8\nEhLiefkylpSUFLS0tFi+fDHnz59FLBZTq1ZtBgz4g7VrV6KtrUOXLt0JCwuld+/pyGTyLCWaMpmM\nFSuWcP36VdLS0unQwZW2bTsQHByEn98qDAwMefToARYWlZg0aRoBAduIjY2lc+euiMW6lC9fjPj4\nl6SlpeHs3IoKFSy+3QUuYCQmJtKp00GCgjwAKfv3+7N5s6syS/U9KFOmLL/91h9PT0/S0qSoqqri\n5TWaUaPGM378SDIy5BgZGeHruwRPz774+EzFw6MLWlpaTJgwBcj83GYPmipWtKRFi1b89psHADVq\n1OLmzRs0adIMkUiEpWUlLC0rMXOm92eDsaFDRzBnzgwOHNiLiooKI0aMo0qVd0F2poXF+y8WnJ1b\n4ezcStlmzpwFyv9Xq2avfOkgIPBfQwjmBAQEBHKBvn4hOnaMYf36GKTSYpQuvZ+ePb9+Hs/HMDev\nwNKlC1m+fDF169bH1raq0iTd2bk1t2/fYtKkaSQmJipN0uvUqUfNmrWVfeRkkg7ZM0I5maQDlCxZ\nkufPnxWYYM7JqT7Hjp390cPINc+fP1PaXxw7dpgqVazYt29Pri0x4uLe0KlTVyIiIpBIUtm1K4DU\n1JR/Pxur6d69E8uWLWLlSj/Wrl3F4cP72b59Mx06uHL27Gm2bNkFoLTkeL880sfHm6lTvSld2iJL\nieb+/XuV2bi0tDQGDOij/Ezdv3+PTZsCMDYuTP/+vbl5M4TWrduxaNFqrl//i4wMQyIjT7JhgwG2\ntt/uu1FQ2bDhLEFBPQExoMrp093Yu/ckLi7/y/djfWxO5s2bN9i8eT0gx9KyEiNGjEVNTQ0Xl9Y0\nauTEpUsXSEh4J9yiq6tLr159cHRsDLz7jsXERHPunCKgmj17AdOmTSQlJQWAUaPGY2VlQ9++PYmI\neMy4cSNo2bINwcFBbNu2mTlzFhAf/5axY4cTHR2NpqYWDx7cx9m5FdHRUcyc6Z2jR+GX8PBhJMuX\nh5CRIaZLl7LY22d9efCz/WYICHwJX2faIiAgIPAfYsaMtqxaFcK0aTvZtcvim86XyzQmNzcvz+rV\ny1i/fg0tWrThyJFDnDhxJJtJup1ddfbs2cWsWdOUfXxokr5u3RbWrduCv/82fH0XK9t9yiQ90+Kh\nYPBzeZaWKlWa3bt30L27K4mJibi4uDFt2mxWrFhMz55d6dWrK7dv3/jo/rq6elhZ2TB+/BRiYqLx\n919DerqU4cMV3mdt2rTj4cP7tGjRmEOH9pGQkMjz58/Q0dFFXV0DH5+pnDlzSilYkUlmiaa9vT2g\nKNHM5MqVixw+fIBevbrSr19P4uPfEhn5FJFIRKVKVShc2ASRSET58hWJiYnh8uWbpKVpkXlvYmIa\nsW/fgxzPJzExkd27d37RtXRxaU18/Nsv2vdX5enTCDp0cGXTpgB0dHTYunUTM2d6M3XqLPbt24dM\nJlNeb5FIhJ6eHv7+2+jYsRMLF85Xrs9K9u+YkZERCxYsxc9vE97eM/nzT0Wpd//+g7GxsWPdui10\n6tQ1yz5r167EwqIS/v5b6ddvINOnTwIUnnuXLoVQpkxr5s5dxLp1qz86p+5zvH79hp49b+Lv78bG\nja707fuasLDHnz0fAYFfDSEzJyAgIJBLRCIRrVo5fJdjFRST9IJIcnIyY8eOICEhHplMym+/9ade\nvYZs2bIBdXV1XFzcWLRoPg8e3GfhwuVcvXqFAwf+UqrnfS/EYjETJ2Y9ZoUKFVmyZFW2tosXr8yy\n7Orahb//Pq3cZ9iwUezatYPw8LtKURszs5I0auSUo4rf6tX+BAVd5vTpEwQG7mDhwuXZ2mTyoRCF\nl9coatSonWVdcHBQlkBfLFZBJpNSpIghItH7D+MS9D6irJ2QEM/u3QG0b++SbZtUqigJ/BCZTIZY\nLM5xrlRBw929Pvv3r/+3zFKGo+MW2rbNfq75xYdzMtevX0Px4maUKFESUJQlBgbuoFOnLgA0adJM\n+e/ixb65Pk56upQFC2Zz/344KioqREY+BT4tYPKhUmZMTAxubu2JjIwhMbE2s2e7cfjwegoXLsSb\nN68pXPhdpcDBg/u4ezeUYcNGAXDkyEF27tyOVJrOw4cPOHnyAs2bO1K5cm2Sk6MpWXIX0dHLiIxs\nSkDASiIjvUlNTcHBIXcG5QICPztCMCcgICBQAHn48D5Lly5ERUWEqqoqI0aMA8DJqTlv376lVKky\nAMTGxjJzpjdyuSKD9vvvgwFo0aI18+b5KAVQpk2bzcKF80hMTEQmk9K5c9dswdyHKoUFFQ0NDXx8\n5qKtrUNcXBy//96LevUaYmtbjW3bNuHi4kZYWChSqRSpVEpIyDWqVq32RccaOfIPpkyZ8Unp9EGD\n+jJo0DAsLbOKfaSlpfHPP+epU+fLXgB8WKZpY2NLePhdbt9+xMaNMaSkpPDixZVsKn6FC5uQmppC\nnToOWFvb0rlzW0Dx8C2XK0RNdHX1uHr1KqVKVeTo0UPKY9asWYfAwJ3Y2dmjqqpKRMQTihQpCsCL\nF8/x8OiCSCQiLU1CqVKl2bRpFerqbyhduiOxsb9Tt+4btLTeZCmla9asBSdOHCUtTcLz589p3tyR\nFi1aU7p0WZYtW6hU8ty+fQ/z5vkQFHQFNTVVtLV1cHHpTJEixYiNfcGgQX3R1y/EkiWrkEgkzJ8/\ni7t3QxGLxQwaNIxq1ew5eHAf5879jUQiISoqkgYNHBkwYMgXXf+8oqurS0BAS7Zt24Oamgg3t46o\nqalx9uxpSpYsTZkyZfP1eB/OydTV1cuSvXxfbfJj+4rFYjIyFEFZRkYGUml6trbbt2/G2LgwEydO\nQyaT0ahR3VyN7/1gLyUlmebNf2PFCglyuS4gIiTEg5o11yGVZs3MvT/mx48fcfLkMVas8EMsFuPo\nWJujRw+RmpqKjY01u3aNQF9/N4UK7eD16y6EhR3C3b0LzZq1IDAwIFfjFBD42RGCOQEBAYECSM2a\ntbPMf8vkxo3rtG7dTrlcvnyFHC0aGjZsRMOGjZTLuckI2dlVx86uunJ548aNBdKYVi6Xs2LFEkJC\nrqOiIuLly1jevHmNhYUld++GkpychLq6OpaWlQgLC+XGjevKt/x5Pc6cOX9+NiuU03ZT0+K4u3ty\n8eKXB3OZZZqzZk2lTJlytG/vwo4dW/HyesqDB4qytqJFZYwc6YWamhhQqPhpa2szZsxw0tLSADmD\nB3spx5k51HHjJjN16lRksgxq1KitPIfWrdsRExNN797dkcvlGBoaMXPmXJ49iyEy8imBgfvR1y/E\n7NnTOXr0EAMHDsXOrjqbN29ER2caGzacZd261Tx9GqE0nXZza09iYiLTp89h7doVynLNnTu3IZPJ\n2LQpgNu3b/4ryjOZ8eNHUqpUaW7eDKFFizYMHtwXIyNjlixZhVisOM/AwABUVFTw999GRMRjhg0b\nxNatgYBibt/69VtQVVWja9eOuLq6YWJS5IvuQV7R0dGhd+9mWdb9/fdpHBzq53sw92Gwb2lZib17\nA4mKisTEpBJHjhzM8hLjxImjdO/ekxMnjiozesWKmXL3biiNGjXh3Lm/kb5zVFeSnJykvH6HDx9Q\nll5ra+uQnJyU49jeV8ocPXoYGRkZnDq1BS2tsqioSHjzxhOx+BlJSXGMGTMMNTU1hgwZjrW1bZZ+\nTp48yqVL/+DkVB8dHV1kMhkxMdGoqanRu7c7UVH72bkzA1XVqzRurM7jx9HKDGSzZs4sX744p+EJ\nCPxSCMGcgICAwE/CtzRJT09PZ8uW06SmyujUqTaGhgb5foz84ujRQ7x9G4ef3ybEYjGurm2QSNIw\nNFTF1NSMgwf3YW1ti7l5eYKDFZmr0qXL5KrvmJhovLwGUaWKNXfvhvL48SMOHDiOvn4h1q9fw9Gj\nhzAwMKRIkaJYWFRS+pOdOnWc+fNnkZiYwJgxk6hSxYo1a1aQlpbGjRvX6dHDk0aNmuTpPHMq0+ze\nfSS//95Yufz8uSfu7vqMHJk1gFi92j9bf+9bClhYWLJ3715lsJ6ZvRKJRPTrN5B+/QZm2Tc5OYnO\nnbuir6+wIBg9egKtWjmxYMEcAAoV0gfkSCSSbKbT+vqF0NTUomJFhThFpk1DTEw0IpGIsWOHK0V5\nAgK2EhZ2hxcvnvP2bRyRkRFYW9ty8OB+Dh8+oPTIu3kzBBeXzoDCA65YMVOePo1AJBJRvXpNtLV1\nAIW6Y0xMdJ6Duc+V7Do7t2Tt2lWkpaUpzcU/VBGtWbM2DRv+j/Pnz3L9+jX8/dcyffoczMzyx3Lh\nw2C/c+duVKlizcSJowE5FStWol27d2WeCQkJeHh0QV1dXVma26ZNe8aMGU7Pnl2pVasOWlrayvaZ\nAX779q6MHz+Kw4cPZmlTvnwFxGIxPXt2pUULhXJp5suCD5UyjY0L4+fnj5vbAF6+1AFeY239O4UK\nGTNrlkKVcsSIwWzaFJAlo3f69Elq1arL7Nm+BAYGsHz5Yjw9+7J1q+IF1qRJrahe/S8uXXrK1Kmd\naNkya7mygMB/ASGYExAQEPhJ+FYm6VKpFA+PAI4f9wDUCQjYyI4d/+Ps2WMEBV37oqzWtyQpKQlD\nQyPEYjHBwUE8exaj3GZrW5WtWzcxbtxkypUzZ9EiXypVqpyn/qOiIpk4cSqVK1vh6toGgNDQ25w5\ncxJ//22kp6fj6dk9S1llRkYGq1f7888/51m3bhV//rmM337rz927oQwdOvKLzjOnjF+FCsXR1b1H\nYqLCX05F5SVmZprZ2uU3IpFI+ZB94EAQhw69JjFRwqpVvhQvXixb+w9Np98n06ahWLHiFC9uppSY\nDw4OYs2aFVSsaMmQIcNZsmQBaWlpjBgxlrNnz/DyZSy9e/dg7dqNnxxrVgNr8ReJ+HyqZNfcvDz+\n/n78+ecyNDU12bRp/UdVRHV0dKlXrwEODvWzZMrzg5yC/erVa+DntxkTE71sWfVu3dzp339wlnWG\nhkZZTNIzt79vCVCiRMkshumZbVRVVbPNxczM7Ovr6ys9MQFcXdsgFosZMqQrx4+foVmzyyxd+gJd\nXRPGjlVkjpOTk5WKmZm8ePECiSSNN2/e/JtpW5Tl+w6K+cGZ5u/W1racOHGUpk2dOXr08EevnYDA\nr4SgZikgICDwH+fUqSCOH+8AqAMq3Ljhjp/fPz96WNnIDG6aNm1OWFgoHh5uHD58gNKl35Wv2dhU\n5fXrV1hZWWNoaISGhobScDu3FC1qSuXK7zyv5HI5N2+GUL++I2pqamhra+PgUD/LPg0bKuTnLSws\nlQ+bijlqHxeJ+BTvP0y/j7V1Rby8HlGy5G6KFduPu/sBunT59kIP1arV4NSp4+zde46hQw3YubMx\nr183wt19PunpinlW4eH3Prr/y5exREQ8Jjk5WWnTkJSUQEJCPKB4ofDo0QP09PRQUVEhJiaa27dv\nAYrgWl1dne7de2JgYMDz58+xta2qnOsXEfGE58+fUbp0mRyv95fcgw9Ldq2srJUluxoaGjx+/JD+\n/T3p1asrhw8f/KyK6Jd+Dj5F3kRhfvxk2MOHgwkMvM3bt2k0bVobkLNqlb9SZTcw8ABaWlpZzkss\nVqFPn9/x8hrI7797IpFIePXqVY4+dgB//DGCwMAAPDzcePky9qcQzhEQ+FqEzJyAgIDAL8DnysLq\n1HFg06b1yOVy6tSpp3y77uRUn6pV61Kq1J+8eDEVdfXHGBmt4syZdNTVv49yZ27JNBIuVMjgo8bb\n9vY1OXXqXSCaOY8qL2hp5ZTpEn3wQJ714TxT6VFFRfzFUuu5ITExETOzOC5fbk9s7AuWLPkTkajD\nNzteJmXLlsPd3ZMFC3woVMgQTc3KvHgxAZFoBO7unRGLValatRojRihsEz58hi5e3IyjRw+RkBDP\nqVPHadWqHZ6e/Vi+fBE9e3ZFJpPSsWNnZDIZoaG32bVrO1ZW1gAsW7aQ2NgX9O/fh1q1alOhQkVK\nly7DvHk+eHi4IRaLGT9+CqqqqlmMqTP5kgd6VdWPl+yampphb18rTyqi+R1UfCzY/xgBAXvz9fh5\nJTExlbFji6OiUgMNjet4ee2mRo3aBARso2vXHgCEh9+lQgWLLN8za2tb5PIM1q3bwu7dO1m2bBFV\nqlgpfwsAHB0bK33yTE2LZ/lt+O23/t/pDAUEfhxCMCcgICDwC/CpsrCSJUuxYsUS/Pw2oaurh5fX\nIM6ePU39+o6kpqbi7NyEFy9iiYoywtR0OAYGrqxf3xJv77GUK/dzmT9HRT1nzZqrZGRAr162lClj\n9tV9ikQibGxsmTNnJj169EIqlXLhwjnatv10EKWjo0NycvJXH/993pf3L1bMlOnTZ+dr/5/C2bkV\nISEifH07AopSxrQ0N5YsqYCxsbGy3ftz8wB8fZcwevSwbCWBQBYxH4B27Tpma5OTEqm6ujrjxk3O\ncYzOzq2Uy5klnF/Cx0p2q1Sxxtd3dq5VRBU2ITkLhfxXSEmRk5xcAV3dSECds2cNOXHCiz//nIuH\nRxdkMpnyZcD7Afkff4zA23sCmzf7U69ew08GxSdPhrBnzzM0NaX88Yc9ZmZFv9PZCQj8WIRgTkBA\nQOAX4FNKjg4ODahWzZ5ChRSiJk5Ozbl+/Rr16zuioqLC//7XhIYNM5g7dxl375qyeHEHdHV1adGi\nBaGhHy+dK2i8fv2Gbt0uc+dOF0DE8eMBBASoUbx43sQvsj4wKv5vaVmZevUa4OHhhpGRMebm5dHV\n/ZhdgWIfOzt7Nm1aT69eXb9IACUnVqxYTFRUJL16daVEiVI8efKIDRu2c/DgPs6ePU1qaiqRkU9x\nc+uGRJLG8eOHUVNTZ+7chejr6xMVFYmv7xzi4t6gp6eDl9cYpc1Fbhg6tDG3bq3j4kVLdHVfM2iQ\napZA7mPkJTPl53eaPXvSUFWV0aePCS1a2Odqv4sXb3Ht2lNq1CiDvX2lz+/wGWxt7di4cR1WVtZo\naGgqS3YNDAwYP34KU6aMIy1NUWL6KRXRxo2bMnv2DHbu3M60abPyTQDlZ8LC4g/u3DEgPr490J6K\nFbdjaGiEt7dPtrbvB+S5zbT9808oAweKefXKBZATHLyRv/5qhra2do7tBQR+JYRgTkBAQOAX4NNl\nYQr58Xe8859SV9dAJBIhFotxcLBGKn2pDFK+xTyfb8nevZe5c6czmcFUeLgLu3cHMHCgc677+LB8\n7f3ytC5deuDp2ZfU1FQGDeqLhYUiYHjf3qFQoUKsWOGHRCJBX1+f1as3fOVZZaV//yE8evSQdeu2\n8OxZDKNGDVVuy1wvkUjo3LktAwb8gZ/fZhYv9uXw4QN06tSFOXNmMHLkOEqUKEl09ENmz579SUPx\nD9HU1GTjRjfi4t6gpVVJKTzxKfJSEnjy5DWmTatAUpIlAOHhp6lcOZIyZT4dAPn7n2HatNLEx3fC\nwCCYqVPP4+b2dWXC1avX+GjJbrVq9jne25xURK2tbdm0acdXjeVnx8vLmnv3tnH7dk2KFLnPkCGG\nn93n9u1wbt9+SsOG1hQtavLJtseOPeHVK9d/l0TcuOHEtWuhODhU/+R+AgK/AoIAioCAgMAvQmZZ\nWNWq1bC1tWPPnl1UrGhBpUpVuH49mLdv45DJZBw/fjTH0rVKlay4fj2Y+Pi3SKVSDh/+udTgChfW\nQSR6/d6aBAwN1fOt/zlzZtCrV1d69+6Oo2MjKlSwyLI9Pj4eV9cd1KwZhYPD3wQEXMy3Y2fyfoD9\nYbBtZ2ePlpYWBgYG6Orq4eCgEEYpV648z55Fk5KSws2bN5g4cTS9enVl8uTJvHr1Ks9jEIlEGBoa\n5SqQyyvBwS+UgRzA8+d1uHQp7LP7bd2aSny8Yo5dXFw1tmxJzPex5YV//gmlY8cDODkdZ/Lkv366\nFyP5jYVFaQ4ebMSxYy84c6YinTrV+WT71atP07ZtBoMGNaNVqztcvhz6yfaGhgDvlDB1dSMwMyuc\nDyMXECj4CJk5AQEBgV+Ej5WFGRsX5vffBzFkyO/I5XLq1q1PvXqKB/33y98KFy6Mp2df+vXrha6u\nHjY2VnxDLY98p1UrBzp33smuXfbI5WJatbqAm1unfOt/8uTpn9zu43OGv//2BFRISIDZswNp2zYN\ndfX8Cyg/RVZJfhXlsoqKCjKZDLk8Az09Pdat2wKQo3z9j8bKygiT9TckAAAgAElEQVRNzQekppoD\nULhwENWrf8m8zR8XPKWmpjJy5BPu3XMD4MaN1xQrdoL+/b+8zDYxMZFjxw7Tvr3LR9s8exbDzZsh\nODk1/2RfMTHRjB49jA0btn/xeL4ELS0tbG2rfLadXC7Hz09CfLyivPbJk1asWLGdmjU/Xjrbv38T\nQkI2ceaMBRoaifTtm0qZMk75NnYBgYKMEMwJCAgI/CJ8qiysSZNmNGnSLNs+76vCAbRo0ZoWLVoD\nBfNh/1OIRCIWLnRhyJCHZGRIqVCh83eVJo+P1+D9gpe4uMIkJSWirm6Ub8fQ1tbOs6hKZlZIW1uH\n4sWLc+rUcf73vybI5XLu3w+nfPmCI3LTvHkNRo48xt69IaiqSunTx5Dy5W0+u1/Xrlrcv3+D+Hgb\nDAyu0q2b/ncYbc48exbDw4fvsrZyuRH37+fd6+593he++RjR0VEcO3bks8Hcz4BUKs6ynJ4u/khL\nBaqqqqxe3Zk3b16jqaklzJUT+E8hBHMCAgIC/2Ey3/gXK1aF06efYGqqjofH/34qf6ZDh/azbdtm\nRCIR5ubladTICX//tUil6ejrF2Ly5OkYGhpx7dpVFi2aDygCv6VL16ClpcWWLRs4deo4aWnpNGjg\nSO/e/b5oHI6Oeuzff4+UlIpABtWq3cPAoGo+nqnClsHa2hZ3986ULl1WeZ+yS/Jn9eHK3DZp0nTm\nzZuFv78fkIGjY5MCFcwBDB7sxODBn2/3Pu7uDbC0vM3VqzuoWbMs1avX/TaDywXFiplSvvxpwsIU\nQaiKykssLD4djHyO94VvatSohVwOly5dQCQS4e7em8aNnVixYgkREY/p1asrrq4uVKtWh2nTJimN\nuL28RmFl9fnA+EcjEolo2TKFVaueIZUWw8AgCBeXQrnaz8jo82I8AgK/GiJ5ASnk/pne/grkjZ/t\n7b5A3hDu789NTEw0Awf+TmjoLF69qoVI9IZu3fbg6+uS473N/JNRUIK9hw8fMH78SFauXIe+fiHi\n4+MRiUTo6ekBsG/fHp48ecygQUMZPXoYPXr0wsrKhtTUVNTU1Lh69QqnT59g1KjxZGRkMGbMcLp1\nc8+z0XgmAQH/cOZMAoUKSRgzxlE5joKI8N39dly5cpc5c+6TmKhO3bqpTJjQ6qu+M5liNxs2bOf0\n6RPs3RuIr+8S4uLe0KePO6tWrSci4glbt25izpwFmJjoERkZi0ikgrq6Ok+fRuDtPYE1azb8sDLL\nvCCXy9m58zyPHiVSv34p6tSp/KOHVKAQvru/LiYmef+bIWTmBAQEBP7DrFixmNjYWHR0fBCJ6iKT\nGXPhwjbc3XfSokVz3Nx6EhMTjZfXIKpUseb27ZukpaWRmJiISARSqRQHh/oYGRVm//49SKVSrK1t\nmTv3TzQ0NJkxYwoaGpqEh9/lzZvXjBkzkYMH9xEWdofKla2UXmGXL1/Ez28VaWlpmJmVYNy4yWhp\naX12/MHBV2jUyAl9fcWbe319fR48uM+kSWN4/foV6enpFC+u8JqztrZl0SJfmjZtTsOGjTAxKcLl\nyxe5cuUSvXp1BSAlRSHt/6XBnKtrHVxdP9/ueyOVSvH1PcajR2IqVJAzdKgwn+hbUqOGBQEBFp9v\nmEvef+9+48Z1nJyaK4VoqlatRmjoHXR0dLLsk54uZcGC2dy/H46KigpPn0bk23i+NSKRCFfXej96\nGAICPwWCmqWAgIDAf5j+/YegoWFIRMQekpProqb2hPT0AaxZs4Hbt28TEnINgKioSDp0cMXXdwmx\nsS9ITU1h2bK11KlTjzt3bvP27RuOHTvLtGmziI2NZf9+haS/SCQiMTGBlSvXMWSIF2PGDKdrV3c2\nbtzBgwf3CQ+/R1xcHBs2+LFw4TL8/DZhYWHJ9u2bczV+kUiUTSlwwYI5uLi44e+/jZEjxyGRSADo\n3r0nY8ZMRCKR0L9/byIiHivXr1u3hXXrtrBtWyAtW7bJp6tbcBg7dh/z5rVi166OzJrlxJQp+3/0\nkAS+kJw+8zll/bZv34yxcWH8/bexZs1G0tPTv9cQBQQEviNCMCcgICDwH0Yul2NsrEGFCjvR1j6J\nru5JTE0X0a9fTx49ekRk5FMAihY1pXJlKwCKFCmGqakZ5cqZY2lZCV1dXUxNzRgwoA/Lli0iJiaa\nR48eKY/h4FAfgLJlzTEyMqZcOXNEIhFly5bj2bNobt++yePHD/n9d0969erK4cMHef78Wa7GX61a\nDU6dOk58/FsA4uPfkpycROHCCl+qQ4feBS1RUZGUK2dOt24eWFpWJiLiCbVq1ebAgb+U84piY1/w\n5s2br7yqBY/gYD0gUxSiEEFBn896ChQc3he+sbGpyokTx8jIyODNmzeEhFyjcuUqaGlpk5ycpNwn\nOTlJOYfs8OEDZGR8nQiLgIBAwUQosxQQEBD4j6Ohoc6BA7WYOvU4Fhbt6NdPIQCSOS8jJiYaLa13\nnmJqaqqoqWXK3iuEHfbu3c3ChcvQ1tZmwIA+pKVJ3mv/TiL/Q/l8mUyGiooYe/taTJky47NjXbt2\nJdraOnTp0h2AsmXL4e7uyaBBfVFREVOxogWenn2ZOHE0enr6VK9uz7NnMQAEBGwlODgIkUiFcuXM\nqV3bAVVVVR4/fszvv/cCFA/NEydOw9Dw86bGPxMGBikfLKd+t2P/DHO0voSzZ09TsmRpypQp+82P\n9b7wTe3adSlfvjw9e3ZBJBIxYMAfGBoaoaenj1gspmfPrnTq5EL79q6MHz+Kw4cPUqtWHbS03ik8\nFpQ5rwICAl+PEMwJCAgI/IfJfONvYGCAm1tH1qxZQUqKO1paWjx//py3byWf7wRIS5NgZGRMYmIC\niYm5n5gvEomoUsUaX9/ZREVFYmZWgpSUFF6+jKVEiZLKNu+3/xBn51Y4O7fKsq5evYYAyGQyxGJF\nwDl06Mhs+6anp9OmTXtcXd0+O9bZs6fj5tad0qXLsGGDH+7unkDuPMB+NOPHWzJy5GYiIkpStuwT\nxo8v+KqGBZ2//z6Ng0P97xLMQXafwwED/siyrKqqysKFy4F3L2L8/bcqt/fvr5AINTUtjr//tm88\nWgEBge+FEMwJCAgI/If58I2/k1NzZZZKX1+PsWOnZJO9zy6DD02aNKVv355oa2tnM8n+VDB27tzf\nrF27EhUVFYYM6YeGhhYxMVFYW9vy5s1r5s5dxJEjBzh8+ACGhkYUKVIUCwuFeXBUVCS+vnOIi3uD\npqYmo0ePp1SpMsyYMQV1dXXCw+9hY1OVQYOG5njuvr5H2bBBjFSqRsuWL5k1q/1HMxYZGRmMHj1B\nubxx43plMJcbD7AfTbVqFTh2zJz4+LcUKlT1izIz69ev4ejRQxgYGCrvg719DebO9UEikWBmVoKx\nYyehp6dHWFgoPj5TEYlE1KxZ6xuc0ddz5MhBdu7cjlSaTuXKVgwfPgZf39mEhYUikaTi6NhYaVOx\nfPlizp8/i1gspmbN2jRs+D/Onz/L9evX8Pdfy/TpczAzK/GDzyhnNm8+x7p1ichkYtq1gz/+EMRv\nBAR+JQRrAoFvjiCh+2sj3N9fl299bxUP/N7Mn78YVVU1Bg/uy6RJ0+jduwcrVvhRubKVss2qVf7I\nZFI8PbvTrl1H3Ny688cf/Rk5chwlSpTk9u1brFq1lIULlzNjxhTi498ya5bvR4OWoKDb9Ox5grS0\nUsTF9aBIkYlYWQWzbds2rl69wv79ezl37m/atu1AUNBlvLxGsWrVMgYNGsapU8fZtm0T5cqZU7as\nOTKZjHPnzlCqVGlq1KjNgAFDcvSui4mJZsSIIdjY2HHrVggmJkXw8ZmPhobGN7vGnyIv9zc09DZz\n5sxg1Sp/0tPT8fTsTtu2HTh8+ABeXqOwtbVj7dqVJCUlMmTIcDw83PDyGoOtbVWWLVvIxYsXClSZ\n5ePHj1i+fBEzZ85DLBYzb94srKysqVu3Pvr6+shkMoYOHcDQoSMpXLgw/fv3ZsuWXQAkJSWio6PL\nzJneODjUp2HDRj/4bLKTeW9DQx/Qtm0qcXG1AdDUfMTKlQ9xdq75g0co8DUIf3d/XQRrAgEBAQGB\n78alS6EsXPiIlBRVmjRRYeDAJnna//r1YOLji+Dg8AhNzQTq1ClLSMi1LGIrN25co0GD//0b8Gjg\n4NAAgJSUFG7evMHEiaOV/aWnSwFF9u9//2vyyezTvXvRxMc3w9BwI3FxPVBXf0BSUipSqZQbN65T\ntWo1jh8/QpUqVsrMXmZGsn//wQQGBrBu3RZA4QH26NED5fLlyxeJjHzK6tUblN51ISHXKFKkKJGR\nT/H29mH06PFMmjSWM2dO0rSpc56u24/g5s0Q6td3RE1NDTU1NRwc6pOamkJiYoLSxqF585ZMnDiG\nxMREEhMTsbVVGKY3a9aSixcv/MjhZ+Pq1cvcvRtGnz49AEhLS8PY2JiTJ4/y1197kMlkvHr1kseP\nH1GmTFnU1TXw8ZlK3br1lYI+QDZVyYLG1asPiIt7p86amlqWO3eCcC74HzkBAYFcIgRzAgICAgJ5\nJiEhnj/+iObhw84ABAU9plixC3TsWDfXfVy4cI+wMCvevGkMwD//nKN69bgsYivwYUCmeHiWyzPQ\n09NTBlAfoqmpmeP6TBo3tsPM7Boy2W1EokTU1FKoWtWasLBQQkKuMXToSFRUVHB0bPzZ8/jwgf5j\n3nVFihTF1NSM8uUrAGBhYUlMTPRn+y8YZJfDzy0FNeBxdm5Fv34DlcvR0VF4eQ1izZqN6OoqMm9p\naRLEYjGrV/sTFHSZ06dPEBi4Qzk3raALidSrV5lixc7x7Nn/ANDXv0WNGmY/eFQCAgL5iWBNICAg\nICCQZ0JDH/LwYXXlskRShpCQvJX9qKmVQVf3DCJRKiJRMurqtzA0NM3SpmpVO/7++zQSiYTk5CTO\nnz8HgLa2DsWLF+fUqeOAImC4fz8818cuWrQwK1eWxchIjQYNJuPkZE7jxo4EB18hKipKmY350of1\nj3nXva/mee/eXRISfo5SKRsbW86fP0taWhrJyclcuHAWTU0t9PT0CQm5Dijk7+3sqqOrq4uurh43\nbijWHz166EcOPUeqV6/JqVMnlDYU8fFvef78GZqaWujo6PD69StlNjElRZGBrFPHgcGDvbh//x6g\nEA9KSkr66DEKAmXKlGDePBUcHQOoV28X3t4RNGhg/aOHJSAgkI8ImTkBAQEBgTxTvnxJihe/RXS0\nQnFSRSWWsmXVP7NXVtq1q8mJE28oVcoVAG3tilSvbs2+fe8CqIoVLWnc2ImePbtgaGhE5cpVlNsm\nTZrOvHmz8Pf3QyqV0qRJU2XWKzdBmI1NeTp1asKBA3/Rvv1kypUzZ9EiXypVqvzZfVVVVZFKpaiq\nqmbxAAOoVas2q1evoGlTZ7S0tIiNfYGqqlq2PkJD76Cjo/PZYxUELC0rU69eAzw83DAyMsbcvDx6\nerqMHz+FefN8SE1NxcysBOPGTQZg3LjJ/wqgQI0atQtcBqtMmbL89lt/vLwGkpEhR01NjWHDRlGx\nogVdu3akSJFi2NjYAgq/tjFjhpOWlgbIGTzYC4DGjZsye/YMdu7czrRps76bAEr//p4sX+730e0u\nLq3Zu3cPoFBxbdq0Gk2bftmxnJzqc+zY2S/b+V/27NmFpqYmzZu35ODBfdSsWYfChQt/VZ8CAgLv\nEARQBL45wkTdXxvh/v66fO7eHjx4lUWLnpOSoo6jYzJTprTO80P74cNX2b//FWpqaQwbVp1SpUw/\nv1M+cvXqFUaMGMLhw6fQ0NCkS5cOtG/vQqdOXWnatCFHj55Rth08uB+DBg3DwsLyX3XDv7GwsGTi\nxGl4e0/gwYNw7O1r8fTpE8LD7xEf/xZDQyN0dfVQV1dHIkklJiaGbdsCuXHjOt7eE9HR0aZo0WIs\nX+733YVQ8vrdTUlJQUtLi9TUVAYN6svo0eOpUMEiWzu5XK4UCSloQdx/AVfXNuzZs5v0dPFX9+Xk\n1IBjx/7Oh1EpGDy4HwMHDsXSslK+9flfRPi7++vyJQIoQjAn8M0RfnR+bYT7++vys93bK1duEhn5\nmiZNqqGnp/iDGBYWyuHDBxg6dMRH9wsPv8fLl7HUqePw1WM4ffoEly5dZPTo8YBC+XDEiCHMmuVL\noUIGnDhxlMuXLzJ27KQsweGPIK/319t7Ao8fPyQtLQ1n51Z0794zW5t79yIYOjSIBw/MKFXqGXPm\nVMbOrkI+jvrH8ezZS0aO/JsnT/QpUyaeuXMbUrSo8XcfR2a27OXLl0yePJbk5CRkMhkjRozFxqZq\nlmBu7NgRvHjxnLQ0Ca6uXWjTpr2yD1fXLly4cA4NDQ1mzZqPoaER0dFReHtPIDU1BQeHBgQEbMtz\nMHfo0H62bduMSCTC3Lw8ZmYl0NLSxtTUlBkzvDExMUFDQ4O+fQfw11978PGZB8CVKxfZvXsXM2fO\nzfdr9qvxs/02C+QeQc1SQEBAQOA/yZQp+1mzphppabZYWe1h48ZamJkVxdKy0mezAOHhd7l7NzRf\ngjlz8wosXbqQ5csXU7duffT0dHn48AEDB/YlKiqJtDQRGhradO4cBRRccZCc+NC0OiemT79OUJAH\nAG/ewPTpW9i169cI5saMOcuRI+6AiLAwOaqqG/Hza//Njjdy5B9MmTIDHR3dD7Yosp3Hjh2mVq06\nuLt7kpGRQWpqarY+xo6dhL6+PhJJKr/95oGjY2P09fVJTU3FysqGvn0HsGzZIv76azceHr1ZuHAe\nHTq40qxZCwIDA/I85ocPH7Bhgx8rV65DX78Q8fHx7Ny5DZEIHB0bs2vXjiwvMJYs+ZO3b+MoVMiA\nAwf20apV2zwfU0Dgv44ggCIgICAgUGBJSUlh5Mg/6NmzK+7unTlx4hhBQZfx9OyGh4cbPj5TiYmJ\nYdMmM0QiCSVL9iY+fgd9+vQlOTmZ4OAgRo0apuxr5kxvfvvNA0/Pbpw7dwapVMqaNSs4ceIYnp7d\nOHHiGG5uHYiLiwMUZuFubu15+zYuV+MtWbIUfn6bMTcvz+rVyzh9+iRly5qjodGV27fPEB5+hlu3\nDjFhQhBQ8NUQ88rr11pZll+90vpIy5+PqCg93qmriv5d/nbMnbswWyAnl8uVLwAqV67CwYP78PNb\nxYMH99HW1s7WR0DAVnr27Eq/fp68ePGcyMgIANTU1Khbtx4AFhaVePYsBoBbt27QpEkzAJo1y7t/\nQXDwFRo1ckJfvxAA+vr6PHnymNevXyvb7Nmzk6Cgy/8eowVHjhwkISGB27dvUbt27tVwsx733fdc\nQOC/hpCZExAQEMhHZDIZYvHXz1URUHDp0gUKFy7C3LkLAUhMTMTdvTOLFq2gRImSTJ8+mQMH9pKW\nVh9TUy9iYv5EIrHC0XFztjloGzb4YW9fk3HjJpOQkEDfvh7Y29fit9/6c/duKEOHjgQgIuIxR48e\nolOnLgQFXaZ8+YoUKmSQq/G+fPkSPT09mjZ1RkdHlz17dhIXF8fr169QBALpqKs/ISZGl5IltUlK\nSszPy/XDqV49lcuX4wF9IBU7u/gfPaR8o2zZeEJCMlC8B8+gbNm3+dZ3TuWQLi6t8fPbRFJSEl5e\ng6hSxZq7d0OVwZytrR1Ll67mwoVzzJw5hc6du9G8eUtln8HBQVy9eoWVK9ehoaHB4MH9/hVxAbH4\n3eOfiooImUyWZTw7dmxRBnWfYseOLbRt2wENDYUViEiU3cLiyZPHqKm9EwBq185FmZlr0aINo0cP\nQ11dnUaNmqCiIuQYBATyihDMCQgICOSB9evXcPToIQwMDClSpCgWFpW4cOEsFSpU5MaNkH8VFSuy\nbNlCZDIZlpaVGTFiLGpqasqHM339QoSF3WHp0oUsXryStWtXEh0dSVRUFHFxcXTr5k7r1u1+9KkW\nCD4sW9TW1qZ4cTNKlFCoaDo7tyIwcAcNG6Zz544xEokVJUocoWtXy2xB9eXLFzl//m+2bt0IQHp6\nOs+fP8uS7QBo2bINY8YMp1OnLhw4sJeWLVvnerwPH95n6dKFqKiIUFVVY8SIsaioqDBkyBhKlTqJ\nSJTBmzfuVKiQQosWrZk3zwdNTc0fIoDyLZg0qQV6eke4e1eF0qXTGT0699euoDN/vhNqapuIiNCl\ndOkEfHy+UCIyB7KXQzbKkrWNiopk4sSpVK5shZNTAwCePXuGiYkJrVu3Iy1NQnj43SzBXHJyEnp6\nemhoaPDkyWNu37712XFYW9ty4sRRAgK2IZVKc2wTExPNiBFDsLGx49ChfZw7dwYrK1siIp4QEfGY\np08jePAgnKlTfThx4hgPHoTz4sUzunbtSGpqCitXLqFNm/Y4Ojbm8eOHREZGsGDBXOrXb0h6erry\nt9LZuRXnz59FJpMybdosSpUqw507t1i0yJe0NAkaGhqMHTuZUqVKf+XVFxD4uRGCOQEBAYFcEhp6\nmzNnTuLvv4309HQ8PbtjYaGYj6Uo19uARCKhS5cOWTJHu3fvpFOnLp8sqXv48AErV64nJSWZXr26\nUadOvWzy3R+TCf+Vpb8zyxb/+eccq1cvo3r1Glm2ZwZhkyY1Y/To07i5BdCypQWVK5fNsb8ZM+ZS\nsmSpLOvu3Mn6kFukSFGMjIy4evUKoaF3mDJlZq7HW7NmbWrWrJ1t/e7dW5g06QiPHmlRr14y06Y1\nRVdXl4YNG+XYz6BBfRk0aNhPp/onFosZMaL5jx7GN0FPT4+lS7/NHLmAgK2cPatQTn3x4gVPnz7N\nsr1oUVMqV7YC3pXmXrsWxNatG/+1x9BhwgTvLPvUqlWXPXt20b27KyVLlsbK6p2/3Pu/Renp6Vy5\ncomePbsikaSyYsUSXrx4ztatG5FIFPPw5s3zISwsFIkklerVaxIZ+RQHh4aIRCLu3r3Lw4cPcXHp\nRJs27Zk/fzb//HMBZ+dG1KpVBzU1NRwcGjB+/BTOnDnJzJnePH0agb19TWbO9MbTsx+nTh1HU1Mr\ny2+lgYEhfn6b2L17J1u3bmL06AmUKVOWpUtXIxaLuXLlEqtWLWX69Dn5f0MEBH4ihGBOQEBAIJfc\nvBlC/fqOqKmp/fuAUl+5rXFjxVv6iIgnOWaOOnXq8tF+RSIR9eo1RF1dHXV1dapVsyc09Bb16zt+\n2DLH/du166j8/6FD+ylXrvwPCeYSExM5duww7du7EBwcxLZtm5kzZ8FX9flh2WJgYADPnsUQFRWJ\nmVkJjhw5iJ1ddcqWLYeqqpTWrUthaVmW5OQkZelXJjVr1mbnzm0MGzYKgHv3wqhY0TKbTxxA69bt\nmDp1Is7OrfJlXpumpiY+Pq1YtuwEkZFaHDlyg44dPz4/SCQS/XLz6QRyJudySEmWNlpa7z7LmXYZ\nzs6tcHZula2/gIC/MDDQIz09gXnzFuV4zPctN9TV1ald2yGLAmv37p3Q0NCgRo3adO/uSokSJVmy\nZBXdu7ty8+Z1Chc2oVGjxuzcuZVOnboQEnKNq1eD8PNbjYqKGD09PczMSlC8uBnq6ho4ONQnODiI\nAwf20bBhI6pXr8H06ZOJj3/Lhg1rady4KQ0bNmLYsEHK38rMFx0VK1py5sxJABISEpg2bTJRUU8R\niUQfzR5msnbtSrS1dejSpfsn2wkI/MwIwZyAgIBArsk+HyQTTc2chR7kcrnyoVwsFpORodhfIknL\n1nbLlg2oqyuMt/fv/4udO7ezcOFyrl69wv79ewFYtWpZNjnxzAcWU1NTwsJCmTp1grJ079GjhyxZ\nsoCUlBQKFTJg/PjJGBt/m0AvISGe3bsDaN/eJd/6/LBs0ctrFMnJyUycOBqZTEalSlVo184FVVVV\npk71YcGCuUgkEjQ1NVmwYOm/QZGir549+7Bo0Xw8PNzIyMigeHEzZs9egJ2dPZs2radXr650796L\nxo2dcHBowMyZ3rRokX9lgsOH72bLFldAl61bH/D27WmcnSsyfPhgLC0rc+9eGGXKlGPixKwZlnnz\nZhEWdgeJJBVHx8b07t0PUGSKFy2aT0pKKmpqaixatAJ1dXVWrFjC9etXSUtLp0MHV9q27ZBv5yCQ\n/7xfDvn48aNclUPmJ+bmFfD1nUvnzl4UKWLOpElugKK0c8IEb6ysbPj9d0/c3NoTF/eGxMTELGIr\nYrEKGRkZ3LwZgra2NkWKFOXx40c8fvwQU1OFb+SHLyZOnTpOZGQ86ekZFC9uSq9efXnwIJz3m6mr\nqyn7z5zTt2bNCuzta+DjM49nz2IYPLjfJ89NeCEi8F9ACOYEBAQEcomNjS1z5sykR49eSKVSLlw4\nS5s2igflzCCvVKnSxMREZ8kcVa1aDYBixUwJC7tD7dp1OXPmhLJfuVzOuXNnGDp0JFu3biQ8/B7G\nxsaA4s3zjRvXqVq1GsePH8lRTjwzYPlQ+lsqlfLnn3OZPfudx9mqVcsYO3bSN7k+K1YsJioqkl69\nuqKqqoqmphYTJozm0aMHWFhUYtKkaYDC+y2nADM8/C5z5/ogkUgwMyvB2LGTqFmzNhs2+FGxogU3\nboRw8eIFDh7cz9atu1BVVSUpKZEuXTqybVsglpaVWblyXZYx2dlVx86uOgAaGhqMHDku27j19fVZ\nvXoDoBCwCQq6QUzMU8qXr5iv83HOnzcEFOqEqanmHD9+HWdnePo0gnHjJmNlZcP/2TvzgBjzP46/\npmu6iwodEklFypH7vuW2ZLGI3NbNSm65rXWvYyMikZy5WbfcVO4rQqdC0TXVzPz+mF+jUVjk2n1e\n/3iO7/U8M43n83w+n/dnzhwfduzYptJvwIAhGBoaIpVKGTlyCJGRD7C2LsXUqRPw8ZmLg4Mj6enp\naGlpsXfvbvT19fH13UBWVhZDhvSjevWamJtbFNp1CBQu7w6HfGOIfEmjJClJQlTURNLScjAy2kqX\nLjMwMpJjamqGk5MzsbExxMfHYWdXjqioR5QpU5Z79+6ojCESiVBTU6NKlWr4+MyhU6fW2NiUYdCg\nYVy6dJG0tDQMDAyV7S9cuMb9+2uxtBzG1auehIU95ty5/TYq9kAAACAASURBVEoBFLlcztq1qwkL\nu4JEkoWmpuJxNS0tjVu3bhASspOXL18oX4qdPXuaa9fC6N27O1ZWVkye7JPPMy8g8G9FMOYEBAQE\n/iEODuWpW7c+Hh5dKVrUBFvbsujr66uExInFYiZMmJrPcwTQp88A5s71Yc0afSpXrqrsoyiua8eK\nFUu5c+c2w4eP4vTpk5QpY8udO7eJiAhj5Mjf8smJX758ocB15hqWT55E8ehRJCNHDgEUMvsmJmZf\n7P4MHjycR48esm5dIGFhV/D2HkNAQDAmJqYMHtyXa9fCKV/e6Z0G5syZUxk92gsXl8qsXbuadev+\nYvjwMcpwqjVrFAZXXFws586doV69hvz992EaNmxcKAqi2dnZ9OkTzKVLLzE23ouTU3MVz+rnoqcn\neWtf8SBarFhxnJycAYVUe3DwFpV2x44dJiRkF1KplOfPk4iKegiAiYmpMqcu11Ny6dJ5IiMfcOKE\n4mVBWloa0dFPBWPuO0ZTU7PAcMh16wLIzs7G3NwCf/8tBfQsHLZvDych4RfkcjEymQEyWQBGRmqA\n4nckLS0NLS2xUpHy2rVwdHS0kUiyUFNTVypkurhUJjT0FP369URf3wCZTEZcXCz6+voEBm4kJyeb\nYsWKk5WlS0aGBnK5AfHxszEzm8/s2c9p2rQmGhqKOTIzJTx8GIm//xYuXbrA+PGjef48CWfnSvz1\n159YW5eibduOHDy4//9zV+Hp0yfMm7cIX9+V7N27m06dfv5i90xA4HtCMOYEBAQEPoJu3Xri6TmA\nzMxMhg4dgIODYz7lyapVq+HntylfXxeXSmzevKPAcW1t7Zg0aTojRgxBLpdTsaILtrZluXr1EjEx\nMdjYlP6gnHguucaHXA6lS9uyapXfp17uR5E3BFUul+PoWAFTU4XxWLZsOeLj49DX1y/QwExLSyU1\nNRUXl8oAtGzZmsmTxyvHy81JBEU+W2DgBurVa8iBA3vx8ppUKOtft+4Yhw97ANq8fDmex49jOHbs\nIk2a1CgUQZJRo0yZOnUfsbFlqVAhjDFjVAUtgHzGY2xsDFu2bGLNmo3o6+sze/Z0srKyeJ99OXr0\nOKpVyy/CIvDjMGfOfjZsMCYnR0yLFn+zdGnnLybbL5XGY23dGblcHblck+TkrtSpc5Xdu7fRt29P\n1q7diJqaGteuhSOVyrCxKU27dh05efIopqamnDhxjOzsbPT19Zk/f7HSQ6+oxReDtrY2GzYEKfNo\nvbwmsXfvVIyNN5GYOIEnT3ZSp44f48e7c/ToEQCaN29B2bLlEIlEVK9ek0aNmnL79i2SkhIZOfI3\nqlRxxdDQkP79BwOgr6/P69ev8fDoSnp6BjVq1Poi90pA4HtEKOghICAg8BHMnz+LPn2607dvDxo2\nbIydnf1njbd790X8/e+zbNl9Jk3a/X+DL4BKlarg4lKZXbu2U65cufeOoZDWV2zr6r6pXWZtXYrk\n5JfcuHEdUChuPnr08LPW+zFoamopt/PmvZQubcu6dYGsWxeIv/8WFi5cxjtSEZXkzUmsWNGFuLg4\nrl69jFQqpXTpMoWy3rQ0OfAmNEsmK8rLlwphlM/xzuWKNLRvX52TJ505efI1+/c3xsHBBoCEhHjl\nZ3TkyEGcnV0AxeealpaGtrYOenp6vHjxnPPnzwJgbW3D8+dJ3LlzC1DkXUmlUqpXr8WOHduUcz55\n8pjMzMxPXvv3wODBnu8937lzW169Kpyab82a1ftwoy/MpUs3WLnShefPW5CS0pCtWzsTEHDii803\nffpAHBzakpAwlVevfqV/fyk9evSiVCkbbGxs6NHDnbJl7di+fR9z5y4kOfkl27dvRV1dg2LFihMY\nuB03tzbY2tqxbp0vGRmZjB07nk2bgnF1rabytyMSgY6ODt27N0BLK5JSpdywt69P/fqqnvWC6tUB\nZGVlsWrVJWrVklCr1k3WrFEIucyePZ0xY8bj778FT8/++QRkBAT+zQieOQEBAYGPYOrUmYU21vPn\nz5k0KYeEhFUAREY+Z9iw9bx48Rwnp4qIxdqIxWKlt0r1oUh1O3f37dplM2bMY8mSBaSmpiKV5vDz\nz90Lzfh5m4JUId/G2tpGaWA6OVUkJyeHp0+fULp0GQwMDImICMfFpRIHD+5T5roVRMuWrfDxmUzv\n3v1UjuetgXXjRgRmZsWYM+cPxowZpvSsJScn079/L4KDQ9i/fw+nT58gMzOTqKgo7OwukJTkiIHB\nXnR1k6lb9y/l2IcO7WfevBlIpVK8vafg6FiBjIwMFi2az6NHD5FKc/D0HEDdug3Yv38PJ08eIzMz\nE5lMxrJlqwEwMjLOV4Dc2roUO3duZe5cH2xsytCxY2dCQ08jEomwsytHuXL2dO/eiWLFSigNvYIE\nXxYvXkHbth2Ii4ulb98eyOVyihQpyuzZv3/U5/i9sXLl+z3LhZtP9u0FM6KinpGZWSnPESOePcsv\nmFRYaGlpsWFDV2JiotHRscHEpCpxcbGoq6szefIMlbbvii6oWbMZCxfeJDOzDi1bimnTpgGASoho\nlSquVKniCsDo0a3p0KECT548o1q1Cujp6amM5+xcmd27d+Dm1oaUlBQiIsIYOnQkQUFXSUhIJCvL\nkcTEyixevIMuXVLIyEinaFETcnJyOHRoP8WKFQd4p2CVgMC/CcGYExAQEPhGREY+JSGhvHJfLjch\nM9Oa48fPKY/lfXDKKyfesGETGjZsAoCn5wDl8QYNGqvULrOzK8fy5W8Mki+JkZExFSu60KvXz4jF\nYooWNcnXRkND450G5sSJ01iwYA6ZmZlYWloxYcLUd87VrFlLfH1X0qxZi3znoqOfMn36HLy8JjJl\nijcnTx57r9R/bp6fRCLB3b0dDRqIKF36ZzQ0rnHq1HG6dOmGXC5HIslk3bpAIiLCmDPHhw0bgtiw\nwQ9X1+pMmDCV169fM2CAB66uNQC4f/8e/v5bMDAweO99K+ihOdf4A955H94WfJHL5aSkJNO370AG\nDvz1vXP+SOTWV0xKSmLqVG+lF3LsWG+cnSuptPX2HsuzZwlkZUlwd+9Gu3YdlWO4u3dTUYJNSEhg\n+/Ygnjx5zMuXL5QvTYB8c40Z442Li+pcoPAK+vkFYGhoVGjX27RpFRwc9nLnjjsA5ubHaNnSrtDG\nLwiRSKQsp5L32D8hPT2dwYNvcfu2oqTAqVP3MTa+QIcONfK1vXs3ipUrbyKTqdOtmzUNG1YvcM4G\nDRpx8+Y1evdW1JwbMmQERYoURUvLkdTU4lhbd0Iu10QisefVKwf69RvEgAG9MTY2pkIFJ+VLpbwv\nugQE/q0IxpyAgIDAN6J8eVvKlj3PgweKhyht7UhcXY0/0OufExQUyvnzqRQvLmX06KbKsgdfknd5\nLnNru8G7DUw7u3L51ChB1bDJ5dq1cBo1aoqenn6+c+bmlpQtq3j4tbd3IC4u9r1rrlzZFR0dHXR0\ndDA0NGT27CGYmpqyb5+UyMj7gOKhsGlTheHo4lKZtLQ0UlNTuXjxPKGhp9i8eSOgEFFJSIhHJBLh\n6lr9g4Zc7tify/PnL+jf/2+uX7fFxOQZkycXo3Xrd3s2fywU9+fIkYPUqFGLXr08kclkBYaPentP\nwdDQEIkkk/79PWjYsAmGhoZkZmYWqAT76lUKP/3kTnT0UyIjHyjHyTuXXC4nIyOj4JV9AUuhSBFj\n1q934s8/t5CTozB6nJxsC32e9/ExoiuRkVHcvv3GKMvMtOPixXA6qKYSk5T0Ak/Pu9y/ryh9cPLk\ncTZtilS5trwvrIYMGcGQISNUxmjRwow9e0rz+PEAQEq9en5YWFjSoUNnOnTonC/nNO+LLgGBfyuC\nMScgICDwkUil0kJRT9TXN2D58lIsXryZzEwtmjfXpEOHRoWwQli37iRTppRHIikDSHjwIIA1a7oU\nytjfkjNnIli79i9evoxi+fJVBbbJrU8FoKamjlQq+X+NP0XO3tv5NKrt1ZT7ampq7xSZAZRv/GfN\n+p2SJa1Vzt26dQMdnYJrD+alsJQKZ806w5kznoCIlBSYM2czrVoVnhLn90D58hWYM8eHnJwc6tVr\niJ2dai5pXFws/ft7KEV3oqOfsnLlUqKjFQWm163zZdmyhTRr5kZ8vCLn8sKFcwwfPoYVK5YAIjIz\nM7h2LZySJa2ZPn0iO3YEY2BgwLhxE6lY0YWUlGSmTZtIUlIiTk7OXyyMr0wZK/74w+qLjF3YWFmV\noHjx2yQk5IZvv8LSMr8kw5EjV7l/v51yPy6uEYcPB3+Uodq8eRWWL7/KoUPbMDDIZuzYNqipqXH7\ndhReXuFERxtgZ5fC4sX1MDf/csq9AgLfE4IxJyAgIPAW69ev4fDhAxgbF6FYseLY2zty9uxp7OzK\nce1aBE2bNqds2XKsWLEEqVSKg0N5xo71RlNTUyXs6s6dW/z55xKWLVvN2rWriY2NJiYmhuTkZH75\npRdt23bA2rooBga7UVdP49QpKdWqFSkwnOtjOXlS8n9DDkDMpUtm5OTkoKHx4/7sr1t3kpkzy/D6\n9QaMjcM5deox3buX/HBHFEbT3bu3cXSsoJTt/xBvq3MeO3aEKlVciYgIR1/fAD09fapXr8m2bVuU\nnsd79+5QrpwDERFh3L59E4BTp05gbV0KG5vSH3nF/5yUFG3y5nu9fGlEVlYWYrH4i835tXFxqcyf\nf/py9uwZZs+exs8//0LLlq2V52/evE5mZgarV69DLBbz888dlMa4mpoavr7+nDsXysqVy1RUSUuU\nMKd9+05oaKizcaM/zs6VmDZtIlOmzCQ5+SVBQYFMnuzFrl0HWbfOFxeXyvTu3Y9z586wd+/ur34f\nvjeKFCmKj48GixcHkZ4upn79ZAYP7pivXenSxdDWfkRmpkI0SiR6QYkSH//9bN68Cs2bqx6bNCmC\n8+d7AhAdLWfKlE34+rb/+IsREPgB+XH/VxcQEBD4Aty+fZOTJ4/h77+F7OxsPD17YG+vePDLrXUm\nkUjo1u0nli5dhZVVSWbOnMrOndvo0qXbez0hDx9Gsnr1ejIy0unT5xdq1ar7j8O5PhZ9fdUQNEPD\n9ELxJn5LtmzJ5PVrRY5hcnIltmx5QPfu+du9/RmIRCK6devB5MnehITspFatuuQaPm/n0snlb4y4\nvOdEIhFaWlp4ev6iFEAB6N27H0uX/oGHR1dkMhkWFpbMm7dIZczTp09Qp069L2rM1a6txaFDT8jK\nsgakVKoU+68y5ADi4+MxMzOjbdsOZGVJuH//rooxl56ejpqaGmKxmMePo4iPj1PmweV+9+3tHUhO\nfqnsY2BgyNGjhwG4e/dNIeyLF88TFfUQkUhEWloqaWnpZGRkEBERxuzZCwCoVauuSiHs/zIdO9ag\nY8f8pTXyUrOmM4MHHyAgIJKcHDGtWj2hW7dOhTJ/QkJeARURz57pFsq4AgI/AoIxJyAgIJCH69cj\nqFevIZqammhqalKnzhup8txaZ0+ePMbCwlIpGODm1oYdO7bSpUu3d44rEomoW7cBWlpaaGlpUaWK\nK7dv3/hg6Nin4uVVncjI9dy44UyxYk8YO9bshw+5e3v5IlH+ELe3wxa7deuh3Pb336zczq1P5ebW\nhkqVqtCt209UqFARHR1tduwI5uzZ02RlZVO/fkMA5s9fzJQp45HJ5MjlcmJjY3FwKM8vv3TO54kF\nRfkELS0tbty4RmjoacLDw/D3X8vMmfOxtCz88Lm+fRuioXGSCxcuUaSIhAkT2hT6HAXxJQRA3ib3\nexsWdpnNmzeioaGBrq4ekyZNV2lXtWo15HI5PXq4U7JkKaWiYd4x1NTUkclkyuOlS9uyY0cwMTHR\n2NqWVbbLyclGKpWiqamJpaUVkyZNV4bNCgqJ7+ZDvzHe3m4MH56GTCbFwKD6e9t+DI6OKdy7JwXU\ngUwqVCicl2ICAj8CgjEnICAgoELB9Y1AtdZZXvK+jVbkZin6SyTvlxMXidQ+GDr2qZQsWYI9e9oT\nHx9H0aK10NX98d9U9+ihy8OHYSQnV8LE5DK9ehWeARETE83kyT6kpaVy/PhRfH03IJPJGD9+DBER\nYSQnv8TUtBi//64w1tLT0wDVh9f7959y40YSder8jYXFZapXF+Pk5EzduvWpU6eeisrol8DDowEe\nHl90iny8qx5YYZIriuHm1gY3t/xGanBwCAC6unqIxWJWrFiDtrYOw4YNpEQJc+LiYlm+3FfZXkdH\nhwkTpnL16mW0tbVZunQlW7YEkJaWxuLFKwCoU6c+dnb2dO+uCN27f/8eJUqY4+JShSNHDuLh0Zdz\n50J5/frVF732fyNvlyEoDBYvbomR0WZiY3Wws8ti0iS3Qp9DQOB7RSgaLiAgIJAHZ2cXQkNPk5WV\nRXp6OmfPnlaey31otbYuRVxcLDEx0YCi/lilSlUARf5NbiHnkyePqvQ9c+YkWVlZpKQkExZ2BUfH\n8sTHx2NsXIS2bTvQpk0H7t+/W2jXoqGhgZVVyX+FIQfQo0c9tmzJYcaMYIKC1OnUqVahjV28uDnl\nyztx4cJ5Ll26oCwM/+TJY6Kjn1KmTFkuX77AypXLiIgIR1c3/wPp8uWRpKQU5/79joSF1eLixafK\nc/8Gb05GRga//TaC3r2706vXzxw9egSAbduC8PTsgYdHV548iQLg1asUvL3H4OHRjYED+yiVIj08\nupKWlopcLqdVqyYcPLgPgBkzpnDp0oXPWp+Ghga9e/ejf38PRo8eSqlSNkD+UNq8uYW5h+vUqc/J\nk8dp3botw4atxtW1FXfv3sLDoxs9enRh925FiRBPz/5ERITRs2cXTp06QYkS5p+1ZoHCQU9PjwUL\n2hMY2Jzp09ugqan54U4CAv8SBM+cgICAQB4cHMpTt259PDy6UrSoCba2ZdHX11d5IBSLxUyYMJXJ\nk72QSqU4OlagQ4fOAPTpM4C5c31Ys0afypWrquRc2draMXz4IJKTk+nTpx8mJqYcOLD3vaFjPyJx\ncbF4eY1iw4agTx4jLOwKmpqaODk5qxyvUsWBKlUcPneJ+dDR0VZu9+jRm/btf8rXxs9vE+fOncHX\ndwWurtXp3buf0hMrk8lITMybo6ZOWtqbHMUfPcQV4MKFsyreybS0VFatWoaxcRH8/ALYuXMbmzcH\n4OU1ibVrV2Nv78icOX9w9eplZs6cwrp1gVSs6MK1a+EUL14CS0tLrl0Lp2XL1ty8eYNx4yZ89ho7\nd+5K585d33ne2NiY4GCFaEneItYlS1qjqdmBS5d6cumSNocOXWHx4k5Mn+6q0t/Q0IiFC5d/9joF\nBAQECgvBmBMQEBB4i27deuLpOYDMzEyGDh2Ag4MjbduqFk2qWrUafn6b8vV1camkUug7L7a2dvmM\ntXeFjv3XuXr1Mrq6evmMuS9NjRo18fVdRfPmbujo6JCY+AwNDU2kUikGBgY0b+6Gnp4++/YpQvty\nPbE1a9bG1PQySUm5RpuEIkUUuVm6urqkpaV91ev4Etja2vHnn0tYuXIZtWvXU6qu5oaPlivnwMmT\nxwBF7umsWb8DCqMpJSWF9PQ0nJ0rEx4eRokS5nTo0JmQkJ0kJSViYGCAWKxd8MRfgdevX3HmjC2g\nWENyclX27dtOq1Zv2oSEnOPu3RRq1LCgfv2v+70UEBAQeBeCMScgICDwFvPnzyIq6iFZWVm4ubXB\nzs6+UMbN65yZO3c/e/ZooaEhpW9fXXr1qvfujj8gUqkUH5/J3Lt3BxubMkyePJ1Hjx6xfPkiMjIy\nMDIyZuLEqZiYmBIcvIXdu3egrq5O6dJlGDRoKCEhO1BTU+fw4f2MHDmuUMo1vI9cz1m1ajWJiopi\n0KA+gMIQmzTJh5iYaP78cwlqaiI0NDQYO1bhRcrriW3SxJGjR69QqtQOzMzCcHCwARTCOfPmzWLb\ntiBmzJj7RQRQvgYlS1qreCerVq0GvKnTp66uWpcvf2ipiEqVKrNjx1YSEuIZMGAIp04d5/jxo8ow\n5W+FWKyNru4rXiqFLuVoa7/JeV2w4CBLllRHIimFoeF1fHxO0737v+tvVkBA4MdEMOYEBAQE3mLq\n1JmFPqan5wDl9s6doSxfXoesLMVD/YwZF6hWLRJHx39ePPd758mTx3h7T8HJyZk5c3zYvn0rp0+f\nYM6chRgbG3P06GH++msF3t5T2LTJn23b9qChoUFaWip6evq0b98JXV1dunbt8eHJPpO3FTDd3bvi\n7q4aqmdpaUX16jXz9X3bE+vllbvVjPT0dORyORUruhAQsPWz1zlv3kx+/vmX95Y4OH36BCVLfpma\ndklJSUrvpL6+AXv27HpnW2fnyhw+fIDevftx9epljI2LoKuri66uLsnJyUilOVhYWOLsXInNmzcy\nerTXO8f6GmhpaTF4sDoLFpwgObkUlSqdZsyYOsrzISHqSCSlAHj1qiK7dt0tsCyGwIc5cGAv1arV\nxNTUFPg6iqgCAv9mBAEUAQEBga/M/fuvlYYcQEqKC9euRX27BX0BihUrrgyRbNGiFRcunOfhw0hG\njRpCnz7d2bDBj8TEREARvjdt2kQOHz6AmtqbPLMfVTPk8eM42rbdQdWqETRtuodLlwpH1MbLa9IH\njbRTp04QFfWwUOZ7m4cPHzBgQG/69OnOunW+eHj0Ja+YCLzJK/X0HMDdu3fw8OjGX3+tYNKkacpW\nFSo4UbKkwjBydq7E8+dJODt/Wc/rP2HAgIacPm3F4cPxhIS4YWFRTHlOU1Om0lZdXfZ29/8cv/02\ngrS01I/qI5VK2b9/D0lJicpjX0MRVUDg34xI/p38BSUmvv7WSxD4QpiZGQif778Y4fP9eE6fvoGn\npw4pKYoHWCurQ+zZUwZLyxLfeGWqfOpnGxcXy7BhA9m2bQ8AV65cYvv2rbx48ZxVq/zytZfJZISH\nXyU09DQXLpzF338L/v5r0dHRVakT96l8qrdKEf65HXt7ByZPnvGP+/XtG8KePb8o92vWDCQkpG2+\ndnFxsYwZMwwHh/Iq4ajXr19jxYolSKVSHBzKM3asN5qamgwdOoBhw0Zjb+9As2b1cHfvxtmzZxCL\nxcyd+wfR0U/x8hqNnp4++vp6H6xpJ/zt/nMCAs7g41OE5OTKWFicYuFCHRo3dvnWy3onX/qzPXhw\nH9u2BSGV5lC+vBNjxoxn4cJ53LlzG4kkk4YNm9C370BA4Xlr0qQ5ly5doGvXX/j99zmYmZmhra3N\nihVr6dHDHTe3NoSGnkYqzWHGjLlYW9t8sbX/GxD+dv+9mJkZfHQfwTMnICAg8JWpV8+J2bMTaNx4\nGy1aBLN4seF3Z8h9LgkJ8dy4cR2AI0cOUr58BZKTXyqP5eTk8OjRQ+RyOQkJ8VSp4srgwcNITU0l\nIyMDXV1dZS23zyEnJ+eTvVW7dm1j8eIV/8iQy8nJUW6/eKFaj/D584LrEwI8ffqEn35yJyAgGD09\nPTZvDmD27On4+MzF338LUqmUnTu3AaqKmJmZmTg5ObN+fSAuLpUJCdlJxYou1K1bn6FDR7BuXeB3\nmZuXlPSCfv124uZ2hF9/3U5q6o8hDNOjR11CQrRZunQ/e/aU+q4NuS9FXFws3br9xPjxY1i4cB53\n795m8eIV3Lp1k9mzpzNgwK+sWbOBBg0ac+jQAR4+fEBg4AaeP0/i4MF91KlTj+bN3ShdugwSiYQy\nZcrSv38vZDKZUhG1Q4fObN4c8K0vVUDgh0LImRMQEBD4Bri718bd/VuvonB428NUooQFVlYlGTly\nCFpaWshkMkaOHEvr1u0ZOXIwMpkMHR0dBg8eRsmS1owbN5KYmBhAjplZMfT19alatTrDhw8iMHAj\nRYsWJTs7mypVXLlx4xqvX7+mePHivHjxnCJFiiKVSklPT8fQ0AiZTIpUKqNGjZpcuxZB/foNCQ09\nTXh4GP7+az/orcrl999nExsbw5gxw3Bza0NERBixsbFoa2szbtxEbG3LsnbtajZs8KN8eSeMjY15\n+vQJDg7lycm5TunSS0lMnIC29mV0dPYzYsQO6tdvQKdOP6vM83Y46vr1a7CwsMTKqiSgUDvdsWMr\nXbp0U+mnqalJ7dp1AbC3d+Ty5Tc12r6TgJsCGTPmBAcO9AJEXLkiRV19E0uXdvzWy/pHODjY4uDw\n78lr/RRiYqKpW7c+d+7cIisri2HDBpKRkU54+FWOHTtMSMguHj16iI6ODkeOHCIlJRlTUzOWLl3F\nokW/ExERBsCzZwn89JM75cs74e7erkBFVAEBgX+G4JkTEBAQEPhs8nqYTExMaNu2I0WKFKFHj94c\nPHgcV9fq7Nq1je3b93LkyGnKli2HkZExr1+/Ji0tjcDAbRw7dpY1azYCcOzYEUaN+o2jR88wZ84C\nkpISadOmHbVr18XWtiwtWrTG3z+Iv/7yp1+/QWhpadG//2A2bAjCyMiInJwc1qzZQK9enp/krfrt\ntwmYmpqxbNlq4uJisbd3xN9/MwMH/srMmVOU7eRyOUuWrGTOnD9o1KgpcXGx7NgRRMuWnbG0HE3j\nxlL27duFuroagYEb882T19sml8vR11cNsXmXYaau/uZdrJqaSEVF8nuuaRcVZcibPDt1Hj36+JAi\ngW9H8eLmlChhjptbG8zMirF8+V8EB+9BXV2dTZs2MHLkWBwcHKlbtz737t3h0qULJCY+w8trNE+e\nPCY6+ikAJiamlC/vpBz3XYqoAgICH0bwzAkICAgIfDZve5iCgzcD0KRJMwBu375JlSquGBkZA9Cs\nWUvCw8NQU1OnUqUqGBoa0bdvMNeuFcXUNA0joxOEhp5i8+aNZGdno6amhomJGRUqVCQ09DT79oWg\npaXJwYP7SU19TWxsDFu2bERf3wB1dXWaNGmusr5P9VbJ5fJ31kwTiUSoq6ujpaVFXFwsu3Zto0uX\n7jx9+oTY2GNADqmpF0lM7ERi4jOeP0+iT5/uVKtWkyFDhgNvwlGdnCpy5MhBHBwc2b17BzEx0Vha\nWnHo0H4qV676j9erra3zXde0s7Z+ze3buXtySpYUnY+NGwAAIABJREFU8n5+JHR0tKlatTrjx49B\nKlWIwLx6lULFii5cvnyRc+dCqVWrDtu2BWFnV44ePXoTELCe5ctXK9UqDx8+gKam5re8DAGBfxWC\nMScgICAg8Nm87WESiRSBHzo6OsrzqgaVqnE1a9ZR9uzpBWjw+DHY2/uyadNyrK1LKQVVSpWyoVQp\nG+RyOZs2bWDJkj+YNm0WjRs3Y+XKZURHP8XXdwUJCfFoa6vmqX2ut+qfGoMaGprs3r0dd/duzJs3\nE4lEgq/vSp4/T0IkUmPVKj+uX7+Gp+cvZGZmoqurx/btQUyfPpHs7GyCgnYhk0np2rUjpUuXwd7e\nkaCgQDp37opEImHRovlkZ2cjkWTy5EkU1tY27NgRzPPnSQwY0BsLC0sCAzcWWNOuIAn4jIwMpkwZ\nT2JiIjKZFA+PfhgZGRUowNK5c9vPFqqYN68O4E9MjCFlyqQwe3bTj+ov8O2xsSlN//6DmTFjMkOG\n9EdbW5uff/6F8+fPsnnzRsqXr4izswvm5pbs2xeCTKYw+hITn6GhoUmjRk1YsuQPPD1/YeXKtwWR\nRN+1Z1lA4HtECLMUEBAQEPhs3hY8cXZWFYhwcKhAePhVUlKSkUql/P33YSpXrkqFChUJD7/K06cZ\ngAZqaskApKc7ERQUqOwfHx/HjRvXiY2N4erVy7i5tUZLS4v4+HiePn3CmTMnKVvWjm7deirru+Wi\nq6v7Wd6q3JppQJ6aaXrvNPCcnJzZuNGPnJwcoqOf4O7elRYtWiMSiVQETv74YxkikQhHxwps2bIT\nLS0txGIxqalpODiU57ffJtCqVVulx1NHR4dJk6azdu1GVq70448/5gFgbFwEI6MiLFmykmnTZhEQ\nsBU/vwAVQ04qlRb4kHzhwllMTYuxfn0gGzYEUaNGrfcKsHyuUIWFRTE2bvyJY8easmZNJ4yNv//a\nYnFxsfTq9fOHGxbA1auXGTduVCGv6NuR+x1q0qQZZmbFWbHClzVrNtCsWQtMTExwcanMihW+zJw5\nn19/HUGzZi3R19dn2LCBTJkynoyMdGrUqI21dSn8/DYhFosJDg5RvmBwcHBk6dJV3/ISBQR+OATP\nnICAgIDAZ2NtXYqdO7cyd64PNjZl6NixM9u3vymUbWpqyqBBQxk+fBByuZzatetRt259AMaNm8jM\nmXMpVWonOTnFiYlZg7V1JSASD4+uZGVloaOjw86dW7l48TwSSRYWFhaYmJiyc2cw+/fv4cWLF+zc\nuY3ixUtQooS5iuHSpElz5s2bVaC36v0ovASengOYM8cHD49u/zeopinOFmAciUSKENIKFSrSvXtn\n1NTUyMrKAkAsFnP16mWlwElcXCwGBoZERFylS5duWFpa8fhxFHfu3KJr118IDw9DJpMikUjYvHkD\n169fY9CgPkgkWVhZWfH69Wt69x7A7duvgGSaN/+JNm3q4+XlDUCzZvVo374Tly9fZPTocco1SiSZ\nTJgwjrZtW2FrW54//1zCypXLqF27Hrq6uu8VYBGEKv67mJtb4O+/RbkfHLxb5Xzec7m4u3fF3b1r\nvuP+/lt4/vwFq1adQyZTo2dPF2xsLAp/0QIC/wEEY05AQEBA4LNRV1fPJ+EfHByist+0aQuaNm2R\nr2/NmrXZs2c38+cf4OpVDWrWDGDq1MaUKKHwhsTFxeLlNeqjar3lpWJFFwICtn644VvkfVidM2dB\nvvOengNUvFNFihSla9cexMREk5j4il9/nYSv7zwiIx/Qp08//v77EPr6Brx6lQIoHo69vCayc2cw\nAC4ulTl37gzq6hpUrVqdgwenIpPJadmyNceOHcHAwABLSytycnJYsWIN/v5r8fV9iLr6bRISJvP6\ndQf09VtTu/YJ6tVrSGZmJhUqODF06EjlGtPT05kyxRs3tza4u7uTmPgaP79NnDt3Bl/fFVStWk3l\nGhUhs2+M1v+qUIVUKsXHZ/I/qgd4/vxZli1biFisjbNzJUQixX3s1q0Tq1b5YWxsjEwmo3v3Tqxe\nvU6ZR/pvRiaTsXDhYW7dUsfCIpORI2vSrdspIiI8ABEHDwYTFKSOlVXxb71UAYEfDsGYExAQEBD4\nbD43z0UkEuHl1eqjxt+37yJhYS9wdDSgU6c6yuPJySnMnHmS58+1cXUVMWRI0y+Wh5N33Nzt8eOX\ncvfufeRydcTidExMzDA0NEJHR5fr1yOQSCQFCpy4uFRmxowptGrVFmNjY1JSUkhOfkmjRk3w9V1B\n8eIlSE1NxdW1Grdv3+LcuVCyshzR0DBDLtcH1DEwqEJ4eBj16jVETU2Nhg2bKNcnl8sZP34Mv/zS\ni2bNWgKQlJSEgYEBzZu7oaenz44dwcTHx6msr1KlKl/k3v1IPHnyGG/vKTg5OTNnjg+bNwcQErKT\npUtXYWVVkpkzp7Jz5zbat/+J+fNnsWzZaiwtrZgyReElFYlEtGjhxuHDB+jSpRuXL19UKrr+F5g1\naz/LlrUCjIBsLl/+g4iIUeQqm96/78727cGMGNHyWy5TQOCHRDDmBAQEBAQ+i7fDr77G+H/9dYyZ\nMyuQmdkELa1oHj06yNixigfBgQMPcfx4H0CNgwfjEYmOMmTIlxHaOHz4pMoaHz9+wsmTfcnMdEVD\nIxpLy/6sXLkJP79VlCtnz+TJPty4cY3Jk72QSqU4OlagQ4fOAMrC6i4ulQEoW9aOly9foKGhgbm5\nJc7OLhw+fIBTp05w5MhBsrNzsLe3JzJSBogwNLyGk5NYaVRqaYnzGZvOzi6cP39Wacw9fPiAqVO9\n0dc3wNi4CGPHepOa+rrA9b0pKaDY/lQD+f79eyQlJVKrVp0PN/5O+Kf1ACtXroqFhaUylLd5czdC\nQnYC0Lp1O8aPH0OXLt3Yt283rVu3/TYX8w2IiNBGYcgBaBIXZ42a2ktkshL/P5aBvr4g4yAg8CkI\nxpyAgICAwA/H3r1SMjPLApCVZcWBAxqMHQsSiYQbNyzI1feSSktw5crXK6Kdnp5BVpYBkADsB9IR\niRoREDBY2aZq1Wr4+W3K11cs1ubYsbPK/XHjJiq3XVwqsW9fCBMmTKVMGVv69u2Js3Mlhg/vR8+e\nx2nYMI7GjTU5ePABjRvnz1HKpV+/Qfj5+fLHH/OYO3cm1avXpF69htSuXVfFi1fQ+vKGneYKVeSK\nwHyMYXf//l3u3r39UcZcTk4OGhrf7pGloHqAueGyuccK5s3xYsWKU7RoUa5cucTt27eYNm32l1ru\nd0eRIqoCRNbWUL/+frZvr0VOjpiWLY/g4dHlG61OQODHRjDmBAQEBAR+OMTinLf2swHQ0tLC1DSZ\nxMTcMzJMTDK+2rrKlStLw4aBHDumD7gBezh/vg27dl2gQ4canzyui0tlNm5ch5NTRcRibcRiMWlp\nOmzceJXevfty6NBWAgNVhWXyG1iKfXt7B5YuXUitWkepVq0m6urqhIeHERS0iefPnzNkyHAaNmxC\neno63t5jSUh4RlJSBmZmzRg8uAl2diaMHj2UChUqcvfubX7/fSkBAeu5c+cWEkkmDRs2oW/fgYCi\nvuDSpX+QkZGJlpYWixYtZ82aVWRlZXHtWjg9e3pSq1YdFi2az6NHD5FKc/D0HEDdug3Yv38PJ08e\nIzMzE5lMxrJlqz/5/n0ub9cDPHv2NCYmply/fo2goE3o6OhQuXJVSpWyIS4uVhmmeuTIIUAhRnPk\nyGnatu2Aj89k3NzaIBKJOH36BCVLlsLGpvQ3u7avwdSpNUlMXM/9+8WxsHjO1KmOVK1qx8CBd8nK\nekmlSl1RUxM8cwICn4JgzAkICAh8BbZuDaR9+58Qi7ULZbyCaoZ9DPv37+Hu3duMGjXuw42/QwYP\ntuDBgwPExNSkWLEwBg0yBRQGzNSp1vj4BJKUZISTUzyTJn29PBx1dXV8fKpz/LgmOTklefx4DwBn\nztyiQ4dPH/fu3dv8+usIxGJtlixZQGKinKNH56CjE4al5Q4aN7YlPv4poaGn0NTUpG/fgcyZswBv\n77FK8ZZx47yZPNmbZ8/iCQraia2tFZGRMSxfvogXL56zcqUfUVGPGD9+NA0bNkEsFjNkyBi6d48l\nJqYm1tZdGTq0PkuXPiYmJprJk30oX94JgAEDhmBoaIhUKmXkyCFERj7A2roUU6dOwMdnLg4OjqSn\npyMWi+nffzB3795m5MjfAFi9+k9cXaszYcJUXr9+zYABHri6Kgzf+/fv4e+/BQMDg8/4VD4PkUiU\nT61VU1OTCROmsmjRPJVwVA0NDcaNm8i4cSMRi7VxcalMbGw0uYZ0nTr1mT17Oq1aKUIsT506QZ06\n9f71xpyVVXF27eqERCJBLBYrjzs5OXzDVQkI/DsQjDkBAQGBr0Bw8BZatGhVaMbcPwlrmzdvJj//\n/As2NqWRyWQf9eY7Li4WT8+x+PkFfrjxN6BxYxcOHkwkLOwizs5lsLAokeecM40aVSQ7OxstLa2v\nvjZzc3NKlIggLi631p4EM7PPU350canCli0BdO7clbCwq7x8qQtooKNzhcTEn5DLM1izZpbSmHr4\n8AFVq1Zj4cJ5vHjxggULThMaGoJYrEGDBlWVLwEMDQ0BqFevAaAoCP3ixQtAETq4YMFC1NSSsbL6\nCw2NZyQmOnHu3HaKFzdXGnIAx44dJiRkF1KplOfPk4iKegiAiYkpDg6OgKLeH8CyZYto0KCRsu/F\ni+cJDT3F5s0bAcjOziYhIR6RSISra/VvasgBlChhzqZN21SONWtWn6pVqzFr1u94eY1i/PjJZGZm\nMn36JB49ekipUqVJSkqkZcvW2Ns70KxZff76awXHj/+NXA4GBgZcvx5BaOhpwsPD8Pdfy8yZ8z+i\nbMaPSV5DTkBAoHAQjDkBAQGBQiYjI4MpU8aTmJiITCalUaOmJCUlMnz4IIyNFcWdFyyYw507t/OF\npXXu3BY3tzaEhp5GKs1hxoy5WFvbkJKSzLRpE0lKSsTJyVklR8fbeyzPniWQlSXB3b0b7dp1BODv\nvw+hp6evrDP29OkTAgLWo69vQNmy5ZQy8z8qxYub0bKlWYHnRCLRNzHkAAwMDPH21mDx4mBev9an\nZs0YRo3q+Flj2ts7cPfubdLT0xCLxWRllURb+8b/jbmJJCauxNNzh9KYevToEWXKlKVFi1ZMmLCI\nXbu8sLZeR1xcbySSE3h7y1TG19R8813I/W4dPnwATU0ZCQnzycx0oHTpxmhqPsHSUo/bt9+8lIiN\njWHLlk2sWbMRfX19Zs+eTlZWFgW9b3hX8fJZs36nZElrlWO3bt1AR0fnc27bV2XHjmCMjIwICNjK\nw4eR9OnTHYArV+6RkZHBrl3hiESp1K1bj5CQnXh49KVu3frUqVNPWb/veye3TMiGDUHfeikCAgL/\nRzDmBAQEBAqZCxfOYmpajN9/XwJAWloq+/fvYdmy1UqPyIABv6qEpT18+IAyZcoiEokwNi6Cn18A\nO3duY/PmALy8JrFunS8uLpXp3bsf586dYe/eN2IU3t5T0NTUZOLE31i8+HeCgjbh6TmQjIwMihQp\nyvr1gTRtWhd1dXWKFSuBSCTiwYN7VKjgRExMNNOnT0IiyaROnfoEB2/hyJFTKtcjlUpZtWo54eFX\nyMrK5qef3Gnf/qevd0N/QLp2rUWXLjKysrLQ1q772ePlKlru37+H6tVrkpWVxIULB9DSisLRcS8v\nX15l3brAPMaUBIBWrdqxcWNfDAxOkJrqRnp6LSQSX54+fUzx4s4qIh5vk5aWRrlyZRg69Dpbt+5D\nQyOGBg1W0qzZGIKCljBixGCWLFnJ5csXSUlJ5uzZ0/j7ryU6+ikvX76kWbOWPH+eRJMmdejY0Z2L\nF88zevQ4RCIRGRnpyuLlRYsWZcuWTTx7Fk9iYiIZGekMGPDre0RFvk+uX49QFlcvU8YWW1s7Xrx4\nyYgRmWhoaHL58maKFr1I+/YXiI+PVvb70a5TQEDg+0LINhUQEPhPM3ToAO7cuV2oY9ra2nH58gVW\nrlxGREQ4enr6+docO3YYT88eeHr24NGjhzx69Eh5LvctfblyDsTFxQIQERFGixaKOmy1atXFwMBQ\n2T44eDM9e3bhzp3baGlpMXHiNGrWrAWgLAKdmZmJg0N5Nm4MolKlKhgbF0Eul7NkyQJ+/rk7/v5b\nKFas4IK9e/fuRl9fH1/fDfj6+rNnzy7lugTejZqaGtrahRNWCwpFy82bA6hUqQp//DGI0qV34+ho\nwsKF1dDT00dPT48XL55z/vwbRUxTU1OMjAwoWnQFKSk/kZVVFh2dmkyaNJ727duzfPlioOB6ec2b\nt+TOndvcvLmejh1jsbCwoEQJRbioRCIhIyODnJwckpISKVHCnFmzpmFsXIS6desTFxfDuXNn8PGZ\ng0Qi4cSJo+jq6uLgUB6xWExU1CPatGmOlZUVM2fOJzY2mhs3riOV5mBjU5qaNWshEn16+YNvxduG\n2dWr94mKak7uu/MXL6pz61aSSsH1s2fPIJFkfs1lfhYymYx582bRs2cXRo8eikQiUfkdTU5Oxt29\nHaDIzfX2HsOoUb/i7t6O7duDCAzciKfnLwwc2IdXr14BEBKyk/79e9G7d3cmTRqnvB+zZk1j8eIF\nDB7sSZcu7Tlx4ui3uWgBge8YwTMnICDwn+ZLPDCWLGmNn98mzp07g6/vCqVBlUvBYWkS5fnc8Ed1\ndTWVh76C3uBfvXqZK1cuMW/eIsaPH41MJuPWrVuUL++Empqa8to0NDQwMysGgL29IxERYVhYWHDz\n5nXmzl0IQLNmLfjzzyX55rh06TyRkQ+UD1JpaWlERz/F3Nzic26TwEfytqKloaEBrVo1o2JFZ8qV\ns6d7904UK1YCZ2cXlX4DB/Zg0aJVFCkSjqHheSZN6oazc1nMzAxITHydb57c2nlGRsasWuWnPJ6T\nk0P37p0wMjLCycmZMmVsuXPnNteuhdOqVTvu3bvDxInTAMULgPDwMIYNG4W6ujrBwSHK76LiXxHj\nxk1Q1rsbPdqL0aOHUrt2PSwsSnHp0i0aNWqKm1ubL3AnvwwVK7pw7NjfVKniyqNHD3n48AGdOvVC\nX//NyyJ19QTMzMSAQqpfV1eXI0cOKcVt/ikfmwNbmDx9+oRp02bj5TWRKVO8OXny2Ht/Rx89esi6\ndYFIJBJ+/rk9Q4aMwM9vE8uWLeTgwX106dKNhg0bK8PDfX1Xsnfvbjp1+hmgQHEeAQGBNwjGnICA\nwH+CuLhYxowZhoNDee7du4ONTRkmT56u0mbBgrn55NWvXLnEtm1BSkXAS5fOs3PndmbP/p2LF8/j\n5/cXWVlZWFpaMWHCVHR0dOjYsRVNmjTn6tXLuLpW5969u+jq6pGWloahoRFpaWloa+uoeFIqV676\n3vW7uFThyJGDeHj05dy5UF6/VrzRTk9Pw8DAAFvbskyfPochQ/qyd+9OXr9WDZ/T0NAkPPwqr16l\nIJfLiI2NUQpT/BNGjx5HtWo1/3F7gcKnatVqHD9+Trm/efMO5faECVPf2e/69Qh+/dWT1q2bfdb8\neUM9K1Z0wda2LFevXiImJhpzc3Pu3s3r4ZZ/VPHy3BcgY8cuY+3abaSm1qJ06ccEBjajSBHjz1r3\nl6AgT2Z2dhY3b16jR48uZGdnoampRaVK5enZcz9//51J6dK9KVo0iitXpBgZGf+/rxopKcl07Nia\ncuXKsWrVunf+rnTu3JYmTZpz6dIFfvnFgyZNPu/z/FTMzS0pW9YOUORyfshLX7myKzo6Oujo6KCv\nb0CdOorSGWXKlCUy8j4AkZEP8PVdSVpaKunpGdSooYgsEIlEBYrzCAgIvEEIsxQQEPjP8PTpE376\nyZ2AgGD09PTYsUNVoW7AgCGsWbOB9es3Ex5+VakI+ORJFCkpyQDs27eHNm3ak5yczIYNfixZsgI/\nvwDs7R0IClIUWpZKczhy5CByuYywsCv07t2Pdu06MGbMMEaMGIydXTmlJ2X69Mn5PClvePO229Oz\nPxERYfTs2YVTp05QooQ5ADVq1EYqldK160/4+6/F2bkSTZq04N69u6ojiUR4eg5g4MA+rFmzCn19\nA0QiERUqVOT4cYXH7e+/Dxe4iurVa7FjxzZychS13Z48eUxm5o8TFvZfZevWUFq2bEdo6BWaN3cr\nlDHzhnq6uFRm167tlCtnj6NjBcLDr5KSkoxUKuXvvw9TqVKVd47Tr98gDAwM+eOPeQAkJSWRmJjI\noUMdeP58ONraj7lypQ9Ll54plHUXNrneS3NzC/z9twBQpUo1LCysCAjYir6+ATk52ZiYmFKqFIwb\nN57Dh2dw4MAegoP3oK9vwMOHDxgxYgzm5hbs2rWfVavWvfd3RSQSYWRkjJ9fwDcz5AAV4SQ1NXWk\nUinq6urIZIoogrxRBvnbqyn3RSKRMvJg9uzpjBkzHn//LXh69lcZoyBxHgEBgTcInjkBAYH/DMWK\nFcfJyRmAFi1aERy8ReX82/LqeRUBDx3aj5tbW27evMGUKTM4dy6UqKiHDBrkCUB2dg4VKyrGFou1\nWb78L4oXfyOXb2/voAwbgnd7UoKDQ5TbDg6OLF26CgBDQyMWLlxeYJ8FC5Zy8eJ5/vxzCWpqIk6d\nOs6YMeNJTX0TQicSiWjVqi2tWrXlxImjnD17hpEjfyM6+ik+PpPZuHEd1avXRF8/f35f27YdiIuL\npW/fHsjlcooUKcrs2b+/+0YLfHP+/PMIc+dWRSI5jkj0gmnT9jNrVvvPHreg4uUuLpUxMTFl0KCh\nDB8+CLn8nxUvHzlyLLNnT2fFiqW4ulZn4cL5mJmpI5MZ8OzZNEBEdvaP85hSqlQpzp49Ta9eXYmL\ni6FWrTo8eHCfa9fCGTnyN44f/5tdu3YQG5uCRJLKypW7mT9/tMoYN29ef+fvCvBNjbj3YW5uwd27\nt3F0rPBJeW0ZGekULWpCTk4Ohw7tf2f+roCAQH5+nF9JAQEBgc8k70OlXC5X2X9fHlurVu3w8hqF\nlpYWjRs3VeaquLrWYNq0WQXO9bUl1atXr0n16qphkMuWrQYgMDCUnBxv6tU7QqtWWXh7t1bmnZiZ\nmfHXX+sBRSmDp0+fAIqHsz179pCY+BqRSMTAgb8ycOCvX++CBD6LY8dESCSlAJDLi3LqlF6hjPu+\nUM+mTVvQtGmLfH1yvVi5BAe/UWLN+1Jj06ZgevYM4u+/+wBaWFoeonNn20JZ99fA0NAIZ+fK1KtX\nn5SUFJUwVLFYzJYtm9DQ+Inw8IEULz6JXbvKYGGR3xv+Pf2uFMTbxrlIJKJbtx5MnuxNSMhOatWq\nS67Bnj+XTjU8Nfdcv36DGDCgN8bGxlSo4ER6enqB8/1ogjgCAl8DwZgTEBD4z5CQEM+NG9dxcqrI\nkSMHcXZ2ITT0NHK5/L15bKamppiamuLvrwh/Aihf3omFC+cRExONpaUVGRkZJCUl5quV9a2JjHyM\nj48+L14ocpMePYrBzi6Uzp3rAHDnzh0WLZqPXC7HwMAAb+8pKv3XrDlBQIAEmUxEp04iRoz4Pj0D\nAqro6KiGuunqSt7R8tsSFRXL1KmXePZMFyenVFavbouf325SU6F9ezucnH4cYw7ehKFOmDCVMmVs\nWbp0IY6O5f//+6JNRIQ56uov0dM7RXp6Da5ckaKrq6vMp/0Svytv14YLDNxIZmYGBgaG7N69A3V1\ndWxsSjN9+uwPjpU3rBSgW7ceym1//83K7f79BwPg5tZGRcQmryGf91yHDp3p0KFzvvnejmB4+8WA\ngICAYMwJCAj8h7C2LsXOnVuZO9cHG5sydOzYmdDQ04hEIpU8toIUAZs1a0lKSgrW1jYAFClShIkT\npzFt2gSysrIBRc7d92bMXb8exYsXDZX7WVmW3L//RrrexaUS69cHFtj3woWbzJ1bilevFGFef/wR\nSfnyl2nWzPWLrvlH4F3Fk9euXY2LS2VcXasX2O/06ROULFkKG5vSX3R9o0bZERUVzL17VbCyus3I\nkeZfdL5PZdSoC4SG9gLgyhUJuro7mD79x1GwfJt3haGWLWtHuXIOREb+TokSu8nIqArIKVo0gwYN\nOjJmzDDMzIqxZMnKd/6uyGQypk6dwMuXL5HJpHh49MPS0orlyxeRkZGBkZExEydOxcTElK1btxIY\nuJns7BxMTEyRyd4Uic/1bm3a5M+2bXvQ0NAgLS31W9yud3L0aDhr1sQjlYpwdzfA3b32t16SgMB3\ni0j+nWSTFiSPLPDv4F3y1wL/Dn6Uz/ddD9//lIUL52Fv70jr1u0KPJ+c/BKZTEbRoiafs8xCJyEh\nkVat7vL0qUIAw8DgJmvWvKRRo3eJrrxh69YTDB3ahryhUZMmbWP48PyhdP81PvX7NGvWNOrUqfdR\n8uq5AhMfS1paGlFRTyhZ0kJZrD4v3/pvVyaTUbnyceLiOiiPNW++nYCA5t9sTV+aU6duMmXKQxIT\ni+LgEMeqVY0wM3v3b4ZcLufRo0doamoSGXmPCxfO4+U1EYC0tFTGjh3O3LkLMTIy5ujRw1y8eB5v\n7yloakrJzlZ8ZxYtms+JE8fYvfsgAJs3B5CRkc7NmzfQ0dGhfv2G1KvX8LsI4QR49Cia9u3jiY9v\nBICxcRj+/hJq1arwjVf2/fCt/3YFvhxmZgYf3UfwzAkICPxn+NR8C0/PHujq6jJ8+JgCz0+cuJut\nWy0ANdq1O86CBZ2+m9yO4sXNWLQogdWrt5KdrUa7djo0alT/H/Vt0qTi/9g784CcsjeOf963fZUt\nS0mpVKRIYxiyJcY2zNjXMBh+1rGHoiyhLNmXlLJkZDD2nRFZxhZmZEmJNor29X17f3+8ekkhS9b7\n+Wfee+65555z79Wc55zn+T5UqhRCQoK8ftmyF2nU6PPaefyUFCRPvnEjjIoV9fH0XIi3t6fCWFu1\nahlnzoSgpKREgwYNadasBWfOhHD16hUCAtYze/YCMjMz8PKSJ9Y2MDDExcUNHR0dRo4cSs2aFly7\nFkbjxg7s37+XoKA/FbsoAwb0YevWHa818rStySyMAAAgAElEQVS0tKhdu+TpJ96XtzVwxWIxRkap\nxMUVlORhZJRN164d8fPbVKwB+qXTtGltTpyoRU5OzhsTykulUoYO3caBAw1RUsqgc+co4uPPs2rV\nMn74wQEdHW3u3Ytg7Nj/AfLvsXz5igDcvn0bL6+FZGSkk56eVkgdsiAht7e3D1euXMLffx2enh60\naOGIm9vsdx7b+vVrqFvXjvr1v2PkyKGMHPn7W6U/KeDUqRvEx/+iOE5OrsfZs8GCMScg8AoEY05A\nQOCb4OVYj7fBz2/TK88dPHgOf/8WSCQGAGzebM0PP4TQpUvJDKaPQdOm1jRtav3W11laGrNw4X38\n/ILJz4eePXX57rsv390pPPwmBw/uY+zYCe/VzuuSJ6ekJBMScpItW/4E5LsoWlraNGnSlMaNHWjW\nrCUAzs49GTduMra29Vi/fg3+/msZPXo8IpEIiUSCr28gIDeUzp49jYNDc44ePUzz5i3fabfuc8Pb\n244ZMzbz+LEmtWql4ObWnn79fD91t0oVkUj0RkMOIDDwBHv29AG0kEjgzz8r4etrhrJyJuvWrcTO\nzh4TE9NCid0LmDJlCnPnLsTU1Iy9e/9i0aL5pKamoK6uQWjoab7/vhEJCfHY2dnj7e2Jjo4u48dP\nea9x/frrb4XG+K4LWnXr1qBMmaukpMhjltXVI7GyKvdefRMQ+JoRjDkBAQGB9+Dhw2QkkufxSPn5\nFYiPz/iEPfqwtG5tR+uvzOvN0tLqnXYMXuZ1yZO1tXVQVVXD09ODH35woHFjB8W5guiG9PR00tPT\nsbWtB8CPP7bH1fX5hNrR8fmD79ixM1u2BOLg0JwDB/YyefL09+5/aSCVSvHwcOX27XCMjWvg6urO\n9evXWLnSB6lUiqVlLSZMcEFFRYWLFy+wcqUPampSWrV6Xl5ATk42U6dOokWLlrRq9SOurpN5/Pix\nIl7sc5Xp/1CkpEiA5yqkMpmYpKRM+vVri5aWNrt2bSc5OVkh6iSRSHjwIBoTkxpkZj6X+j969BBm\nZuYMGeJMxYr6GBubKN5TVNQ9UlNTKVeuPLt2/UlIyN/k5uagpqaGi8sMjIyqs3//HkJCTpKdnc3D\nhw/o2bMPOTm5HD16EBUVVby8fNDV1S3iQiyTydi3bzcREXcUXg27d+/k/v1IRo0aV9yQAbC1rcm0\naX8TELAdqVRM5875tG0ruHYLCLwKwZgTEBAQeA86dKjP+vW7iYiQx/0YGe2jXTubN1wlUBpkZWXh\n5jal0IS/atWq+PgsJDs7GxUVFXx8VhEe/h9bt25mwQK5cMTixQuIjLyHVCph0KChNGnSjP3793D6\n9ClycnKIiXlI06bN+d//RhMXF8v//jeYrKws0tPT+OmnNkyf7s6OHcE8eZKEqqoaly79w44dwSgr\nK3P16mUSEuLZsWMbU6a4cvbsGa5evUxgoD/Dho0C4PLli/j5rUVNTZ2oqHt4eLgCoK6uUchVMykp\nkcuXLyKVSjExqfEpH/UriY6+j4uLG9bWNnh6ehAUtIndu3eydOlqDA2rMXv2DHbu3E6nTr8wd657\nkfLu3XsBkJmZiZubC23bdqBNm3acPHmMChX08fLyAfjsBDtKg06d6hAUtJvIyJ8AGRYWfhw+fJbj\nxzeirKzChAkuiMVifHy8SU9PRyqV0KNHb0xMajBmzJgiUv8FypDx8XEkJaUwcOAQVFRU6NbtJ9av\n34iysjI9e/ZFSUmJf/45z9q1K5g9ewEAkZH3WL58LYcOHWDt2pV06NCZChX0qVatGgcP7qN7915F\nduNEIhEtWzoRGOjHiBFj2bDBl5Mnj+Hu7vnGsQ8Y0IwBA0rjqQoIfH0IxpyAgIDAe1C5ckX8/U3x\n9d1Gfj4MGGCGiYnBp+7WN8n586FFJvwDB/bBw2MelpZWZGZmoqamVuiawEA/7O0bMHXqDNLS0hg6\n1Bl7++8BuHv3Nhs2bEFZWYXevbvQrVtPUlJSePz4EdOnu7N5cwDKysocO3YYA4NqVKhQkcTEx5ia\nmhMe/i9r1mxAW1uHwYP7k5aWQrly5WjevCVWVrWpU8cWd/fp6OjoEhFxl7t3b9O+/U9Ur27Mf//d\nICcnh/T0tEKumoGB6/HwcGXAgMEf98G+Bfr6lbC2li9mtGnTjg0bfKla1QBDw2qAXI5+x45t2NnZ\nF1vevXsvZDIZU6aMp0+f/jg5yVNqmJqas2KFjyJezNa27qcZ4EfExMSADRty2bRpG0pK+fzvf32p\nXPn3IvWWL19bpKxXr160alVUFdTb+xCrVlUhPb0ijRr9ycaN7RXn0tLSmDVrBjExDxCJREilUsW5\nevXsycvL49ChfWhr62BtbcODB9HUqGFGRMSdV45BQ0MDO7vvOHMmBFvbemzZEkiNGoXTTbxrfN3+\n/Xu4desmv/8+6a2uExD42hCMOQEBAYH3xNLSGG9v40/djW+elyf82tralC9fQTFJ1NTULHLNhQvn\nOHPmFEFBGwHIy8sjISEekUhE/foN0NSUu7kZG5sQFxdLVFQUmpqa2NrWIyhoI6am5tjbN2DlyqXk\n5GSTnp7OkydJ5Ofn06dPV0BEfr6UIUOGk5cnISbmIXv37kYsFgEiVq/2w919GlJpPnFxcUydOoPV\nq5dz5colNDW1CrlqtmnTnoAAP5ycPl+Xsxd3ZmQyGdraOqSmphQqK44Xy0UiETY2tpw7F6ow5qpV\nM8LPbzNnz55m3bqV2Ns3+KyN2g+FlZUJc+aUPI3F7t0X2Lz5CaqqKnTrpsNPPz1PkZGQkMDq1fqk\npclzTIaGmuHjsw2QP39f39XY23+Hp6c38fFxjBr1PAZOVVWF1auXERPzEKlUysaNfmhpabNz53Zi\nYh6QnJyMsrJ8ShkefpM7d27h4TGdSpWq0K1bT/7660/u349CVVW+mNK1a0ccHVvzzz/nyc3Nfaf4\nus9FZEpA4FMj/tQdEBAQEBD4Nrh8+SKTJhXdWfhQFEz4TU3NWLduJX//fbxE182Z44W//xb8/bew\nffseqlc3BuQT2ALEYiWkUikikQixWKwQ1BGLxaioqFCuXHlmz16AsbEJpqbmbNoUzPHjoRw/foaT\nJ8/Rp48zf/yxGTMzc06cCOXw4VNIJHmYm9dk3LjJ1KtXn7lzvdDW1kZJSUzfvs5YWdVi3boAmjd3\nJDQ0hIkTx9CiRSu0tLRL4/F9EBIS4rlx4zoAR44cxNLSiri4WGJiHgJw6NB+6tWrj5FR9WLLCxg8\neBg6OrosXDgfgMTERFRVVWndui29evXj1q3wjzyyz59r1+4wZYoWJ05049ChzkyZos21a893zVJT\nU8nIqPDCFWKysp6v6WdkZFChglwNc9++3UXaHz58NAYGhlSsqM+gQb9x584tnJx+xMmpLbGxMTx5\nkoRUKmXJEi9MTExxdZ1F+/YdOXnyGI8ePSIhIR5VVVU8PFxJTHzMuXNnWLXKl7Jlyyru4e09j8GD\n+9OvX3fWr1+jKL9581+GDx/EgAG9GTp0AJmZmYUWAEJDTzNs2KBCCwcCAt8KgjEnICAgIPBV8PKE\n/+bNf3nyJInw8P8AyMzMKOQ6BtCgQUO2b3+ucnr7ttxIKG4HSSQSUbOmBVlZ2Qqxk9zcXEU7+/fv\nUfz29V1VpM3MzAxFHsKDB/cVSuRcHAVxeeXKGfLvv7FERt7D2fnXkj+Qj4xIJMLIqDo7d26jb99u\npKen06NHH6ZOnYGr62ScnXuipKRE585dUVVVLbZ8/vzZSCQSAMaOnUBOTjYrVy7l3r27DB06gIED\ne7Nhg+8H35ULDt5K377daNu2JZs3BwByqf2goFcr2X5unDlzl8TEhorjxMTvOXPmruK4Ro0aNG58\nBpD/G9DXD6FDByNAHuvWu3d/Vq9ezqBBfZ59m/Kdr4JYuOf/JuS/raxqo6uri1gswsysJllZWTx+\n/JjIyAju3r2Nu/s0AgP9ePz4MS1btqJMmbI8eZLEL790o2JFfapXN2HHju2FxjB06P/w9Q1kw4Yg\nrl69TETEXfLy8pgxYypjxkxkw4YtLFmyEjU1NcXO3N9/n2Dz5gC8vZd+leksBATehOBmKSAgIPCN\nUJxAiIGBIcuXy4VAypTRY9q0GZQvX4GHDx8wceICHj9OQiwWM3v2fKpWNWDFCh/Onw9FJBLRv/+v\nODo6KQQ89PTKEhkZgYWFFW5uswA4dy6UZcsWoaamjo1N6cY53bt3lxUrfBCLRQqBCJksn8WLvRR5\nvRYvXvFsciq/ZsCAwSxduhBn557k5+dTtaoB8+cvfqW0uq6uLvr6+kybNpH8fBnJyU9p0cKRAQMG\nM2+eBw8fPuDUqRNkZWUVafPnn7sxbdokDh7cz/ffN0JD47nbZ3EeY5mZGYwaNYKoqCzy8rR5+tSV\nRYsusHjx55nrr3LlKmzevL1Ief363+Hnt7lE5S+rdBaIdoDcSC4tdu3ajo/PKsXOFHx5bnx16lRF\nSyucjAxLALS0wrG2fq60q6SkREBAR5YuDSYjQ4n27avTqJEVwcF/AWBtXYegoB2K+kOGDAfk8Yxt\n23ZQLGAEB//F5csXUVFRVZxbvHgBHTp0wsLCkhMnjhZJlzBp0u8YGFQlLy9HEVPp5NSG/fv3Fqp3\n/Phhdu/ehVQqJSkpkaioewDFukvLZDIuXbpIePhNFi9eUawbtYDAt4BgzAkICAh8IxQnEDJhwmjm\nzVtEmTJ6HDt2mLVrV+Li4oa7+3RGjvwftrbfk5eXR36+lJMnj3H37m0CAraSnPyUwYP7U7euXFb/\n7t3bbNoUTPnyFRg+/FeuXw+jZk1LFiyYw7JlazAwMMTNzaVYo+VD0aBBw2In/GvW+Bc6rlevvsKl\nT01NjYkTpxa5pmCSWsCCBYsVv4ODi7qgAcyYMee1/TM0rEZAQJDiePhwuZqlnZ09dnb2ivLRo8cT\nEXGP3NxcKlUayNGj3RTn9u49iZvbE8qW/fR5t+LiYhk/fhTW1jZcvx6GpWUt2rbtgJ/fWpKTk5kx\nYxahoafR1NSiV6++APTr1x0vr6WUKVOm0MLCgAFDaNmyVSExjHPnQlmzZgUJCSmAFr/9No5Onb7/\noGNwcnKgdeu2xMQ8pE+fbvz661BiYh6SnJzM5csXqV27Dr169WXkyKFYWFgSFnaVrKxMpk93JzDQ\nn8jIezg6OikMn09Jkya2TJ58nKCgf1FRUaJLFxEODi0L1dHS0sLFpf0rWng9mpqaZGZmvraOkZEx\nyclPuXHjOmZm5vj47OXo0dXUqVMbPb1yQJSirkxW2GCOjY1h69bN+PpuRFtbm7lz3Z/F0xV/L5FI\nhIGBAXFxsURH3/8g6UYEBL5EBGNOQEBA4BvhZYEQHR1t7t2LYOzY/yGVSklNTaVGDTPOnj1DRMQd\nWrVqxePHac9yf6lw/XoYeXm53L8fhbGxCXXr2nHz5n9oaWlhZVVbsathZlaTAwf2sn37H1StaoCB\ngSEArVu3ZffunZ/wCZQOoaFh3LwZR8uW1piYGL5XW9nZ2Tg77+TUqSaoqz/B2PhWofNKSnmfVbLw\nmJiHzJ69ABcXNwYP7s+xY4dZvdqP06f/JjDQH3PzmoXqyyfvsmIXFgrOi0Qinj59yoIFc9DQ+ImL\nF0cjFmczZkwMOTmhdO/+IRPXi5g4cSoXLpxj/fqNnDkTQnZ2Nrdu3eTnn7sqdntEIhEqKqr4+gYS\nHLyVKVPG4++/GR0dXXr06EyPHn3Q1dX9gP16N4YNa8mwYVCxog6PH6d90LbLlNGjTh1b+vfvgZqa\nmsJl+EWUlZWZNWs+ixcv4L//YsjIKMPTpxNITVWlUaMdpKQkK2IqT548ho2NLWfOhCCTycjIyEBd\nXQMtLS2ePEni3LnQZ/GVxiQlJRIe/h+WlrXIzMxATU0dmUxG5cpVGDFiDFOnTmLWrHmfbcoOAYHS\nRDDmBAQEBL4RXlYEtLOzx8TElNWr/YiLi2Xy5N9ZtGgZZ8+efmUbbdt2xNj4ubpewcq6svJzsRAl\nJTESSV4xVxevZPgl4+NzhMWLrcjMbIyBwVGWLk3GwcH6ndtbufIEJ04MBFTIyIC7d1UwMlpOdPRv\nqKo+oFevxM8qLqhKFQOF1LyJSQ3s7Rs8+21KfHxsEWNOjui1qQZkMhn//nsda2sbgoPrAKrk56uS\nmanL0aP/0b37hx+HRCJh+PBf6dPHmZCQv8nJyWbXrj9p3tyRmJiH3L17h6SkRK5fD6NDh5+oUcNU\nYcxUrWpAQkL8Z2HMlTYzZswutvzF9ADm5jXp128EnTrVAOSLG6mp0KpVAkZG8ezcuQ01NTVyc3P5\n+eeunDkTgkgkwty8JjVrWtC7dxf09StjY2MLyA1EDw/PV7hLizAyMmbGjFm4uk5hwYLFVK0qpIYR\n+LYQjDkBAQGBb4TExER0dHRo3botWlra7Nq1neRk+Up5cPCWZ65m8t0IZWUVevbsyZMnTzE3r4mL\nixs2NvWYN28WNWqYUblyFQ4fPoCGhib//HOOsmXLsW/fbjZt2kB6ejrVqxtjbFyDGzeuERPzEAMD\nQ44cOfSpH8EHRSaTERQkIzNT7t4VE+OEn9+29zLmMjLEwHPDOCenCh4e1XnyZDfVqpWnWbOiucM+\nJYUVP8XPdnHlv6VSKUpKSshkz4VeCgRj3pRqQCQSoaSkhI5OGklJBaUytLSyS31MjRs7cOvWTZo3\nd0RTU5MFC+ZQrZoR48dPQSKRsHChJxUq6Bfq65vEbL41dHQ0UVFJJU+xpiOlTJkydO8+kfPnn9Kw\nYROGD2+FkpISy5Y9V618MUbyRSwtaxVxl37RFdrc3IJNm7aVxlAEBD57BGNOQEBA4BuhOIEQsViM\nj483T58+JT8/n169+mJgUI3Jk39/NqFW5vTpEM6ePUPz5o74+Hgzc+ZU1NXVAahf3x5HRycCA/3w\n81uLn98mfH1X888/5zEzM2fSpGlMmjQWNTV1bG3rERv78BM/hQ+LVFpYFDo///2CAn/6qQZ//nmM\n2FhHIJ8GDQ7i6NipSLLzL4UqVapy5kwIALduhStENF5eWHhRCl8kElG7dh0WLpzHr79+z8qV+0hK\nKouNzS0mT25W6n0uUG2UyWTk5uZy48Y1xGKx4rtPTU0rZMwJFMXa2oLevXewebMyEkk5vv9+F3p6\neoweXZ2srFZAOnfvBrNkSdd3av/gwUv88UciSkoyfv1VLuQiIPCtIhhzAgIC3zzp6ekcOXKQn3/u\nyuXLF9m6dXMhwYuvhVcJhCxfvlbhZtmhQ2eF8MOmTYE8fpyGt/c8cnPlS+wGBoaMHPk7165dxcfH\nm/nz59Cv3wB+/rkbISEnKVNGj/Hjp7B9+1aio6Np0KBhIYXD+Pg4jhw5qEgG/SUjEon45ZdcVq6M\nJifHiIoVz9Knz/tN8m1tzfH1vcXOncGoqkoYO7bNZ23Ivaz4WHD86FECDx8+oFmzlhw8uI9+/bpT\nq5Y11apVB15eWFBmwoTCIjR6enpMmjSNKVPGYW1tiK5uGZYtW6NITF1a43hRxbTgv9raOhgZVWfk\nyN+xsLDkypVLbN365aQs+FR4ef1C9+7XSUm5j4NDZ3799W+yssyfndXm9OmyyGSyt1YNvXz5NuPG\nqZGYKDcEr1w5wF9/xWNoWPkDj+DjUvA3ODDwj0/dFYEvDMGYExAQ+OZJS0tl585gfv753VaJv0ZU\nVFQVv5WUxEilkkLnd+3ajrq6BgcPngAgJORkodxsf/8dzqVLiWzZcoxOnbJwde0IyBXrjhw59FUY\ncwAuLu2wtT3H3bvnad7cFBubd3exLMDe3gJ7e4sP0LvSpSBxegEvusjp61fC0LAaampqLFq0vMi1\nlStXLnZh4UWXu4YNf6BiRX1WrVpf6nGCK1asY/Lk32nbtgN169oxefLvDBo0FICzZ0/zyy/dsLCw\nRCaToaOjy/z5zxd7XuyzQGG++66O4remZm6hc5qaOe+U/uHUqUiFIQfw4EFrTpzYSb9+X7Yx97ZI\nJJJSW9wQ+LIQvgIBAYFvntWrlxET85CBA3ujrKyMuroG06dPLpIzLTz8ZrE52b4GSiI7XkBg4Hpi\nY2OQSCRs2yaPtevXbxAeHq6IRCL+/fc/7t1LJiurPioqAezencfVq+vZuHELq1cvJzo6ioEDe9O2\nbUe6d+9VyiMrfdq1K738Z18yUqkUDw9Xbt8Ox9i4Bq6u7ly/fo2VK32QSqVYWtZiwgQXVFRUuHjx\nAitX+pCXl0dmpi6amr9gZSVTLBDk5GQzdeokWrRoSYcOnT9YH180Jl71281tNt7e8wgI8OPJk1SS\nk2uTl9caJ6cM3N07fnH56D4V48ZZc/t2EP/99x2VKt1l9OiiapglwcREGxWVGPLy5EInWlo3sbJ6\nPxXZklCQisPSslaJvumuXTvSsqUT58+HoqqqxsyZczAwMGTOnJk0buxA8+aOgDw9xpEjIUXuNXv2\nDLKysgAYN24S1tY2XL58EV/f1VSoUI47d+4Wygso8O0iGHMCAgLfPMOHjyYy8h7+/lu4cuUSLi7j\nC+VMu3btKrVqWbNkiRfz5xfNyfY1UBLZ8QL69/+V27dvkZycjI6OXMGvQoUKmJnV5MSJY+jolCc9\nvRWammeIj19IdnY9unffjJqaGsOHjyIoaNNX6cYqUJjo6Pu4uLhhbW2Dp6cHQUGb2L17J0uXrsbQ\nsBqzZ89g587tdOr0C3PnurN06WpmzTrP1av/kZOTzaFDXbGxWUxmZiZubi60bduBNm3afdA+Hj78\nN1B4l/HF3zk5OYjFYry8lhATE0fr1gkkJcnj9qKiHmFhcYo+fUo/ju9rwNLSmAMHKnHvXhRVq1q9\nc67ETp0aExa2l7/+0kBZWUq/fmLs7Vt94N4Wz4MH0UydOuON33T37r0QiUTo6OgQELCVgwf34eOz\nkAULFhdj/BddDChXrhyLF69AVVWVBw+icXefjq9vIAB37txiyZJ9qKp+/eqpAiVDMOYEBAS+eV50\nD5TJZEVypsXHx6GtrU1kpDwnG0B+fj7ly1f8JP0tLUoiO/6iS9n27bsV4haJiYloa+syZsx46tVr\nSKdOEaSkVKNiRU/y8+tQv37DZ8qGX196AoHi0devhLW1DQBt2rRjwwZfqlY1wNCwGiBXI9yxYxt2\ndvaK8n//vUVq6i/o6W0hOdmZ3FyYMmU8ffr0/+iuufv3X8bdPYn4eEOsrc/Qu7cGSUnPjQapVJ97\n97I+ap++dDQ0NKhd+/3FStzcOjB9en6hGMePQUm/6QKPg1at2ij+u2zZohLfJy9PwuLF87l79w5i\nsZiHDx8ozllZ1cbAwOCD5xEU+HIRv7mKgICAwLdF0XgxKSDPneXvvwV//y0EBGxl0aJlJWpv+PBB\nwHPxj6+N8PBYmjaN4OBBNRYtukl+vozly8uhr7+MmjWb0bFjBitXziE6OqpE7W3btoWcnNKXoBco\nXV6cZMtkMrS1dQqdL86wr1ixsHGkpAQ2NracOxdaOp18DQsWxBEZ+QtZWQ34558BnDiRjInJ8xyM\n2tr/8f33gqrlp0IsFn90F9eSfNOv6lNBuZKSEvn58m8/Pz+/2Jycf/yxmfLlKxAQsBVf342KlB4A\n6uoa7z0Oga8LwZgTEBD45ilJvJiRkTHJyU+5ceM6IA8+j4y8V6L2V63yA56Lf3xN5Ofnc+2ahMTE\nVujohBIdbc/ChRdp1MgKNTUxQUH98fCYhqVlLaKj76OlpU1mZsZr2wwO3kp29rdrzK1fv4agoM9T\nLTEk5CRRUZElqpuQEK/493LkyEEsLa2Ii4slJkaenuLQof3Uq1cfI6PqivKZM60wMVmEsrImjRtv\nQE9PjcGDh6Gjo8vChfNLbVwvk5+fT0qKeqGynJwKLF9elbZtt+Lo+CceHpG0bm1XKvd3cnIotnzO\nnJmcPHmsVO4p8GZK8k3Xrfv8mzh27LDivwU7epUrV+HWrZsAnD59ComksLgUQGZmhsLV/eDBfUIe\nQ4HXIrhZCggIfPOUJF5MWVmZWbPm4+PjTXp6OlKphB49emNiUuON7RcEuL8s/mFv3wBPT3ckEgn5\n+TLmzFmgcNf5/BEpkiXn5akWKs/KkkvpSyQS+vfvgUwGKSnJxMY+RCKRIBKJGDCgN61atSEs7DKP\nHz8mP1+Ks/Ngnj5NIjHxMaNHD0NPryw+Pqs+zfA+IR9rt6Fgx/ltOHXqJI0bO2BsbPLaeiKRCCOj\n6uzcuY158zwwNq5Bjx59qF27Dq6uk5FKpVhZ1aZz564oKyszdeoMRfmPP9Zm7NgJqKur062bPE5o\n7NgJzJ3rzsqVS/nf/0a/03jfBrFYjL39E2JicgFVVFWjcXBQ5rvvLAkIsCz1+xcXRwV8dLdCgcKU\n9JsuIC0tDWfnXqiqqjJz5hwAfvrpZ6ZMGc+AAb35/vtGaGhoKuoXvNuff+7GtGmTOHhwfzF1PtJg\nBb4YRLLPJIBB8P39eqlYUUd4v18xwvt9M05OTTly5BRXrlwqJP6xZIkXtWrVoXXrH5FIJEil0o+e\nU6xAoc3a2obr18OwtKxF27Yd8PdfS1paKtOmuWNgYIinpwexsbGoq6szadI0TE3NSElJZubMaVy7\nFsGjRw5oal4gKWk2Xl4ZaGo+Yc6cmZiammFlVZv//W802to6tGrVBGVlFSpXrkKLFo74+a2lTx9n\nQkNPo6ysjJfXEoYOHcD69RtLXY7+cyIgYD0HD+6jbNly6OtXwsLCCnv77/Dy8iQnJwcDA0NcXNyQ\nSPKYMGEM69dv5M6d2wwa1Ic//9yLvn4levToTGDgVry956Glpc2pUyd48iQJAwNDzM0tsLCwIjQ0\nBHPzmly7FsbPP3fC1LRWsQqtu3fvZM+eneTlSTA0NMTV1YPbt28xefI4tLS00dbWYvbsBRgYlL6K\n4Idg/fo1aGpq0atX3xJfk5OTg4uLL8nJqjRvbkH//k2LKBF+CLZu3cT+/XsA6NChM92791L8zZDJ\nZCxevICLFy+gr18JFRUV2rf/6Y33F/HSUSIAACAASURBVP4uf3jeNg9ct24/fZC/YzKZjKioKJSU\nlDAyMgKE9/s1U7GizpsrvYTgZikgICBQAq5eDSco6ChxcY/euY2X185q167Dxo1+bN4cQHx83CdL\nDh0T85CePfuyZcufREff59ixw6xa5cekSZMIDPTHz28tFhZWBAQE8dtvI5g9W67g6e+/Dlvbehw8\nuId27bRRUYnF2zuZevUqcfz4EdTU1PH33wJAjx4DadKkI9nZ2WRlZbF48XJatnRCKpUSHX2f33+f\nhJ2dPbt37/wkz+BTEh5+k+PHj7BhQxDe3j6Eh/8HwOzZMxkxYgwBAUGYmprh77+WsmXLkZubQ2Zm\nBteuXcHSshZXr14hPj6OsmXLoaYmdw2MirpHmTJl8PXdSF5eHuHhNxX3k0gk+PoG0rdvX5Ys8WLO\nnAWsX7+R9u07snbtSgCaN2/JunWBbNiwherVTdi79y/q1LGlSZOmjBw5Bn//LaVqyGVkZLB9+wmO\nH7/wQURz3mU3S01Njdq1lXFy0qB//6bv3M7rCA+/yYEDe1m3LoA1azawZ89O7ty5pTh/6tQJHjyI\nZvPm7Uyf7sH169eEnblPyNs9+/d/T/n5+Qwb9gdNmsho0iSbceO2CyJSAkUQ3CwFBAQE3sCKFcdY\nuNCE9PQOGBkdZtWqJ3z33fu7Wjk5/Ujt2nUIDQ1hwoQxTJo0FTs7+w/Q47ejShUDatQwBcDEpAb2\n9g0AqFmzJnFxsSQkxDFnjhcAdnb2pKSkkJmZQVjYFebO9UZFRYX588fQrt1uWrSow9Gjh7h1K5zs\n7CwGDuzNgwexpKToc//+MczNbZBItElLS8PIqDoqKio0bdqcdetWUrZsObS1tT/6+D81165doWnT\nFs+MeTUaN25KdnYW6elp2NrWA+DHH9vj6joFAGtrW65dCyMs7Cr9+g3k/PlQQKaoKxKJKF++AjY2\ndTEzMyc5OZmOHZ/nZnN0bA3AvXv3XqnQGhFxl3XrVpGRkU5mZhbff99IcX1pTyafPHlKr17HuHKl\nB0pKT+jZ808WLery1kZMcbudMTEPWbRoAcnJT1FXV2fy5GkYGRlz+vQpAgP9kEjy0NUtw4wZs8nO\nzmb37h2IxUocOXKAMWMmAnD16hX++GMzSUlJ/O9/o99rl+7atavP3r3cCG/WrCVXr15RnL969QpO\nTj8iEomoUKEC9et//L8PAnJeTFlREoKD/3rve27ZcpydO3sAuuTlQVCQAa1ancXZuc17ty3w9SAY\ncwICAgKvQSaTERAgJT3dFoDo6PasWfPHOxlzmppahcQ/YmNjqFrVgK5de5KQkEBExN1PYsypqqoo\nfovFYlRU5MfymDgpYrHKKyfwrypv27YD27f/gb//Fnr1mklcXFlACZlMCZEohadPn6CpqYWSkjKt\nW7dFS0ubDRt8MTGpgaamJhkZGV+dm+WBA3v57ruGVKjwcqL5tzNS6tatR1jYFRIS4nFwaMamTRsQ\niUT88MNz0YwX00C8/I4K1PBkMhkmJqasXu1X5B5z57ozb94iTE3NOHBgL1euXHre21LeGVq9OpQr\nVwYAIqRSTbZt+54hQ25Tq5ZFidt4cbdTKpUwaFBfLCysWLBgLhMnujxLg3CDhQvn4+OzClvbeqxd\nuwGAPXt2sXlzICNHjqVTpy5oamrSs6fcPXPv3l08eZLEqlV+REVFMmXKuPcy5op7li8WiUSlbzwL\nfL48fZoHPM8nJ5VW4PHj9E/XIYHPEsHNUkBAQOA1yGQypFKlQmUvH7+JggmbmZk5SkpKDBjQm23b\ntnD8+BH69evOwIG9iYyM4Mcf23+wfn9IbG3rcfjwAQAuX76Inl5ZNDW1sLW1U6RaOHv2DGlpqYhE\nIurXb8CJE88V9+rXt0JD4zzVq3dEJJIgFlegfPkK3Lt3l5ycbAYO7M2GDeto3rwlIBcIGD9+FGPG\nDP/4gy1F9u/fQ2Li4yLldevW49Spk+TkyN0nz5wJQV1dAx0dXcLCrgJyRbt69eoD8vdx6NB+DA2r\nIRKJ0NXV5ezZM9jY1FW0Wb26MWfOhDyTNJcRGhqiOFdgHJiYmLxSoTUrK5Ny5cojkUg4dGi/4toC\nQ7s0kUrFvGjg5uWpk52d++oLiuHF3U5NTS0aN25Kbm4ON26E4eo6mYEDe+PtPZekpCQAHj1K4Pff\nR+Ds3JOgoI1ERT1Xqn3RlhKJRDg4yJOEGxub8OTJk3cfKGBrW/fZu5e7H586dUKxwyo/b8exY0fI\nz88nMTGRy5cvvaY1ga+Nzp3rYWq6S3FsZRVMx47ffcIeCXyOCDtzAgICAq9BLBbTvn0Gvr7xSCSV\nKV/+PN27F1W7fB2HD/8NyBUxX1Zn7Nt3wIfq6jvz8u7Ai8cikYiBA4fg6emBs3MvNDQ0mD59JgCD\nBg1h5sxp9OvXHWtrWypXrgLIJ7lDhgxn0yZ/nJ17oaysjKNjM27erMHTpzNYsWIZhobVMDSshoaG\nJv7+W7h1K4L9+w+Sl5dHly496NKlx0cb//vwsniFg0OzQiIJW7ZsJDs7ixo1TAkPv4mb2xQSEx9z\n4MAJRYxkzZqWODo6MWBAL8qWLUetWrURiWDatJl4e3uSnZ2NgYEhU6fOAOQ7usnJyQoJdFvbes+S\ntj93UTU0rEaTJk1xdu5JTk4OpqZmaGtrF1JDVFVVfaVC6+DBwxg6dAB6enrUrm2tSN3h6Nia+fPn\nsH37H8yaNa9U4ub69KnD/v07iIj4BcjGyekwtrY937KVojteBXnBCuI4X2Tx4gX06tWPxo0duHLl\nEn5+a1/ZcsHOdUGb70PNmpa0a9eBIUOcAejY8WfMzS0U76hZsxZcvvwPfft2o1KlytSpY/Ne9xP4\nsqhWrTIBATkEBv6BWCxj6FB7ypUr+6m7JfCZIahZCpQ6gurS18238H5lMhk7dpzh/v0MmjY1xt6+\n5O5exXHx4i2OH4/EwECd3r2bfbaCBh/r3S5ffpRFi4xITzfFwuIwvr61sLCoXur3fV/Cw2/i6enO\n2rUbyM+XMXSoM25us5g1y01hzAUFbXoWOziEUaN+o2fPvqxZs7zEinjFcfnyRbZu3axQRX0VWVlZ\naGhokJ2dzciRQ5k8eRrm5s+/3c/t3+6LaoH378cxb94KxOJcRKJkzM0tuHr1ElKpFBcXN6ysar+2\nrdu3w5kzR/5u5G6W/ejU6RdOnTpO9+69adGiFTKZjIiIu5iZmTNoUB8mT3bFwsKSuXPdiYuLZdmy\nNWzduomMjAx+/fU3QO5++sMPTRSulQWqk58bn9u7FfiwCO/36+Vd1CyFnTkBAQGBNyASiejSpckH\naevIkSuMGaNMYmI3xOIkwsJ2smDBLx+k7S+JHTvOsmdPGioqWYSGqpOeLnchvHWrOytWbGXp0s/f\nmHuTeEUBL66Zyt12pXh4uHL7djjGxjVwdXVny5aNhIaGkJOTg7W1DZMmTQPg4cMHeHl5kpKSjFgs\nZtaseYXavnnzX7y85jJ79gL09MoyZ84xEhPVqF9fmbi4Y0RFRZKbm0vbth0KGXIlJS0tjcmTjxAZ\nqUO1aul4ejanfPnS3xmoXr0KTZtakpWVyZUrl8jJycbffwthYVfw9PR4ozH8qt1ON7fZeHvPIyDA\nD4lEQqtWrZ8Zc0NxdZ2Mjo4u9evbEx8fB0Djxk2ZPn0yZ86cUgigvLxzXVpER8exefNVVFRk/Pab\nAzo6bz/JExAQ+PoRjDkBAQGBj8j27Y9JTOwCQH5+eQ4cqMDs2bmoqqq+4cqvh2PHrjJpUhVSU1sD\nWYjFRwqdz839Mv7XJBKJOHBgL40bN8XS0gqZTEZGRjr5+c+Nt5yc7CKT/+jo+7i4uGFtbYOnpwc7\ndmynS5ceDBw4BIBZs9w4cyaExo0dcHefTv/+A3FwaE5eXh75+VISEuIBuH49jCVLvJk3bxH6+pVw\ndt7GgQPOgDJ79sQxdaqUGTPmvNcYJ08+wvbt/QAxly7JkEgC8fP7+IsPrVrJ1ftsbeuRkZFBRkY6\nWlqvVz7t338Q/fsPKlK+cOHSImVNmjSjSZNmRcqrVTMiICAIkBvitWtbo6z8/PsscKH+0Dx8mEDv\n3mHcvt0dkHLy5AaCgzuioaFR4jYOHNjL1q2bEYlEmJmZM3jwMObOdSclJQU9vbJMnepGpUqVmTNn\nJmpq6ty5c4unT58wZYor+/fvITz8P2rVsla49zo5OfDTTz9z4cI5ypWrgLv7XPT09IrNSaimps6c\nOTPR0tLm1q3/Cil/zp49g2bNWuDg0BwAd/fpODo6Ffv8BQQE3owggCIgICDwEVFWlhY6VlHJQ0np\n7QRVvnRCQuJJTa3z7EiD/Pw4IAWA8uXP8csvFT9Z394GW9u6JCcnk5eXS1ZWFiEhJ2nY8AeSk5+Q\nmppCbm4uoaGnFfU1NTXJzMxAX78S1tby2Kc2bdpx7dpVLl/+hyFDnHF27snlyxeJirpHZmYGSUmJ\nikmvioqKYhfw/v1IvLzmsmDBYvT1K5Gfn8/Vq+UpWKOVSKpw4cL7R1FERenwfKogIjJS93XV3wsl\nJaVChnBubs4r635s1+Rduy7g4HAAO7vTDBmy7ZmwzPsRHLyVvn27MWuWa5Fz27Zd4fbtbs+OlLhw\noTuHD/9T4rbv3LlDYKAfy5atZsOGLYwePZ5FixbQrl1HAgKCaN36R5Ys8VbUT09PY80af0aPHseU\nKePp3bs/GzduIyLiLnfv3gEgOzsbS8tabNy4jXr17PD3l8cVFpeTsIAC5c8FC5awevVyADp06MT+\n/Xuf3TedGzeuF1JiFRAQeDu+jOVPAQEBga+E4cNrcvnyTiIi2qCldZsBA/Lfy5gbPnwQq1b5ER8f\nx/XrYTg5/fhO7XTt2hE/v03o6pYhOHgrf/31JzY2dZg0ye2d+/YqDAyUkRtv8tQDeno1GTZsFzk5\nWrRsacT339t98Hu+SFxcLOPHj8La2obr18OwtKxF27Yd8Pdfy9OnycyYMQsAH5+F5ObmoKamhovL\nDIyMqpOTk83cue5ERNzFyMiYMmXKMHv2DNTU1LCxqceSJd6oqqrRpUsHzMxqYmxsorhvu3YdWbHC\nh6SkRHJy5O3KZDJEIhGLFskTd1esqI+f39pnxkLxBktBHrm8vFxu3w6nUaMmiMViypXLIi6uoJYM\nPb2s935WhoZpXLwoe9YXGUZGpRenU65ceYUhrK6uQWjoaUV+u+PHj2BnZ09Y2FW0tXXQ1NQqtX68\nTEZGBh4eGTx8KBfl+euvHMzMdjF5crv3anfXru34+KyiQoWiixfq6gC5gHzHXix+Qpky6iVu+9y5\nc7Rs6aRI76Grq8t//13H01NuwLVq1YZVq+Q7lCKRiMaN5caUiYkp5cqVL5R3Mj4+FjMzc8RisSJH\nYevWbZk2Te52+qqchK9S/qxb146FC+eRnJzMyZNHadGiJWKxsLcgIPCuCMacgICAwEfE2tqUPXv0\n+PvvY5ibV8XGxum92lu1Sp4jLDY2hiNHDr2zMffiTkfBJNPKqkapBNn/+mtLbt7cwYkT5VBTy2X4\ncE2cnT+u615MzENmz16Ai4sbgwf359ixw6xa5cfp038TGOiPq6sHK1asQ0lJiX/+Oc/atSuYPXsB\nO3duR0NDk02bgomIuMugQX1YuzaASpUqM336JHx8VqKmps6mTRuQSCQMGDBYcc9mzVpSs6Yl3bt3\n4s6d21hb1+HIkYPY2Nhy48Y1dHXLkJmZyYkTR2nZ0glNTU0qVtQnJOQkDg7Nyc3NRSbLV6gyuri4\nMnbsCNTVNahXrz7TplXD3T2IR4/0sbK6z/TpLd/7Oc2f3xKpNJCoKF2qVUtj/vzS20FRVlZmwIDB\nDBniTMWK+lSvbqw4p6qqyqBBfRQCKMXxooDKh+TJkyQePTJ6oUSNR4/ebzfdy2susbExjB8/irZt\nOxAWdoXY2FjU1dWZNGkagwY1Z/fuocTGaqOiEkOVKjLu3XPg9Ol9xMXFkpAQz6hRv3P9+jX++ecc\nFSroM3/+IpSVlQkPv8nGjRtJT0/n5s3/mDZtBuXLVyAtLY1lyxZz48Y1HB0L/91RUVEhPT2dI0cO\nFsk7KZVKeZmCRQh4fU7CVyl//vhjew4d2sexY0eYNm3mez1LAYFvHcGYExAQEPjIVKhQni5dmn+Q\ntpycHDhyJITVq5cTHR3FwIG9adu2I/b2DfD0dEcikZCfL2PuXC8MDAw5dGg/27f/gUSSR61a1owf\nP0WxKi6TyQpNMrt370b79l0+SD9fRCwWs2hRV6RSKWKx+JOoeVapYlBo98HevsGz36bEx8eSnp7G\nrFluxMQ8QCQSKSa0YWFX6dZNLpNvamqGqak5AP/+e52oqHsMGyaP0crLk1Cnjg1btpzmjz8yUFKS\n0b9/WRo2NMLIqDo7d25j3jwPjI1r8PPPXUlLS6N//x6UK1eeWrWsFf10dfXAy2suvr5rUFFRwcPD\n81l6AShbthwLFixmwoTRTJ06g1atbGnZsg4ZGeno6DT4IM+pbFk91q//eIZ216496dr1eRoCiUTC\n+fNncXRsw+jR4z9aP16kSpWq1Kmzm0uX5Hn81NUjaNjw/dxNJ06cyoUL51i2bA3r16/BwsIKT8+F\nXL58kdmz3fD330KXLrU4evQIY8ZMpVGj+vj5rSUuLpalS1cTGXmP334bwNy53owcOZapUydy9uxp\nGjVqwpIlXsybN49JkybTooUja9euZMSIMWhraxMVdQ9f30D2799TKJ8dQFpaKocOHUBFpfipYX5+\nPidOHMXRsfWzRQj59S/nJNTXr/TG8bdr15HBg/tToULFQkb727Jt2xY6dfpF4X78vvUEBL5EBGNO\nQEBA4ItGbggNHz6KoKBNCrn6JUu86NatN61b/4hEIkEqlRIVFcnx40dYvdoPJSUlvL3ncfjwAUWy\ncpFIVGiSaWpqWKry129yL42Li2XChNHY2NTjxo0wKlbUx9NzIYmJj1m0aAHJyU9RV1dn8uRpGBhU\no2fPXwgO/ou0tDTat3dk2bK12NrWZcSIIUydOqNQTrSXdx8KdhAKdiJ8fVdjb/8dnp7exMXFMnr0\nsFf2s2DHwd7+e2bOfC44cvbsDfr31yQlxRaAW7fOsn17Fps3by/SxpAhwxkypGiSdEPDaoVyE2Zk\nZHD9ejT9+snrVqpUmY0btxUai45O6cW1fUzCwu4wdmw4aWkShg07h5eXMg0bWr72mvz8fObPn1Po\ne4mOjsLLy5OcnBwMDAxxcXFDR0eHkSOHYmFhSVjYVbKyMpk+3Z3AQH8iI+/h6OikeB/Hjh1GX387\ntrZ+qKgY0a1bN7p1a/FBxiiTybh+PYw5c7wAsLOzJyUlhczMDJSUlGjbth0//GAPyP99Nmz4A0pK\nStSoYYpMJlO4NJqamhEXF0d09H0iIyPw8PAgNzeXxYsXoKysjEwmw9DQiNTUVJyde1G2bFmFsElB\n26tXLyMhQe6nu3KlD3p65ThzJoTLly9y9+4d1NU1+O+/f5k/fw4go2JFfXbv3qnISRgfH4e5eU3C\nw2/y6FECKiqq+PquZuXKpYwePb7Qok3ZsuUwNq5B06bN3+v5BQdvpU2bdm800kpaT0DgS0Qw5gQE\nBAQ+Ia9yDVu/fg22tvUUO0Zv4uWUobVr1yEw0I/HjxNo1qwlhobVuHTpArduhTN4cD8AcnJyKF/+\n7RKgf2wePnyAu7snkydPw83Nhb//Ps6+fXuYONEFQ8Nq/PvvDRYunI+PzyqMjKoTGXmP2NiYZ5P0\ny1hZ1eLRo0dvldy6QJWyIJapICk4QN269Thy5CB2dvbcu3eXiIg7iEQiateuw6JF84mJeYiBgSFZ\nWVkcO3aVlJTnBtrjxw0JDd2OlVWNd3oW8fGP6ds3hGvXOqKu/pDfftvHtGnt36mtL4F5827x77+9\ngd4ALFgQxI4drzfmHjyIZubMuYW+l82bAxk3bhK2tvVYv34N/v5rFcaF3OAIJDh4K1OmjMfffzM6\nOrr06NGZHj368ORJEsePH8HPb6NiAaRKlcwPPtZXpfx92fhQVn6+6KCk9HwK93z3WIaJiSl//hlc\nZCFm1KjfGDduMhYWhZ9hgVFnYWFFZOQ9AgP/4MKFc5w8eYx9+46Sn5/PlCnjkUqljBr1O87Ov6Kr\nq0tOTjZDhjizfPk6OnfuioPDd/z22wi+/74RU6dOJCsrk4CArURG3mPOnBkcPvz3s53WMJSVRTx8\nGI2TU5sSP6OsrCzc3Kbw+PFj8vOltGjRisTEx4wePQw9vbL4+KzC29uT8PCb5ORk07y5I7/++hvB\nwVuL1Ltw4ZwiNtXAwJCpU2egoaHBqlXLOHMmBCUlJRo0aMiIEWNK3D8BgU+FYMwJCAgIfIYUJCl+\nV5ycfqR27TqEhoYwYcIYJk2aCkDbth347bcRH6KLH4UqVQwwM5O7MlpYWBIXF8uNG2G4uk5W1MnL\nkwBydcmwsMvExsbSt+9A9uzZSd26dlhZ1SrS7suunS8ei8VievXqz5w5MwgIWE+jRk0o2AHt3Lkr\nc+e607dvN6pXN8bSUt62np4e06bNZObMqeTm5gHQqNGPaGndIiPD4lmdy9jbv5shB7B06XmuXesP\niMjOLseGDUkMH55EuXKft0H+rqSmqhU6Tkl5867Ky99LTMxD0tPTFC6FP/7YHlfXKYr6TZo0BaBG\nDVNq1DBVPMuqVQ1ISIjn2rUrpb4AYmNTj8OHDzBgwGAuX76Inl5ZNDW1XmngvQ4jI2OSk59y9epV\nDAxMkUgkPHgQjYnJm7+7gvuFhJwkJORvrly5xMCBckM6KysbkJ8PDg4iJESekuHRowQePoymVi1r\nVFRUCu0UqqqqKnYR4+LiyM7Opn//nVy4UJlKlbwwNbVBQ0OzxGM7fz6UChX08fLyASAjI539+/ew\nbNkahdDL0KEj0NXVRSqVMnbs/7h37y7duvVk27YtinrJyckEBvoVim/944/N/PJLN0JCTrJly5+K\n9gUEvgQEY05AQEDgE1Oca5i3tyeNGzvQvLljiVaLNTW1yMzMUBzHxsZQtaoBXbv2JCEhgYiIu3z3\n3fdMmTKe7t17U7ZsWVJTU8jMzKJy5cofc7hvRWF3SCVSU5+gra2Dv/+WInVtbe3YuTOYpKREBg8e\nRlDQRq5cuVQkNqhKlaoEBGxVHL/obvbiuaCgHYryApc7NTU13N3nFttXOzt71q0LfKn0GH/+eQOx\nOJ/+/bWxtX335PN5eSq8qHCZk6NDTs6r5fs/Fi/HI02cOIaZM+e8Mg/c+vVr0NTUolevvq9tt2HD\nbC5efIJMVg5IpUGDN0+uX/5e0tNf7yasoiJXiyzYpSvgxTjJ0lsAESESiRg0aCienh44O/dCQ0OD\n6dNnKvrwcjjpi8dFFyTkIjKzZs3H29ubp09TkEol9OjRu0TGHIBMBqdOnSQ1NYW+fQfQqVPheMnL\nly9y6dI/rFnjj5qaGqNG/aZI0/DyTuGLu4hy1+UTnDw5EFAhMvInoqOjCA29QuPGJVOvNTU1Z8UK\nH1atWsYPPzhga1u3SJ3jxw+ze/cupFIpSUmJREZGUqOGWaE6r4pv1dLSRlVVDU9PD374wUGh8Ckg\n8LkjGHMCAgLfNC4uE3j0KIHc3By6devFTz/9/NH7UJxrmHwiJyIlJfm1q8UFEzozM3OUlJQYMKA3\n7dp1IDc3l0OH9qOsrEz58hXo338QOjo6DBkynHHjRpCfL0NZWZnx4ycXY8x9fEGSkqKlpUXVqgac\nOHGUFi1aIZPJuHv3DubmNalVqzazZrliYFANVVVVzMzM+euvHYqV/NIiPT0dL6+TpKaq0LJlOTp2\n/E5xbtgwR4a9OtzurejevTqHDx8nLq4lkEWrVleoXLnHh2n8HZFKpUXikd70vEsqeDN9egfKlTvK\nrVv5mJqKGDXqp7fun5aWNrq6uoSFXcXWti4HD+6jXr36JbpWJBJRv36DUlsACQ5+no+tIGXAiwwa\nNPS1xy8mLH/xnLa2NomJiZibW3L7djjnzp3FyelH7Ozs8faeS05ODtbWNkyaNA2AkSOHUrOmBVeu\nXOLx40ckJSWirKzMpUv/YG1tw5w5M58pZapw/34UERF3UVNT4/79KP7990aJx5uVJQKeG9tSaVme\nPr1Z4uurVTPCz28zZ8+eZt26ldSv/12h87GxMWzduhlf341oa2szd677K3MVvhzfWsC6dQFcvHiB\nkyePsWPHtkLxqgICnyuCMScgIPBN4+LiVij+o3nzlgqXnY9Fca6EBWhr67x2tbhgQqesrFxk4tG3\n74Ai93J0dCoiSw4QHLz7hd9/FTn/qSjOHdLNbRbe3vMICPBDIpHQqlVrzM1roqKiQqVKlaldW64G\naWtbj2PHjmBqalZc0x8EmUzGoEH7nu04KLF793Vksgv89NO7q0kW5PmzsLBkyhQ3JkwYQ2pqMv36\nDWLjRmP27QtGT0/EkCFdS10JtLjFDicnBzp16sLFixdo3rxlkXikF3MWHjiwl61bNyMSiTAzM2f6\ndPdC7cfEPCwiZmNkZAzI3/WIEW+XuqO472Xq1Jl4e3uSnZ2tiI8q7rriHqWxsUkJF0A+L6Kiopg8\n2ZXateswbNgIpkyZw++/D2HgwCEAzJrlxpkzITRu7IBIJEIikeDvvwV39+mcPXuG2rWtsbP7Dg+P\n6cTExDBx4ljmzvUiLi6WihX16du3G9WqVcfauo7insXtFL54rls3G3bs2ElExM9APg0abKdVq5Ib\n6ImJiejo6NC6dVu0tLTZu/cvNDW1yMjIQFe3DBkZGaira6ClpcWTJ0mcOxeqMNw1NTUV9WrVsi4S\n35qY+JgKFSqSnZ1Fo0aNqVPHlh49Or37CxAQ+IiIZO/ilF0KlKZimsCnpWJFHeH9fsV86e93/fo1\niviP+Pg4Fi5cpjAGPgYvC6AEBW0iKyuT+Pg4fvihCc2bO5KXl6dYLY6PjyuV1eLQ0KtERCTQunV9\nKlWqAHz57/Zj8OjRIxo0eExmHMSaCAAAIABJREFUZkNFWc+e21m6tOTCDi/Tp09XRTLpGzeu4+u7\niiVLVn6I7haiJO83NTX1JbGLtbT/P3vnGRDV0YXhZ5dd6tJEsGChWFBRhGCvUTEaNdEoAWzYWxJ7\nrLFHsKAGjYoSaSrYjd1YY8GosYFG8TN2moKKtKXsst+PlZUqFlCT3OfX3r0zc2fuXS5zZs55T5cO\nzJ3rzaefdgDA1fUL1q1br1kEyT1OTExk+vTvWbMmECMjY1JSUjA0NCQgYC36+vq4u/dlzJiRfP/9\nNI2Yzdq1K4XdkHckLi6WMWNG0LOnG8ePqzh40BYTk18xNi6HtXUkKpWS5ORkevVyo08fT777bjhD\nhozQuCN7ec3RvHsADh06yI0bf/Hdd+Pw8PgKf/8QjIzeTjH17t0YQkMj0dZWMWJEKwwNDV+77vnz\nZ1m50hexWO3COXHiVK5di2D79i2Ym1vg67saL685XL0agYVFRQwNZbRo0ZrOnbuyffvmfOUuXbrA\n6tXLNfGtw4aNws6uDlOmTHjhNqrCw6OfRun3Y0N4N/97MTd//b+JXISdOQEBgf8sRcV/ZGdnlfl1\n3yS5sVwuL/PVYm/v/axe7UhGRjNsbffyyy/W1Kv39kIdH4qnT5+xadNZ9PUl9OnTNl/C4rJCJpNh\nbHyNdI3AYQ6Ghq//G9q0aYNGLbNr1+48eHBPk+evY8fO7NnzK0lJzxg4sDc//rjojVQ5S4P8YheP\nefjwIWKxWDPRLw6VSsWlS3/Srp2LxsgrOHGXy+VcvRpZpJjNx8DduzEsWuTHw4eXMDWV0a1bdxQK\nBdraUnr1cmf58iXcvv03vr6ruXjxT/bt283MmfNwcWmFq6sHZ86cRkdHhwULlmBqWu61r6tQKJBI\nJMUevw4ikYjQ0A1cvLgELa0cVCoxmZnHsbQcwPz57holx1x0dfUK1c+lbdt2BAau5ZNPnLGzq1Oi\nIRcefoVbtx7RsaMjlStb5DtnbW3J9OmWbzSWXBo3bkrjxk3zfVe7th09e750NS5q1xWgZ0+3fOWK\njm9Vu1kKCPzTEIw5AQGB/yzp6WkYGhq+VfxHaVKcq5xIJCI9PS3favE334wt1WtnZGSwcaOMjIxa\nANy+3Z01azazfPk/y5h7/PgJbm6n+euvPkAGR46EEBzsVmIuu3dFX1+fCROk+PjsJSmpIp98coXJ\nkzu/Vt2oqBscOLAXf/9gcnJUDBvmycyZ8zh37g+N8l7duvb58geWBampqRw+fJAePXpx6dIFNm3a\nyKJFy4oRu8hEW1vntdw7RSLRKxUZVaocDA2LFrP50ERF3cPT8xQ5Obd58GAXn322kT17djJ58gw2\nb95Ir17uREXdQKFQoFAoiIi4TMOGaiGPjIwM7O0bMGzYKCZOHIOnpwflyplha1sDLS2tfLteLi6t\nOHxYncvtl1/8MDIy4v79e0yaNB1//9UYGRnx4MF9NmzYyurVK7hy5SJZWdl89ZUrX375FZcuXSAg\nYC0mJqbcvXub2rXrMHToSGJiYtDS0qJSpRloaaXz9Olg9PQuo6WlR3p6OsePH6Fdu5curHmfU65L\nYi7a2to0adIMH58FTJ0685X3bdGiA/z8syMZGc1ZtWo/a9c+p2HDmm/9HIpa+IqKusHBg/sYO3bi\nW7dbsP3PPx/N0aNp6OtnMmlSYywtS058LiDwsSD+0B0QEBAQ+FA0adIcpVJJ376u+Pn9nC/+o6zJ\nVbCcNGksZmblyczMJCYmmgsXzhMefoqYmGisrW0wMyuPlZU19vb10dHR5c6d26XaD6VSiVIpKfBd\n2RpAZUFQ0LkXhpwI0OPQoS84derie7l2//6tOHPmE86dM2T7dtfXdkGLjLxC69afoqOji56eHm3a\ntOPKlcv5yryPSIiUlGR27txa6Pu8ix337t0tdrGj4OQf1Iack1Mjjh8/QnLyc0DtspmLSqVWYK1c\nuTLHjx958Z1azOZjIDT0Bs+eGZCa2hGVypTDh9tRr15Drl+/xs2bN0hPT0NbWxt7+/pERd0gMvKK\nxk1RKpXSvHlL7ty5za1bN2nUqAlBQaGMGVOU8fHSKL516yZjx35PWNgOVCqV5jg0dDt79vyKTCbD\n3z8Ef/9g9uz5VRNb+/ff/2Ps2Ils2LCV2NgYbt68gY2NDVKpNjLZc+TyBjx/7o5EUo+//vqFCRO+\no27d/K7keY3z9u07Ehq6nkGD+hIbGwNAhw6dEIvFhXbG8pKVlcXGjTpkZNQEtLh3rxv+/qX7vgKw\ns6tTKoZcLsnJcqZMqc6uXT0JC/Ng8ODTZGdnl1r7AgJljbAzJyAg8J9DpVLx+PFjxGIxPj7LP0gf\nilKwzJsMOzh4B0OGTMLWdgAGBsloaalYsyaw1AUvDAwM6No1nvXrE8nJKU+lSsfp0+f9uvKVBoVv\niwKp9P0ZpTKZDJmsaCn+4ijqWZaxnkmR+PmtICYmmoEDeyORSNDV1eOHHyZz587fpKena8QubGxs\n8PX1ITMzg/Hjv2P69FmYmZWnatVqeHj0RCqV0Ly5WqBHLs8gLGw9CoWS7t07Y2ZWHkfHTzRucLnj\nnDnzx0JiNrliQB8SsVj54pPamJZIMpBIxIjFIipVsmT//j3Ur++ArW0NLl36k5iYaKpXtwJeSvRf\nuqRWg8zdHS7JyK9Tpx4VK1Yq8vjPP89y+/bf/P77UQDS0tKIjn6IRCKhTp16mgT3NWrU4vHjx0gk\nEkxNTfH3D2b//ks8f76Hr76aRoUKhXPkrVixJt9x/foObNiwJd93kZFX6NLli1e+f1QqVaGFoZyc\n0vtBx8REM2PGZDp06MSVK5dYtGgZ69at4dGjeOLiYnn0KJ6vv/agVy93AIKCfuHQoQOYmJhiYVGB\n2rXr4OHRl6ioG3h7z0UkEtG4cRPS0pSkp9dBJMrEwmI2T55cZMCAMCZMmIKTkzP79+/h1KnfycjI\nIDr6Ie7ufcjMzOLIkYNIpdosXuz71jGEAgKlgbAzJyAg8J9CpVIxZsw2mjRJoGnTGKZO/fW97H4U\n5FXJsN3cerJ69QaePROzfbsbZ86k4+TUBJFIRFxcLP37F5ajX7duDRcunH+rvixa1ANf37NMm7aV\n0NByNG9er1CZb78dRlSUWkb82LEj9O3rypgxI9/qemXBkCHNadgwCFAASXTrdoBmzRxLqPVhcXBo\nyMmTv5OZmYFcLufkyeOFcuLlZf/+PSxbtqjU+zFy5GgsLasQGBjKqFFjXuwITWTjxm1UrFiJSZOm\nM2/eAiQSKb6+qzl58jxdunRj7Vq1KMu1a1c5cuQUhw+fYtKkaWzduptff92Gs3Njtm7dxa5dvyGR\nSBg/Xh0bN2jQMNzd+5KU9Iw7d+KYNm0WQUGhbNiwhQEDhpTauLZsCSUzM+Ot6o4a1YwqVeKRyQ4j\nld7k668vEhl5GQcHJxwcGhIWtoGGDZ1wcHDk11+3U6tW7UJtqA2f/O8WLS0tcnLU3+Xk5KBQvNwB\nKhi3VvB4/PhJBAaGEhgYypYtu2jUqAkqlSpffjwtLTE5OUrNsVgspnfv9owc2blIQ+5VbNoUztCh\nh+natS/79+/B1dX9leV1dHT44ounaGk9AqBixd/p3fvt4uMK8uDBPWbMmMz06XOoU6duvnMPHz5g\n2bKV+PsHExjoj1Kp5MaNvzhx4hjBwZvw8VlOVNQNzQKCt/ccxo+fTFCQ2r1XIlEBmZiYbATEZGSM\nY9q0WcyfP1sTV3j37h28vHzw9w9h7dpVGBgYEBCwEXv7+hw8uK9Uxigg8LYIO3MCAgL/KTZv/p3N\nm7u/SEQMwcE2tG17js8+a1qsDPu7iBkUx6uSYfv4HOT4cVfN+ZQUS27dintle4MHD3/rvohEItzc\n2pRYJndVfu/eXUye/AP16zu89TVLG1NTE3bs6MT27buQybTp0cMNsfjjXq+sVcuOzz/vytChngB0\n69ajUILjvJL5ZZWGIO9ihkqlKrTTEx8fh0wm4+7d24wdOwpQGyJmZuoytrY1mT17Oq1bt6VVq7aA\nWnkwPPwkYWHrAcjOzubx43hN2oHw8OuMHfuY+/c/oXLlSLy99ejc+fWSR78OReW/exMsLMzYvbs/\nXl4J3Lw5goQEPbp160HNmrV4/jyJ9esDNa7POjo6GiP8++/HaNpwcmrE+vWBNGyolsdPTn5OxYqV\n2Lo1DCMjI9LT01EoXk/wpXHjZuzYsQ0QsXXrJr79diwWFkXHdRkbm7Bnzx4+/7yLRo7/Tdm58yxT\npliTnl4b6IGzcxD6+gYl1ps//0ucnU/z8GEaHTrUKhUhpWfPnjF16kS8vHyoXt2KS5cuaM6JRCKa\nN2+JRCLB2NgEU9NyPH36hKtXI2jVSi2CJJVKNSldUlJSSE1N1SQc/+yzLvzxRzjduq3n6tXf0NJq\nxJgxWtSrV5+KFSvx8OEDRCIRjo7O6Onpoaenh0xmSIsWrQGwsanB7dsfh2uwwH+XMjPmTp48iZeX\nFzk5OfTq1Ythw4aVXElAQECgjElIyNAYcgAKRUXi4s4CReecyytmsGrVcnbv3omn5+BS71feZNhV\nq+qhpfUYLa2nZGXZIZEkU7lydU3Z3Hi7a9ciMDe3wNt7CT4+3rRo0Yq2bdvTq1c3XFw6cfZsOGKx\nFpMmTcfPbwWxsTF4ePSje/eeJCYmMmvWVNLT01AqlUyYMBUHh4acP39Wo3RnY2PFhAnT0dNT7xCo\nVCoCA/25ejUCb++5tGzZmlGjxhQ3pPeOTCbD07NjmbQtl8uZOXMKCQkJ5OQo8fQcgqVlFX7+eRly\nuRxjYxOmT59FamoqP/44S6OKFxcXy5Qp4wkO3kRU1I1C5d3c+nDq1Alq1arNb7/tR6lUMHbs94wb\n9w2ZmZmYmZUvMrlxURTlViaTydi9ewfZ2QqqVKnCjBlz0dHRZf782ZiYGBIZeY3ExATEYjE//jiL\ny5cvanaOABITH7Nu3RqkUilaWhJWrVqHnp4eq1evIDz8FJ6eHjRq1ITmzVsSHn6KkJAA/PwCiY5+\noDEIPT2H4Oe3AhMTUwCioq4zY8ZM7t//DTOzFeTkPGDBgssEB4vo06c/3bp159KlC6xbtwYDAwOi\nox/i5OTMhAlTEIlEHD58kA0bglCpVDRr1pKRI78DKDH/3ZuiTjxdWB3R2bkxx4//oTnOjXFTqVT5\nEqZbW9swfPi3hIWtZ8CA3tSqVZuRI7/jzJnT/PzzTzRp0gw9PX1N+YJ52fIed+vWnbi4WBYv9iIp\n6RlLlizAy2txsfnxAL74ogcTJnynkeN/E06ffv7CkAMQERHhRGxsDNWqVX9lPZFIxFdftXplmTdF\nJpNRoUIlIiIua1xZ8yKR5F0YE6NUKoGC4jsqTf/yolKpEIlErFvnxpQp5+jV6xOcnQvniMy/+CbW\nHL+8noDAh6NMli2VSiXz5s3jl19+Yd++fezbt4/bt0s/CFZAQEDgTenatQFWVns1x7Vq/crnn6tX\nzrduDWPAgN4MHz5II8OeK2YAULt2HeLjX71D9roUlwx7797dHDjgR/36PahYcTVVquykbt1kbG1f\nxrE9fPiAnj2/Zv36Lchkhpw4cSzfzplIJKJChYoEBobSsKEjXl6z8fLyYc2aIAIC1gJw+PBBmjRp\nRmBgKEFBYdSsWYukpCRCQgLw9V1FQMAG6tWrx+bNG/P1ceDAodjZ1WHWrPkflSFX1pw7d4by5S0I\nCgolJGQzTZs2w9d3MfPnL2LduvUat8Pq1a1QKLI14hRHjx6iffuOKBQKfvqpcHlAk7T5l19CcHfv\ny7Vr6Zw7N5JjxxYSFVWOdevUz+xV7sDFuZW1afMp/v4hBAWFUr26NXv37tJcMyUlhTVrAhk2bBTx\n8XH07t2fadNmkZ6exq1b/yMpKYmoqBv06eNJUFAYAMuXLyU5+TknT/7OvHkLCAoKpXPnrjg5OTNy\n5HekpqZy+vQJKlWyxNm5seZeFRSUyMl5Of3Q1r6FltYI1qwJIDDQn8TExBdjus64cZPYsGErMTHR\nnDhxjMTEBPz8fmb5cj8CA0OJirrOqVO/A2oVyXr17AkKCmXAgCGUL2/OihVrSi1v3aZNG+jf343+\n/d3YsiWM+Pg4PDy+4scfZ9G/vxvz5m2mbdsOjBmzieTkZIKCfmH9+kBkMkOsrKyxtrbF1LQc1apV\nZ8CAwYwc+R1GRkasW7eGn3/+ifj4OB48uAeoXRafP3/OoEF9GDlyEA8fPmD48G+YMmUGDRt+gq/v\nagwMZDg6fsLChS+VTseNm0Tnzl0BtRx/aOj2txq/mZkCeJm6wNz8AeXKvbtHwtsglUrx8lrMwYP7\nOHz4YL5zRf9NiGjQwIHw8FNkZWWRnp7OmTOngdzYVkMiI68AcOjQAU0tR8dPOHLkNwAePLjPo0fx\nVK9uVYIi60eRqlngP06Z7MxFRkZSrVo1qlRRTz66dOnC0aNHsbW1LYvLCQgICLw21taWBARkEBy8\nBbFYxdCh9bGwMCtWhj1XzABALBaVyipspUqVCQ7epDn28Oir+bxkiVqQJS0tjYiIG9jYWFKxYocC\n9QvH2xWkZUu126SNTQ3kcrnGRUgqlZKWlkrduvXw9p6LQqGgVau21KxZi8uXL3Lv3h1GjBgEqKXj\n69QpOoH6f20SY2tbk5UrfVm9egXNm7fC0FDGnTtFux22a+fC0aOH6Nt3AMeOHWHevAU8eHCvWDdF\nUCsIgnoHcM2aRKTS/VhaJvDsWTYnTmgxsQTxvqLcylQquH37b/z9V5OWlkp6upwmTZpp6nz66acA\n2Ns3QE9Pj9mzp6Gjo4O+vj7x8bE8fvyI5ORkgoPXsXPnVmQyGeHhJ7l+/RqPHsWxePF8XF092LIl\njPT0NFQqFa6u7tSrV5/U1BQiIq7w9ddfoqOjQ0pKcr7+lisHOjr3UalEyOVN6NEjC2NjE5ycnLlx\n4xoymSF169ajUqXKAHTo8BmRkVeQSCQ4On6CsbEJAC4unbhy5TKtWrV9rfx3b0tRaSQcHZ1eiHLM\nZe/eh/j4uGBtHcbmzT2Ji1uNgcFZgoM3kZ2dzaBBfbGzqwNQaOHFxMSUgIAN7Ny5jbCwDUye/ANW\nVtasXOmPlpYWf/55jrVrV/LjjyXHS169eotz5/7ms88cqFq18luPd8KEDty6FcL581UwNExh3Dhj\nZLI3T2ZcGohEInR1dVm06CfGjRuFp+eQfK7HRe1M2tnVpWXL1nh6umvSQuQKFE2bNuuFAAo0atRU\n8yx69HDFx8cbT093tLS0mD59NhKJJN/zenHVfH0rK/dnAYHXpUyMuUePHlGp0ktFpgoVKhAZGVkW\nlxIQEBB4Y+ztbVm8OP/i0uvKsL8Pbt68z7Bh17hxowVmZreYPv02ffu21JwvGG+nVGYWaiOvG1De\n5Nm5bkEODo6sXOnPmTOn8fKajZtbHwwNjXB2bqJx6zM3NyQhIaXIPv7XJjBVq1YjIGAjf/xxGn//\nVTg5OWNtbYufX0Chsu3auTBjxhTatGmHSCTC0rIKt2//XWx5eCl2kZ6ejpbWfhITx5GW9il6eucp\nV+7Vub3UFJ3TzctrLgsWLMHWtgYHDuzl8uWX6RpyfxdisZiKFStpcnl5ec1BqVQiFmvRunXbIt08\ns7OzuXDhPL//fhRtbW1Wr16X73xgYCh//HGa3bt38sknjfjtt/0a983MzCwsLU3p2zeKnTv/wsxM\nwrRp37wciaiw01CuO1xhXn7/uvnv3oa8aSQA2rRpR0TEZSpUqETduvbMnx8LmOaOgHv3njJiROGY\nraJo06YdoI6hPHHiGKCO7Zo3bxYxMQ81O7clsW3bWWbMMOLJE1d8fC4xe3Y47u4t3mq8Ojo6BAa6\nIZfL0dHR+WDxp3kXvnJTMwC0bKmOWRs0KH8IT958dB4e/Rg0aBgZGRl8++0watdWG9O1a9tpxE8A\nRo0aDajz6RWVdLxz566a3U6ArVt3FXtOQOBDUCbG3Nu8TM3NP8yKj8D7QXi+/27+Dc+3S5eO7N+/\nC09PN6ytrXF0bIiJiT5isUgzPmNjfXR1pWU+3jFjorhxQ60c9+RJVfz8tjF2rAyRSERmpgESiZam\nDzKZDmKxEl1dKUZGepibGyIWizAzk2FiYohMpoOenramfO659PQUatashp1dP3R0xDx4cJfhw4fj\n67sYufwZ1apVIz09nbS0J1hZWSGVamFqqo+5uWG+z/8VHj9+jKWlGX36fE3lyuaEhYWRmppMTMxt\nGjZsSHZ2Nvfv36dGjRqYm9dBR0fKpk3BfPllN8zNDTE2rkdKyvMiy0ulWpiYqJ9d+fIyjIySiI9X\nKw+amQVgbq6Dubkhhoa6+Z5lXlq3bsasWbMYP3402dnZnDsXjpubGxkZ6dSqVR1DQ12OHz9ExYoV\nMTc3RFdXbciZmxsW+k3p6koxNtbH2dm50O/h8ePHWFhYIJdn8cUXnfj00xZ06NBBU1cdf7me69eN\nqFQJPDw8OHz4N6pXr0Zc3F1q1mzN+fOnkEq1GD78M7Ky/sfRo0cxMdElLS2NyMjL/PDDVO7cuUNU\n1HUyM59TuXJlTp8+jru7Ow0bNmTFiqVIJAqMjIw4efIY/fr1w9zcEJEo/7vI0FCGjk7pvJ8MDXVR\nKjM0benrayOT6WJoaIC5uSGWlgryKlcaGyvR13/5rPT0pMhkupp7n/dvtVIlU0xMDDEzkyEWq/u7\nZMl82rZtRd++fYmJidGM0cREHx0dSZFjCgtL48kTdSLwp08/YdOmnXz33buO/Z/7Nz5hwmxu375N\nZmYmPXr0oHlz53dqb9mygxw7loWRUSbz5jXHxqZ0lDrflv/S+1fg1ZSJMVehQgXi4l7GlcTHx1Oh\nQtGqS7kUt/or8M/nVav7Av98/inP99Sp36latTpWVtaAWmr/22/HaVyfALy8luark5qayi+/rCcu\n7hkSiQQnp+Y4OTUv8/EmJ+dfEEtNlfLo0XO0tLR4+jQNpTJH04fU1Ezk8kwyMrJJTpaTkJBCTo6K\nJ09Syc7WIjVVfS63fE4OPHmSSnj4KcLC1iORSNDXN+CHH+agVEqZMmUmo0ePISsrG4lEzKBBIzAw\nMCM7W8mzZ+kkJKTk+/xf4c8/I1i50hexWIREImXixKmIxWK8vReSmpqKUqnAza03xsbq/3WtW7dn\n9erl9Os3VHOfZs/2LrJ8draSpCS5pty0aWNZsGAYIEWlSuX5cz06dHDByMgYU9Ny9OrlyrNnScya\nNQ8AX98lZGVlkpSURMeOn2FhUQEdHT127PgVY2NTWrZsiYmJKZ9+2p70dPVzy8jIRiQSkZCQUug3\nlftbKvh7ABg2bBR2diqmTJnwQrZdxbffjtPUnT9/L/7+lpibr+DmTTEXLybh57eYjIwM5s6dq4nz\nUijU10tPz6J6dRs8PPqQlJRE//6DAF2SktKxs6vLjBmzXgigNKJhQ3XC6qFDR9GnT19UKhXNm7ei\nfv1GL64vyveb7NLlSwYOHPRWAiAFsbWtw/z5c/jqKw9yclQcPPgbM2bMRaFQkpCQwpQpzbh/P4iH\nD9Oxt9/G0KHNOHgwlK++6o1CoeDo0WN8+eVXmntf1N9qUlI62dnq9p48SUJXV/1uXb8+jJwcFQkJ\nKSQlpZOZqSjyby8rK78LeGZmzn/qb7QgU6bMznf8Lvdi/fqTTJ1an6ysagBERQWzb9+XmhyC75t/\nyv9dgTfnbYx0kaoMAh8UCgWdOnUiKCgICwsLXF1dWbp06Stj5oQf5b8X4aXz7+af8nznz5+tUXsE\n+O674Xzzzdh8xlxetm49h5eXnISEqjRocJmAgDZUrKiOcVIoFEgkZZfZZcuWM0yZUoXU1HpAMh4e\n2/H17VVm1yuOf8qz/bcSFxeLu3sPAgNDsba2YciQ/tSoUZOpU2dy+vQJ9u3b80KdUgctLS1Onz7J\nwYN7+eGHufTr9zUKRTYbN25DIpHSu3dPVq9eh7m5hab9sni+ffoc4vDhnppjS8tfuXSpXbEeOwEB\na9HT088XNwpw6dIFNm3ayKJFy4qs9yHYvHkj+/btBtRpJFq1asPkyePyxb+6un7BunXrMTIyJiBg\nLYcPH6RcOTNMTU1p2rQ5Xbt2x8trDi1atKJNm3b5ykdF3WDVKl+WL/fj2rWrzJ8/Cz09PZo1a8mh\nQwfZunUXly5dYPPmjflET3IJCTnJ3LnVSU62x9DwBtOm/c3gwW3f1+35VzNmzG+Ehb18B+vqXuDs\nWRmVK3+Y3Tnh3fzv5W2MuTKZjUgkEmbMmMHgwYM1qQkE8RMBAYG35U3yv8XFxeLtPZfnz59jYmLK\ntGkzefz4EeHhp7hy5TIhIQHMm7cQgOPHj7BkyQJSU1OYMmUmDg4NUSqVrF69grCw40gkMvT0+nDh\nwgCmTfNCVzcSIyMj7t+/R1jYjjIZq1wu588/N+PoeJ/U1GwcHdvh5TWuTK71JmzfHs7//pdCkyYV\nadeu4Yfuzr+e48cj8fW9ikplwq5dfzNunC3W1jYa2XRra1vi42NJTU1h3ryZxMQ8fPE3ksXgwWqx\nDZnMSJMbzMrKmri42HzG3Juye/cFli17TGqqDi1aPGPJkh6FdiYqVEgHcsgVy65UKbXE0IuiTr9K\ncv9V7N9/gXPnEqleXZuBAz8t1Rg6N7c+uLn1yfddXkMOYOvW3ZrPxcVs5Y3Lylvezq4Oy5f7AWBv\nXz/fO2bo0JEAODk54+RUtLtg//6tqVHjKhcvbqNtW1vq12/7FqN8e1JTUzl8+CA9evQiMTGRn35a\nzI8/LnyvfSgrKlVSABmAOmayYsUHlCtXuikYBATeljJbWm7Tpg1t2rw6Ca2AgIDA6/Am+d+WLVvM\n5593o1OnLuzbt5uffvLB29uHli1ba1bDc8nJycHfP5g//ggnMHAtP/20ir17d6Gnp0dS0iQSE9tR\ntaoHaWktyMjQ5uHDm6xfv4WKFSu9orfvRq4Efm6+qrS01NcSH1i3bg0ODo5F5kh6V7y99/Hzz63I\nzrZEJrvOnDkn6devdakXG+jbAAAgAElEQVRfJy9xcbFMnjwun6ABlO04PxaePXvKxInPiIvriqXl\nPpYsaUKVKuH5xGxyhWx++cUPZ+dGeHv7EB8fx3ffDWfjxm3s37+HmzdvaNoUi7XIycl56z4lJz9n\n1qw0YmLcALh/PxUbm4OMHv1ZvnJz5rTnyZNgbtwwxcIijXnz6r6y3YICFrk4On6Co+Mnb9THkJCT\nzJpVg7S0TxGJnvD337vw8ur+Rm2UJosWzefevTtkZWXRuXNXatasXXKlPMTHJzJ16iliYw2xtU1m\n0aKOGkXG4mjevD7Nm9f/IDs3KSnJ7Ny5lR49elG+fPl/jSEHMH68C/fuhXL+fDmMjTOYNKkSurpv\nnoxeQKAsKDs/IQEBAYFSYuvWME6dOgFQbP63CxfOAXD9+lW8vX0A+Oyzz1m9ermmnYJe5W3afPqi\nvp0mf9yff57l9u2/qVhxF3p6qxGL05DJztCwoQ7R0fXK1JCDwhL4Dg4vd8Fy+1/UbsPgwcPLrE8H\nDmiTna12J0pNrcvu3dfp16/MLvdKynKcHwvXr9/l4UMnJBK18ZWVVZWrV89ScO6oUqlIS0vVJOfO\ndQEsjneJqoiPf0xsbI0838iIiSlcztDQkODg9+8SDHDwYAZpaWqDSaUy49ixDysQMWvWj+9Uf+LE\nkxw61B8QcflyDlLpRnx9P5xxWhJ+fiuIiYlm4MDeVKlSjfv37xISspn9+/dw6tTvZGRkEB39EHf3\nPmRmZnHkyEGkUm0WL/bFyMiImJholi5dRFLSM3R1dZk8eTrVqllx7NgRgoL8EYu1kMlk/Pzz2vc+\nNm1tbfz8XF+hqiog8OEQjDkBAYGPmrfJ/1bcpLXgP2GpVPtFfa189cePn0SDBo4sXXqUxEQJLVoY\nYmVlzqZNpZNixc/vZywsKvDVV66AerdJX98AlSqH48ePoKury8OHD/D3X0Xt2nacOXOaevXqc/Pm\nDRYvXs66dX7cvHkDkUhEly5f8vXXHvliAi9cOM+qVb4olUrs7OoyceJUpFIpvXp1o3PnroSHn0Kp\nVDBv3gKqVbMqsb9SqbLA8dvv8LwJOTk5LFw4n2vXIjA3t8Dbewk+Pt6acfbq1Q0Xl06cPRuOWKzF\npEnT8fNbQWxsDB4e/ejevWfJF/kIqVPHiipVrhAf3wAAqTSGunUNuHMn/29YLBbj4dGf+fNnERy8\njmbNWpKbA6uo/FfvMgmtVq0q9eod4do1OwB0dO7i7Pxxqenp6ORPTK6rm1VMyX8G9+8b8zKnmZi7\ndw0+ZHdKZOTI0dy9e4fAwFDi4+OYNGms5lzu95mZmbi5fcmoUWMICNjIihVLOXhwH19/7cGiRfP5\n/vtpVKlSlb/+usaSJQvx9V1NcPAvLF26kvLly5OWlvoBR/jfS8ki8M/gwyQOERAQEHhN3jT/m719\nA44ePQTAoUMHcHBwBEBfX5+0tLQSr9e4cTN27NiGlpYWU6d+zpgxtfn8c8d3H0ge2rd34dixw5rj\n48ePYmJiQnT0Q7y9l7J2bTAKhYKmTZtz585tYmKi+eorV9av30JS0jMSExMICdlMcPAmunTpBryc\nvGdmZuLlNYe5cxcQHLwJpVLJzp3bNGVyExR3796LsLANr9XfoUONKFfuNJBK1aoHGDGiaqnej+J4\n+PABPXt+zfr1W5DJDDlx4lihhMsVKlQkMDCUhg0d8fKajZeXD2vWBBEQ8P5X70uLcuXMWLjQkEaN\nwjEzG8SYMadxd2/FtGmzNG7Cufm3cmOrAgI2MnToSE0OrM6duzJ27PeaNhctWkbDhk5v3SddXV1W\nrbLniy/C6NBhOzNnRuLq2vzdBlrKjB5dC1vbnUAs5csfZdQokw/dpXeiWrXneY5yqF79wxoyJZF3\nEa3ggpqjozN6enqYmJggkxnSooXaTdvGpgbx8bHI5XKuXo1kxozJDBzYGx8fL548eQJA/foOzJ8/\niz17fs236CYgIKBG2JkTEBD4qGnSpDm//rqdvn1dqVq1Ovb29YH8K6R5P48dOwlv7zmEhq7H1NRU\nIzbQvn1HFi6cz7Ztm5k3b0ERV1K30a1bd+LiYhk8WC19bmpaDi+vxW8tyFAUNWvWfmGUJfLs2VMM\nDQ25c+c2f/55josX/yQhIQGVKoe7d28zduz3REdHU7euPQCWllWIjY3hp58W06xZSxo3bqppV6VS\n8eDBfSpXtqRKFbXB1blzV3bs2MLXX3sARScoLgl39+Y0afKAq1dP0qSJHRUqmJfOjSiBSpUsqVGj\nJqB2hY2Liy1UpmVLdWy2jU0N5HI5enp66OnpIZVKSUtLxcDg1TFGHysuLg1xcXk7oZkdO8K5eTMF\nZ2cLXFze3oAriJ2dFb/8YlVq7ZU2jo41OXSoApGR/6NGjeolpkT62PHxacXUqeuJjTXExuY53t4d\nNede5XL9MaKtLdV8FovFmuPc2E+VKgdDQ0MCA0ML1Z04cSrXr1/jjz/CGTy4n0b9U0BAQI1gzAkI\nCHzUSKVSfHyWF/r+0KETms9t27bXpByoWLFikTml6td3YMOGLZrjFSvWaD6bmJhodjREIhHDh3/D\n8OHf5Kv/NoIMr+LTTzvw++9HePLkCe3buxAfH0/fvgP48suv8pWLi4tFT+9lsJQ6JmkT586d4ddf\nt3Ps2GGmTp2pOV9wclcwxiN3EqWlJX6jVW5r62pYW1d7ozG+K/kngFpkZ8uLLZNXHCT3+L+4ir9o\n0QGWL29OVlYVDAyimDXrBAMG/HfEyAwNjWjRovT+TovKR1malORyrVJl4+bWlsGDhxMXF8vQof01\nLtft2rmQkpLM6NETANi9eyf379/lu+/Gl0lfS0JfX5/09PQ3qpNrlOrrG1C5cmWOHz/Cp592QKVS\ncfv239SoUZOYGPViVt269pw9G87jx48FY05AIA+CMScgICBQDCqVipMnL/D4cQqdOzcuUUnuTWjX\nzoWFC3/k+fMkVq70JzR0FwEBIdSq1YA6dWqQkPAYiURaqN7z50lIJBLatGlH1arV+PHHlzLnIpGI\natWqExcXS0xMNJaWVfjtt/1FutclJiZw48ZfpTaeVxEXF8uECd9hb9+Aq1cjsLOrS+fOXQkMXKtJ\nfm1pWQVv77nExsaiq6vLwIFDAfXkNjY2moiIK+jrG2BjY0toaAgbNgSRmJjA9evXaNq0xTuJe3xM\nBAX9wqFDBzAxMcXCogK1a9dBJpOxe/cOsrMVVKlS5UVuOV3mz5+Njo4ut27d5Nmzp0yZMoM9e3ZQ\nqVIQGRkOPHrkze7df1G37lkCAtaSlZWFpWUVpk1T5y8TKJl32fl6nXyU7du74Ou7RGPMHT9+lD59\n+nP1agT+/iHk5OQwZcoEIiIuY2FRgZiYaGbMmEvduvbI5XIGDPDgm2/GoqWlxYEDe/j+++lv3d93\nxdjYhPr1Hejf343q1a3zuUPnv4/5vSpyz82c+SM+PgsIDg5AoVDQoUNHatSoyapVvkRHP0SlUuHs\n3FizWy8gIKBGMOYEBAQEimHSpJ1s3NgWhcICB4ethIa2xtzcrFTatra2QS5Px8KiAkuXniYgwBWZ\nzITBg8dhaSmhfHlTZsyYV2gilJCQgJfXHFQqtQjJiBHf5WtXW1ubadNmMWPGZJRKJXXq1KN791x1\nwaInVO+DmJhofvxxEVOnzmTIkP4cPXqI1asDOH36BCEhgVSooDZcvL2XcOnSBZYuXajJYXb//n26\nd+9JVlYWBw7spWvXL/H0HMxXX3XB13cJTZu2eOWE8Z/CjRt/ceLEMYKDN5Gdnc2gQep8cW3afEq3\nbmoVQ3//1ezdu4uePd0QiUSkpqawZk0gp0+fYMqUCejoDOLmzW+oVq0n2tpRiMXJhIRsw9d3FTo6\numzYEMTmzRsZMGDIBx7tu/M6iwRWVjYsW7aIu3fvoFQqGDRoGC1btnlthUWA337bz8KF81AqlUyd\nOpM6deohl8uLbffEiWNkZGSQk5PD7NnzmTlzKunpaSiVSiZMmJpPofZVLtcDB/YGQC5X99HCogIV\nKlTSuFzr6enh5NSI8PBTVK9uhUKhwMbmw+b0LUrBs3PnrnTu3FVznOsFUfBcpUqVWbKksBfG/PmL\ny6CnAgL/HgRjTkBAQKAI7t69y6ZNDVAoqgMQEdGPVas2M2tWl1K7RnDwJtLT03F2voRCUZmkJE+S\nkjxxctrCzz93zlculxo1ahIQUFi4JG8i4k8+aURAwMZCZfJOomxta1CxYqVCapEPHtxj8WJvMjMz\nsbGxYvz4aSgU2UycOIZ169Zz69b/GDSoD9u378XCogJff/0l69dvQUdH55VjrVTJUjPRzJv82sam\nBnFxsTx6FKeZtDk5OZOens6GDVvYtGkjLVu2pm/fAQDs2LGV338/yu+/H8XY2Jjnz5+TkZHxygnj\nP4WrVyNo1aotUqkUqVRKixatUKng9u2/8fdfTVpaKunpcpo0aaap06KFOnGxtbUt5cqZ0bNnbWbO\nPEVmphUVK+6hQwd99u69w4gRgwDIzlZQv36DDzK+sqCkRQIrK2ucnRszbdosUlJSGDbME2fnJsDr\nKSyqVCoyMzMIDAwlIuIy3t5zCQnZTEhIQLHt3rr1P4KDN2FoaEhY2AaaNGlG//6DUKlUyOWFXYXf\n1uUaoFu3LwkJCaB6dWu6dPmijO7yh+P8+WtERkbTunUdatWq/qG7IyDwUSIYcwICAgJFkJmpQKHI\n64omQqksfQFg9Y6SqsB3pe8yGBh4gl27MpFKlQwdWpH69Svw8OEDZs/2YvLk6cycOZUTJ46xcWMI\n48dPwsHBkbCwQAID1zJ69ASysjJJT08jMvIydnZ1uXLlMg0aOFCunFmJhhwUFkDIjW8TiUTk5CgR\ni6XFukrq6OSdwKpYuzZYUz8jIwMvr8PExOhQt24O48Z1fK0k6x8noiLvgZfXXBYsWIKtbQ0OHNjL\n5csXNefyJhHX1pbi6tqURo2iWbAggU6dHDA3Nyc+vgmzZ89/b6N4nxS3SGBtbUt8fCwJCY8JDz9J\nWNh6ALKzs3n0KB6RSKRRWNTT0yuksHj79i1A/fvs0EGdGN3BwZG0tDRSU1M5f/5sse06OzfG0FCd\ntqFu3Xp4e89FoVDQqlVbatasVWgMBV2ub9++hb+/Hx07dkZPT69Yl2t1+/Y8fvyY//3vJiEhm9/6\nPrq4tOLw4VNvXb8s8PM7xqJFtqSm9qJChZMsXfoUF5fSVRYWEPg38E/9jycgICBQptSqZUvHjicB\n9Uq6tfVu+vQpfREEPT09evVKQls7GlBRvfp+Bg2qUWK9N+Ho0cvMnVuDM2d6cuLE13z/fTZxcY8L\nqUXGxESTmpqiSefQo0cPrly5DIC9vQORkRFERFyhX7+BRERcIjLyCg0avJ3iYkEcHBw5dOgAoM4t\naGJi+kIIIr9x06hRU7ZufblTOXz4Ovz8XNmzpycLF7rg7b2/2GukpqZq0jRcunSBSZPGlUrfS4sG\nDRwIDz9FVlYW6enpnDmjnlzL5WmUK2eGQqHgt9+KH18uVlZVqFatAqamJtSrV5+rVyOIiYl+0Zac\nhw8flOk43ifFLRLkFcCZP38xgYGhBAaGsm3bHqpXtyqybkGFxeLI9eYtrt288YgODo6sXOmPubkF\nXl6zOXhwX6H28rpclytnRqNGTXFx6cSIEQPx9HRn5swpyOXpL65d2H24XbsONGjQ8B1jel/fLVml\nUr2XGNXQUAWpqfaAiEeP2hAY+KjMrykg8E9E2JkTEBAQKAKxWMy6dV8TFLSf5OQcevSoj7W1ZZlc\na86cbjRrdpb79//gs88aYGVVuVTbv3DhEWlprTXHcXHNuXx5UyG1yNTUlHz18k7YGjZ0JCLiMo8e\nxdOqVRs2bAhCJBLRvHmr1+rDqxJYi0QiBg4cirf3XFq1akS9evX54YfZmnN5q44dO5GlSxfi6emB\nUqnk7l0LIHccJly6VPwuYUpKMjt3bqVHj17FlvmQ2NnVpWXL1nh6ulOunBm2tjWQyWQMGTKCYcMG\nYGJiQr169vkUA4tL0ZGLiYkJ06fPZvbsaWRlqZNqDxs2iqpV368y6YeiceOmbNu2iXHjJgHwv/9F\nUauW3SuNkYL50o4dO4yTkzMREVeQyQwxMJC9drvx8fGYm5vTrVt3srKyuHXrJp06FXbVzutKDeDq\n6o6rq3uJ5QAiIyNwd+/zirvw+qSnpzN16kRSUpJRKhUMHTqSli3bEBcXy/jx32qUNBcvXs7Bg3sL\nifV4ePQlJiaapUsXkZT0DF1dXSZPnk61alZv3JecnILKvP+8OFgBgfeBYMwJCAgIFINEImHIkI4l\nFywFOnVqWnKht6RePVN0dO6RmWkFQPnyF7G3t+LEifzlDAxkGBkZERFxBQeHhuzatUuTjsHBwZE1\na1bi6PgJIpEIIyMj/vgjvJAAS1HkJrjOJW9836lTv7N2bRA6Orp4e/vQsqUzfn4BmvODBg3TfM7M\nzCQpKYkpU2Zqdj+6dduT71qmpoVjknLx81tBTEw0Awf2RiKRoKurxw8/TObu3dvUrl2HmTPnARAV\ndYOff16GXC7H2NiE6dNnYWZWnm+/HUbt2nZERFxBLk/nhx/mEBISyN27d2jf3oWhQ0eWeC9KwsOj\nH4MGDSMjI4Nvvx2GnV0datasnUfE5iV572Nx91ilUlGrlh1r1wb/Y3KSvQklLRIMGDAEX18fPD3d\nycnJoXJlSxYuXPbaCosikQhtbW0GDeqjEUABGDBgCMuXLymx3cuXLxAWth6JRIK+vgE//DCn1Mb+\n559XmDt3KnXq1MHJyblU2tTR0cHbezH6+gYkJSUxYsRATS7HvEqaxYn1ACxaNJ/vv59GlSpV+euv\nayxZsrDIdDEl0asXLF16h4wMG8zMzuPhYVoqYxQQ+LchGHMCAgL/KPbv38PNmzc0K+ICJdO1axNu\n3fqNPXsuIZUqGDLEhGrVqhc5EZ42bTY+Pt5kZGRgY2PFhAlqqfOKFSsBaNIcODg4kpiY+FquXa9K\ncLx16yY+++zzAnFxakJDQzh+/AhZWdnUrt2AQ4cacOuWNVZWQ6hcOR09PW2++KILcvkGnj37AwOD\ny6SnG7Jy5V2++WZMofZGjhytEb24fPkiU6dOYMOGrZiZlWfkyMFERl6hbl17fvppMQsXLsXY2ISj\nRw+xdu0qpk6diUgkQirV5pdfQti6dRNTpkwgMHAjhoZGuLl1x82tj0YB8W1ZtGg+9+7dISsri86d\nu1KzZu23buvy5Vt8//1fREdXwsYmmp9+akStWh92Ry4uLpbJk8dp4rtCQ9eTkSHH0NCIXbt2oKWl\nhZWVNXPmeBWrGJnLqxYJ8p77/vtphfrxugqLefNR5kVHR+e12i14XFqsXn2MxYurkpp6iMePj3Dp\n0v9wciocj/emqFQq/Px+JiLiCmKxiMTEBJ49ewqQT0mzKLEeULvxXr0ayYwZkzVtZmcr3qovY8e6\nUK/eBaKiLtCihQ1OTo3fcXQCAv9OBGNOQEDgH8W/cXfhfTBu3GeMKxAilnci7OHRV/N5zZpAAMzN\nDUlIeOl6uWPHy3iffv0G0q/fwGKvV9Atq06dety5c5vMzAzatm3P4MHD2bp1E4mJCYwePQITE1PN\n6v3atas4cuQ35PJ0QkI2Y2xsQseO7iQnP8DS8hFKZSqpqd3ZunUUfn4raNbsL8LDT9OmzafMmvUj\naWmpRfYp16h0cWnFwoXLqFOnHuXLmwNQo0Yt4uPjkMlk3L17m7FjRwGQk5ODmZm5po2WLXNFMmyx\nsVErSAJUrmzJo0fx72zMFSXt/rbMmxdFZGQ/AJ4+hR9/3EhIyMflXpn797xxYzDbtu1BIpFonl9x\nipG6uoUN/4+NR4+eEBR0DrEYhgxpjqmpSam1rVKpCAxUkJqqXli5e/cL/Pw2s3btuxtzhw4d4Pnz\nJAICNqClpYWr6xdkZmYBFFDSLCjWo3rRtxwMDQ0JDAx9574AuLg44+JSKk0JCPxrEYw5AQGB98pv\nv+1n27bNKBTZ1K1rz4QJU1i6dCFRUTfyTfRBnXdr+fIlyOUZaGtr89NPqwB1wusJE0YTExNN69Zt\nGTVq9Icc0n8CuVzO6NH7uHrVlHLl5MyYUYNmzexeWSevW1ZycjJGRkYolUrGjh3FnTt/4+rqzpYt\noaxYsQYjI2NNPXv7BmRlZbFnz68MGOBBuXJmZGREI5e7kJz8FVWq9CclZRMREc2RSrWJjY2latWq\naGtrc+LEcc0uQfGoDQipVFvzjZbWS9ELa2vbfK6eecmtk7tLp2lRJCInJ6eE675fnj7Nnxj82bOP\nN1G4rW1NZs+eTuvWbWnVqi1AkYqRjx/Hv1X81fvkyZOnuLuH89dfvQEVR44EsX1753cUKHlJTk4O\n2dla+b4rePy2pKWlYWpaDi0tLS5dukB8fFyR5Ro0cGDRIi/69RuIQqHgzJnTfPnlV+jrG1C5cmWO\nHz/Cp592QKVScfv230KibwGBMkQw5gQEBN4b9+7d5dixw/j5BaClpYWPzwIOHTrAsGHf5Jvo3779\nN9WqVWfWrGnMnbsAO7s6pKeno6Ojg0ql4tat/xEUFIpEIqV37564urpjbm5R5DXj4mKZOHE0DRo4\nFptPzdKyClOnztTIiQsUZv78I+za1ReQcucOTJ++kaNHa79ypzSvW9axY4fYvftXlEolT54kcvfu\nXWxsilbtbN68JZcuXaBduw4ATJ78A+3atUEmO4aBwUlycgyRSJ7j57cCqVRKq1Zt6NPHkwsXzvP7\n70fZsWNLkTE6+vr6+cRDVCoVK1f6cu7cGRITE8nJycHFpRPx8XEMHNgHS8sq3LnzN1WqVGXRop8A\niIy8wty5P7yYUCuYNGkcixYte9vbWqY0bJjM9esZgC6QjKNj8fGE7wstLS1ycl7u6GRmZgDg4+PL\n5csXCQ8/RUhIgGbXeP78xf84sZbNm8+9MOREgIjLl/uxY8du+vcvnfhbLS0tunRJZd26xyiVFpQr\nd55evcq9U5u5f8cdO3Zi8uTxeHq6U7t2HapXty5UBooX6wGYOfNHfHwWEBwcgEKhoEOHjoIxJyBQ\nhgjGnICAwHvj4sXz3LwZxZAhatevrKwszMzMCk307927A4CZWXlNUL2+vj6gnlB88klj9PUNALCy\nsiYuLrZYYw4gOvohc+Z4F5tPbd26NZp8agJFEx+vw0vVSIiLM0cul2ueS1HkumXFxsawadNGfvll\nPTKZDC+vOWRlZb7yek2aNGXJkoUaY1BXV4svv3Tj1q1satQwYPToTvzxRzgrV/pqlDibNWtB/foO\nuLl9WWSbxsYm1K/vwJEjv7F69XJyclQoFNkEB29iwYJ5HD16mAEDhjB48HAWL/YmOzsbsVjM/fv3\nuHo1gpycHNavD2Tt2iDi4mKZM+cHPmav38WLu2Fm9isPHkipXTuH8eNLP3brTSlXzoykpKckJz9H\nV1ePM2dO06RJMx49isfJyZkGDRpy9Ogh5HJ5sYqRHzv6+hIgE7URDZCKTFZyLsY3Yd68L7C3P8WD\nB+m0bl2Npk0bvVN7hw6p1ZCMjU2K3ZUuqKRZUKyndm31u7pSpcosWbL8nfojICDw+gjGnICAwHul\nc+euDB/+jeY4NjaG8eO/LTDRz3rlJLmgpH5J7m0l5VPr1KkLM2ZMeYdR/fuxt1exZ89TVCr1DkDt\n2rHo6zd7rbppaWno6uphYGDA06dPOHv2jEYlU19fn7S0tHxulqDOJ9egQUNOnfodT093RCIxOjrR\n9O/vzMqVvvTtuw4DAwMaNnQiOzuLSZPGkZWVhTp2R0Ry8vNCbYI6Ju306ZP4+4ewfPkSatSohUgk\nYurUmSgUM7lx4zoAtrY1CAzcCICPzwLi4mIZO3Yivr5LqFixEjk5Klxd+3LlynmgeKGMD4lUKmXG\njMIy+B+ShITHaGlJGDrUE3NzC6ysrFEqlcydO4O0tFRUKhWuru7IZDIGDBjCmDEj6dfva0Adl3jp\n0oWPLrl1Qfr0acuRI8EcOvQFoKBbtwN07+5WqtcQiUS4u7cuuWAZUlCsJzo6k8mT9yOXS2nfXsHE\niZ0/aP8EBP4rCMacgIDAe+OTTxozZcoEvv66N6ampiQnP+fRo/giJ/rVqlnx5EkiUVHXsbOrS3p6\nGjo6ukXmhyopgW1J+dQESmbMmI5kZh7g4kUp5cpl8MMPLUusk+uWVbNmLWrVqk3v3j2xsKhIgwYO\nmjJffNGDCRO+w9zcAl/f1ZodV1C7W4JapfD58ySWLl3IypXhKJVKnJwaMXHiFAIC1qKvr4+/f7Cm\nnqvrF6+V1FgkKijioI5Hio5+SEpKsua7l/F06vH4+PzG6tWVUalkVK8er4kHFHg9jI2NNWqWxZGa\nmkpiYgKJiQmsW7ceY2O1gIiLy4c1YF4HqVRKcLAbp05dRCrVolkzN8Ri8YfuVqmTV6wnKekZ7dtH\n8PCh2miNjIyhSpXTuLuX/J4QEBB4NwRjTkBA4L1hZWXN0KEjGT/+G3JyVEilUsaNm1TkRF8ikTB3\nrjfLli0mMzMTXV1dli1bWUR+qDdXuCyYT+3gwX2anSKBohGJREye/Plrl3+VbHxeevZ0o2fPl7sW\nue5eAG3btqdt2/aA2v1rzhzvQvWtrBrg77+cvXt3IxaL8PQcAsC2bZsJDz+FUqlg3rwFVKtmRXLy\nc7y955KRIWf48IG0a+fC0aOHiYuL5d69u4SHn3whrR5BSkoKAwf2pl+/QZprVatWnejohxw+LCYl\npTkVK27n2bNqLFt2klmzPrwL45vi6OjIoUMnC6ULKGtyd+L+978orKxsmDFjDlevRrJqlS9KpRID\ngwqcO/c5aWl/U778Y4YPH0yFChb51E7PnDmNjo4OCxYswdT03eLFygItLS3atv3vSOlHRd3j4UNH\nzXF2tiV//XXmA/ZIQOC/g2DMCQgIvFfat3ehffv8WtP16tkXWdbOrq5GJj+XgnmbXkd8oqR8apaW\nVYo1NgQ+PlQqFWtzuAAAACAASURBVPv3n2H9+rNcvFgBHR0HDA0b4e9fHWvrCvj5rcDExJSAgA3s\n3LmNsLANTJ78A+vWraF27TpcvHiB4cO/YcWKpTRq1IRff92OXJ7O1Kmz6NixE35+P3P06CGNvHpE\nxCVAnVusX7+BLFz4M4aGG8jIqA+IyMiQvqK3AgV58OA+U6fOxN6+Ad7ecwkL28Du3TtZvtyPKlWq\n0qrVEFJS0khKmoKx8SEMDNzw9VW7WmZkyLG3b8CwYaNYtWo5u3fvxNNz8AcekYCdnRVVq17h4cMq\nAEilsdSta1BCLQEBgdJAMOYEBAT+MRw5cpktWx4jkSgZMaImDRqUrJBWcIfI3b0PV69GkZiYxsqV\n/kgkwmvwn4RKpWLcuO1s2tSZnJwOSKVrKFfuFM+fm7B06U0CAkYC0KZNOwBq1bLjxIljgDrR8fz5\nixkwQL179/z5cwYMGIKurh5isZiOHTsB6h24XBdPIF+C+s8++5xNm5ScODEIC4v5SCQGdOtWtczH\nLZfLmTlzCgkJCeTkKPH0HMLq1ctxcenE2bPhiMVaTJo0HT+/FcTGxuDh0e//7J1nQBRXF4af3aU3\nqQpWigZUBLEX7L3Ghl0Ru35qbFHRWFGJXWygKFgRxa7BCPYSY0Ox90pTUEDqwrL7/diwiqAxStFk\nnl87s3PvnDtDmTPnnPfQqVNXUlNTcXefSFLSW7KyZAwZMiJH4+2ioHjxEtjbOwDK67lx43pKlixF\n6dLK6yiV1kRb+zIJCa4AJCe/awGhrq6uuje2thW5fPlCIVsvkBeGhkYsWmTEihU7SE9X1sz16iXU\nzAkIFAbCU4yAgMB3weXL9/jpJzViY7sBEBa2jwMHjCle3OSz51AoFPz88x62b69NZmZJGjUKYsuW\nzt9FE+KCpEWLBv9IVOLq1Suoq6urHsgLkydPnrBrlxNyuTkAmZkjefbMCF1dPSIjV+Pvr/y3ll0n\n+X7/OPh4faWm5rufgQ8juQqFgoMHz/HyZTJi8Qt0dI5TrdpmdHVL8tNPY6hXr1K+rjEvLlz4A1PT\n4ixa5AVASkoyPj4rKVHCHH//AFauXMr8+bPw8fFHKpXSv38POnXqiqamJp6ei9DR0SUhIYHhw92K\n3Jl7//oqFAr09PR5+zZRta9SpSSuXs3661gZ9eu/66Emkbx7bBGLRTnurUDR0rSpA02bFv7fBAGB\n/zr/vopcAQGBfyUnTjwhNraeavvx41acOhX+j+a4dOkGAQHOZGZWBEpz6pQb69adzF9Dv0v+Wc1h\nWNhlbty4XkC2fBqZLAu5/F1ao0QSi0Khjr6+Hj16dOH+/XsfHevg4ERIyGFAuQZDQyN0dHRzOXgf\n9qP7+ec9DB1ajWnTuuLnV45fflnA778fZvfuDTRs6EhhYGNTgcuXL+DtvZLw8Gvo6ip7emU7ZtbW\n5alcuQra2toYGhqirq6uUof08VmFq2svxo0bSVxcLPHxbwrF5o/x8mUMN2/eACA09Hfs7CoSHR1F\nZGQEADY2b3F21qJfv12UKCHGxSX/61nv3r3D8uWL831eAQEBgcJGiMwJCAh8F5QurYVYHIdcbgqA\nru4DfvihFHK5/LOV4uLjU5DJ3hdLUCet6PsoFzgBAZvR0NCgW7eerFixhEePHuLl5c2VK5c4dGg/\nkLeoxPHjx1m5cjUyWSYGBsWYOXMu6enpHDiwB7FYQkhIMGPHTsLRsWqhraVChfK0bbuDAwfKAXqY\nm3tTokQIRkZ6nD+vy4QJUz5oM/FOMGfgwKF4es7B1bUX2tra/PLLLOURIlGOVhhOTjXYunUjbm69\n6dixC7t22SGXlwDgwYOubNgQyK+/Fm4j6zJlyuLnt43z58/i67uG6tWVfcWyI5BisRh19fdVW8XI\nZDJOnTpMYmICfn5bkUgkuLh0RCrNKFTb30ckElG2bDn27t3Jr7/OwdLSmh49+lC5chWmT59MVlYW\nFStWxsvLHTU1NXbvTsihdvp+VO+fCh+9j51dRVUPy89BJpN9Vkp2TEw0N26E06JF6y+27XOZN28W\n9es3UIkECQgI/DcRnDkBAYFC5ciRYHbt2oFMlkmlSvbY2FQgJiaKkSN/AiA4+CD37t1h3LhJuY51\ndX3D4cMm6Ou7Y2tbm2XLXtK4cVPu3buLp6fyLfulS3+yd+9u5s9flOvcjRo5UavWXi5edAPEWFvv\nw8Ulb/GVfxOOjtUIDNxKt249uXv3DjKZDJlMxvXr16hatRpHjx7JU1SiRo0arFu3EYCDB/exbdtm\nRo0ay48/dkVHR4eePfsW+lpEIhFr17rQqNFREhIy6Ny5D6VLj89xTFDQftVnO7uKrFjhA4CBgYHq\n5+R9Bg4cmmPbwMAAX9/NgDKKBB9Gsgo/qSUuLg59fX1atmyDnp4+Bw/uy/H9x9JHU1JSMDIyRiKR\nEBZ2mZiY6MIw96OYm1uwbduuXPurV6+Jn9+2XPs/pXZqa1uRdevWMH/+bG7cCMfOrhJt2rTH338d\n8fEJzJzpAYCX1xIyMqRoamri7j6TsmXLERZ2mcDAbSxcuEylchoVFYWWlhaTJk3DxqY8GzasJSoq\ngqioKMzNLXJI8X+MqKhIQkOP/CNn7nMdxQ/JS9lXQEDgv4fgzAkICBQaT58+4fjxUHx8/JBIJCxZ\nsgBtbW1Onz6pcuaOHw/F1XVQrmMXL/6VJk008fCoS7NmmfTr144mTZoD0KdPNxITEyhWzJDffjtI\n+/Y/5nl+LS0ttm9vw5o1O8nMFNOzpz3W1qULbf1Fha2tHffu3SE1NQUNDQ3s7Cpy9+4dwsOvMnbs\nzx8VlYiOjsbDYx5v3rwmMzOTkiVLqeb8jDZuBYZEIqFfv/yNRhw+fIXff49DRyeDn392xtjYCFCK\ndXTqdJbt28ujUBhjbb0fN7fPj+jkF48fP2T1ai/EYhFqauq5IpC5H+yV2y1btmby5PG4uvbE1rYi\n5cpZ5RiT1+dvkVevXnPmzHUqVCiFg8MPqv2RkRHMnbsQd/cZDB7cn2PHQvD29uPs2VNs3uzP9Olz\nWL3aF4lEwqVLF1i3bjVz5y7MMXe2yqmn5xLCwi4zd+4MlZLps2fPWLNmPSdOHGXIEFfVi6W2bTuy\ncOE8fH03kZWVxdChrsye7YmPzyqeP3+Km1tv2rTpQLduPfD2Xsm1a1fIyMikSxcXfvyxC2Fhl1m/\n3gcDAwOePXvKpEnT2LBhLYaGRjx58ghb24rMmKF0RjduXM+5c6eRSqXY2zswadI0le2f009RQEDg\n343gzAkICBQaV65c5N69uwwe3A+AjIwMjIyMKFmyFLdu3aR06dI8e/aMKlUc2b17R45jpVIpJiYm\naGhoIBaLc6QWtWrVliNHgmnTpgO3bt1UPQTlhb6+PpMntyvYhX5jqKmpYWFRiuDgg1Sp4oiNTXnC\nwi4RGRmJpaXVR0Ul5s6dS7duvahfvwFXr17Bz29dUS2hQAkNvcZPPxmQkNAYUHDrlj979nRGTU0N\nkUjEsmVdcXY+TWxsKu3bO1GmjHmh21irVh1q1aqTY9/7EcgPW3a8/52Pj1+OcfHxb0hKektYWBix\nsUm5FF+/NcLDHzBs2AseP26Bnt49Jkw4xv/+p/z9t7AohbW1DQBWVtbUqFHrr882xMREkZychIfH\nDCIjXyASiZDJZLnmz1Y5BahWrQaJiYmkpqYgEolwdm5IVFRkrpdQL148w9m5Ib6+3kil6bRq1RZr\naxtGjBjN9u1bVS1T9u/fg56eHr6+m8nIyGDkyMGq+/jgwT22bNmJubkFYWGXefjwPlu3BmFiYsqI\nEYO4fv0aDg5V6dKlu0qB1cNjBufOnaF+/QYFe9EFBAS+GwRnTkBAoFBp06Y9w4b9L8e+3347wPHj\noZQrZ0mjRk0+eSyAhoZmjkhC27YdmTx5HBoaGjRt2vyza+j+Szg6VmX79q1MnToTa2sbVqxYSsWK\nn1ZhTE5OxtTUDIDDhw8ByjTYS5cuUKNGLTZsWIuOji69en1+uuU/Vc4sDI4efUlCQre/tkRculSb\n58+fqZwEkUhEt25FqwCZHygUCsaO3U1wsBUSSQZDhlxgwoQWfz+wiPH2fsDjx8pUy+RkJ/z9nzBi\nhBx4VzMIOesGxWKliun69T7UqFETT8/FxMREM3r0sDzP8SmV0w9fQkmlUoyNjXFzG8KgQf3Q1NRU\nta/4cJ5Ll/7k0aOHnDx5DFCmvUZEvEAikVCxYmXMzS1Ux1asWFn1+1a+/A/ExETj4FCVsLBLBARs\nQSpN5+3bt1hb2wjOnICAgArhiUdAQKDQqF69FidOHCM+Ph6At28TiYmJoWHDJpw5c5KjR4/QvHnL\nTx6bF6amppiamrJpkx/t2nUonMV8Zzg6OvHmzWvs7atgZGSMpqYmjo5OwMfT7UaNGsX06ZMZNKgf\nhoaGqlS+lJQUfv89mP379xAdHaU6PizsMpMmjfsbS769dL5ixWTAu4iNoWEUhoaGRWdQAbF9+0kC\nAzuRmNiYN29asny5I+fOXStqs/4WmUySYzszU4JcLv/bcQqFgpSUdy8kfvvtQJ7HfY7KqbIWLwB/\n/wACAnbj5jaEhIQE0tPTSEtLRSqVftSO8eMnqcbu3LmfmjVrA6ClpZ3jOHX1d/30sltqSKVSli5d\nyLx5C9m0KZAOHTqRkVF4AjbR0VH0798j1/4NG9Zy+fLFT47dsGEt27dvLSjTBAQE/kKIzAkICBQa\nlpZWDBkygvHj/4dcrkBNTY0JEyZjbm6OpaU1z549wc6u0t8em1d9T4sWrUlMTKRsWctCXtWXER0d\nxcSJY3BwcOLmzXDMzIrj6bmE58+fsmiRJ1KplFKlSuPuPgN9fX1GjRpK5cpVCAu7THJyElOmzPhH\nKpLVq9fkxInzqu3t2/eoPoeEnOLw4UMEBm5DJBJhY1Oec+fOEBCwET09PfT19enVqx+6unrs37+H\nqlWrMX78JPz81qGtrQMoa5e8vVfy/Pkz/ve/IUyePI2yZS2Jiopk9uxfSE9Po379hvl3AfORceOa\ncvOmH+fPV0Ff/zWjRyswNv78/oXfCy9fSlEojFTbUmkZnj4Np379IjTqM+jWzZRz5y7w+nVtxOJY\n2rRJVAmGfPi34P1tsVhMr179mTdvJps2baBuXWfef5mQfejfqZxWr16LKVMm0L17b4yMjHj7NpHU\n1FSWLVvIkCEjiIqKxNt7BePGTUJHR5fU1BTVOWrVqsuePbtwcqqBmpoaz58/o3jxEp+99mzHzcCg\nGKmpqZw4cZSmTYs+mjpoUN4Rzvf51uswBQT+LQjOnICAQKHSrFkLmjXL/TAikUjQ09OnX7/uuLj0\nomPHzvz66xxcXHrxxx9n/6r7KklqagrFihmqFOBiY2MZMWIgdevWp0OHTkWwoi8nIuIFs2d7Mnny\nNGbMcOfUqeNs27aZ8eMn4ejoxIYNa/H3X8eYMRMQiUTI5XJ8fTdx/vw5/P3XsXz5mnyx4/HjR2zc\nuJ6SJUsRHx/P3bt3qFatBo0aNSI09ChPnz7BzW0AERGjSU9/g4nJUZWs+/PnT+nTpxuxsa9o2LAJ\nRkbGuLoOYsmSBXh5eePltZguXVxo1aote/YE5Yu9+Y22tjbbtvUkLi4OHR0rdHV1i9qkAqFNGzu2\nbAkhIkIZ/a5Y8Tdatcr/Hm75TevW1TE2vsPJk0GUKaNFz57K3/MPa/2mTp2p+vz+d++/uBgyZASg\njPQXK6aMvn6OyumHL5YaNGiEuroGzZu3Qi6XM3z4QMLCLuPgUBWJRMKAAb1p27YDLi49iY6OYtCg\nvigUCoyMjJk/f1Gudhgfbmejr69Phw6d6N+/B8bGJlSqlFN9tzAcJrlczoIF83K8dFq82FPVFuH8\n+bOsWrUcLS1tqlRxICoqSlUz+PTpY0aPHsbLlzF0796Lbt16Fri9AgL/NUSKb0QKKTY2qahNECgg\nzMz0hfv7Lya/7u/bt28xMDBAKk1nyBBXVq1aR7t2zVmwYBn16jmzZs0KdHV1cXUdxPz5s3F2bsSR\nI4kEB99HW/s3TExM2Lt3xxdJfBcF0dFRjBs3isBA5YPmtm2byMjI4NCh/ezeraxPi4yMYPr0Kfj5\nbWX06GEMG/Y/7O0dePPmNSNHDiYwcG++2LJrVyDXroWhr1+MyZOVSnk3b17H338tcXGvycjI4Pnz\neKKiFiGRxGNk5IujY3Vq1dJnz54gVq/2ZcgQVzQ01FEoFJQsWYrMTBlbt+6kXbtmHDgQgkQiISUl\nmU6d2hIaejpf7Bb451y6dJeAgKeIxXKmTauDsbHx3w/6l3H27Cm8vVfi7j4Te/sqOb5bsSKU/ftB\nIpExaFAxevSoV0RWfh359Xc5OjqKnj07s2HDVsqXr8CMGe44Ozfk8uWL1K/fgDp16tOrVxfWrFmP\nubkFs2ZNIy0tlQULlv2VinmBlSvXkZKSTO/eXVV/CwS+DuG56t+LmZn+Px7zfTz1CAgI/OsJCtrO\nmTPKHlKvXr3ixYsXH5XM79ChE0uWLCMkZDUWFod4/nwHL16oc/LkVZo3r1lka/in5BRvkJCc/Ol/\nztk1NWKxRKU4mR+IRCIMDY24cOE83t4rqVevAb6+a6hVqwZnz/5BSkoKkIaGxkOyspTph2lp6iQm\nJmBgYICFhQX6+vr8/PNUDhzYq3orL/BpNm/2o3//gYV6zpo17ahZ0w747z4QOjs3wtk5t6BNcPBF\nFi92Ij1d2b5h5sw/cHJ6Snj4n+zfvxtbWzumT/+4Um5BolAoOHHiEi9fvqVt25oUK1as0M5tYVGK\n8uUrAMo2J9l1sgqFgufPn1KyZCmVkEvz5q04cED5kkkkElGvXgPU1NQoVswQIyNj4uPfqGoYBQQE\n8gdBAEVAQKDICQu7zJUrl1i71p+NGwOoUOEHMjKkH5XMr1LFkbi4ONTVnwFZZGSU/6v+53URrSB/\n0NXVw8DAgPBwpSjF77//hpNTwafBVatWkytXLuHl5Y2NTXl8fFby4sUzdu7cybx5C3Fyqo6amg4i\nUToAYnEGFStqqsbr6OhSsmRJrl9X2q1QKHj48AGgvFfHjoUAEBLye4Gv5Xtiy5aNRW3CN8vHhDcK\nklu33qgcOYA3b2pw5cpD9u3bxfLla4rMkQNwd99H374V+OmndnTqdJqoqFeFdu4PXzrlfJH0YZpn\nzmQvNbWcaqMyWf69hBIQEFAiROYEBASKnNTUFPT19dHU1OTp0yfcunXzb8e0bt2aN29GExs7HgBL\ny2Batfp8QZBvgbzEG6ZOncXixZ6kp6dTqlTpHHVAH4zONzusrKzp3NmFSZPGoqamjqGhISVKWHD3\n7m0mTRpH1apOqKtnUKPGTeTyGJKSMmnQoCL3798jKektkZERzJgxl+HDB5KWlka/fj1o3rwl5ctX\n4KefJjJ79i9s27YJZ+dG/1lRBHf3ibx69ZKMDCkuLr2IiookI0OKm1tvrK1titRR+KdER0cxYcJo\n7O0duHEjHDu7Sn+pPa4jPj6BmTOVa/HyWkJGhhRNTU3c3WdStmw5goMPcvRoCG/exJGeLqVhw8aM\nHDmGQ4f28/jxQ8aMmQDA0aNHeP06rlDXVb26OXp6t0lOVoowmZuf486dE0RFRTJhwmiaNWtJZGQE\njx8/IitLxsCBQ3F2bsTPP//E8OGjsbEpj5tbbxo1asqAAYNZv96HEiXMv7qWNyoqku3bbZHJygJw\n61YvvL134OGRf/0yv7RlSNmy5YiKiiQmJhpzcwv27dvN27dvAaGhuYBAYSE4cwICAkVO7dr12Ldv\nN337ulCmTDlVHcvHJPMBevXqye7dATRvnoGa2g4GD65QJM2cv5QPxRve79W2dq1/ruNXrPBRXQND\nQ8McTaHzg+zm4SKRshfWxInuXL58jgMHDnL37h2aNWuJubkFbm5DmD9fKcM+ZMgIHByqMmnSWDQ1\ntWjatAVRUREsWPAuzdLComSOptXZAhT/NdzdZ+SqCd29eyf+/gFFbdoXERkZwdy5C3F3n8Hgwf05\ndiwEb28/zp49xebN/kyfPofVq32RSCRcunSBdetWM3fuQgAePXpA+fIV8PRcQu/eXXFx6UmzZi3Z\nssWf//1vLBKJhJMnj6Gjo8ucOdO5f/8ulpbWTJ8+mydPnrBq1TLS0tIoVsyQadNmYmJiSkTECxYt\n8iQxMQGxWMzcuQswMjJmypQJJCW9JStLxpAhI3B2bkR0dBSTJ49j8+YdAAQEbCE9PY2BA4fStetc\nLl78A5FIxA8/lGXOHG+6deuAlZUNu3fvRF1dnbFjJ+LoWI2hQ12pUaM2jo5OhIdfxdzcHDU1NW7c\nuA7A9evX+PnnqV99rTMyMsnK0nxvj4isrPxOrPr4S5aPvYARiURoamoyYcIUJkwYjZaWNjo6Oqp0\n8Y+JuggICOQvgjMnICBQ5Kirq7N48Ypc+0NCTqk+N27cjMaNm6m2r1+/RrNmLfjll86FYmNRsXHj\nadavTyUjQ43WrVOZPbtDgUS3atWqQ61adXLsc3auSZ8+g3Idmx0tjIl5SVKSCC+vtZia5pTyj4h4\nyZQpfxARoY+19VuWLGmKkdG/r3fb55JXTej3woeRuHLlLDE2NmbRonnExydQpkwZKleugrv7BJ49\ne8arVzHcunWDPXuCePjwvirCNnBgXzp27ExWVha3bt1g2LABaGpqERMTTZUqxalWrSbnzp2hXDlL\nZDIZMTHRzJw5F3t7Bzw957B7907OnDmJp+dSDA0NOXYshHXr1uDuPoPZs3+hf383GjRoTGZmJnJ5\nFmpq6nh6LkJHR5eEhASGD3fLs1Yuu38iwIMH5zh27CBqamqkpCQDkJSURNWq1Xjx4jlSaTrTp0+h\nbFlLMjMzefUqBkdHJ3btCsTCoiR16zpz+fJFpNJ0oqOjKFOm7Fdf/3LlytGmzQ727y8P6GFpeZC+\nfW2/eL4Po8QdOyr/hq5cuZSLF//E2NiU2bPnY2hoSHJyEpqaWri69srVKqVECWWdnI1NeTIyMti0\nKZA2bZoA4ObWmz59Bqj6hgIq51lAQCB/EZw5AQGB74qQkDC8vPxITX3A2LGTi9qcAuX+/SfMm2dC\nYqIyncrX9yWVK5+hR4+i79e2f/9Fpk1T8OqVE2XKXGDpUiMaNXqnDDh58nlCQ/sDcPu2Ak3NLXh7\n/7sd74/xfk2opqYmo0cPIyPj402mv0Xej8S5uvYkPT1dFYlbtmwRCoUCe3sHxoyZwJgxw5kxw53B\ng4chl2cxatRYVq1azpo16zl69AgODo7IZFksXLiMSZPGqWqwOnT4kc2b/ShXzoqmTZuTnJyMvb0D\nAK1atWXTJj8eP37EuHEjAaVkvomJGampqbx+HUeDBo0B5cshUEcmk+Hjs4rw8GuIxSLi4mKJj3+T\n5/qyUwJtbCowa9Y0GjZsrJovI0NKUNB2VSqhiYkpHh6eqp6WMpmMu3fvULJkaWrWrE1iYgL79+/F\n1rZivlx7kUiEj48L9euHkpAg48cfq2BlVeqL5/swSty4cVPS09Ows6vE6NHj2bhxPf7+6xg3bhJz\n585k/PjJebZKyXaAjxwJJjb2FQMG9MbGpgImJuaEh9di/Hh1rKz2sHx57a+yV0BA4NMIAigCAgLf\nDdevP2DcOHUuXdrErVt/MHu2Nk+fRhW1WQXG7dsvSEx811cqK6sET56kFqFF7/D2juPVq+aAKS9e\ntGP16pyRpogIvfe2RB9sf3u0aNGgwOb+WE2ompoaMpmswM6bn1hYlMLa2gaRSETp0mVVzeKtrcuT\nmprKs2dPadWqLQBaWlpkZEjR1zegShVHli1bREpKMklJbxGLcz92ZDtSlSrZ8+rVK0JDf6d+/Zz1\nlQqFAl1dXaysbPD3D8DfP4BNmwJZunQlH4puZBMScpjExAT8/Lbi7x+AkZExUmkGEokEufzdGKk0\nXfV50aLldOniwr17dxkypL/K0Zw2bTbdu/emRo1a7Np1kLJlLbl//y6gvI9mZsU5ceIo9vYOODg4\nERi4lapVnb7iiudE2buuOWPHtv5qxygoaDsDBvRm2LCBqiixWCymWTNlFK1lyzZcv36NlJRkkpOT\ncXRUrqN163Zcu3Y113w//tgVM7PibN26k44dO3P5cjwXL/YjJqYj58+7MmvW5a+yV0BA4NMIzpyA\ngMB3w4kTj4iNfdf3KSKiOcePXy9CiwqW+vUrY2l5QrVtaHiVevW+jTfcUqlGju2MDPUc21ZWibx7\nyM7C2jq5cAz7YgquuKd27XpkZWXRt68La9euVtWEduzYmQEDeuHhMb3Azp1f5FQ0FKscLZFI9Jcz\nJlI5ZWKxGB0dHfz81nH8eCj16jVAoYARIwbx5s1rPrzW7zttTZs2x8GhKrq6urx8GcPNmzcACA39\nncqV7UlIiFftk8lkPHnyGB0dXczMinPmzEkAMjIykErTSUlJwcjIGIlEQljYZWJiogEwNjYhIeEN\nb98mkpGRwR9/nFWt4+XLGKpVq8GIEaNJTk4mLS0NTU1NDh7cw4ABg5HJZPTs2Zl+/bqzYcNald1V\nq1bDyMgYDQ0NHB2rEhcXq3KCYmKiCQ39tJJrXi8TgoMPsmyZss4wOTmZvXt3fXKOz+FjysHwzqlW\nKBR/m8otkUhQKOQAuaLMaWk5k75iY3W+2m4BAYGPI6RZCggIfDfY2BRDQyOCjIzSAOjo3KNixZJF\nbFXBYWZmwurV5nh77yAzU0Lnzvo0bPhtNDFu1SqD+/ejyMwsibb2I9q2zdkIeMmSpmhobCEqSg8r\nqyQ8PVsXmC1paWnMmDGF2NhY5PIsXF0H4+Ozkg0btmBgUIy7d2+zerUXK1euJTU1leXLF3Hv3h1A\nxMCBQ2nUSFnns27dGv744yyampr8+usSjIzyp6H2x2pCnZyqM2LE6Hw5R2Gio6PD6NHjVNvFixen\nevVahIQcZsCAwfz000RWrVqOn99WIiMjKFWqNJMmTeWXXyZjaWlFzZp1WLlyKUCunoTXr4fTs2cf\nRCIRZcuW1bCnbgAAIABJREFUY+/enfz66xwsLa3p1q0ntWrVxctrMcnJyWRlyejRozdWVtZMnz6H\nRYvms379WtTU1Jg7dwEtW7Zm8uTxuLr2xNa2IuXKKdsOqKmpMWDAYIYMccXMrDiWlsr9WVlZeHjM\nICUlGYVCgYtLT/T09Ni//wgrVixh6FBX5HI55cpZ5hD5ARg8eDiDBw8HwNTUjNOnL6q+i4qKJDT0\nCC1a5P4dkMlkqKmpkdfLhPcdqqSkt+zdG0Tnzt0++z5lO2fvz/OxKLFcLufkyWM0a9aS0NDfcXBw\nQldXD319ZasUR8eqOVqlWFiU5O7d29jZVeLkyWOq+XV1dTE0TAYyAXUgHXv7b/1FjoDA943gzAkI\nCHw3tG9fl+HDD7FvnxZisZw+fUTUrduiqM0qUN5v8vwtMWVKW6ysznD//jmcnIxp375pju9NTIxY\nt65wauQuXPgDU9PiLFrkBUBKSjI+PivzPHbjxvXo6+urlESTkpTKe+npadjbOzB06EjWrFnBgQN7\ncXXNLf7yNWzbdpZt21IA6NlTm/79i7728XPIysrKs43G+5/d3Ibg6TkHV9deaGtr88svswBlSl9Y\n2GVEIjHW1jbUqVMfyE4b7E2bNu24eVOfK1dEpKWtompVW6pVqwHAtm25I1EVKvzAqlXrcu0vXboM\nXl7eufa/r6T6Pt269aRbt5659q9Zsz7XvqNHQ7h58wYikRhbW1sGDx7OmDHDSUxMxNDQiKlTZxAb\nm8qYMXNJTdVHR+cpRkZyRo8eS+PGzfDxWcXz509xc+tNmzbt0dc34OTJY6SnpyOXy5k3bxFSqRRX\n115oaGggEomQyWTEx79RNev28VlJZGQEbm69qVmzDiNHjiEgYDMnThwlIyOThg0bM2jQMKKjoxg/\nfhTVq1cjPPw6ixevoESJdyq/H1MO1tLS5vbtW2zatAEjIxPmzJkPwLRpebdK6dWrL9Onu3PgwF7q\n1nUm2xl1cqqBmZk/1ao1R0enCY6Otkyd2j7PeyAgIJA/iBTfSCOQ2NikojZBoIAwM9MX7u+/mKK4\nv3m9cRbIf76X390XL54zfvwomjZtQb16DXB0rIqLS8c8I3ODBvVjzhxPSpUqnWOOpk3rcfz4HwAc\nOxbK5csXmDz5l3yz8fLlO/Tpk4lc/ozExN4YGFxn1qzLXL/+R67o1NcQHR3FxIljcHBw4ubNcMzM\niuPpuYS4uFiWLl1IQkI8WlpaTJ48jerVq7B3729s3uyHTJaJgUExZs6ci5GRMRs2rCUqKoKoqCjM\nzS2YOXNuvtn4PgsWBLNkSTtAWVNZqVJbDh1aj56efoGc75/w9u1bBg7cSlTUAbS1BzJzZmWqVi3D\n3Lkzadq0Oa1bt+O33w5w9uxp7txpwKNHNxGL04mOXkaHDstJSfmNwMC9XL16he3bt6ruc3DwQdav\n92HTpkD09fVZtmwh+/fv5eTJ81y6dIFVq5axaVMgu3btwNfXmyNHThITE82kSWNVipAXL/7JyZPH\nmDRpGnK5nClTJtCnT3+KFy9Bjx6d2LFjBxYWVp9aXr6hUCh4/Pgx6urqlC379eqdAn/P9/K3WeCf\nY2b2z//2CTVzAgIC3x3vK6kJCJQpUxY/v23Y2JTH13cN/v6+OUQupNKMHMfn9Q5TInmXqCIWi1TC\nF/nFlStPSUoqh6HhdgDevnXg/v3Yr5rzYzZGRLyga9fubNmyEz09fU6dOs7ChfMZN+5nNmzYwsiR\nP7FkyQIAHB2dWLduI35+22jWrCXbtm1WzfPs2TO8vLy/2pGLjX3N//63j+7dQ/DwOJTD7ocP1VA6\ncgpAQUzMWESib+PRZM6cE1y/bkF8vAs3bgxi1qzHGBgYcPv2DVXKZKtWbblx4xoxMXqAiOTk5oCI\nxMSyvHmjVM788OdNJBJRo0Yt9PWVD203boQjkSjTlJ2cqhMVFUm/fj3Yvn0z6elpxMe/yTXHxYt/\ncunSBdzcejNoUF+eP39GRIRShKhECQscHBwK8Mq8IysriyFDdtCggQRn5zR+/nmP0CxcQKCQEdIs\nBQQEBAS+a+Li4tDX16dlyzbo6upx6NB+VU1PnTr1OHXqXU1PzZq12bNnJ2PGTACUaZbZD9X5TWDg\nVoKDDwJQtWpdSpYMRl39OWXLdiIrqwJ2dpU5fz6cX36ZzJMnj7C1rciMGR4A3L17J8/m2KNGDeWH\nH2y5fj2c5s1bUry4ORs3+iIWS9DT02PatFlYWJRSpefZ2toRHR3FzZvhTJ/+rpVHZqZSRfPVq5fM\nmDGFN29ek5mZScmSSoEdkUiEs3NDNDRyCt38HT4+qyhevARdurgAsGHDWnbvvkd0dCYSyVuePJES\nF3cPL68JREdH8ezZEkqUOI+W1l0iI9dhZvYLMtnuXNevfftOdO/e65MNv4OCAtm/fw8SiQRLSytm\nz57/Rfctm7g4bSCNbCGf2Fh9lfrohw6Lre0bwsIUKBTqQBqVKkk5f/7jTo22tnae+0NCDiOXK1iz\nZj1nzpxk+fLFuV5GZNO37wB+/LELAIcPHyIwcBsBAVtISkokMjKSiRMn5UgFLVHCnHnzZqGpqcWD\nB/eIj3/DlCnTCQ4+yN27t6lUyV6VRtmiRQM6d+7G+fPnMDExZfDgEfj4rOTVq5eMGTMBZ+eGSKVS\nhg8fw82biZQsGUxs7BS2bWuKufkKXr+OQCqVEhkZQcOGjRk5csw/uPICAgL/BMGZExAQEBD4rnn8\n+CGrV3shFotQU1Nn4kR30tPT+fXXOaxfr4eTU3VVJNfVdRBLly6gf/8eiMUSBg4cSsOGjXPVgH0t\nd+/e4fDhQ/j6bkIuVzB0qCuurl0JCnqMsXFfevXSwc5Om82b77F1axAmJqaMGDGI69evUamSPcuX\nL2LBgqUUK5azOXZ2PdX69Zv/Wk9Pli5djampKSkpybx9+/YD5UkJb9++QU9PH3//gFx2Llu2kF69\n+lG/fgOuXr2Cn9+7ejRNTa1/vO5mzVrg5bVE5cydOHGUmJheREV1RaHQQyx+Q3h4e0DpTKenx1On\njgkPHw6iXr2TZGYqa8byun5OTtVypV++H6Xftm0Tu3blbPj9NTg5wbFjtlhYbCE+fgCVKsWSmpqC\nvb0Dx46F0KpVW0JCDuPo6MTkya3o1+8wCoU2Tk5vmD69DW3aLAJAR0eX1NQU1bwfOoIODk48fvwY\ngHv37qCtrY2+vj7Pnj1RjdPR0SE19V1bktq16+Dr60PLlm2Ijo7C39+XxYtXoK6uzsSJY/Dw8KBt\n2w6qVNDlyxfj6bkYgOTkJNau9efs2VNMmTIBHx8/rKysGTy4Pw8fPqB8+Qqkp6dTvXotRo78ialT\nf2bDBh+8vLx58uQx8+bNxNm5IXv2BJGZCc+e/Ya6+mNKlx7E06cHSExM4+HD+2zcGICamjq9e3fF\nxaUnZmbFv/qeCAgI5EZw5gQEBAQEvmtq1apDrVp1cu3fvn1Prn3a2tpMmzYr1/6QkFOqz40bN6Nx\n42ZfZdP169do2LCJyiFq1KgpxYopsLTUY/PmVoBSJr5ixcqYmpoBUL78D8TERKOnp8eTJ48YOzZn\nc+xssvuBAVSp4si8eTNp2rSFSpXzQ3R1dSlZshQnThylSZPmKBQKHj16iJlZNVJTU1TnP3z4kGrM\nl6bKVahgS0JCPHFxccTHv0Ff34DixUVkZCxFW/syCoUYufytqnl3iRIW+PqOUo13cfFFoVDkef3C\nw6/i7Nwo1zk/1fD7axgzpgUKRSinTtWgWLEOaGgYsGrVbcaOnYSn52wCArZgZGTE1KkzMTAwoHbt\nctSvX5VGjZRiQNlOZvnyFVSCL23bKgVQ3n9hMHDgUPbuDcLVtRfq6uqYmprh6tpTpSYJUKyYIVWq\nONK/fw/q1KnPyJFjePr0KcOHu5GYmIBIJP5LFVOp1nnt2jVmzfoVUKaCenuvUNlUv76yDYKVlQ3G\nxiZYW9v8tW1NTEwU5ctXQF1dndq16/51XcujoaGBRCLB2tqG6Ghli4cbN8Lp3bszDx4c4MmTjmRm\nlqRixXU4Olqjq5uFjo4uAJaWVkRHRwnOnIBAASE4cwICAgIC/1liY18zd+5Z3r7Vpm5dNYYObfr3\ngz6DvKJ7eQX81NXfpTFKJGJVPZmVlc1HlRi1tN6l6E2c6M7t2zc5f/4cgwb1Y/78RXkqT86Y4cHi\nxb+yaZMfMpmM5s1bUrduNQYOHMr06ZPR1zegevUaql5syojXP142AE2aNOfkyaO8fv2a5s1b8urV\na3buvEZKygAqVUokOXmjKnVQWzvv6N+Ha8juffZ3Db+vXQvj3LkzbN7sx6ZNgapatC9BJBIxdmxL\nxo5tCUzJ8V1eypnZKYrZZL8gUFNTy3V8mzbvFB4NDAw4derC39rzYe2ii0tPXFx6snv3Dl6/fq1K\nkd20KZAOHVp81CFXV1dGbsVica7+gdk/f+/XkIpEyoj3h8cAmJubsmmTKVu37uTatVimT6/Kmzev\nckWH5XL5365PQEDgy/g2qowFBAQEBAQKGYVCwZAhx9i+vQ+//daV2bOr4e9/6u8HfgaOjlU5ffok\nUmk6aWlpnD59gipVquZIlfsYZcta5tkcOy8iIyOoVMmeQYOGYWhoiEgkVrVdAKWEvJvbECwsSrJk\nyQo2bgxg69adDBgwGABn50bs3LlfJYyyYoUPoIwW9ezZ94vW3rRpC44eDeHkyWM0adIcU1M9fvzR\nnosXW/DTT5a8evXyk+NFIlGu63fmzEkcHJwwMjL+7Ibf6elpX2R/UbBo0WFatTpKx46HCQm5+o/G\nlitXnoCAfdSseZDOnfcRFnYLJycnjh0LAVClguY3jo5VCQk5jJ2dJUOHVkJLK4Pq1avl6UQKoigC\nAgWHEJkTEBAQEPhPkpAQz61b5cnukZWZWYZLly7h5vb1c//wgx1t27ZnyBBXADp06IytrV2OVLm6\ndevnGf1SU1PDw2NBns2xP2TNGi8iIl6gUCioUaOWSvjkSzh37jaHDr1ASyuT8eMbfbEwjJWVNWlp\nqRQvXgJjY5OPNu+GvCKYyu28rl+FCj8AfHbDb11dvS+yv7AJCjrH8uX1yMxUtst4/vww1au/xsTE\n5LPGr137ghcvJmFs7MeLF2ImT9YkOHg5EydOypEKms3n1Ifmju7m/q5zZxcWL/bE1bUnEomEadNm\noaamlqfasKA+LCBQcAh95gQKHKEfyr8b4f7+e/m331uZTEaDBkd59Mglew/DhgXh4fHfaHL8/v09\nf/4OgwdnEBvrDMipV8+PoKAuqpS8r6Vbtw74+W3FwKAYLVo0IDT0TL7M+29g1qzfWbPG5b09L9m1\n6xYNG9b8rPEdO4by559dVNt2dnu5c6fzv/p397/Ov/1v83+ZL+kzJ0TmBAQEBAT+k6ipqTFzZgnm\nzw8kPl4PJ6eXuLt/H45cbOxrAgMvoqkpxtW1MZqaml8136FDz4iNzXYoxJw/35CHDx9TsaLt1xuL\nMjKTlpaOh8cM0tPT6d+/B66ugylWrBhr1niRlZWFnV0lJk50R11dnW7dOtCiRWv+/PMcYrGESZOm\n4eOzkqioSHr16kenTl0BCAjYTGjoEaKiEjAwqMygQT1o3bp6vthcWDg4GKCp+RypVNlwu3TpK9jb\nV/rs8XZ2Kfz5ZwagAcixs3tbMIZ+BgqFgt9/P8+rV0l06FALY2OjIrNFQOC/guDMCQgICAj8Z2nd\n2olWraoik8nyLQpV0Lx69Zru3c9y+3YfIJPQUH8CAly+yn59/SxARvZjgZ7eS4yMSnzRXO7uE3n1\n6iUZGVJcXHrRsWNnAK5cuYipaXG0tLTZvHkHycnJ9O/fgxUrfChdugxz585k795ddO/eC5FIRIkS\n5vj7B7By5VLmz5+Fj48/UqmU/v170KlTVy5e/JMXL54TF9eJq1ddKVnyf0yYEIWamoTmzat+8bUo\nbLp0qcfz5yGEhl5BSyuT//2vJMbGn5diCTB3blvU1IJ4+FCLUqVS8fBo+feDCoiJE/cQENCKrCwz\n/P2D2LatJqVKfdnPkYCAwOchOHMCAgICAv9pRCLRd+PIAWzefPEvR04EaHDqVDeOHbtE69b1vnjO\nMWOacOWKP+fO1UVXN44RI15jbv5lDpG7+wwMDAyQStMZMsSVxo2VCqFWVtb4+/uSmZlBePg1dHR0\nKFmyFKVLlwGUCo979uyke/deAKo2BNbW5UlLS0NbWxttbW3U1dVJTk7m4sU/OX/+HDEx4ZQtewCx\nOI3k5BaEhr6kefMvvhRFglI188vGamhoMH9+x/w16At4/vw5QUH2ZGVZAHD7di98fALx8GhXxJYJ\nCPy7EZw5AQEBgSIkLS2NGTOmEBsbi1yehavrYEqVKs2qVctIS0ujWDFDpk2biYmJKZGRESxdupCE\nhHi0tLSYPHkaZctaFvUSBAoZZTsxOZAtuy9FQ+PLJfhB2ZQ6MNCFp0+fYmBQFjOzL09VDArazpkz\nSlXQV69e8eLFCwBKlSqNn982fvyxFb6+a6hePWdNWHb7gWyy5e3FYnEOZ1spjy8DoHv33nh4/MDr\n19neWybFiu36YtsFvpysrCyysnK+FJHLBeETAYGCRmhNICAgIFCEXLjwB6amxdm4MYDNm3dQp05d\nvLwWMW/eQjZs2EK7dh1Yt24NAAsXzmPcuJ9VMvJLliwoYusFioLBgxtQs+ZGIB14TYcOh2jc+PPE\nMj6FRCLBxsYGMzOzvz/4I4SFXebKlUusXevPxo0BVKjwAxkZUgBev37zV/NpNXr16sfNmzeIiYkm\nMjICgCNHgqlatVquOfPSaROJRNSuXYeTJ48xalQC5uaH0NcPpnHjVYwb93UN3wW+DEtLS9q3vwQo\na/ZsbPbRv3/lojVKQOA/gBCZExAQEChCbGwqsHq1F97eK6lXrwH6+no8fvyIsWNHAiCXyzExMSMt\nLY0bN64zffpk1djMTFlRmS1QhOjp6REU1J4DB35HT0+Ttm17IBZ//N1sXjVsLVo0oHfv3hw/fgIT\nE1MGDx6Bj89KXr16yZgxE3B2bohUKmXJkl+5d+8OEomEUaPGUa1aDYKDD3L27GmkUimRkRE0bNiY\nkSPHAHDq1HHu37/HqFFDKVHCnPDwdz3Tnj59zKxZU0lPT2PjxvVMnOhOcnIS06dPJisri4oVK9Op\nU7e/js4pn59T2l75uWbNOjx9+pRDh/xxdJSjqanF7Nnz0dbWJjk5mdDQ3+ncWTlfWNhlAgO3sXDh\nsvy5CQK5EIlEeHu70KDBceLjM+jUyYkyZcyL2iwBgX89gjMnICAgUISUKVMWP79tnD9/Fl/fNVSr\nVgMrKxt8fPxyHJeSkoy+vj7+/gFFZKnAt4SOjg49e346AhUUFMj+/buxtrZhw4YtOWrY0tPTqVu3\nLm5uI5g69Wc2bPDBy8ubJ08eM2/eTJydG7JnTxBisbIJ+fPnTxk3bhTbt+8B4OHD+2zcGICamjq9\ne3fFxaUnIpGIc+fOYG9fhdjYV4SFXcbExPQva0RUr16DJk2a0bJlI3x9N6ns9PPbloft+1Wf27Rp\nT5s27fP8zsWlJy4uPXONT0p6y969QSpn7mvJyspCIvm6VNb85syZk5QpU07Vay8/+Nq2EWKxmL59\nhciogEBhIjhzAgICAkVIXFwc+vr6tGzZBl1dPfbt20VCQgI3b97A3r4KMpmMFy+eY2VlTcmSJTlx\n4ihNmjRHoVDw6NHDr2oSLfDvZt++XXh5ebN//x4GDOgNvKthU1dXp0GDBsTGJmFjU/6v9EcJ1tY2\nREdHA3DjRjjduvUAoGxZS8zNLXjx4jkikYjq1Wuho6MLgKWlFdHRUSQkJODkVJ1p02YBsGtXIC9e\nPMfJqXoOBywk5NRXr00mk7F//1mkUhldujizb98ugoMPAtC+fSdu3bpBZGQEbm69qVmzNnXrOpOW\nlsovv0zmyZNH2NpWZMYMDwDu3r2TZ43qqFFD+eEHW65fD6dFi1b06NHnq+3OT06fPkn9+g3+kTMn\nk8lQU/vUo59Q4yYg8L0hOHMCAgICRcjjxw9ZvdoLsViEmpo6Eye6IxaL8fJaTHJyMllZMnr06I2V\nlTUzZsxl8eJf2bTJD5lMRvPmLQVnTgCAwMCtOZyZ58+fEhUVyciRgwARW7bsRFNTk9Gjh5GRIUUi\neffvXyRS/uxBtrhI1t+eL1ucRDlGQlZWFqIP/ACZLIuwsGdMmHCEOnUMcHGp+/ULRRklGzBgJyEh\nvQF1tm1bhLHxH6xfvxm5XMHQoa7MmOHBkyePVJHssLDLPHhwj61bgzAxMWXEiEFcv36NSpXsWb58\nEQsWLKVYMUOOHQth3bo1uLvPQCQSIZPJWL9+c77Y/XdER0cxceIYHBycuHkzHDOz4nh6LuHIkWAO\nHtxLZqaM0qVLM336HO7fv8e5c2e4du0qmzf74eGxAE/POYwaNQ47u4q8efMGF5euBAUdIDj4IKdO\nHSc9PR25XM7ChcuZMmUCSUlvycqSMWTICJVyqICAwPeH4MwJCAgIFCG1atWhVq06ufavWrUux3Zc\nXBxZWVksXuz1Qf2QQF6MGjWU0aPHY2trR7duHfDz24qBQbGiNqtAuHv3DocPH8LXd1MOZ+bChfMM\nHjyCY8dC0NTU5OnTJ9y6dfOz53V0rEpIyGGqVavB8+fPePkyhnLlLLl3706uY0UiERUrVmbFiqUk\nJSWhra2Nn18QkZENiI3tRlDQY5KTT+Hm9vVOw8GD5wgJ6QnoA/DgQWmaNi2HpqYWAI0aNeXatau5\nxlWsWBlTU6W4S/nyPxATE42enh5PnuSuUc2mWbPC7dkWEfGC2bM9mTx5GjNmuHPq1HEaN26q6tXn\n6+vNoUP76dq1B87ODalfvwGNGilbP+SuLXzHgwf32bQpEH19fbKysvD0XISOji4JCQkMH+4mOHMC\nAt8xgjMnICAg8I0zb95vbNxojlRqQNOmO/D17fpd9UUrCt5/qM0v51cul39SaKSouH79Gg0bNsnT\nmalWrQYhIYfp29eFMmXKYW9fBch9Td7fzP6uc2cXFi/2xNW1JxKJhGnTZqGmpvZRp8HU1Ix+/dwY\nMsQVAwMDEhNLIJcrHej0dGtOnLiKm9vXr1cmyyLn44sYuTyn4mVet1xdXUP1WSJ5F4HMq0Y1Gy0t\n7a819x9hYVFKFW23tbUjOjqKR48e4uvrTUpKMqmpadSu/S7CmZfSZ17UrFkbfX191Rgfn1WEh19D\nLBYRFxdLfPwbjIyM839BAgICBY7gzAkICAh8w9y8eZ+1ayuRnu4AQHBwJdatO8j//te6iC0rHAIC\nNqOhoUG3bj1ZsWIJjx49xMvLmytXLvHbbwdo06YdGzasIyMjg1KlSjN16ky0tXM/gO/atQMDA4N/\nPE+3bh1o1qwlly5doE+f/ujrG+Dn9/fnK0zycqyyd2loaLB48Ypc379ftzZw4NA8v9PQ0GDq1Jm5\nxn4oSPK+QmSLFq3p2LEzmZmZNGkygPT0Kqrv9PTSP3NFn6Zjx/oEBGzj7Fk3QIK5eRCJiWlIpenI\n5QpOnz7BtGmzCQzMLawCykjmtWtXsbOrxK1bN3ny5HGeNapFQe70VSnz58/h11+XYGNTnsOHD3H1\n6hXVMe/fe4lEgkIhByAjIyPHvFpaWqrPISGHSUxMwM9vKxKJhLZtmzJixCAqV7YH4KefRpKUlEif\nPgO4fPkCPXr0+Whd3tmzp3n69DF9+w746JqCgw9y794dxo2b9PkXQkBA4LP59l4xCggICAioiI5+\nQ3p6qff2aJGYWDDnGjFiYMFM/BFiYqIJDf39k8c4OlYjPPwaoHwIT0tLQyaTER5+FRub8ixa9CsL\nFizDz28rtrZ27NiR9wO8vb3DJ+fZtMmP5cvX5JpHJBJRrJghfn5bqV69Fps3++Hllfu4osTRsSqn\nT59EKk0nLS2N06dP4OjoVGjnT0tLY8KEfXTpEkrfvj/j6tqTAQN6Ua1aSYyMUlBTu07VqluYNKlG\nvpxPQ0ODgIDOzJ27n5kzd3HwoA9durgwZIgrw4YNoEOHztja2lGliiP9+/dgzZoVf0UTlePt7CpS\ntary+qipqVGnTj18fFYyYEBv3Nx6c+vW9Y+eWyYr/HYgaWmpGBubIJPJOHIkWLVfR0eHlJQU1baF\nRUnu3r0NwO+/f/z3KiUlBSMjYyQSCWFhl3n79i0zZ85l+nQP5HI5IpFSYbRZsxZMnvzLJwVWnJ0b\nftKRg/yLjAsICOSNEJkTEBAQ+IapV68Kjo6HCA/vD4iwsDhOu3Y2+Tb/+/23vL3zTjUrCGQyGVFR\nkYSGHqFFi49HGW1t7bh37w6pqSloaGhgZ1eRu3fvcP36NZydGxITE8Xo0UORSCRkZsqoUsUhz3nK\nl6/wyXmePn2scmY/nKdZsxYA3Lp1g6dPHzN8eN7HFRU//GBH27btGTLEFYAOHTpToYJtoZ1/8uTD\nBAb2QflI8SOdO29h7dquACQnJxMXF0upUu3yJTU4LS2NGTOmEBsbi1yehavrYKZN+5nRo8fTo0cf\nWrRoQGzsS/r1646JiSmTJv2Cj89KTp48xpgxEwDlz3x0dDTjxk0iOPgghoaGzJ49n7NnT7N5sx97\n9gRx9GgIc+Z4YmRkzIYNa4mKiiAqKgpzcwtmzpz71ev4GHk5PoMHD2Po0AEYGhpSubI9qampgLKe\nb8GCeezatYO5cxfQq1dfpk9358CBvTRr1pRsZcr302IDA7dy8OA+YmKiOXHiGLq6SkVSD48ZtGvX\nkYwMKXfv3mbgwD65RFX+/PMP1q1bg1wux9DQkOXL1+SIumVfP5ksEwODYsycOVdI3RQQKAQEZ05A\nQEDgG0ZXV5etWxuyYkUgmZnquLiUw9GxYBQss3tMhYVdxs9vHSYmRty5c5cmTZpjZWXN7t07yMjI\nYP78xZQqVZp582ahoaHBvXt3SUlJZvTo8dSr5/zJZtPZqnpZWVlkZmby7NkT3Nx606ZNBxo2bIyH\nxwwN72x0AAAgAElEQVTS0tIAGD9+Evb2Dujp6ePm1gd1dXWePXtCWNhlUlJSePz4EQqFApFIhIFB\nMby8vD+6NjU1NSwsShEcfJAqVRyxsSlPWNglIiMjsLAoRY0atZk1a16eY99Po/zUcUVJjx596NGj\nDzKZjPj4eORyOUFBBwrl3A8e6PPucULCgwfvhGb09PTQ09NTbW/YsBYdHV1SU1NwdHSiRo1ahIdf\nZdEiTzQ01PH29mP9eh/+/PMcdes6q5qRZ3Phwh+YmhZn0SIvQNl/cd++Xarv09PTqV69FiNH/vTR\n/nkAWVky9u7dhaampmqso6MT7u7zWL78KlFR4cyZs4BlyxYA8OzZM9asWY+GhgYFhYVFSTZtClRt\n9+rVV/X5XTP1d1Sp4sjWrTtz7Nu0aTsAZmb69OkzCHiXFpstlOPntzWHUM7UqT/j4+OHgUExKlWy\nZ/v2rarU2WxHMD4+noUL57FmzXrMzS1ISkpSfZ+No6MT69ZtBODgwX1s27aZUaPGfnZdX0GQ/fP2\n/rV8n4Lo1ScgUNgIzpyAgIBAEfO5kuQeHnPQ1NRi3rxZaGpq8eDBPeLj3zBlynSCgw9y9+5tKlWy\nV9U5Xbz4Z571XX/++QcrVy5FU1MLB4eq71ny7sHs4cMHrF79OxkZYlxcOtKhQ6f/s3eWAVFlbxx+\nhu6yQLBAKSXEVuze1V1XxVpFReVv69qFhaBgIq4oJuiCK3Z3d6CirphgEAoiHQIz/w8jIwgoKpj3\n+TQz99xzzo2B+877nt+PVav8CQraxNat/8qyHM+fR7N6tT/Pnj1l5MjBbNq0/b1m07lV9a5du5rn\nwTEjI53Fi/9GSUmJp0+fMGvWNFav9sfEpCoHDuxl1ix3bG3t6Ny5AzVqWOHsPIw9e3YxZcoMzMws\nSEtLIzY2hgoVKhZ4nm1sbAkM3MiUKTMwNjZh6dJFWFhYUr26FYsWeRAR8QxDQ6NC+7G0rFGkdl+L\n8+dDmTjxAc+eVcLE5Aze3jUxN69c4uMaGCQDEnLuH+n7gsl5+B8w4H+yzw4d2o+jY3/atGkPwO7d\n29m//3iBWSoTk2r8/bcXPj7eNGzYGBsb2zzBgqKiokwgpDD/PJBmhrdvD6JHj7cP+U+ehDN4sCvp\n6QqIRJmEhWlx4kQIIpEIe/sm+QK5qKhIJk78C3//f4t4pr4uuYVyrl69R1xcBUaODEQsTpO1KSjw\nkkgk3L59E1tbO/T1DQBkYiq5efHiOdOnTyIu7iWZmZmUL2+Yr82X5kMlnp/i1Scg8K0hBHMCAgIC\n3wAfI0kuEolITk5i5cp1nDlzkkmTxrJixVqqVDFm4EBH7t+/R5kyZWXru5SVVdi4cT3//vsPPXv2\nwdPTDW/vlRgaGjF9+uQClf8sLCwpXbo0MTFJGBlVkD0gGxubEBx8BZA+KLVoIS1BNDKqQPnyhjx+\nHP5es+natevmUdXLTWZmFosXe/DgwX3k5OR49uwpIH2AB2jUqDHKyiooKytTrlw5dHR00NHRwcNj\nDtnZUuEHZ+eh7wnmarJhwzpq1LCS9WNjUxMdHR2mTp3JzJlTeP06s9B+dHV1i9TuazF37gNCQ3sC\ncONGQ9zd/8Hfv3KJj+vu3pj09A2EhWlRsWIi7u55rTb8/NZw4MBedHX1KFu2HGZmFri7z6JhQ3uS\nk5M4fvwoly5d5MKFc6SmppCWloaT05/07t0fO7vaLFw4l+fPowEYOXIsa9f+w9y5s5k2bQKKiopk\nZmaSlJTEtGkTyMrKYtAgR0aOHItIJCI4+ApPnz4hKiqSlJRktmzZhLFxVR4/DiMpKYk1a1agra0D\ngIfHXJ49G0BKSldUVS9RqpQ3x49HUqkSMqXQ75mcwCY5OZmRI8OJi6tOdrYeenqX2bnzEn36tP7g\nvu9j8WJPevbsQ6NGjbl27Spr1/p+cJ+SoKD7bffuHezate2DXn1Xr17O5+n3I1x7gR8bIZgTEBAQ\n+Ab4WEnyRo0aA1JZdT29Uhgbm7x5b0x0dCQvXjwvcH3XkyePKV/eEENDIwDatGnPrl3b880nt4y7\nSCSSvReJRO81lf7QQ9/7lB///fcfSpUqjYuLK9nZ2bRo0RAAU1MzGjSwR0lJGYlEQrt2v2BubgmA\nsrIyS5Ysz+ch5+29UvY6p9ywVq06HD9+XvZ5TrYQpBL+q1blN4d+t1SxsHbfAvHxKu+8/zIqm/r6\npQkI+KPAbaGhdzh27DDr1weSnZ2Fk1NvzMwsAOm90qFDJ0JCbuTxS2vduonM7HvmzKl069YLa2tb\noqOjGT16CH5+mzAxqcqjRw+oVKkKqakpBAT406/fAC5evICrqyfjxo2Q/dDw9OkTvL1X0qqVPUuX\nLqJy5SpkZ4tRV9egQ4dObN8eRN++PYmOjkBV9RUpKaCjsw5l5dtcvfqE+/dVaNOmnex45s6djUgk\nom7deiV9aosVGxtb3NxmYW1dlwcPLKlYcT3R0Z5IJOsICUkqdD+RSET16lYsXDiPqKhIDAzKk5iY\ngJaWdp4fZFJTU2Q+fvv37ynx4ymIgu43c3MLmjZtTseOnYD3e/VpamoW+AOagMC3jBDMCQgI/NTk\nrBOLjY1hyZIFzJnjUaT2xc3HSpLniEnIycm9s6/UP0tOTr7A9V337997Z+RPX88ikUg4fvwI7dt3\nIDIygsjICCpVqpzHbNrHx5vbt28yc+YUTEyq8fTpY7ZtC6JzZwfU1TV49OgBgYEb6dmzN8HBl4mK\niuLixQsYGBggFouJiorE1XU6AI6O3fPJ7Oco+mlpaRMcfI9Fi+6TlqZEixYwbFjhmYZPOdbNm0/w\n8mU6v/1mh5FRuWLru7ioUyeR0NBUQA2RKI569TK+9pQICbn2prRPGVCmUaMmBbYrbF3VlSuXePw4\nTPY+KSmJQYMciY+PR15env79B/H330v4779bLF7sSUZGOpMnjyE1NZWsrCxEIhENG9pz//5dsrPF\nVKxYkV69+jJ/vhvq6hrs3r0DKytb3Nw8mTFjMqdOLUNHJxBIQE1NnZ07dzJq1BAuX75Iv34DmTt3\nFmPGTMLGxpbly71K4IyVHDlCOUuWuGFikk5s7AAyMiwAMZUrS/+G5Fb9zI2Ojg4TJkxl6tTxiMUS\n9PT0WLRoWR5xFScnZ1xcJqKpqUWtWrWJjo7K1eeXUbQs6H6TSCiyV1/+dvULGEVA4NtCCOYEBAR+\ncqQPGaVLl/lgIJe7/ZfgXUnysmWLFkDk/JJe0PquSpUqExUVKfv88OGDefZ7+7rwvnO2iUQiypXT\nZ9CgvqSkJDN+/GQUFRVlZtPdu/9BTMwLPD0XY2VlS8+enbG1rcmxY4fp3NkBE5OqxMe/Yu/eXURE\nPKVUqTIkJ6cgEkFY2COZOEVsbAxWVjYsX74633x+++0Pxo4dgZ5eKa5d68L9+z0AuHTpMWXLnsPB\noWGRztn7kEgkjBgRRFBQZyQSHTZs2Mn69emYmVX67L6LEw+P3yhXbg+PH8tjYSFi2LBfv/aU+Pzv\niwRfX798Sphr1/qiqqqGubkF3t4r6dChVaHtFBQUuXnzBv37D+TEiaPY2trRunU7zp49hUgkws3N\nEwBn52E8ffqUJUv+pl+/XmzbtheAyZOn4+IyieTkZJKTk7Gxka4zbdv2Vy5cOPeZx/dlyRHK2b8/\nmCVLokhL206TJgMZOlTqG1izZi1q1qwla587w12/fkPq18/7fcrtOWhv3xR7+6b5xnzXl7BkKfh+\nK6pXn7v7LObNW1RgOwGBbxXBZ05AQEAAqZiBo6O0nGbfvt1MmTKesWNH0qNHZ5Yvz2+6HB8fz+DB\nTpw/f5bY2FiGDRtE//69cHTsLvMz+xjeJ0k+ZMiAfAv08wZeeff181uLoqICU6fOZMqU8bRo0ZDB\ng51Ys2YFW7duZsKEqUyYMBonp97o6ZWSBWc5ZtF2drXx8HhrBO3tvRIzM3NA+rCXe1udOvVYvdqf\n2NgYGjSwB96aTXfp0o0//3Skbt0GqKqq0qHD71Svbk18/CtiY2MJC3tEtWpmbNy4GRUVVe7cuY2c\nnOhNwCgnMxnW1y+fJ5D7668JsofDLl26ExCwFWfnUdy/bydrk5FRiWvXEot49t9PZGQEu3bZIJHo\nAiIePuyEn9/tYum7OFFQUGDChPZ4eDTE0DBBtmZswoS/vtqcbG1rvvHAyyA1NYWzZ99mtYuiclin\nTn2Cgt4qPObPLBe1nSifUEpmZmaBfcnLy5OVlc0ff+yiUaMjTJ16ALFYnK/d11Rp/Fzat7fj4MFf\nOXWqFXPm/F5o5qwo3pM3blyjd+9uODn9SVzcS/76awfduh1i2rRd+czLi4tr165y69ZbP8AdO7Zy\n4MDeQu+3tLSUInn1FebpJyDwLSNk5gQEBAQK4MGDe6xfH4CCgiK9enXBwaEHZcqUBeDVqzgmThyD\ns/NQateuS2DgRurVa4CjoxMSiUQmrV9UPlaSPEetsqB9c2+zs6vNvHkLmTjxL/z8AmWCBPXqNeCf\nf97KuRfG+9bG5Sf/w+C7D4hSGwFo3rwVJ04c4eXLl7Rq1Ua2vXfvfvz+e+c8+0RFRaKqqiLbf/bs\nPZw9q4aGRhrjxpnQsKF0/VWlSoYYGt4gIsL4zdhxVKnyab5mmzZtZN++3QB06NAJMzNzDAxmkJJi\nj6rqNbKyyiEWN/ukvr8ESUmJbN8exB9/5L93vjSmpua0bNmafv16oqurh6Vlddm2wn6QyP169Ohx\nLFrkQd++PcnOzsbW1o5x4ya9aUeR21lb2+Dp6S77fl65cgkDA0MePw5j5sypzJzpxoEDe6lZsxbq\n6hrEx8sTHm5Genpt4uK8sLLSe2OzoElIyHWsraWlxJ9LUNAmdu7cipmZOS4urp/dX3FTFO/J3Gqk\nAwduZdcuR0COEycyyMoKYt683/O0z8rKQkHh8x4/g4OvoKamTo0aUp/HTp26yLa9e7+JRDBw4OAP\nevW5us4rtJ2AwLeMEMwJCAgIFECtWnVRU5Ma6lauXIXo6CjKlClLVlYmo0YNYezYSdjY1ATA0rI6\nc+fOJisri8aNm1GtmmmRxti/fw+bNv2DSCSiatVqDBw4GHf3WSQkJKCjo8uUKdMpV04fN7eZqKtr\ncPfuf7x8+ZKhQ0fSrFlLYmNjmTFjMqmpKWRnZzNu3GSsrW3p2rUja9duREtLm61bN/P06ROGDh2I\nRCLHvXv3uXjxLhDH/ft3SU1NRUdHBy+v5VSsWJlhwwYRFvaQjIwMTE1NmTbNlalTJ/DyZSwVK1bC\n2tqWo0cPsWTJcu7fv8exY4e5ezeU9PQ01qxZydmzp8nOzsLVdZ5McKF3776IxRJOnz6Bi4srCgoK\neHjMISEhnr//XgVAvXr1WbVqBW3atEdVVZWYmBcoKOQNxnx9j7F8eVskklIAPH++hSNHKqOqqoq2\ntg7u7posWbKJ1FRlmjRJZuDA39895R8kx4tr1So/mRdXzZquKClFExXVnhcvZmFq6oCVVcqHO/tK\nrFjhTUTEM/r374WCggIqKqpMmzaRsLCHmJlZMH26NGg4f/487u5zyc7OxtzcknHjpGWyPj7enD17\nGnl5eerWrc+wYaN49epVPlVJKyubIs3H0dEJR8fCMzy5f4CAtxliAG1tHWbNmptvHycn5zzvi9LO\n3r4Jhw8fYMECd0xMqlK/fkPMzCxZsGAuffv2lNl3iMVikpJ6UqbMfOTk0nj9uiKKim1kc5UKoEiz\ngZ+7FmzHji14efnIhEOgeIKd4uJd70kdHd0899Hu3TtkaqQXL54jNLQppUvPR139DADXr0tLNoOD\nr7B69Qq0tLR4/DicCROmsmbNSjQ1NXn48EGhXpYFGZGnp6eza9c25OTkOXRoH6NHT+DKlYsyP7kG\nDRpx5swpUlJSSE9P59dff0dTU5MjRw5RvboVwcFXSE5O4saN69jY2Mq8+pKSErGwsMbff9N7hZoE\nBL41vo2/FgICAgLfGPkFSaRZKgUFBczNLblw4ZwsmLOxqcnff6/i3LkzuLvPpHv3P2nX7v3rlR49\neoi//1pWrlyHlpY2iYmJzJkzg19+6Ui7dr+yd+8ulixZwNy5CwCIi3uJj89awsPDmDRpDM2ateTw\n4QOyjKBYLCY9PR14m9kIDb3DuXOnMTQ0olmzznh5zebVK0ceP66AgcEc1qzxo3JlY7p27cjcua64\nukqNvq2ta+LpuZhJk0YzZco4xo+fgpfXAoYOHcXEiX9hYFCe+fPdsbCwpH79hrIHUR0dXdau3cj2\n7VsIDNzIxInT+OWXDgwa1BeAjh3/kAW6aWmplC1bDj09aWBWp059wsPDGTy4PyAtf3Jxcc0jnnD/\nfrYskAMICzMnOjqKKlWk2bj27e1o3/6TLreM3F5cAE2btuDGjWsYGhoxcWIi0dFbyc6uR1ZW+ucN\nVIIMGTKSsLBHrFsXwLVrV5k8eSwbNwZRqlRphgwZwM2bNzA1NWfy5MksXrwcI6MKzJkzg+3bt9Cu\n3S+cPn2CgICtgNSUG8DLa0EeVclx40awcWPQ1zzMj+b33zsTGlqOiAh5wsN96d27P9WqmbJy5bp8\nbY2NVTl9ehPSjHMKDg5SVVMzM3PWrw+QtXvX1PxjmD/fncjICMaOHcHz59E0atSEyMgI9PUNGDVq\nHAsWuOcLntPS0li82JOwsEdv1BqdC1ynVnzk9p68l+c+Cgm5TseOnbh5860a6cWL80hJieLx413I\ny79EQ6M9L19Kv//3799lw4bN6OsbEBx8hQcP7hMQsAVNTa1CvSwLMyL//fcuqKmpyXwCr169JMvU\nzpkzgzFjJmJjU5M1a1aybp2vzKpCLBazapUf58+fZd06X5YsWQ7A/v3BTJ2awLNnplhYHOfvv82o\nUcOkBM+rgEDxIQRzAgICAh+FiMmTpzNt2gT++cePP//sS3R0NGXKlKFjx068fv2a+/fvfjCYCw6+\nTIsWrWWS+lpaWvz3301Z8Na27S/4+EjX6olEIho3lj6wVa5chbi4OOD9GUGJREJIyDXq1WvAkSOH\n8PX1JjGxI1lZZRGJ5BCLJbi6zkBBQZ709HRiY2O4c+c2pUqVpk2bdigoKNCmTRvc3NxYuHAejx+H\n4+npRlpaKq1bt2PVKh/KldPHxqYmVlbWLF7sKZP3NjU15+TJY8BbwYV3yV0amoODQw8cHHoU2tbM\nTAE5uVjE4tIAGBvfwcCg2XvP88dSWKZFSUmRDh0aARAYuJG0tG+3/Cr3Wi6JRIKFRXVZwF21qilR\nUZGoqKhiZGSEkVEFQCpSsW3bZrp06YaSkjJz586mYcPGMguMd1UlU1NTSU9PR0Xl+/Hg6t17DDEx\nYkSi1yQmdiYo6CEuLhYFtvX2bszMmf8QG6uOlVUakya1JzLyBfv2BVOhgi5t236+LcH48VO4dOkC\n3t4r2bLlX86dO8Py5atRUlLKZ8mQEzz7+6+ldu26TJkyg6SkJJyd+1K7dr0vch3evY+io6Oxts7b\npk6ddLKz9VBU3ImxcTwmJnW4c+c/1NXVsbCoLjMdl/ZnKfsxpzAvy/cZkRe0ZDElJUekRvpDW7t2\nv+LiMkm2vWnT5oA0KM9R2wRYvDiKZ8+kf3vu3DFnwYJA1q8XgjmB7wMhmBMQEPipKWitzvuktHO2\nzZzpzsSJY1BTU0dFRYXAwA0oKCigpqbOtGmzijRuQQIKhYkq5Fbpy2nz4YygCIkENDQ0EIuVUFB4\nTkaG2ZttCvj4rEZDQ5MJE/6iZ8/esixMzoOhWCxGUVGJdesCmDfPlapVqxEaegdra1uys7O4dSuE\nkSPHyOaTk82Ul5f7yPV2+fH03M/u3QrIy2fj5KSOo2NjBg5szvPnezlzRhlNzdeMG1e12B9i3y0N\nPXXqOC4uswv04vteyO0ZmHNtClrPKN0uz6pVfly5cokTJ46ybdtmvLx8KExV8nvi1au+PHnSSfY+\nJGRroW3Lly+Lr+/bMt07d8JwcnrEw4ddUFSMwslpF66uvxXLvHLOvb19E5SUpNeqoOA5LS2NS5cu\ncPbsKQIDNwCQmZnJixfRVKxYuVjm8j7y30dZ+dro6WkxfHhVfv21JQCurjdk95qKSt7SxaJ4WRa3\nEXnOGLmrLQCSk/P+HUlJUUJA4HtBCOYEBAR+anLW5+QWEnlXStvTc3G+9oqKiixa5C37/GOlt+3s\n6jBlyjh69PjzTZllAjVqWHP06CHatv2FQ4f2y35dLoy8GcGMPBlBkUiErW1Ndu7ciry8PG5u0xk+\n3InsbG0yM01QUVHi8uWLNG/eColEQmRkBPXrNyQu7qVsDd6xY8coV64cx48feVPutJyOHTthamqG\nvLw8GRkZqKmps337h8VUPoZdu87j7V2fjIyKALi6XqJ27QdYWlZl2rSSlTjP8eLKXRqqqamVL/j5\nUr5Zn4KamtoHhRsqVqxERESEzKLi4MF91KxZi7S0NNLT02jQoBFWVjZ07y4NaHLUInv16gPA0KED\nmT9/CRIJHD58QCa2Ehx8hU2b/snzncnBw2MO3bv/mU+Z9UtRunQKjx7lvJNQunTRs6urV4fy8GE3\nADIzDdm8WZ/x4xPymdV/DjmlvTnzKyx4dnObT4UKFYtt3OIgJyC1tq7Jzp3baN++AwkJCdy4cY3h\nw0cTFvboAz0UTGFG5O+qUErnAOrqGmhqasnWw+WI2nyIRo0SePAgAdBGWfkJzZsLYu8C3w9CMCcg\nICDwidy9G8a1aw9p2NCSihXLf9S+VaoY4+joxPDhzsjJyWNqasbo0ROYO3cWAQEb0NXVzSMMUVAG\n8dq1K+/NCJqamtOwoT1btvyLr+8iGjduxPXr52nXrgzJyc3Ys2cXfn5riYh4RqlSpfj1198wM7Ng\nzRpfNm36h1atWvLXX5NYsGAe0dGRxMS8IDk5GTk5OUxNzXj27Bl9+/YoYM3O55kE37uXIAvkABIS\nbLl+fQ+WllU/uc+PIXdpqEQi4fLlEAYNGicTpsitNvotoq2tg5WVDY6O3VFWVpaVsuVGSUkJd3d3\nXFwmkp2djYVFdTp16kp8fDyTJ499IykvYcSIMUDBapHq6hpERUUWWTlz4sRpxX2oH8Xs2RZMmbKR\nqChNTE1fMXNm0deavZswF4vlCrQrKC7eDZ7v379HtWqm1K1bny1bNslsO+7dC8XU1LzE5lEU78nc\n7Zo2bc7t2yH069cTkUjE0KGj0NXVIzw8LM/+hZmTv7utMCPyRo2aMG3aRM6ePcWoUePzzG/q1Jks\nWDCX9PR0mahNISPJXnl4dKJy5SM8fizBzk6DHj1aFX6wAgLfGCLJN2KUEhOT9LWnIFBClCmjKVzf\nH5if9foGBp5l1ixt4uLsKF/+DAsXKtGype3XnlaxUti1zcrKIj09DQ0NTeDtr/LFla06c+YW/fur\nkpAgPZ+GhofZtasSFSoYfGDP4kUikTB8eBBbtzZHLFahRYvd+Pt3lZXCfe+877sbEOCPkpISXbv2\nYOnShTx8+AAvLx+uXr3Mnj07uXUrhNWr/Vm0yIMzZ05RsWIl6tSpR4MG9qxd64u2tk4+9czhw50Z\nMWIMZmbmtG7dGAeHnpw7dwZlZWXmzVuIrq7eFznu7Oxs5OXlP2qf69fvM3BgFE+e/IqcXAy9e+9l\nwYIuH97xAzg4/M7q1X5s3bo5j6BHQkI8ixZ5EB4ensdqISMjg6VLF3LrVghisZjy5Q3z+D7m8LP+\nXf5ZEK7vj0uZMpofvY+QmRMQEBD4BNasSSIurh0AkZEt8fXd/MMFcwURGHiOhQuTSUrSoU6dhxgZ\nyXP0aCkUFTMZNEiV/v0/X1nP3r4Gc+acZevWLSgoiHF2NvzigRzAoUMX2LLlVyQSfQCOHevH+vW7\ncXZu+8Xn8qWxsbFj06aN3Lp1k5Mnr5CRocavv+6gQYNQbG3tuHUrBJFIlEc5E6Rllvfv382nnmll\nZZMn2E9PT6dGDWucnYeyfPlSdu3aTt++A77IsX1sIAdga1uNzZtV2b9/M/r66nTu3PnDOxWBoKCd\nQNGsFtLT03n27ClDhoyQ/ZAi8GHOn7/DiROPMTJSoXfvpt90ibSAwKcgBHMCAgICn0BWVt4HwszM\nj39A/N5ITk7CwyOTyEhpRuLQoSbAv4BUVMLd/QL29mFUq/b5a6K6d29E9+6f3c1nkZCQhkSSe02U\nEqmp30QxS4ljZmbO3bt3yMjQJTm5FKmpDXj2zIqkpH/o3bs7GzeuBwoW7MmvnhmVz5NOUVGRhg3t\n34xlwZUrF0v2gIoBY2Mjhg0z+ipj3779iOHD/+POHRsMDa/g6qrFL798eC3Yz87evZcZM0aDV68c\nEIlecvPmdjw9iycQFxD4VhBWeAoICAh8Ar/9JkZZ+TEAmpq36dz5+5Fo/1Ti4+OJjc39MKsIaMne\nJSRYEhr67IvPq6To0KE+tWoFAtK1Uaamm+nWze7rTuo9REVF0qtXF9zdZ9GzZ2dmzZrGpUsXGDzY\niR49OnPnzm3WrFlJYOBG2T59+nQjOlrqZbZ//x769u1Jv369mDfPFQMDQ5KS4hGLVVBTO0X58s5k\nZcV8UHyjKKqH8vJvf0uWkxN9tvrpj87Chbe5fbsnYrElT5/+xqJF0V97St8FO3a84tWrugBIJKU4\neFCXrKz896OAwPeMkJkTEBAQ+ATGjGlLtWoXCA29RJ06BjRr1uRrT6nEMTAoT82a27h40QYQoaJy\nCzm5RHKEE6tUOUGDBtbv7eN7Qk1NjX//bcvKlUFkZYno3duO8uXLfu1pvZeIiGfMmePJ5MnTGTjQ\nkaNHD7FixVrOnDmJv/+6PF6E8Had47sm9klJSQQFBXL9+jUkkrI8e/YPlSr9jpxcfJ4yxaIoZwp8\nPikpynnevyulL1AwCgp5AzclpUzk5IQ8hsCPhRDMCQgICHwiHTvWp2PHrz2LL4e8vDxr17bAwyOQ\nlBRlWrTQRF6+LDt2bEVRMYthw6pSunR+5cTvGS0tLcaP/+VrT6PIGBgYYmwsNTuuUsWY2rXrvvOR\nnUEAACAASURBVHltQnR0ZL5gTookn4m9pqYmNjY1EYtXU69eJV6+PEFsbHY+Vcfcypn16zeiQYNG\n71U9zKEgdVaBwmnWTI5z5568UXlNolGj+K89pe+CoUNNuX59Ow8ftkRT8x4DBigKwZzAD4cQzAkI\nCAgIFJkyZUqxYEHeCLaYtCAEioEc43YAOTk5mU+ZnJycTMVRInkrqS+1ICjYxL5WrTq0b/8rDRvW\np1mzlkAbWreWZqCDgnbJ2s2YMSfPfrl9vXIk9AG8vVfKXuf4NQI0a9byTf8ChTFkSCt0dc9w9eol\nKlYUMXRopw/vJICVVVX27SvFmTNnqFatPObmzb/2lAQEih3h5wkBAQEBAYH3MH78KFJSkklOTs5j\nkB4cfIUJE/76ijP7eAwMynP3bigAt2/fJioqEhBhZ1eH48ePkJiYAEBiYmKJjJ+QkMCkSbsYOvQQ\nmzadLZExflR69LBn/vy2jBjR5pMUOX9WdHV16dixMebmJl97KgICJYKQmRMQEBAQEHgP8+d7AVKB\nka1bNwHwxx9duX//Lnfu3P6icwkOvsKmTf/g6ZnfWwzylyy+W87YtGkLDhzYS58+3bCzq0mFCpWA\ngk3sc8yWi6skUiKR4OR0gNOnnQA59u69h5zcObp1a/jJfQoICAj87Aim4QIljmBu+WMjXN8fl5/l\n2n6MQfbp09LywK5du1O6dBnWr1+DnV3tfAbZuRGLxcW2TudDwdzH8KWvb2xsLHXrRpGc/DZ469Zt\nK8uWtflic/hZ+Fm+uz8rwvX9cfkU03ChzFJAQEBA4KfGxsaOGzeuAxAaeoe0tDSysrIICbmOra3U\niiDHIFtRUQmRSMTlyxfZsWMrKSnJvH6dAcDFi+cJCZH207VrR3x8vHFy6s3x40c4fPgAffv2wNGx\nOz4+3rKxW7duLHt9/PgR3N1nAVJVSmfnfvTt2wNf3+WytWoAaWmpTJs2kT//7Mrs2S4lck7S09MZ\nP34nf/xxmBEjthdL2aWmpia6us9zfZKNru7rz+5XQEBA4GdGCOYEBAQEBH5qcgyyU1NTUFJSokYN\nK0JD73DjxjVsbGrK2kkkEvT09DA0NGLdugCsrW0Ri8VMmDCVjRuDkJeX4/Jlqfm1SCRCW1uHtWs3\nYmNTkxUrlrF06QrWrQsgNPQ/Tp8+8abXgksYvbwW0L17L/z8NlG2bLk8871//y6jR49j48YgIiMj\nZAFkcTJ16n78/Lpz9mxn/v23N3/9dfiz+1RWVmbSJE0qVdqKtvYJmjdfy8SJzT5/sgICAgI/McKa\nOQEBAQGBnxoFBQUMDAzZt283VlY2mJhUJTj4MhEREVSuXKXQ/apUMUZLS5vSpcsAoKOjS1zcS9n2\nli1bA3Dnzm3s7Gqjra0DQOvW7bh+/RqNGzcrtO/bt28yb96iN+3b8vffXrJtFhbVZWNWrWpKdHQU\n1ta2n3bwhXDvngZSU3gAOe7f1y6Wfh0c6tGpUyYpKcloa9sJtgSfyKZNG9m3bzcAHTp0okmTZowZ\nMxxzc0vu3QvFzMyUCRNcUFZWITT0DsuWLSYtLQ1tbR2mTp1BqVKlGT7cmerVrQgOvkJychKTJk3H\nxqZ47yMBAYGSRwjmBAQEBAR+emxsbAkM3Mj//jecVat8SEtLw8LCkoCADSQlJbFz5zb2799DZGQE\nKiqqgDRIS09PA8DNbSYvX8Zy6tQJLl++SFpaGqqqqojFYnbu3MbNmzeIjY1BQUGBcuX0ZX3kDmYy\nMjKKNFdFRSXZa3l5qeVAcVO+fAogISdzaGiYXGx9KyoqoqOjW2z9/WyEht5h//49rFrlh1gswdm5\nLzVr2vH06ROmTJlBjRrWLF48l23btuDg0IMlS+bj4bEIbW0djh49hK/vciZPno5IJEIsFrNqlR/n\nz59l3TpflixZ/rUPT0BA4CMRyiwFBAQEvnOioiJxdOz+tafxXRMf/4rnz6M5evQg8vLyKCsrY2NT\nUxZsbdmyCX//f7G3b0p6ehrLly8lPDxMtn9qaiqJiUnY2zfB03MJSUnSNWYnTx4jMzMTTU0tRo0a\nx61bN7l584ZsLZ6enh6PH4cjFos5deq4rL/q1a04fvwoAEeOHPrg/IOCNtG7twOursWzhs7NrQlt\n2/pTteoOmjXbwNy5dYulX4HPJyTkOk2aNEdZWQVVVVWaNm3B9evXKFu2HDVqWAPw22+/ERJynSdP\nHhMW9pDRo4fSv38v/P3XEhMTI+uraVOp75qZmTnR0VFf5XgEBAQ+DyEzJyAgICDwQ9G6dWMOHz5N\nbGwMS5YsYM4cD/bt283du3fymFjn5urVy+zYsZ/MzEwmTvyLwMBtAAQGbqRbt57cvn2LmTOn0rRp\nc+Tl5Th//gzh4WHo6+sD0gybiooy1ta2VK5cBbFYaswdEnKDdu1+RVFRkRkzJpOdnY2hYQXs7aWC\nJoMHD2fChNFoa+tgYWFJWpo00zdy5Fhmz3Zhw4Z11K1bHw0NDdlcC6pM3LFjC15ePrLyS4CsrCwU\nFD7t33zp0nps2CC4wX+LFFSaKhLl/Vwikbx5L6FKFRNWrFhbYF85WV45OfkSyfAKCAiUPEIwJyAg\nIPADIBaL8fBw49atG5QpU5a5cxdy8OA+du/eTmZmFkZGRri4zEZZWYVjx46wfv0q5OTk0dDQYNky\n3689/WJG+lBbunQZ5szxkH7ynrVZ8+e7ExkZwdixI4iKikJZWVm2LSDAj9at2zN27ESGD3cmPPwR\nERHPsLdvRlhYGLGxCYwfPxYVFQU0NDQJCblBQIA/IpEIZWUVRCKIi3tJcPBVFBQUUVRUwM6uFiAt\nzVRSUkJbWwdra1uGDx8tG7dMmTL4+q4H4MiRgzx9+gQAO7va2NnVlrX7668Jeeb//Hk0jRo1ITIy\nAn19A/73v2G4u88iISEBHR1dpkyZTrly+ri5zURHR5OQkFu8ehXHpEku7Nu3m9DQ/7C0rCHzmBMo\neUJD73DgwF5Gjx5XpPY2Nra4uc2id+++iMUSTp06jovLbLy8FnLr1k1q1LBiz5492NjYUrFiZeLj\nX8k+z8rK4unTJ1SpYlzCRyUgIPClEMosBQQEBH4Anj59Qpcu3diwYTMaGpqcPHmMZs1asGqVP+vX\nB1CpUhX27NkJgJ/fahYt+pv16wPw8Fj0lWdecuQuP81tqXru3BkGD3YiISGeS5cu8PDhAyQSMDAw\npHNnB9LSUklMTOD169ekpqYikUiIjY0hNjaGiROnoaWlzblzVojFirx40ZnTp38hOTmVly9jZddA\nJBJx8uQxrKxsCAjwZ/Toccyfv4TMzCx27dohm0tsbAwrV67LE8gBhIaG0q9fL/r27cmOHVtl2x8/\njmTDhoPcuHFP1nb8+CmULl0Gb++VdOvWi/DwMLy8fJgxYw6LFnnyyy8d8fMLpE2bdixZskC2X1JS\nEitXrmPkyDFMmjSWXr0c2bBhMw8fPuD+/XsIfBnMzS2KHMgBmJqa88svHRg0qC//+18/Onb8A01N\nLSpWrMT27Zvp3duBpKQkOnXqioKCAq6uHqxY4U2/fr3o378Xt2+HFNLzlxGjGT7cmdDQO19kLAGB\nnwEhMycgICDwA2BgYEjVqtUA6fqXqKhIHj58wKpVPqSkJJOamka9eg0AsLKywc1tBi1atJatmflZ\nOHnyOJs3B7BgwVKysrLw91+Ll9dyevfuRtWqVQkJuUGdOvUYNKgvZcqURVFRiezsbJYuXYicnBzz\n57tjbl6Xdev6UrXqMkDEo0edqFTJD11dPdk1kJOTIyoqEgeHniQlJeHo2ANFRQUkEkhNTQGk2cLm\nzVsVmDW0sbFl/fqAd+Z+k9GjU4iI6ISW1k0mTTrJwIFNZdtzAtbGjZuipCQtn/vvv5vMnSsN4Nq2\n/QUfn6W5xm4GQJUqJujplcLY2OTNe2OioyOpVs20mM76j09UVCRjx46gRg1rbt68gbm5Je3bd2Dd\nOl9evYpnxgypmbyX10Jev85AWVmZyZNnULFipTxG8GvWrOT582iioiJ5/jyabt160rVrj3zjde/+\nJ927/5lnfHl5eVxcpOPkNpWuVs20wOy7t/dK2WsdHR2CgnYWy7nIuQ8Ly4aLRCJBxVRAoBgRgjkB\nAQGBHwAlJUXZa+n6lwzc3Wczb95CTEyqsn//Hq5duwrAuHGT+e+/W5w/f5YBA/qwZs0GtLSKR3r+\nW+bq1SuEht5h8eK/UVNT4+zZ04SHP2LwYCdiYl5w9OgR1NTUqF+/IXPnLgSgR48/6NXLEYlEzMSJ\nf+Hv/y8nTlzG3z8GUCAmZhqQRv36v3Lt2jbZWM7Ow94oXUrQ0dFl166DJCTE4+zcL8/6JRUVlSLP\nf/XqCCIiHABITKyJn99DBg7M305ZOW+fubOSuVFUlN4zcnJy79w/JaOQCTBkiBM+PgWv3/reiYh4\nxpw5nkyePJ2BAx05evQQPj5rOXPmJP7+63Bxmc3ff69CXl6ey5cv4uv7N3PmeObr5+nTJ3h7ryQl\nJZlevbrwxx8OyMvLf3D8DwVI69ev5tCh/WhpaRMZmU1mZkVMTKqgqHiVxMREVFRUmDhxKhUrVsbN\nbSbq6hrcvfsfL1++ZOjQkTRr1hKAgAB/jh8/wuvXmTRp0owBA/5HVFQkY8YMp3p1K+7evcP8+UvZ\nuHE9oaH/kZGRTrNmLRkw4H+fdmIFBATei1BmKSAgIPCDkpaWip5eKbKysjh4cJ/s84iIZ1ha1mDA\ngP+ho6PDixcvvuIsvwwikQhDQ0PS0lJ58uSx7PPateuxbl0AZcqUZcWKNXTv3ou7d0MBuHs3lKio\nyHx9NW1amz//PIpEIkJO7jbt22+ga9cGedqIxdlcunSfxYtPk5qaSteuHRk2zJm+fQfw8uXLfH0W\nhezsvA/rWVkffsCvUcOao0elapiHDu3PY4L+NfhRAzmQZseNjU0QiURUqWJM7dpSBdAqVUyIjo4k\nOTmJadMm4ujYnWXLFhMW9ihfHyKRiIYN7VFQUEBbWwddXT1evYorwtjl8fPbVOj2O3duc/LkMfz8\nNiGRtCY6OplHj6w5ezaYlJSarFmzgaFDR7FwoYdsn7i4l/j4rMXTcwkrViwD4NKlCzx79pRVq/xZ\nt+4f7t4N5caNa4D070rnzg5s2LAZfX19nJ2Hsnq1P+vXB3L9ejAPHz74qPMpICBQNITMnICAgMAP\nQEG/yg8c+D+cnfuho6ND9eo1SE1NBWD5ci+ePXuKRCKhdu26stLAHxmJRIK+vgHDho1iypQJuLrO\nw9KyBosWeRAR8QwQkZ6egYlJNQ4c2EufPt2wtKxBhQqVZH3knGORSMSCBV2YO/c/goOHY2Jig6Ji\nM9l2iURCUFAIoaGWxMU5YGAAlSptJjs7nYCADbRq1UZ2zj+m3MzBQZsrV64SH18LZeUndOr0OtfW\nt/3k7nL06AnMnTuLgIAN6Orq5hE2yT32u/MoqTK4HKXR4OArrF3ri46OLmFhDzEzs2D6dNcSGfNL\n8W52M3fmMzs7m9WrV1C7dh3mzl1AdHQUI0YUnKlSUMjbT1bW52dJb968QePGzVBUVOThwzIkJ7dA\nJMpAVfU6oaGP6d//GACZmVmA9Po3biwt4a1cuQpxcdKA8tKlC1y+fJH+/XsBkJaWzrNnTylbthzl\nyhlgaVlDNuaxY4fYtWsH2dnZvHwZS3h4GCYmVT/7WAQEBPIiBHMCAgICH8GnlonlXhdTVNasWYma\nmjo9e/Z+b7t3f5XP3b5Tp6752ru5zS/yHL5HCgpSctbpVKxYmRkzXHFxmYSn52KmTp3JzJlTUFNT\nY8KE0Tg7D2XRomUF9vtu5mPyZJcCt0dHRxEcPIj09DoAREU5oKsrYsGCtoDU0y409C5jx07Ko5z5\nIf74oz76+rc5fz6IatW06djxF9m2nPVOTk7OefbR19fHy8snX19TpsyQratKTX1N375/kZycjIaG\nRgkrWb69Ng8e3GPjxiBKlSrNkCEDCAm5jrW1bQmO/fWQSCSkpCTLrCP27t1VaLuSQSTr28AgmchI\nADFisSbGxgNYt65Tvj1ygtF359W7dz9+/z2vbUVUVCSqqm/LeyMjI9i06R9Wr96AhoYG7u6zeP06\no3gPSUBAABCCOQEBAYGPojjLxD7kfVbc2ZGnT6M5ePA6lSrp0br1j2sCfejQSSBvkNu+fQfat+8A\nQLVqZmzcuBmA8uUNWbXKv1jHV1FRQUXlGenpOZ9IUFTMBCAo6AKzZ2fz4kUVrKwO4utrh7GxUZH7\nbtCgOg0aVC+2uS5bdoRFiwxJTrbB0vII69fXpHJlw2Lr/32Ymprj4TGHmJgYXryI5ujRQwQHX+Hs\n2VNkZGRQo4Y1EyZMBaQKiGZm5ty4cZ20tFSmTZuFv/86wsIe0bJlawYNGgLAwYP72LLlX7KyMrG0\nrMHYsZOQk/syK0rel92Uk5OjZ09H3Nxm4Oe3hgYN7Ckomyr90aH452ZtbYOnpzt9+vTHxcUGZ+fl\nZGQ0QFVVmQ4dpJk/iUTCw4cP3pupr1evPqtWraBNm/aoqqoSE/MiTyYxh5SUFFRUVFFXVycu7iUX\nLpyjZs1axX9gAgICQjAnICAg8DEUpUzszp3bLF26kLS0dBQVFfNlRnIybjo6OgD06dON+fOXoq+v\nj5/fGg4c2Iuurh5ly5bDzMwCkK5HWbTIk/j4V3mECopKSMh9BgyI4PHjrigpPWXgwN3MnNmxeE7K\nd0ZmZiZz5hzg4UNlKlRIZ/r01qiqqhZb/7q6ejg5vWT58uukpxthbb2PUaPskUgkLF4cx/PnUruE\nkBAzFiwIYPnyogdzxUlGRgarVsmRnCwN7P/7rydeXptYvPjLBHMpKSkYGlZg/nwvFi/2pEoVY1q0\naEO/flJVF1fX6Zw9e5pGjRojEolQVFRi9Wp/goI2MWnSWNat+wdNTS26d+9E9+5/Ehf3kmPHDrNi\nxVrk5eVZsGAehw7tp127X0v8WN7NjufObubelmNGD8gC0Nzege9mVv39/y2W+ZmbW2Jv34S+fXug\np1cKe3tb6tevSa1a/2PBgnns2bOVrKysQkuAc17XqVOf8PBwBg/uD4CamhouLq75FCqrVTPF1NSM\nXr26ULasPtbWNsVyHAICAvkRgjkBAQGBjyJvmdjChctwc5vBpUsX6Nz5V6ysbLh27Sq6unpkZGQw\natRYHj16yKJFnsTEPGfIECdMTc1RU1N/26NIxNWrl9m8+R+ys7MZPnw0fn5rOHnyGOHhj+jUqQue\nnm6MHz8FI6MK3L59i4ULPQosnyuMNWse8PhxNwBev65IUJAuEyakoqamVnyn5jth6tS9rF/vAKgA\nmSQl/cOyZV2KdYxJk9rz++/3iYi4RoMGbVBXVyc7O5uUlLxKk6mpRS+zLG4yMzNJT1fP89nr1/mz\nLCWFmpo6V65cxMfHm9jYGMzNLQkOvkxAwAYyMtJJTEzE2NiERo0aA2Bv3wQAY2MTjI2ldgogza4+\nfx5NSMg17t4NZeDAPoA0WC1VqtQXO55PRSKR4OGxj1OnlNDQyGDcuKrUrWte7OP07NkHJydn0tPT\n32Q6LTAwKM/ChUvztX231DYn2w3g4NADB4f8dgnvliEXVq6b2xJBQEDg8xGCOQEBAYFPxMKiOnp6\nekREPKNp0xY0atSYDRvWI5FI8PML5MyZk2zeHIiLy2xGjx7H5s2BODj0wMtrAb/++rtsHUpKSjI7\ndmyhVau2JCYmEhi4EW/vlfj6+vDkSTgbN67n5s0QXFwmysbOESooKhKJ6J33ciW4PqdgPmW94enT\nJ6hQoRKVK1cptnncvq2BNJADUOT27ZKxZbCwqIaFxduSNXl5eeztXxIUlAqooaZ2l9atiy8j+LFo\naGjQokU4W7cmAxro6V2gU6cyJTpm7uyNmpoqa9f+w/nzZzh4cB8SiYRbt26yZs0GypQpy9q1vrx+\n/VbkRVFRSdZHzuuc9zlWCu3bd+B//xtWosdQ3KxdewIvrxZkZ5cFIDIyiMOHKxVrthjA09ON8PBH\nvH79mvbtO1Ctmlmx9l8QZ8/e5syZpxgbq9O1q73gLycgUAIIwZyAgMA3QXJyMocPH+CPP/ILduQQ\nFRUp8/r6Fsh5oDQwMERXVxexWIyhoRHZ2dJAK7ck+dq1K3nw4D7Pn0cRHx+PRCIGpN5nr169YsEC\nby5fvkhMzP1c3mcxyMmJ0NHRQVNTk3XrAgqdy4dwdKzC2bMHefq0LQoK0XTq9AJ1dfUP70jxnfdP\nWW946tQJGjVqXKzBXNmyKe+8Ty22vj+El9cfVKu2j+fPRdSvr83vvzf+YmMXxLJlXbGxOUhMjJhW\nrSrRoEHJWhfkZHjs7GpTsWJllJSUaNOmPRoamuzevQORCLS0tElNTeX48SO0aNG6SP2KRCJq1arL\npElj6datF7q6uiQmJpCamoa+vn5JHtJnc+fOa1kgB/DggSVRUZEyE/fiYsaMOcXa34fYseMCEybo\nER/vgIJCNDdv7mb27N++6BwEBH4GhGBOQEDgmyApKZHt24PeG8x9q+SWJNfU1CQ5OZnQ0P/Q1tYh\nMzMTX9/lmJqao6GhxZgxExg4sA9374ZSv35DtLW1CQ9/RGRkJLa2Ndm8OQA7uzpMmjQNJ6c+dOrU\nmR49ejNkiBPHjx+hefNWRRIqeJfatc3YtOkJhw5txshIk99++70kTsV7ad26MZ6eSwgM3ChT9Vy0\nyAMLi+q0b98BHx9vzp49jby8PHXr1qdp0+acPXua69ev4ee3hjlzPDE0/Pz1ZTNn1iUhwY9Hj7Sp\nWDGJ2bO/nPeagoICo0e3/WLjfQh5eXkGD27zxcZLSkoiIOAsCgpymJur4Ovrg5ycCAUFRcaNm8yp\nU8dxdOyOnl6pPDL3uSlIJGTHjq0yIZQxY4YhFktQUFBg7NiJ30QwFxS0iZ07t2JmZo6LS14LhmrV\n5BGJ4pBI9ACoUuUe+voNAdi8OYDff++czwj+e2DbtkTi46XBeFaWPvv3qzNrlkTIzgkIFDNCMCcg\nIPBNsGKFNxERz+jfvxd2drV58OABSUmJZGdnMWjQEOztm+ZpHxHxDBeXiUyYMA1NTU2ZOIiioiJ1\n6tRjwICCPZzeR1HsA/KKAuTfLicnR48ef7J48XySk5OJjo7EyKgihoZGREVFsnfvLlRUVElKSmTl\nyr9RV9fA0NAIb+9FzJ27gNat2xEQ4P/G2Ls6mZmZPH36hOnT57BgwTz8/NbmEyooKtWqVaRatYof\ntU8OYrEYDw83bt26QZkyZZk7dyEHD+5j9+7tZGZmYWRkhIvLbDIzs+jXrydbtuwGIC0tjT//7EpQ\n0C4kEgnLly/lyZPHDBs2iIkTp8rOZ2JiAqdPnyAgYCsgLT1VV9fA3r4JjRo1pmnTFp8074KoVMmA\nbds6I5EID5ZfksTERBwcDnLtWj8gi8aN1xEY6IeS0tuSSTMzc5kwSG5yr7OqWbNWHmXEd9dgtWxZ\ntGzel2THji14efnIrAly4+zcksjI3Zw9q4aGRjpjx1aRrWUNCtpE27a/fJfBnKJi9nvfCwgIFA9C\nMCcgIPBNMGTISMLCHrFuXQDZ2dlkZKSjpqZOfHw8gwf3zxPMPXkSzsyZU5k6dRYmJlUZNWqITBzk\n5MnjuLpO/6RgrijkLhOzs6tNVFQkIpFIZi9w7dpVjIwqsHLlOqKiIpk0aQy9e/fDzW0GqqqqVK1q\nikgkx6JFy9i/fw93795h9Ojx3L9/V+Z9VqdOPXx8lvLw4QMePnyAsXFVGjVqXKBQQVH53FLJ8PAw\nZs50Z+LEqUyfPpmTJ4/RrFkLfvvtDwBWrfJhz56ddOnSnWrVTAkOvoKdXW3OnTtNvXoNkZeX5/Xr\n1zg49ODIkUP07TuAhQs9ZOWT6uoaKCkpM3fubBo2bCwTvYCS894SArkvi7//2TeBnBygxOnTf7J7\n93G6dGn2Uf2sX7+aQ4f2o6OjK1N8ffDgHklJWmRlKZOdfQ8vL28g7w80ly5dkK3DMzQ0YsoU6Xey\na9eOtGrVlu3bg8jMzKRcOX0GDhyCoaERy5YtJi0tDW1tHaZOnUGpUqULVZZ1c5uJuroGd+/+x8uX\nLxk6dCTNmrVk/nx3IiMjGDt2BG3atOf06ZO8fp2BsrIykyfPoGLFSkyf/is+Pt5cunSe1avlePGi\nExKJhNjYGEaOHIyOju5HCR59CwweXIWQkD08ftwCLa3bODkpCd85AYESQAjmBAQEvglyP7BLJBJW\nrFjGjRvXkZMTERsbw6tXcQC8evWKyZPH4e6+gEqVKpOamsqtW2/FQaKiosjISKd//17UqVMPiQQu\nXjyHSCTC0XEALVu2lmWI3v08N3fu3Gb+fHfmzPGkfPnCpdo/R5K8IO+z8PDHZGTI4+3ti4qKCsHB\n99iy5REHD+5hxIgGlCnzddT5RCKRLBNoZmZOVFQkDx8+YNUqH1JSkklNTaNevQYAtGjRmmPHDmNn\nV5sjRw7RpUs3UlNTEYvFrF3rS1xcHDExz8nMzJKdW3l5eVat8uPKlUucOHGUbds2yx5ei/IA+LHB\n6v79e6hTpz6lS5f+lNMh8AnIyYmA3IF5NvLy7/eACw29w4EDexk9ehxr1qwkOTmJ69eD8fPbRGZm\nJk5OvdHXN+DQoaNERMwjObktVas24sqV29SuXZ1jxw7TqlVb4uPj8fdfi5fXcpSVVdi4cT3//vsP\n/foNRCQSER//ihYt2mBqasa9e6HUr9+AceNGMm/eIrS1dTh69BC+vsuZPHn6e5Vl4+Je4uOzlvDw\nMCZNGkOzZi0ZP34Kly5dwNt7JQoKCvTo0Rt5eXkuX76Ir+/fzJnjya5d23n+PJr16wORk5MjMTER\nLS0t/v03AG/vlWhplYxIT0lSp445e/aU5syZI1hYVMDSstnXnpKAwA+JEMwJCAh8cxw6tJ+EhHjW\nrt2IvLw8Dg6/kZEhVbXT0NCgXDkDbty4RqVKlZFIxGhovBUHiY6OYsKE0axbF8CJE0fZZKTsEgAA\nIABJREFUuXMbfn6biI9/xcCBjtja1uTmzRs8eHAv3+c53Lx5gyVLFjBv3iLKli33xY570aJDeHsb\nkZJiQs2ae5k6tRKjRmUSEeEASLhwYT07dvzyyXYCRS2VVFZWITIyguHDZ5CUlIyNjV2efuTk5MnO\nzsDdfTbz5i3ExKQq+/fv4dq1qwA0atQEX9/lJCYmcu9eKLVq1SE1NQUQsXTpCoYNG4Svrx/p6ek4\nOfXG2tqWtLQ00tPTaNCgEVZWNnTvLl3Tp6amRkpKyruH8tns27ebKlVMhGDuC9K3b2P27VvHpUuO\nQCatWgXSoUP39+5jbm6BubnUa1EkEhEVFUnjxs1QVFREUVGRRo0aExf3iqwsBaSPNPIkJrbB13cf\ntrZmnD9/lmHDRhMcfEUmLARSNVgrK2vZOL/88huuri68fp1BeHgYz59H8+jRQ0aPHgpIvzulSpUh\nLS2tUGVZkUhE48bSCoLKlasQFxeX73iSkpJwdZ1BRMTTPCqcV69eolOnrjKDcy0trY8/wd8g5cqV\npkuX5l97GgICPzRCMCcgIPBNoKamRmqqVFUwOTkZXV095OXlCQ6+QnR0lKydoqIi7u7zGTNmOKqq\nqrRu3Y7y5cvLxEHEYrFMzjwk5DqtW7dDJBKhq6uHra0dd+78x82bNwr8XF1dnfDwR8yf787ixX9T\nqtSXe9BPSIhn1SpNUlLqAXDtWl9cXecRETH5TQsR16//xrlz12jVqt4njfH06ZMil0p6eS2gV69e\nNGzYgvXrVxfYX1paKnp6pcjKyuLgwX2ywFdNTQ1zc0u8vObLDJ/V1TWQkxNx+/ZNmjdvRZ8+3dHR\n0cHMTCqPnpqawqRJY99cOwkjRowBoGXLNnh4uLFly7+4us57rwBKdnY2s2e7cO9eKJUrG+PiMouw\nsLB8pXIhIdcJDb3D7NnTUFZWZvToCWze/A9ubvM5ffoEM2dO5eDBk2RnZ9OnTzc2b95ZaGndq1ev\nWLhwLs+fRwMwcuRYrKxsWLNmJc+fRxMVFcnz59F069aTrl3ze3P9KERFRTJ27AjM/8/eWYdFlbZx\n+B6GlBIUMQETkEZsbF111bWwXQEDYy1s7Mbe1V0DXUEUY0WxVtfuTsDCVhqR7piZ749ZRhAwcY3v\n3Nfl5cw5b533zDDvc57n/T1mtQvM/+3bwaxZsxKJRIKZWW22bRvL/v1/c+3aEeLiXjBo0N/Ur9+A\nESPGcPLkcTZt2oCSkhgtLS3++GN9oX2sr1694sGDvRw9eph+/QYAoKKijNzjJwMkKCkl8fDhafr2\nPY+urq5C4t/BoT6zZy8ocvwmJiZ4e29l9+6/uHDhLKdPn6Rq1eqsW1dQgTUtLfWtyrIqKq/FkN4M\nD5bJZPz55zocHOri6bmMqKhIRo8eVmx5AQEBgfdBMOYEBAS+CnR1S2NlZcOAAb0wM6tNaOgLnJ17\nY2pqjrHxa1l6kUiEuro6S5b8hrv7CEqV0iwgDpKRkfGvF0hetrgF0pvH80L5ypY1ICcnm4cPQ2jY\n0PEzXW1h5OMunX9EiMXqQCZ5OdHU1aMwNCxdVPX3okKFSu8dKnnnTjAbNngRH59OkybN2bixcKLf\nwYOH4ubmQunSpbGwsFQY4yAXoZg500MhTpGUlEiZMmX5++/9xMW9QllZmQYNGuHiMlhRZ8MG30J9\nWFnZ4Oe3872uLzT0BR4eM7G0tMbTcy67d+/k3LnTeHquoHTpgqFyAQH+jBzpjqmpGbm5uTx69BCA\noKBAqlWrwf37d8nNzcXCwgqg2NC6lSuX0bNnX6ytbYmOjmbChFH4+fkDcuP599+9SEtLpW/f7nTt\n2gOxWPxe1/ItEhYWytSpsxTzv327H/v372HVqnVUrlyF+fNncfjwQX766UcOHVpXQOwGwNf3T1as\nWE3ZsmUVx/Ijk8lIS0tFV7c0S5euxM3NGRUVFVq1+gE1NSnKyuHo6m6ibNlUSpcuTc2apjx+/JCo\nqEhq17ZkxYrFRESEU6lSZTIyMnj1KpYqVeSCQPHxcVSoUIkGDRpx8uQx7t+/S2JiInfu3MbS0orc\n3FzCwkKpWrVagYdHH6osm5aWphBBOXTogOK4g0N99u0LwN7eAbFYrAizzPNMf4thlgICAv8NgjEn\nICDw1fA+eZDy9qBpaWmxYcNmxfE8cZCkpEQGDfoZAGtrW/bt20P79h1JSkoiKOgWI0eORSKRsG9f\nQKHjz549RUtLGw+PGYwd+wvq6hoFVPM+J4aG5Wne/DT//GMDqGFgcJ7Jkx3w9vbl5MmGqKkl4+IS\nhpVVx4/uI38KhXeFSuanfPnyqKu/TmDcp09/xesuXQqmkli8eD69evWjefNWnD17FYBXr2IZNWoo\n/fu70L17z7eOUSKRsHz5VqKjE3F2bo+NTU3Onz/L8+dP6d/fhY0bvShVSrPAGPIoV84QS0t56Fzb\ntj/i6+vN06dPcHcvGCqXR55Br6ysTKVKlXnx4jkhIffo3bsfgYG3kEol2NjYvjW07vr1q7x48Uxx\nPD09nYyMDEQiEY0aOaKsrIyubmn09PRJSIgvUs3we+HN+d+06U8qVqxE5cpVAPke0YCAnXTv3rNI\nsRsrKxsWLJhFy5ZtaNascGieSCSideu2iEQiRo8eSm5uLoaGhmhqaqKursK0aZmcP3+WlJQ4UlIy\nOXv2NOXLlyc8PIy6deszbdpsZs+eSnZ2DgBubiMUxtzz58+YNWsa2dnZxMW9Ytq0OSgpKbFy5TJS\nU1ORSHLp1asvVatWe6uybEG12/x7PUWIRCL69h3AggWz8PXd+O/DInmZTp26EBYWirNzH5SVlfnp\np65069aDn37qyvjxozAwKPfNCaAICAj8NwjGnICAwDfNzp0X2bUrBbFYypAhFWnZ0kbh4WvQoBE1\natTAxaUPIpGIESPGoKenT7NmLbh7N7jQ8efPnyESgZ6ePkuW/MqECaOZOnUW5uYWn/06RCIRGzZ0\nZ+3a/SQkQPv2xtSvb06zZtaEhoaioVEeQ0ObEu+3uFBJKysbDh48SMOGLTh69PB7tzd58vRCx3R0\ndLG1Hcy5c2Kys8/Tp0/RHk+ZTMYvv+zi+HE91NTiOX48mXXr7uHo2BRHx6bA28VQ8p+TyWRoamoW\nGSpXVHkbGzsuXTqPWKxMnTr1OHx4FlKpjF9+GYNUKnlLaJ2M9et9C4TX5aGsnN94ViI39/uWZn9z\n/rW0tElOTipwDIoXu5kwwYN79+5w6dIFBg36mY0btxTZT58+PzNwoBtz5kzn3r27VK9eg3LlyuHs\nPIBHj+7g5jaYunUbFKpnb+9Q4AEQyI3vKVPmU7myIb6+2wvV+eOP9YWOVahQsUhlWQ+PmQXmIE/5\nFsDffx8AlpZWRYohicViRo1yZ9Qod0D+4EEmk9G9ey+6d3/7vkIBAYH/bwRjTkBA4Jvl/Pk7TJtW\nnqQkeRLm+/dPsG9fZCEP34gRYwrVHTFiTKHj+fNXGRqWZ8uW9wvvKylUVVUZM6ZdgWNKSkqYmJiU\nSPtFGULFhUqOGTOBhQtnsW6dF46OzYqsm5GRwcyZU4iNjUUqleDsPJg9e/wZNWocpqZmtGnThB49\n+rBjxz4yM9N49uwku3encOGCB/Hxj4iNjUFNTZ3SpfWQSHJp2NCRmzcvYmgYjkwmJjv7Bl5e1iQm\nPuHBg/uK9A/FERMTrQiLO3bsMBYWlhw4sLfIUDl5+NrrUD4bGzvmzZvJjz92onTp0iQlJZGYmEC1\natUBig2tq1u3Af7+O+jbV+4NfvToITVr1vroe/Qt8+b8m5mZs29fgCK08ciRQ9jZ1SlW7EaeW9GS\n2rUtuXz5Ai9fvizQvkwm4/z5M4SGvuDZsye8ePGcfv0GYGJSTVGmXr2GBATsws7OAWVlZUJDX1Cu\nnCHq6oXztD19Gs6gQbe4e9cRff3HTJ36kAEDmhQq9y4OHbrB0qXRJCerU69ePKtWdSnSuC+O/AnF\np0+fy+TJezh+XBdV1RyGDtXA1bXZuxt5C5+alkRAQODrRjDmBAQEvlkuXw4nKamH4n1kZFPOnz+A\nsXHFj2rv2LFb7N37ElXVHMaOrYOxcYWSGuoX580UCm8Llcwrv2PHDmJjUwCKTOR85cpFypYtx9Kl\nKwH53qe9e3cpzmdmZmJhYUliogGqquvR1d1JfPxw7t27jZ1dNcqUKYOpqTmuroM5fvwoXl6rSUmZ\nQ05OJnp63iQkuFCuXCYi0buFIUQiEUZGxuzZs5NFi+ZiYlINJ6fe1KvXsMhQuR9/7MSyZZ6oq6uz\nbp0PtWtbkJiYgI2NXNW0Ro2ainQYQLGhdWPHTmDFisU4O/dBIpFga2vPhAlT/h3TO4f9XfHm/Pfq\n1Q8LCytmzJiMRCLB3NyCLl2cSExMxMOjsNjNmjUrCQ8PQyaT4eBQjxo1anLr1g3FPIpEIqpXr0l4\neBjZ2TlMnOhBx45dFLkeQR6uGBUVyaBB/ZHJZOjp6bNw4dIix/vrr4HcvdsXgPh4I1av3kX//lKF\nouT7kJGRwaxZibx4IRe3CQvLxMRkH5Mn//jebeRPKO7jcxJf307IZPoAeHpeoGXLMIyNq7x3ewIC\nAv9fiGRfiXxS3oJB4PvDwEBbuL/fMV/y/v7992WGD69FVpYxALq6V9m7VwULixof3NbFi/cYNAji\n4uRKkZaWWzlwoBWampolOuavHX//S5w7l4yeXg5Ll/5Eerq02LJhYaGMGzeSli3b0KhRE2xsbBk1\naqhCWKRly0acPHmRJk38yM7+i/R0R2Ji5mNqaomzszNBQbcYOvQXLC2t2bzZmz//9EJLqxzx8dmI\nRNmoqNTBz288d+/eICTkHu7uk/D2Xo+GRqki98wJfBgl+d39Fr0/Q4YcZd++7or3hoaHuHGjPqqq\nqu/dRkREOA0aZJKV9Tq1Sb9+u/j117bvVX/p0oUcOnQAIyNj2rfvyM6dRwgLkyKVahATM5fs7LIM\nGTIPS0tTxWf+5597snTpKmQyKRMmjMba2q5AuhE1NTViYl4wadIURCIR9erV5/Lli9/UvRF4O8K6\n6vvFwED7g+u8/+MnAQEBga+Mjh0bMGbMVczMArCw8GfGjNiPMuQAjh8PVRhyAHfutOLWrfslNdRv\ngh07LjBhggk7djixdm1Pevb0f2v5KlWM8PbeSvXqNdiwYQ0+PhsKnBeL5cEfw4YZIhanIxa/xN7e\nF1XV1z89efvKlJREaGhocOjQAVxdu1O3ri2HDs3CyOjb8o7GxSUwbNgefvrpGO7ue8nIyPiodiZO\nHENaWiqpqans2fPa23nz5nUmTXIvkbFevXqVO3eCS6QteL/k7p+TR49CWb36Hw4cuPBe5du310ZH\nJ+/602jaNOqDDDmQh2PXrv16DlVVX+Dg8P55ICdOnErZsgb8/rsXUVGRWFmZkpg4l1ev3ClffjI1\napylcuWCojn55zk8PIzu3XuyZctOtLS0OXPmJAAeHh6MGzeZTZuKTqEgICDw/SCEWQoICHzTTJjQ\njgkTPr0dAwMlIB2QL8S0tZ9TuXK5T2/4G+L06TQyMvL2e4m5fNmYlJRktLWLTmD86tUrtLW1+eGH\n9mhqavH33/sKnM/KygSgV68W+PrOw9FRyuzZ7XByWs39+3cBkEolpKXJE5Nv2LCOhIQEVFVFhIbe\nJzU1DS0t7QJpJIoLJsmvcrlxoxc2NnY4ONT7xBn5cMaNO8k//zgDIi5fzkUk2s6KFV0+uJ280NWo\nqEj27PGna1d5KGx8fByBgYUVRz+GK1euIJMpKxQoP4U3w3j/ay5fvsfw4clERPRERSWSa9f2M3fu\nT2+t061bQ7S1b3H2rD+GhiKGD+/+1vJFoayszLp19Vm8eBupqeo0bSqmX7+WH9yOTCbj9u0gFixY\nio3NC/bvT+H580iWLDHkzp2nxdYrKt1Iaqr8IYCNjS0Abdt24PLlix88JgEBgW8DwTMnICAgAAwZ\n0pKuXbehp3eCChX2MW5cNCYmRl96WP8pOjpZyBMvy9HTS6BUqeLDTJ8+fYybmwuurn3ZtOlPnJ0H\nFTifl85AWVmZFi1aExh4g0mTxlKvXgOioiK5f/8e8+bN5Pnz51SpYoyOjg7jxv3CwYP7iY+Px919\nBCdOHEMkEim8EfLXhceS31sxaNDQL2LIATx9qkue3Dwo8/hx0fO3bdtmdu2SGz+rVi1nzBj5nsQb\nN64xZ850evT4iaSkRNat+52IiHBcXfuyZs1KQIRUKmX69Mn06+fE3LkzFG1ev36VgQP74ezcG0/P\nueTkyCX4nZw6KVQlQ0LuMWrUUKKjo/jrr7/YuXMbrq59CQoKLHKc/v476N+/B/PmzSjy/KFDB/j1\n1yWA3KDevt3vQ6arEHv37ubw4YMfXG/z5lAiItoAkJNTkYAAXbKyst5Zr00bO+bNa8fIkW0/Ogdg\n1aqVWLeuE35+bXBz+3BDLj8ymYxu3RqyadMP6OurY2lZDbFYjEz2OtxZvtdQTuF0I4UVU7+S3TQC\nAgKfCcEzJyAgIIDc4PDy6kVKSjIqKqpFqt9970yZ4sjDh94EBlpQpkw0c+YYvnWBW69eA+rVKygB\nn5ckHFAsQG/evE5ERDj16jXk2bMnlCtnqEgYff/+XVauXE5qaiqGhhVYtWodISH32LFjK0uW/Mqq\nVctRU1NnzBi5+/XUqeMsXSqXhff13cjhwwfR09OnXDlDzMzMAViwYDaNGzehefNWODl1on37jly4\ncA6JJJd58xZhZGRCQkICc+ZMIy7uFZaW1ly7dgVvb79PTs5cqVIKDx4oZoBKlQonvwawsbFnxw4/\nnJx6ExJyn9zcXHJzcwkODsTW1p47d4IRiUQMHz6aZ8+eKtIiHD9+hKysLKRS+dxevnyR7dv9CAy8\nwaNHD1m1ah1RURF4es5nz55d9OzZp8jwx/LlK9C7d29kMjG9exe//zC/OEdRFJ9X7ePo0uXDvWPy\nvgu+V1KSffGwzw/F2tqOo0f/wcVlMDdvXqd0aT1KldKkQoWKXLhwDoAHD0KIiop8aztaWlpoa2sT\nHByItbUtR4/+818MX0BA4AsheOYEBAQE8qGtrfNVG3Il4f0oDn19PQICnLhypSwXLjSmX7/Gn9ji\n68X048cPGTt2An5+/kRGRnD7dhA5OTlMnTqJJ0/qcOHCTEJCunL16pN/a8iYPHkPGzaks3JlFr/8\nshOpVKpYoIeE3OfkyWNs2rSdZctWEhJy73Wvb3jySpfWw9vbjy5dnBRz5+OzHgeHemzZspPmzVsR\nExP9idcqZ/HiBrRqtQVz87106LAZT8+iPTWmpmY8eHCf9PQ0VFVVsbS0IiTkPkFBtxSKmlC0V0Um\nk9G3789s3bqL0qX1ePDgPk+fyo3kypWrcPDgAbp06UZQ0M13jvdtTpulSxcSGRnB+PGj2LHDDw+P\n8Tg792HoUFeePHn81nYfPXqAm5sLzs59mDp1IikpKSQkxDNo0OsUDk2a1OXlyxgAevXqQlZWZoHP\n98iRbqxd+ztDhjjTp083hfcwMzOTGTOm0L9/T6ZOnYibmwutW0OVKv8AMtTUXtCzZ9oH73/7csg/\nrwMHuvHgQQjOzn1Yv34N06fPBqBZs5akpCTz8889CQjYSZUqxq9rvmGw5r339PRkxYoluLr2LbKc\ngIDA94PgmRMQEBD4hvjcizIlJSUMDQ1LvF1zcwuFd6dGjVpERUVSqpQmKSmqBAfLpfwfPmzIkiXb\nmT+/PK9eJXL4cEt0dJKRyUqxa1d3Gjc+9W9rMoKDb9G0aQvU1NQANRo3blps382ayQ2qWrXMFAIR\nt28H4em5HID69RsWuy/wQzE2rsD27e/eI6esrEyFCpU4dOgAVlY2VK9eg5s3rxEREYGJSdW31lVV\nVVPsczMxMSE09AWNGzfh4sXzpKSkcPfuHTp06MTDhyGAPCG1VCq32rKysott900mTpzK1auX+f13\nLzZu9MLU1BxPz+XcvHmd+fNn4uOzrZCxmffxnD9/FuPGTcbGxo6NG73w8VnP6NHjyc7OIj09jeDg\nW5iZ1SYw8BbW1jbo6emjpqZeIIxWJJKHlG7Y4MulSxfw8VnPb7+tISDAH11dXfz8dvL06RNcXfsy\nfrwJu3frcOzYLkxM9GnTpsN7X+eXJi+hOICn57JC59XU1Fix4o8i6xaXbsTCwqKA+MmIEaNLYqgC\nAgJfIYJnTkBAQOArx9d3I336dGPEiMGEhr4AYNSooYSEyNU2ExMT6dFDLvYgkUhYvXolQ4YMwNm5\nD/v2BXyxcedHReW1l0QsVkIikSASgURS8GcoIUG+zy4zM4fc3PKAGJACusTF5eTbL/SmUVu8iylv\nX1Fev4oaX3gvkY2NLdu3+2Fra4+NjR179+6mVq2CCcdLlSqlSOSeR357XiaTGz29evXj5csYdu/e\nQcuWrTl69DC2tvaAPKQyz3N55swJRV1NTU3S09PeOc48cY62beW50+ztHUhKSiq2bp4KZ56HsV27\nDgQG3gLA0tKG4OAggoIC+flnV4KCbhIcHFjAG5mfZs1aAHJPZnR0FCA3xFu1+gGAatWqU726XADE\nxKQiQ4a0o02bL7Nf8mtAJpNx9ux1AgJOv9eeQQEBgW8fwZgTEBAQ+Ip5Vzjhm/z99z60tLTYsGEz\nGzb4cuDA3nfusflSGBmZoKKSjIbGWQBEojgcHOSJug0N9TA3DyAnpxJqavcwMTmAlVWpf69FhK2t\nHWfPyhes6elpXLhw/oP6trKy4eTJYwBcvXqZlJTkEr2298HGxo74+DgsLa3+9UypFTJqdHVLY2Vl\nw4ABvVizZhUgIisrizt3bgMQFvaCKlWMqFChIqam5mzatJGzZ08hFosVyeBdXd1YuXIZgwcPQCxW\nVnxuWrRowdmzp3F17UtwcNECKPkpbPx+uJfY1taOoKBbxMRE06RJMx49evhWYy7vIcCb4h5f2hD/\nGpHJZIwe7U/PntXp3t2WHj32kppa9J5NAQGB7wchzFJAQEDgK+ZDwgkBrl27zJMnjzl9Wu6BSUtL\nIzw8jAoVKv4Hoy1IQXGMwueVlZVZufI3PDxmkJqag7q6MgsW+PLkySNUVVXw9bVh3bqr3LnzgFKl\ngrl82U6xX6hWLTNatWqDi0sf9PT0qV3b4n1GpBiTq6sbs2dP48iRQ1hYWKOvX+atyp2fgzp16nLq\n1CXF++3bX3tR/f33K17PmjVf8To6OgpjYxP27NnJokVzMTGphofHTAB69OjNrl1/sW6dd4F+5B7A\nwh5aExMTfH23v9dYixbnKJhPTSaTIZOBpqYW2to6BAUFYmNjy+HDB7Gzq/PvWOzw8lqNnV0dRCIR\nOjo6XLp0gWHDRuVr5+1jkRvix7G3d+DZs6c8ffr2/XtfA6mpqRw7dliRYuJzcPVqMP7+LZBK5Sq8\nly+7sn79LsaN+/Gz9SkgIPDlEYw5AQEBga+aor0f8n1Qck9FdnbBcKpx4yZRt26Doqr9pxw9egaQ\nh+XZ2zsojru7T1K8Nje3YO/egoaGnV0dxeJ/0aIuQNF70AYMGMiAAQMLHZ86dZbidX6jyMzMnFWr\n1gFyxb8VK35HLBZz504wDx7cQ1n56/9JLF++Alu37ipwLCbmJbdvP+XKlct06vT2/Xo5OTns2XMO\niUTK0KHvs8h/Lc7h6TkXZ+c+aGhoKMQ5iksbMW3abJYt8yQyMgI9PT3WrNmoGD+gCAG1sbHj1atX\naGlpve6xWIefiI0bvVBVVSUxMYH+/XtibGxM1arVCtT/GKRSKUpK8mClkSPdGDnSXaGOWhKkpCQX\nyBf4OUhPz0Qqzf9AQkx2tiB8IiDwvSOSfWSswuLFizl9+jQqKioYGRnh6emJtrY2AF5eXuzevRsl\nJSWmT5+Oo6PjO9uLjU35mGEIfAMYGGgL9/c7Rri/n5eHD0NYsGAO69dvQiLJZeDAn+ncuRuhoc8x\nNTWjSxcndu7chr//Dvz997N//x4uXbrAvHmLUFZWJjT0BeXKGX6UQuf3eG/T0tJYuvQUL1+mEB29\nEx0dDVRUlBk/3qNEF+//FYcO3WDKlBxUVX9HRSWXuXMn0aZNnSLL5uTk0L+/P6dO9QfEtGixHV/f\njp9VvdXbez0aGqUKiHN8anvq6ho4OfVCVVWViIhwxo79he3bdxdrjEdFRTJ+/CjMzGrz8GEIJibV\nmDFjDv369aBVqx+4du0K/foNQFtbB2/v9Tx58hgLC0s8PZejoaHB2rW/c+HCOcRiMfXqNeCXX8aQ\nkJDA8uWeChXU0aPHY2Vlw8aNXsTERBMVFUlMTDQ9e/bByak3s2Z5cP78WYyMjKlbt8FnESTJycmh\nT59dnD3rCqhQs+Yutm+3xsioQon3JfBl+R7/NgvIMTDQ/uA6H/0Y0tHRkYkTJ6KkpMSyZcvw8vJi\nwoQJPH78mEOHDnHw4EFiYmJwdXXlyJEjiideAgICAv/PtGnThGPHzr13+XPnzmBgYFAgnFAkkivX\nzZjhwf79e2jY0JE8D16nTl2Iiopk0KD+yGQy9PT0Wbhw6We6mm8LmUzGwIEHOHVqICBGS6sRy5dH\n0bXrl/difixr1sQQHd0LkCfMXrfuL9q0Kbrs7t1nOXXqZ0AeHnnqVH/8/PYyeHC7Eh3Tm/n/TE3N\niYgIZ8WKJSQmJqCurs7kydPQ1y+Li0sfdu06AEBGRgb9+jnh77+f6OioQuWNjEzIzs4mKOgZ/v49\nUFMTExv7En19fWbO9MDDYyba2tqMHOlGzZqmBAbeQCKRMGTIcMLCQhk/fgrKyspcuHCOXr26IpFI\n0NUtzdq1fzJ79jSuXr2Cg0M9qlathrFxVf76ayvduvXg3LnTiryIaWnyPWgrVy6jZ8++WFvbEh0d\nzYQJo/Dz8wcgLCyUqVNnMWXKOHx8NtC1a49C+QJLkvyeRD+/rmzYsAdlZXU6drShSpXyJdZPVFQk\nkye7s3nzXyXWpoCAwKfz0cZc48av8w/Z2Nhw5MgRAE6cOEGHDh1QUVGhcuXKGBntZbufAAAgAElE\nQVQZERwcjK2t7aePVkBAQOCb58PCnkQiEXXq1GPZslWFzuXf7zRkyHBAvtjs3Lkbbm4jhNxSbxAX\nF8e1axbIFTIhNdWSkycf0LXrlx3Xp5CZqVLgfVaWSjElITdXSt61y1FCIilZIZH8gj1yT3J/TE3N\nWbJkIRMnelC5chXu3r3D8uWLWblyLTVr1uLmzevY2ztw8eI56tdvhFgsZsmSBUycOLVA+XnzFrNz\n5yPCwxuRmLgUC4vWLF/+G/b2DgXSH4hEIrKyMvHx2UZQ0C0WLZpHuXKGXL9+FQeHerRr14Ht27dw\n9eplHB2bsmfPLjIzM9HQUCcsLJTQ0OckJiZQp05dNDW1UFVVw9NzLo0aNaFx4yYAXL9+lRcvnimu\nOz09nYyMDEQiEY0aOaKsrIxYLEZPT5+EhPjPKtiSP9RVXV2dUaPaC54bAYH/I0pkg8Du3bvp0EGe\n0+Xly5fY2NgozpUvX56YmJiS6EZAQEDgu2Lbts2cOnWc7OwcmjZtzqBBQ4GiPRvvw9q1J/njD1VS\nU8vQqNEpvL27oqGh8Tkv4ZtCS0sLXd2XvBb4k6Kt/W3Jtw8fPpC1a18LnLRrJyEkJJzs7Mqoqz+j\nfXvYuXMbnTt3Q02tYPikk1MT/P03c+mSKyCiQYOt9OtXjBvvIylKsCc7O4s7d4KYMWOyolxOTi4A\nLVu24eTJY9jbO3D8+FG6d+9Jeno6t28HFyq/adNFIiPtADFKShLS05WJjMzE3l6e/mDGjCmK8q1b\ntwXke/IyMtJRUhJz9eplLlw4S1ZWFomJiYB8L1tQUCB2dnXQ1S3N7NkLGDiwP5MnT8fU1AyADRt8\nuX79KqdPnyAgYCcrV64FZKxf74uKSmHjWVlZfkwikRAX94qRI92oVKkKMpkMH58NXLx4jqysLCwt\nrZk0aRoA/v472LcvALFYjIlJVebMWUhGRga//rqEZ8+eIpHkYmtbh9u3g0hLSyUyMgIDg3IkJydh\nZGRCZmYmP//ck7lzF1G+fAVcXEYRF5eARJLLkCHDcXRspgg3tbS05vbtIMzMatO+fUd8fNaTkJDI\nrFnzMDe3YONGLyIjw4mIiCAxMZF+/QYU2ospkUhYt+4PAgNvkJ2dQ7duPejcudunfXgEBAQ+irca\nc66urrx69arQcXd3d1q2lCdhXbt2LSoqKnTq1KnYdoSnwwICAgIFuXr1MuHhYWzYsBmpVMqUKeMJ\nCrqFmpp6Ic/G++zlio2N5bfftElIkP9tPnHCjpUrdzNlSuHkyYcOHeDBg/u4u0+id++utG37I66u\nQ0r8Gr821NXVGT9enWXLDhAfX4E6dQKZPLlkQww/N/kNOYAJE9phYnKB+/cvYW1dms6d29Cjx0+0\nbftjIWNOXV2dHTs6s3nzHqRSGePGdSMjo6Q9RoV/72UyGVpa2kWGGDZu3JT169eQnJzMw4ch1KlT\nl/T0NLS1C5dfterwR48qNvYl6uoaLF68gq1bfTExqcru3TupUkWu/GhsXJV9+wKIiAgHICsrk7Cw\nUMqWNSAzMwMTk6rcvRtMQkICAHXrNsDffwd9+/4MwKNHD6lZ83WOwJ07t/HixXMqVqzEb7+txcvr\nD+7du42jYzNOnTrO5s1/MW/eTC5cOEfjxk3YutWXXbsOoKysrAjl3LzZGweHekydOou7d+8wZsxw\n9u07zNatm9i+3Y9Bg4YSHBzIgQN72bVrB23b/kjVqtWQSCT88ccfZGTISExMZNgwVxwdmwEQHh7G\n/PlL8PCYyeDBAzhx4ihr13pz/vwZNm/2USQtf/r0CV5em8jISMfVtR+NGhXUPsifAiU7O5sRIwZT\nr16DL6KaKyDw/85bjTkfH5+3Vg4ICODMmTP4+voqjhkaGhIdHa14Hx0djaGh4TsH8jEb/gS+HYT7\n+30j3N/3RySSz9edOze5ceMqQ4bIF4MZGRkkJr4kLS2N9u3bUblyWQDatGmNpqbaO+c4NjaSpKRK\n+Y6okJ1dCgMD7QJKfQDa2upoaKhiYKBNxYoV6NChbbHtl9S9bdmyJQEBcs/DgQMH6Nu3LwBXrlzB\nx8eHdevWlUg/RREeHs7w4cM5cOAA7u7tcHNLIzExkQoV7L+5/dx2dnbcunWLK1eu8Mcff6Cnp8ej\nR4+wsLBg8OBlbN68mVevYnF3H4G+vj6+vr78/fffeHl5AdCsWTOmT5+gaO8TRSAL0aKFI1OmTMHd\nfRQ5OTlcuXKBXr16YWRUhRs3LtCuXTtkMhkPHjzAzMwM0MbGxpp1636jdetWlCunA+gUWX7ChLbs\n3z+OsLAGSKXKaGpKqV1bHwMDbXbsOE7jxg0xMNBGRUXMxYunadu2BdevX0dHRwcdHR3EYjHDhrnS\noEEDhgxxZefObZQpo4WjY0MCA6+yZMlipk+fyKNHj1i8eB4eHh5UqVKO8eMnkJqaSmRkJPPmzcPA\nQJt582Yzd+5cBg3qh0QioW7dujRqNBtNTTU0NdVJTVVDR0cHLS1NypTRpH//Pty+HcjkyWOJi4tj\n4MC+JCUlYWVVGwMDbczNzfD0nEXr1q1p3bo1pUqV4ubNq1y5cgF//23Ex8eTnZ3F8OGuREdHI5VK\n8fffRk5ODiKRiNDQZwwY0A83twHk5OQglUoRi8WIxWIiIsJZt+43rly5QpkyZfDwGEfXrl2Jjo4k\nOTmRlJRYHBxs8PX9EwMDbbS01Gnb9gcqVSoDlKFRo4aEhz/BzMwMZWUxBgbaBAff4MGDB5w/fxrI\nSxQfh4GBacl+oASKRfjdFcjjo8Msz549y8aNG9myZcu/4RRyWrZsyfjx43FxcSEmJoYXL15gbW39\nzvaE2O7vFyF2//tGuL8fhkwm/3uXnp5N377OXL58kZcvYxCJlEhKSiczM4sHDx7TqVNnpFIpKSlJ\nODn14cWLGBYunMPFi+eoUsWYgQPdiIgI5+7d29y5E0xychI1ayqTlNSC6OhfqVHDmsDAslhbz6Bc\nOQPq12/ElSuX0NTUJDk5mZSUFEJDI3j69Bnbt/szZowRI0e6YWFhxc2b10lNTWHRIk+MjU3JzMxk\nwYLZPHv2FCMjY169imXcuMkfpP4olcqIi0slLS2NLVv8aNNGHs2RmJhOVlbuZ/0MxcenkZsrKdCH\nqqoOcXFpn63P9+FjJPDzPj+Jiencu3cPPz9/ypQpy/Dhgzh58jzt23fF29uH335bi46OLvfvP2XJ\nkqV4e/uhpaXNuHEjCQg4QJMmzT/Ld9fAoArNmrWiQ4eO6OnpU6uWOWlpWUydOodlyxbx+++ryc3N\npXXrHyhTRv7wwdGxBTNnevD7716K8RRV3sVlME5O1Xnw4A729gdxcFjG4sVLC6Q/iI1NISdHglQq\nol279kREhDN3rie///4r9eo1RCqVcOdOMF26dKVmTVNycsS0bt2RI0fGMGzYMOrVa4Cqqjrjxk1W\nhFmuWeNNVFQkEyeO4fz5y6xb54WBQTk8PZfz6lUsK1Ys4cqVa9jY2ODt7YeRkQmrVi1HJpPh7b2N\nkJD7zJw5lZSUVCSSXCpVqoy39za8vdcTH59MbGwKCxYsJzDwJhcunGP16jX4+u4gN1fKnDmLqFLF\niN27/+LVq1cMHfoL7u6/cP36VdzdJ1O+fEW6dm3Py5cvmTRpMr//7kVwcCC+vn/SrVtPevXqR7Nm\n9RGJVFm4cDmTJ7uTkZGBikopGjduilgsZs0aLwYMGEhWVjaxsSmkpWUhk8kU9yIzM4eUlKwC36Os\nrBzGjJlQKAWK8Fvw3yD87n6//KdqlvPnzycnJ4eBA+U5fmxtbZk9ezY1atSgffv2dOjQAbFYzKxZ\ns4QwSwEBAYE3qF+/ARs2rGP+/CWUK1eO8PAwJk92x919ImvWrGTz5r8oW7YsLi79EIlg06Y/0dTU\nonLlKvj6biclJYVHjx5w/fpV1NTUOHz4NO7uI8nOVkdHZxdXr2bRtKkjY8dO5MWL5wwY0IudO/dz\n/PhhduzYStu27enUqSsuLn0K5AmTSqVs2ODLpUsXWL16NUuWrCIgwB9dXV38/Hby9OkTXF37vvXv\nuofHBF6+jCE7O4sePfrw009yhRGZTMa6db8TERGOq2tf6tatT8OGjmRkpDN9+mSePXuCqak5M2fO\nA+QiE2vWrEQikWBmVpsJEzxQUVHByakT3t5+6OjoEhJyj9WrV/L7714kJCQwZ8404uJeYWlpzbVr\nV/D29gPkecQWL17AnTtBioV4/geRxfGmV7MkyS9c8TGYm1tQtqwBADVq1CIqKgorK5sCZe7fv4u9\nvQO6uqUBaNOmHYGBt2jSpPlH9/suisv/t3x5YREfgObNW3H27NUCxypUqFhk+WHDRhZ47+Xlo0h/\nkD/XXNu2HejRow+TJ7tTvXrNf0NsJ7/ZHABqamq4uAxmx46tLFhQvPJrWFgos2cvZPLkacyc6cGZ\nMyc5ePAAEyd6IBaLGTNmuELYRSaDlJQUxo3bwPPnf1OjRlXs7R348891iMVi0tPTOXXqOC1btkEm\nkxETE429vQPW1racOHGUjIwM6tVrwK5dO3B3n0SdOvUYN24UPXv2pXZtC27fDqJsWQNmz54KyENo\nZTIZlStX4dKlC5ibmxMcHEjNmqZIJBKFcEsezZq15MGD+1SsWIkbN64VOCeTyTh//gw//+xKRkY6\nt27dYPjwUWRnZyvK1KvXkICAXdjZOXxyChQBAYFP46ONuaNHjxZ7btiwYQwbNuxjmxYQEBD4bslb\nvNet24Dnz58zaFA/0tLSUFJSQklJzJ07tzExqcbkye7o6eljaWkFwI0b1xg5ciz3798BQFtbm5iY\nGCpUqEiZMmWZN28mxsbGqKqq4O7eFkfHady4cQ1X176kpqaioqJKZmYGd+7cViwgq1evgb5+mQLj\na9asBQCmpmZEREQAcPt2ED179gGgWrXqVK9e863X6OExEx0dHbKyMhkyxJnmzVsqrv1NifabN6/z\n6NGDAh6m27eDqFXLjIUL57Bq1ToqV67C/Pmz2LNnFz179inWAPLxWY+DQz3693fhypVL/P33PsW5\nohbiP/zQvkjDs02bJnTu3J3r168ybtwk7t27w6FDcvn8jh270LNnn0Iy7du2bSEzM4OBA90KeTin\nTJmJjY0tWVmZLFw4hydPHmNkZEJWVtYnqRyqqKgqXovFSkgkuYXKiESiN/r4fKqKJUVReeGmT59D\n//49ijTiAR4/fsiwYQNJTExESang56NChYqMGTOBSZPcWbLkV27duvGv5wzi4lKwsBiMiUlasQ8V\nQkLus3z5IpSUxKxZs4pp02ZhampGcHAQN29eY8CA3mhqliItLQ1VVfkDgkuXnpCbq8fx46Foa0ej\npVWWrl2dCA19waFDBxg/fhS1a1sCcjGRefNmkpaWikwmo0eP3mhpaeHiMphVq5bj7NwbqVSKnp4e\n48b9QkZGJllZWQwePABlZWWMjEzQ19fnwYP73Lx5nR9+aMfBg3uJiopCU1MLZWVlxf7JvO+Oqqpc\npEVJSQmJRFLgnEgkonr1mowePYzExERcXQdTpkxZoqIiFWWEFCgCAl8PJaJmKSAgICDwfhw9ekbx\nunr1GlSpYsyvv65GTU2NUaOGUrOmKaGhLxQLyTyOHZPn65RKXy/Gc3NzABnLlq3k1q0bbNniw4MH\n9xk9ehwikRILFy6jShUjzp07zZkzpzA2Nvm3ZvEL+jwDQUlJTG7ua+PgQ4wOf//tnDsnv86XL18S\nFhb21nYKe5giUVfXoGLFSlSuXAWA9u07EhCwU2FUFsXt20F4ei4HoH79hmhr6yjOVahQiRo15Eao\nqakZUVGRQNGGZ2ZmJhYWlowcOZaQkPv888/fbNjgi1Qqw83NGTs7e7S0CobC5Peyvenh9PFZz2+/\nrWHPnl1oaJTCz8+fJ08eM3Bgv88SuVKqlNyw0NHRxczMgt9+W0ZSUiJaWtocP34UJ6feJd5nSZOX\nq83S0hpPz7kEBPgXO1cymYwnTx6zfv1rwY6yZcsW8CTlZ8cOP8aPn4KX10P++acTV67ooa+/i4oV\n77FzZ4DioUJwcCC1a1vy229LmTjRg/nzZ9GhQyfWr1+DiUk1Tp06jo6ODgcPnmDNmpVcvnxRYdy/\neKFDfPxQkpOd0NAIIienF2pq6nTr1pM7d4JZu3ZjgTGtWfNnoXGqqakxceLUQsejoiLp2bMzixat\nwNLSikWL5lGxYiUiIyMwNCyPrm5prKys6NixC05Ovbl16waGhuXQ0dHF13cHPXr8BMDUqbMICbnH\n5csXqVChIr6+OxR9VK9ek+nT5xToN38ZqVSKm9sIhg79pbhbKCAg8B/xbe38FhAQEPiOyFPtU1NT\n4/nzZ9y+HcyxY1e5deuGwthITk4CoG7d+pw6dYLExHiSk5OIj4/jxYtnREdH8fjxQ2xs7JBKpchk\ncjEVsViJXbvkCy9zc0uuXbtCcnISVlbWnDp1ApFIxNOnj4mPj3vnOK2sbDh58jgAz5495enTx8WW\nvXnzOjduXMPLy4dNm7ZRs2YtsrPfLv9f2MMkKbRwl8lkimNisVhh1GZlZRcqVxR5ngiQG6p53gh/\n/+24uPRl6NCBCsNTSUmJ5s1bARAcHPiv1L46GhoaNGvWkqCgW0UaFvn7zu/hjI6OAiAoKJAffmgP\nyA35d3k4iyJ/v8XZgT/91JXx40cxZsxwypYty7BhIxk9ehiurn0xM6uNo2PTD+73v6ZcOUMsLeX7\n7du2/ZHbtwOLLSsSiWjSpBmqqqro6pbG3t6Be/fuFFveysqGlSuXc+nSfcRiCSAmI8MEZeVKlC1r\ngEgkokaNWkRHRxEa+pxnz54wd+4MwsPD2LzZm9jYWLKzs8jNzcXIyIRTp47Ttm0HZDIZjx8/AkBZ\nWYJIJEMq1f733xMAjh7955Pm5dGjF+zZc5by5SuyZ89O+vfvQWpqKr169WPq1FnMmDEZZ+feiMVi\nunRxypuhN2eswOuiPsvFfbYyMjJwdd2Jnd05mjc/xOHDtz7pegQEBD4dwTMnICAg8IWoX78Re/fu\npl+/HiQkKJOcbIWPTzuqVoVJk9wRi8Xo6+uzYsUfODsPYsWKxYjFynTu3I5KlapQu7YFhoYVGD58\nEFKplFKlStG/vzNaWlqoqqqSm5ubL0RLn6FDXdHU1EJDQ4MjR/4hNvalwiNWFHmLvG7dejB//iz6\n9++JsbExVatWK7A/KT9vGqh37xZcVJcqVYr09PR3zo2RkTFRUZFERIRTqVJljhw5hK2tPQDly1cg\nJOQeM2ZMLpD/Sm50HqNfP2euXr1MSkryW/vIb3jmeUazs7NQVVUr4GXLT55Rmd+gBLmUff6y+T2c\neYZjSZDn2bW3d8De3kFx3N19kuJ19+696N69l+J969ZtFXnXvhXyz6V8zpXeasQXrl/8s+r+/V1o\n1MiRfv18qFKlD+Hhcq+YsvLrOnkPFQCqVq3OrFnzmTJlnMIztWmT3LM2c+Y8li1bRGRkBNHRkZw/\nf4YaNWpSt646p08/ISHhJWJxW2Syvbi6nqNu3QYf7Y3dv/8qHh6qxMa6UqaMBb17pzJjxmsPfp06\ndfH23goUFMjw938dbpybm4uXl7fCa21mZs6qVQWVZAcOdCt2DIsWneDgQWdAmehomDPHn9atc1FW\nFpaTAgJfCuHbJyAgIPCFUFFRYdmyVdy//5BWrTTIza0NwL179WnV6i9mzHidI05DQ4Np02a/d9tH\nj5794PHk7T8CKF26NCdOnCA2NgWRSMT06XNQV1cnIiKcsWN/wdCwfJFt5Bmo/fv3oEoVY8Wevzxv\ngDwEzIYBA3rRoEFjGjZsXKQXQFVVVeFpkEgkmJtbKDwNrq5uLFo0l8zMLMRiZcXi2NXVjdmzp3Hk\nyCEsLKzR1y9DqVKapKWlFWmUvcvwBLCxsWXBgjn07++MVCrj3LnTzJgxDz09fYWXVF1dg4sXz9Ow\nYeO3zq+trR3Hjh3G3t6Bp08f8+TJo7eW/1SysrLw8TlFVhb07l0XQ8My7670lRATE82dO7extLTi\n2LHDWFvbkJ6eRkjIPRo0aMSZMycUZd9HsCM/ERHhVKtWg19+ac+qVS9RVb2CkdFzqlbVKVTWyMiE\nxMQE4uLi/lWYzCUsLBQXl0GcOnWc2NiXLF++ijVrVnH58gVcXAYDsGLFPCIjo3jwIBh7+5/Q1R2g\naHPEiNEfNScbN8YRG9sTgLi4Rnh776R79/evf/jwLebMiSE21pDatU+wYUNLDA3LftAYXr1SJf/S\n8eXL8iQlJVGmzLfz2RIQ+N4QjDkBAQGBL0xurgSpNP+fYxFSacnupfL3v8SmTUlIJCK6d1dlyJAW\n76wjkUgYPXoXp07poK29mrJlZejpaTJhwpRin8TnGaiF+3/tHZg1a36Bc3Z2dRSv83uYHjy4z48/\ndsLJqTerVi1n/PhRrFy5ltzcHMzMavPq1StUVFRITk5m6FBX5s1bxIoVv5OcnMzMmVNITU1l+PCB\njB49Hl/fHWzc6EVkZDiRkZGUL1+Bn37qxpIlC2nZshGqqmqYmFQFCnqFatUy48cfOzJkiDMAnTp1\nVSSHdnEZTIcOrbGxsVPULRp5e126OLFwoVzIw9jYBDOz2m+p82nk5OTQv38AZ864ACoEBGxn586G\nH7x4/1IYGRmzZ89OFi2ai4lJNbp27YG5uSWLFs3lzz+1sLOr80GCHfJy8v/9/bdz8+Z1RCIlWreu\nRLdulcjN1WfPnieFxqGsrMy8eYtZuXIZqany1AK9evWlatVqTJ06C0/PuYhEFOlxe/z4FTduJJCV\n9Yh27ep+8pxIJOIC73NzxcWULIxMJsPTM4InT+R7Ti9fbsKCBVtZtarzB42hbl019u4NJyenMiDD\nwuIR+vo276wnICDw+RDJPkVKqwQR8mV8vwj5UL5vhPv76UilUgYP/ou//+4HlKJWLX/8/GwwMalY\nIu3fu/eEbt0yiI9vCICm5gO8vV/SooXtW+v5+Z1j3LgmgFzso3Tps5w+bUjFiiUzrndx9+4dduzw\nY968RYwYMZjc3FzWrPmTLVt80Ncvw7Jlnixe/CuNGjmyZs0qcnNzCQy8QWRkJGXKlGHSpOn8+WcQ\nt2/vwMFhKDVrRnDt2hXWrPkTVVVVZs+eRrduPbC2tiU6OpoJE0bh5+f/QWNs06Ypx459uBf0c3P0\n6CX697cH8ow3GRMn+jNxYntFma/1u/umUui3yNat55g1y4jkZEvU1Z8yfvwtxoz54ZPa9PE5w9y5\nNUhLM6NUqYdMmXKPYcNaFVn2zXsrkUiwsztHdHQnxbH27Xfj6/vhY/LyOsHFixJ0dTOZNq3RN/OA\n4Hvia/3uCnw6/2meOQEBAQGBkkFJSYkNG3qybdsxkpNz6NbNgQoVit/L9qFcu/aI+PjX8VhpaaYE\nBd2mxTuccxERueQZcgCJiTV4/vzRf2bMmZqa8eDBfdLT01BVVcXMzJyQkPsEBd1i7NiJqKio0KiR\n479lzbl+/Qre3lvp2LENqqqqTJgwjYQEPcRiMVu2dKZJk3F07NgUVVX5frbr16/y4sUzRX/p6elk\nZmYWyJW1bdtmVFVVFd7BJ08es3LlWm7cuKZIfbB+/RouXjyPmpoaixYtR09Pn4SEBJYv9yQmJhoA\nS8sfiI8vS2rqecqUUSEqKpKYmGh69uzzWdQl1dSUEYkyef24Voqy8lfx7Pa9+Jrz0z56FMbs2bd4\n9UqT2rWTWLy4o+Izlcfu3RkkJ8tTD2RmVmPfvkDGjPm0fl1dm2FsHMitW7extS1Hq1ZFG3JFIRaL\nsbd/yaFDuYAyKirhNG78cUvAoUNbMXToR1UVEBD4DAjGnICAgMBXgFgs5uef339x9iE0aGBK2bKX\nePVKvqdLS+s+dnZF73nLT5s2lfnzz0CSkuQePHPzc1hb/3dKiMrKylSoUIlDhw5gZWVD9eo1uHnz\nGhEREZiYVEUsfv0TpqQkyic0ImP9el86dDhDaGhXRZm4ODVFvq385VRUVCgOGxt7duzww8mpNyEh\n98nNzSU3N5fg4EBsbe05fvwIlpbWuLmNYM2aVezfvwdn50GsXLmMnj37Ym1ty/z5f7F16188f36M\ncuUeUr36GXbv/ou0tFT69u1O1649EIvfP2TufWja1IHOnf9i797OgBb16m1nyJAfS7SPz8WbMvlf\nG+PHX+fyZfkeuFu3stDW3s3cuZ0KlJGrZL5GWblkRHBatrSlZcuPq7t2bScWLfInNlaVevXUcHH5\nyIYEBAS+KgRjTkBAQOA7x9S0KgsWXMbHxx+JRISTkzrNmjV7Z722be1ZuvQ4Bw/uRlU1m7FjrYtV\nsfxc2NjYsn27H1OnzqJateqsWrUCc/O37zWrW7cB/v47KF9e7lVUVQ0hO9sMLa3sIsv17fszAI8e\nPaBmTdMCZT7GOwgFvX5PnqQgEskQidLJzdUlJcUMZWVldHVL/+vFi3+rqujHIBKJWLeuJ926XSY1\nNYsOHTqhoaFRon38PyKVSnnxIr9QihrPnqkVKjd4sCEhIaeJiWmMnt4NXFw+PHSqpNHQ0GDOnI5f\nehgCAgIljGDMCQgICPwf0LVrA7p2fXe5N+nSpT5dury73OfCxsaOLVt8sLS0Qk1NHTU1NWxs7IA3\nc669fj127ARWrFhMbu4jatdeR1ZWDapUaUXDhpULKGfmlXN27oNEIsHW1p4JE6YU6P9TvYMqKip0\n6XKIixd75WuTfHWUyM0tudQF+VFSUqJdu0afpe3/V5SUlDA2TiYqKu9IFlWrFs6j2LatPaam4Vy4\ncAB7+2qYm79d6VRAQEDgYxGMOQEBAQGBr5Y6depy6tQlxfvt2wMUr/NyrgE0b95KkehbV7c0c+Z4\nvrPt9y33Kd7Bvn1/ZtSoSoSFrScsrBM6Og9xdCwsgS/w7bB8uQOzZ28lNrYUlpYpTJ/eochyJiaV\nMTGp/B+PTkBA4P8NwZgTEBAQEPi/4+nTcDZtCkYkkuHm5kClSobFlv0U7/xS3VsAACAASURBVGCe\n169bt9o0b16LwMByQk6ub5yaNauwdWuVLz0MAQEBAUBITSDwHyBI6H7fCPf3++VrubcfKlX/4sVz\nZs2aipKSEvPmLaJSpYLekcjIl/ToEcijR90BGbVrbycgwBF9fb3PMPqvl6/l/gqUPMK9/b4R7u/3\ny8ekJlD6DOMQEBAQEBD4Ypw9e5oWLVrh7e1XyJAD2LPnxr+GHICIe/d6sn//1c8+rri4eHbtOsWt\nW/ffu87IkW6EhLx/+fdh4sQxpKWlkpKSwp49uxTHb968zqRJ7iXal4CAgIDA50UIsxQQEBAQ+OqR\nSCTMnTuDhw9DMDGpxowZc3j27Bl//PErGRkZ6OqWZtq0WTx8GMKuXdtRUhJz8+Z1Vq5cy44dfhw6\ndACAjh27ULp0OZSVQ6hceQwZGbaoqwehojKAbds2c+rUcbKzc2jatDmDBpVcMq17954xePATHj9u\nj4bGY8aMOcq4ce9O2CwSiT4o55pUKkVJ6e3PaZcuXQlAUlISe/b407Wr03u3/zYkEkmJp1h4kzZt\nmnDs2LnP2oeAgIDAt4RgzAkICAgIfPWEhr7Aw2MmlpbWeHrOZffunZw7dxpPzxWULl2aEyeOsn79\nGjw8ZtK5c3dKlSpF7979CQm5zz///M2GDb5IpTLc3JyZPn0ubdse5OHDUGJj3WjZ0pQaNcpy5kww\nGzZsRiqVMmXKeIKCbin2xn0seSGiOjoDePy4N3p6GxGJMti58yCqqvcICrpFamoKU6bMxMbGlqys\nTBYunMOTJ48xMjIhK+u1UuLVq5fx9l5PdnY2lSpVZurUWWhoaODk1IlWrX7g2rUr9OvnTKtWbRR1\njhw5xK5df5Gbm0Pt2paMGzeZXr26sHHjFlavXkFERDiurn2pW7c+DRs6kpGRzvTpk3n27AmmpubM\nnDkPgJCQ+4UM5zJlyjJypBu1apkSHBxEmzZt6dWr3yfN17v5epOJCwgICHwJBGNOQEBAQOCz4+TU\nCW9vP3R0dD+qfrlyhlhaWgPQtu2P+Pp68/TpE9zdRwByj1SZMvJcbTKZjLzd4MHBgTRt2kKRLLxZ\ns5bcvh3IwoUdGT58L1u21KBq1aqsXr2Sa9eu4OraF4CMjEzCw8Pe25g7f/4sz58/pX9/lyLPSyR5\nP7eif8eohESSy4YNvsyfPwsfn/X89tsa9uzZhYZGKf7H3n0GNHW1ARz/BwhhJYg4UARFRBxMte5t\naaWOalUcxYWr1FG34sCtddVVd0VxK65XrVqte9Sq4N4DZYsDgQgEEvJ+SEmhYB1FcZzfp+Tm3nvO\nvWHkyTnnedauDeHu3Tv4+emCo2fPnrF6dRDz5i1CJjNh7dpVbNq0jm7deiKRSLC0LERQ0Nocbd6/\nH86hQwdYsiQIQ0NDZs+ezv79e/WjfUOHDuXGjZusXLke0E2zvH37JmvXhmBtXQR//x5cunSBSpVc\nmDt3JtOn/4SlZc7AWSKRoFar+eWX1a90nwACAoYSH/+Q9HQV7dp1pGXL1nh51aNdu46cOnUCmUzG\njz/OxsqqMDEx0UyYMIa0tFTq1Hl3BesFQRA+FCKYEwRB+Mj988Nz8+ZfM23aRG7evI5EIqFZs5b4\n+HRCqVRy4MA+WrduS1jYObZv38SkSTNfuZ29e3fz2Wc1KVKkSK7XXmeqYF6yH6/VajE3N8fBwZEl\nS4L+dd9/tqvVavXBjEIhp2zZsvrXfH278fXX37xR/+rWrU/dui8ONtq3L8Hx4yfIyACJJJXChdNp\n0kQ3zfLo0UNYW+vu2cWLF2jXrgMAjo7lcHR0AuDq1cvcv3+P777zAyAjQ42rq5v+/NlH47KEhp7h\n5s0b9OypK4qenp6OldXfSV7yyn9WsWJlfQHzcuXKExcXi4WFBeHhdxk4MHfgrGv75dNFswsICESh\nUKBSpdGrV1caNmxMWloaLi5u9O79PYsWzWfnzu107dqDefNm8c037fjyy6/Yti3ktdoRBEH4FIhg\nThAE4SP3zw/Pzs4Vefz4kT47pFKpBCA5Oek/raHas2cXDg6OzJ79Y66RlyxeXvXo0aNPjjVsPj4d\nCQ5ewbZtIdSuXY/Dh3/HxsaGpUtXIZPJuHPnNnFxsXTs+A116zZg9+4d+Pp2Y9euHVy5chkXF1fU\najWRkRE4OJTN0Sd3dw+mTJmAr29XMjO1HD9+hLFjJ+UIZAIChhIefpdHj+JRqzNo06Y9GzasYceO\nrSgUlpQr54SxsTGDBg3nxIljrF4dhFqdgUJhybhxk7GyKsyePbu4efM6gwYNZ8qU8ZibW3Dz5jXi\n4+PJzMykYUM3Fi48w+TJSwENGRkZhIff4/jxo6hUKuLiYpk0aWye9zWrr9Wq1WD8+Cl57mNqaprn\ndm/v5vTp0zfHtr17d7/wPZRKjfWPDQ0N9EXQXxQ4A5iY5N32i4SEbOD4cV2NwPj4eCIjI5FKpdSu\nXRcAZ+eKnDv3JwBXrlxi6tRZAHz5pTeLFy94rbYEQRA+diKYEwRB+Mj988NzRkYGMTHRzJ07k1q1\n6lK9ek0AlixZoF9DZWRkhFxukef6qVWrfuHkyWOoVCpcXNwYPnw0hw//zo0b15k4cQxSqZRly4IB\nrX7kJUtmpjbXGjZPzyp88YU3QUHLaNPGB41GTWTkA44ePcQXX3izaNE8bGxKUKlSZfbs2Ulmppa2\nbTtQvXot5s2bhVKpRKNR0759J30wlzUgV758Bb76qjm9enUFoEWL1jg5lSc2NkY/apcV7G7YsIaF\nC+exbVsIMTHRLFu2CgcHR374wR8np/KArubcsmWrANi1awfr1q2mX7+BuUYAnz59wuLFQdy9e5vu\n3b8lKSmRyMhryOUymjf/mrCwc9jZlaZFi1Zs27aZQoWsGDt2Eps2rePAgX1UqVKNe/fucPfubSQS\nCZUru/LTT9OJjo7C1rYUqampPH78CDs7+xe+71WrVmfkyCH4+HTCysqKpKREUlJS9K+bm5vneP4i\n9vZlePYs4aWB86sICztHaOhZli5diUwmo3//PqSnqzA0/PvjiIGBRB9ECoIgCP9OBHOCIAgfsbw+\nPKvVGQQHb+TPP0+xY8dWDh06QEBAIP7+AwgPv8fKles5fz6UUaOGsmbN5hzrp9zcPPjmGx+6desJ\nwKRJgZw8eZxGjT5n27YQypVzIiLiAX36dOfx43iSk5OJjIxEpVIxffoUMjM1mJtb0KePHzKZjOrV\na3Hx4nmio6OwsJBTrpxuWmHJkrbExsYQFhZKXFwcZcs6kpDwjIkTf2TevFnIZDKcnMrz88/Lcl2z\nn1/vHM/bt/82V2KOEiVKEhy8EcgKdo8AEoyNjfH2bk5ExAOcnJwBaNSoCZGREQDExz8kMHAkT58+\nISMjg5IlbYGcUxYlEgn16jUAwNHRCSMjI3r16oqpqRlKpZLz50NJS0v7x2iaLhhs1aotU6dOwNe3\nHaVLl6FChUoAFCpUiNGjxzN+/CjS0zMA6N37+38N5sqUcaBXL38GD+5LZqYWqVTKoEHD9W1ZWVnh\n6upOly7tqVmzDrVq1SGv2bBGRkZMmjT9hYHz60hJeY5cLkcmk3H/fjhXr1751/1dXd05eHA/X3zh\nzf79+167PUEQhI+dCOYEQRA+Ytk/PD94cJ+rV6/w7FkCGo2aBg0aY2dnz6RJgUDOgESr1eLm5pZr\n/ZSbmwdhYWdZv34NKlUaSUlJlC3rSJ069QAwNpZx48Z1tm37lUGD+nLnzm1SU1NIT0/H1dWNo0cP\nUaxYccaPn8KiRfO5fv0qJUuWRCKR5Ehrb2BgQEZGBosXz8fKyooVK9Zw8OB+tmzZmK/358cfV7Jr\n137S0nrx+eepyOX7KF26DA8e3M92L/7ef86cGXTs2Jk6depx/nwoQUG5g0kAqVSqf2xoaMSmTTsA\nePLkMadOnWDbts1/jXhWACSEhPwPAJlMxoQJU/M8Z5Uq1Vi+PHeikZCQnS+8viZNvHKtp8tqC2Dc\nuMk5XvP0rKp/rAv8dF4UOC9YsPSFbeelRo3a7NixFV/fdtjZlcbFxRV48TrHH34YyoQJY1i3Lpi6\ndRv857WXgiAIHxsRzAmCIHzE8vrw/OjRI/r3/w6tNhOA777rn+exxsa510+pVCp++mkGK1asoWjR\nYvpU+VksLQuRlpaGRqP+a/80IiIekJ6um5JpZGREePg9VKo0HBwc2bVrBwMGDCEqKipH21otJCY+\nIyLiPhkZGXTs2BpjYxkpKamYmprky72JjIxm7VoJRkZliYlpx7p1YTg6TqdFi9ZcuBBGcnIypqam\nHD16SD9imJLyXB/g/tvas7zExcVRtGhRWrRoRXq6itu3b9K0aTOMjIxQq9UYGb36v+RLl25z8eI9\n6tVzoUwZ29fqx3+lUqmYM+cgjx4ZUq9eIVq1qvHKx0qlUmbNmp9r+/79R/WPGzZsQsOGTQDdCGr2\ntXq9evn/h54LgiB8fEQwJwiC8BF70YfnrIyJ2ZmZmb10DVVW4KZQWJKSksLhw7/TuLGX/vhy5ZyQ\nyWR06NAaCws59vZluHXrBhqNBnv70kilxvo1bEqlEjs7O/16tH+OukgkEhwcHBkwYDDTp0/BwEBC\n3br1uXHj2hvdi3+6dSuK+Pi22NpeonTpr8jIcMDS0p5ixYrRuXN3evXqikKhoHTpMpibWwC6KZxj\nx45ALldQtWo14uJi9X190ehS1uPz58+xYcMajIyMMDMzZ8yYCQC0bNmabt064uxcgbFjJ7203ytX\nHmXq1BIkJraiZMkjzJnzmEaN3PPlnryK77/fwa5dXQBjtm69SVraSTp0qJOvbSQlJbFlyx/I5ca0\nadPgpYXQBUEQPlUimBMEQfhEbN9+mjVrEtFqoVMnOe3a1c7xuqVlIf0aKplMho1N8VznkMvltGjR\nii5d2lO4sDWVKrnoX/vqqxbMnTsTqdQIQ0MjhgwZSdmyjvTo0Zm6devra8xlrWE7fPh3/vjjJAAW\nFhZ06OCrP1e9eg2oU6c+vr7tSE1NIzh4A2q1mkWL5lOxYqV8uR+ffVYBR8fT3L27HACF4jIDBybi\n4eGOs3NFWrZsjVqtZvToYdSv3xCAunUbULdug1zn8vZujrd3cwBGjRqX47WsUafs+2Tn798ff/+8\nR0fzsmpVKomJuumQMTGf88svm99ZMKdSqTh92hbQjdqmpDhz8OAVOuT+buCNPXnylPbtj3Dpki+g\nZN++TSxf3v6FAV3W9GAxBVMQhE+RCOYEQRDekL+/H4sXB/H48SPmzp3F5MnTC7pLL3Tp0m1GjbLk\nyRPdKNrVq6E4OFynWrWKOfbLvoaqaFE5jx4lAznXT/Xq5Z/ndLcGDRrToEFjQkPPMnToAFxcXJHJ\nTJDJZPri2/82evXPz+KGhoa0adONiRPHkZycgFarxd6+DPPnL37Du5CTQmHJ0qVlmD9/I+npUlq0\nsKBxY12AGxS0jHPn/iQ9PZ3q1WtRr17DfGkzy5495zh58jG2tgZ8993nrzXypFYb5niu0by7USup\nVIpcruTRo6wtWszN0/K1jRUr/uTSpS7oErVYsmvXl8yYMZNr18IAXTmL+vUbMmhQXypXduXmzevM\nmjWf4sVt8rUfgiAIHwKJNq+qoQUg6wOD8PHJ/oFQ+PiI9/fDsHTpXsaO9cmxbdy4zfTt6/3CY17l\nvX306AlLlpxGozHg228r4+T04uyKr0Or1TJgwBZCQpqQmSmnVq2trF/fAnNz83w5f0Fat+44Y8aU\n5fnzCkASnTptZe7cV6/tN2PGXhYsqIFKVZpChcKYNu0RbdrUfvmB//Cmv7ubN//B1Kkq4uNL4+ER\nxi+/1KNkyWKvfZ4XmT59L7NntyMr66ZMdpAaNaawbt0mfTmLwMBJ9OjRmSVLgnKMDgs64u/yx028\nvx+vokXlr32MmIQuCILwhry8dBkcY2Nj6NKl/Ttt+3Xb9PCwQ6G4pH9uYXENd/eS/6kPSqWSTp2O\nsGBBexYt8qFz59uEh0f/p3Nm2bBhL5s2NSQzszRQmD/+6M7y5cfy5dwF7bffUv8K5AAUHDtmRWZm\n5isfP3y4N0uW3CIgYAvBwelvFMj9Fz4+tThxoip//CFh586v8zWQA+jWrTqVK68HtEAKVapspmlT\nb2QyE0xNTWnQoDEXL56nePESIpATBOGTJ6ZZCoIgvLEPZ41OjRoujBlzlHXrbqHVSujQQUrduo3+\n0zn37j3DxYvtyboP9+61Yvv2EAYP/m/ZFR8/fsKPP0YATbJtNSI9/cO53//G1DQjx3Mzs/TXTvDR\nrFlNmjXLz169HgsLORYWr/8N8qsoXtyarVvrsWlTCBYWRhgbe6JUKnPtl19ZTQVBED5kYmROEATh\nA6XRaJg4cSy+vu0YM2YEKlUaN25cp1+/3vTo0ZnBg/vz5MljAK5fv8rhw4spVSqY1q2vc+zYEkA3\nwte3by/8/Hzx8/PlyhXd6F1Y2Dk6d+7MmDEj+PbbtkycODZX+0WLyjE0fJRtSypy+X//t3L06CXi\n4noDuwA1AIULr6RDh3eXsfFtGjy4MpUqbQLuU6zYAfr3L1TQXXrvFC5shb+/N507e+HpWZVjx46g\nUqWRmprKsWOH9WswBUEQPnViZE4QBOEDFRHxgICAQFxc3Jg2bSJbt27m+PEjTJv2E4UKFeLgwf0s\nW7aIgIBApk6dwMiRgVSu7MKSJT/rk48ULlyYOXMWYmxsTGRkBBMmjOGXX3SFqa9fv86aNZuxti6C\nv38PLl26gJubh779Bg2q4eu7nQ0bPFCrTfjiiyN07+6TV1dfi5OTLebm4Tx/3gH4FXhO376G2NuX\n+M/nfh84O5dhz55i3LhxF3v7chQpUqSgu/ReK1++gr6cBUCLFq2RyxUie6UgCAIimBMEQfhgFStW\nHBcXNwC+/PIrgoODuHfvLoMGfQ9AZmYm1tZFUSqVpKamUrmybn2Rl1dTTp06DkBGhpo5c6Zz585t\nDAwMiIqK1J/fzc1NXyC7XLnyxMXF5gjmJBIJM2d+Q58+d1GpkqhYsUO+1ANzcyvPoEEHWLUqnIwM\nKd7eGfTr1/o/n/d9YmZmRpUqrgXdjQ9GVjmL7IKDNxZQbwRBEN4fIpgTBEF4Qy9Ks18Q7Wu1WszN\nzXFwcGTJkqAc+yUn58x6lj2J8aZN67C2LsLYsZPQaDT61PwAxsbG+seGhgZoNJo8+1GunON/uo68\nDBjgRd++GjQaTY5+CJ+uGzfuM2PGFZRKYxo0kNC3r1dBd0kQBKHAiTVzgiAIb8jRURfElChRskBG\nCR4+jOPKlcsAHDiwj8qVXXj2LEG/Ta1WEx5+D7lcjpmZGdeuXQHg4MH9+kAwJeU5hQtbA7Bv36+v\nlVXxbTM0NBSBnABAeno6ffteZvfujhw50oZp0z5j3brjBd0tQRCEAieCOUEQhNdw4sRVOnTYQ6tW\n+/Hw+PblB7wlEokEe/vSbN++GV/fdiiVStq27cCkSdNZsmQB3bp1onv3Tly9qktoMnLkWKZPn0L3\n7p1IS0vDzExXr61163bs3fsr3bp1IiLiAaamZgV2Te+T9etXs2WLLkCfP382P/ygK5IeGnqWiRPH\ncvbsab77zg8/P1/Gjh1JampqQXb3vfcqXxIolUq2b98C6BLwDB8+SP9adHQU16//PS01Pd2OsLCU\n/O+oIAjCB0ZMsxQEQXhFCQlPGTToEQ8e6Oq7xce7U6KEBS1b1njnfbGxKcG6dVtybXdyKs/PPy/L\ntd3BwZHg4A0ArFmziooVKwFQqpSdfjuAv39/AKpUqcaXXzbSF6YdNGh4vl/D+8zdvQobN66lbdsO\n3LhxnQcP7nPw4H4ePLiPo2M5goODmDt3ESYmJqxdu4pNm9bRrVvPgu52gQkIGEp8/EPS01W0a9eR\nli1b4+VVj6+/bsO5c2cYPHg4sbExbNmyCbU6g0qVXBgyZGSONZbJyUls3x5C69a5C6gXK1YcW9vT\nPHiQFdA9p1Qpba79BEEQPjUimBMEQXhFFy/e4cGD6tm2GBAWlkDLlgXWpVd26tQJ1q5diUajwcam\nJKNHj8tzv4iIOMaN+5OHD82pUkVFYKDXJznV0dm5AjdvXicl5TnGxsZYWVkRExPNpUsXqFu3Pvfv\n38Pf3w/QJZFxdXUr4B4XrICAQBQKBSpVGr16daVhw8akpaVRubIL/foN5P79cNatC2bJkiAMDQ2Z\nNetH9u/fS9OmfxfLW7JkAdHRUXTv3gkjIyNMTEwZM2YE4eF3cXauyMSJrfnpp40olSeQy68QFmbG\njBmhDB8+GoB+/XpTubIrYWHnUCqTGTkyEHd3jxd1WRAE4aMggjlBeAdiY2MYMWIQq1dveqX9z58P\nRSqV6jMVCu+HihXLUKzYZeLji/+1RUv58h/GtMQmTbxo0uTlCSMGDTrF8eO6FPDnzqWj1W5hypQW\nb7t7BWLjxrXs2bMLgObNW1G/fkOGDOmPm5snV65cRKlMZufO7bi6unPhQhh3797h3r27pKSkUK1a\nDcaPn8LZs6fZvn0rI0aMKeCrKVghIRs4fvwoAPHx8URGRmJgYEDDhrrC76GhZ7h58wY9e3YGQKVS\nYW1tneMc/v4DCA+/x8qV6zl/PpSAgCGsXRuiL41ha2vAgQPNSEqqh0KhAGDSpEBOnjxOnTr1kEgk\nZGZmsnx5MH/8cZKVK5cxd+6id3gXBEEQ3j0RzAnCeygs7BxmZuYimHvPFC9ejKlTH7Bw4SZUKmO0\nWg0dO9Yr6G7lG61WS3i4ZbYtxty7Jyuw/rxNN25cZ+/e3SxfHkxmppbevbvi6VmFqKhIJkyYxogR\no+nc2Yc1a1YyceKPREQ84OzZP/Hw8OTu3Ts8ehRPdHQUv/66iy++aEpkZAR2dvYFfVkFIizsHKGh\nZ1m6dCUymYz+/fuQnq7C2FiWI+Oqt3dz+vTp+8LzZM+yqtVqqVixcp6lMcLCzrJ+/RpUqjSSkpIo\nW9aROnV0v4cNGjQCdCOrcXGxb+NyBUEQ3isimBOEd0Sj0TBx4lhu3bpBmTJlGTNmAr6+7QgKWotC\nYcmNG9dYuHAeo0ePZ+fObRgYGLJ//x4GDhwupgq9R1q2/Ew/rdLLa+JHVbhYl1QliaiorC1q7Ow+\nzsQely5doH79RshkJgA0aNCYixfPU6KELeXKOQHg4uLGr7/uxMXFld9+24NUaoS7uyfOzhV59Cie\nsWNHcu/eHcLD79K7d98PMpjbu3c3GzeuQyKR4OhYjsaNvQgOXoFanYFCYcm4cZOxsirMihVLefgw\njtjYGB4+jMPHpyNt23YAdBlR5XI5MpmM+/fDuXr1Sq52qlatzsiRQ/Dx6YSVlRVJSYmkpKRiY2Pz\nwr5JpblLY6hUKn76aQYrVqyhaNFiBAUtIz09PdcxBgaGLyylIQiC8DERwZwgvCMREQ8ICAjExcWN\nadMmsm1bSJ6BgI1NCb7+ug1mZmZ06OBbAD0VXtXHFMhlmTmzGoGB63j40AxPz1QmTPiioLv0Vrzo\nvTM2luof29uXoVu3nvqAb+DAYTRs2ITY2BiGDx9IixatefLksT5pzIfm9u3brF4dxNKlK1EoLElK\nSkIikbBs2SoAdu3awbp1q+nXbyAAkZERLFiwlOfPlXTq1IbWrdthaGhIjRq12bFjK76+7bCzK42L\niy5JSfZ7XKaMA716+TN4cF8yM7UYGRkxZMiIHMGcmZkZKSn/nqEyK3BTKCxJSUnh8OHfadxY1JsT\nBOHTJYI5QXhHihUrrp82+eWXXxESsuFf99eKRG3vvf37jxZ0F/Kdk5MdGzbYAVC0qFyfzfJj4+7u\nwZQpE/D17UpmppZjxw4zduxEdu7c/q/HXb16j/79r5GQYMW9e8sZMGDEO+px/jt9+jSNG3uhUOim\n1ioUCu7evUNg4EiePn1CRkYGJUvaArrArHbtuhgZGWFpWQgrq8IkJDylSJGiSKVSZs2an+v8//z9\neNm6TUvLQri6utOlS3tkMpm+/mF2crmcFi1a0aVLewoXtqZSJZd/ucK392XL666DfpEVK5bi7u5J\ntWrVX76zIAhCHkQwJwjvSPZvqbVaLRKJAYaGhmRm6qI2lSr9RYcKBUypVDJ//lFSUw35+msHqlVz\nfittHDiwL8+07EL+K1++Al991ZxevXTJXlq0aI1crsg1Ypf9uUQiYebMq1y50gm5XIGBwRpWrsyg\nfft32vV8I5FIcqxTA5gzZwYdO3amTp16nD8fSlDQ32UujIz+HrU0MDBArX61aYz79oUxb14cqanG\nNGyYyrhxzV84Mjpu3OQ8t2cvjdGrlz+9evnn2mfMmAlkBXCFChUiJOR/r9S/gtSjR5+C7oIgCB84\nUTRcEN6Rhw/juHLlMgAHDuzDzc0dG5sS3LhxDYCjRw/q99VNN3peIP0UcsrIyMDXdzdz57Zj6dJ2\n+Pklce7czXxvJ6vG1vvg34o3vw2xsTF06fLuI6L27b9l9epNrF69iXbtOmBjU4Lg4I361zt29KV7\n914AjBo1jgYNGpOcrJtyaWoaSmJiO/3zD1HNmjU5fPh3kpISAf5ax/Zcn3Rk797d+n3/GfS9qmfP\nEhg1SkloaHuuXWvNkiVerFp15D/3PTutVsvAgVuoUeMxNWvGM2zYtjfu7+vIzMxk+vQpdO7sw+DB\n/VCpVOzcuZ1evbrQrVsnxowZjkqVhlKppG3bvzPCpqam8s03zVCr1UyZMp4jR3R/+9u2bcGKFUvx\n8/Ola9cORETcByAhIYGBA7+nc2cfpk+fTNu2LfTvmSAIggjmBOEd0CWWKM327Zvx9W2HUqmkdet2\ndO/em3nzZtGzZxcMDY3031bXqVOfY8eO0L17Jy5dulDAvf+0Xb16i1OnGgOGAMTFNWbnzvB8byd7\nja1Fi3JPWXuX3qfA8m04dOh3fH3b8cMP/pw/H8qVK5de6bgDBy4QHn4de/vmmJkdx8pqGUWLrn3L\nvX17ypUrR5cufvTr15tu3Trx889z8fPrzdixI+jRozOFChXS/02ShAqJvAAAIABJREFUSCS8yRLR\nu3cjiYr6eypkZmZR7txR5dclALBt23E2bmxJSkotnj+vzdq1Tdm9+1S+tpGXyMgI2rTxYc2azVhY\nyDl69BANGzZm+fLVrFq1ntKlHdi9+39YWFjg5FSesLBzAJw6dZwaNWpjZGT01339+x4XKmRFUNBa\nWrVqy4YNup+tlSuXUa1addas2UzDhk14+DDurV+bIAgfDjHNUhDeARubEqxbtyXXdnd3DzZs2AZA\nQsJT7tyJJDk5CTs7e4KD/31NnfBuWFnJMTN7QkqK419bNJiZ5X+WvOw1tgpaVmDZqlUrQJKreHNg\n4CQAzp07w6JF89BoNFSoUImhQwOQSqW0bdsiV5bWBQuWkpCQwIQJo3ny5DEuLm6cPfsnQUG6D6xZ\noxxXrlykaNFiTJs2G5ns7ZRF2L37f4wYMQZXV3dWrFj6SmVAEhOfMXJkIlFRo4H9lC49gXLlPmft\n2pePWqrVaoyM3s9/t97ezfH2bp5jW926DXI812g0tGnjo19bB7zyWrHy5cvg6HiWu3dLAyCTReDu\nLv+Pvc4pPv45mZmFs/W3GHFxb3etZ3z8QwwMDPSZT52dKxAbG8Pdu3dYvnwxz58rSUlJpUaNWoBu\nWurGjeuoUqUav/++nzZtfPI8b4MGjQHdNOCjRw8BcPnyRaZNmw1AjRq1kMsVb/XaBEH4sLyf/10E\n4ROzZ08oo0alEBPjgqPjWebNs6V69QoF3S0BKF3anj599rJ0aSapqdbUrXuE/v3zv4j2u5gW9qqy\nAssdO3awf/+RXMWbL1++SPnyFZg6dQLz5y+hVCk7Jk8ex/btW/Dx6fjC9VBZIwy+vt34888/2L37\n7zVNkZERjB8/lREjRhMYGMDRo4f44gvv/3wtAQFDiY9/SHq6inbtOvL06ZO/PhxPxNHRiUuXzuvL\ngAwaNBw7u9LMnj1NP/oxYMAQXF3d+fnnBWRkKLGzW4eh4WOMjBJ5+vQwISE2XLx4npiYGExMTBg+\nfDSOjuVYsWIpMTFRxMTEYGNT4oVrwd53e/eGMWlSHI8eFcPF5R7Ll39OkSKFX37gX+RyBXPnlmTe\nvE2kpEhp0kSLj0/+Zkht0aIKwcH/4969VgCUK7edFi2q5msbefv751xXCkHF1KkT+fHH2Tg6lmPv\n3t2cPx8KwOjRE+jatQNJSUncunWDqlU/y/OMWdlUs0oxZHmf/j4IgvB+EcGcILwH5s+PIyZGV7Pp\n7l175s7dyPr1Iph7XwQEeNOlSzTPnj3D2bntezvKkl9eVrw5NjYGExNTSpa0pVQpXeZLb+/mbNu2\nGR+fji8877+NMGSv75Y1ypEfAgICUSgUqFRp9OrVlZ9/XkZo6Fn69RvEiRNHSUl5zmef1aBDB1+W\nLl3I3LmzsLcvjUqlQq3WEBg4ku3b95KRkYZcfpI7d86i1Rrj6OhJ1ap1iY2Nwdm5ItOmzSYs7ByT\nJwfqR1cfPHjAokW/YGxs/JJevp+0Wi3TpsVy547ub9PJkw2ZMmU9c+a0fK3z1KhRkfXrK76NLgJQ\nqlRxgoNTCQrajESSSY8ertjYFH1r7f1Nq68damhoSN269VEqk5g5cwrp6RnExcXqs1TOmTMDa2tr\n5s2bSXJyEkFByzh58jixsdGUL69LqKTRZDJq1DASE59RqpQdV69eJikpEVdXdw4dOsC333blzJnT\nJCcnvYNrEwThQyHWzAnCeyA11fhfnwsFz9bWlsqVK7y1QO5VamwVlLyKN/9z9E2XoVXy1z4vztL6\nohGG7PXd8rPgc0jIBrp160SfPn7Ex8cTGRmpf61Zs5bcuXMbrVY3zfPQoQNERUVy6tRxDAwMMDQ0\n4MmTJ0RHRyGVSilatBANG26nZs1tyGQaqlZ15PLli3z55VcAVKlSjcREXRIRiURC3br1P9hADnTJ\nf54+zT4lUkJiommB9effODuXYfp0b378sRlOTu+meHtGRgbffNOOtWtDMDY25tq1q8jlCh49eoSh\noSEVK1bi1q0bgG49nKurBwcO/Iapqal+bZy9fRlOnDgGgFKZhIdHFdas2UzVqtX1NfW6d+/NmTN/\n0qVLew4fPkjhwtaYmZm/k2sUBOH9J4I5QXgPNGyYioHBYwBksvt4eX18xaiFf5e9xlZBJ0B5lcDS\n3r40sbExREdHAfDbb3vw8KgC8MIsrVkjDMA7GWEICztHaOhZli5dyapV63FyKk96+t/JN2xsSiCT\nyXj0KJ4zZ07j5OSMRqOmf//BrFq1gTVrNuPl1ZTw8HsAWFiYsmnTV+zc6YWx8d9B/YsC1Kxi4x8q\nY2NjqlSJA3SBtVQaRa1aH/eo9KsqVqw4xYvb6Nda+vsPIDNTS2LiMxQKBWp1Bo8fP6JkyVL6Y1xc\nXDl27AzGxjL92rgBA4boX7e1LUXz5l8D0LZte/0aRQsLC376aQGrV2+iWbMWWFtbf/SzAwRBeHXi\nr4EgvAfGj29BmTJHuHtXhaengjZtPi/oLgkF4H1ZV5UVWLZo0QJDQ6M8izcbGxszatQ4xo4dgUaj\noWLFyrRqpauR1717b378cSK//GKBp2dV/Yhd9+69GT9+NL/9tofKld30IwzPnz//1/pubyol5Tly\nuRyZTMaDB/e5evVKrn1cXd25fPkiT548olmzlty7d4czZ07TooVu/VVychISiQQDA4Nc008B3Nw8\n2b9/L9269SQs7ByFCllhZmb+0axxWry4GVOnbuLJExnVqxvj59eooLtUoPz9/Vi8OAjIXTvU3Nwc\nBwdHliwJeul5XmVtXHq6mhYtDpCWpqFQodWUKKFAKpUyfPiY/LocQRA+AiKYE4T3gEQioXv3T/tD\n0qdoxYojnDiRjkKRyujRdSlWLHfQVFDGjZtM0aJyHj3KmRUwe/HmqlU/IyhoXa5js2dpzS5rhMHQ\n0JArVy5x8+Y1jIyMKFGiZK76bvmhRo3a7NixFV/fdtjZlcbFxTXXPr6+3fDz+5Y7d27xzTc+dO7c\nncWLF9ClSwcyMnSjKwEBgYSGnuXp06dkZGSQlpZGeroKAwMJfn69mTZtIl27dsTU1JQxY8YDb57K\n/31jbm7OlCn5n/DnQ5UVyMHftUNdXFw5cGAflSu7sGvXDv02tVpNZGQEDg5lX+nc2dfGHTy4n9TU\nFO7ebU1mZiGgHa1aHcTf3+stXZkgCB8qEcwJgiAUgNWrjzJ+vBsqVWlAy/37QezY0S5fRqTeVw8f\nxhEYOJLMTC1SqRHDh49h4cIDHDmixcwsnSFDKuDmVi7f2pNKpcyalXvK6oIFS/WPHRzK0qxZS+Ry\nBe7unri7exIefpfTp09hbCwlICAQK6vCDBgwGCMjIzp3bk/JkiWpV68BJiamKBQKpk2blasNP7/e\n+XYdwvvDy6seBw4cJyEhAWNjY4YNG0BaWhouLm4MGjSc6tVrMW/eLJRKJRqNmvbtO70kmJPkOXJd\nokQpNBorMjOz1sZZEhWV/yVRBEH48Em078lckH9++yt8PPL6dl/4eIj3983067efzZvb6J/L5Sc5\nc8YWa+v3Z3Qu672NjY1hxIhBr1xb7FVt3HiCoUMrk56uS1hRocImfvutEaam7y7JRmZmJj16+DJ5\n8gxsbUvleO3WrQjmz79MWpoRDRtqqVu3Avb29hgY5L3cPCkpifHjD/PokSmurmqGDm36wn3fhdTU\nVAIDR/Lo0SMyMzV07doTS0tLfW1ADw93+vUbilQqffnJBAC8vOpz4MAxNmxYS0ZGOl26+KHVaklN\nTcXMzOyNzxsdHUto6C2qVStPyZIlOHv2TwYNmsytW0cAMDG5w/z592nVqsYrnU/8Xf64iff341W0\n6OvX4RQjc4IgCAXA2jodUJP1Z7hIkVgUireXvv1VZBXQTkl5jru7J97eTXK8HhZ2jo0b1zFjxpx8\nae/8+ef6QA7g5k0PIiOjKF/e6bXP9SaFucPD7zFixCAaNGicK5BTKpX07HmFGzc6AP9j5045RkYa\nGjXaSFBQmzwLmn///W/s398NMOC33xLQavcxYsRXr30t+eXPP09RpEgxZs6cB+iuqUuX9vragLNm\nTdbXBhReT6VKlZk2bSJqtZp69Rri5FT+jc/1v/+dYfRoCQkJpbG398fWVoKVlZyAgAFs3bqBtDRj\nvLyMadWqYf5dgCAIHw2RzVIQBKEAjBzZBG/vYIoV+xVn5w0EBhYp8BGSrOlePXr00dfHypKZmcmG\nDWs5fz6UwYP7oVKpuH37Jr17d6Nr146MGjWM5ORkEhKe0qNHZwBu375FvXqfER//EAAfn69RqVQk\nJCQwZsxwrl9fjL19a0xMwoBMHB27olD8nXK9Q4fWJCQk6Pfv1asLvXp14fLli4Au+Jw0aSz+/j2Y\nMmX8a1+vg0NZNm/+H337/pDrtbCw69y40QAIB2yBxqjVHhw40J2FCw/l2l+r1XLtWmH+/rdqxaVL\nBft+Ojo6ce7cnyxevICLFy8QGxuTozZgq1atuHgxrED7+KFyd/dk4cLlFC1ajKlTx7Nv369vfK6l\nS58QH9+YjAxX7t49iETSm+XLV9OsmRdBQc1Zv/4LundvmH+dFwThoyJG5gRBEAqAqakpwcE+pKen\nI5VKC2ytXHDwCvbt+xUrq8IUK1YcZ+eKTJ06gdq169KuXStOnz7FnDkziI6OwsnJGU/PqpiYmHD0\n6CHWrVvN4MHDcXf3ZMWKpaxcuYwBA4aQnq4iJeU5ly6dp0KFSly4cB43N3cKF7ZGJpMxbdpEfHw6\nMXGiG4MGrebcuR8wN+9L5cpVCAsL5auvSnL16hVKlCiJlZUV48ePxsenE25uHsTFxTF0aH/Wrg0B\n3l5hbgeHElha3vmrrlqJbK8Yk5yc+72SSCQUK/acqKisLVqKFHmer316XXZ29gQFreOPP06wfPki\nqlb9rED78zGJi4ujaNGitGjRivT0dG7fvknTps3e6Fzp6TmDfpVKfDQTBOHVib8YgiAIBaggi0rf\nuHGdQ4cOsGrVBjQaNX5+viQmPiM5OZk6deqhUqmYMWMKY8dOZPr0KX8VCwdn5wpER0ehVCbj7u4J\nQNOmzRg7diQALi7uXLp0kYsXL9C5c3f+/PMUoNXve+7cGR48CNf3o3hxCevXN+HOHTtWrvyFr75q\nwcGDv9GkiVee+6ekpJCamvpWC3Pb2ZVixIi7LFkSRWzsCTIyBgASihc/SrNmDnkeM2GCM2PHruPh\nQ3MqVkxg3Lgmee73rjx+/Bi5XM4XX3hjbm7Btm0hxMXFEh0dha1tKf73v//h6Vm1QPv4ocn60uX8\n+XNs2LAGIyMjzMzMGTNmwhuf09tbzc2bEahU9pia3qF5c8P86q4gCJ8AEcwJgiB8oi5dOk/9+o3+\nWv8lo06d+ty7dxfQTRu8d+8eJUvaYmNTAmNjKV984c3OndsxMDBEqXzx4nsPD08uXjzPw4dx1KvX\ngLVrVyGRSKhdu95fe2hZtiw417TS3bv/x/3793j27BnHjx+jW7deufbv1683I0cGYmpqyubN62nf\nPn/KGOSlZ88G+PllEh//mMWLN5KWJqVVK3uqVXPOc/8aNZzZv9+ZzMzMAk18kuXevTssXDgPAwMJ\nRkZShg4NQKlM1tcG9PT00NcGFF7N/v1HAfD2bo63d/N8OeeQIU1xcDjF9et/4ulpzVdfNc6X8wqC\n8GkQwZwgCMInSzfKkDXVMjU1FSurwhgaGhIbG8Pq1SuIiIhk5sypfxU2/jv5sbm5BQqFgosXL+Du\n7sG+fb/qR3nc3T1ZunShvmC4QqHgjz9O8t13/QH47LOahIRspFOnrLV1N3FycmbkyLEsWjSPBQtm\n4+DggEKhyLW/RCIhIuIBzs4V9P3PT1lJYLJq3RkYGGBjU4wJE/L+4P748SPmzp3F5MnT9dveh0AO\noHr1mlSvXjPX9qzagCIj3uvJzMwkMHAX586ZUahQGqNGVcLNzTFfzv3NN7Xz5TyCIHx63o//OIIg\nCMI75+HhyYED+/j9999YtOgXZDIZ8fFxAGzZspmGDRuiVqu5fv0asbEx7N+/j9u3b7Jhwxq2b99C\ntWo1WLRoHp9/Xo/9+/dy7tyfdOnSnoSEpwBUquTC1KkTuHfvDs+eJXDhQigA/fsPYvfuHTRuXJtG\njWozZ85MAPr1642jY3n279+HSqWiZ88udO7sQ9GiRbl58xpdu3bk+vWrHD2aPQGJlhUrlrJ58wb9\nlqVLFxISspE38TprF9VqNUWKFM0RyL2vjh27Qo8eO2jXbgJ//HEdgIcPHzJmzIgC7tn7Yf361WzZ\novuZmT9/Nj/84A9AaOhZJk4cy6xZP9KyZWt++20dDx5Ec+hQJ4YMucKiRfPx9fWha9eOLFw4ryAv\nQRCET5QYmRMEQfhElS9fATs7e65evcLo0cOpXNmVhw8fkpKi5PlzJbdv32batFnMmTOT58+VxMXF\nkpGRzu7dvwPw/LkSc3ML+vfvg52dPcOHj+bixfNMmzaRbdt+ZenShVSrVp1Ro8aRnJxM795dqVat\nBkePHsbR0Ym1a0MwMDAgKSkJ0AVSZco4cPz4WZKSklAoFGg0GgYO/J6BA4fh6FiO/v374OZWk6NH\nz2JhYUGbNj6kpKQwatQwfHw6kpmZyaFDB1i+fPUr34eskUmFwhKNRkOTJl/QrVsnzMzMWLToF549\ne0avXl0ICdnJnj27OHr0EGlpaWRmZjJ69HiGDfuBNWs2s2fPLk6cOIZKpSI6Oor69Rvy/fcDANi9\newfr1q3GwkJOuXJOGBsbM2jQ8Px/U9GVIDhwYB+tW7clLOwcv/yynJMne/HoUSNsbTfTt+8ztmyJ\nokaNih9EIPouuLtXYePGtbRt24EbN66jVqtRq9VcvHgeD48qNGzYhHv3qnDmTCtKleqGsfFN7t+3\n5ujRlWzatAPQ/T4IgiC8ayKYEwRB+IR99llNKlSoRI8efQBYsGAOFhYWbN68nqtXrxIVFY2xsRQj\nIyMqV3YlMfEZc+fOpFatujmm8H3++ZeAborl8+fPUSqVnDlzmpMnj7FhwxoAMjIyePgwjtDQM7Rq\n1VY/HTFrOmV2hw7tZ+fOHWg0Gp48ecz9++GUKePAjRvx7NhRCpWqIpUqPUetVmNjUwJLS0tu377J\nkydPKF++Qp7nzEv2JDDR0ZH4+XWmSZMv/no171G627dvERy8EblcTmxsTI7RvDt3brFq1XokEgM6\nd/ahXbsOSCQSgoODCApah6mpKT/84J+jLtnLintXqFCJoUMDkEqltG3bAi+vppw+fRIDA0OGDx/N\nkiULiImJpmPHzrRq1Ybk5CRWrlzOnj07SUxMJDk5jdjYRtjYDEIqjUCrXcSMGUWYP38CPXv2YvXq\nTezZs4vjx4+QlpZGVFQkHTp8i0qVzu+/70MqNWbmzHkoFAqio6P46acZPHuWgImJCSNGjMbevgyH\nDv3OqlXLMTAwxMLCgp9/XvZK9/994excgZs3r5OS8hxjY2MqVKjIjRvXuXTpAgMHDuPQof08eLCa\n0qVXYmj4BGPju9jZPcHU1JRp0yZSu3Y96tSp9/KGBEEQ8pkI5gRBED5hHh6eTJkyAV/fbmg0ak6e\nPM7XX3+DTGaCoWEJYmM7YGCwlwYNKvPDD0Po06cvf/55ih07tnLo0AECAgLzPG9WfDNlykzs7Oxz\nva7VanNtyxITE83Gjev45Zc1WFhYMHXqBNLTVWzbdownT4oBxYESpKZaEBJygj59vqZ581b8+usu\nEhKe0KxZy1e+/uxJYIKDV6DVZrJp01pSUlKwsyvNmDEjuHPnFgkJCfpjypd3JiBgCKmpqZiYmPy1\nnhDWrFmFiYkJAwb44+X1JUWKFGXYsIGkpKSQkZFBeroKuVxOo0ZNiIyM0J/vZcW9J08epy/uLZFI\nKF7chpUr17NgwU9MnTqeJUtWolKp6NKlPa1atWHKlPEkJj7D2toahcKShIRn2Nn5YGiYgFYrJT7+\nJ9q3j6Bjx46YmJgCEBsbw5kzp9m9+3fOnTvD6NHDKFKkKIUKFcLZuQL79v2Kj09HZsyYwrBhoyhV\nyo6rV68we/Z05s1bTHDwL/z000KKFCnyQY5QGRkZUaKELXv27MLV1R1Hx3KEhZ0lOjoKmUzGxo3r\nWL9+DdOmHeaPP/ZSseIRpkzpQOXKqzl37gxHjhxk27bNzJu3uKAvRRCET4xYMycIgvAJK1++Ak2a\neNGtW0eGDv2BSpUqI5FAmTJNiI6+zpMnvxAdLePkSTvi4mLRaNQ0aNCYXr2+4/btm4AuMDt06AAA\nFy9ewMJCjrm5BdWr19SvQwK4desGANWq1eB//9umD4Kypllmef78OSYmppibm/P06RNOnz711/YM\ncv7bkpCaqgagQYNG/PnnKW7cuE6NGrVe4w78Parm7z8ACwsL2rf3xc6uNBER9xk4cCjz5i1Go1Fz\n6dIFNBoN165dZcqUGaxYsYZGjT7n6dMnujNJdOf75ZfVtGnTngcPwunZsw/9+w+kVCk7li1b9Nf9\nytmDlxX39vZunqO4d926DQAoW7YclSu7YmpqSqFChZBKpSiVSuzs7PWjhcnJSaSlpeDhURy1uiMG\nBml07LiNRo2q5lofaGEhx9TUlN27d2BpWYhly4JZtGgFTk7OxMXFkJqayuXLlxg7dgTdu3di1qyp\nPHmiu3ZXV3emTBnHrl079O/rh8bd3YMNG9bi4VEFd3dPduzYSvnyzvqfR4VCwfDh9bC0vE+/fp44\nO5dCqUymVq069O8/mDt3bhX0JQiC8AkSI3OCIAgfqX9mZnyRLl386NLFL8e2gwf3ExdXg8KFl2Fs\nHEFi4k3u36/IsmWL0GozAfTZKSUSCcbGxvj5fYtGo9GP1nXr1pP582fTtWsHMjMzKVnSlunT59Ci\nRSsiIyPo2rUjRkZGtGzZmm++aadv28mpPOXLO9OpUxuKFbPBzc0dgDZtarFixWwMDIYDUqTSR1hY\nPGbRonl8//0PVK36GfHxD5k7dyaDBg3nt9/2sGXLJtTqDCpVcmHIkJEYGBjg5VWPdu06curUCbTa\nTNRqDb6+3UhJeU5KSgoAhQsXJi0tlSJFirJ583qkUilxcbE8ffqYpKREBg78HgCVSoVardb33c5O\nF4BFRNwnJSWFn3+eg1RqTGRkBAYGhqjVao4ePUS5ck7Zjvn34t5arTZH4GVsrCvpYGBgkKO8g4GB\nARqNGq0WChWyYuXK9YSFnWPNmpXMmTOVqKhIevUypUmT0nn+HBgY6NpwdXXnzJnT7Nu3my+//Aoj\nIyM0Gg1abSZyuZyVK9fnOnbo0ACuXbvCH3+cpEePzqxYsQaFwjLPdt5X7u6erFmzEhcXV2QyE2Qy\nGe7unpQr55Tnz2NKynNGjhxCeno6oKV//8EFewGCIHySRDAnCILwkXqdzIxZTp++xoYNEdy5cxOl\ncghK5VcAeHqupkaNWtSsmXcK9S+/bMaAAUNybJPJZAwbNirXvoaGhvTvP4j+/Qfl2L5gwVL941Gj\nxuU67v79cD77zAkbm6aAIenpZ7C1tSU4eAXffdefq1cvY25uweeff8n9++EcOnSAJUuCMDQ0ZNas\nH9m/fy9NmzYjLS0NFxc3evf+nkWL5nPr1g26deuIubkFMpkMiQQaN/ZizpyZ+Pl9S61adQHJXyNO\nEhQKS31AExsbw8iRWR/idfXcQDf6ZmZmxsiRgXh4VGHnzu2sX7+G77/vSenSZTAzM9df18uKe//2\n2x48PKrkuh95TVWVSCR4enpy4MBeUlNT9fslJCQgl8vRaNRoNGr9+5Ale0Dq69uNLVs2oVKp8Pfv\nwddffwOAmZk5JUuW5PDh32nU6HO0Wi13796hXDknoqOjqFTJhUqVXDh9+iTx8fEfXDBXtepnHD78\nh/75hg3b9I//+fOYnJxEYmKi/udLEAShoIhgTsg3/v5+LF4cBMDChfM4ffoktWrVZdy40QXcM0H4\ndGRlZrSyKkyxYsVxdq7I7ds3mTlzGiqVClvbUgQEBCKXy3Mls2jbtiuDBklJSrLA2vo0Dg6NMTZW\nULJkV8aOdX+j4DA/hYae4d69uyQk6AKp9PR0Hj1K59EjCY0bN6F+/Tpcv34ZV1d3tm7dxM2bN+jZ\nU1fLTqVSYW1tDYBUKqV27boAODtXJDk5iblzF5GY+IwePTrToYMvYWHn8PCowowZcwD0RdK//bYr\ne/fu5sqVy7i4uFK0aDHGj58KgLW1NR076tqzty+NpWUhjIykaLVaGjRojKurO3Z29owePYz69Rvq\nr+tlxb0rVqycrbj33++BRCL5x3uie1yzZh2MjWV89113UlNTSE5OJjU1hZIlbbG2LsKyZYsID7+H\njY0NMTExANy8eV1/fHR0FFKpMe3adSA8/B5Pnz7RtxMYOJlZs34kODgItVrN559/QblyTixaNI+o\nqEi0Wi3VqlXPMfL4sQkKOsqcOYY8e2bDZ59tYdUq71dOuCMIgpDfRDAn5JusQA5g167t7N17uMA/\n/AnCpyR7ZkaNRo2fny/OzhWZPHk8gwcPx93dkxUrlrJy5TIGDBiSK5nFmDETiYnZS+nSLYiKCkKj\nseLHH0Pw82vxr+1mH1F727y9m9OnT18Adu48w4ABpTAyuoKx8W2OHn1M69b18tw3O0PDv//1GRhI\n9Gu8LC0L4erqTpcu7ZHJZBQubK3fLyMjg9DQs3h7N+fbb7syfPgPFCtmg0ajpn37Tjg4lAX+Hg2V\nSqVMmjSdsWMDiYpSkpmZhExmSIkShalRozb16jXUn/tlxb2zCwn5X47r8/ZunudrNWvW5u7d21ha\nFsLBwZGSJW31bVWoUAlv7+Y8eHCTkSMD6NmzC56eVfWjcyEhGzA1NaF//+8oW9aRvn0HYmSku2cl\nSpRk9uz5ufo1ZcrMXNs+RkqlkjlzJDx86A3AiRPuzJy5iUmT8i4qLwiC8LaJYE7IN15e9Thw4Dgj\nRgwiNTUVP79v8fXtTocO3xR01wThk5A9MyPIqFOnPmlpqSjHQyKVAAAgAElEQVSVybi7ewLQtGkz\nxo4dmSOZRRa1+jkSyVNSU6tgYzOStLSq2NuXLaCrya1q1eqMHDkEH59OWFlZcfx4FOnpFUlL88Le\nfjHJyVaUK/dNnvsmJSWSkpKKjY3Nv7YxbtzkPLd37tydESN000JtbUvh4uKuH7XL8s+g1sbGhhs3\nviMmJisYTqV589388EPTN7j61/Oi68he265atWo5phJmGThw2Cu3o9VqOX48lMTEVLy8PsPExOT1\nO/sBUSqTSUwslm2LAUql9IX7C4IgvG0imBPyke4b6enT5+DlVT/PRfKCILxNrz4SnlcyC61Wy5Ah\nW9m9uz4SyR3q1LnI0qVbqF7d9b1Y/1SmjAO9evkzeHBfMjO1PH2agpGRM2lpNUlPL4ep6WVq166V\n575GRkYMGTICGxubHDMGXnX2wJIlC4iOjqJ7904YGRlhYmLKmDEjCA+/i7NzRQIDJwG60dGff57D\n06fPSExU8/BhIFJpBCVKDCQiYhtxcRIiIyMYN24UQUFr8/8m/Qdbt/7B8eNJWFllMHx4E0xNTf91\nf61Wy8CBW9m0qQmZmYWoXj2EDRu8kcvl76jH716xYsWpWfMYR45UAQxRKC7g5VWkoLslCMInTARz\ngiAIH4kX1YyTyxVcvHgBd3cP9u37FU/Pqi9MZvHTT23p0+c6pUp9TZkyfWjVqvV7lcyiSRMvmjTx\nAkCj0TB48HaOH3+IuXljBgxok2NqZPZ9s9u//6j+ccOGTWjYsMlL2/X3H0B4+D1WrlzP+fOhBAQM\nYe3aEKyti+Dv34NLly5QqZILc+fOpFu3AfTtqyQhIYMiRX7m4cNNZGZaIJcfoGZNBXv27HqtWnjv\nwqZNpxg+vAypqc6Amtu3V7J2bYd/PSYs7CqbN9clM1NXR/DMme4sWRLCsGFfvYMeFwwDAwOCgpox\ne/ZmkpKkeHkVpWnTqgXdLUEQPmEimBMEQfhIZK8ZZ2VVWF8zbvTo8cyaNY20tDRsbUvpM/P9n737\nDIji6ho4/l+WooA0ARHsiqCiYDeW2KKxPyZ2LIAtajRqSGLvsWM3KhZQsPfXxB5iN5ZY0BixYqHY\nkN7Z3fcDYZWIHUTw/L64M3tn7p0dBA537jkvS2bh5+dDSMh9lEodXFyqf7TJLJRKJQsWdEStVqOj\n8/qyqXfuhPPjj6e5d68QpUvHMHduPWxtrV97HGTOHKnRaKhQoRKWllYAlCtXngcPwjE2NiY4+BYT\nJ45DoTCjcGEVaWmmgC/W1kWoVWsTX301h27dZrJihd87XXNOOXw4/t9ADkCXs2fLEBcXh7Gx8UuP\nSUxMQqVKz8ppZuaHqelGzp414tgxQ4oXL0mpUqXfe1zh4WGMGDEcP79N732u7GJsbMyECbJGTgjx\ncZBgTggh8pGsasYBeHv7vrDvdcksrKwK8fhxbPYPMpu9SSAHMGbMGY4cSc82GRwMY8b44+vb/p36\n1NPT175WKnW0SVRKly5LyZIdmDevMxkFzgsUOM/8+ZWZNGkUJ08ew9GxwkeX/dDYOAnQkPGorqlp\n5Gsfs6xTx4WGDbdw5Ig7pqYb0NPrzKxZn7Fu3Qrq1WuQLcGcEEKIV5NgTmSbd1mHIoT4OERGRjFs\nWAA3b5pRtGgc06dXw8qqYm4PK1s9eGD0yu1XMTQ01BYUf5kSJUoRFRVJ796W/PmnD6dOfYmR0WV6\n9ozC0bEttWt/hpfXDG1R9Y/JiBH1uH7dhwsXqmBpGYanp8lL66dt3LiWPXt+BeCrr1pjZNSH27fv\nYme3kaNHYzhx4hgXL15gzZpVTJ06G41Gk6kExogRYyhRohRTp07EyMiYa9f+ISIigkGDvsvykVeV\nSsXkyeO4fj2IUqXKMG7cJIKDg1m8eB6JiYmYmpoxZswEChe2JCTkPrNnTyc6OgodHR1+/nkm5uYW\njBzpSWxsDCpVGv36DaR+/YaEh4fh6TkEJ6cqXL4cqM3y6eu7nMjIKCZMmEKFCpVITExk3rxZBAff\n/jdLbH/q12+Yo/dDCCHelARzIlukpaXh7e3L06cRWFgUzrQmRQjx8Rs79hB797oBCm7cgJEj/Tl6\nNH8Fc+XLR3P5sgpQAmmUL//ms46vKluQQVdXlylTZrJggRfm5jE0aLCOL79sTf/+3wDwxRctOHr0\ncJZlCHKbpaUFO3Z05OHDB5ialsDQ0DDLdkFBV9m79zdWrFiDWq2hf383xo+fwujRP+LtvQoTE1NC\nQu5Tr14DGjZsAsDQoQMzlcCYM2cmCxYsBeDp0wiWLvXhzp1gRo78Pstg7t69u4waNR4npypMnz6Z\nbds2c+zYYaZPn4uZmRkBAQdYvnwJo0aNZ9KksfTq5UGDBo1ITU1FrVahq6vH9OmzMTQ0IioqigED\nPLTBWGhoCD//PItRo8bTt28vAgIOsHSpD8ePH8HPz5fp073w8/OhRo1ajB49gdjYWPr3d6NGjdr5\nPnOnECJvkGBOvLeYmBh69drLmTN1MDW9wdChlxgwoHFuD0sI8RbCw415PhtmWFj+y0jo5dWCggXX\nc/++IaVLJzBpUsu3Ov5N0v3b25dn8eLlmd6PjY3hzJmrBAWdpnXrdh/tkws6OjoULWr7yjaXLl38\nt/xFeiDTsGETLl688EK7jDWGCQkJ/P135hIYqanp9ewUCgUNGqQHVaVKlebp06dZ9mltXQQnpyoA\nfPllK9as8eH27VsMHz4IALVaTeHCViQkJBAR8URbw09PTw/QIy0tjWXLFhMYeBEdHQVPnjwmMjK9\nr6JF7ShTpiwApUuXoUaNWv++LsuDB+kF1c+cOcWJE0fZsMH/3/Gn8ujRA0qUKPXKz0oIIT4ECebE\ne/PyOsrJk70BHSIinFi0aB+urtEfTfY7IcTr2dsncPx4CqAPaLC3j3rnc/3441AmTpyKkZGxtv7k\nx5DIwsjIiLlz322N3Lu6du0uffv+Q2zsAQwM7tCzZ78P2n92yyoQzSo2zWin0agxNi700lI16QEX\n/7bVZNnm+T41Gg1GRkaULl2WZct8MrVLSIjP8vgDB/YSHR2Fj89alEolnTq1Izk5BQB9/Wf96+jo\naMejo/NsHSSkryMtXrxElucXQojc9GarxoV4hYQEfZ7/UoqNtSIuLi73BiSEeGtTprSkd+8t1Ku3\nnY4d/Zk/v8k7n2v27AUYGWVkQfw4Z6E+lIULL3PtWmfCwlYSHPw7q1frolarc3tY78zZ2YWjRw+T\nnJxEYmIiR48e0hakz2BoaEh8fHpgZWRkrC2BAenB2M2bN96qz4cPH/D335cBOHhwH5UqOREVFand\nl5aWRnDwbQwNjbCysubYscMApKSkkJycRHx8PObmFiiVSs6f/4sHD8Lfqv9ateqwdetG7fb160Fv\ndbwQQuQkCebEe/vySyvMzM7+u5VGnTrnsbEpmqtjEkK8HX19fWbMaMeOHc1YsuQrLCzMX9p2/Xo/\n7S+3CxfOYejQgQCcO3eWSZPG0qlTO2Jioj/IuD92KSn6mbaTkwuQlpaWS6N5f+XLO9KqVRv69XPj\nm2/cadv2K+ztHTK1adq0OevX+9O7dw/CwkIZP/5nfvttF+7urvTs2YXjx5+tqX5d4iyFQkGJEiXZ\nsWMzPXp0Ii4ujo4duzJlykyWLVuEu7srHh6uXLlyCYBx4yazdesm3Ny6MXBgH54+fUrz5i0ICrqK\nm1tX9u3bTcmSpV/aZ1bjcXfvS1paGm5uXenZszOrVnm/xycohBDZS6F52XMNH1heSH8tXi4g4CIH\nDjzE2DgVT88mmRbP55X05uLdyP3Nv152b69c+ZuNG9cyZcoMBg1K/0V3yZKV+Pv7YmFRmLVrV7Nq\nlT8mJqY0a/Y5Bw8e/Sges8wNv/56Bk9PM6KiqgPRuLpuZ/78jrk9LCD//9/9VL/mIP/f20+d3N/8\ny8rq7dery5o5kS2aNnWh6YtJyIQQ+ZCDgyPXrl0lISEefX19HB0rEBR0lcDACwwb9iNr167O7SF+\nNNq2rYWp6WWOHNlC0aK69O79dW4P6aPwf/93mv37oylYMJkff6yDjY1Vbg/pBcnJyXh6/sqlS+YU\nLpzAuHGOVKtmT3h4GN9/PwQ9Pd1PMlAUQnxcJJgTQgjxVnR1dSla1I49e36lcmVnypYtx/nzZwkN\nDZVC0Vn4/PPKfP555dwexkdj//7zeHpaExPzBaDhypXV7NrVDn19/dce+7bep0adpWVzNm/uh6Xl\nPBITjzN0aBQTJw6jYkUnkpKSePw4mlGjPLl16yaNG39B6dJl2LZtEykpKUyb5oWdXTEiIyOZM2c6\nDx8+AOC77zypXNk5269TCPHpkjVzQggh3pqzswsbNqzFxaUazs5V2blzG+XLl8/tYYk84NChx8TE\nVPl3S8GFC3W5fftOjvR1795dvv66E2vXbsHIyIht2zazYMFsfv55FqtW+dO6dVuWL18CwKRJY+nY\nsTOrV6/H29uXx48LY2x8CAODa9y9u4unT7/jl18WEBkZiUajIikpiXPnzhIZ+ZQtWzZw4cJfqFRq\n7t27S/fuHRk8+Btmz55K48bNMDExIzk5hWHDvuXevfRrnTp1IvPnezFwYG86d/4fhw8H5MhnIITI\n3ySYE0II8dacnavy9GkETk6VMTe3wMDA4IWshvD6BBfi02NpqQaStdsWFvewtn6xCHt2+G+NutOn\nT2lr1Hl4uOLn58Pjx4+zrFFXoYIOBQueIja2DaCgTJkEqlatzq1bN3jy5AkAc+cupkmTZiiVuvz5\n50mUSh0mTpxGuXLlUSjg1KmTzJgxhYcPw1EqdTA0NGT27Gna8WUUTZ81az7Lli3Okc9ACJG/yWOW\nQggh3lr16jU5dOhP7faGDdu1r7ds2UViYiIpKSkcOJCeubBoUVvWrNn4wnnEp2Hw4P4MGfI9Dg6O\n/PnnL7Ro8ZDTpytibBzDkCH6WFjkTDD3PjXqPD2bc/Hibp4+TcbWNpHx4+vg738RhUKBqakpiYlJ\nODlVISUlhT/++J3IyEiioiKZOHE0aWlpFChQELVajUaTpq1fZ2lpSWRklHZsb1I0XQghXkVm5oQQ\nQmQbjUaDp+dWatQ4T82ax5k790BuD0l8ABqN5qVFvyFzUKWjo8PixV9x7lwFTp1qiLv75zk2rvep\nUZeWlso333SgYsUwVq5sg6lpQQIDL1CunD2g0BZL12g06OjoYGhYkIoVnZgzZxGffVafgweP0qBB\nI/T0dPH1XY+v73pGj57A2rWbteN7k6LpQgjxKjIzJ4QQItusXXuYtWv/h0ZjAcCCBZdo0iQIFxfH\nXB7Zp6lRozrY2tphZmaOtXURHBwq8PnnjZg7dxZxcdHo6urj7FyVo0cPkZKSQqlSpXn8+DHx8XEM\nGvQdjRqlpylev96PQ4d+JyUllc8/b0SfPt/8m9VxMJUqVebatavMnr2QtWtXExT0D8nJSTRq1JQ+\nfb7JclwajYZNm9ZTqJAJnTt3A8Db+xcsLArTqVPXbLn252vUzZgxmVKlytCxY1dq1fqMBQu8iIuL\nQ6VKo0sXV0qXLsO4cZOZPXsaK1d6o6ury88/z6Rhw8ZcuXIJd/duKBQKBg0aiqmpGdHR6bNrf/99\nmYMH96HRqNHR0eHhwwfcuRMMpBdH79SpG+fPn6VDhzYULGiIi0tV2rfv+G9AKIQQ70+COSGEENkm\nPDxZG8gBJCaW49at3yWYywVXr15BpVKxZs1GUlNT6d27Bw4OFZg1axo//jiKqlUrcvjwnwwdOoCN\nG3fg7f0LFy6cY/DgYZQqVYaRI7+nUaOmnDlzipCQ+6xY4YdarWbkSE8CAy9gbV2E0NAQxo2bTMWK\nTgD07z8IExMTVCoVw4YN4tatm5QtW+6FsSkUClq3bsfo0T/SuXM31Go1f/xxkBUr/LLt+m1sirJu\n3dYX9tvbl2fx4uUv7C9WrDgLFix9Yf+gQUMZNGiodvvBg3Ds7IoRFRWJp+dgNBoN1avXomdPd+bM\nmcGSJQtRq1Vcvx5E374DWLnSHy+vGUREPOHixQtYWlppgzlZUyqEeF8SzAkhhMg2zZqVxc/vOI8e\n1QfA3n4vjRrVyOVRfZouXw5EqdRFT08PPT09ChcuzObN63jy5AmDBvWlSBFrwsLCSEpKwtNzCEql\nLtHRUfzyy0KMjY2IiEhP8nHmzCnOnj2Nh4crAImJSYSE3MfaughFihTVBnIAf/xxgF27dqJSqYiI\neMKdO8FZBnOQHmyZmppy48Y1IiIiKF/eERMTk5z/YN6TjU1RNm3a+cL+tLQ0Vq70R6FQsHfveRYu\nfICXlz6NGp3Dy2vBC8Ha6NETMm1nrC8VQoi3IcGcEEKIbFO1qj2LF19i8+Zt6OqmMXBgBQoXtnj9\nge8oLi6Ogwf38dVXHTl//i82blzHrFnzcqy/vEUBpK/DOnPmFLGxsfzvfx3YsWMrDg6OfPvtQEqW\ndKBTp3YsWuTN4sXzMTIyomPHLjRs2IRmzZ6tZevRw53//S9zwfPw8DAKFiyg3Q4LC2XjxnWsXOmP\nsbEx06ZNIiUlmVdp06Y9u3f/SmRkBK1bt8u+S/+A0tLS+O67HZw4YYmRUSJ9+ypZuNCQsLAuAAQF\nPaZkySN4eDQCID4+nsmTAwgPL0ClSmn8+GMLdHQkhYEQ4t3Idw8hhMiHwsPD6NWrS5bvDR7cn6Cg\nqznWd6NGVViypDkLF7aiQoWcLSIeGxvDjh1bcrSPvKpKFWdUKhUpKSmcOHGMu3eD2blzK/HxcQQF\nBXH37l00Gg2pqamZjvtvIo7ateuwe3d6hlKAx48fERkZ+UJ/8fHxFChQECMjI54+jeDUqZOvHWPD\nho05ffokQUFXqV37s/e42tyzeHEAW7d2Izy8HTdvdmHmzEeEhT2brVSrrbh1K0m7PWTIXnx9u7Jv\nXwfmzGnB9Ol7c2PYQoh8QmbmhBDiE6NQKPLN+pxlyxYRGhqCh4crurq6FChQkLFjRxAcfAsHhwqM\nHz8FgKCgqyxePI/ExERMTc0YM2YChQtbMnhwfypVqsz5838RFxfLyJHjcXZ2yeWryh6OjhVRKnVx\nc+tKUlISZcuW4+uvO1G9ei28vGbg5+fH8uUrSUp6Fmg8/7WR8W/NmnW4c+cOAwZ4AGBoaMi4cVNe\n+Dqyty9P+fIOuLp2wNrahipVnF87Rl1dXapXr0mhQiZ59mvywQOAgtrtyMjaFCv2JyEhJQAwMLiL\ns3Mh7ftXrpgDyn+3TAkMfJbRUggh3pYEc0II8RHau/c3Nm5ch0KhoFw5e/r2HcC0aZOIjo7GzMyc\n0aPHU6SIDVOnTqRevQbarIPNmjXg4MFjmc6VnJzEtGmTuHXrJiVKlCI5OTnfpEEfOPA7goNv4+u7\nngsXzjFqlCdr126hcGFLBg7sw6VLF6lY0Yn582czc+ZcTE3NCAg4wPLlSxg1ajwKhQK1Ws2KFWv4\n888T+PouZ/78Jbl9WdlGV1eXDRu2c+LEUSZMGEPJkmUoWtSWkSPHYmNjjkqlR6dO6Y83jh49gfnz\nZxMfn15z7fk1XJ06dc0yy+R/awf+dx1YhkWLvLWvt2zZpX2tVqu5cuUyP/88690vMpfVrWvOxo3X\nSEhwAKBKlSAmTizFkiWbSEzUo0kT6NSpmba9lVU8wcEZWxoKF0748IMWQuQbEswJIcRH5vbtW/j5\n+eDt7YuJiSkxMTH8/PMEWrVqS4sWrdm9exfz53sxfbpXFrMZL85u7NixlYIFDVm7dgu3bt2kd+/u\nH80syKpV3hgaGtGtW49M+8PDwxgxYjh+fpteefzzQalGo6FChUpYWloBUK5ceR48CMfY2Jjg4FsM\nGzYISA8gChe20h7XsGFjABwcHHnwIDxbrutjkZqagoeHKykpKVSvXgMvr2kAFCxoyPz5cylQwIzn\nv2aaNm3OzJlT2bp1E1OmzMDOrliOjGv16iNs336P6Gg/Pvusdo718yG0a1eLuLjjHDz4N4aGyXh6\nVqNMmWLUr18ly/YTJzoyZsxawsNNcHB4ysSJDT/wiIUQ+YkEc0II8ZE5f/4sTZo0w8TEFAATExP+\n+ecy06d7AfDll61YunThG58vMPCidlalbNlylC378dS4yu6gUk9PX/taqdRBpVIBULp0WZYt83nl\nMTo6Sm37DyWrmdTsdOTI6Ze+Z2VViEePYli6dJX2PlSu7JypqHVOOHDgPJMmlSM+vg0wiLi4vQwf\n/pgiRaxee+zHytW1Pq6ub9a2Ro3y7N9fHpVKhVKpfP0BQgjxCpIARQghPjIKhSLLxyCz2qdUKlGr\n0/er1WrS0lJfaJOd1q/3Y+vW9EfrFi6cw9ChAwE4d+4skyeP4+DBfbi5daVXry4sXbpIe1yzZg20\nrw8d+p1p0ya9cO6goKu4uXXD3d31jZOaGBoakpDw6sfUSpQoRVRUJH//fRlIzz4YHHz7jc6f83Jv\nhlSlUjFgwCZq1w6lVq2/mTVr3wfp9+zZR8THP6s7eP9+Xc6evf5B+v6YSCAnhMgOMjMnhBAfmWrV\najJ69A907dr938cso3FyqkJAwAG+/LIVBw7sxdm5KpBe8+ratas0afIFx48fJS0t7YXzubhU5eDB\nfVSrVoPbt29y69aNF9qEh4fh6TkEJ6cqXL4ciKNjRbp27cT8+QuIjIxiwoT0RCL79u3h0aOHBAQc\nICUlFaVSybff9qNUqTIUL16CZcsWY2FRmJ9+GsPixfM4duwwDRo04vmg5b+zcRmb06dP4vvvR+Ls\n7MKSJQve6LMyNTWjcmVnevXqgoGBARYWhV9oo6ury5QpM1mwwIu4uDhUqjS6dHGldOkyWZwxe4Or\n9ev90NfXp2PHrixcOIdbt26yYMFSzp07y2+//R8Ay5cv4eTJ4xgYGDBjxhzMzS0IDw9j+vTJL6yR\nzE6LF+9jxw5XwAiAX365SOvWN6hUKWdnbh0cCqGvH0pKih0AlpYXqVKlVI72KYQQ+ZXMzAkhxEem\ndOky9OrVm8GD++Pu7srixfMZNuwn9uz5FTe3bhw4sJehQ38AoF27r7h48Tzu7q5cuXKZggUNtefJ\nCJrat+9IQkICPXp0YtUqbxwdKwLQsWNbYmKite1DQ0Po2rUH69dv4969u+zZs4elS30YPHgofn6+\nlCxZmhUr1mBiYoKrqxuPHj3AyakyVatWY9eu7RgbF8LBwRGNRoO9fXmaNWuBt/cSDh8OeO01x8XF\nERcXp80k+eWXrd/485ow4Wf8/DaxYoUfM2c+qzE3fPhPtGzZBkjPtLh48XJWr16Pv/9m2rRpD6Qn\n5nBwSJ8lMjMzY8uW/3vjft+Es3M1AgMvAukzj4mJiaSlpXHp0kVcXKqRlJSIk1MVVq9ej7NzVXbt\n2gHAvHmzadWqLWvWbKB58xbMn+/10j727v2NJ0+evPXYHj9WkRHIASQmluT+/cdvfZ7nhYeH0b17\nR2bOnErPnp35/vvBJCcnc+PGNfr3d8fNrRvnz+/km28O4Ojoh4NDYyZNSiE5OZ4GDWry6NFDADp3\n/h/Jya+uUSeEEEJm5oQQ4qPUsmUbbSCSYcGCpS+0Mze3wNvbV7s9cOAQAIoWtdVmGjQwMGDSpGkv\nHPvfxzmLFrWjTJmyQHpAWbdu3X9fl+XBgzDi4mKZN282kZFPmTNnOmq1mipVXLh9+xYajQZra2vu\n37/PV191/PeMGhSKF0shvMkv6R8i2+bOnae4eDGKChWM6dKlfo704eDgyLVrV0lIiEdfXx9HxwoE\nBV0lMPACw4b9iJ6eHnXr1v+3bQX++it9jdvbrJHcs+dXSpcui6Wl5VuN7X//c2DFiqM8epReHNzJ\naS/16zd+l8vMJCTkPpMmTWfEiDGMHz+KI0f+YN06P77//iecnauyapU38fF3OXrUk549N9C6dWX2\n7v0NR8eKXLx4gSpVnLGwKIyBgcFL++jYsS0+PmsxMTHN8XWHQgjxMZNgTggh8qm4uHh++mk/N2+a\nYmMTganpIWJjo1GrVbi59QVg69ZNnDhxjKSkRCA9gIqJiebChb+4dOkC5uar8fDoh0ql4qefhmNr\na0u3bj3ZtWsHcXFxFC1qx8KFc1EoFERHR3P3bjBXrlxm69aNREZGYmtrh0ajwcLCgrt371C8eAmO\nHj2EkZExkB60aTRgbGyMsXEhLl26SJUqLhw4kLOFlH/55XdmzHAhObk0enphBAfvZuTIN58NfFO6\nuroULWrHnj2/UrmyM2XLluP8+bOEhoZSqlRplMpnP4Z1dBSoVCrCw8OIiYlh1qxp/PPPZQoXtkSj\ngRs3rjF79nSSk5OxsyvGqFHj+euv0wQFXWXy5LEUKFCApUt9XhkEPa9mTQe8vSPZsmUL+voqhgyp\nibGx8Xtfc9GidpQrl/6opoODI6GhIcTFxWofDW7RojXjxo0EwMnJmUuXAgkMvEjPnh6cPn0S0FCl\nyqtr/WV+VPfjyMwqhBC5QR6zFEKIfGrMmANs3dqDixfbc/x4Ma5fT2X16vX4+W2iTp3PADAzM8fH\nZy3Nm7ckOjoKSC8XYGJixsiRI/nmm29ZvHgeGo2G1NQUjIyMcHauytOnESgUCkxNzTAwMECpVLJq\nlTd2dsU4evQQSqWSZs1aEBoagkKhYMCAwfz00zAGDuyjLR0AGbN26a9Hj57A3Lmz8PBw1b6XU/bt\nU5OcXBqA1FRbDh58swDoXTg7u7Bhw1pcXKrh7FyVnTu3Ub58+Vceo1arsbOzw99/M/Hx8RQtasvP\nP0/k22+HsmbNBsqWLYev73IaN/4CR8cKTJgwFR+fdW8cyGWoV68S8+e3YNas1hQvnj1r8vT19QgP\nD6NXry7o6CiJi4slPj4eH5/lbNmyEU/PIdy5E8zEiWNwcamKn58PV65cpkGDhty4cZ1582ZTsmT6\nvRk16gf69OlJz56dtY+gvsyUKeM5duywdnvSpLEcP37k5QcIIUQ+IDNzQgiRT927ZwykZ8xLTnYg\nLu4uS5cuom7dBly79g8ajYaGDZsAUKZMWW3ylMuXA/EeK58AACAASURBVLGzK4ZCoaBq1RrExsZi\nYmJC5crOHDt2hFu3btKzpwfr1q0BYMOG7TRr9jnGxsaUKFGKnj09aNWqLQAREelrsBo1aqotbP68\n3r37a187ODiyevV67fagQd9l/4fyLwODzIliChRIybG+nJ2r4u/vi5NTZQwMCmBgYKCdpXo+YH3+\ntY2NLefOnSUg4CDJyUnUr/85hw4FZDm7BR/msdR3ZWRkTIECBjx48IBdu3bQunU7kpKS6N27H7Gx\nscydO5NixYqjUCgwMTEhMTGBChXS13WOGjUeExMTkpOT6NfPjUaNmmJiYpJlP23btmfTpvU0aNCI\nuLg4/v77MuPGTf6QlyqEEB+cBHNCCJFPlSwZz4kTKkBJamopihXrTdmyZqxYsYSbN29gZGSEvr4e\nANbWRbSJUSA9kHJ2duTx41iUSiXe3r5s2bKR7t174eraC4CAgAPa9hqNBrVaTdGiRd86sEhMTGTj\nxmPo6kKXLo3Q19d//UHvaciQ4gQH/8b9+zWxsQlk0CDrHOurevWaHDr0p3Z7w4bt2tcHDjybOcoI\neENDQyhQwEC7RnLDhrU8efLolX18LEXgIatspQqaNv2SP/44SHx8PLt376J37/7o6CixsSkKgK1t\netFwZ+eqnD//F4aG6YlZtmzZwLFj6Z/Ro0cPCQm5R8WKTln26+JSjTlzZhAVFcXhw7/TuHETdHTk\nASQhRP4m3+WEEOIjFx4ehqtrB6ZNm0S3bl8zadJYzpw5xYABvena9WuuXr3CqlXebNiwVntMz56d\nGTq0Ch06+FKhQjsqVaqHQrEVpVKXkiVLER8fR0TEE0aN+uGF/qpUqapds3b+/F+YmZljaGhE0aK2\nXLsWBMC1a0GEhYUyePBeatWaRVJSIq1adcLZuRoBAQdRq9U8efKE8+fPvfLaEhIS6Nx5FyNGtMPT\nsw3du28jJSXnZskyNGxYmd9/r8yWLVf4/fdytG5dI8f7fBP791+gc+fD3LyZTMeOm3n6NBJIn90y\nMTHRZsbct283VatWB9Jr7cXHx+XamJ+XkXgno/5ht2498PDoR6FChWjbtj0HDhxh/Pgp3L17h379\neqFSqXB17aWdievZ0wNr6yJA+tfeuXNn8fb2ZfXq9djbO7z2a6NFi9bs37+bPXt+o3Xr/+X49Qoh\nRG6TYE4IIfKA/5YNCAg4wLJlz8oGZDUbUrBgQbp0saRVq4rMnDkRAwN9/PxWcf36NQoXLkzhwlba\njInwLONk7979uXYtiHbt2rF8+RLGjp0IQMOGTYiNjaFnz85s374ZPT0LTp7swJ07U1CpjNi504iG\nDRtTvHhxevToxNSpE6hcucorr8vf/yinT7sDeoABR470YNu2o9n62b2MubkFDRvWxNra6vWNPwCN\nRsPPP4cSHNwWlcqYo0d7M3VqepZGhULB6NETWbJkAW5u3bh16yYeHv0AaNWqLV5e0+ndu/tHk87f\nwqIwUVFPiYmJJiUlhZMnj6PRaHj48AHVqtVg4MAhxMXFERMTQ0hINIcOHUOlUnHtWhDh4WEAJCTE\nU6hQIQwMDLh79w5Xrvz92n5btWrL5s0bUCgUlCxZKoevUgghcp88ZimEEHnAf8sG1KhR69/X6WUD\n7O2zSqihoGxZe375ZQEmJiZ8//0IbR23Tp3asWqVPyYmpgA4OlZg4cJlAJiYmDB9uhdWVoV4/DhW\nezYDAwPmzl2s3T5z5iBpaemFn2/dOo+Ozg4iIiLo0cOD4cN/eourk8yEACkpKTx9akpaWjHu3v0V\ngKioAnTr1k7b5vkyFBkaNmyiXfv4sdDV1cXdvS/9+rlhZWVNqVKlUalUTJ48jvj4ODQaDV991ZE+\nffZz4sRgbG2H88UXrWjWrC7Fi5cEoHbtuuzcuY0ePTpRvHhJnJwqZ9nX83/IMDe3oFSpMnz+eaMP\ncZlCCJHrJJgTQog8IGNtG4COjg56enra1yqVCqVSiUaj1rbJeBytePES+Pis488/j7NixRJq1KiF\nu3vfbBlTxYoJnDiRCBQEVKSmXqR2bXPUaj3atTvCvHkdXruWq0ePBuzatZqzZ90ANZ9/7k+HDh2y\nZXx5jYGBAS4uYRw8mL7OUU8vlNq1X/wxvX79cQICEjAySmLEiNrY2RV56742b15Pnz5u2u0ffxzK\nxIlTMTIy1tZtCw8PY8SI4fj5bXqn6+nYsSsdO3Z96fve3vs4frw9oEdoqB+hoVEMHXqc0aMnaNt4\neWVdX2/Lll3a18+vO0xKSiIk5B7Nmn35TmMWQoi8RoI5IYTIB4oWteXEifRH8p5/VO3JkycUKlSI\n5s1bYmRkzO7d6b8Ep6+zitfOzL2LiRNboau7k2vX9FGp/uHEiX6kpaUnstiwoQJ16x6hc+dGrzyH\nkZERW7a0Yf36Hejq6tCt29cfJAHKx8rbuzVTp24iIsKA2rX16d07cxHvHTtOMWpUGRITHQANN26s\nYdeudtrg/k1t2bIRV9fOZPwaMHv2AgDi4uK0WU3f1NSpE6lXr0GW2UpfJTVVQ+ZfQwxISnq7vjNc\nuHCD5cv3c+PGDnr27K5NoCKEEPmdBHNCCJEHZLUm7vnXDRs2Yd++3fTs2ZmKFZ20j6rdvn2TX35Z\ngI6OAl1dXX74YTQA7dp9hafnEKysrLVZE9+Wnp4ekya1AcDHR8ORI7ba9zQaCx4+THyj8xgaGtK3\nb4t3GkN+Y2xszPTpbV/6/okTMf8GcgAKAgOrERoaQqlSpV96TGJiIuPHj+Tx48eo1SoaN/6CJ08e\n06tXLwoVMmXBgqV07NgWH5+1xMfHv3Uwl14r8O0fj+3e/TN27vTj0qVegIrPPltL+/bt3/o8ly/f\nonfvJ4SGjgJGsnGjL126JFGgQIG3PpcQQuQ1EswJIcRHLiNDYIbnH0N7/r3n17NlsLGxoVatOi/s\n79ChCx06dMm2MbZuXZ1Vq3Zw40b6I5IlS/5GmzavTn4i3p6VVRqQDKQXB7e2vkfhwlVfeczp0yex\ntLTWzr7Fx8exZ8+v+Pv7k5qaXocwIxhbtmwRGo0GDw9XKlSoRETEE3r16oJCoaBXrz40bdoMjUbD\nvHmz+OuvM1hbF8k0K+jru4KTJ4+RnJyMk1MVfvppDKGhIYwbNxIfn/Rsq/fv32PChNH4+Kxl8+bG\n+PtvQVcXPDzavVMAtmvXDUJDO/27peDcuTacOnWJRo1qvfW5hBAir5FslkII8QlITk5m7NhduLoe\nYNSoXSQlJWXr+YsUsWT1akfc3TfRq9dmfHzsKF3aLlv7EDB8+Be0beuPtfUeypTZzJgxBhQqlHUR\n7Qxly9rz11+nWbp0EYGBFzEyMn5p24EDv0OhUODr+6wUwJo1G5k/fwlLliwgIuIJR48e4v79e6xb\nt5WxYydz+fIl7fEdOnRhxQo//Pw2kZyczIkTx7CzK4axsTE3blwHYM+eX2ndOj2pi4WFOUOHtuTb\nb1tiaGj4Tp+JsTHAs5IFBQo8wMrK7J3OJYQQeY3MzAkhxCdg9Og9+Pt3BfSBVOLi1rNo0dfZ2oe9\nfQlmzSqRrecUmenr67NqVReSk5PR19d/o8cb/5sEp3r1mi9t+3zB96CgfzA2NkahUGBuboGLSzWu\nXv2HwMALNGvWAoVCgaWlJdWrP6vRd/78Wdav9yc5OYmYmBjKlClLvXoNaNOmPXv2/MqQIcP544+D\nrFjh934fxHMGDGjMmTO+/PFHAwwMound+w6VKrXJtvMLIcTHTII5IYT4BFy5Ykx6IAegxz//vHo2\nR3zcDAwM3rjtf5Pg/Pbb/2FoaERcXBwGBq9KgPPyQPH5oC9DcnIyc+fOYtUqf6ysrPHxWa6te9ew\nYWN8fZdTvXoNHB0rYGKSfV9/BgYG+Pt35dat2xgZmWNr65Rt5xZCiI+dPGYphBCfgCJFEjJtW1sn\nvKSlyG9u375J//7ueHi4snr1Stzd+9KuXXv69u3L0KEDM7V9/lHHChUqEBcXj1qtJjIyksDAC1Sq\n5ISzczUCAg6iVqt58uQJ58+fA56VwzAxMSUhIYFDh37XzhwaGBhQu/ZneHnNoFWrdmQ3HR0d7O3L\nYWsrj/YKIT4tMjMnhBCfgEmTahIb60dwsAklS8YyeXL13B6S+EBq1arzQhIcBwdHBgzoqy0Kv2XL\nLu1s2xdffEmvXl2oU6cuX33VAXf3bigUCgYNGoq5uQUNGzbm/Pmz9OjRiSJFbKhcOT3RTaFChWjb\ntj29enXBwqIwFStmniH74osWHD16OMuEPEIIId6NQpPVsxK5IOMHish/rKwKyf3Nx+T+5i0ajeaN\n08jLvc3fnr+/06fvYft2fZRKNd276zBkyBfZ2ldsbAxr165GV1ePfv0Gvv4A8V7k/27+Jvc3/7Ky\nKvTWx8jMnBBCfELepR6YyN927z7FkiWfkZycnrxmzpyr1Kx5mTp1KmfL+VevPsrSpb5oNDHY2LSj\nU6dozMzevVi9EEKIZ2TNnBBCCPEJu3kzShvIASQkOPDPP6HZcu6EhATmz1cRHLyVO3cOcOrUIGbN\nOpYt5xZCCCHBnBBCCPFJa9SoLFZWJ7XbdnZ/0KRJ9szKxcfHEx1t9dweHeLi9F7aXgghxNuRYE4I\nIYT4hDk72zNvXiotW26hdevNLFxoRKlS2ZMV0tLSklq1/gZUAJiYXOKLL8yz5dxCCCFkzZwQQog8\nKjw8jBEjhuPntym3h5LnNW9ejebNs/+8CoUCH5+2eHltJiZGj6ZNC9OqVa3s70gIIT5REswJIYQQ\nIscYGRkxYUKb3B6GEELkS/KYpRBCiDxLrVYzc+ZUevbszPffDyY5OZnQ0BA8Pb+jT5+efPttP+7d\nu5PpmIEDewPw4EE4Bw/uy4VRCyGEENlDgjkhhBDZIi4ujh07tn7QPu/fv0eHDp3x99+MsXEhjhz5\ng1mzpjF8+I+sWuXPoEFDmTNnZqZjli71ASAsLJSDB/fn6PiaNWuQ5f6dO7exb9/ulx53/vxf/PTT\n8JwalhBCiHxCgjkhhBDZIjY2hh07tnzQPosWtaNcOXsAHBwcCQ8P4++/Axk3bgQeHq54eU0jIiIi\n0zEZAdayZYu5dOkCHh6ubN68IYdGmHVdv/btO9CiResc6vPVOnZsS0xMdK70LYQQInvJmjkhhBDZ\nYtmyRYSGhuDh4UrNmrXRaOD06ZMoFAp69epD06bNsr1Pff1nae51dJTExDzF2LgQvr7rX3FUeoA1\ncOAQNmxYy6xZ8965//Xr/dDX16djx64sXDiHW7dusmDBUs6dO8tvv/0fAMuXL+HkyeMYGBgwY8Yc\nzM0tWLXKG0NDI7p160FIyH1mz55OdHQUOjo6TJkyA4VCQWJiAmPHjiA4+BYODhUYP37KO48z09VL\n4XghhMg3ZGZOCCFEthg48Dvs7Irh67ueihWduHnzOmvWbGT+/CUsWbKAiIgnOT4GIyMjbG3tOHTo\ndwA0Gg03b97Isq1Go3nv/pydqxEYeBGAoKCrJCYmkpaWxqVLF3FxqUZSUiJOTlVYvXo9zs5V2bVr\nB5AeUGXEVJMmjaVjx86sXr0eb29fLC0t0Wg03LhxjWHDfmDt2i2EhYVy6dLFtx7fqFE/0KdPT3r2\n7KztO0NCQgI//jgUd3dXevXqQkDAQQD++usMvXt3x82tK9OnTyY1NfU9PiEhhBA5SYI5IYQQ2eL5\n4OjSpYs0a9YChUKBubkFLi7VuHr1n2zv87+zTAqFgvHjp/Dbb7twd3elZ88uHD9+JNv7zeDg4Mi1\na1dJSIhHX18fJ6fKBAVdJTDwAs7OVdHT06Nu3fr/tq3AgwfhmY5PSEggIuIJDRo0AkBPTw8DgwKo\n1WoqVKiEpaUVCoWCcuXKv3Dsmxg1ajyrVvmzcqUfW7du1D5eqdFoOHbsGJaW1qxevR4/v03UqfMZ\nycnJTJs2icmTZ7BmzUZUKtUHXwcphBDizcljlkIIIbKdQqF4YeYrux/vK1rUljVrNmq3u3XroX09\nZ87C1x5vaGhEQkL8G/cXHh7GDz98R5UqVfn770CsrKyZPn0OFhaF6dfPjbi4OB4/fgzA/fv3+eGH\n71Aq03/MJiYmMnfuDOrWbUBoaAj79+8lJSWZw4f/IC0tDYCpUyeir6/PjRvXsbGxQU9PX9u3UqmD\nSqV647Fm2LJlA8eOpQezjx494v79+0D6vXBwcGD69BksXbqIunUb4Ozswo0b17G1taNYseIAtGzZ\nhu3bN9O5c7e37lsIIUTOk5k5IYQQ2cLQ0JCEhAQAqlRxISDgIGq1msjISAIDL1CxYqUc7T8tLY2d\nO4+ybdthUlJSXtouI6gsV84epVKJu/ubJ0AJCbn/QvbMiIgI4uLiGD9+CkOGDGPHjm04Ojpib19e\nG4CdPHkMe3sHFAoFs2ZNpU6dz+jc2ZUhQ74nOTmJY8cOA+kB18KFS2nfvuP7fRikZ8Q8d+4s3t6+\nrF69Hnv78qSkJGvfL1WqFD4+6yhbthwrVixh9eqVLwTc2fEoqhBCiJzz3jNzPj4+zJo1i1OnTmFm\nZgaAt7c327ZtQ0dHh7Fjx1K/fv33HqgQQoiPm6mpGZUrO9OrVxfq1KlLuXLlcHfvhkKhYNCgoZib\nW+RY32lpabi5bebgwa6Akg0bNrBu3VcYGBi80PbAgfSZKl1dXRYsWPpW/WSVPfPJk0ekpqayaNFc\n7Yyks3NVzM0t+PPPEwD8/vsBKld2JjQ0hMuXLxEcfBsdHR0OHNiDubkFW7du4vr1IIyNC/H06dNM\na+reVUJCPIUKFcLAwIA7d4K5cuXvTO8/evQIfX19mjdviZGRMbt378LVtRfh4WGEhoZgZ1eM/fv3\nULVq9fcbiBBCiBzzXsFceHg4J06cwNbWVrvv5s2b7Nmzh927d/Pw4UM8PDzYv38/OjoyCSiEEPnd\nhAk/A+nFvNPS0hg0aOgH6Xf79qMcPNgdMAbg6FE3/P130bfvl9o2cXFxzJp1mOhoPRo3NqN9+9pv\n3U9W2TNNTEz5v/97sfh4QkICVlbWxMTEcP16ENOmzSYhIZ5z585m2X7atEnUrVsfW1s7LCwKM27c\ns+yVw4f/9NZjrV27Ljt3bqNHj04UL14SJ6fK/76THiVev36dadNmoKOjQFdXlx9+GI2+vj6jR09g\n3LgRqFQqKlSolC2zhEIIIXLGewVz06dP58cff2TQoEHafQEBAbRu3Ro9PT2KFStGiRIluHTpEi4u\nLu89WCGEEB+/detOsGhRLHFxRtSrF8Yvv3RAVzdnl2inpKgAvef26JKaqtZuaTQaevfezeHDHoCS\nXbv+QaM5xVdf1Xmvfp/Pntm48Rfa7Jn29uUxNDTE0bEiCxbMpl69BigUCoyMjLG1tc3U/tatm9rZ\nPgAvr/2sWVOA1FQDvvwyhHnzvn6nP4jq6enh5fXi2sEtW9JLJpQtW581a549XhoTE83hw2ewty+G\nj8+6d/g0hBBCfGjvPF32+++/Y2Njg6OjY6b9jx49wsbGRrttY2PDw4cP332EQggh8oynTyOYPl2H\n27c78ehRK3bs6M6iRb/neL8dOtSnVi1/QAWocXFZQ/fu9bTvN2vWgL/+qggoAYiPr8gff7x94ew3\nyZ554sRR7ftNmzbj4MH9NG3aXLtv/PifX5pt886dMBYtcuDhwzY8fdqMDRu+Yu3anMvGmeHSpZu0\nbHmazp0r07TpQ9auPZ7jfQohhHh/r/xTqYeHB0+evFgXaNiwYSxfvhwfHx/tvlctkn6TDGZWVoVe\n20bkXXJ/8ze5v/nX297b8PD7PHpU+rk9BYmNNfgAXyOF+OOPbixbtgeVSsM333TA1NRE+66Ojg4W\nFhHExWXsUVOkyMuvLzY2ll9//RVXV1dOnz6Nr68vy5YtY8+e3do23303kLFjx2JkpIufn2+W5+nU\nqT2dOrXPtM/KyiHL9vPmebFxYwCJiWWe22tGbGzO/R/LOK+3921u3OgAwNOn1qxYsY3hw+X/dV4m\n35fzN7m/IsMrgzlf36x/OF2/fp2QkBDatWsHwMOHD+nQoQObN2+mSJEiPHjwQNv2wYMHFClS5LUD\nefw49m3GLfIQK6tCcn/zMbm/+de73FsLC2ucnfcRGJj+2KCR0T/UqPHhvkZ69WoEQEpK5p8rGg0M\nH66Hl9dmlMoNGBtHEBRkxI4dBahfv+ELZQdMTEyJjo6iWbO2BAb+w+nTp2nTpi01atTm9OmT+Plt\nYs+eX1GrFZiYWPP4cSw//TSMbt16UrVqdby8ZhAU9A/JyUk0atSUPn2+AeDPP4+zePF8ChQoiJNT\nFS5c+JsaNbrx+ecl2bNnMzdv3qB8+SmEho4jPr4pRYoco169Yjny+T1/f2NiMv9BNi5OyaNHMdle\nTkJ8GPJ9OX+T+5t/vUuQ/k6LGMqXL8/Jkye1202aNGH79u2YmZnRpEkTPD09cXd35+HDh9y9e5cq\nVaq8SzdCCCHyGAMDA1aurMWcORtISNDnyy+NadWqbm4PC4Du3evRrl0sERGOlChRkpiYGAYM8KB+\n/YZAetmBSZOmM2LEGL7+ujVPn0bg4eHKvXt3KVmyNLa2duzevQu1+tlavCNHDtGqVTvs7ctz48Z1\npk2bhKGhIU2bNueHH0aiUqkYNmwQt27dpFix4syePZ0lS1ZiY1OUr792IzjYlH37OlGq1FB69SqF\nj88ELlz4hx9+GE758g/p1q0ULi72L7ukbNOmjTEnT/5NbKwTCsVTmjaNlkBOCCHygGxZkf78N/xy\n5crRsmVLWrdujVKpZMKECfIDQQghPiElSxZl4cI2uT2MLBUsWJDt2zcTGHgRHR0FT548JjLyKZC5\n7EDz5i3ZvXsXixYtx9W1A2FhIcyaNY+oqCi++caDy5cDM533+vVrpKSkMGHCz7i4VGPTpnX07t0D\nlUpFRMQT7ty5jVqtwtbWDhuboiQnJ3P7dh3gNgBq9X127LjAuXN/AGBurs+oUZUpUaLUB/lcunSp\nR+HCFzh5cgvFiunj4fHVB+lXCCHE+8mWYC4gICDT9oABAxgwYEB2nFoIIYTINgcO7CU6Ogofn7Uo\nlUo6dWpHcnJ6gfHMZQcUQMajhxoqVKiEpaUVUVFRGBjoEx4ejlKp1La3sytGUlIiW7ZsICwslB07\ntrJypT/GxsZMmzbp3yLmz/6wqVAo0NFR89wkH6VLd2Plyh45ePWv9sUXVfnii1zrXgghxDuQ4m9C\nCCE+GfHx8ZibW6BUKjl//i8ePAh/ZXtjY2MMDAqQlJQMQEDAAUCBSpWGjY0tiYkJaDQaEhLiUSp1\nsbd3YN++3cTERGNkZMTTpxGcOpW+LKFEiZKEhYXy4EE4+vr6lCt3Dh2dBECNnp4N1tb/aPu9fj0o\npz4CIYQQ+UjOFv4RQgghPgIZj/s3b96CESO+x82tKw4OFShZsvQLbQD09PRJTU0FwNW1J0uXLsbD\nwxUXl+ro6aXP4Dk7u6Cvb8DYsSMoXboM9vblcXGpxuefN+Lbb/vj6toBa2sbqlRxBtLXE3p6jsTT\ncwgFChSkVq2KWFndp0aN7TRqNJitW/1xc+uKWq3G1taOmTPnfaiPRwghRB6l0LyqpsAHJFl58i/J\nupS/yf3Nvz71eztp0lhu3bqBnp4elpZWzJw5D3//1QQE7KdLl+60bNmGIUO+YfDg4SiVSqZNm4RG\nk/7c5IABQ6hd+7MXzpmYmEjBggUBmDNnJsWLl+DzzxsTFHSPqlXLY2pq9sGu71O/v/mZ3Nv8Te5v\n/vUu2SwlmBM5Tr7p5G9yf/MvubfpAgIOsnatLyqVChsbW8aMmZAp6Lp3L4Rr10KoUcMBc3PzV55r\n8+b17N37G6mpaTg4OFCuXHOmTCnAkyeVKFPmTxYvLkaNGg45fUmA3N/8TO5t/ib3N/+SYE58lOSb\nTv4m9zf/ys/3tmPHtvj4rMXExPS9zrNu3XEmTzYiMtKJMmWOvnUw1rjxPq5c6aTdbtlyE2vWtHqv\nMb2p/Hx/P3Vyb/M3ub/517sEc5IARQghxCdHoVCQHX/L9PaOJTLyc8CC27fb88svN9/q+KQkvUzb\nycmylF0IIcSbk58aQggh8rXExETGjx/J48ePUatVuLn1BWDr1k2cOHEMlSqNKVNmUKJEKRITE5k3\nbxbBwbdRqdLo3bu/tqh4VpKTMwdjKSl6L2mZtaZN4wgOfoJabYmh4TVatDB4+wsUQgjxyZJgTggh\nRL52+vRJLC2tmT17AQDx8XEsW7YIMzNzfHzWsmPHVjZsWMuIEWPx8/OhRo1ajB49gdjYWPr3d6NG\njdoUKFAgy3M3axbPypWPUautKFToCm3bGr7V2KZMaYe9/RHu3EmiZk1LWrV6eeAohBBC/JcEc0II\nIfK1smXt+eWXBSxduoi6dRvg7OwCQMOGTQAoX96RI0f+AODMmVOcOHGUDRv8AUhNTeXRoweUKFEq\ny3NPmdIOR8ejBAcnUq+eDU2a1H+rsSkUCtzcGr3bhQkhhPjkSTAnhBAiXytevAQ+Puv488/jrFix\nhOrVawKgr5/+SKRSqYNKpdK2nzp1NsWLl3ijcysUCnr0kNk0IYQQuUMSoAghhMjXnjx5gr6+Ps2b\nt8TVtRfXr197adtateqwdetG7fb160EfYohCCCHEO5FgTgghRL52+/ZN+vd3x8PDFV/fFbi59QEU\nz7VQoFCkb7u79yUtLQ03t6707NmZVau8c2XMQgghxJuQOnMix0k9lPxN7m/+Jfc2f5P7m3/Jvc3f\n5P7mX1JnTgghhHgHarWaESO2U69eAM2a7WH37nO5PSQhhBDitSQBihBCiE+et3cAvr7/A8wAGDv2\nN+rXj8LU1Cx3ByaEEEK8gszMCSGE+OTdu6cmI5ADCA2tQEhIeO4NSAghhHgDEswJIYT45FWrVggD\ngzvabUfH85QuXTLXxiOEEEK8CXnMUgghxCevU6e6REQEEBBwDkPDFDw9HTA0NMztYQkhhBCvJMGc\nEEIIAQwY0JQBA3J7FEIIIcSbk8cshRBCCCGE/MFDnAAADGJJREFUECIPkmBOCCGEEEIIIfIgCeaE\nEEIIIYQQIg+SYE4IIYQQQggh8iAJ5oQQQgghhBAiD5JgTgghhBBCCCHyIAnmhBBCCCGEECIPkmBO\nCCGEEEIIIfIgCeaEEEIIIYQQIg+SYE4IIYQQQggh8iAJ5oQQQgghhBAiD5JgTgghhBBCCCHyIAnm\nhBBCCCGEECIPkmBOCCGEEEIIIfIgCeaEEEIIIYQQIg+SYE4IIYQQQggh8iAJ5oQQQgghhBAiD5Jg\nTgghhBBCCCHyIAnmhBBCCCGEECIPkmBOCCGEEEIIIfIgCeaEEEIIIYQQIg+SYE4IIYQQQggh8iAJ\n5oQQQgghhBAiD5JgTgghhBBCCCHyIAnmhBBCCCGEECIPkmBOCCGEEEIIIfIgCeaEEEIIIYQQIg+S\nYE4IIYQQQggh8iAJ5oQQQgghhBAiD5JgTgghhBBCCCHyIAnmhBBCCCGEECIPkmBOCCGEEEIIIfIg\nCeaEEEIIIYQQIg+SYE4IIYQQQggh8iAJ5oQQQgghhBAiD5JgTgghhBBCCCHyIAnmhBBCCCGEECIP\nkmBOCCGEEEIIIfIgCeaEEEIIIYQQIg+SYE4IIYQQQggh8iAJ5oQQQgghhBAiD5JgTgghhBBCCCHy\nIAnmhBBCCCGEECIPkmBOCCGEEEIIIfIgCeaEEEIIIYQQIg+SYE6I/2/v7kKzrB8/jn98uPkF1cl0\nbJJYoJRFrA6DDkpbc2s6FM0jBTWwDkKWppAPGPYgaxAdFQpp5YFgaCFoBLpSpFYY0QSDEmQo6UzN\npzrYXNf/IBr/KP3lw4953bxeZ7vu2/GVD0Pe933NGwAASkjMAQAAlJCYAwAAKCExBwAAUEJiDgAA\noITEHAAAQAmJOQAAgBIScwAAACUk5gAAAEpIzAEAAJSQmAMAACghMQcAAFBCYg4AAKCExBwAAEAJ\niTkAAIASEnMAAAAlJOYAAABKSMwBAACUkJgDAAAoITEHAABQQmIOAACghMQcAABACYk5AACAEhJz\nAAAAJSTmAAAASkjMAQAAlJCYAwAAKCExBwAAUEJiDgAAoITEHAAAQAmJOQAAgBIScwAAACUk5gAA\nAEpIzAEAAJSQmAMAACghMQcAAFBCYg4AAKCExBwAAEAJiTkAAIASEnMAAAAlJOYAAABKSMwBAACU\nkJgDAAAoITEHAABQQmIOAACghMQcAABACYk5AACAEhJzAAAAJSTmAAAASkjMAQAAlJCYAwAAKCEx\nBwAAUEJiDgAAoITEHAAAQAmJOQAAgBIScwAAACUk5gAAAEpIzAEAAJSQmAMAACghMQcAAFBCYg4A\nAKCExBwAAEAJiTkAAIASEnMAAAAlJOYAAABKSMwBAACU0A3F3JYtW9LS0pLp06ens7Nz6PqGDRvS\n1NSU5ubmHDhw4IYPCQAAwF+Nvt4/2N3dna6uruzcuTOVSiVnz55Nkhw5ciS7d+/Orl270tfXl4UL\nF+bTTz/NyJHeBAQAALhZrruwtm7dmsWLF6dSqSRJampqkiR79+5Na2trKpVKxo8fnwkTJqSnp+fm\nnBYAAIAkNxBzvb29OXjwYObOnZv58+fn0KFDSZJTp06lvr5+6Hn19fXp6+u78ZMCAAAw5Kq3WS5c\nuDCnT5/+2/X29vYMDg7m/Pnz2bZtW3p6etLe3p69e/f+4/cZMWLEzTktAAAASf5LzG3evPmKj23d\nujVNTU1JkoaGhowcOTJnz55NXV1dTp48OfS8kydPpq6u7r8epLb2zn97ZkrIvtXNvtXLttXNvtXL\nttXNvvzpum+zbGxsTHd3d5Lk6NGjGRgYSE1NTaZOnZpdu3alv78/x44dS29vbxoaGm7agQEAALiB\n/81y9uzZWblyZWbMmJFKpZKOjo4kyaRJk9LS0pLW1taMGjUqa9eudZslAADATTaiKIpiuA8BAADA\ntfHhbwAAACUk5gAAAEpIzAEAAJTQsMZcT09P5syZk5kzZ2b27Nnp6ekZemzDhg1pampKc3NzDhw4\nMIyn5Hpt2bIlLS0tmT59ejo7O4eu27Z6bNq0KZMnT865c+eGrtm3/Do6OtLS0pK2trY8//zzuXjx\n4tBj9i2//fv3p7m5OU1NTdm4ceNwH4cbdOLEicyfPz+tra2ZPn16PvjggyTJuXPnsnDhwkybNi2L\nFi3KhQsXhvmkXK/BwcHMnDkzzz33XBLbVpMLFy5kyZIlaWlpyVNPPZXvvvvu2vcthtG8efOK/fv3\nF0VRFJ9//nkxb968oiiK4scffyza2tqK/v7+4tixY0VjY2MxODg4nEflGn355ZfFggULiv7+/qIo\niuLMmTNFUdi2mvz000/FokWLiilTphS//PJLURT2rRYHDhwY2q2zs7Po7OwsisK+1eDy5ctFY2Nj\ncezYsaK/v79oa2srjhw5MtzH4gacOnWqOHz4cFEURXHp0qWiqampOHLkSNHR0VFs3LixKIqi2LBh\nw9DPMeWzadOmYunSpcWzzz5bFEVh2yqyYsWK4sMPPyyKoigGBgaKCxcuXPO+w/rOXG1t7dArvhcv\nXhz6cPG9e/emtbU1lUol48ePz4QJE/7yrh23vq1bt2bx4sWpVCpJkpqamiS2rSbr16/P8uXL/3LN\nvtXh0UcfzciRf/zz8NBDD+XkyZNJ7FsNenp6MmHChIwfPz6VSiWtra3Zu3fvcB+LG1BbW5v7778/\nSXL77bdn4sSJ6evrS1dXV2bNmpUkmTVrVvbs2TOcx+Q6nTx5Mvv27cvTTz89dM221eHixYs5ePBg\n5syZkyQZPXp07rzzzmved1hjbtmyZeno6Mjjjz+eN954I8uWLUuSnDp1KvX19UPPq6+vT19f33Ad\nk+vQ29ubgwcPZu7cuZk/f34OHTqUxLbVYs+ePamvr8/kyZP/ct2+1Wf79u157LHHkti3GvT19WXc\nuHFDX9fV1dmwihw/fjzff/99GhoacubMmYwdOzZJMnbs2Jw5c2aYT8f1eP3117NixYqhF9iS2LZK\nHD9+PDU1NXnppZcya9asrF69Or/99ts173vdHxr+by1cuDCnT5/+2/X29vZs2bIlq1evzpNPPplP\nPvkkK1euzObNm//x+/jg8VvP1bYdHBzM+fPns23btvT09KS9vf2Kr/7a9tZ0tX03btyYTZs2DV0r\nrvJxlfa9NV1p3xdeeCFTp05NkrzzzjupVCqZMWPGFb+PfcvFXtXr119/zZIlS7Jq1arccccdf3ls\nxIgRti+hzz77LGPGjMkDDzyQr7766h+fY9vyunz5cg4fPpw1a9akoaEhr7322t9+j/nf7Ps/j7kr\nxVmSLF++PO+9916SpLm5OatXr07yxyuFf97Wk/zxFvOft2By67jatlu3bk1TU1OSpKGhISNHjszZ\ns2dtWyJX2veHH37I8ePH09bWluSPV/pnz56dbdu22bdErvbzmyQ7duzIvn378v777w9ds2/51dXV\n5cSJE0Nf27A6DAwMZMmSJWlra0tjY2OSZMyYMfn5559TW1ubU6dODf26A+Xx7bffpqurK/v27Ut/\nf38uXbqU5cuX27ZK1NfXp66uLg0NDUmSadOmZePGjRk7duw17Tust1nefffd+frrr5Mk3d3dueee\ne5IkU6dOza5du9Lf359jx46lt7d36C9KOTQ2Nqa7uztJcvTo0QwMDKSmpsa2VeDee+/NF198ka6u\nrnR1daWuri47duzI2LFj7Vsl9u/fn3fffTdvv/12/vOf/wxdt2/5Pfjgg+nt7c3x48fT39+f3bt3\n54knnhjuY3EDiqLIqlWrMnHixCxYsGDo+tSpU/PRRx8lST7++OOhyKM8li5dmn379qWrqytvvvlm\nHnnkkXR2dtq2StTW1mbcuHE5evRokuTLL7/MpEmTMmXKlGva93/+ztzVrFu3LuvWrUt/f39uu+22\nvPLKK0mSSZMmpaWlJa2trRk1alTWrl3rLeSSmT17dlauXJkZM2akUqmko6MjiW2r0f/fz77V4dVX\nX83AwEAWLVqUJHn44Yfz8ssv27cKjB49OmvWrMkzzzyT33//PXPmzMnEiROH+1jcgG+++SY7d+7M\nfffdl5kzZyb5IwIWL16c9vb2bN++PXfddVfeeuutYT4pN4ttq8eaNWvy4osvZmBgIBMmTMj69esz\nODh4TfuOKK72yy4AAADckob1NksAAACuj5gDAAAoITEHAABQQmIOAACghMQcAABACYk5AACAEhJz\nAAAAJSTmAAAASuj/AKSSWUR2kw4CAAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def plot(embeddings, labels):\n", " assert embeddings.shape[0] >= len(labels), 'More labels than embeddings'\n", " pylab.figure(figsize=(15,15)) # in inches\n", " for i, label in enumerate(labels):\n", " x, y = embeddings[i,:]\n", " pylab.scatter(x, y)\n", " pylab.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',\n", " ha='right', va='bottom')\n", " pylab.show()\n", "\n", "words = [reverse_dictionary[i] for i in range(1, num_points+1)]\n", "plot(two_d_embeddings, words)" ] }, { "cell_type": "markdown", "metadata": { "id": "QB5EFrBnpNnc" }, "source": [ "---\n", "\n", "Problem\n", "-------\n", "\n", "An alternative to skip-gram is another Word2Vec model called [CBOW](http://arxiv.org/abs/1301.3781) (Continuous Bag of Words). In the CBOW model, instead of predicting a context word from a word vector, you predict a word from the sum of all the word vectors in its context. Implement and evaluate a CBOW model trained on the text8 dataset.\n", "\n", "---" ] } ], "metadata": { "colab": { "name": "5_word2vec.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_deep_learning/6_lstm.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "D7tqLMoKF6uq" }, "source": [ "Deep Learning\n", "=============\n", "\n", "Assignment 6\n", "------------\n", "\n", "After training a skip-gram model in `5_word2vec.ipynb`, the goal of this notebook is to train a LSTM character model over [Text8](http://mattmahoney.net/dc/textdata) data." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "MvEblsgEXxrd" }, "outputs": [], "source": [ "# These are all the modules we'll be using later. Make sure you can import them\n", "# before proceeding further.\n", "from __future__ import print_function\n", "import os\n", "import numpy as np\n", "import random\n", "import string\n", "import tensorflow as tf\n", "import zipfile\n", "from six.moves import range\n", "from six.moves.urllib.request import urlretrieve" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "RJ-o3UBUFtCw" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found and verified text8.zip\n" ] } ], "source": [ "url = 'http://mattmahoney.net/dc/'\n", "\n", "def maybe_download(filename, expected_bytes):\n", " \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n", " if not os.path.exists(filename):\n", " filename, _ = urlretrieve(url + filename, filename)\n", " statinfo = os.stat(filename)\n", " if statinfo.st_size == expected_bytes:\n", " print('Found and verified %s' % filename)\n", " else:\n", " print(statinfo.st_size)\n", " raise Exception(\n", " 'Failed to verify ' + filename + '. Can you get to it with a browser?')\n", " return filename\n", "\n", "filename = maybe_download('text8.zip', 31344016)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "Mvf09fjugFU_" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Data size 100000000\n" ] } ], "source": [ "def read_data(filename):\n", " with zipfile.ZipFile(filename) as f:\n", " name = f.namelist()[0]\n", " data = tf.compat.as_str(f.read(name))\n", " return data\n", " \n", "text = read_data(filename)\n", "print('Data size %d' % len(text))" ] }, { "cell_type": "markdown", "metadata": { "id": "ga2CYACE-ghb" }, "source": [ "Create a small validation set." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "w-oBpfFG-j43" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "99999000 ons anarchists advocate social relations based upon voluntary as\n", "1000 anarchism originated as a term of abuse first used against earl\n" ] } ], "source": [ "valid_size = 1000\n", "valid_text = text[:valid_size]\n", "train_text = text[valid_size:]\n", "train_size = len(train_text)\n", "print(train_size, train_text[:64])\n", "print(valid_size, valid_text[:64])" ] }, { "cell_type": "markdown", "metadata": { "id": "Zdw6i4F8glpp" }, "source": [ "Utility functions to map characters to vocabulary IDs and back." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "gAL1EECXeZsD" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1 26 0 Unexpected character: ï\n", "0\n", "a z \n" ] } ], "source": [ "vocabulary_size = len(string.ascii_lowercase) + 1 # [a-z] + ' '\n", "first_letter = ord(string.ascii_lowercase[0])\n", "\n", "def char2id(char):\n", " if char in string.ascii_lowercase:\n", " return ord(char) - first_letter + 1\n", " elif char == ' ':\n", " return 0\n", " else:\n", " print('Unexpected character: %s' % char)\n", " return 0\n", " \n", "def id2char(dictid):\n", " if dictid > 0:\n", " return chr(dictid + first_letter - 1)\n", " else:\n", " return ' '\n", "\n", "print(char2id('a'), char2id('z'), char2id(' '), char2id('ï'))\n", "print(id2char(1), id2char(26), id2char(0))" ] }, { "cell_type": "markdown", "metadata": { "id": "lFwoyygOmWsL" }, "source": [ "Function to generate a training batch for the LSTM model." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "d9wMtjy5hCj9" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['ons anarchi', 'when milita', 'lleria arch', ' abbeys and', 'married urr', 'hel and ric', 'y and litur', 'ay opened f', 'tion from t', 'migration t', 'new york ot', 'he boeing s', 'e listed wi', 'eber has pr', 'o be made t', 'yer who rec', 'ore signifi', 'a fierce cr', ' two six ei', 'aristotle s', 'ity can be ', ' and intrac', 'tion of the', 'dy to pass ', 'f certain d', 'at it will ', 'e convince ', 'ent told hi', 'ampaign and', 'rver side s', 'ious texts ', 'o capitaliz', 'a duplicate', 'gh ann es d', 'ine january', 'ross zero t', 'cal theorie', 'ast instanc', ' dimensiona', 'most holy m', 't s support', 'u is still ', 'e oscillati', 'o eight sub', 'of italy la', 's the tower', 'klahoma pre', 'erprise lin', 'ws becomes ', 'et in a naz', 'the fabian ', 'etchy to re', ' sharman ne', 'ised empero', 'ting in pol', 'd neo latin', 'th risky ri', 'encyclopedi', 'fense the a', 'duating fro', 'treet grid ', 'ations more', 'appeal of d', 'si have mad']\n", "['ists advoca', 'ary governm', 'hes nationa', 'd monasteri', 'raca prince', 'chard baer ', 'rgical lang', 'for passeng', 'the nationa', 'took place ', 'ther well k', 'seven six s', 'ith a gloss', 'robably bee', 'to recogniz', 'ceived the ', 'icant than ', 'ritic of th', 'ight in sig', 's uncaused ', ' lost as in', 'cellular ic', 'e size of t', ' him a stic', 'drugs confu', ' take to co', ' the priest', 'im to name ', 'd barred at', 'standard fo', ' such as es', 'ze on the g', 'e of the or', 'd hiver one', 'y eight mar', 'the lead ch', 'es classica', 'ce the non ', 'al analysis', 'mormons bel', 't or at lea', ' disagreed ', 'ing system ', 'btypes base', 'anguages th', 'r commissio', 'ess one nin', 'nux suse li', ' the first ', 'zi concentr', ' society ne', 'elatively s', 'etworks sha', 'or hirohito', 'litical ini', 'n most of t', 'iskerdoo ri', 'ic overview', 'air compone', 'om acnm acc', ' centerline', 'e than any ', 'devotional ', 'de such dev']\n", "[' a']\n", "['an']\n" ] } ], "source": [ "batch_size=64\n", "num_unrollings=10\n", "\n", "class BatchGenerator(object):\n", " def __init__(self, text, batch_size, num_unrollings):\n", " self._text = text\n", " self._text_size = len(text)\n", " self._batch_size = batch_size\n", " self._num_unrollings = num_unrollings\n", " segment = self._text_size // batch_size\n", " self._cursor = [ offset * segment for offset in range(batch_size)]\n", " self._last_batch = self._next_batch()\n", " \n", " def _next_batch(self):\n", " \"\"\"Generate a single batch from the current cursor position in the data.\"\"\"\n", " batch = np.zeros(shape=(self._batch_size, vocabulary_size), dtype=np.float)\n", " for b in range(self._batch_size):\n", " batch[b, char2id(self._text[self._cursor[b]])] = 1.0\n", " self._cursor[b] = (self._cursor[b] + 1) % self._text_size\n", " return batch\n", " \n", " def next(self):\n", " \"\"\"Generate the next array of batches from the data. The array consists of\n", " the last batch of the previous array, followed by num_unrollings new ones.\n", " \"\"\"\n", " batches = [self._last_batch]\n", " for step in range(self._num_unrollings):\n", " batches.append(self._next_batch())\n", " self._last_batch = batches[-1]\n", " return batches\n", "\n", "def characters(probabilities):\n", " \"\"\"Turn a 1-hot encoding or a probability distribution over the possible\n", " characters back into its (most likely) character representation.\"\"\"\n", " return [id2char(c) for c in np.argmax(probabilities, 1)]\n", "\n", "def batches2string(batches):\n", " \"\"\"Convert a sequence of batches back into their (most likely) string\n", " representation.\"\"\"\n", " s = [''] * batches[0].shape[0]\n", " for b in batches:\n", " s = [''.join(x) for x in zip(s, characters(b))]\n", " return s\n", "\n", "train_batches = BatchGenerator(train_text, batch_size, num_unrollings)\n", "valid_batches = BatchGenerator(valid_text, 1, 1)\n", "\n", "print(batches2string(train_batches.next()))\n", "print(batches2string(train_batches.next()))\n", "print(batches2string(valid_batches.next()))\n", "print(batches2string(valid_batches.next()))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "KyVd8FxT5QBc" }, "outputs": [], "source": [ "def logprob(predictions, labels):\n", " \"\"\"Log-probability of the true labels in a predicted batch.\"\"\"\n", " predictions[predictions < 1e-10] = 1e-10\n", " return np.sum(np.multiply(labels, -np.log(predictions))) / labels.shape[0]\n", "\n", "def sample_distribution(distribution):\n", " \"\"\"Sample one element from a distribution assumed to be an array of normalized\n", " probabilities.\n", " \"\"\"\n", " r = random.uniform(0, 1)\n", " s = 0\n", " for i in range(len(distribution)):\n", " s += distribution[i]\n", " if s >= r:\n", " return i\n", " return len(distribution) - 1\n", "\n", "def sample(prediction):\n", " \"\"\"Turn a (column) prediction into 1-hot encoded samples.\"\"\"\n", " p = np.zeros(shape=[1, vocabulary_size], dtype=np.float)\n", " p[0, sample_distribution(prediction[0])] = 1.0\n", " return p\n", "\n", "def random_distribution():\n", " \"\"\"Generate a random column of probabilities.\"\"\"\n", " b = np.random.uniform(0.0, 1.0, size=[1, vocabulary_size])\n", " return b/np.sum(b, 1)[:,None]" ] }, { "cell_type": "markdown", "metadata": { "id": "K8f67YXaDr4C" }, "source": [ "Simple LSTM Model." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "Q5rxZK6RDuGe" }, "outputs": [], "source": [ "num_nodes = 64\n", "\n", "graph = tf.Graph()\n", "with graph.as_default():\n", " \n", " # Parameters:\n", " # Input gate: input, previous output, and bias.\n", " ix = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n", " im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n", " ib = tf.Variable(tf.zeros([1, num_nodes]))\n", " # Forget gate: input, previous output, and bias.\n", " fx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n", " fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n", " fb = tf.Variable(tf.zeros([1, num_nodes]))\n", " # Memory cell: input, state and bias. \n", " cx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n", " cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n", " cb = tf.Variable(tf.zeros([1, num_nodes]))\n", " # Output gate: input, previous output, and bias.\n", " ox = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n", " om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n", " ob = tf.Variable(tf.zeros([1, num_nodes]))\n", " # Variables saving state across unrollings.\n", " saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n", " saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n", " # Classifier weights and biases.\n", " w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], -0.1, 0.1))\n", " b = tf.Variable(tf.zeros([vocabulary_size]))\n", " \n", " # Definition of the cell computation.\n", " def lstm_cell(i, o, state):\n", " \"\"\"Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf\n", " Note that in this formulation, we omit the various connections between the\n", " previous state and the gates.\"\"\"\n", " input_gate = tf.sigmoid(tf.matmul(i, ix) + tf.matmul(o, im) + ib)\n", " forget_gate = tf.sigmoid(tf.matmul(i, fx) + tf.matmul(o, fm) + fb)\n", " update = tf.matmul(i, cx) + tf.matmul(o, cm) + cb\n", " state = forget_gate * state + input_gate * tf.tanh(update)\n", " output_gate = tf.sigmoid(tf.matmul(i, ox) + tf.matmul(o, om) + ob)\n", " return output_gate * tf.tanh(state), state\n", "\n", " # Input data.\n", " train_data = list()\n", " for _ in range(num_unrollings + 1):\n", " train_data.append(\n", " tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size]))\n", " train_inputs = train_data[:num_unrollings]\n", " train_labels = train_data[1:] # labels are inputs shifted by one time step.\n", "\n", " # Unrolled LSTM loop.\n", " outputs = list()\n", " output = saved_output\n", " state = saved_state\n", " for i in train_inputs:\n", " output, state = lstm_cell(i, output, state)\n", " outputs.append(output)\n", "\n", " # State saving across unrollings.\n", " with tf.control_dependencies([saved_output.assign(output),\n", " saved_state.assign(state)]):\n", " # Classifier.\n", " logits = tf.nn.xw_plus_b(tf.concat(outputs, 0), w, b)\n", " loss = tf.reduce_mean(\n", " tf.nn.softmax_cross_entropy_with_logits(\n", " labels=tf.concat(train_labels, 0), logits=logits))\n", "\n", " # Optimizer.\n", " global_step = tf.Variable(0)\n", " learning_rate = tf.train.exponential_decay(\n", " 10.0, global_step, 5000, 0.1, staircase=True)\n", " optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n", " gradients, v = zip(*optimizer.compute_gradients(loss))\n", " gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n", " optimizer = optimizer.apply_gradients(\n", " zip(gradients, v), global_step=global_step)\n", "\n", " # Predictions.\n", " train_prediction = tf.nn.softmax(logits)\n", " \n", " # Sampling and validation eval: batch 1, no unrolling.\n", " sample_input = tf.placeholder(tf.float32, shape=[1, vocabulary_size])\n", " saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]))\n", " saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]))\n", " reset_sample_state = tf.group(\n", " saved_sample_output.assign(tf.zeros([1, num_nodes])),\n", " saved_sample_state.assign(tf.zeros([1, num_nodes])))\n", " sample_output, sample_state = lstm_cell(\n", " sample_input, saved_sample_output, saved_sample_state)\n", " with tf.control_dependencies([saved_sample_output.assign(sample_output),\n", " saved_sample_state.assign(sample_state)]):\n", " sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "RD9zQCZTEaEm" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initialized\n", "Average loss at step 0 : 3.29904174805 learning rate: 10.0\n", "Minibatch perplexity: 27.09\n", "================================================================================\n", "srk dwmrnuldtbbgg tapootidtu xsciu sgokeguw hi ieicjq lq piaxhazvc s fht wjcvdlh\n", "lhrvallvbeqqquc dxd y siqvnle bzlyw nr rwhkalezo siie o deb e lpdg storq u nx o\n", "meieu nantiouie gdys qiuotblci loc hbiznauiccb cqzed acw l tsm adqxplku gn oaxet\n", "unvaouc oxchywdsjntdh zpklaejvxitsokeerloemee htphisb th eaeqseibumh aeeyj j orw\n", "ogmnictpycb whtup otnilnesxaedtekiosqet liwqarysmt arj flioiibtqekycbrrgoysj\n", "================================================================================\n", "Validation set perplexity: 19.99\n", "Average loss at step 100 : 2.59553678274 learning rate: 10.0\n", "Minibatch perplexity: 9.57\n", "Validation set perplexity: 10.60\n", "Average loss at step 200 : 2.24747137785 learning rate: 10.0\n", "Minibatch perplexity: 7.68\n", "Validation set perplexity: 8.84\n", "Average loss at step 300 : 2.09438110709 learning rate: 10.0\n", "Minibatch perplexity: 7.41\n", "Validation set perplexity: 8.13\n", "Average loss at step 400 : 1.99440989017 learning rate: 10.0\n", "Minibatch perplexity: 6.46\n", "Validation set perplexity: 7.58\n", "Average loss at step 500 : 1.9320810616 learning rate: 10.0\n", "Minibatch perplexity: 6.30\n", "Validation set perplexity: 6.88\n", "Average loss at step 600 : 1.90935629249 learning rate: 10.0\n", "Minibatch perplexity: 7.21\n", "Validation set perplexity: 6.91\n", "Average loss at step 700 : 1.85583009005 learning rate: 10.0\n", "Minibatch perplexity: 6.13\n", "Validation set perplexity: 6.60\n", "Average loss at step 800 : 1.82152368546 learning rate: 10.0\n", "Minibatch perplexity: 6.01\n", "Validation set perplexity: 6.37\n", "Average loss at step 900 : 1.83169809818 learning rate: 10.0\n", "Minibatch perplexity: 7.20\n", "Validation set perplexity: 6.23\n", "Average loss at step 1000 : 1.82217029214 learning rate: 10.0\n", "Minibatch perplexity: 6.73\n", "================================================================================\n", "le action b of the tert sy ofter selvorang previgned stischdy yocal chary the co\n", "le relganis networks partucy cetinning wilnchan sics rumeding a fulch laks oftes\n", "hian andoris ret the ecause bistory l pidect one eight five lack du that the ses\n", "aiv dromery buskocy becomer worils resism disele retery exterrationn of hide in \n", "mer miter y sught esfectur of the upission vain is werms is vul ugher compted by\n", "================================================================================\n", "Validation set perplexity: 6.07\n", "Average loss at step 1100 : 1.77301145077 learning rate: 10.0\n", "Minibatch perplexity: 6.03\n", "Validation set perplexity: 5.89\n", "Average loss at step 1200 : 1.75306463003 learning rate: 10.0\n", "Minibatch perplexity: 6.50\n", "Validation set perplexity: 5.61\n", "Average loss at step 1300 : 1.72937195778 learning rate: 10.0\n", "Minibatch perplexity: 5.00\n", "Validation set perplexity: 5.60\n", "Average loss at step 1400 : 1.74773373723 learning rate: 10.0\n", "Minibatch perplexity: 6.48\n", "Validation set perplexity: 5.66\n", "Average loss at step 1500 : 1.7368799901 learning rate: 10.0\n", "Minibatch perplexity: 5.22\n", "Validation set perplexity: 5.44\n", "Average loss at step 1600 : 1.74528762937 learning rate: 10.0\n", "Minibatch perplexity: 5.85\n", "Validation set perplexity: 5.33\n", "Average loss at step 1700 : 1.70881183743 learning rate: 10.0\n", "Minibatch perplexity: 5.33\n", "Validation set perplexity: 5.56\n", "Average loss at step 1800 : 1.67776108027 learning rate: 10.0\n", "Minibatch perplexity: 5.33\n", "Validation set perplexity: 5.29\n", "Average loss at step 1900 : 1.64935536742 learning rate: 10.0\n", "Minibatch perplexity: 5.29\n", "Validation set perplexity: 5.15\n", "Average loss at step" ] }, { "name": "stdout", "output_type": "stream", "text": [ " 2000 : 1.69528644681 learning rate: 10.0\n", "Minibatch perplexity: 5.13\n", "================================================================================\n", "vers soqually have one five landwing to docial page kagan lower with ther batern\n", "ctor son alfortmandd tethre k skin the known purated to prooust caraying the fit\n", "je in beverb is the sournction bainedy wesce tu sture artualle lines digra forme\n", "m rousively haldio ourso ond anvary was for the seven solies hild buil s to te\n", "zall for is it is one nine eight eight one neval to the kime typer oene where he\n", "================================================================================\n", "Validation set perplexity: 5.25\n", "Average loss at step 2100 : 1.68808053017 learning rate: 10.0\n", "Minibatch perplexity: 5.17\n", "Validation set perplexity: 5.01\n", "Average loss at step 2200 : 1.68322490931 learning rate: 10.0\n", "Minibatch perplexity: 5.09\n", "Validation set perplexity: 5.15\n", "Average loss at step 2300 : 1.64465074301 learning rate: 10.0\n", "Minibatch perplexity: 5.51\n", "Validation set perplexity: 5.00\n", "Average loss at step 2400 : 1.66408578038 learning rate: 10.0\n", "Minibatch perplexity: 5.86\n", "Validation set perplexity: 4.80\n", "Average loss at step 2500 : 1.68515402555 learning rate: 10.0\n", "Minibatch perplexity: 5.75\n", "Validation set perplexity: 4.82\n", "Average loss at step 2600 : 1.65405208349 learning rate: 10.0\n", "Minibatch perplexity: 5.38\n", "Validation set perplexity: 4.85\n", "Average loss at step 2700 : 1.65706222177 learning rate: 10.0\n", "Minibatch perplexity: 5.46\n", "Validation set perplexity: 4.78\n", "Average loss at step 2800 : 1.65204829812 learning rate: 10.0\n", "Minibatch perplexity: 5.06\n", "Validation set perplexity: 4.64\n", "Average loss at step 2900 : 1.65107253551 learning rate: 10.0\n", "Minibatch perplexity: 5.00\n", "Validation set perplexity: 4.61\n", "Average loss at step 3000 : 1.6495274055 learning rate: 10.0\n", "Minibatch perplexity: 4.53\n", "================================================================================\n", "ject covered in belo one six six to finsh that all di rozial sime it a the lapse\n", "ble which the pullic bocades record r to sile dric two one four nine seven six f\n", " originally ame the playa ishaps the stotchational in a p dstambly name which as\n", "ore volum to bay riwer foreal in nuily operety can and auscham frooripm however \n", "kan traogey was lacous revision the mott coupofiteditey the trando insended frop\n", "================================================================================\n", "Validation set perplexity: 4.76\n", "Average loss at step 3100 : 1.63705502152 learning rate: 10.0\n", "Minibatch perplexity: 5.50\n", "Validation set perplexity: 4.76\n", "Average loss at step 3200 : 1.64740695596 learning rate: 10.0\n", "Minibatch perplexity: 4.84\n", "Validation set perplexity: 4.67\n", "Average loss at step 3300 : 1.64711504817 learning rate: 10.0\n", "Minibatch perplexity: 5.39\n", "Validation set perplexity: 4.57\n", "Average loss at step 3400 : 1.67113256454 learning rate: 10.0\n", "Minibatch perplexity: 5.56\n", "Validation set perplexity: 4.71\n", "Average loss at step 3500 : 1.65637169957 learning rate: 10.0\n", "Minibatch perplexity: 5.03\n", "Validation set perplexity: 4.80\n", "Average loss at step 3600 : 1.66601825476 learning rate: 10.0\n", "Minibatch perplexity: 4.63\n", "Validation set perplexity: 4.52\n", "Average loss at step 3700 : 1.65021387935 learning rate: 10.0\n", "Minibatch perplexity: 5.50\n", "Validation set perplexity: 4.56\n", "Average loss at step 3800 : 1.64481814981 learning rate: 10.0\n", "Minibatch perplexity: 4.60\n", "Validation set perplexity: 4.54\n", "Average loss at step 3900 : 1.642069453 learning rate: 10.0\n", "Minibatch perplexity: 4.91\n", "Validation set perplexity: 4.54\n", "Average loss at step 4000 : 1.65179730773 learning rate: 10.0\n", "Minibatch perplexity: 4.77\n", "================================================================================\n", "k s rasbonish roctes the nignese at heacle was sito of beho anarchys and with ro\n", "jusar two sue wletaus of chistical in causations d ow trancic bruthing ha laters\n", "de and speacy pulted yoftret worksy zeatlating to eight d had to ie bue seven si" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "s fiction of the feelly constive suq flanch earlied curauking bjoventation agent\n", "quen s playing it calana our seopity also atbellisionaly comexing the revideve i\n", "================================================================================\n", "Validation set perplexity: 4.58\n", "Average loss at step 4100 : 1.63794238806 learning rate: 10.0\n", "Minibatch perplexity: 5.47\n", "Validation set perplexity: 4.79\n", "Average loss at step 4200 : 1.63822438836 learning rate: 10.0\n", "Minibatch perplexity: 5.30\n", "Validation set perplexity: 4.54\n", "Average loss at step 4300 : 1.61844664574 learning rate: 10.0\n", "Minibatch perplexity: 4.69\n", "Validation set perplexity: 4.54\n", "Average loss at step 4400 : 1.61255454302 learning rate: 10.0\n", "Minibatch perplexity: 4.67\n", "Validation set perplexity: 4.54\n", "Average loss at step 4500 : 1.61543365479 learning rate: 10.0\n", "Minibatch perplexity: 4.83\n", "Validation set perplexity: 4.69\n", "Average loss at step 4600 : 1.61607327104 learning rate: 10.0\n", "Minibatch perplexity: 5.18\n", "Validation set perplexity: 4.64\n", "Average loss at step 4700 : 1.62757282495 learning rate: 10.0\n", "Minibatch perplexity: 4.24\n", "Validation set perplexity: 4.66\n", "Average loss at step 4800 : 1.63222063541 learning rate: 10.0\n", "Minibatch perplexity: 5.30\n", "Validation set perplexity: 4.53\n", "Average loss at step 4900 : 1.63678096652 learning rate: 10.0\n", "Minibatch perplexity: 5.43\n", "Validation set perplexity: 4.64\n", "Average loss at step 5000 : 1.610340662 learning rate: 1.0\n", "Minibatch perplexity: 5.10\n", "================================================================================\n", "in b one onarbs revieds the kimiluge that fondhtic fnoto cre one nine zero zero \n", " of is it of marking panzia t had wap ironicaghni relly deah the omber b h menba\n", "ong messified it his the likdings ara subpore the a fames distaled self this int\n", "y advante authors the end languarle meit common tacing bevolitione and eight one\n", "zes that materly difild inllaring the fusts not panition assertian causecist bas\n", "================================================================================\n", "Validation set perplexity: 4.69\n", "Average loss at step 5100 : 1.60593637228 learning rate: 1.0\n", "Minibatch perplexity: 4.69\n", "Validation set perplexity: 4.47\n", "Average loss at step 5200 : 1.58993269444 learning rate: 1.0\n", "Minibatch perplexity: 4.65\n", "Validation set perplexity: 4.39\n", "Average loss at step 5300 : 1.57930587292 learning rate: 1.0\n", "Minibatch perplexity: 5.11\n", "Validation set perplexity: 4.39\n", "Average loss at step 5400 : 1.58022856832 learning rate: 1.0\n", "Minibatch perplexity: 5.19\n", "Validation set perplexity: 4.37\n", "Average loss at step 5500 : 1.56654450059 learning rate: 1.0\n", "Minibatch perplexity: 4.69\n", "Validation set perplexity: 4.33\n", "Average loss at step 5600 : 1.58013380885 learning rate: 1.0\n", "Minibatch perplexity: 5.13\n", "Validation set perplexity: 4.35\n", "Average loss at step 5700 : 1.56974959254 learning rate: 1.0\n", "Minibatch perplexity: 5.00\n", "Validation set perplexity: 4.34\n", "Average loss at step 5800 : 1.5839582932 learning rate: 1.0\n", "Minibatch perplexity: 4.88\n", "Validation set perplexity: 4.31\n", "Average loss at step 5900 : 1.57129439116 learning rate: 1.0\n", "Minibatch perplexity: 4.66\n", "Validation set perplexity: 4.32\n", "Average loss at step 6000 : 1.55144061089 learning rate: 1.0\n", "Minibatch perplexity: 4.55\n", "================================================================================\n", "utic clositical poopy stribe addi nixe one nine one zero zero eight zero b ha ex\n", "zerns b one internequiption of the secordy way anti proble akoping have fictiona\n", "phare united from has poporarly cities book ins sweden emperor a sass in origina\n", "quulk destrebinist and zeilazar and on low and by in science over country weilti\n", "x are holivia work missincis ons in the gages to starsle histon one icelanctrotu\n", "================================================================================\n", "Validation set perplexity: 4.30\n", "Average loss at step 6100 : 1.56450940847 learning rate: 1.0\n", "Minibatch perplexity: 4.77\n", "Validation set perplexity: 4.27" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "Average loss at step 6200 : 1.53433164835 learning rate: 1.0\n", "Minibatch perplexity: 4.77\n", "Validation set perplexity: 4.27\n", "Average loss at step 6300 : 1.54773445129 learning rate: 1.0\n", "Minibatch perplexity: 4.76\n", "Validation set perplexity: 4.25\n", "Average loss at step 6400 : 1.54021131516 learning rate: 1.0\n", "Minibatch perplexity: 4.56\n", "Validation set perplexity: 4.24\n", "Average loss at step 6500 : 1.56153374553 learning rate: 1.0\n", "Minibatch perplexity: 5.43\n", "Validation set perplexity: 4.27\n", "Average loss at step 6600 : 1.59556478739 learning rate: 1.0\n", "Minibatch perplexity: 4.92\n", "Validation set perplexity: 4.28\n", "Average loss at step 6700 : 1.58076951623 learning rate: 1.0\n", "Minibatch perplexity: 4.77\n", "Validation set perplexity: 4.30\n", "Average loss at step 6800 : 1.6070714438 learning rate: 1.0\n", "Minibatch perplexity: 4.98\n", "Validation set perplexity: 4.28\n", "Average loss at step 6900 : 1.58413293839 learning rate: 1.0\n", "Minibatch perplexity: 4.61\n", "Validation set perplexity: 4.29\n", "Average loss at step 7000 : 1.57905534983 learning rate: 1.0\n", "Minibatch perplexity: 5.08\n", "================================================================================\n", "jague are officiencinels ored by film voon higherise haik one nine on the iffirc\n", "oshe provision that manned treatists on smalle bodariturmeristing the girto in s\n", "kis would softwenn mustapultmine truativersakys bersyim by s of confound esc bub\n", "ry of the using one four six blain ira mannom marencies g with fextificallise re\n", " one son vit even an conderouss to person romer i a lebapter at obiding are iuse\n", "================================================================================\n", "Validation set perplexity: 4.25\n" ] } ], "source": [ "num_steps = 7001\n", "summary_frequency = 100\n", "\n", "with tf.Session(graph=graph) as session:\n", " tf.global_variables_initializer().run()\n", " print('Initialized')\n", " mean_loss = 0\n", " for step in range(num_steps):\n", " batches = train_batches.next()\n", " feed_dict = dict()\n", " for i in range(num_unrollings + 1):\n", " feed_dict[train_data[i]] = batches[i]\n", " _, l, predictions, lr = session.run(\n", " [optimizer, loss, train_prediction, learning_rate], feed_dict=feed_dict)\n", " mean_loss += l\n", " if step % summary_frequency == 0:\n", " if step > 0:\n", " mean_loss = mean_loss / summary_frequency\n", " # The mean loss is an estimate of the loss over the last few batches.\n", " print(\n", " 'Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr))\n", " mean_loss = 0\n", " labels = np.concatenate(list(batches)[1:])\n", " print('Minibatch perplexity: %.2f' % float(\n", " np.exp(logprob(predictions, labels))))\n", " if step % (summary_frequency * 10) == 0:\n", " # Generate some samples.\n", " print('=' * 80)\n", " for _ in range(5):\n", " feed = sample(random_distribution())\n", " sentence = characters(feed)[0]\n", " reset_sample_state.run()\n", " for _ in range(79):\n", " prediction = sample_prediction.eval({sample_input: feed})\n", " feed = sample(prediction)\n", " sentence += characters(feed)[0]\n", " print(sentence)\n", " print('=' * 80)\n", " # Measure validation set perplexity.\n", " reset_sample_state.run()\n", " valid_logprob = 0\n", " for _ in range(valid_size):\n", " b = valid_batches.next()\n", " predictions = sample_prediction.eval({sample_input: b[0]})\n", " valid_logprob = valid_logprob + logprob(predictions, b[1])\n", " print('Validation set perplexity: %.2f' % float(np.exp(\n", " valid_logprob / valid_size)))" ] }, { "cell_type": "markdown", "metadata": { "id": "pl4vtmFfa5nn" }, "source": [ "---\n", "Problem 1\n", "---------\n", "\n", "You might have noticed that the definition of the LSTM cell involves 4 matrix multiplications with the input, and 4 matrix multiplications with the output. Simplify the expression by using a single matrix multiply for each, and variables that are 4 times larger.\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "4eErTCTybtph" }, "source": [ "---\n", "Problem 2\n", "---------\n", "\n", "We want to train a LSTM over bigrams, that is pairs of consecutive characters like 'ab' instead of single characters like 'a'. Since the number of possible bigrams is large, feeding them directly to the LSTM using 1-hot encodings will lead to a very sparse representation that is very wasteful computationally.\n", "\n", "a- Introduce an embedding lookup on the inputs, and feed the embeddings to the LSTM cell instead of the inputs themselves.\n", "\n", "b- Write a bigram-based LSTM, modeled on the character LSTM above.\n", "\n", "c- Introduce Dropout. For best practices on how to use Dropout in LSTMs, refer to this [article](http://arxiv.org/abs/1409.2329).\n", "\n", "---" ] }, { "cell_type": "markdown", "metadata": { "id": "Y5tapX3kpcqZ" }, "source": [ "---\n", "Problem 3\n", "---------\n", "\n", "(difficult!)\n", "\n", "Write a sequence-to-sequence LSTM which mirrors all the words in a sentence. For example, if your input is:\n", "\n", " the quick brown fox\n", " \n", "the model should attempt to output:\n", "\n", " eht kciuq nworb xof\n", " \n", "Refer to the lecture on how to put together a sequence-to-sequence model, as well as [this article](http://arxiv.org/abs/1409.3215) for best practices.\n", "\n", "---" ] } ], "metadata": { "colab": { "name": "6_lstm.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_deep_learning/Dockerfile ================================================ FROM gcr.io/tensorflow/tensorflow:1.0.0 LABEL maintainer="Vincent Vanhoucke " # Pillow needs libjpeg by default as of 3.0. RUN apt-get update && apt-get install -y --no-install-recommends \ libjpeg8-dev \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* RUN pip install scikit-learn pyreadline Pillow imageio RUN rm -rf /notebooks/* ADD *.ipynb /notebooks/ WORKDIR /notebooks CMD ["/run_jupyter.sh", "--allow-root"] ================================================ FILE: courses/udacity_deep_learning/README.md ================================================ Assignments for Udacity Deep Learning class with TensorFlow =========================================================== Warning: These files DEPRECATED. see https://tensorflow.org/tutorials for modern examples of how to use tensorflow. Course information can be found at https://www.udacity.com/course/deep-learning--ud730 ## Getting Started with Docker If you are new to Docker, follow [Docker document](https://docs.docker.com/machine/get-started/) to start a docker instance. Kindly read the requirements of Windows and Mac carefully. Running the Docker container from the Google Cloud repository ------------------------------------------------------------- docker run -p 8888:8888 --name tensorflow-udacity -it gcr.io/tensorflow/udacity-assignments:1.0.0 Note that if you ever exit the container, you can return to it using: docker start -ai tensorflow-udacity Accessing the Notebooks ----------------------- On linux, go to: http://127.0.0.1:8888 On mac, go to terminal and find the virtual machine's IP using: docker-machine ip default Then go to: http://(ip address received from the above command):8888 (likely http://192.168.99.100:8888) On Windows, use powershell to find the virtual machine's IP using: docker-machine ip default Then go to: http://(ip address received from the above command):8888 (likely http://192.168.99.100:8888) FAQ --- * **I'm getting a MemoryError when loading data in the first notebook.** If you're using a Mac, Docker works by running a VM locally (which is controlled by `docker-machine`). It's quite likely that you'll need to bump up the amount of RAM allocated to the VM beyond the default (which is 1G). [This Stack Overflow question](http://stackoverflow.com/questions/32834082/how-to-increase-docker-machine-memory-mac) has two good suggestions; we recommend using 8G. In addition, you may need to pass `--memory=8g` as an extra argument to `docker run`. * **I want to create a new virtual machine instead of the default one.** `docker-machine` is a tool to provision and manage docker hosts, it supports multiple platform (ex. aws, gce, azure, virtualbox, ...). To create a new virtual machine locally with built-in docker engine, you can use docker-machine create -d virtualbox --virtualbox-memory 8196 tensorflow `-d` means the driver for the cloud platform, supported drivers listed [here](https://docs.docker.com/machine/drivers/). Here we use virtualbox to create a new virtual machine locally. `tensorflow` means the name of the virtual machine, feel free to use whatever you like. You can use docker-machine ip tensorflow to get the ip of the new virtual machine. To switch from default virtual machine to a new one (here we use tensorflow), type eval $(docker-machine env tensorflow) Note that `docker-machine env tensorflow` outputs some environment variables such like `DOCKER_HOST`. Then your docker client is now connected to the docker host in virtual machine `tensorflow` * **I'm getting a TLS connection error.** If you get an error about the TLS connection of your docker, run the command below to confirm the problem. docker-machine ip tensorflow Then if it is the case use the instructions on [this page](https://docs.docker.com/toolbox/faqs/troubleshoot/) to solve the issue. * **I'm getting the error - docker: Cannot connect to the Docker daemon. Is the docker daemon running on this host? - when I run 'docker run'.** This is a permissions issue, and a popular answer is provided for Linux and Max OSX [here](http://stackoverflow.com/questions/21871479/docker-cant-connect-to-docker-daemon) on StackOverflow. Notes for anyone needing to build their own containers (mostly instructors) =========================================================================== Building a local Docker container --------------------------------- cd tensorflow/examples/udacity docker build --pull -t $USER/assignments . Running the local container --------------------------- To run a disposable container: docker run -p 8888:8888 -it --rm $USER/assignments Note the above command will create an ephemeral container and all data stored in the container will be lost when the container stops. To avoid losing work between sessions in the container, it is recommended that you mount the `tensorflow/examples/udacity` directory into the container: docker run -p 8888:8888 -v :/notebooks -it --rm $USER/assignments This will allow you to save work and have access to generated files on the host filesystem. Pushing a Google Cloud release ------------------------------ V=1.0.0 docker tag $USER/assignments gcr.io/tensorflow/udacity-assignments:$V gcloud docker push gcr.io/tensorflow/udacity-assignments docker tag $USER/assignments gcr.io/tensorflow/udacity-assignments:latest gcloud docker push gcr.io/tensorflow/udacity-assignments History ------- * 0.1.0: Initial release. * 0.2.0: Many fixes, including lower memory footprint and support for Python 3. * 0.3.0: Use 0.7.1 release. * 0.4.0: Move notMMNIST data for Google Cloud. * 0.5.0: Actually use 0.7.1 release. * 0.6.0: Update to TF 0.10.0, add libjpeg (for Pillow). * 1.0.0: Update to TF 1.0.0 release. ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l01c01_introduction_to_colab_and_python.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "YHI3vyhv5p85" }, "source": [ "## **Introduction to Colab and Python**" ] }, { "cell_type": "markdown", "metadata": { "id": "OVi775ZJ2bsy" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "F8YVA_634OFk" }, "source": [ "Welcome to this Colab where you will get a quick introduction to the Python programming language and the environment used for the course's exercises: Colab.\n", "\n", "Colab is a Python development environment that runs in the browser using Google Cloud.\n", "\n", "For example, to print \"Hello World\", just hover the mouse over [ ] and press the play button to the upper left. Or press shift-enter to execute." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "X9uIpOS2zx7k" }, "outputs": [], "source": [ "print(\"Hello World\")" ] }, { "cell_type": "markdown", "metadata": { "id": "wwJGmDrQ0EoB" }, "source": [ "## Functions, Conditionals, and Iteration\n", "Let's create a Python function, and call it from a loop." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "pRllo2HLfXiu" }, "outputs": [], "source": [ "def HelloWorldXY(x, y):\n", " if (x < 10):\n", " print(\"Hello World, x was < 10\")\n", " elif (x < 20):\n", " print(\"Hello World, x was >= 10 but < 20\")\n", " else:\n", " print(\"Hello World, x was >= 20\")\n", " return x + y\n", "\n", "for i in range(8, 25, 5): # i=8, 13, 18, 23 (start, stop, step)\n", " print(\"--- Now running with i: {}\".format(i))\n", " r = HelloWorldXY(i,i)\n", " print(\"Result from HelloWorld: {}\".format(r))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lHNmDCh0JpVP" }, "outputs": [], "source": [ "print(HelloWorldXY(1,2))" ] }, { "cell_type": "markdown", "metadata": { "id": "kiZG7uhm8qCF" }, "source": [ "Easy, right?\n", "\n", "If you want a loop starting at 0 to 2 (exclusive) you could do any of the following" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "m8YQN1H41L-Y" }, "outputs": [], "source": [ "print(\"Iterate over the items. `range(2)` is like a list [0,1].\")\n", "for i in range(2):\n", " print(i)\n", "\n", "print(\"Iterate over an actual list.\")\n", "for i in [0,1]:\n", " print(i)\n", "\n", "print(\"While works\")\n", "i = 0\n", "while i < 2:\n", " print(i)\n", " i += 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "vIgmFZq4zszl" }, "outputs": [], "source": [ "print(\"Python supports standard key words like continue and break\")\n", "while True:\n", " print(\"Entered while\")\n", " break" ] }, { "cell_type": "markdown", "metadata": { "id": "5QyOUhFw1OUX" }, "source": [ "## Numpy and lists\n", "Python has lists built into the language.\n", "However, we will use a library called numpy for this.\n", "Numpy gives you lots of support functions that are useful when doing Machine Learning.\n", "\n", "Here, you will also see an import statement. This statement makes the entire numpy package available and we can access those symbols using the abbreviated 'np' syntax." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4Dxk4q-jzEy4" }, "outputs": [], "source": [ "import numpy as np # Make numpy available using np.\n", "\n", "# Create a numpy array, and append an element\n", "a = np.array([\"Hello\", \"World\"])\n", "a = np.append(a, \"!\")\n", "print(\"Current array: {}\".format(a))\n", "print(\"Printing each element\")\n", "for i in a:\n", " print(i)\n", "\n", "print(\"\\nPrinting each element and their index\")\n", "for i,e in enumerate(a):\n", " print(\"Index: {}, was: {}\".format(i, e))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "RTa8_9G3LV03" }, "outputs": [], "source": [ "print(\"\\nShowing some basic math on arrays\")\n", "b = np.array([0,1,4,3,2])\n", "print(\"Max: {}\".format(np.max(b)))\n", "print(\"Average: {}\".format(np.average(b)))\n", "print(\"Max index: {}\".format(np.argmax(b)))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9YaGj5n4LW7P" }, "outputs": [], "source": [ "print(\"\\nYou can print the type of anything\")\n", "print(\"Type of b: {}, type of b[0]: {}\".format(type(b), type(b[0])))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "V6ilVhi9LXn_" }, "outputs": [], "source": [ "print(\"\\nUse numpy to create a [3,3] dimension array with random number\")\n", "c = np.random.rand(3, 3)\n", "print(c)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W_Q-DkFCLYGA" }, "outputs": [], "source": [ "print(\"\\nYou can print the dimensions of arrays\")\n", "print(\"Shape of a: {}\".format(a.shape))\n", "print(\"Shape of b: {}\".format(b.shape))\n", "print(\"Shape of c: {}\".format(c.shape))\n", "print(\"...Observe, Python uses both [0,1,2] and (0,1,2) to specify lists\")" ] }, { "cell_type": "markdown", "metadata": { "id": "c-Jk4dG91dvD" }, "source": [ "## Colab Specifics" ] }, { "cell_type": "markdown", "metadata": { "id": "G0cGd8sHEmKi" }, "source": [ "Colab is a virtual machine you can access directly. To run commands at the VM's terminal, prefix the line with an exclamation point (!).\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cLkfhyzq0W2y" }, "outputs": [], "source": [ "print(\"\\nDoing $ls on filesystem\")\n", "!ls -l\n", "!pwd" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gR2WTN1cOZ1n" }, "outputs": [], "source": [ "print(\"Install numpy\") # Just for test, numpy is actually preinstalled in all Colab instances\n", "!pip install numpy" ] }, { "cell_type": "markdown", "metadata": { "id": "QuWRpQdatAIU" }, "source": [ "**Exercise**\n", "\n", "Create a code cell underneath this text cell and add code to:\n", "\n", "\n", "* List the path of the current directory (pwd)\n", "* Go to / (cd) and list the content (ls -l)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "xU-cJbMCR61P" }, "outputs": [], "source": [ "!pwd\n", "!cd /\n", "!ls -l\n", "print(\"Hello\")" ] }, { "cell_type": "markdown", "metadata": { "id": "7b5jv0ouFREV" }, "source": [ "All usage of Colab in this course is completely free or charge. Even GPU usage is provided free of charge for some hours of usage every day.\n", "\n", "**Using GPUs**\n", "* Many of the exercises in the course executes more quickly by using GPU runtime: Runtime | Change runtime type | Hardware accelerator | GPU\n", "\n", "**Some final words on Colab**\n", "* You execute each cell in order, you can edit & re-execute cells if you want\n", "* Sometimes, this could have unintended consequences. For example, if you add a dimension to an array and execute the cell multiple times, then the cells after may not work. If you encounter problem reset your environment:\n", " * Runtime -> Restart runtime... Resets your Python shell\n", " * Runtime -> Restart all runtimes... Will reset the Colab image, and get you back to a 100% clean environment\n", "* You can also clear the output in the Colab by doing: Edit -> Clear all outputs\n", "* Colabs in this course are loaded from GitHub. Save to your Google Drive if you want a copy with your code/output: File -> Save a copy in Drive...\n", "\n", "**Learn More**\n", "* Check out [this](https://www.youtube.com/watch?v=inN8seMm7UI&list=PLQY2H8rRoyvwLbzbnKJ59NkZvQAW9wLbx&index=3) episode of #CodingTensorFlow, and don't forget to subscribe to the YouTube channel ;)\n" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l01c01_introduction_to_colab_and_python.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l02c01_celsius_to_fahrenheit.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "HnKx50tv5aZD" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "IwtS_OXU5cWG" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "YHI3vyhv5p85" }, "source": [ "# The Basics: Training Your First Model" ] }, { "cell_type": "markdown", "metadata": { "id": "_wJ2E7jV5tN5" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "F8YVA_634OFk" }, "source": [ "Welcome to this Colab where you will train your first Machine Learning model!\n", "\n", "We'll try to keep things simple here, and only introduce basic concepts. Later Colabs will cover more advanced problems.\n", "\n", "The problem we will solve is to convert from Celsius to Fahrenheit, where the approximate formula is:\n", "\n", "$$ f = c \\times 1.8 + 32 $$\n", "\n", "\n", "Of course, it would be simple enough to create a conventional Python function that directly performs this calculation, but that wouldn't be machine learning.\n", "\n", "\n", "Instead, we will give TensorFlow some sample Celsius values (0, 8, 15, 22, 38) and their corresponding Fahrenheit values (32, 46, 59, 72, 100).\n", "Then, we will train a model that figures out the above formula through the training process." ] }, { "cell_type": "markdown", "metadata": { "id": "fA93WUy1zzWf" }, "source": [ "## Import dependencies\n", "\n", "First, import TensorFlow. Here, we're calling it `tf` for ease of use. We also tell it to only display errors.\n", "\n", "Next, import [NumPy](http://www.numpy.org/) as `np`. Numpy helps us to represent our data as highly performant lists." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-ZMgCvSRFqxE" }, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "y_WQEM5MGmg3" }, "outputs": [], "source": [ "import numpy as np\n", "import logging\n", "logger = tf.get_logger()\n", "logger.setLevel(logging.ERROR)" ] }, { "cell_type": "markdown", "metadata": { "id": "AC3EQFi20buB" }, "source": [ "## Set up training data\n", "\n", "As we saw before, supervised Machine Learning is all about figuring out an algorithm given a set of inputs and outputs. Since the task in this Codelab is to create a model that can give the temperature in Fahrenheit when given the degrees in Celsius, we create two lists `celsius_q` and `fahrenheit_a` that we can use to train our model." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gg4pn6aI1vms" }, "outputs": [], "source": [ "celsius_q = np.array([-40, -10, 0, 8, 15, 22, 38], dtype=float)\n", "fahrenheit_a = np.array([-40, 14, 32, 46, 59, 72, 100], dtype=float)\n", "\n", "for i,c in enumerate(celsius_q):\n", " print(\"{} degrees Celsius = {} degrees Fahrenheit\".format(c, fahrenheit_a[i]))" ] }, { "cell_type": "markdown", "metadata": { "id": "wwJGmDrQ0EoB" }, "source": [ "### Some Machine Learning terminology\n", "\n", " - **Feature** — The input(s) to our model. In this case, a single value — the degrees in Celsius.\n", "\n", " - **Labels** — The output our model predicts. In this case, a single value — the degrees in Fahrenheit.\n", "\n", " - **Example** — A pair of inputs/outputs used during training. In our case a pair of values from `celsius_q` and `fahrenheit_a` at a specific index, such as `(22,72)`.\n" ] }, { "cell_type": "markdown", "metadata": { "id": "VM7_9Klvq7MO" }, "source": [ "## Create the model\n", "\n", "Next, create the model. We will use the simplest possible model we can, a Dense network. Since the problem is straightforward, this network will require only a single layer, with a single neuron.\n", "\n", "### Build a layer\n", "\n", "We'll call the layer `l0` and create it by instantiating `tf.keras.layers.Dense` with the following configuration:\n", "\n", "* `input_shape=[1]` — This specifies that the input to this layer is a single value. That is, the shape is a one-dimensional array with one member. Since this is the first (and only) layer, that input shape is the input shape of the entire model. The single value is a floating point number, representing degrees Celsius.\n", "\n", "* `units=1` — This specifies the number of neurons in the layer. The number of neurons defines how many internal variables the layer has to try to learn how to solve the problem (more later). Since this is the final layer, it is also the size of the model's output — a single float value representing degrees Fahrenheit. (In a multi-layered network, the size and shape of the layer would need to match the `input_shape` of the next layer.)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "pRllo2HLfXiu" }, "outputs": [], "source": [ "l0 = tf.keras.layers.Dense(units=1, input_shape=[1])" ] }, { "cell_type": "markdown", "metadata": { "id": "_F00_J9duLBD" }, "source": [ "### Assemble layers into the model\n", "\n", "Once layers are defined, they need to be assembled into a model. The Sequential model definition takes a list of layers as an argument, specifying the calculation order from the input to the output.\n", "\n", "This model has just a single layer, l0." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cSp-GpLSuMRq" }, "outputs": [], "source": [ "model = tf.keras.Sequential([l0])" ] }, { "cell_type": "markdown", "metadata": { "id": "t7pfHfWxust0" }, "source": [ "**Note**\n", "\n", "You will often see the layers defined inside the model definition, rather than beforehand:\n", "\n", "```python\n", "model = tf.keras.Sequential([\n", " tf.keras.layers.Dense(units=1, input_shape=[1])\n", "])\n", "```" ] }, { "cell_type": "markdown", "metadata": { "id": "kiZG7uhm8qCF" }, "source": [ "## Compile the model, with loss and optimizer functions\n", "\n", "Before training, the model has to be compiled. When compiled for training, the model is given:\n", "\n", "- **Loss function** — A way of measuring how far off predictions are from the desired outcome. (The measured difference is called the \"loss\".)\n", "\n", "- **Optimizer function** — A way of adjusting internal values in order to reduce the loss.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "m8YQN1H41L-Y" }, "outputs": [], "source": [ "model.compile(loss='mean_squared_error',\n", " optimizer=tf.keras.optimizers.Adam(0.1))" ] }, { "cell_type": "markdown", "metadata": { "id": "17M3Pqv4P52R" }, "source": [ "These are used during training (`model.fit()`, below) to first calculate the loss at each point, and then improve it. In fact, the act of calculating the current loss of a model and then improving it is precisely what training is.\n", "\n", "During training, the optimizer function is used to calculate adjustments to the model's internal variables. The goal is to adjust the internal variables until the model (which is really a math function) mirrors the actual equation for converting Celsius to Fahrenheit.\n", "\n", "TensorFlow uses numerical analysis to perform this tuning, and all this complexity is hidden from you so we will not go into the details here. What is useful to know about these parameters are:\n", "\n", "The loss function ([mean squared error](https://en.wikipedia.org/wiki/Mean_squared_error)) and the optimizer ([Adam](https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/)) used here are standard for simple models like this one, but many others are available. It is not important to know how these specific functions work at this point.\n", "\n", "One part of the Optimizer you may need to think about when building your own models is the learning rate (`0.1` in the code above). This is the step size taken when adjusting values in the model. If the value is too small, it will take too many iterations to train the model. Too large, and accuracy goes down. Finding a good value often involves some trial and error, but the range is usually within 0.001 (default), and 0.1" ] }, { "cell_type": "markdown", "metadata": { "id": "c-Jk4dG91dvD" }, "source": [ "## Train the model\n", "\n", "Train the model by calling the `fit` method.\n", "\n", "During training, the model takes in Celsius values, performs a calculation using the current internal variables (called \"weights\") and outputs values which are meant to be the Fahrenheit equivalent. Since the weights are initially set randomly, the output will not be close to the correct value. The difference between the actual output and the desired output is calculated using the loss function, and the optimizer function directs how the weights should be adjusted.\n", "\n", "This cycle of calculate, compare, adjust is controlled by the `fit` method. The first argument is the inputs, the second argument is the desired outputs. The `epochs` argument specifies how many times this cycle should be run, and the `verbose` argument controls how much output the method produces." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lpRrl7WK10Pq" }, "outputs": [], "source": [ "history = model.fit(celsius_q, fahrenheit_a, epochs=500, verbose=False)\n", "print(\"Finished training the model\")" ] }, { "cell_type": "markdown", "metadata": { "id": "GFcIU2-SdCrI" }, "source": [ "In later videos, we will go into more detail on what actually happens here and how a Dense layer actually works internally." ] }, { "cell_type": "markdown", "metadata": { "id": "0-QsNCLD4MJZ" }, "source": [ "## Display training statistics\n", "\n", "The `fit` method returns a history object. We can use this object to plot how the loss of our model goes down after each training epoch. A high loss means that the Fahrenheit degrees the model predicts is far from the corresponding value in `fahrenheit_a`.\n", "\n", "We'll use [Matplotlib](https://matplotlib.org/) to visualize this (you could use another tool). As you can see, our model improves very quickly at first, and then has a steady, slow improvement until it is very near \"perfect\" towards the end.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "IeK6BzfbdO6_" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "plt.xlabel('Epoch Number')\n", "plt.ylabel(\"Loss Magnitude\")\n", "plt.plot(history.history['loss'])" ] }, { "cell_type": "markdown", "metadata": { "id": "LtQGDMob5LOD" }, "source": [ "## Use the model to predict values\n", "\n", "Now you have a model that has been trained to learn the relationship between `celsius_q` and `fahrenheit_a`. You can use the predict method to have it calculate the Fahrenheit degrees for a previously unknown Celsius degrees.\n", "\n", "So, for example, if the Celsius value is 100, what do you think the Fahrenheit result will be? Take a guess before you run this code." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oxNzL4lS2Gui" }, "outputs": [], "source": [ "print(model.predict([100.0]))" ] }, { "cell_type": "markdown", "metadata": { "id": "jApk6tZ1fBg1" }, "source": [ "The correct answer is $100 \\times 1.8 + 32 = 212$, so our model is doing really well.\n", "\n", "### To review\n", "\n", "\n", "* We created a model with a Dense layer\n", "* We trained it with 3500 examples (7 pairs, over 500 epochs).\n", "\n", "Our model tuned the variables (weights) in the Dense layer until it was able to return the correct Fahrenheit value for any Celsius value. (Remember, 100 Celsius was not part of our training data.)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "zRrOky5gm20Z" }, "source": [ "## Looking at the layer weights\n", "\n", "Finally, let's print the internal variables of the Dense layer. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kmIkVdkbnZJI" }, "outputs": [], "source": [ "print(\"These are the layer variables: {}\".format(l0.get_weights()))" ] }, { "cell_type": "markdown", "metadata": { "id": "RSplSnMvnWC-" }, "source": [ "The first variable is close to ~1.8 and the second to ~32. These values (1.8 and 32) are the actual variables in the real conversion formula.\n", "\n", "This is really close to the values in the conversion formula. We'll explain this in an upcoming video where we show how a Dense layer works, but for a single neuron with a single input and a single output, the internal math looks the same as [the equation for a line](https://en.wikipedia.org/wiki/Linear_equation#Slope%E2%80%93intercept_form), $y = mx + b$, which has the same form as the conversion equation, $f = 1.8c + 32$.\n", "\n", "Since the form is the same, the variables should converge on the standard values of 1.8 and 32, which is exactly what happened.\n", "\n", "With additional neurons, additional inputs, and additional outputs, the formula becomes much more complex, but the idea is the same.\n", "\n", "### A little experiment\n", "\n", "Just for fun, what if we created more Dense layers with different units, which therefore also has more variables?" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Y2zTA-rDS5Xk" }, "outputs": [], "source": [ "l0 = tf.keras.layers.Dense(units=4, input_shape=[1])\n", "l1 = tf.keras.layers.Dense(units=4)\n", "l2 = tf.keras.layers.Dense(units=1)\n", "model = tf.keras.Sequential([l0, l1, l2])\n", "model.compile(loss='mean_squared_error', optimizer=tf.keras.optimizers.Adam(0.1))\n", "model.fit(celsius_q, fahrenheit_a, epochs=500, verbose=False)\n", "print(\"Finished training the model\")\n", "print(model.predict([100.0]))\n", "print(\"Model predicts that 100 degrees Celsius is: {} degrees Fahrenheit\".format(model.predict([100.0])))\n", "print(\"These are the l0 variables: {}\".format(l0.get_weights()))\n", "print(\"These are the l1 variables: {}\".format(l1.get_weights()))\n", "print(\"These are the l2 variables: {}\".format(l2.get_weights()))" ] }, { "cell_type": "markdown", "metadata": { "id": "xrpFFlgYhCty" }, "source": [ "As you can see, this model is also able to predict the corresponding Fahrenheit value really well. But when you look at the variables (weights) in the `l0` and `l1` layers, they are nothing even close to ~1.8 and ~32. The added complexity hides the \"simple\" form of the conversion equation.\n", "\n", "Stay tuned for the upcoming video on how Dense layers work for the explanation." ] } ], "metadata": { "colab": { "collapsed_sections": [], "name": "l02c01_celsius_to_fahrenheit.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l03c01_classifying_images_of_clothing.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "vasWnqRgy1H4" }, "outputs": [], "source": [ "#@title MIT License\n", "#\n", "# Copyright (c) 2017 François Chollet\n", "#\n", "# Permission is hereby granted, free of charge, to any person obtaining a\n", "# copy of this software and associated documentation files (the \"Software\"),\n", "# to deal in the Software without restriction, including without limitation\n", "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", "# and/or sell copies of the Software, and to permit persons to whom the\n", "# Software is furnished to do so, subject to the following conditions:\n", "#\n", "# The above copyright notice and this permission notice shall be included in\n", "# all copies or substantial portions of the Software.\n", "#\n", "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", "# DEALINGS IN THE SOFTWARE." ] }, { "cell_type": "markdown", "metadata": { "id": "jYysdyb-CaWM" }, "source": [ "# Classifying Images of Clothing" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "FbVhjPpzn6BM" }, "source": [ "In this tutorial, we'll build and train a neural network to classify images of clothing, like sneakers and shirts.\n", "\n", "It's okay if you don't understand everything. This is a fast-paced overview of a complete TensorFlow program, with explanations along the way. The goal is to get the general sense of a TensorFlow project, not to catch every detail.\n", "\n", "This guide uses [tf.keras](https://www.tensorflow.org/guide/keras), a high-level API to build and train models in TensorFlow." ] }, { "cell_type": "markdown", "metadata": { "id": "H0tMfX2vR0uD" }, "source": [ "## Install and import dependencies\n", "\n", "We'll need [TensorFlow Datasets](https://www.tensorflow.org/datasets/), an API that simplifies downloading and accessing datasets, and provides several sample datasets to work with. We're also using a few helper libraries." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "P7mUJVqcINSM" }, "outputs": [], "source": [ "!pip install -U tensorflow_datasets" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "_FxXYSCXGQqQ" }, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1UbK0Uq7GWaO" }, "outputs": [], "source": [ "# Import TensorFlow Datasets\n", "import tensorflow_datasets as tfds\n", "tfds.disable_progress_bar()\n", "\n", "# Helper libraries\n", "import math\n", "import numpy as np\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "590z76KRGtKk" }, "outputs": [], "source": [ "import logging\n", "logger = tf.get_logger()\n", "logger.setLevel(logging.ERROR)" ] }, { "cell_type": "markdown", "metadata": { "id": "yR0EdgrLCaWR" }, "source": [ "## Import the Fashion MNIST dataset" ] }, { "cell_type": "markdown", "metadata": { "id": "DLdCchMdCaWQ" }, "source": [ "This guide uses the [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) dataset, which contains 70,000 grayscale images in 10 categories. The images show individual articles of clothing at low resolution (28 $\\times$ 28 pixels), as seen here:\n", "\n", "\n", " \n", " \n", "
\n", " \"Fashion\n", "
\n", " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
 \n", "
\n", "\n", "Fashion MNIST is intended as a drop-in replacement for the classic [MNIST](http://yann.lecun.com/exdb/mnist/) dataset—often used as the \"Hello, World\" of machine learning programs for computer vision. The MNIST dataset contains images of handwritten digits (0, 1, 2, etc) in an identical format to the articles of clothing we'll use here.\n", "\n", "This guide uses Fashion MNIST for variety, and because it's a slightly more challenging problem than regular MNIST. Both datasets are relatively small and are used to verify that an algorithm works as expected. They're good starting points to test and debug code.\n", "\n", "We will use 60,000 images to train the network and 10,000 images to evaluate how accurately the network learned to classify images. You can access the Fashion MNIST directly from TensorFlow, using the [Datasets](https://www.tensorflow.org/datasets) API:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "7MqDQO0KCaWS" }, "outputs": [], "source": [ "dataset, metadata = tfds.load('fashion_mnist', as_supervised=True, with_info=True)\n", "train_dataset, test_dataset = dataset['train'], dataset['test']" ] }, { "cell_type": "markdown", "metadata": { "id": "t9FDsUlxCaWW" }, "source": [ "Loading the dataset returns metadata as well as a *training dataset* and *test dataset*.\n", "\n", "* The model is trained using `train_dataset`.\n", "* The model is tested against `test_dataset`.\n", "\n", "The images are 28 $\\times$ 28 arrays, with pixel values in the range `[0, 255]`. The *labels* are an array of integers, in the range `[0, 9]`. These correspond to the *class* of clothing the image represents:\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LabelClass
0T-shirt/top
1Trouser
2Pullover
3Dress
4Coat
5Sandal
6Shirt
7Sneaker
8Bag
9Ankle boot
\n", "\n", "Each image is mapped to a single label. Since the *class names* are not included with the dataset, store them here to use later when plotting the images:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "IjnLH5S2CaWx" }, "outputs": [], "source": [ "class_names = metadata.features['label'].names\n", "print(\"Class names: {}\".format(class_names))" ] }, { "cell_type": "markdown", "metadata": { "id": "Brm0b_KACaWX" }, "source": [ "### Explore the data\n", "\n", "Let's explore the format of the dataset before training the model. The following shows there are 60,000 images in the training set, and 10000 images in the test set:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "MaOTZxFzi48X" }, "outputs": [], "source": [ "num_train_examples = metadata.splits['train'].num_examples\n", "num_test_examples = metadata.splits['test'].num_examples\n", "print(\"Number of training examples: {}\".format(num_train_examples))\n", "print(\"Number of test examples: {}\".format(num_test_examples))" ] }, { "cell_type": "markdown", "metadata": { "id": "ES6uQoLKCaWr" }, "source": [ "## Preprocess the data\n", "\n", "The value of each pixel in the image data is an integer in the range `[0,255]`. For the model to work properly, these values need to be normalized to the range `[0,1]`. So here we create a normalization function, and then apply it to each image in the test and train datasets." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nAsH3Zm-76pB" }, "outputs": [], "source": [ "def normalize(images, labels):\n", " images = tf.cast(images, tf.float32)\n", " images /= 255\n", " return images, labels\n", "\n", "# The map function applies the normalize function to each element in the train\n", "# and test datasets\n", "train_dataset = train_dataset.map(normalize)\n", "test_dataset = test_dataset.map(normalize)\n", "\n", "# The first time you use the dataset, the images will be loaded from disk\n", "# Caching will keep them in memory, making training faster\n", "train_dataset = train_dataset.cache()\n", "test_dataset = test_dataset.cache()" ] }, { "cell_type": "markdown", "metadata": { "id": "lIQbEiJGXM-q" }, "source": [ "### Explore the processed data\n", "\n", "Let's plot an image to see what it looks like." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oSzE9l7PjHx0" }, "outputs": [], "source": [ "# Take a single image, and remove the color dimension by reshaping\n", "for image, label in test_dataset.take(1):\n", " break\n", "image = image.numpy().reshape((28,28))\n", "\n", "# Plot the image - voila a piece of fashion clothing\n", "plt.figure()\n", "plt.imshow(image, cmap=plt.cm.binary)\n", "plt.colorbar()\n", "plt.grid(False)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "Ee638AlnCaWz" }, "source": [ "Display the first 25 images from the *training set* and display the class name below each image. Verify that the data is in the correct format and we're ready to build and train the network." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oZTImqg_CaW1" }, "outputs": [], "source": [ "plt.figure(figsize=(10,10))\n", "for i, (image, label) in enumerate(train_dataset.take(25)):\n", " image = image.numpy().reshape((28,28))\n", " plt.subplot(5,5,i+1)\n", " plt.xticks([])\n", " plt.yticks([])\n", " plt.grid(False)\n", " plt.imshow(image, cmap=plt.cm.binary)\n", " plt.xlabel(class_names[label])\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "59veuiEZCaW4" }, "source": [ "## Build the model\n", "\n", "Building the neural network requires configuring the layers of the model, then compiling the model." ] }, { "cell_type": "markdown", "metadata": { "id": "Gxg1XGm0eOBy" }, "source": [ "### Setup the layers\n", "\n", "The basic building block of a neural network is the *layer*. A layer extracts a representation from the data fed into it. Hopefully, a series of connected layers results in a representation that is meaningful for the problem at hand.\n", "\n", "Much of deep learning consists of chaining together simple layers. Most layers, like `tf.keras.layers.Dense`, have internal parameters which are adjusted (\"learned\") during training." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9ODch-OFCaW4" }, "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " tf.keras.layers.Flatten(input_shape=(28, 28, 1)),\n", " tf.keras.layers.Dense(128, activation=tf.nn.relu),\n", " tf.keras.layers.Dense(10, activation=tf.nn.softmax)\n", "])" ] }, { "cell_type": "markdown", "metadata": { "id": "gut8A_7rCaW6" }, "source": [ "This network has three layers:\n", "\n", "* **input** `tf.keras.layers.Flatten` — This layer transforms the images from a 2d-array of 28 $\\times$ 28 pixels, to a 1d-array of 784 pixels (28\\*28). Think of this layer as unstacking rows of pixels in the image and lining them up. This layer has no parameters to learn, as it only reformats the data.\n", "\n", "* **\"hidden\"** `tf.keras.layers.Dense`— A densely connected layer of 128 neurons. Each neuron (or node) takes input from all 784 nodes in the previous layer, weighting that input according to hidden parameters which will be learned during training, and outputs a single value to the next layer.\n", "\n", "* **output** `tf.keras.layers.Dense` — A 128-neuron, followed by 10-node *softmax* layer. Each node represents a class of clothing. As in the previous layer, the final layer takes input from the 128 nodes in the layer before it, and outputs a value in the range `[0, 1]`, representing the probability that the image belongs to that class. The sum of all 10 node values is 1.\n", "\n", "> Note: Using `softmax` activation and `SparseCategoricalCrossentropy()` has issues and which are patched by the `tf.keras` model. A safer approach, in general, is to use a linear output (no activation function) with `SparseCategoricalCrossentropy(from_logits=True)`.\n", "\n", "\n", "### Compile the model\n", "\n", "Before the model is ready for training, it needs a few more settings. These are added during the model's *compile* step:\n", "\n", "\n", "* *Loss function* — An algorithm for measuring how far the model's outputs are from the desired output. The goal of training is this measures loss.\n", "* *Optimizer* —An algorithm for adjusting the inner parameters of the model in order to minimize loss.\n", "* *Metrics* —Used to monitor the training and testing steps. The following example uses *accuracy*, the fraction of the images that are correctly classified." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Lhan11blCaW7" }, "outputs": [], "source": [ "model.compile(optimizer='adam',\n", " loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { "id": "qKF6uW-BCaW-" }, "source": [ "## Train the model\n", "\n", "First, we define the iteration behavior for the train dataset:\n", "1. Repeat forever by specifying `dataset.repeat()` (the `epochs` parameter described below limits how long we perform training).\n", "2. The `dataset.shuffle(60000)` randomizes the order so our model cannot learn anything from the order of the examples.\n", "3. And `dataset.batch(32)` tells `model.fit` to use batches of 32 images and labels when updating the model variables.\n", "\n", "Training is performed by calling the `model.fit` method:\n", "1. Feed the training data to the model using `train_dataset`.\n", "2. The model learns to associate images and labels.\n", "3. The `epochs=5` parameter limits training to 5 full iterations of the training dataset, so a total of 5 * 60000 = 300000 examples.\n", "\n", "(Don't worry about `steps_per_epoch`, the requirement to have this flag will soon be removed.)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "o_Dp8971McQ1" }, "outputs": [], "source": [ "BATCH_SIZE = 32\n", "train_dataset = train_dataset.cache().repeat().shuffle(num_train_examples).batch(BATCH_SIZE)\n", "test_dataset = test_dataset.cache().batch(BATCH_SIZE)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "xvwvpA64CaW_" }, "outputs": [], "source": [ "model.fit(train_dataset, epochs=5, steps_per_epoch=math.ceil(num_train_examples/BATCH_SIZE))" ] }, { "cell_type": "markdown", "metadata": { "id": "W3ZVOhugCaXA" }, "source": [ "As the model trains, the loss and accuracy metrics are displayed. This model reaches an accuracy of about 0.88 (or 88%) on the training data." ] }, { "cell_type": "markdown", "metadata": { "id": "oEw4bZgGCaXB" }, "source": [ "## Evaluate accuracy\n", "\n", "Next, compare how the model performs on the test dataset. Use all examples we have in the test dataset to assess accuracy." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "VflXLEeECaXC" }, "outputs": [], "source": [ "test_loss, test_accuracy = model.evaluate(test_dataset, steps=math.ceil(num_test_examples/32))\n", "print('Accuracy on test dataset:', test_accuracy)" ] }, { "cell_type": "markdown", "metadata": { "id": "yWfgsmVXCaXG" }, "source": [ "As it turns out, the accuracy on the test dataset is smaller than the accuracy on the training dataset. This is completely normal, since the model was trained on the `train_dataset`. When the model sees images it has never seen during training, (that is, from the `test_dataset`), we can expect performance to go down. " ] }, { "cell_type": "markdown", "metadata": { "id": "xsoS7CPDCaXH" }, "source": [ "## Make predictions and explore\n", "\n", "With the model trained, we can use it to make predictions about some images." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Ccoz4conNCpl" }, "outputs": [], "source": [ "for test_images, test_labels in test_dataset.take(1):\n", " test_images = test_images.numpy()\n", " test_labels = test_labels.numpy()\n", " predictions = model.predict(test_images)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Gl91RPhdCaXI" }, "outputs": [], "source": [ "predictions.shape\n" ] }, { "cell_type": "markdown", "metadata": { "id": "x9Kk1voUCaXJ" }, "source": [ "Here, the model has predicted the label for each image in the testing set. Let's take a look at the first prediction:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3DmJEUinCaXK" }, "outputs": [], "source": [ "predictions[0]" ] }, { "cell_type": "markdown", "metadata": { "id": "-hw1hgeSCaXN" }, "source": [ "A prediction is an array of 10 numbers. These describe the \"confidence\" of the model that the image corresponds to each of the 10 different articles of clothing. We can see which label has the highest confidence value:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "qsqenuPnCaXO" }, "outputs": [], "source": [ "np.argmax(predictions[0])" ] }, { "cell_type": "markdown", "metadata": { "id": "E51yS7iCCaXO" }, "source": [ "So the model is most confident that this image is a shirt, or `class_names[6]`. And we can check the test label to see this is correct:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Sd7Pgsu6CaXP" }, "outputs": [], "source": [ "test_labels[0]" ] }, { "cell_type": "markdown", "metadata": { "id": "ygh2yYC972ne" }, "source": [ "We can graph this to look at the full set of 10 class predictions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DvYmmrpIy6Y1" }, "outputs": [], "source": [ "def plot_image(i, predictions_array, true_labels, images):\n", " predictions_array, true_label, img = predictions_array[i], true_labels[i], images[i]\n", " plt.grid(False)\n", " plt.xticks([])\n", " plt.yticks([])\n", " \n", " plt.imshow(img[...,0], cmap=plt.cm.binary)\n", "\n", " predicted_label = np.argmax(predictions_array)\n", " if predicted_label == true_label:\n", " color = 'blue'\n", " else:\n", " color = 'red'\n", " \n", " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", " 100*np.max(predictions_array),\n", " class_names[true_label]),\n", " color=color)\n", "\n", "def plot_value_array(i, predictions_array, true_label):\n", " predictions_array, true_label = predictions_array[i], true_label[i]\n", " plt.grid(False)\n", " plt.xticks([])\n", " plt.yticks([])\n", " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", " plt.ylim([0, 1]) \n", " predicted_label = np.argmax(predictions_array)\n", " \n", " thisplot[predicted_label].set_color('red')\n", " thisplot[true_label].set_color('blue')" ] }, { "cell_type": "markdown", "metadata": { "id": "d4Ov9OFDMmOD" }, "source": [ "Let's look at the 0th image, predictions, and prediction array. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "HV5jw-5HwSmO" }, "outputs": [], "source": [ "i = 0\n", "plt.figure(figsize=(6,3))\n", "plt.subplot(1,2,1)\n", "plot_image(i, predictions, test_labels, test_images)\n", "plt.subplot(1,2,2)\n", "plot_value_array(i, predictions, test_labels)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Ko-uzOufSCSe" }, "outputs": [], "source": [ "i = 12\n", "plt.figure(figsize=(6,3))\n", "plt.subplot(1,2,1)\n", "plot_image(i, predictions, test_labels, test_images)\n", "plt.subplot(1,2,2)\n", "plot_value_array(i, predictions, test_labels)" ] }, { "cell_type": "markdown", "metadata": { "id": "kgdvGD52CaXR" }, "source": [ "Let's plot several images with their predictions. Correct prediction labels are blue and incorrect prediction labels are red. The number gives the percent (out of 100) for the predicted label. Note that it can be wrong even when very confident. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "hQlnbqaw2Qu_" }, "outputs": [], "source": [ "# Plot the first X test images, their predicted label, and the true label\n", "# Color correct predictions in blue, incorrect predictions in red\n", "num_rows = 5\n", "num_cols = 3\n", "num_images = num_rows*num_cols\n", "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", "for i in range(num_images):\n", " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", " plot_image(i, predictions, test_labels, test_images)\n", " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", " plot_value_array(i, predictions, test_labels)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "R32zteKHCaXT" }, "source": [ "Finally, use the trained model to make a prediction about a single image. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "yRJ7JU7JCaXT" }, "outputs": [], "source": [ "# Grab an image from the test dataset\n", "img = test_images[0]\n", "\n", "print(img.shape)" ] }, { "cell_type": "markdown", "metadata": { "id": "vz3bVp21CaXV" }, "source": [ "`tf.keras` models are optimized to make predictions on a *batch*, or collection, of examples at once. So even though we're using a single image, we need to add it to a list:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lDFh5yF_CaXW" }, "outputs": [], "source": [ "# Add the image to a batch where it's the only member.\n", "img = np.array([img])\n", "\n", "print(img.shape)" ] }, { "cell_type": "markdown", "metadata": { "id": "EQ5wLTkcCaXY" }, "source": [ "Now predict the image:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "o_rzNSdrCaXY" }, "outputs": [], "source": [ "predictions_single = model.predict(img)\n", "\n", "print(predictions_single)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6Ai-cpLjO-3A" }, "outputs": [], "source": [ "plot_value_array(0, predictions_single, test_labels)\n", "_ = plt.xticks(range(10), class_names, rotation=45)" ] }, { "cell_type": "markdown", "metadata": { "id": "cU1Y2OAMCaXb" }, "source": [ "`model.predict` returns a list of lists, one for each image in the batch of data. Grab the predictions for our (only) image in the batch:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2tRmdq_8CaXb" }, "outputs": [], "source": [ "np.argmax(predictions_single[0])" ] }, { "cell_type": "markdown", "metadata": { "id": "YFc2HbEVCaXd" }, "source": [ "And, as before, the model predicts a label of 6 (shirt)." ] }, { "cell_type": "markdown", "metadata": { "id": "-KtnHECKZni_" }, "source": [ "# Exercises\n", "\n", "Experiment with different models and see how the accuracy results differ. In particular change the following parameters:\n", "* Set training epochs set to 1\n", "* Number of neurons in the Dense layer following the Flatten one. For example, go really low (e.g. 10) in ranges up to 512 and see how accuracy changes\n", "* Add additional Dense layers between the Flatten and the final `Dense(10)`, experiment with different units in these layers\n", "* Don't normalize the pixel values, and see the effect that has\n", "\n", "\n", "Remember to enable GPU to make everything run faster (Runtime -> Change runtime type -> Hardware accelerator -> GPU).\n", "Also, if you run into trouble, simply reset the entire environment and start from the beginning:\n", "* Edit -> Clear all outputs\n", "* Runtime -> Reset all runtimes" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l03c01_classifying_images_of_clothing.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l04c01_image_classification_with_cnns.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "6uQP3ZbC8J5o" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "vasWnqRgy1H4" }, "outputs": [], "source": [ "#@title MIT License\n", "#\n", "# Copyright (c) 2017 François Chollet\n", "#\n", "# Permission is hereby granted, free of charge, to any person obtaining a\n", "# copy of this software and associated documentation files (the \"Software\"),\n", "# to deal in the Software without restriction, including without limitation\n", "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", "# and/or sell copies of the Software, and to permit persons to whom the\n", "# Software is furnished to do so, subject to the following conditions:\n", "#\n", "# The above copyright notice and this permission notice shall be included in\n", "# all copies or substantial portions of the Software.\n", "#\n", "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", "# DEALINGS IN THE SOFTWARE." ] }, { "cell_type": "markdown", "metadata": { "id": "jYysdyb-CaWM" }, "source": [ "# Image Classification with Convolutional Neural Networks" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "FbVhjPpzn6BM" }, "source": [ "In this tutorial, we'll build and train a neural network to classify images of clothing, like sneakers and shirts.\n", "\n", "It's okay if you don't understand everything. This is a fast-paced overview of a complete TensorFlow program, with explanations along the way. The goal is to get the general sense of a TensorFlow project, not to catch every detail.\n", "\n", "This guide uses [tf.keras](https://www.tensorflow.org/guide/keras), a high-level API to build and train models in TensorFlow." ] }, { "cell_type": "markdown", "metadata": { "id": "H0tMfX2vR0uD" }, "source": [ "## Install and import dependencies\n", "\n", "We'll need [TensorFlow Datasets](https://www.tensorflow.org/datasets/), an API that simplifies downloading and accessing datasets, and provides several sample datasets to work with. We're also using a few helper libraries." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5HDhfftMGc_i" }, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "uusvhUp9Gg37" }, "outputs": [], "source": [ "# Import TensorFlow Datasets\n", "import tensorflow_datasets as tfds\n", "tfds.disable_progress_bar()\n", "\n", "# Helper libraries\n", "import math\n", "import numpy as np\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "UXZ44qIaG0Ru" }, "outputs": [], "source": [ "import logging\n", "logger = tf.get_logger()\n", "logger.setLevel(logging.ERROR)" ] }, { "cell_type": "markdown", "metadata": { "id": "yR0EdgrLCaWR" }, "source": [ "## Import the Fashion MNIST dataset" ] }, { "cell_type": "markdown", "metadata": { "id": "DLdCchMdCaWQ" }, "source": [ "This guide uses the [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) dataset, which contains 70,000 grayscale images in 10 categories. The images show individual articles of clothing at low resolution (28 $\\times$ 28 pixels), as seen here:\n", "\n", "\n", " \n", " \n", "
\n", " \"Fashion\n", "
\n", " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
 \n", "
\n", "\n", "Fashion MNIST is intended as a drop-in replacement for the classic [MNIST](http://yann.lecun.com/exdb/mnist/) dataset—often used as the \"Hello, World\" of machine learning programs for computer vision. The MNIST dataset contains images of handwritten digits (0, 1, 2, etc) in an identical format to the articles of clothing we'll use here.\n", "\n", "This guide uses Fashion MNIST for variety, and because it's a slightly more challenging problem than regular MNIST. Both datasets are relatively small and are used to verify that an algorithm works as expected. They're good starting points to test and debug code.\n", "\n", "We will use 60,000 images to train the network and 10,000 images to evaluate how accurately the network learned to classify images. You can access the Fashion MNIST directly from TensorFlow, using the [Datasets](https://www.tensorflow.org/datasets) API:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "7MqDQO0KCaWS" }, "outputs": [], "source": [ "dataset, metadata = tfds.load('fashion_mnist', as_supervised=True, with_info=True)\n", "train_dataset, test_dataset = dataset['train'], dataset['test']" ] }, { "cell_type": "markdown", "metadata": { "id": "t9FDsUlxCaWW" }, "source": [ "Loading the dataset returns metadata as well as a *training dataset* and *test dataset*.\n", "\n", "* The model is trained using `train_dataset`.\n", "* The model is tested against `test_dataset`.\n", "\n", "The images are 28 $\\times$ 28 arrays, with pixel values in the range `[0, 255]`. The *labels* are an array of integers, in the range `[0, 9]`. These correspond to the *class* of clothing the image represents:\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
LabelClass
0T-shirt/top
1Trouser
2Pullover
3Dress
4Coat
5Sandal
6Shirt
7Sneaker
8Bag
9Ankle boot
\n", "\n", "Each image is mapped to a single label. Since the *class names* are not included with the dataset, store them here to use later when plotting the images:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "IjnLH5S2CaWx" }, "outputs": [], "source": [ "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" ] }, { "cell_type": "markdown", "metadata": { "id": "Brm0b_KACaWX" }, "source": [ "### Explore the data\n", "\n", "Let's explore the format of the dataset before training the model. The following shows there are 60,000 images in the training set, and 10000 images in the test set:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "MaOTZxFzi48X" }, "outputs": [], "source": [ "num_train_examples = metadata.splits['train'].num_examples\n", "num_test_examples = metadata.splits['test'].num_examples\n", "print(\"Number of training examples: {}\".format(num_train_examples))\n", "print(\"Number of test examples: {}\".format(num_test_examples))" ] }, { "cell_type": "markdown", "metadata": { "id": "ES6uQoLKCaWr" }, "source": [ "## Preprocess the data\n", "\n", "The value of each pixel in the image data is an integer in the range `[0,255]`. For the model to work properly, these values need to be normalized to the range `[0,1]`. So here we create a normalization function, and then apply it to each image in the test and train datasets." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nAsH3Zm-76pB" }, "outputs": [], "source": [ "def normalize(images, labels):\n", " images = tf.cast(images, tf.float32)\n", " images /= 255\n", " return images, labels\n", "\n", "# The map function applies the normalize function to each element in the train\n", "# and test datasets\n", "train_dataset = train_dataset.map(normalize)\n", "test_dataset = test_dataset.map(normalize)\n", "\n", "# The first time you use the dataset, the images will be loaded from disk\n", "# Caching will keep them in memory, making training faster\n", "train_dataset = train_dataset.cache()\n", "test_dataset = test_dataset.cache()" ] }, { "cell_type": "markdown", "metadata": { "id": "lIQbEiJGXM-q" }, "source": [ "### Explore the processed data\n", "\n", "Let's plot an image to see what it looks like." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oSzE9l7PjHx0" }, "outputs": [], "source": [ "# Take a single image, and remove the color dimension by reshaping\n", "for image, label in test_dataset.take(1):\n", " break\n", "image = image.numpy().reshape((28,28))\n", "\n", "# Plot the image - voila a piece of fashion clothing\n", "plt.figure()\n", "plt.imshow(image, cmap=plt.cm.binary)\n", "plt.colorbar()\n", "plt.grid(False)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "Ee638AlnCaWz" }, "source": [ "Display the first 25 images from the *training set* and display the class name below each image. Verify that the data is in the correct format and we're ready to build and train the network." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oZTImqg_CaW1" }, "outputs": [], "source": [ "plt.figure(figsize=(10,10))\n", "i = 0\n", "for (image, label) in test_dataset.take(25):\n", " image = image.numpy().reshape((28,28))\n", " plt.subplot(5,5,i+1)\n", " plt.xticks([])\n", " plt.yticks([])\n", " plt.grid(False)\n", " plt.imshow(image, cmap=plt.cm.binary)\n", " plt.xlabel(class_names[label])\n", " i += 1\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "59veuiEZCaW4" }, "source": [ "## Build the model\n", "\n", "Building the neural network requires configuring the layers of the model, then compiling the model." ] }, { "cell_type": "markdown", "metadata": { "id": "Gxg1XGm0eOBy" }, "source": [ "### Setup the layers\n", "\n", "The basic building block of a neural network is the *layer*. A layer extracts a representation from the data fed into it. Hopefully, a series of connected layers results in a representation that is meaningful for the problem at hand.\n", "\n", "Much of deep learning consists of chaining together simple layers. Most layers, like `tf.keras.layers.Dense`, have internal parameters which are adjusted (\"learned\") during training." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9ODch-OFCaW4" }, "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " tf.keras.layers.Conv2D(32, (3,3), padding='same', activation=tf.nn.relu,\n", " input_shape=(28, 28, 1)),\n", " tf.keras.layers.MaxPooling2D((2, 2), strides=2),\n", " tf.keras.layers.Conv2D(64, (3,3), padding='same', activation=tf.nn.relu),\n", " tf.keras.layers.MaxPooling2D((2, 2), strides=2),\n", " tf.keras.layers.Flatten(),\n", " tf.keras.layers.Dense(128, activation=tf.nn.relu),\n", " tf.keras.layers.Dense(10, activation=tf.nn.softmax)\n", "])" ] }, { "cell_type": "markdown", "metadata": { "id": "gut8A_7rCaW6" }, "source": [ "This network layers are:\n", "\n", "* **\"convolutions\"** `tf.keras.layers.Conv2D and MaxPooling2D`— Network start with two pairs of Conv/MaxPool. The first layer is a Conv2D filters (3,3) being applied to the input image, retaining the original image size by using padding, and creating 32 output (convoluted) images (so this layer creates 32 convoluted images of the same size as input). After that, the 32 outputs are reduced in size using a MaxPooling2D (2,2) with a stride of 2. The next Conv2D also has a (3,3) kernel, takes the 32 images as input and creates 64 outputs which are again reduced in size by a MaxPooling2D layer. So far in the course, we have described what a Convolution does, but we haven't yet covered how you chain multiples of these together. We will get back to this in lesson 4 when we use color images. At this point, it's enough if you understand the kind of operation a convolutional filter performs\n", "\n", "* **output** `tf.keras.layers.Dense` — A 128-neuron, followed by 10-node *softmax* layer. Each node represents a class of clothing. As in the previous layer, the final layer takes input from the 128 nodes in the layer before it, and outputs a value in the range `[0, 1]`, representing the probability that the image belongs to that class. The sum of all 10 node values is 1.\n", "\n", "> Note: Using `softmax` activation and `SparseCategoricalCrossentropy()` has issues and which are patched by the `tf.keras` model. A safer approach, in general, is to use a linear output (no activation function) with `SparseCategoricalCrossentropy(from_logits=True)`.\n", "\n", "\n", "### Compile the model\n", "\n", "Before the model is ready for training, it needs a few more settings. These are added during the model's *compile* step:\n", "\n", "\n", "* *Loss function* — An algorithm for measuring how far the model's outputs are from the desired output. The goal of training is this measures loss.\n", "* *Optimizer* —An algorithm for adjusting the inner parameters of the model in order to minimize loss.\n", "* *Metrics* —Used to monitor the training and testing steps. The following example uses *accuracy*, the fraction of the images that are correctly classified." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Lhan11blCaW7" }, "outputs": [], "source": [ "model.compile(optimizer='adam',\n", " loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { "id": "qKF6uW-BCaW-" }, "source": [ "## Train the model\n", "\n", "First, we define the iteration behavior for the train dataset:\n", "1. Repeat forever by specifying `dataset.repeat()` (the `epochs` parameter described below limits how long we perform training).\n", "2. The `dataset.shuffle(60000)` randomizes the order so our model cannot learn anything from the order of the examples.\n", "3. And `dataset.batch(32)` tells `model.fit` to use batches of 32 images and labels when updating the model variables.\n", "\n", "Training is performed by calling the `model.fit` method:\n", "1. Feed the training data to the model using `train_dataset`.\n", "2. The model learns to associate images and labels.\n", "3. The `epochs=5` parameter limits training to 5 full iterations of the training dataset, so a total of 5 * 60000 = 300000 examples.\n", "\n", "(Don't worry about `steps_per_epoch`, the requirement to have this flag will soon be removed.)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "o_Dp8971McQ1" }, "outputs": [], "source": [ "BATCH_SIZE = 32\n", "train_dataset = train_dataset.cache().repeat().shuffle(num_train_examples).batch(BATCH_SIZE)\n", "test_dataset = test_dataset.cache().batch(BATCH_SIZE)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "xvwvpA64CaW_" }, "outputs": [], "source": [ "model.fit(train_dataset, epochs=10, steps_per_epoch=math.ceil(num_train_examples/BATCH_SIZE))" ] }, { "cell_type": "markdown", "metadata": { "id": "W3ZVOhugCaXA" }, "source": [ "As the model trains, the loss and accuracy metrics are displayed. This model reaches an accuracy of about 0.97 (or 97%) on the training data." ] }, { "cell_type": "markdown", "metadata": { "id": "oEw4bZgGCaXB" }, "source": [ "## Evaluate accuracy\n", "\n", "Next, compare how the model performs on the test dataset. Use all examples we have in the test dataset to assess accuracy." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "VflXLEeECaXC" }, "outputs": [], "source": [ "test_loss, test_accuracy = model.evaluate(test_dataset, steps=math.ceil(num_test_examples/32))\n", "print('Accuracy on test dataset:', test_accuracy)" ] }, { "cell_type": "markdown", "metadata": { "id": "yWfgsmVXCaXG" }, "source": [ "As it turns out, the accuracy on the test dataset is smaller than the accuracy on the training dataset. This is completely normal, since the model was trained on the `train_dataset`. When the model sees images it has never seen during training, (that is, from the `test_dataset`), we can expect performance to go down. " ] }, { "cell_type": "markdown", "metadata": { "id": "xsoS7CPDCaXH" }, "source": [ "## Make predictions and explore\n", "\n", "With the model trained, we can use it to make predictions about some images." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Ccoz4conNCpl" }, "outputs": [], "source": [ "for test_images, test_labels in test_dataset.take(1):\n", " test_images = test_images.numpy()\n", " test_labels = test_labels.numpy()\n", " predictions = model.predict(test_images)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Gl91RPhdCaXI" }, "outputs": [], "source": [ "predictions.shape\n" ] }, { "cell_type": "markdown", "metadata": { "id": "x9Kk1voUCaXJ" }, "source": [ "Here, the model has predicted the probability of each label for each image in the testing set. Let's take a look at the first prediction:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3DmJEUinCaXK" }, "outputs": [], "source": [ "predictions[0]" ] }, { "cell_type": "markdown", "metadata": { "id": "-hw1hgeSCaXN" }, "source": [ "A prediction is an array of 10 numbers. These describe the \"confidence\" of the model that the image corresponds to each of the 10 different articles of clothing. We can see which label has the highest confidence value:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "qsqenuPnCaXO" }, "outputs": [], "source": [ "np.argmax(predictions[0])" ] }, { "cell_type": "markdown", "metadata": { "id": "E51yS7iCCaXO" }, "source": [ "So the model is usually most confident that this image is a Shirt, or `class_names[6]`. Let's check the label:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Sd7Pgsu6CaXP" }, "outputs": [], "source": [ "test_labels[0]" ] }, { "cell_type": "markdown", "metadata": { "id": "ygh2yYC972ne" }, "source": [ "We can graph this to look at the full set of 10 class predictions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DvYmmrpIy6Y1" }, "outputs": [], "source": [ "def plot_image(i, predictions_array, true_labels, images):\n", " predictions_array, true_label, img = predictions_array[i], true_labels[i], images[i]\n", " plt.grid(False)\n", " plt.xticks([])\n", " plt.yticks([])\n", " \n", " plt.imshow(img[...,0], cmap=plt.cm.binary)\n", "\n", " predicted_label = np.argmax(predictions_array)\n", " if predicted_label == true_label:\n", " color = 'blue'\n", " else:\n", " color = 'red'\n", " \n", " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", " 100*np.max(predictions_array),\n", " class_names[true_label]),\n", " color=color)\n", "\n", "def plot_value_array(i, predictions_array, true_label):\n", " predictions_array, true_label = predictions_array[i], true_label[i]\n", " plt.grid(False)\n", " plt.xticks([])\n", " plt.yticks([])\n", " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", " plt.ylim([0, 1])\n", " predicted_label = np.argmax(predictions_array)\n", " \n", " thisplot[predicted_label].set_color('red')\n", " thisplot[true_label].set_color('blue')" ] }, { "cell_type": "markdown", "metadata": { "id": "d4Ov9OFDMmOD" }, "source": [ "Let's look at the 0th image, predictions, and prediction array. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "HV5jw-5HwSmO" }, "outputs": [], "source": [ "i = 0\n", "plt.figure(figsize=(6,3))\n", "plt.subplot(1,2,1)\n", "plot_image(i, predictions, test_labels, test_images)\n", "plt.subplot(1,2,2)\n", "plot_value_array(i, predictions, test_labels)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Ko-uzOufSCSe" }, "outputs": [], "source": [ "i = 12\n", "plt.figure(figsize=(6,3))\n", "plt.subplot(1,2,1)\n", "plot_image(i, predictions, test_labels, test_images)\n", "plt.subplot(1,2,2)\n", "plot_value_array(i, predictions, test_labels)" ] }, { "cell_type": "markdown", "metadata": { "id": "kgdvGD52CaXR" }, "source": [ "Let's plot several images with their predictions. Correct prediction labels are blue and incorrect prediction labels are red. The number gives the percent (out of 100) for the predicted label. Note that it can be wrong even when very confident. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "hQlnbqaw2Qu_" }, "outputs": [], "source": [ "# Plot the first X test images, their predicted label, and the true label\n", "# Color correct predictions in blue, incorrect predictions in red\n", "num_rows = 5\n", "num_cols = 3\n", "num_images = num_rows*num_cols\n", "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", "for i in range(num_images):\n", " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", " plot_image(i, predictions, test_labels, test_images)\n", " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", " plot_value_array(i, predictions, test_labels)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "R32zteKHCaXT" }, "source": [ "Finally, use the trained model to make a prediction about a single image. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "yRJ7JU7JCaXT" }, "outputs": [], "source": [ "# Grab an image from the test dataset\n", "img = test_images[0]\n", "\n", "print(img.shape)" ] }, { "cell_type": "markdown", "metadata": { "id": "vz3bVp21CaXV" }, "source": [ "`tf.keras` models are optimized to make predictions on a *batch*, or collection, of examples at once. So even though we're using a single image, we need to add it to a list:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lDFh5yF_CaXW" }, "outputs": [], "source": [ "# Add the image to a batch where it's the only member.\n", "img = np.array([img])\n", "\n", "print(img.shape)" ] }, { "cell_type": "markdown", "metadata": { "id": "EQ5wLTkcCaXY" }, "source": [ "Now predict the image:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "o_rzNSdrCaXY" }, "outputs": [], "source": [ "predictions_single = model.predict(img)\n", "\n", "print(predictions_single)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6Ai-cpLjO-3A" }, "outputs": [], "source": [ "plot_value_array(0, predictions_single, test_labels)\n", "_ = plt.xticks(range(10), class_names, rotation=45)" ] }, { "cell_type": "markdown", "metadata": { "id": "cU1Y2OAMCaXb" }, "source": [ "`model.predict` returns a list of lists, one for each image in the batch of data. Grab the predictions for our (only) image in the batch:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2tRmdq_8CaXb" }, "outputs": [], "source": [ "np.argmax(predictions_single[0])" ] }, { "cell_type": "markdown", "metadata": { "id": "YFc2HbEVCaXd" }, "source": [ "And, as before, the model predicts a label of 6 (shirt)." ] }, { "cell_type": "markdown", "metadata": { "id": "-KtnHECKZni_" }, "source": [ "# Exercises\n", "\n", "Experiment with different models and see how the accuracy results differ. In particular change the following parameters:\n", "* Set training epochs set to 1\n", "* Number of neurons in the Dense layer following the Flatten one. For example, go really low (e.g. 10) in ranges up to 512 and see how accuracy changes\n", "* Add additional Dense layers between the Flatten and the final Dense(10), experiment with different units in these layers\n", "* Don't normalize the pixel values, and see the effect that has\n", "\n", "\n", "Remember to enable GPU to make everything run faster (Runtime -> Change runtime type -> Hardware accelerator -> GPU).\n", "Also, if you run into trouble, simply reset the entire environment and start from the beginning:\n", "* Edit -> Clear all outputs\n", "* Runtime -> Reset all runtimes" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l04c01_image_classification_with_cnns.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l05c01_dogs_vs_cats_without_augmentation.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "TBFXQGKYUc4X" }, "source": [ "##### Copyright 2019 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "1z4xy2gTUc4a" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "FE7KNzPPVrVV" }, "source": [ "# Dogs vs Cats Image Classification Without Image Augmentation" ] }, { "cell_type": "markdown", "metadata": { "id": "KwQtSOz0VrVX" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "gN7G9GFmVrVY" }, "source": [ "In this tutorial, we will discuss how to classify images into pictures of cats or pictures of dogs. We'll build an image classifier using `tf.keras.Sequential` model and load data using `tf.keras.preprocessing.image.ImageDataGenerator`.\n", "\n", "## Specific concepts that will be covered:\n", "In the process, we will build practical experience and develop intuition around the following concepts\n", "\n", "* Building _data input pipelines_ using the `tf.keras.preprocessing.image.ImageDataGenerator` class — How can we efficiently work with data on disk to interface with our model?\n", "* _Overfitting_ - what is it, how to identify it?\n", "\n", "
\n", "\n", "\n", "**Before you begin**\n", "\n", "Before running the code in this notebook, reset the runtime by going to **Runtime -> Reset all runtimes** in the menu above. If you have been working through several notebooks, this will help you avoid reaching Colab's memory limits.\n" ] }, { "cell_type": "markdown", "metadata": { "id": "zF9uvbXNVrVY" }, "source": [ "# Importing packages" ] }, { "cell_type": "markdown", "metadata": { "id": "VddxeYBEVrVZ" }, "source": [ "Let's start by importing required packages:\n", "\n", "* os — to read files and directory structure\n", "* numpy — for some matrix math outside of TensorFlow\n", "* matplotlib.pyplot — to plot the graph and display images in our training and validation data\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oSdjGwVWGshH" }, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nlORkUyFGxWH" }, "outputs": [], "source": [ "from tensorflow.keras.preprocessing.image import ImageDataGenerator" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wqtiIPRbG4FA" }, "outputs": [], "source": [ "import os\n", "import matplotlib.pyplot as plt\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "GHHqtPisG3R1" }, "outputs": [], "source": [ "import logging\n", "logger = tf.get_logger()\n", "logger.setLevel(logging.ERROR)" ] }, { "cell_type": "markdown", "metadata": { "id": "UZZI6lNkVrVm" }, "source": [ "# Data Loading" ] }, { "cell_type": "markdown", "metadata": { "id": "DPHx8-t-VrVo" }, "source": [ "To build our image classifier, we begin by downloading the dataset. The dataset we are using is a filtered version of Dogs vs. Cats dataset from Kaggle (ultimately, this dataset is provided by Microsoft Research).\n", "\n", "In previous Colabs, we've used TensorFlow Datasets, which is a very easy and convenient way to use datasets. In this Colab however, we will make use of the class `tf.keras.preprocessing.image.ImageDataGenerator` which will read data from disk. We therefore need to directly download *Dogs vs. Cats* from a URL and unzip it to the Colab filesystem." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "rpUSoFjuVrVp" }, "outputs": [], "source": [ "_URL = 'https://download.mlcc.google.com/mledu-datasets/cats_and_dogs_filtered.zip'\n", "zip_dir = tf.keras.utils.get_file('cats_and_dogs_filterted.zip', origin=_URL, extract=True)" ] }, { "cell_type": "markdown", "metadata": { "id": "Giv0wMQzVrVw" }, "source": [ "The dataset we have downloaded has the following directory structure.\n", "\n", "
\n",
        "cats_and_dogs_filtered\n",
        "|__ train\n",
        "    |______ cats: [cat.0.jpg, cat.1.jpg, cat.2.jpg ...]\n",
        "    |______ dogs: [dog.0.jpg, dog.1.jpg, dog.2.jpg ...]\n",
        "|__ validation\n",
        "    |______ cats: [cat.2000.jpg, cat.2001.jpg, cat.2002.jpg ...]\n",
        "    |______ dogs: [dog.2000.jpg, dog.2001.jpg, dog.2002.jpg ...]\n",
        "
\n", "\n", "We can list the directories with the following terminal command:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ssD23VbTZeVA" }, "outputs": [], "source": [ "zip_dir_base = os.path.dirname(zip_dir)\n", "!find $zip_dir_base -type d -print" ] }, { "cell_type": "markdown", "metadata": { "id": "VpmywIlsVrVx" }, "source": [ "We'll now assign variables with the proper file path for the training and validation sets." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "sRucI3QqVrVy" }, "outputs": [], "source": [ "base_dir = os.path.join(zip_dir, 'cats_and_dogs_filtered')\n", "train_dir = os.path.join(base_dir, 'train')\n", "validation_dir = os.path.join(base_dir, 'validation')\n", "\n", "train_cats_dir = os.path.join(train_dir, 'cats') # directory with our training cat pictures\n", "train_dogs_dir = os.path.join(train_dir, 'dogs') # directory with our training dog pictures\n", "validation_cats_dir = os.path.join(validation_dir, 'cats') # directory with our validation cat pictures\n", "validation_dogs_dir = os.path.join(validation_dir, 'dogs') # directory with our validation dog pictures" ] }, { "cell_type": "markdown", "metadata": { "id": "ZdrHHTy2VrV3" }, "source": [ "### Understanding our data" ] }, { "cell_type": "markdown", "metadata": { "id": "LblUYjl-VrV3" }, "source": [ "Let's look at how many cats and dogs images we have in our training and validation directory" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "vc4u8e9hVrV4" }, "outputs": [], "source": [ "num_cats_tr = len(os.listdir(train_cats_dir))\n", "num_dogs_tr = len(os.listdir(train_dogs_dir))\n", "\n", "num_cats_val = len(os.listdir(validation_cats_dir))\n", "num_dogs_val = len(os.listdir(validation_dogs_dir))\n", "\n", "total_train = num_cats_tr + num_dogs_tr\n", "total_val = num_cats_val + num_dogs_val" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "g4GGzGt0VrV7" }, "outputs": [], "source": [ "print('total training cat images:', num_cats_tr)\n", "print('total training dog images:', num_dogs_tr)\n", "\n", "print('total validation cat images:', num_cats_val)\n", "print('total validation dog images:', num_dogs_val)\n", "print(\"--\")\n", "print(\"Total training images:\", total_train)\n", "print(\"Total validation images:\", total_val)" ] }, { "cell_type": "markdown", "metadata": { "id": "tdsI_L-NVrV_" }, "source": [ "# Setting Model Parameters" ] }, { "cell_type": "markdown", "metadata": { "id": "8Lp-0ejxOtP1" }, "source": [ "For convenience, we'll set up variables that will be used later while pre-processing our dataset and training our network." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3NqNselLVrWA" }, "outputs": [], "source": [ "BATCH_SIZE = 100 # Number of training examples to process before updating our models variables\n", "IMG_SHAPE = 150 # Our training data consists of images with width of 150 pixels and height of 150 pixels" ] }, { "cell_type": "markdown", "metadata": { "id": "INn-cOn1VrWC" }, "source": [ "# Data Preparation " ] }, { "cell_type": "markdown", "metadata": { "id": "5Jfk6aSAVrWD" }, "source": [ "Images must be formatted into appropriately pre-processed floating point tensors before being fed into the network. The steps involved in preparing these images are:\n", "\n", "1. Read images from the disk\n", "2. Decode contents of these images and convert it into proper grid format as per their RGB content\n", "3. Convert them into floating point tensors\n", "4. Rescale the tensors from values between 0 and 255 to values between 0 and 1, as neural networks prefer to deal with small input values.\n", "\n", "Fortunately, all these tasks can be done using the class **tf.keras.preprocessing.image.ImageDataGenerator**.\n", "\n", "We can set this up in a couple of lines of code." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "syDdF_LWVrWE" }, "outputs": [], "source": [ "train_image_generator = ImageDataGenerator(rescale=1./255) # Generator for our training data\n", "validation_image_generator = ImageDataGenerator(rescale=1./255) # Generator for our validation data" ] }, { "cell_type": "markdown", "metadata": { "id": "RLciCR_FVrWH" }, "source": [ "After defining our generators for training and validation images, **flow_from_directory** method will load images from the disk, apply rescaling, and resize them using single line of code." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Pw94ajOOVrWI" }, "outputs": [], "source": [ "train_data_gen = train_image_generator.flow_from_directory(batch_size=BATCH_SIZE,\n", " directory=train_dir,\n", " shuffle=True,\n", " target_size=(IMG_SHAPE,IMG_SHAPE), #(150,150)\n", " class_mode='binary')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2oUoKUzRVrWM" }, "outputs": [], "source": [ "val_data_gen = validation_image_generator.flow_from_directory(batch_size=BATCH_SIZE,\n", " directory=validation_dir,\n", " shuffle=False,\n", " target_size=(IMG_SHAPE,IMG_SHAPE), #(150,150)\n", " class_mode='binary')" ] }, { "cell_type": "markdown", "metadata": { "id": "hyexPJ8CVrWP" }, "source": [ "### Visualizing Training images" ] }, { "cell_type": "markdown", "metadata": { "id": "60CnhEL4VrWQ" }, "source": [ "We can visualize our training images by getting a batch of images from the training generator, and then plotting a few of them using `matplotlib`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3f0Z7NZgVrWQ" }, "outputs": [], "source": [ "sample_training_images, _ = next(train_data_gen) " ] }, { "cell_type": "markdown", "metadata": { "id": "49weMt5YVrWT" }, "source": [ "The `next` function returns a batch from the dataset. One batch is a tuple of (*many images*, *many labels*). For right now, we're discarding the labels because we just want to look at the images." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JMt2RES_VrWU" }, "outputs": [], "source": [ "# This function will plot images in the form of a grid with 1 row and 5 columns where images are placed in each column.\n", "def plotImages(images_arr):\n", " fig, axes = plt.subplots(1, 5, figsize=(20,20))\n", " axes = axes.flatten()\n", " for img, ax in zip(images_arr, axes):\n", " ax.imshow(img)\n", " plt.tight_layout()\n", " plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "d_VVg_gEVrWW" }, "outputs": [], "source": [ "plotImages(sample_training_images[:5]) # Plot images 0-4" ] }, { "cell_type": "markdown", "metadata": { "id": "b5Ej-HLGVrWZ" }, "source": [ "# Model Creation" ] }, { "cell_type": "markdown", "metadata": { "id": "wEgW4i18VrWZ" }, "source": [ "## Define the model\n", "\n", "The model consists of four convolution blocks with a max pool layer in each of them. Then we have a fully connected layer with 512 units, with a `relu` activation function. The model will output class probabilities for two classes — dogs and cats — using `softmax`. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "F15-uwLPVrWa" }, "outputs": [], "source": [ "model = tf.keras.models.Sequential([\n", " tf.keras.layers.Conv2D(32, (3,3), activation='relu', input_shape=(150, 150, 3)),\n", " tf.keras.layers.MaxPooling2D(2, 2),\n", "\n", " tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n", " tf.keras.layers.MaxPooling2D(2,2),\n", " \n", " tf.keras.layers.Conv2D(128, (3,3), activation='relu'),\n", " tf.keras.layers.MaxPooling2D(2,2),\n", " \n", " tf.keras.layers.Conv2D(128, (3,3), activation='relu'),\n", " tf.keras.layers.MaxPooling2D(2,2),\n", " \n", " tf.keras.layers.Flatten(),\n", " tf.keras.layers.Dense(512, activation='relu'),\n", " tf.keras.layers.Dense(2)\n", "])" ] }, { "cell_type": "markdown", "metadata": { "id": "PI5cdkMQVrWc" }, "source": [ "### Compile the model\n", "\n", "As usual, we will use the `adam` optimizer. Since we output a softmax categorization, we'll use `sparse_categorical_crossentropy` as the loss function. We would also like to look at training and validation accuracy on each epoch as we train our network, so we are passing in the metrics argument." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6Mg7_TXOVrWd" }, "outputs": [], "source": [ "model.compile(optimizer='adam',\n", " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { "id": "2YmQZ3TAVrWg" }, "source": [ "### Model Summary\n", "\n", "Let's look at all the layers of our network using **summary** method." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Vtny8hmBVrWh" }, "outputs": [], "source": [ "model.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "N06iqE8VVrWj" }, "source": [ "### Train the model" ] }, { "cell_type": "markdown", "metadata": { "id": "oub9RtoFVrWk" }, "source": [ "It's time we train our network.\n", "\n", "Since our batches are coming from a generator (`ImageDataGenerator`), we'll use `fit_generator` instead of `fit`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "KSF2HqhDVrWk" }, "outputs": [], "source": [ "EPOCHS = 100\n", "history = model.fit(\n", " train_data_gen,\n", " steps_per_epoch=int(np.ceil(total_train / float(BATCH_SIZE))),\n", " epochs=EPOCHS,\n", " validation_data=val_data_gen,\n", " validation_steps=int(np.ceil(total_val / float(BATCH_SIZE)))\n", ")" ] }, { "cell_type": "markdown", "metadata": { "id": "ojJNteAGVrWo" }, "source": [ "### Visualizing results of the training" ] }, { "cell_type": "markdown", "metadata": { "id": "LZPYT-EmVrWo" }, "source": [ "We'll now visualize the results we get after training our network." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "K6oA77ADVrWp" }, "outputs": [], "source": [ "acc = history.history['accuracy']\n", "val_acc = history.history['val_accuracy']\n", "\n", "loss = history.history['loss']\n", "val_loss = history.history['val_loss']\n", "\n", "epochs_range = range(EPOCHS)\n", "\n", "plt.figure(figsize=(8, 8))\n", "plt.subplot(1, 2, 1)\n", "plt.plot(epochs_range, acc, label='Training Accuracy')\n", "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", "plt.legend(loc='lower right')\n", "plt.title('Training and Validation Accuracy')\n", "\n", "plt.subplot(1, 2, 2)\n", "plt.plot(epochs_range, loss, label='Training Loss')\n", "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", "plt.legend(loc='upper right')\n", "plt.title('Training and Validation Loss')\n", "plt.savefig('./foo.png')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "kDnr50l2VrWu" }, "source": [ "As we can see from the plots, training accuracy and validation accuracy are off by large margin and our model has achieved only around **70%** accuracy on the validation set (depending on the number of epochs you trained for).\n", "\n", "This is a clear indication of overfitting. Once the training and validation curves start to diverge, our model has started to memorize the training data and is unable to perform well on the validation data." ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l05c01_dogs_vs_cats_without_augmentation.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l05c02_dogs_vs_cats_with_augmentation.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "TBFXQGKYUc4X" }, "source": [ "##### Copyright 2019 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "1z4xy2gTUc4a" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "FE7KNzPPVrVV" }, "source": [ "# Dogs vs Cats Image Classification With Image Augmentation" ] }, { "cell_type": "markdown", "metadata": { "id": "KwQtSOz0VrVX" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "gN7G9GFmVrVY" }, "source": [ "In this tutorial, we will discuss how to classify images into pictures of cats or pictures of dogs. We'll build an image classifier using `tf.keras.Sequential` model and load data using `tf.keras.preprocessing.image.ImageDataGenerator`.\n", "\n", "## Specific concepts that will be covered:\n", "In the process, we will build practical experience and develop intuition around the following concepts\n", "\n", "* Building _data input pipelines_ using the `tf.keras.preprocessing.image.ImageDataGenerator` class — How can we efficiently work with data on disk to interface with our model?\n", "* _Overfitting_ - what is it, how to identify it, and how can we prevent it?\n", "* _Data Augmentation_ and _Dropout_ - Key techniques to fight overfitting in computer vision tasks that we will incorporate into our data pipeline and image classifier model.\n", "\n", "## We will follow the general machine learning workflow:\n", "\n", "1. Examine and understand data\n", "2. Build an input pipeline\n", "3. Build our model\n", "4. Train our model\n", "5. Test our model\n", "6. Improve our model/Repeat the process\n", "\n", "
\n", "\n", "**Before you begin**\n", "\n", "Before running the code in this notebook, reset the runtime by going to **Runtime -> Reset all runtimes** in the menu above. If you have been working through several notebooks, this will help you avoid reaching Colab's memory limits.\n" ] }, { "cell_type": "markdown", "metadata": { "id": "zF9uvbXNVrVY" }, "source": [ "# Importing packages" ] }, { "cell_type": "markdown", "metadata": { "id": "VddxeYBEVrVZ" }, "source": [ "Let's start by importing required packages:\n", "\n", "* os — to read files and directory structure\n", "* numpy — for some matrix math outside of TensorFlow\n", "* matplotlib.pyplot — to plot the graph and display images in our training and validation data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "in3OdvpUG_9_" }, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "L1WtoaOHVrVh" }, "outputs": [], "source": [ "from tensorflow.keras.preprocessing.image import ImageDataGenerator" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ede3_kbeHOjR" }, "outputs": [], "source": [ "import os\n", "import numpy as np\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "markdown", "metadata": { "id": "UZZI6lNkVrVm" }, "source": [ "# Data Loading" ] }, { "cell_type": "markdown", "metadata": { "id": "DPHx8-t-VrVo" }, "source": [ "To build our image classifier, we begin by downloading the dataset. The dataset we are using is a filtered version of Dogs vs. Cats dataset from Kaggle (ultimately, this dataset is provided by Microsoft Research).\n", "\n", "In previous Colabs, we've used TensorFlow Datasets, which is a very easy and convenient way to use datasets. In this Colab however, we will make use of the class `tf.keras.preprocessing.image.ImageDataGenerator` which will read data from disk. We therefore need to directly download *Dogs vs. Cats* from a URL and unzip it to the Colab filesystem." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "OYmOylPlVrVt" }, "outputs": [], "source": [ "_URL = 'https://download.mlcc.google.com/mledu-datasets/cats_and_dogs_filtered.zip'\n", "\n", "zip_dir = tf.keras.utils.get_file('cats_and_dogs_filterted.zip', origin=_URL, extract=True)" ] }, { "cell_type": "markdown", "metadata": { "id": "Giv0wMQzVrVw" }, "source": [ "The dataset we have downloaded has following directory structure.\n", "\n", "
\n",
        "cats_and_dogs_filtered\n",
        "|__ train\n",
        "    |______ cats: [cat.0.jpg, cat.1.jpg, cat.2.jpg ....]\n",
        "    |______ dogs: [dog.0.jpg, dog.1.jpg, dog.2.jpg ...]\n",
        "|__ validation\n",
        "    |______ cats: [cat.2000.jpg, cat.2001.jpg, cat.2002.jpg ....]\n",
        "    |______ dogs: [dog.2000.jpg, dog.2001.jpg, dog.2002.jpg ...]\n",
        "
" ] }, { "cell_type": "markdown", "metadata": { "id": "VpmywIlsVrVx" }, "source": [ "We'll now assign variables with the proper file path for the training and validation sets." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "sRucI3QqVrVy" }, "outputs": [], "source": [ "base_dir = os.path.join(os.path.dirname(zip_dir), 'cats_and_dogs_filtered')\n", "train_dir = os.path.join(base_dir, 'train')\n", "validation_dir = os.path.join(base_dir, 'validation')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Utv3nryxVrV0" }, "outputs": [], "source": [ "train_cats_dir = os.path.join(train_dir, 'cats') # directory with our training cat pictures\n", "train_dogs_dir = os.path.join(train_dir, 'dogs') # directory with our training dog pictures\n", "validation_cats_dir = os.path.join(validation_dir, 'cats') # directory with our validation cat pictures\n", "validation_dogs_dir = os.path.join(validation_dir, 'dogs') # directory with our validation dog pictures" ] }, { "cell_type": "markdown", "metadata": { "id": "ZdrHHTy2VrV3" }, "source": [ "### Understanding our data" ] }, { "cell_type": "markdown", "metadata": { "id": "LblUYjl-VrV3" }, "source": [ "Let's look at how many cats and dogs images we have in our training and validation directory" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "vc4u8e9hVrV4" }, "outputs": [], "source": [ "num_cats_tr = len(os.listdir(train_cats_dir))\n", "num_dogs_tr = len(os.listdir(train_dogs_dir))\n", "\n", "num_cats_val = len(os.listdir(validation_cats_dir))\n", "num_dogs_val = len(os.listdir(validation_dogs_dir))\n", "\n", "total_train = num_cats_tr + num_dogs_tr\n", "total_val = num_cats_val + num_dogs_val" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "g4GGzGt0VrV7" }, "outputs": [], "source": [ "print('total training cat images:', num_cats_tr)\n", "print('total training dog images:', num_dogs_tr)\n", "\n", "print('total validation cat images:', num_cats_val)\n", "print('total validation dog images:', num_dogs_val)\n", "print(\"--\")\n", "print(\"Total training images:\", total_train)\n", "print(\"Total validation images:\", total_val)" ] }, { "cell_type": "markdown", "metadata": { "id": "tdsI_L-NVrV_" }, "source": [ "# Setting Model Parameters" ] }, { "cell_type": "markdown", "metadata": { "id": "8Lp-0ejxOtP1" }, "source": [ "For convenience, let us set up variables that will be used later while pre-processing our dataset and training our network." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3NqNselLVrWA" }, "outputs": [], "source": [ "BATCH_SIZE = 100\n", "IMG_SHAPE = 150 # Our training data consists of images with width of 150 pixels and height of 150 pixels" ] }, { "cell_type": "markdown", "metadata": { "id": "RLciCR_FVrWH" }, "source": [ "After defining our generators for training and validation images, **flow_from_directory** method will load images from the disk and will apply rescaling and will resize them into required dimensions using single line of code." ] }, { "cell_type": "markdown", "metadata": { "id": "UOoVpxFwVrWy" }, "source": [ "# Data Augmentation" ] }, { "cell_type": "markdown", "metadata": { "id": "Wn_QLciWVrWy" }, "source": [ "Overfitting often occurs when we have a small number of training examples. One way to fix this problem is to augment our dataset so that it has sufficient number and variety of training examples. Data augmentation takes the approach of generating more training data from existing training samples, by augmenting the samples through random transformations that yield believable-looking images. The goal is that at training time, your model will never see the exact same picture twice. This exposes the model to more aspects of the data, allowing it to generalize better.\n", "\n", "In **tf.keras** we can implement this using the same **ImageDataGenerator** class we used before. We can simply pass different transformations we would want to our dataset as a form of arguments and it will take care of applying it to the dataset during our training process.\n", "\n", "To start off, let's define a function that can display an image, so we can see the type of augmentation that has been performed. Then, we'll look at specific augmentations that we'll use during training." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "GBYLOFgOXPJ9" }, "outputs": [], "source": [ "# This function will plot images in the form of a grid with 1 row and 5 columns where images are placed in each column.\n", "def plotImages(images_arr):\n", " fig, axes = plt.subplots(1, 5, figsize=(20,20))\n", " axes = axes.flatten()\n", " for img, ax in zip(images_arr, axes):\n", " ax.imshow(img)\n", " plt.tight_layout()\n", " plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "rlVj6VqaVrW0" }, "source": [ "### Flipping the image horizontally" ] }, { "cell_type": "markdown", "metadata": { "id": "xcdvx4TVVrW1" }, "source": [ "We can begin by randomly applying horizontal flip augmentation to our dataset and seeing how individual images will look after the transformation. This is achieved by passing `horizontal_flip=True` as an argument to the `ImageDataGenerator` class." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Bi1_vHyBVrW2" }, "outputs": [], "source": [ "image_gen = ImageDataGenerator(rescale=1./255, horizontal_flip=True)\n", "\n", "train_data_gen = image_gen.flow_from_directory(batch_size=BATCH_SIZE,\n", " directory=train_dir,\n", " shuffle=True,\n", " target_size=(IMG_SHAPE,IMG_SHAPE))" ] }, { "cell_type": "markdown", "metadata": { "id": "zJpRSxJ-VrW7" }, "source": [ "To see the transformation in action, let's take one sample image from our training set and repeat it five times. The augmentation will be randomly applied (or not) to each repetition." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "RrKGd_jjVrW7" }, "outputs": [], "source": [ "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "i7n9xcqCVrXB" }, "source": [ "### Rotating the image" ] }, { "cell_type": "markdown", "metadata": { "id": "qXnwkzFuVrXB" }, "source": [ "The rotation augmentation will randomly rotate the image up to a specified number of degrees. Here, we'll set it to 45." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1zip35pDVrXB" }, "outputs": [], "source": [ "image_gen = ImageDataGenerator(rescale=1./255, rotation_range=45)\n", "\n", "train_data_gen = image_gen.flow_from_directory(batch_size=BATCH_SIZE,\n", " directory=train_dir,\n", " shuffle=True,\n", " target_size=(IMG_SHAPE, IMG_SHAPE))" ] }, { "cell_type": "markdown", "metadata": { "id": "deaqZLsfcZ15" }, "source": [ "To see the transformation in action, let's once again take a sample image from our training set and repeat it. The augmentation will be randomly applied (or not) to each repetition." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kVoWh4OIVrXD" }, "outputs": [], "source": [ "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "FOqGPL76VrXM" }, "source": [ "### Applying Zoom" ] }, { "cell_type": "markdown", "metadata": { "id": "NvqXaD8BVrXN" }, "source": [ "We can also apply Zoom augmentation to our dataset, zooming images up to 50% randomly." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tGNKLa_YVrXR" }, "outputs": [], "source": [ "image_gen = ImageDataGenerator(rescale=1./255, zoom_range=0.5)\n", "\n", "train_data_gen = image_gen.flow_from_directory(batch_size=BATCH_SIZE,\n", " directory=train_dir,\n", " shuffle=True,\n", " target_size=(IMG_SHAPE, IMG_SHAPE))\n" ] }, { "cell_type": "markdown", "metadata": { "id": "WgPWieSZcctO" }, "source": [ "One more time, take a sample image from our training set and repeat it. The augmentation will be randomly applied (or not) to each repetition." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "VOvTs32FVrXU" }, "outputs": [], "source": [ "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "usS13KCNVrXd" }, "source": [ "### Putting it all together" ] }, { "cell_type": "markdown", "metadata": { "id": "OC8fIsalVrXd" }, "source": [ "We can apply all these augmentations, and even others, with just one line of code, by passing the augmentations as arguments with proper values.\n", "\n", "Here, we have applied rescale, rotation of 45 degrees, width shift, height shift, horizontal flip, and zoom augmentation to our training images." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gnr2xujaVrXe" }, "outputs": [], "source": [ "image_gen_train = ImageDataGenerator(\n", " rescale=1./255,\n", " rotation_range=40,\n", " width_shift_range=0.2,\n", " height_shift_range=0.2,\n", " shear_range=0.2,\n", " zoom_range=0.2,\n", " horizontal_flip=True,\n", " fill_mode='nearest')\n", "\n", "train_data_gen = image_gen_train.flow_from_directory(batch_size=BATCH_SIZE,\n", " directory=train_dir,\n", " shuffle=True,\n", " target_size=(IMG_SHAPE,IMG_SHAPE),\n", " class_mode='binary')" ] }, { "cell_type": "markdown", "metadata": { "id": "AW-pV5awVrXl" }, "source": [ "Let's visualize how a single image would look like five different times, when we pass these augmentations randomly to our dataset. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "z2m68eMhVrXm" }, "outputs": [], "source": [ "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "J8cUd7FXVrXq" }, "source": [ "### Creating Validation Data generator" ] }, { "cell_type": "markdown", "metadata": { "id": "a99fDBt7VrXr" }, "source": [ "Generally, we only apply data augmentation to our training examples, since the original images should be representative of what our model needs to manage. So, in this case we are only rescaling our validation images and converting them into batches using ImageDataGenerator." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "54x0aNbKVrXr" }, "outputs": [], "source": [ "image_gen_val = ImageDataGenerator(rescale=1./255)\n", "\n", "val_data_gen = image_gen_val.flow_from_directory(batch_size=BATCH_SIZE,\n", " directory=validation_dir,\n", " target_size=(IMG_SHAPE, IMG_SHAPE),\n", " class_mode='binary')" ] }, { "cell_type": "markdown", "metadata": { "id": "b5Ej-HLGVrWZ" }, "source": [ "# Model Creation" ] }, { "cell_type": "markdown", "metadata": { "id": "wEgW4i18VrWZ" }, "source": [ "## Define the model\n", "\n", "The model consists of four convolution blocks with a max pool layer in each of them.\n", "\n", "Before the final Dense layers, we're also applying a Dropout probability of 0.5. It means that 50% of the values coming into the Dropout layer will be set to zero. This helps to prevent overfitting.\n", "\n", "Then we have a fully connected layer with 512 units, with a `relu` activation function. The model will output class probabilities for two classes — dogs and cats — using `softmax`. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "Evjf8jZk2zi-" }, "outputs": [], "source": [ "model = tf.keras.models.Sequential([\n", " tf.keras.layers.Conv2D(32, (3,3), activation='relu', input_shape=(150, 150, 3)),\n", " tf.keras.layers.MaxPooling2D(2, 2),\n", "\n", " tf.keras.layers.Conv2D(64, (3,3), activation='relu'),\n", " tf.keras.layers.MaxPooling2D(2,2),\n", "\n", " tf.keras.layers.Conv2D(128, (3,3), activation='relu'),\n", " tf.keras.layers.MaxPooling2D(2,2),\n", "\n", " tf.keras.layers.Conv2D(128, (3,3), activation='relu'),\n", " tf.keras.layers.MaxPooling2D(2,2),\n", "\n", " tf.keras.layers.Dropout(0.5),\n", " tf.keras.layers.Flatten(),\n", " tf.keras.layers.Dense(512, activation='relu'),\n", " tf.keras.layers.Dense(2)\n", "])" ] }, { "cell_type": "markdown", "metadata": { "id": "DADWLqMSJcH3" }, "source": [ "### Compiling the model\n", "\n", "As usual, we will use the `adam` optimizer. Since we output a softmax categorization, we'll use `sparse_categorical_crossentropy` as the loss function. We would also like to look at training and validation accuracy on each epoch as we train our network, so we are passing in the metrics argument." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "08rRJ0sn3Tb1" }, "outputs": [], "source": [ "model.compile(optimizer='adam',\n", " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { "id": "uurnCp_H4Hj9" }, "source": [ "### Model Summary\n", "\n", "Let's look at all the layers of our network using **summary** method." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "b66qAJF_4Jnw" }, "outputs": [], "source": [ "model.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "N06iqE8VVrWj" }, "source": [ "### Train the model" ] }, { "cell_type": "markdown", "metadata": { "id": "oub9RtoFVrWk" }, "source": [ "It's time we train our network.\n", "\n", "Since our batches are coming from a generator (`ImageDataGenerator`), we'll use `fit_generator` instead of `fit`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tk5NT1PW3j_P" }, "outputs": [], "source": [ "epochs=100\n", "history = model.fit_generator(\n", " train_data_gen,\n", " steps_per_epoch=int(np.ceil(total_train / float(BATCH_SIZE))),\n", " epochs=epochs,\n", " validation_data=val_data_gen,\n", " validation_steps=int(np.ceil(total_val / float(BATCH_SIZE)))\n", ")" ] }, { "cell_type": "markdown", "metadata": { "id": "ojJNteAGVrWo" }, "source": [ "### Visualizing results of the training" ] }, { "cell_type": "markdown", "metadata": { "id": "LZPYT-EmVrWo" }, "source": [ "We'll now visualize the results we get after training our network." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "8CfngybnFHQR" }, "outputs": [], "source": [ "acc = history.history['accuracy']\n", "val_acc = history.history['val_accuracy']\n", "\n", "loss = history.history['loss']\n", "val_loss = history.history['val_loss']\n", "\n", "epochs_range = range(epochs)\n", "\n", "plt.figure(figsize=(8, 8))\n", "plt.subplot(1, 2, 1)\n", "plt.plot(epochs_range, acc, label='Training Accuracy')\n", "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", "plt.legend(loc='lower right')\n", "plt.title('Training and Validation Accuracy')\n", "\n", "plt.subplot(1, 2, 2)\n", "plt.plot(epochs_range, loss, label='Training Loss')\n", "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", "plt.legend(loc='upper right')\n", "plt.title('Training and Validation Loss')\n", "plt.show()" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l05c02_dogs_vs_cats_with_augmentation.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l05c03_exercise_flowers_with_data_augmentation.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "TBFXQGKYUc4X" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "1z4xy2gTUc4a" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "FE7KNzPPVrVV" }, "source": [ "# Image Classification using tf.keras" ] }, { "cell_type": "markdown", "metadata": { "id": "KwQtSOz0VrVX" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "gN7G9GFmVrVY" }, "source": [ "In this Colab you will classify images of flowers. You will build an image classifier using `tf.keras.Sequential` model and load data using `tf.keras.preprocessing.image.ImageDataGenerator`.\n" ] }, { "cell_type": "markdown", "metadata": { "id": "zF9uvbXNVrVY" }, "source": [ "# Importing Packages" ] }, { "cell_type": "markdown", "metadata": { "id": "VddxeYBEVrVZ" }, "source": [ "Let's start by importing required packages. **os** package is used to read files and directory structure, **numpy** is used to convert python list to numpy array and to perform required matrix operations and **matplotlib.pyplot** is used to plot the graph and display images in our training and validation data." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "rtPGh2MAVrVa" }, "outputs": [], "source": [ "import os\n", "import numpy as np\n", "import glob\n", "import shutil\n", "\n", "import tensorflow as tf\n", "\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "markdown", "metadata": { "id": "Jlchl4x2VrVg" }, "source": [ "### TODO: Import TensorFlow and Keras Layers\n", "\n", "In the cell below, import Tensorflow as `tf` and the Keras layers and models you will use to build your CNN. Also, import the `ImageDataGenerator` from Keras so that you can perform image augmentation." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "L1WtoaOHVrVh" }, "outputs": [], "source": [ "#import packages" ] }, { "cell_type": "markdown", "metadata": { "id": "UZZI6lNkVrVm" }, "source": [ "# Data Loading" ] }, { "cell_type": "markdown", "metadata": { "id": "DPHx8-t-VrVo" }, "source": [ "In order to build our image classifier, we can begin by downloading the flowers dataset. We first need to download the archive version of the dataset and after the download we are storing it to \"/tmp/\" directory." ] }, { "cell_type": "markdown", "metadata": { "id": "_lPjfOmNVrVs" }, "source": [ "After downloading the dataset, we need to extract its contents." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "OYmOylPlVrVt" }, "outputs": [], "source": [ "_URL = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", "\n", "zip_file = tf.keras.utils.get_file(origin=_URL,\n", " fname=\"flower_photos.tgz\",\n", " extract=True)\n", "\n", "base_dir = os.path.join(os.path.dirname(zip_file), 'flower_photos')" ] }, { "cell_type": "markdown", "metadata": { "id": "2yge5MKnnjMd" }, "source": [ "The dataset we downloaded contains images of 5 types of flowers:\n", "\n", "1. Rose\n", "2. Daisy\n", "3. Dandelion\n", "4. Sunflowers\n", "5. Tulips\n", "\n", "So, let's create the labels for these 5 classes: " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FiYVs1MEmNHf" }, "outputs": [], "source": [ "classes = ['roses', 'daisy', 'dandelion', 'sunflowers', 'tulips']" ] }, { "cell_type": "markdown", "metadata": { "id": "G1ymuCPS0_eu" }, "source": [ "Also, the dataset we have downloaded has following directory structure.\n", "\n", "
\n",
        "flower_photos\n",
        "|__ daisy\n",
        "|__ dandelion\n",
        "|__ roses\n",
        "|__ sunflowers\n",
        "|__ tulips\n",
        "
\n", "\n", "As you can see there are no folders containing training and validation data. Therefore, we will have to create our own training and validation set. Let's write some code that will do this.\n", "\n", "\n", "The code below creates a `train` and a `val` folder each containing 5 folders (one for each type of flower). It then moves the images from the original folders to these new folders such that 80% of the images go to the training set and 20% of the images go into the validation set. In the end our directory will have the following structure:\n", "\n", "\n", "
\n",
        "flower_photos\n",
        "|__ daisy\n",
        "|__ dandelion\n",
        "|__ roses\n",
        "|__ sunflowers\n",
        "|__ tulips\n",
        "|__ train\n",
        "    |______ daisy: [1.jpg, 2.jpg, 3.jpg ....]\n",
        "    |______ dandelion: [1.jpg, 2.jpg, 3.jpg ....]\n",
        "    |______ roses: [1.jpg, 2.jpg, 3.jpg ....]\n",
        "    |______ sunflowers: [1.jpg, 2.jpg, 3.jpg ....]\n",
        "    |______ tulips: [1.jpg, 2.jpg, 3.jpg ....]\n",
        " |__ val\n",
        "    |______ daisy: [507.jpg, 508.jpg, 509.jpg ....]\n",
        "    |______ dandelion: [719.jpg, 720.jpg, 721.jpg ....]\n",
        "    |______ roses: [514.jpg, 515.jpg, 516.jpg ....]\n",
        "    |______ sunflowers: [560.jpg, 561.jpg, 562.jpg .....]\n",
        "    |______ tulips: [640.jpg, 641.jpg, 642.jpg ....]\n",
        "
\n", "\n", "Since we don't delete the original folders, they will still be in our `flower_photos` directory, but they will be empty. The code below also prints the total number of flower images we have for each type of flower. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "a-AL030LmcdD" }, "outputs": [], "source": [ "for cl in classes:\n", " img_path = os.path.join(base_dir, cl)\n", " images = glob.glob(img_path + '/*.jpg')\n", " print(\"{}: {} Images\".format(cl, len(images)))\n", " train, val = images[:round(len(images)*0.8)], images[round(len(images)*0.8):]\n", "\n", " for t in train:\n", " if not os.path.exists(os.path.join(base_dir, 'train', cl)):\n", " os.makedirs(os.path.join(base_dir, 'train', cl))\n", " shutil.move(t, os.path.join(base_dir, 'train', cl))\n", "\n", " for v in val:\n", " if not os.path.exists(os.path.join(base_dir, 'val', cl)):\n", " os.makedirs(os.path.join(base_dir, 'val', cl))\n", " shutil.move(v, os.path.join(base_dir, 'val', cl))" ] }, { "cell_type": "markdown", "metadata": { "id": "8Lp-0ejxOtP1" }, "source": [ "For convenience, let us set up the path for the training and validation sets" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "uh68rmWspp0U" }, "outputs": [], "source": [ "train_dir = os.path.join(base_dir, 'train')\n", "val_dir = os.path.join(base_dir, 'val')" ] }, { "cell_type": "markdown", "metadata": { "id": "UOoVpxFwVrWy" }, "source": [ "# Data Augmentation" ] }, { "cell_type": "markdown", "metadata": { "id": "Wn_QLciWVrWy" }, "source": [ "Overfitting generally occurs when we have small number of training examples. One way to fix this problem is to augment our dataset so that it has sufficient number of training examples. Data augmentation takes the approach of generating more training data from existing training samples, by augmenting the samples via a number of random transformations that yield believable-looking images. The goal is that at training time, your model will never see the exact same picture twice. This helps expose the model to more aspects of the data and generalize better.\n", "\n", "In **tf.keras** we can implement this using the same **ImageDataGenerator** class we used before. We can simply pass different transformations we would want to our dataset as a form of arguments and it will take care of applying it to the dataset during our training process. " ] }, { "cell_type": "markdown", "metadata": { "id": "2uJ1G030VrWz" }, "source": [ "## Experiment with Various Image Transformations\n", "\n", "In this section you will get some practice doing some basic image transformations. Before we begin making transformations let's define our `batch_size` and our image size. Remember that the input to our CNN are images of the same size. We therefore have to resize the images in our dataset to the same size.\n", "\n", "### TODO: Set Batch and Image Size\n", "\n", "In the cell below, create a `batch_size` of 100 images and set a value to `IMG_SHAPE` such that our training data consists of images with width of 150 pixels and height of 150 pixels." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "QyPkET61yMMX" }, "outputs": [], "source": [ "batch_size =\n", "IMG_SHAPE = " ] }, { "cell_type": "markdown", "metadata": { "id": "rlVj6VqaVrW0" }, "source": [ "### TODO: Apply Random Horizontal Flip\n", "\n", "In the cell below, use ImageDataGenerator to create a transformation that rescales the images by 255 and then applies a random horizontal flip. Then use the `.flow_from_directory` method to apply the above transformation to the images in our training set. Make sure you indicate the batch size, the path to the directory of the training images, the target size for the images, and to shuffle the images. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Bi1_vHyBVrW2" }, "outputs": [], "source": [ "image_gen = \n", "\n", "train_data_gen = " ] }, { "cell_type": "markdown", "metadata": { "id": "zJpRSxJ-VrW7" }, "source": [ "Let's take 1 sample image from our training examples and repeat it 5 times so that the augmentation can be applied to the same image 5 times over randomly, to see the augmentation in action." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jqb9OGoVKIOi" }, "outputs": [], "source": [ "# This function will plot images in the form of a grid with 1 row and 5 columns where images are placed in each column.\n", "def plotImages(images_arr):\n", " fig, axes = plt.subplots(1, 5, figsize=(20,20))\n", " axes = axes.flatten()\n", " for img, ax in zip( images_arr, axes):\n", " ax.imshow(img)\n", " plt.tight_layout()\n", " plt.show()\n", "\n", "\n", "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "qXnwkzFuVrXB" }, "source": [ "### TODO: Apply Random Rotation\n", "\n", "In the cell below, use ImageDataGenerator to create a transformation that rescales the images by 255 and then applies a random 45 degree rotation. Then use the `.flow_from_directory` method to apply the above transformation to the images in our training set. Make sure you indicate the batch size, the path to the directory of the training images, the target size for the images, and to shuffle the images. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1zip35pDVrXB" }, "outputs": [], "source": [ "image_gen = \n", "\n", "train_data_gen = " ] }, { "cell_type": "markdown", "metadata": { "id": "m2lNPWFc3Bre" }, "source": [ "Let's take 1 sample image from our training examples and repeat it 5 times so that the augmentation can be applied to the same image 5 times over randomly, to see the augmentation in action." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wmBx8NhrVrXK" }, "outputs": [], "source": [ "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "NvqXaD8BVrXN" }, "source": [ "### TODO: Apply Random Zoom\n", "\n", "In the cell below, use ImageDataGenerator to create a transformation that rescales the images by 255 and then applies a random zoom of up to 50%. Then use the `.flow_from_directory` method to apply the above transformation to the images in our training set. Make sure you indicate the batch size, the path to the directory of the training images, the target size for the images, and to shuffle the images. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tGNKLa_YVrXR" }, "outputs": [], "source": [ "image_gen = \n", "\n", "train_data_gen = " ] }, { "cell_type": "markdown", "metadata": { "id": "XuoAX0Za4zTk" }, "source": [ "Let's take 1 sample image from our training examples and repeat it 5 times so that the augmentation can be applied to the same image 5 times over randomly, to see the augmentation in action." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-KQWw8IZVrXZ" }, "outputs": [], "source": [ "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "OC8fIsalVrXd" }, "source": [ "### TODO: Put It All Together\n", "\n", "In the cell below, use ImageDataGenerator to create a transformation that rescales the images by 255 and that applies:\n", "\n", "- random 45 degree rotation\n", "- random zoom of up to 50%\n", "- random horizontal flip\n", "- width shift of 0.15\n", "- height shift of 0.15\n", "\n", "Then use the `.flow_from_directory` method to apply the above transformation to the images in our training set. Make sure you indicate the batch size, the path to the directory of the training images, the target size for the images, to shuffle the images, and to set the class mode to `sparse`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gnr2xujaVrXe" }, "outputs": [], "source": [ "image_gen_train = \n", "\n", "\n", "train_data_gen = " ] }, { "cell_type": "markdown", "metadata": { "id": "AW-pV5awVrXl" }, "source": [ "Let's visualize how a single image would look like 5 different times, when we pass these augmentations randomly to our dataset. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "z2m68eMhVrXm" }, "outputs": [], "source": [ "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "a99fDBt7VrXr" }, "source": [ "### TODO: Create a Data Generator for the Validation Set\n", "\n", "Generally, we only apply data augmentation to our training examples. So, in the cell below, use ImageDataGenerator to create a transformation that only rescales the images by 255. Then use the `.flow_from_directory` method to apply the above transformation to the images in our validation set. Make sure you indicate the batch size, the path to the directory of the validation images, the target size for the images, and to set the class mode to `sparse`. Remember that it is not necessary to shuffle the images in the validation set. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "54x0aNbKVrXr" }, "outputs": [], "source": [ "image_gen_val = \n", "val_data_gen = " ] }, { "cell_type": "markdown", "metadata": { "id": "wEgW4i18VrWZ" }, "source": [ "# TODO: Create the CNN\n", "\n", "In the cell below, create a convolutional neural network that consists of 3 convolution blocks. Each convolutional block contains a `Conv2D` layer followed by a max pool layer. The first convolutional block should have 16 filters, the second one should have 32 filters, and the third one should have 64 filters. All convolutional filters should be 3 x 3. All max pool layers should have a `pool_size` of `(2, 2)`.\n", "\n", "After the 3 convolutional blocks you should have a flatten layer followed by a fully connected layer with 512 units. The CNN should output class probabilities based on 5 classes which is done by the **softmax** activation function. All other layers should use a **relu** activation function. You should also add Dropout layers with a probability of 20%, where appropriate. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Evjf8jZk2zi-" }, "outputs": [], "source": [ "model = " ] }, { "cell_type": "markdown", "metadata": { "id": "DADWLqMSJcH3" }, "source": [ "# TODO: Compile the Model\n", "\n", "In the cell below, compile your model using the ADAM optimizer, the sparse cross entropy function as a loss function. We would also like to look at training and validation accuracy on each epoch as we train our network, so make sure you also pass the metrics argument." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "08rRJ0sn3Tb1" }, "outputs": [], "source": [ "# Compile the model" ] }, { "cell_type": "markdown", "metadata": { "id": "oub9RtoFVrWk" }, "source": [ "# TODO: Train the Model\n", "\n", "In the cell below, train your model using the **fit_generator** function instead of the usual **fit** function. We have to use the `fit_generator` function because we are using the **ImageDataGenerator** class to generate batches of training and validation data for our model. Train the model for 80 epochs and make sure you use the proper parameters in the `fit_generator` function." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tk5NT1PW3j_P" }, "outputs": [], "source": [ "epochs = \n", "\n", "history = " ] }, { "cell_type": "markdown", "metadata": { "id": "LZPYT-EmVrWo" }, "source": [ "# TODO: Plot Training and Validation Graphs.\n", "\n", "In the cell below, plot the training and validation accuracy/loss graphs." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "8CfngybnFHQR" }, "outputs": [], "source": [ "acc = \n", "val_acc = \n", "\n", "loss = \n", "val_loss = \n", "\n", "epochs_range = \n" ] }, { "cell_type": "markdown", "metadata": { "id": "E9FROGm-KGEG" }, "source": [ "# TODO: Experiment with Different Parameters\n", "\n", "So far you've created a CNN with 3 convolutional layers and followed by a fully connected layer with 512 units. In the cells below create a new CNN with a different architecture. Feel free to experiment by changing as many parameters as you like. For example, you can add more convolutional layers, or more fully connected layers. You can also experiment with different filter sizes in your convolutional layers, different number of units in your fully connected layers, different dropout rates, etc... You can also experiment by performing image augmentation with more image transformations that we have seen so far. Take a look at the [ImageDataGenerator Documentation](https://keras.io/preprocessing/image/) to see a full list of all the available image transformations. For example, you can add shear transformations, or you can vary the brightness of the images, etc... Experiment as much as you can and compare the accuracy of your various models. Which parameters give you the best result?" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l05c03_exercise_flowers_with_data_augmentation.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l05c04_exercise_flowers_with_data_augmentation_solution.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "TBFXQGKYUc4X" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "1z4xy2gTUc4a" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "FE7KNzPPVrVV" }, "source": [ "# Image Classification using tf.keras" ] }, { "cell_type": "markdown", "metadata": { "id": "KwQtSOz0VrVX" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "gN7G9GFmVrVY" }, "source": [ "In this Colab you will classify images of flowers. You will build an image classifier using `tf.keras.Sequential` model and load data using `tf.keras.preprocessing.image.ImageDataGenerator`.\n" ] }, { "cell_type": "markdown", "metadata": { "id": "zF9uvbXNVrVY" }, "source": [ "# Importing Packages" ] }, { "cell_type": "markdown", "metadata": { "id": "VddxeYBEVrVZ" }, "source": [ "Let's start by importing required packages. **os** package is used to read files and directory structure, **numpy** is used to convert python list to numpy array and to perform required matrix operations and **matplotlib.pyplot** is used to plot the graph and display images in our training and validation data." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "rtPGh2MAVrVa" }, "outputs": [], "source": [ "import os\n", "import numpy as np\n", "import glob\n", "import shutil\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "markdown", "metadata": { "id": "Jlchl4x2VrVg" }, "source": [ "### TODO: Import TensorFlow and Keras Layers\n", "\n", "In the cell below, import Tensorflow and the Keras layers and models you will use to build your CNN. Also, import the `ImageDataGenerator` from Keras so that you can perform image augmentation." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "KzKpWdemHbC_" }, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "L1WtoaOHVrVh" }, "outputs": [], "source": [ "from tensorflow.keras.models import Sequential\n", "from tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout, MaxPooling2D\n", "from tensorflow.keras.preprocessing.image import ImageDataGenerator" ] }, { "cell_type": "markdown", "metadata": { "id": "UZZI6lNkVrVm" }, "source": [ "# Data Loading" ] }, { "cell_type": "markdown", "metadata": { "id": "DPHx8-t-VrVo" }, "source": [ "In order to build our image classifier, we can begin by downloading the flowers dataset. We first need to download the archive version of the dataset and after the download we are storing it to \"/tmp/\" directory." ] }, { "cell_type": "markdown", "metadata": { "id": "_lPjfOmNVrVs" }, "source": [ "After downloading the dataset, we need to extract its contents." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "OYmOylPlVrVt" }, "outputs": [], "source": [ "_URL = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", "\n", "zip_file = tf.keras.utils.get_file(origin=_URL,\n", " fname=\"flower_photos.tgz\",\n", " extract=True)\n", "\n", "base_dir = os.path.join(os.path.dirname(zip_file), 'flower_photos')" ] }, { "cell_type": "markdown", "metadata": { "id": "2yge5MKnnjMd" }, "source": [ "The dataset we downloaded contains images of 5 types of flowers:\n", "\n", "1. Rose\n", "2. Daisy\n", "3. Dandelion\n", "4. Sunflowers\n", "5. Tulips\n", "\n", "So, let's create the labels for these 5 classes: " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FiYVs1MEmNHf" }, "outputs": [], "source": [ "classes = ['roses', 'daisy', 'dandelion', 'sunflowers', 'tulips']" ] }, { "cell_type": "markdown", "metadata": { "id": "G1ymuCPS0_eu" }, "source": [ "Also, the dataset we have downloaded has following directory structure. n\n", "
\n",
        "flower_photos\n",
        "|__ daisy\n",
        "|__ dandelion\n",
        "|__ roses\n",
        "|__ sunflowers\n",
        "|__ tulips\n",
        "
\n", "\n", "As you can see there are no folders containing training and validation data. Therefore, we will have to create our own training and validation set. Let's write some code that will do this.\n", "\n", "\n", "The code below creates a `train` and a `val` folder each containing 5 folders (one for each type of flower). It then moves the images from the original folders to these new folders such that 80% of the images go to the training set and 20% of the images go into the validation set. In the end our directory will have the following structure:\n", "\n", "\n", "
\n",
        "flower_photos\n",
        "|__ daisy\n",
        "|__ dandelion\n",
        "|__ roses\n",
        "|__ sunflowers\n",
        "|__ tulips\n",
        "|__ train\n",
        "    |______ daisy: [1.jpg, 2.jpg, 3.jpg ....]\n",
        "    |______ dandelion: [1.jpg, 2.jpg, 3.jpg ....]\n",
        "    |______ roses: [1.jpg, 2.jpg, 3.jpg ....]\n",
        "    |______ sunflowers: [1.jpg, 2.jpg, 3.jpg ....]\n",
        "    |______ tulips: [1.jpg, 2.jpg, 3.jpg ....]\n",
        " |__ val\n",
        "    |______ daisy: [507.jpg, 508.jpg, 509.jpg ....]\n",
        "    |______ dandelion: [719.jpg, 720.jpg, 721.jpg ....]\n",
        "    |______ roses: [514.jpg, 515.jpg, 516.jpg ....]\n",
        "    |______ sunflowers: [560.jpg, 561.jpg, 562.jpg .....]\n",
        "    |______ tulips: [640.jpg, 641.jpg, 642.jpg ....]\n",
        "
\n", "\n", "Since we don't delete the original folders, they will still be in our `flower_photos` directory, but they will be empty. The code below also prints the total number of flower images we have for each type of flower. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "a-AL030LmcdD" }, "outputs": [], "source": [ "for cl in classes:\n", " img_path = os.path.join(base_dir, cl)\n", " images = glob.glob(img_path + '/*.jpg')\n", " print(\"{}: {} Images\".format(cl, len(images)))\n", " num_train = int(round(len(images)*0.8))\n", " train, val = images[:num_train], images[num_train:]\n", "\n", " for t in train:\n", " if not os.path.exists(os.path.join(base_dir, 'train', cl)):\n", " os.makedirs(os.path.join(base_dir, 'train', cl))\n", " shutil.move(t, os.path.join(base_dir, 'train', cl))\n", "\n", " for v in val:\n", " if not os.path.exists(os.path.join(base_dir, 'val', cl)):\n", " os.makedirs(os.path.join(base_dir, 'val', cl))\n", " shutil.move(v, os.path.join(base_dir, 'val', cl))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "yP85YhYol8ER" }, "outputs": [], "source": [ "round(len(images)*0.8)" ] }, { "cell_type": "markdown", "metadata": { "id": "8Lp-0ejxOtP1" }, "source": [ "For convenience, let us set up the path for the training and validation sets" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "uh68rmWspp0U" }, "outputs": [], "source": [ "train_dir = os.path.join(base_dir, 'train')\n", "val_dir = os.path.join(base_dir, 'val')" ] }, { "cell_type": "markdown", "metadata": { "id": "UOoVpxFwVrWy" }, "source": [ "# Data Augmentation" ] }, { "cell_type": "markdown", "metadata": { "id": "Wn_QLciWVrWy" }, "source": [ "Overfitting generally occurs when we have small number of training examples. One way to fix this problem is to augment our dataset so that it has sufficient number of training examples. Data augmentation takes the approach of generating more training data from existing training samples, by augmenting the samples via a number of random transformations that yield believable-looking images. The goal is that at training time, your model will never see the exact same picture twice. This helps expose the model to more aspects of the data and generalize better.\n", "\n", "In **tf.keras** we can implement this using the same **ImageDataGenerator** class we used before. We can simply pass different transformations we would want to our dataset as a form of arguments and it will take care of applying it to the dataset during our training process. " ] }, { "cell_type": "markdown", "metadata": { "id": "2uJ1G030VrWz" }, "source": [ "## Experiment with Various Image Transformations\n", "\n", "In this section you will get some practice doing some basic image transformations. Before we begin making transformations let's define our `batch_size` and our image size. Remember that the input to our CNN are images of the same size. We therefore have to resize the images in our dataset to the same size.\n", "\n", "### TODO: Set Batch and Image Size\n", "\n", "In the cell below, create a `batch_size` of 100 images and set a value to `IMG_SHAPE` such that our training data consists of images with width of 150 pixels and height of 150 pixels." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "QyPkET61yMMX" }, "outputs": [], "source": [ "batch_size = 100\n", "IMG_SHAPE = 150 " ] }, { "cell_type": "markdown", "metadata": { "id": "rlVj6VqaVrW0" }, "source": [ "### TODO: Apply Random Horizontal Flip\n", "\n", "In the cell below, use ImageDataGenerator to create a transformation that rescales the images by 255 and then applies a random horizontal flip. Then use the `.flow_from_directory` method to apply the above transformation to the images in our training set. Make sure you indicate the batch size, the path to the directory of the training images, the target size for the images, and to shuffle the images. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Bi1_vHyBVrW2" }, "outputs": [], "source": [ "image_gen = ImageDataGenerator(rescale=1./255, horizontal_flip=True)\n", "\n", "train_data_gen = image_gen.flow_from_directory(\n", " batch_size=batch_size,\n", " directory=train_dir,\n", " shuffle=True,\n", " target_size=(IMG_SHAPE,IMG_SHAPE)\n", " )" ] }, { "cell_type": "markdown", "metadata": { "id": "zJpRSxJ-VrW7" }, "source": [ "Let's take 1 sample image from our training examples and repeat it 5 times so that the augmentation can be applied to the same image 5 times over randomly, to see the augmentation in action." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jqb9OGoVKIOi" }, "outputs": [], "source": [ "# This function will plot images in the form of a grid with 1 row and 5 columns where images are placed in each column.\n", "def plotImages(images_arr):\n", " fig, axes = plt.subplots(1, 5, figsize=(20,20))\n", " axes = axes.flatten()\n", " for img, ax in zip( images_arr, axes):\n", " ax.imshow(img)\n", " plt.tight_layout()\n", " plt.show()\n", "\n", "\n", "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "qXnwkzFuVrXB" }, "source": [ "### TODO: Apply Random Rotation\n", "\n", "In the cell below, use ImageDataGenerator to create a transformation that rescales the images by 255 and then applies a random 45 degree rotation. Then use the `.flow_from_directory` method to apply the above transformation to the images in our training set. Make sure you indicate the batch size, the path to the directory of the training images, the target size for the images, and to shuffle the images. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1zip35pDVrXB" }, "outputs": [], "source": [ "image_gen = ImageDataGenerator(rescale=1./255, rotation_range=45)\n", "\n", "train_data_gen = image_gen.flow_from_directory(batch_size=batch_size,\n", " directory=train_dir,\n", " shuffle=True,\n", " target_size=(IMG_SHAPE, IMG_SHAPE))" ] }, { "cell_type": "markdown", "metadata": { "id": "m2lNPWFc3Bre" }, "source": [ "Let's take 1 sample image from our training examples and repeat it 5 times so that the augmentation can be applied to the same image 5 times over randomly, to see the augmentation in action." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wmBx8NhrVrXK" }, "outputs": [], "source": [ "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "NvqXaD8BVrXN" }, "source": [ "### TODO: Apply Random Zoom\n", "\n", "In the cell below, use ImageDataGenerator to create a transformation that rescales the images by 255 and then applies a random zoom of up to 50%. Then use the `.flow_from_directory` method to apply the above transformation to the images in our training set. Make sure you indicate the batch size, the path to the directory of the training images, the target size for the images, and to shuffle the images. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tGNKLa_YVrXR" }, "outputs": [], "source": [ "image_gen = ImageDataGenerator(rescale=1./255, zoom_range=0.5)\n", "\n", "train_data_gen = image_gen.flow_from_directory(\n", " batch_size=batch_size,\n", " directory=train_dir,\n", " shuffle=True,\n", " target_size=(IMG_SHAPE, IMG_SHAPE)\n", " )" ] }, { "cell_type": "markdown", "metadata": { "id": "XuoAX0Za4zTk" }, "source": [ "Let's take 1 sample image from our training examples and repeat it 5 times so that the augmentation can be applied to the same image 5 times over randomly, to see the augmentation in action." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-KQWw8IZVrXZ" }, "outputs": [], "source": [ "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "OC8fIsalVrXd" }, "source": [ "### TODO: Put It All Together\n", "\n", "In the cell below, use ImageDataGenerator to create a transformation that rescales the images by 255 and that applies:\n", "\n", "- random 45 degree rotation\n", "- random zoom of up to 50%\n", "- random horizontal flip\n", "- width shift of 0.15\n", "- height shift of 0.15\n", "\n", "Then use the `.flow_from_directory` method to apply the above transformation to the images in our training set. Make sure you indicate the batch size, the path to the directory of the training images, the target size for the images, to shuffle the images, and to set the class mode to `sparse`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gnr2xujaVrXe" }, "outputs": [], "source": [ "image_gen_train = ImageDataGenerator(\n", " rescale=1./255,\n", " rotation_range=45,\n", " width_shift_range=.15,\n", " height_shift_range=.15,\n", " horizontal_flip=True,\n", " zoom_range=0.5\n", " )\n", "\n", "\n", "train_data_gen = image_gen_train.flow_from_directory(\n", " batch_size=batch_size,\n", " directory=train_dir,\n", " shuffle=True,\n", " target_size=(IMG_SHAPE,IMG_SHAPE),\n", " class_mode='sparse'\n", " )" ] }, { "cell_type": "markdown", "metadata": { "id": "AW-pV5awVrXl" }, "source": [ "Let's visualize how a single image would look like 5 different times, when we pass these augmentations randomly to our dataset. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "z2m68eMhVrXm" }, "outputs": [], "source": [ "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", "plotImages(augmented_images)" ] }, { "cell_type": "markdown", "metadata": { "id": "a99fDBt7VrXr" }, "source": [ "### TODO: Create a Data Generator for the Validation Set\n", "\n", "Generally, we only apply data augmentation to our training examples. So, in the cell below, use ImageDataGenerator to create a transformation that only rescales the images by 255. Then use the `.flow_from_directory` method to apply the above transformation to the images in our validation set. Make sure you indicate the batch size, the path to the directory of the validation images, the target size for the images, and to set the class mode to `sparse`. Remember that it is not necessary to shuffle the images in the validation set. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "54x0aNbKVrXr" }, "outputs": [], "source": [ "image_gen_val = ImageDataGenerator(rescale=1./255)\n", "\n", "val_data_gen = image_gen_val.flow_from_directory(batch_size=batch_size,\n", " directory=val_dir,\n", " target_size=(IMG_SHAPE, IMG_SHAPE),\n", " class_mode='sparse')" ] }, { "cell_type": "markdown", "metadata": { "id": "wEgW4i18VrWZ" }, "source": [ "# TODO: Create the CNN\n", "\n", "In the cell below, create a convolutional neural network that consists of 3 convolution blocks. Each convolutional block contains a `Conv2D` layer followed by a max pool layer. The first convolutional block should have 16 filters, the second one should have 32 filters, and the third one should have 64 filters. All convolutional filters should be 3 x 3. All max pool layers should have a `pool_size` of `(2, 2)`.\n", "\n", "After the 3 convolutional blocks you should have a flatten layer followed by a fully connected layer with 512 units. The CNN should output class probabilities based on 5 classes which is done by the **softmax** activation function. All other layers should use a **relu** activation function. You should also add Dropout layers with a probability of 20%, where appropriate." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Evjf8jZk2zi-" }, "outputs": [], "source": [ "model = Sequential()\n", "\n", "model.add(Conv2D(16, 3, padding='same', activation='relu', input_shape=(IMG_SHAPE,IMG_SHAPE, 3)))\n", "model.add(MaxPooling2D(pool_size=(2, 2)))\n", "\n", "model.add(Conv2D(32, 3, padding='same', activation='relu'))\n", "model.add(MaxPooling2D(pool_size=(2, 2)))\n", "\n", "model.add(Conv2D(64, 3, padding='same', activation='relu'))\n", "model.add(MaxPooling2D(pool_size=(2, 2)))\n", "\n", "model.add(Flatten())\n", "model.add(Dropout(0.2))\n", "model.add(Dense(512, activation='relu'))\n", "\n", "model.add(Dropout(0.2))\n", "model.add(Dense(5))" ] }, { "cell_type": "markdown", "metadata": { "id": "DADWLqMSJcH3" }, "source": [ "# TODO: Compile the Model\n", "\n", "In the cell below, compile your model using the ADAM optimizer, the sparse cross entropy function as a loss function. We would also like to look at training and validation accuracy on each epoch as we train our network, so make sure you also pass the metrics argument." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "08rRJ0sn3Tb1" }, "outputs": [], "source": [ "model.compile(optimizer='adam',\n", " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { "id": "oub9RtoFVrWk" }, "source": [ "# TODO: Train the Model\n", "\n", "In the cell below, train your model using the **fit_generator** function instead of the usual **fit** function. We have to use the `fit_generator` function because we are using the **ImageDataGenerator** class to generate batches of training and validation data for our model. Train the model for 80 epochs and make sure you use the proper parameters in the `fit_generator` function." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tk5NT1PW3j_P" }, "outputs": [], "source": [ "epochs = 80\n", "\n", "history = model.fit_generator(\n", " train_data_gen,\n", " steps_per_epoch=int(np.ceil(train_data_gen.n / float(batch_size))),\n", " epochs=epochs,\n", " validation_data=val_data_gen,\n", " validation_steps=int(np.ceil(val_data_gen.n / float(batch_size)))\n", ")" ] }, { "cell_type": "markdown", "metadata": { "id": "LZPYT-EmVrWo" }, "source": [ "# TODO: Plot Training and Validation Graphs.\n", "\n", "In the cell below, plot the training and validation accuracy/loss graphs." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "8CfngybnFHQR" }, "outputs": [], "source": [ "acc = history.history['accuracy']\n", "val_acc = history.history['val_accuracy']\n", "\n", "loss = history.history['loss']\n", "val_loss = history.history['val_loss']\n", "\n", "epochs_range = range(epochs)\n", "\n", "plt.figure(figsize=(8, 8))\n", "plt.subplot(1, 2, 1)\n", "plt.plot(epochs_range, acc, label='Training Accuracy')\n", "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", "plt.legend(loc='lower right')\n", "plt.title('Training and Validation Accuracy')\n", "\n", "plt.subplot(1, 2, 2)\n", "plt.plot(epochs_range, loss, label='Training Loss')\n", "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", "plt.legend(loc='upper right')\n", "plt.title('Training and Validation Loss')\n", "plt.show()" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l05c04_exercise_flowers_with_data_augmentation_solution.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l06c01_tensorflow_hub_and_transfer_learning.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "W_tvPdyfA-BL" }, "source": [ "##### Copyright 2019 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "0O_LFhwSBCjm" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "9-3Pry4jh1-E" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "NxjpzKTvg_dd" }, "source": [ "# TensorFlow Hub and Transfer Learning" ] }, { "cell_type": "markdown", "metadata": { "id": "crU-iluJIEzw" }, "source": [ "[TensorFlow Hub](http://tensorflow.org/hub) is an online repository of already trained TensorFlow models that you can use.\n", "These models can either be used as is, or they can be used for Transfer Learning.\n", "\n", "Transfer learning is a process where you take an existing trained model, and extend it to do additional work. This involves leaving the bulk of the model unchanged, while adding and retraining the final layers, in order to get a different set of possible outputs.\n", "\n", "In this Colab we will do both.\n", "\n", "Here, you can see all the models available in [TensorFlow Module Hub](https://tfhub.dev/).\n", "\n", "## Concepts that will be covered in this Colab\n", "\n", "1. Use a TensorFlow Hub model for prediction.\n", "2. Use a TensorFlow Hub model for Dogs vs. Cats dataset.\n", "3. Do simple transfer learning with TensorFlow Hub.\n", "\n", "Before starting this Colab, you should reset the Colab environment by selecting `Runtime -> Reset all runtimes...` from menu above." ] }, { "cell_type": "markdown", "metadata": { "id": "7RVsYZLEpEWs" }, "source": [ "# Imports\n" ] }, { "cell_type": "markdown", "metadata": { "id": "ZUCEcRdhnyWn" }, "source": [ "Some normal imports we've seen before. The new one is importing tensorflow_hub which was installed above, and which this Colab will make heavy use of." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "8z5hqr0hHtLv" }, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "WZnAHGETHu7e" }, "outputs": [], "source": [ "import matplotlib.pylab as plt\n", "\n", "import tensorflow_hub as hub\n", "import tensorflow_datasets as tfds\n", "\n", "from tensorflow.keras import layers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FVM2fKGEHIJN" }, "outputs": [], "source": [ "import logging\n", "logger = tf.get_logger()\n", "logger.setLevel(logging.ERROR)" ] }, { "cell_type": "markdown", "metadata": { "id": "s4YuF5HvpM1W" }, "source": [ "# Part 1: Use a TensorFlow Hub MobileNet for prediction" ] }, { "cell_type": "markdown", "metadata": { "id": "4Sh2sPc10V0b" }, "source": [ "In this part of the Colab, we'll take a trained model, load it into to Keras, and try it out.\n", "\n", "The model that we'll use is MobileNet v2 (but any model from [tf2 compatible image classifier URL from tfhub.dev](https://tfhub.dev/s?q=tf2&module-type=image-classification) would work)." ] }, { "cell_type": "markdown", "metadata": { "id": "xEY_Ow5loN6q" }, "source": [ "## Download the classifier\n", "\n", "Download the MobileNet model and create a Keras model from it.\n", "MobileNet is expecting images of 224 $\\times$ 224 pixels, in 3 color channels (RGB)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "y_6bGjoPtzau" }, "outputs": [], "source": [ "CLASSIFIER_URL =\"https://tfhub.dev/google/tf2-preview/mobilenet_v2/classification/2\"\n", "IMAGE_RES = 224\n", "\n", "model = tf.keras.Sequential([\n", " hub.KerasLayer(CLASSIFIER_URL, input_shape=(IMAGE_RES, IMAGE_RES, 3))\n", "])" ] }, { "cell_type": "markdown", "metadata": { "id": "pwZXaoV0uXp2" }, "source": [ "## Run it on a single image" ] }, { "cell_type": "markdown", "metadata": { "id": "TQItP1i55-di" }, "source": [ "MobileNet has been trained on the ImageNet dataset. ImageNet has 1000 different output classes, and one of them is military uniforms.\n", "Let's get an image containing a military uniform that is not part of ImageNet, and see if our model can predict that it is a military uniform." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "w5wDjXNjuXGD" }, "outputs": [], "source": [ "import numpy as np\n", "import PIL.Image as Image\n", "\n", "grace_hopper = tf.keras.utils.get_file('image.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg')\n", "grace_hopper = Image.open(grace_hopper).resize((IMAGE_RES, IMAGE_RES))\n", "grace_hopper " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "BEmmBnGbLxPp" }, "outputs": [], "source": [ "grace_hopper = np.array(grace_hopper)/255.0\n", "grace_hopper.shape" ] }, { "cell_type": "markdown", "metadata": { "id": "0Ic8OEEo2b73" }, "source": [ "Remember, models always want a batch of images to process. So here, we add a batch dimension, and pass the image to the model for prediction." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "EMquyn29v8q3" }, "outputs": [], "source": [ "result = model.predict(grace_hopper[np.newaxis, ...])\n", "result.shape" ] }, { "cell_type": "markdown", "metadata": { "id": "NKzjqENF6jDF" }, "source": [ "The result is a 1001 element vector of logits, rating the probability of each class for the image.\n", "\n", "So the top class ID can be found with argmax. But how can we know what class this actually is and in particular if that class ID in the ImageNet dataset denotes a military uniform or something else?" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "rgXb44vt6goJ" }, "outputs": [], "source": [ "predicted_class = np.argmax(result[0], axis=-1)\n", "predicted_class" ] }, { "cell_type": "markdown", "metadata": { "id": "YrxLMajMoxkf" }, "source": [ "## Decode the predictions\n", "\n", "To see what our predicted_class is in the ImageNet dataset, download the ImageNet labels and fetch the row that the model predicted." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ij6SrDxcxzry" }, "outputs": [], "source": [ "labels_path = tf.keras.utils.get_file('ImageNetLabels.txt','https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt')\n", "imagenet_labels = np.array(open(labels_path).read().splitlines())\n", "\n", "plt.imshow(grace_hopper)\n", "plt.axis('off')\n", "predicted_class_name = imagenet_labels[predicted_class]\n", "_ = plt.title(\"Prediction: \" + predicted_class_name.title())" ] }, { "cell_type": "markdown", "metadata": { "id": "a6TNYYAM4u2-" }, "source": [ "Bingo. Our model correctly predicted military uniform!" ] }, { "cell_type": "markdown", "metadata": { "id": "amfzqn1Oo7Om" }, "source": [ "# Part 2: Use a TensorFlow Hub models for the Cats vs. Dogs dataset" ] }, { "cell_type": "markdown", "metadata": { "id": "K-nIpVJ94xrw" }, "source": [ "Now we'll use the full MobileNet model and see how it can perform on the Dogs vs. Cats dataset." ] }, { "cell_type": "markdown", "metadata": { "id": "Z93vvAdGxDMD" }, "source": [ "## Dataset\n", "\n", "We can use TensorFlow Datasets to load the Dogs vs Cats dataset." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DrIUV3V0xDL_" }, "outputs": [], "source": [ "(train_examples, validation_examples), info = tfds.load(\n", " 'cats_vs_dogs', \n", " with_info=True, \n", " as_supervised=True, \n", " split=['train[:80%]', 'train[80%:]'],\n", ")\n", "\n", "num_examples = info.splits['train'].num_examples\n", "num_classes = info.features['label'].num_classes" ] }, { "cell_type": "markdown", "metadata": { "id": "UlFZ_hwjCLgS" }, "source": [ "The images in the Dogs vs. Cats dataset are not all the same size." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W4lDPkn2cpWZ" }, "outputs": [], "source": [ "for i, example_image in enumerate(train_examples.take(3)):\n", " print(\"Image {} shape: {}\".format(i+1, example_image[0].shape))" ] }, { "cell_type": "markdown", "metadata": { "id": "mbgpD3E6gM2P" }, "source": [ "So we need to reformat all images to the resolution expected by MobileNet (224, 224).\n", "\n", "The `.repeat()` and `steps_per_epoch` here is not required, but saves ~15s per epoch, since the shuffle-buffer only has to cold-start once." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "we_ftzQxNf7e" }, "outputs": [], "source": [ "def format_image(image, label):\n", " image = tf.image.resize(image, (IMAGE_RES, IMAGE_RES))/255.0\n", " return image, label\n", "\n", "BATCH_SIZE = 32\n", "\n", "train_batches = train_examples.shuffle(num_examples//4).map(format_image).batch(BATCH_SIZE).prefetch(1)\n", "validation_batches = validation_examples.map(format_image).batch(BATCH_SIZE).prefetch(1)" ] }, { "cell_type": "markdown", "metadata": { "id": "0gTN7M_GxDLx" }, "source": [ "## Run the classifier on a batch of images" ] }, { "cell_type": "markdown", "metadata": { "id": "O3fvrZR8xDLv" }, "source": [ "Remember our `model` object is still the full MobileNet model trained on ImageNet, so it has 1000 possible output classes.\n", "ImageNet has a lot of dogs and cats in it, so let's see if it can predict the images in our Dogs vs. Cats dataset.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kii_jWZYOn0B" }, "outputs": [], "source": [ "image_batch, label_batch = next(iter(train_batches.take(1)))\n", "image_batch = image_batch.numpy()\n", "label_batch = label_batch.numpy()\n", "\n", "result_batch = model.predict(image_batch)\n", "\n", "predicted_class_names = imagenet_labels[np.argmax(result_batch, axis=-1)]\n", "predicted_class_names" ] }, { "cell_type": "markdown", "metadata": { "id": "QmvSWg9nxDLa" }, "source": [ "The labels seem to match names of Dogs and Cats. Let's now plot the images from our Dogs vs Cats dataset and put the ImageNet labels next to them." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "IXTB22SpxDLP" }, "outputs": [], "source": [ "plt.figure(figsize=(10,9))\n", "for n in range(30):\n", " plt.subplot(6,5,n+1)\n", " plt.subplots_adjust(hspace = 0.3)\n", " plt.imshow(image_batch[n])\n", " plt.title(predicted_class_names[n])\n", " plt.axis('off')\n", "_ = plt.suptitle(\"ImageNet predictions\")" ] }, { "cell_type": "markdown", "metadata": { "id": "JzV457OXreQP" }, "source": [ "# Part 3: Do simple transfer learning with TensorFlow Hub\n", "\n", "Let's now use TensorFlow Hub to do Transfer Learning.\n", "\n", "With transfer learning we reuse parts of an already trained model and change the final layer, or several layers, of the model, and then retrain those layers on our own dataset.\n", "\n", "In addition to complete models, TensorFlow Hub also distributes models without the last classification layer. These can be used to easily do transfer learning. We will continue using MobileNet v2 because in later parts of this course, we will take this model and deploy it on a mobile device using [TensorFlow Lite](https://www.tensorflow.org/lite). Any [image feature vector URL from tfhub.dev](https://tfhub.dev/s?module-type=image-feature-vector&q=tf2) would work here.\n", "\n", "We'll also continue to use the Dogs vs Cats dataset, so we will be able to compare the performance of this model against the ones we created from scratch earlier.\n", "\n", "Note that we're calling the partial model from TensorFlow Hub (without the final classification layer) a `feature_extractor`. The reasoning for this term is that it will take the input all the way to a layer containing a number of features. So it has done the bulk of the work in identifying the content of an image, except for creating the final probability distribution. That is, it has extracted the features of the image." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5wB030nezBwI" }, "outputs": [], "source": [ "URL = \"https://tfhub.dev/google/tf2-preview/mobilenet_v2/feature_vector/2\"\n", "feature_extractor = hub.KerasLayer(URL,\n", " input_shape=(IMAGE_RES, IMAGE_RES,3))" ] }, { "cell_type": "markdown", "metadata": { "id": "pkSvAPvKOWg2" }, "source": [ "Let's run a batch of images through this, and see the final shape. 32 is the number of images, and 1280 is the number of neurons in the last layer of the partial model from TensorFlow Hub." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Of7i-35F09ls" }, "outputs": [], "source": [ "feature_batch = feature_extractor(image_batch)\n", "print(feature_batch.shape)" ] }, { "cell_type": "markdown", "metadata": { "id": "CtFmF7A5E4tk" }, "source": [ "Freeze the variables in the feature extractor layer, so that the training only modifies the final classifier layer." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Jg5ar6rcE4H-" }, "outputs": [], "source": [ "feature_extractor.trainable = False" ] }, { "cell_type": "markdown", "metadata": { "id": "RPVeouTksO9q" }, "source": [ "## Attach a classification head\n", "\n", "Now wrap the hub layer in a `tf.keras.Sequential` model, and add a new classification layer." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "mGcY27fY1q3Q" }, "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " feature_extractor,\n", " layers.Dense(2)\n", "])\n", "\n", "model.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "OHbXQqIquFxQ" }, "source": [ "## Train the model\n", "\n", "We now train this model like any other, by first calling `compile` followed by `fit`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3n0Wb9ylKd8R" }, "outputs": [], "source": [ "model.compile(\n", " optimizer='adam',\n", " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])\n", "\n", "EPOCHS = 6\n", "history = model.fit(train_batches,\n", " epochs=EPOCHS,\n", " validation_data=validation_batches)" ] }, { "cell_type": "markdown", "metadata": { "id": "76as-K8-vFQJ" }, "source": [ "You can see we get ~97% validation accuracy, which is absolutely awesome. This is a huge improvement over the model we created in the previous lesson, where we were able to get ~83% accuracy. The reason for this difference is that MobileNet was carefully designed over a long time by experts, then trained on a massive dataset (ImageNet).\n", "\n", "Although not equivalent to TensorFlow Hub, you can check out how to create MobileNet in Keras [here](https://github.com/keras-team/keras-applications/blob/master/keras_applications/mobilenet.py).\n", "\n", "Let's plot the training and validation accuracy/loss graphs." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "d28dhbFpr98b" }, "outputs": [], "source": [ "acc = history.history['accuracy']\n", "val_acc = history.history['val_accuracy']\n", "\n", "loss = history.history['loss']\n", "val_loss = history.history['val_loss']\n", "\n", "epochs_range = range(EPOCHS)\n", "\n", "plt.figure(figsize=(8, 8))\n", "plt.subplot(1, 2, 1)\n", "plt.plot(epochs_range, acc, label='Training Accuracy')\n", "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", "plt.legend(loc='lower right')\n", "plt.title('Training and Validation Accuracy')\n", "\n", "plt.subplot(1, 2, 2)\n", "plt.plot(epochs_range, loss, label='Training Loss')\n", "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", "plt.legend(loc='upper right')\n", "plt.title('Training and Validation Loss')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "5zmoDisGvNye" }, "source": [ "What is a bit curious here is that validation performance is better than training performance, right from the start to the end of execution.\n", "\n", "One reason for this is that validation performance is measured at the end of the epoch, but training performance is the average values across the epoch.\n", "\n", "The bigger reason though is that we're reusing a large part of MobileNet which is already trained on Dogs and Cats images. While doing training, the network is still performing image augmentation on the training images, but not on the validation dataset. This means the training images may be harder to classify compared to the normal images in the validation dataset." ] }, { "cell_type": "markdown", "metadata": { "id": "kb__ZN8uFn-D" }, "source": [ "## Check the predictions\n", "\n", "To redo the plot from before, first get the ordered list of class names." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W_Zvg2i0fzJu" }, "outputs": [], "source": [ "class_names = np.array(info.features['label'].names)\n", "class_names" ] }, { "cell_type": "markdown", "metadata": { "id": "4Olg6MsNGJTL" }, "source": [ "Run the image batch through the model and convert the indices to class names." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fCLVCpEjJ_VP" }, "outputs": [], "source": [ "predicted_batch = model.predict(image_batch)\n", "predicted_batch = tf.squeeze(predicted_batch).numpy()\n", "predicted_ids = np.argmax(predicted_batch, axis=-1)\n", "predicted_class_names = class_names[predicted_ids]\n", "predicted_class_names" ] }, { "cell_type": "markdown", "metadata": { "id": "CkGbZxl9GZs-" }, "source": [ "Let's look at the true labels and predicted ones." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nL9IhOmGI5dJ" }, "outputs": [], "source": [ "print(\"Labels: \", label_batch)\n", "print(\"Predicted labels: \", predicted_ids)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wC_AYRJU9NQe" }, "outputs": [], "source": [ "plt.figure(figsize=(10,9))\n", "for n in range(30):\n", " plt.subplot(6,5,n+1)\n", " plt.subplots_adjust(hspace = 0.3)\n", " plt.imshow(image_batch[n])\n", " color = \"blue\" if predicted_ids[n] == label_batch[n] else \"red\"\n", " plt.title(predicted_class_names[n].title(), color=color)\n", " plt.axis('off')\n", "_ = plt.suptitle(\"Model predictions (blue: correct, red: incorrect)\")" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l06c01_tensorflow_hub_and_transfer_learning.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l06c02_exercise_flowers_with_transfer_learning.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "W_tvPdyfA-BL" }, "source": [ "##### Copyright 2019 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "0O_LFhwSBCjm" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "9-3Pry4jh1-E" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "NxjpzKTvg_dd" }, "source": [ "# TensorFlow Hub" ] }, { "cell_type": "markdown", "metadata": { "id": "crU-iluJIEzw" }, "source": [ "[TensorFlow Hub](http://tensorflow.org/hub) is an online repository of already trained TensorFlow models that you can use.\n", "These models can either be used as is, or they can be used for Transfer Learning.\n", "\n", "Transfer learning is a process where you take an existing trained model, and extend it to do additional work. This involves leaving the bulk of the model unchanged, while adding and retraining the final layers, in order to get a different set of possible outputs.\n", "\n", "Here, you can see all the models available in [TensorFlow Module Hub](https://tfhub.dev/).\n", "\n", "Before starting this Colab, you should reset the Colab environment by selecting `Runtime -> Reset all runtimes...` from menu above." ] }, { "cell_type": "markdown", "metadata": { "id": "7RVsYZLEpEWs" }, "source": [ "# Imports\n" ] }, { "cell_type": "markdown", "metadata": { "id": "ZUCEcRdhnyWn" }, "source": [ "Some normal imports we've seen before. The new one is importing tensorflow_hub which this Colab will make heavy use of." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zIuDCLW_IAG_" }, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dHenfza_ICJL" }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "import tensorflow_hub as hub\n", "import tensorflow_datasets as tfds\n", "\n", "from tensorflow.keras import layers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gEsgwsqbHFn2" }, "outputs": [], "source": [ "import logging\n", "logger = tf.get_logger()\n", "logger.setLevel(logging.ERROR)" ] }, { "cell_type": "markdown", "metadata": { "id": "amfzqn1Oo7Om" }, "source": [ "# TODO: Download the Flowers Dataset using TensorFlow Datasets" ] }, { "cell_type": "markdown", "metadata": { "id": "Z93vvAdGxDMD" }, "source": [ "In the cell below you will download the Flowers dataset using TensorFlow Datasets. If you look at the [TensorFlow Datasets documentation](https://www.tensorflow.org/datasets/datasets#tf_flowers) you will see that the name of the Flowers dataset is `tf_flowers`. You can also see that this dataset is only split into a TRAINING set. You will therefore have to use `tfds.splits` to split this training set into to a `training_set` and a `validation_set`. Do a `[70, 30]` split such that 70 corresponds to the `training_set` and 30 to the `validation_set`. Then load the `tf_flowers` dataset using `tfds.load`. Make sure the `tfds.load` function uses the all the parameters you need, and also make sure it returns the dataset info, so we can retrieve information about the datasets.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oXiJjX0jfx1o" }, "outputs": [], "source": [ "splits = \n", "\n", "(training_set, validation_set), dataset_info = " ] }, { "cell_type": "markdown", "metadata": { "id": "X0p1sOEHf0JF" }, "source": [ "# TODO: Print Information about the Flowers Dataset\n", "\n", "Now that you have downloaded the dataset, use the dataset info to print the number of classes in the dataset, and also write some code that counts how many images we have in the training and validation sets. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DrIUV3V0xDL_" }, "outputs": [], "source": [ "print('Total Number of Classes: {}'.format(num_classes))\n", "print('Total Number of Training Images: {}'.format(num_training_examples))\n", "print('Total Number of Validation Images: {} \\n'.format(num_validation_examples))" ] }, { "cell_type": "markdown", "metadata": { "id": "UlFZ_hwjCLgS" }, "source": [ "The images in the Flowers dataset are not all the same size." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W4lDPkn2cpWZ" }, "outputs": [], "source": [ "for i, example in enumerate(training_set.take(5)):\n", " print('Image {} shape: {} label: {}'.format(i+1, example[0].shape, example[1]))" ] }, { "cell_type": "markdown", "metadata": { "id": "mbgpD3E6gM2P" }, "source": [ "# TODO: Reformat Images and Create Batches\n", "\n", "In the cell below create a function that reformats all images to the resolution expected by MobileNet v2 (224, 224) and normalizes them. The function should take in an `image` and a `label` as arguments and should return the new `image` and corresponding `label`. Then create training and validation batches of size `32`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "we_ftzQxNf7e" }, "outputs": [], "source": [ "IMAGE_RES = \n", "\n", "def format_image(image, label):\n", " \n", " return image, label\n", "\n", "BATCH_SIZE = \n", "\n", "train_batches = \n", "\n", "validation_batches = " ] }, { "cell_type": "markdown", "metadata": { "id": "JzV457OXreQP" }, "source": [ "# Do Simple Transfer Learning with TensorFlow Hub\n", "\n", "Let's now use TensorFlow Hub to do Transfer Learning. Remember, in transfer learning we reuse parts of an already trained model and change the final layer, or several layers, of the model, and then retrain those layers on our own dataset.\n", "\n", "### TODO: Create a Feature Extractor\n", "In the cell below create a `feature_extractor` using MobileNet v2. Remember that the partial model from TensorFlow Hub (without the final classification layer) is called a feature vector. Go to the [TensorFlow Hub documentation](https://tfhub.dev/s?module-type=image-feature-vector&q=tf2) to see a list of available feature vectors. Click on the `tf2-preview/mobilenet_v2/feature_vector`. Read the documentation and get the corresponding `URL` to get the MobileNet v2 feature vector. Finally, create a `feature_extractor` by using `hub.KerasLayer` with the correct `input_shape` parameter." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5wB030nezBwI" }, "outputs": [], "source": [ "URL = \n", "feature_extractor = " ] }, { "cell_type": "markdown", "metadata": { "id": "CtFmF7A5E4tk" }, "source": [ "### TODO: Freeze the Pre-Trained Model\n", "\n", "In the cell below freeze the variables in the feature extractor layer, so that the training only modifies the final classifier layer." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Jg5ar6rcE4H-" }, "outputs": [], "source": [ "feature_extractor" ] }, { "cell_type": "markdown", "metadata": { "id": "RPVeouTksO9q" }, "source": [ "### TODO: Attach a classification head\n", "\n", "In the cell below create a `tf.keras.Sequential` model, and add the pre-trained model and the new classification layer. Remember that the classification layer must have the same number of classes as our Flowers dataset. Finally print a summary of the Sequential model." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "mGcY27fY1q3Q" }, "outputs": [], "source": [ "model = \n" ] }, { "cell_type": "markdown", "metadata": { "id": "OHbXQqIquFxQ" }, "source": [ "### TODO: Train the model\n", "\n", "In the cell bellow train this model like any other, by first calling `compile` and then followed by `fit`. Make sure you use the proper parameters when applying both methods. Train the model for only 6 epochs." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3n0Wb9ylKd8R" }, "outputs": [], "source": [ "EPOCHS = \n", "\n", "history = " ] }, { "cell_type": "markdown", "metadata": { "id": "76as-K8-vFQJ" }, "source": [ "You can see we get ~88% validation accuracy with only 6 epochs of training, which is absolutely awesome. This is a huge improvement over the model we created in the previous lesson, where we were able to get ~76% accuracy with 80 epochs of training. The reason for this difference is that MobileNet v2 was carefully designed over a long time by experts, then trained on a massive dataset (ImageNet)." ] }, { "cell_type": "markdown", "metadata": { "id": "SLxTcprUqJaq" }, "source": [ "# TODO: Plot Training and Validation Graphs\n", "\n", "In the cell below, plot the training and validation accuracy/loss graphs." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "d28dhbFpr98b" }, "outputs": [], "source": [ "acc = \n", "val_acc = \n", "\n", "loss = \n", "val_loss = \n", "\n", "epochs_range = \n" ] }, { "cell_type": "markdown", "metadata": { "id": "5zmoDisGvNye" }, "source": [ "What is a bit curious here is that validation performance is better than training performance, right from the start to the end of execution.\n", "\n", "One reason for this is that validation performance is measured at the end of the epoch, but training performance is the average values across the epoch.\n", "\n", "The bigger reason though is that we're reusing a large part of MobileNet which is already trained on Flower images. " ] }, { "cell_type": "markdown", "metadata": { "id": "kb__ZN8uFn-D" }, "source": [ "# TODO: Check Predictions\n", "\n", "In the cell below get the label names from the dataset info and convert them into a NumPy array. Print the array to make sure you have the correct label names." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W_Zvg2i0fzJu" }, "outputs": [], "source": [ "class_names = \n" ] }, { "cell_type": "markdown", "metadata": { "id": "4Olg6MsNGJTL" }, "source": [ "### TODO: Create an Image Batch and Make Predictions\n", "\n", "In the cell below, use the `next()` function to create an `image_batch` and its corresponding `label_batch`. Convert both the `image_batch` and `label_batch` to numpy arrays using the `.numpy()` method. Then use the `.predict()` method to run the image batch through your model and make predictions. Then use the `np.argmax()` function to get the indices of the best prediction for each image. Finally convert the indices of the best predictions to class names." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fCLVCpEjJ_VP" }, "outputs": [], "source": [ "image_batch, label_batch = \n", "\n", "\n", "\n", "predicted_batch = \n", "predicted_batch = tf.squeeze(predicted_batch).numpy()\n", "\n", "predicted_ids = \n", "predicted_class_names = \n" ] }, { "cell_type": "markdown", "metadata": { "id": "CkGbZxl9GZs-" }, "source": [ "### TODO: Print True Labels and Predicted Indices\n", "\n", "In the cell below, print the true labels and the indices of predicted labels." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nL9IhOmGI5dJ" }, "outputs": [], "source": [ "print()" ] }, { "cell_type": "markdown", "metadata": { "id": "gJDyzEfYuFcW" }, "source": [ "# Plot Model Predictions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wC_AYRJU9NQe" }, "outputs": [], "source": [ "plt.figure(figsize=(10,9))\n", "for n in range(30):\n", " plt.subplot(6,5,n+1)\n", " plt.subplots_adjust(hspace = 0.3)\n", " plt.imshow(image_batch[n])\n", " color = \"blue\" if predicted_ids[n] == label_batch[n] else \"red\"\n", " plt.title(predicted_class_names[n].title(), color=color)\n", " plt.axis('off')\n", "_ = plt.suptitle(\"Model predictions (blue: correct, red: incorrect)\")" ] }, { "cell_type": "markdown", "metadata": { "id": "7QBKxS5CuKhc" }, "source": [ "# TODO: Perform Transfer Learning with the Inception Model\n", "\n", "Go to the [TensorFlow Hub documentation](https://tfhub.dev/s?module-type=image-feature-vector&q=tf2) and click on `tf2-preview/inception_v3/feature_vector`. This feature vector corresponds to the Inception v3 model. In the cells below, use transfer learning to create a CNN that uses Inception v3 as the pretrained model to classify the images from the Flowers dataset. Note that Inception, takes as input, images that are 299 x 299 pixels. Compare the accuracy you get with Inception v3 to the accuracy you got with MobileNet v2." ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l06c02_exercise_flowers_with_transfer_learning.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l06c03_exercise_flowers_with_transfer_learning_solution.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "W_tvPdyfA-BL" }, "source": [ "##### Copyright 2019 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "0O_LFhwSBCjm" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "9-3Pry4jh1-E" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "NxjpzKTvg_dd" }, "source": [ "# TensorFlow Hub" ] }, { "cell_type": "markdown", "metadata": { "id": "crU-iluJIEzw" }, "source": [ "[TensorFlow Hub](http://tensorflow.org/hub) is an online repository of already trained TensorFlow models that you can use.\n", "These models can either be used as is, or they can be used for Transfer Learning.\n", "\n", "Transfer learning is a process where you take an existing trained model, and extend it to do additional work. This involves leaving the bulk of the model unchanged, while adding and retraining the final layers, in order to get a different set of possible outputs.\n", "\n", "Here, you can see all the models available in [TensorFlow Module Hub](https://tfhub.dev/).\n", "\n", "Before starting this Colab, you should reset the Colab environment by selecting `Runtime -> Reset all runtimes...` from menu above." ] }, { "cell_type": "markdown", "metadata": { "id": "7RVsYZLEpEWs" }, "source": [ "# Imports\n" ] }, { "cell_type": "markdown", "metadata": { "id": "ZUCEcRdhnyWn" }, "source": [ "Some normal imports we've seen before. The new one is importing tensorflow_hub which this Colab will make heavy use of." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "RxwCQNZWIL8y" }, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ivDUkUNdINH2" }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "import tensorflow_hub as hub\n", "import tensorflow_datasets as tfds\n", "\n", "from tensorflow.keras import layers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "NU3IAZ02G6VA" }, "outputs": [], "source": [ "import logging\n", "logger = tf.get_logger()\n", "logger.setLevel(logging.ERROR)" ] }, { "cell_type": "markdown", "metadata": { "id": "amfzqn1Oo7Om" }, "source": [ "# TODO: Download the Flowers Dataset using TensorFlow Datasets" ] }, { "cell_type": "markdown", "metadata": { "id": "Z93vvAdGxDMD" }, "source": [ "In the cell below you will download the Flowers dataset using TensorFlow Datasets. If you look at the [TensorFlow Datasets documentation](https://www.tensorflow.org/datasets/datasets#tf_flowers) you will see that the name of the Flowers dataset is `tf_flowers`. You can also see that this dataset is only split into a TRAINING set. You will therefore have to use `tfds.splits` to split this training set into to a `training_set` and a `validation_set`. Do a `[70, 30]` split such that 70 corresponds to the `training_set` and 30 to the `validation_set`. Then load the `tf_flowers` dataset using `tfds.load`. Make sure the `tfds.load` function uses the all the parameters you need, and also make sure it returns the dataset info, so we can retrieve information about the datasets.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oXiJjX0jfx1o" }, "outputs": [], "source": [ "(training_set, validation_set), dataset_info = tfds.load(\n", " 'tf_flowers',\n", " split=['train[:70%]', 'train[70%:]'],\n", " with_info=True,\n", " as_supervised=True,\n", ")" ] }, { "cell_type": "markdown", "metadata": { "id": "X0p1sOEHf0JF" }, "source": [ "# TODO: Print Information about the Flowers Dataset\n", "\n", "Now that you have downloaded the dataset, use the dataset info to print the number of classes in the dataset, and also write some code that counts how many images we have in the training and validation sets. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DrIUV3V0xDL_" }, "outputs": [], "source": [ "num_classes = dataset_info.features['label'].num_classes\n", "\n", "num_training_examples = 0\n", "num_validation_examples = 0\n", "\n", "for example in training_set:\n", " num_training_examples += 1\n", "\n", "for example in validation_set:\n", " num_validation_examples += 1\n", "\n", "print('Total Number of Classes: {}'.format(num_classes))\n", "print('Total Number of Training Images: {}'.format(num_training_examples))\n", "print('Total Number of Validation Images: {} \\n'.format(num_validation_examples))" ] }, { "cell_type": "markdown", "metadata": { "id": "UlFZ_hwjCLgS" }, "source": [ "The images in the Flowers dataset are not all the same size." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W4lDPkn2cpWZ" }, "outputs": [], "source": [ "for i, example in enumerate(training_set.take(5)):\n", " print('Image {} shape: {} label: {}'.format(i+1, example[0].shape, example[1]))" ] }, { "cell_type": "markdown", "metadata": { "id": "mbgpD3E6gM2P" }, "source": [ "# TODO: Reformat Images and Create Batches\n", "\n", "In the cell below create a function that reformats all images to the resolution expected by MobileNet v2 (224, 224) and normalizes them. The function should take in an `image` and a `label` as arguments and should return the new `image` and corresponding `label`. Then create training and validation batches of size `32`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "we_ftzQxNf7e" }, "outputs": [], "source": [ "IMAGE_RES = 224\n", "\n", "def format_image(image, label):\n", " image = tf.image.resize(image, (IMAGE_RES, IMAGE_RES))/255.0\n", " return image, label\n", "\n", "BATCH_SIZE = 32\n", "\n", "train_batches = training_set.shuffle(num_training_examples//4).map(format_image).batch(BATCH_SIZE).prefetch(1)\n", "\n", "validation_batches = validation_set.map(format_image).batch(BATCH_SIZE).prefetch(1)" ] }, { "cell_type": "markdown", "metadata": { "id": "JzV457OXreQP" }, "source": [ "# Do Simple Transfer Learning with TensorFlow Hub\n", "\n", "Let's now use TensorFlow Hub to do Transfer Learning. Remember, in transfer learning we reuse parts of an already trained model and change the final layer, or several layers, of the model, and then retrain those layers on our own dataset.\n", "\n", "### TODO: Create a Feature Extractor\n", "In the cell below create a `feature_extractor` using MobileNet v2. Remember that the partial model from TensorFlow Hub (without the final classification layer) is called a feature vector. Go to the [TensorFlow Hub documentation](https://tfhub.dev/s?module-type=image-feature-vector&q=tf2) to see a list of available feature vectors. Click on the `tf2-preview/mobilenet_v2/feature_vector`. Read the documentation and get the corresponding `URL` to get the MobileNet v2 feature vector. Finally, create a `feature_extractor` by using `hub.KerasLayer` with the correct `input_shape` parameter." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5wB030nezBwI" }, "outputs": [], "source": [ "URL = \"https://tfhub.dev/google/tf2-preview/mobilenet_v2/feature_vector/4\"\n", "feature_extractor = hub.KerasLayer(URL,\n", " input_shape=(IMAGE_RES, IMAGE_RES, 3))" ] }, { "cell_type": "markdown", "metadata": { "id": "CtFmF7A5E4tk" }, "source": [ "### TODO: Freeze the Pre-Trained Model\n", "\n", "In the cell below freeze the variables in the feature extractor layer, so that the training only modifies the final classifier layer." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Jg5ar6rcE4H-" }, "outputs": [], "source": [ "feature_extractor.trainable = False" ] }, { "cell_type": "markdown", "metadata": { "id": "RPVeouTksO9q" }, "source": [ "### TODO: Attach a classification head\n", "\n", "In the cell below create a `tf.keras.Sequential` model, and add the pre-trained model and the new classification layer. Remember that the classification layer must have the same number of classes as our Flowers dataset. Finally print a summary of the Sequential model." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "mGcY27fY1q3Q" }, "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " feature_extractor,\n", " layers.Dense(num_classes)\n", "])\n", "\n", "model.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "OHbXQqIquFxQ" }, "source": [ "### TODO: Train the model\n", "\n", "In the cell bellow train this model like any other, by first calling `compile` and then followed by `fit`. Make sure you use the proper parameters when applying both methods. Train the model for only 6 epochs." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3n0Wb9ylKd8R" }, "outputs": [], "source": [ "model.compile(\n", " optimizer='adam',\n", " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])\n", "\n", "EPOCHS = 6\n", "\n", "history = model.fit(train_batches,\n", " epochs=EPOCHS,\n", " validation_data=validation_batches)" ] }, { "cell_type": "markdown", "metadata": { "id": "76as-K8-vFQJ" }, "source": [ "You can see we get ~88% validation accuracy with only 6 epochs of training, which is absolutely awesome. This is a huge improvement over the model we created in the previous lesson, where we were able to get ~76% accuracy with 80 epochs of training. The reason for this difference is that MobileNet v2 was carefully designed over a long time by experts, then trained on a massive dataset (ImageNet)." ] }, { "cell_type": "markdown", "metadata": { "id": "SLxTcprUqJaq" }, "source": [ "# TODO: Plot Training and Validation Graphs\n", "\n", "In the cell below, plot the training and validation accuracy/loss graphs." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "d28dhbFpr98b" }, "outputs": [], "source": [ "acc = history.history['accuracy']\n", "val_acc = history.history['val_accuracy']\n", "\n", "loss = history.history['loss']\n", "val_loss = history.history['val_loss']\n", "\n", "epochs_range = range(EPOCHS)\n", "\n", "plt.figure(figsize=(8, 8))\n", "plt.subplot(1, 2, 1)\n", "plt.plot(epochs_range, acc, label='Training Accuracy')\n", "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", "plt.legend(loc='lower right')\n", "plt.title('Training and Validation Accuracy')\n", "\n", "plt.subplot(1, 2, 2)\n", "plt.plot(epochs_range, loss, label='Training Loss')\n", "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", "plt.legend(loc='upper right')\n", "plt.title('Training and Validation Loss')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "5zmoDisGvNye" }, "source": [ "What is a bit curious here is that validation performance is better than training performance, right from the start to the end of execution.\n", "\n", "One reason for this is that validation performance is measured at the end of the epoch, but training performance is the average values across the epoch.\n", "\n", "The bigger reason though is that we're reusing a large part of MobileNet which is already trained on Flower images. " ] }, { "cell_type": "markdown", "metadata": { "id": "kb__ZN8uFn-D" }, "source": [ "# TODO: Check Predictions\n", "\n", "In the cell below get the label names from the dataset info and convert them into a NumPy array. Print the array to make sure you have the correct label names." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W_Zvg2i0fzJu" }, "outputs": [], "source": [ "class_names = np.array(dataset_info.features['label'].names)\n", "\n", "print(class_names)" ] }, { "cell_type": "markdown", "metadata": { "id": "4Olg6MsNGJTL" }, "source": [ "### TODO: Create an Image Batch and Make Predictions\n", "\n", "In the cell below, use the `next()` function to create an `image_batch` and its corresponding `label_batch`. Convert both the `image_batch` and `label_batch` to numpy arrays using the `.numpy()` method. Then use the `.predict()` method to run the image batch through your model and make predictions. Then use the `np.argmax()` function to get the indices of the best prediction for each image. Finally convert the indices of the best predictions to class names." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fCLVCpEjJ_VP" }, "outputs": [], "source": [ "image_batch, label_batch = next(iter(train_batches))\n", "\n", "\n", "image_batch = image_batch.numpy()\n", "label_batch = label_batch.numpy()\n", "\n", "predicted_batch = model.predict(image_batch)\n", "predicted_batch = tf.squeeze(predicted_batch).numpy()\n", "\n", "predicted_ids = np.argmax(predicted_batch, axis=-1)\n", "predicted_class_names = class_names[predicted_ids]\n", "\n", "print(predicted_class_names)" ] }, { "cell_type": "markdown", "metadata": { "id": "CkGbZxl9GZs-" }, "source": [ "### TODO: Print True Labels and Predicted Indices\n", "\n", "In the cell below, print the true labels and the indices of predicted labels." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nL9IhOmGI5dJ" }, "outputs": [], "source": [ "print(\"Labels: \", label_batch)\n", "print(\"Predicted labels: \", predicted_ids)" ] }, { "cell_type": "markdown", "metadata": { "id": "gJDyzEfYuFcW" }, "source": [ "# Plot Model Predictions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wC_AYRJU9NQe" }, "outputs": [], "source": [ "plt.figure(figsize=(10,9))\n", "for n in range(30):\n", " plt.subplot(6,5,n+1)\n", " plt.subplots_adjust(hspace = 0.3)\n", " plt.imshow(image_batch[n])\n", " color = \"blue\" if predicted_ids[n] == label_batch[n] else \"red\"\n", " plt.title(predicted_class_names[n].title(), color=color)\n", " plt.axis('off')\n", "_ = plt.suptitle(\"Model predictions (blue: correct, red: incorrect)\")" ] }, { "cell_type": "markdown", "metadata": { "id": "7QBKxS5CuKhc" }, "source": [ "# TODO: Perform Transfer Learning with the Inception Model\n", "\n", "Go to the [TensorFlow Hub documentation](https://tfhub.dev/s?module-type=image-feature-vector&q=tf2) and click on `tf2-preview/inception_v3/feature_vector`. This feature vector corresponds to the Inception v3 model. In the cells below, use transfer learning to create a CNN that uses Inception v3 as the pretrained model to classify the images from the Flowers dataset. Note that Inception, takes as input, images that are 299 x 299 pixels. Compare the accuracy you get with Inception v3 to the accuracy you got with MobileNet v2." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wVII2H9ZNNQf" }, "outputs": [], "source": [ "IMAGE_RES = 299\n", "\n", "(training_set, validation_set), dataset_info = tfds.load(\n", " 'tf_flowers', \n", " with_info=True, \n", " as_supervised=True, \n", " split=['train[:70%]', 'train[70%:]'],\n", ")\n", "train_batches = training_set.shuffle(num_training_examples//4).map(format_image).batch(BATCH_SIZE).prefetch(1)\n", "validation_batches = validation_set.map(format_image).batch(BATCH_SIZE).prefetch(1)\n", "\n", "URL = \"https://tfhub.dev/google/tf2-preview/inception_v3/feature_vector/4\"\n", "feature_extractor = hub.KerasLayer(URL,\n", " input_shape=(IMAGE_RES, IMAGE_RES, 3),\n", " trainable=False)\n", "\n", "model_inception = tf.keras.Sequential([\n", " feature_extractor,\n", " tf.keras.layers.Dense(num_classes)\n", "])\n", "\n", "model_inception.summary()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "idcaQKWAPgL0" }, "outputs": [], "source": [ "model_inception.compile(\n", " optimizer='adam', \n", " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])\n", "\n", "EPOCHS = 6\n", "\n", "history = model_inception.fit(train_batches,\n", " epochs=EPOCHS,\n", " validation_data=validation_batches)" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l06c03_exercise_flowers_with_transfer_learning_solution.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l07c01_saving_and_loading_models.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "W_tvPdyfA-BL" }, "source": [ "##### Copyright 2019 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "0O_LFhwSBCjm" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "9-3Pry4jh1-E" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "NxjpzKTvg_dd" }, "source": [ "# Saving and Loading Models\n", "\n", "In this tutorial we will learn how we can take a trained model, save it, and then load it back to keep training it or use it to perform inference. In particular, we will use transfer learning to train a classifier to classify images of cats and dogs, just like we did in the previous lesson. We will then take our trained model and save it as an HDF5 file, which is the format used by Keras. We will then load this model, use it to perform predictions, and then continue to train the model. Finally, we will save our trained model as a TensorFlow SavedModel and then we will download it to a local disk, so that it can later be used for deployment in different platforms." ] }, { "cell_type": "markdown", "metadata": { "id": "crU-iluJIEzw" }, "source": [ "## Concepts that will be covered in this Colab\n", "\n", "1. Saving models in HDF5 format for Keras\n", "2. Saving models in the TensorFlow SavedModel format\n", "3. Loading models\n", "4. Download models to Local Disk\n", "\n", "Before starting this Colab, you should reset the Colab environment by selecting `Runtime -> Reset all runtimes...` from menu above." ] }, { "cell_type": "markdown", "metadata": { "id": "7RVsYZLEpEWs" }, "source": [ "# Imports\n", "\n", "In this Colab we will use the TensorFlow 2.0 Beta version. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "e3BXzUGabcI9" }, "outputs": [], "source": [ "!pip install -U tensorflow_hub\n", "!pip install -U tensorflow_datasets" ] }, { "cell_type": "markdown", "metadata": { "id": "a28UtPNlizGI" }, "source": [ "Some normal imports we've seen before. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "OGNpmn43C0O6" }, "outputs": [], "source": [ "import time\n", "import numpy as np\n", "import matplotlib.pylab as plt\n", "\n", "import tensorflow as tf\n", "import tensorflow_hub as hub\n", "import tensorflow_datasets as tfds\n", "tfds.disable_progress_bar()\n", "\n", "from tensorflow.keras import layers" ] }, { "cell_type": "markdown", "metadata": { "id": "amfzqn1Oo7Om" }, "source": [ "# Part 1: Load the Cats vs. Dogs Dataset" ] }, { "cell_type": "markdown", "metadata": { "id": "Z93vvAdGxDMD" }, "source": [ "We will use TensorFlow Datasets to load the Dogs vs Cats dataset. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DrIUV3V0xDL_" }, "outputs": [], "source": [ "(train_examples, validation_examples), info = tfds.load(\n", " 'cats_vs_dogs',\n", " split=['train[:80%]', 'train[80%:]'],\n", " with_info=True,\n", " as_supervised=True,\n", ")" ] }, { "cell_type": "markdown", "metadata": { "id": "mbgpD3E6gM2P" }, "source": [ "The images in the Dogs vs. Cats dataset are not all the same size. So, we need to reformat all images to the resolution expected by MobileNet (224, 224)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "we_ftzQxNf7e" }, "outputs": [], "source": [ "def format_image(image, label):\n", " # `hub` image modules exepct their data normalized to the [0,1] range.\n", " image = tf.image.resize(image, (IMAGE_RES, IMAGE_RES))/255.0\n", " return image, label\n", "\n", "num_examples = info.splits['train'].num_examples\n", "\n", "BATCH_SIZE = 32\n", "IMAGE_RES = 224\n", "\n", "train_batches = train_examples.cache().shuffle(num_examples//4).map(format_image).batch(BATCH_SIZE).prefetch(1)\n", "validation_batches = validation_examples.cache().map(format_image).batch(BATCH_SIZE).prefetch(1)" ] }, { "cell_type": "markdown", "metadata": { "id": "JzV457OXreQP" }, "source": [ "# Part 2: Transfer Learning with TensorFlow Hub\n", "\n", "We will now use TensorFlow Hub to do Transfer Learning." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5wB030nezBwI" }, "outputs": [], "source": [ "URL = \"https://tfhub.dev/google/tf2-preview/mobilenet_v2/feature_vector/4\"\n", "feature_extractor = hub.KerasLayer(URL,\n", " input_shape=(IMAGE_RES, IMAGE_RES,3))" ] }, { "cell_type": "markdown", "metadata": { "id": "CtFmF7A5E4tk" }, "source": [ "Freeze the variables in the feature extractor layer, so that the training only modifies the final classifier layer." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Jg5ar6rcE4H-" }, "outputs": [], "source": [ "feature_extractor.trainable = False" ] }, { "cell_type": "markdown", "metadata": { "id": "RPVeouTksO9q" }, "source": [ "## Attach a classification head\n", "\n", "Now wrap the hub layer in a `tf.keras.Sequential` model, and add a new classification layer." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "mGcY27fY1q3Q" }, "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " feature_extractor,\n", " layers.Dense(2)\n", "])\n", "\n", "model.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "OHbXQqIquFxQ" }, "source": [ "## Train the model\n", "\n", "We now train this model like any other, by first calling `compile` followed by `fit`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3n0Wb9ylKd8R" }, "outputs": [], "source": [ "model.compile(\n", " optimizer='adam', \n", " loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])\n", "\n", "EPOCHS = 3\n", "history = model.fit(train_batches,\n", " epochs=EPOCHS,\n", " validation_data=validation_batches)" ] }, { "cell_type": "markdown", "metadata": { "id": "kb__ZN8uFn-D" }, "source": [ "## Check the predictions\n", "\n", "Get the ordered list of class names." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W_Zvg2i0fzJu" }, "outputs": [], "source": [ "class_names = np.array(info.features['label'].names)\n", "class_names" ] }, { "cell_type": "markdown", "metadata": { "id": "4Olg6MsNGJTL" }, "source": [ "Run an image batch through the model and convert the indices to class names." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fCLVCpEjJ_VP" }, "outputs": [], "source": [ "image_batch, label_batch = next(iter(train_batches.take(1)))\n", "image_batch = image_batch.numpy()\n", "label_batch = label_batch.numpy()\n", "\n", "predicted_batch = model.predict(image_batch)\n", "predicted_batch = tf.squeeze(predicted_batch).numpy()\n", "predicted_ids = np.argmax(predicted_batch, axis=-1)\n", "predicted_class_names = class_names[predicted_ids]\n", "predicted_class_names" ] }, { "cell_type": "markdown", "metadata": { "id": "CkGbZxl9GZs-" }, "source": [ "Let's look at the true labels and predicted ones." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nL9IhOmGI5dJ" }, "outputs": [], "source": [ "print(\"Labels: \", label_batch)\n", "print(\"Predicted labels: \", predicted_ids)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wC_AYRJU9NQe" }, "outputs": [], "source": [ "plt.figure(figsize=(10,9))\n", "for n in range(30):\n", " plt.subplot(6,5,n+1)\n", " plt.imshow(image_batch[n])\n", " color = \"blue\" if predicted_ids[n] == label_batch[n] else \"red\"\n", " plt.title(predicted_class_names[n].title(), color=color)\n", " plt.axis('off')\n", "_ = plt.suptitle(\"Model predictions (blue: correct, red: incorrect)\")" ] }, { "cell_type": "markdown", "metadata": { "id": "mmPQYQLx3cYq" }, "source": [ "# Part 3: Save as Keras `.h5` model\n", "\n", "Now that we've trained the model, we can save it as an HDF5 file, which is the format used by Keras. Our HDF5 file will have the extension '.h5', and it's name will correpond to the current time stamp." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tCnNWTkZ3Ckz" }, "outputs": [], "source": [ "t = time.time()\n", "\n", "export_path_keras = \"./{}.h5\".format(int(t))\n", "print(export_path_keras)\n", "\n", "model.save(export_path_keras)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9tdJWVHmnKxJ" }, "outputs": [], "source": [ "!ls" ] }, { "cell_type": "markdown", "metadata": { "id": "JgqmH1WVUKli" }, "source": [ " You can later recreate the same model from this file, even if you no longer have access to the code that created the model.\n", "\n", "This file includes:\n", "\n", "- The model's architecture\n", "- The model's weight values (which were learned during training)\n", "- The model's training config (what you passed to `compile`), if any\n", "- The optimizer and its state, if any (this enables you to restart training where you left off)" ] }, { "cell_type": "markdown", "metadata": { "id": "yXole25Z799G" }, "source": [ "# Part 4: Load the Keras `.h5` Model\n", "\n", "We will now load the model we just saved into a new model called `reloaded`. We will need to provide the file path and the `custom_objects` parameter. This parameter tells keras how to load the `hub.KerasLayer` from the `feature_extractor` we used for transfer learning." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Rx-z3Qwx5RnB" }, "outputs": [], "source": [ "reloaded = tf.keras.models.load_model(\n", " export_path_keras, \n", " # `custom_objects` tells keras how to load a `hub.KerasLayer`\n", " custom_objects={'KerasLayer': hub.KerasLayer})\n", "\n", "reloaded.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "XxDl2vPkf2ST" }, "source": [ "We can check that the reloaded model and the previous model give the same result" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "MFljA-Hd85Tu" }, "outputs": [], "source": [ "result_batch = model.predict(image_batch)\n", "reloaded_result_batch = reloaded.predict(image_batch)" ] }, { "cell_type": "markdown", "metadata": { "id": "YeCZUUJK9Svv" }, "source": [ "The difference in output should be zero:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "S3p5-uD39PC1" }, "outputs": [], "source": [ "(abs(result_batch - reloaded_result_batch)).max()" ] }, { "cell_type": "markdown", "metadata": { "id": "KqU79kKFo7S2" }, "source": [ "As we can see, the reult is 0.0, which indicates that both models made the same predictions on the same batch of images." ] }, { "cell_type": "markdown", "metadata": { "id": "nKunO4soA4Dm" }, "source": [ "# Keep Training\n", "\n", "Besides making predictions, we can also take our `reloaded` model and keep training it. To do this, you can just train the `reloaded` as usual, using the `.fit` method." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "NEv-fnHEAplx" }, "outputs": [], "source": [ "EPOCHS = 3\n", "history = reloaded.fit(train_batches,\n", " epochs=EPOCHS,\n", " validation_data=validation_batches)" ] }, { "cell_type": "markdown", "metadata": { "id": "5OKoZaAHFH_s" }, "source": [ "# Part 5: Export as SavedModel\n" ] }, { "cell_type": "markdown", "metadata": { "id": "3V47IwQbFTYT" }, "source": [ "You can also export a whole model to the TensorFlow SavedModel format. SavedModel is a standalone serialization format for Tensorflow objects, supported by TensorFlow serving as well as TensorFlow implementations other than Python. A SavedModel contains a complete TensorFlow program, including weights and computation. It does not require the original model building code to run, which makes it useful for sharing or deploying (with TFLite, TensorFlow.js, TensorFlow Serving, or TFHub).\n", "\n", "The SavedModel files that were created contain:\n", "\n", "* A TensorFlow checkpoint containing the model weights.\n", "* A SavedModel proto containing the underlying Tensorflow graph. Separate graphs are saved for prediction (serving), train, and evaluation. If the model wasn't compiled before, then only the inference graph gets exported.\n", "* The model's architecture config, if available.\n", "\n", "\n", "Let's save our original `model` as a TensorFlow SavedModel. To do this we will use the `tf.saved_model.save()` function. This functions takes in the model we want to save and the path to the folder where we want to save our model. \n", "\n", "This function will create a folder where you will find an `assets` folder, a `variables` folder, and the `saved_model.pb` file. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "LtpeKMfoGXrj" }, "outputs": [], "source": [ "t = time.time()\n", "\n", "export_path_sm = \"./{}\".format(int(t))\n", "print(export_path_sm)\n", "\n", "tf.saved_model.save(model, export_path_sm)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5h6B1wITlu-9" }, "outputs": [], "source": [ "!ls {export_path_sm}" ] }, { "cell_type": "markdown", "metadata": { "id": "ktpsqHxJPIQW" }, "source": [ "# Part 6: Load SavedModel" ] }, { "cell_type": "markdown", "metadata": { "id": "v0FDcCn9ncDb" }, "source": [ "Now, let's load our SavedModel and use it to make predictions. We use the `tf.saved_model.load()` function to load our SavedModels. The object returned by `tf.saved_model.load` is 100% independent of the code that created it." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "c7_PM7lofG2V" }, "outputs": [], "source": [ "reloaded_sm = tf.saved_model.load(export_path_sm)" ] }, { "cell_type": "markdown", "metadata": { "id": "esEODdtM0kHB" }, "source": [ "Now, let's use the `reloaded_sm` (reloaded SavedModel) to make predictions on a batch of images." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lpQldy_bm3Ty" }, "outputs": [], "source": [ "reload_sm_result_batch = reloaded_sm(image_batch, training=False).numpy()" ] }, { "cell_type": "markdown", "metadata": { "id": "65upCyVN0u3Q" }, "source": [ "We can check that the reloaded SavedModel and the previous model give the same result." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "hoCwmkGznR_0" }, "outputs": [], "source": [ "(abs(result_batch - reload_sm_result_batch)).max()" ] }, { "cell_type": "markdown", "metadata": { "id": "wxuETjjs01pL" }, "source": [ "As we can see, the result is 0.0, which indicates that both models made the same predictions on the same batch of images." ] }, { "cell_type": "markdown", "metadata": { "id": "7Nom-ka0yuqB" }, "source": [ "# Part 7: Loading the SavedModel as a Keras Model\n", "\n", "The object returned by `tf.saved_model.load` is not a Keras object (i.e. doesn't have `.fit`, `.predict`, `.summary`, etc. methods). Therefore, you can't simply take your `reloaded_sm` model and keep training it by running `.fit`. To be able to get back a full keras model from the Tensorflow SavedModel format we must use the `tf.keras.models.load_model` function. This function will work the same as before, except now we pass the path to the folder containing our SavedModel." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Vi1jaqh8yvt6" }, "outputs": [], "source": [ "t = time.time()\n", "\n", "export_path_sm = \"./{}\".format(int(t))\n", "print(export_path_sm)\n", "tf.saved_model.save(model, export_path_sm)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1VPE2_QQGmAP" }, "outputs": [], "source": [ "reload_sm_keras = tf.keras.models.load_model(\n", " export_path_sm,\n", " custom_objects={'KerasLayer': hub.KerasLayer})\n", "\n", "reload_sm_keras.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "thTbiHE72GL4" }, "source": [ "Now, let's use the `reloaded_sm)keras` (reloaded Keras model from our SavedModel) to make predictions on a batch of images." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-0oCJrNLKdKj" }, "outputs": [], "source": [ "result_batch = model.predict(image_batch)\n", "reload_sm_keras_result_batch = reload_sm_keras.predict(image_batch)" ] }, { "cell_type": "markdown", "metadata": { "id": "jUQaxzFj2Q8k" }, "source": [ "We can check that the reloaded Keras model and the previous model give the same result." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DJCD9JJxKg9F" }, "outputs": [], "source": [ "(abs(result_batch - reload_sm_keras_result_batch)).max()" ] }, { "cell_type": "markdown", "metadata": { "id": "NFa6jNc4PW9F" }, "source": [ "# Part 8: Download your model" ] }, { "cell_type": "markdown", "metadata": { "id": "2W_y1qMVQeCm" }, "source": [ "You can download the SavedModel to your local disk by creating a zip file. We wil use the `-r` (recursice) option to zip all subfolders. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "WPRFoU1xPCGF" }, "outputs": [], "source": [ "!zip -r model.zip {export_path_sm}" ] }, { "cell_type": "markdown", "metadata": { "id": "MBZL85RsQBTj" }, "source": [ "The zip file is saved in the current working directory. You can see what the current working directory is by running:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ALP-DfwSQRL8" }, "outputs": [], "source": [ "!ls" ] }, { "cell_type": "markdown", "metadata": { "id": "IR89aU-SRmsL" }, "source": [ "Once the file is zipped, you can download it to your local disk. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lOXYrlDkNjKQ" }, "outputs": [], "source": [ "try:\n", " from google.colab import files\n", " files.download('./model.zip')\n", "except ImportError:\n", " pass" ] }, { "cell_type": "markdown", "metadata": { "id": "RzLutxq_SeLQ" }, "source": [ "The `files.download` command will search for files in your current working directory. If the file you want to download is in a directory other than the current working directory, you have to include the path to the directory where the file is located." ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l07c01_saving_and_loading_models.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l08c01_common_patterns.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "Ivi7Bm7thgCT" }, "source": [ "# Common patterns" ] }, { "cell_type": "markdown", "metadata": { "id": "tFYaTP1Shi91" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "vidayERjaO5q" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gqWabzlJ63nL" }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "sJwA96JU00pW" }, "outputs": [], "source": [ "def plot_series(time, series, format=\"-\", start=0, end=None, label=None):\n", " plt.plot(time[start:end], series[start:end], format, label=label)\n", " plt.xlabel(\"Time\")\n", " plt.ylabel(\"Value\")\n", " if label:\n", " plt.legend(fontsize=14)\n", " plt.grid(True)" ] }, { "cell_type": "markdown", "metadata": { "id": "yVo6CcpRaW7u" }, "source": [ "## Trend and Seasonality" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "t30Ts2KjiOIY" }, "outputs": [], "source": [ "def trend(time, slope=0):\n", " return slope * time" ] }, { "cell_type": "markdown", "metadata": { "id": "iJjc3G1Maefn" }, "source": [ "Let's create a time series that just trends upward:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "BLt-pLiZ0nfB" }, "outputs": [], "source": [ "time = np.arange(4 * 365 + 1)\n", "baseline = 10\n", "series = baseline + trend(time, 0.1)\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3-4hV2WHTC_F" }, "outputs": [], "source": [ "time" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "eOK7NnaOTGa7" }, "outputs": [], "source": [ "series" ] }, { "cell_type": "markdown", "metadata": { "id": "WKD4nh9sauBf" }, "source": [ "Now let's generate a time series with a seasonal pattern:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "89gdEnPY1Niy" }, "outputs": [], "source": [ "def seasonal_pattern(season_time):\n", " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", " return np.where(season_time < 0.4,\n", " np.cos(season_time * 2 * np.pi),\n", " 1 / np.exp(3 * season_time))\n", "\n", "def seasonality(time, period, amplitude=1, phase=0):\n", " \"\"\"Repeats the same pattern at each period\"\"\"\n", " season_time = ((time + phase) % period) / period\n", " return amplitude * seasonal_pattern(season_time)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "7kaNezUk1S9l" }, "outputs": [], "source": [ "amplitude = 40\n", "series = seasonality(time, period=365, amplitude=amplitude)\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "-Vo433h0bDLD" }, "source": [ "Now let's create a time series with both trend and seasonality:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "AyqFdaIN1oy5" }, "outputs": [], "source": [ "slope = 0.05\n", "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "YVdJ2jNN8OHk" }, "source": [ "## Noise" ] }, { "cell_type": "markdown", "metadata": { "id": "V4taP424sces" }, "source": [ "In practice few real-life time series have such a smooth signal. They usually have some noise, and the signal-to-noise ratio can sometimes be very low. Let's generate some white noise:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3kD3_zjVscBH" }, "outputs": [], "source": [ "def white_noise(time, noise_level=1, seed=None):\n", " rnd = np.random.RandomState(seed)\n", " return rnd.randn(len(time)) * noise_level" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "aLvBwrKrtDzo" }, "outputs": [], "source": [ "noise_level = 5\n", "noise = white_noise(time, noise_level, seed=42)\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, noise)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "GHa6gicgbL74" }, "source": [ "Now let's add this white noise to the time series:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2bRDx8K816N9" }, "outputs": [], "source": [ "series += noise\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] } ], "metadata": { "colab": { "collapsed_sections": [], "name": "l08c01_common_patterns.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l08c02_naive_forecasting.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "_3bi1D2IiCyW" }, "source": [ "# Naive forecasting" ] }, { "cell_type": "markdown", "metadata": { "id": "m_6H00ELiA57" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "vidayERjaO5q" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gqWabzlJ63nL" }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "sJwA96JU00pW" }, "outputs": [], "source": [ "def plot_series(time, series, format=\"-\", start=0, end=None, label=None):\n", " plt.plot(time[start:end], series[start:end], format, label=label)\n", " plt.xlabel(\"Time\")\n", " plt.ylabel(\"Value\")\n", " if label:\n", " plt.legend(fontsize=14)\n", " plt.grid(True)\n", " \n", "def trend(time, slope=0):\n", " return slope * time\n", "\n", "def seasonal_pattern(season_time):\n", " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", " return np.where(season_time < 0.4,\n", " np.cos(season_time * 2 * np.pi),\n", " 1 / np.exp(3 * season_time))\n", " \n", "def seasonality(time, period, amplitude=1, phase=0):\n", " \"\"\"Repeats the same pattern at each period\"\"\"\n", " season_time = ((time + phase) % period) / period\n", " return amplitude * seasonal_pattern(season_time)\n", " \n", "def white_noise(time, noise_level=1, seed=None):\n", " rnd = np.random.RandomState(seed)\n", " return rnd.randn(len(time)) * noise_level" ] }, { "cell_type": "markdown", "metadata": { "id": "yVo6CcpRaW7u" }, "source": [ "## Trend and Seasonality" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "BLt-pLiZ0nfB" }, "outputs": [], "source": [ "time = np.arange(4 * 365 + 1)\n", "\n", "slope = 0.05\n", "baseline = 10\n", "amplitude = 40\n", "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", "\n", "noise_level = 5\n", "noise = white_noise(time, noise_level, seed=42)\n", "\n", "series += noise\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "a1sQpPjhtj0G" }, "source": [ "All right, this looks realistic enough for now. Let's try to forecast it. We will split it into two periods: the training period and the validation period (in many cases, you would also want to have a test period). The split will be at time step 1000." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "_w0eKap5uFNP" }, "outputs": [], "source": [ "split_time = 1000\n", "time_train = time[:split_time]\n", "x_train = series[:split_time]\n", "time_valid = time[split_time:]\n", "x_valid = series[split_time:]" ] }, { "cell_type": "markdown", "metadata": { "id": "bjD8ncEZbjEW" }, "source": [ "## Naive Forecast" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Pj_-uCeYxcAb" }, "outputs": [], "source": [ "naive_forecast = series[split_time - 1:-1]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JtxwHj9Ig0jT" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid, label=\"Series\")\n", "plot_series(time_valid, naive_forecast, label=\"Forecast\")" ] }, { "cell_type": "markdown", "metadata": { "id": "fw1SP5WeuixH" }, "source": [ "Let's zoom in on the start of the validation period:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "D0MKg7FNug9V" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid, start=0, end=150, label=\"Series\")\n", "plot_series(time_valid, naive_forecast, start=1, end=151, label=\"Forecast\")" ] }, { "cell_type": "markdown", "metadata": { "id": "35gIlQLfu0TT" }, "source": [ "You can see that the naive forecast lags 1 step behind the time series." ] }, { "cell_type": "markdown", "metadata": { "id": "Uh_7244Gsxfx" }, "source": [ "Now let's compute the mean absolute error between the forecasts and the predictions in the validation period:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "LjpLQeWY11H8" }, "outputs": [], "source": [ "errors = naive_forecast - x_valid\n", "abs_errors = np.abs(errors)\n", "mae = abs_errors.mean()\n", "mae" ] }, { "cell_type": "markdown", "metadata": { "id": "WGPBC9QttI1u" }, "source": [ "That's our baseline, now let's try a moving average." ] } ], "metadata": { "colab": { "collapsed_sections": [], "name": "l08c02_naive_forecasting.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l08c03_moving_average.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "Nm71sonIiJjH" }, "source": [ "# Moving average" ] }, { "cell_type": "markdown", "metadata": { "id": "C34f-r1Mhzkj" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "vidayERjaO5q" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gqWabzlJ63nL" }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import tensorflow as tf\n", "\n", "keras = tf.keras" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "sJwA96JU00pW" }, "outputs": [], "source": [ "def plot_series(time, series, format=\"-\", start=0, end=None, label=None):\n", " plt.plot(time[start:end], series[start:end], format, label=label)\n", " plt.xlabel(\"Time\")\n", " plt.ylabel(\"Value\")\n", " if label:\n", " plt.legend(fontsize=14)\n", " plt.grid(True)\n", " \n", "def trend(time, slope=0):\n", " return slope * time\n", "\n", "def seasonal_pattern(season_time):\n", " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", " return np.where(season_time < 0.4,\n", " np.cos(season_time * 2 * np.pi),\n", " 1 / np.exp(3 * season_time))\n", "\n", "def seasonality(time, period, amplitude=1, phase=0):\n", " \"\"\"Repeats the same pattern at each period\"\"\"\n", " season_time = ((time + phase) % period) / period\n", " return amplitude * seasonal_pattern(season_time)\n", "\n", "def white_noise(time, noise_level=1, seed=None):\n", " rnd = np.random.RandomState(seed)\n", " return rnd.randn(len(time)) * noise_level" ] }, { "cell_type": "markdown", "metadata": { "id": "yVo6CcpRaW7u" }, "source": [ "## Trend and Seasonality" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "BLt-pLiZ0nfB" }, "outputs": [], "source": [ "time = np.arange(4 * 365 + 1)\n", "\n", "slope = 0.05\n", "baseline = 10\n", "amplitude = 40\n", "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", "\n", "noise_level = 5\n", "noise = white_noise(time, noise_level, seed=42)\n", "\n", "series += noise\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "bjD8ncEZbjEW" }, "source": [ "## Naive Forecast" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Pj_-uCeYxcAb" }, "outputs": [], "source": [ "split_time = 1000\n", "time_train = time[:split_time]\n", "x_train = series[:split_time]\n", "time_valid = time[split_time:]\n", "x_valid = series[split_time:]\n", "\n", "naive_forecast = series[split_time - 1:-1]\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid, start=0, end=150, label=\"Series\")\n", "plot_series(time_valid, naive_forecast, start=1, end=151, label=\"Forecast\")" ] }, { "cell_type": "markdown", "metadata": { "id": "Uh_7244Gsxfx" }, "source": [ "Now let's compute the mean absolute error between the forecasts and the predictions in the validation period:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "byNnC7IbsnMZ" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, naive_forecast).numpy()" ] }, { "cell_type": "markdown", "metadata": { "id": "WGPBC9QttI1u" }, "source": [ "That's our baseline, now let's try a moving average." ] }, { "cell_type": "markdown", "metadata": { "id": "MLtZbFoU8OH-" }, "source": [ "## Moving Average" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "YGz5UsUdf2tV" }, "outputs": [], "source": [ "def moving_average_forecast(series, window_size):\n", " \"\"\"Forecasts the mean of the last few values.\n", " If window_size=1, then this is equivalent to naive forecast\"\"\"\n", " forecast = []\n", " for time in range(len(series) - window_size):\n", " forecast.append(series[time:time + window_size].mean())\n", " return np.array(forecast)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Le2gNBthBWPN" }, "outputs": [], "source": [ "def moving_average_forecast(series, window_size):\n", " \"\"\"Forecasts the mean of the last few values.\n", " If window_size=1, then this is equivalent to naive forecast\n", " This implementation is *much* faster than the previous one\"\"\"\n", " mov = np.cumsum(series)\n", " mov[window_size:] = mov[window_size:] - mov[:-window_size]\n", " return mov[window_size - 1:-1] / window_size" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "F50zyJGoDNJl" }, "outputs": [], "source": [ "moving_avg = moving_average_forecast(series, 30)[split_time - 30:]\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid, label=\"Series\")\n", "plot_series(time_valid, moving_avg, label=\"Moving average (30 days)\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wG7pTAd7z0e8" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, moving_avg).numpy()" ] }, { "cell_type": "markdown", "metadata": { "id": "JMYPnJqwz8nS" }, "source": [ "That's worse than naive forecast! The moving average does not anticipate trend or seasonality, so let's try to remove them by using differencing. Since the seasonality period is 365 days, we will subtract the value at time *t* – 365 from the value at time *t*." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5pqySF7-rJR4" }, "outputs": [], "source": [ "diff_series = (series[365:] - series[:-365])\n", "diff_time = time[365:]\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(diff_time, diff_series, label=\"Series(t) – Series(t–365)\")\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "WDNer84g8OIF" }, "source": [ "Focusing on the validation period:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-O21jlnA8OIG" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, diff_series[split_time - 365:], label=\"Series(t) – Series(t–365)\")\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "xPlPlS7DskWg" }, "source": [ "Great, the trend and seasonality seem to be gone, so now we can use the moving average:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "QmZpz7arsjbb" }, "outputs": [], "source": [ "diff_moving_avg = moving_average_forecast(diff_series, 50)[split_time - 365 - 50:]\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, diff_series[split_time - 365:], label=\"Series(t) – Series(t–365)\")\n", "plot_series(time_valid, diff_moving_avg, label=\"Moving Average of Diff\")\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "Gno9S2lyecnc" }, "source": [ "Now let's bring back the trend and seasonality by adding the past values from t – 365:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Dv6RWFq7TFGB" }, "outputs": [], "source": [ "diff_moving_avg_plus_past = series[split_time - 365:-365] + diff_moving_avg\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid, label=\"Series\")\n", "plot_series(time_valid, diff_moving_avg_plus_past, label=\"Forecasts\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "59jmBrwcTFCx" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, diff_moving_avg_plus_past).numpy()" ] }, { "cell_type": "markdown", "metadata": { "id": "vx9Et1Hkeusl" }, "source": [ "Better than naive forecast, good. However the forecasts look a bit too random, because we're just adding past values, which were noisy. Let's use a moving averaging on past values to remove some of the noise:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "K81dtROoTE_r" }, "outputs": [], "source": [ "diff_moving_avg_plus_smooth_past = moving_average_forecast(series[split_time - 370:-359], 11) + diff_moving_avg\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid, label=\"Series\")\n", "plot_series(time_valid, diff_moving_avg_plus_smooth_past, label=\"Forecasts\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "iN2MsBxWTE3m" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, diff_moving_avg_plus_smooth_past).numpy()" ] }, { "cell_type": "markdown", "metadata": { "id": "WKnmJisHcvTW" }, "source": [ "That's starting to look pretty good! Let's see if we can do better with a Machine Learning model." ] } ], "metadata": { "colab": { "collapsed_sections": [], "name": "l08c03_moving_average.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l08c04_time_windows.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "Ou0PGp_4icRo" }, "source": [ "# Time windows" ] }, { "cell_type": "markdown", "metadata": { "id": "93b0GzKph0jK" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "vidayERjaO5q" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gqWabzlJ63nL" }, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "markdown", "metadata": { "id": "ViWVB9qd8OIR" }, "source": [ "## Time Windows\n", "\n", "First, we will train a model to forecast the next step given the previous 20 steps, therefore, we need to create a dataset of 20-step windows for training." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "bgJkwtq88OIS" }, "outputs": [], "source": [ "dataset = tf.data.Dataset.range(10)\n", "for val in dataset:\n", " print(val.numpy())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ad8C65JV8OIT" }, "outputs": [], "source": [ "dataset = tf.data.Dataset.range(10)\n", "dataset = dataset.window(5, shift=1)\n", "for window_dataset in dataset:\n", " for val in window_dataset:\n", " print(val.numpy(), end=\" \")\n", " print()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "AQtmODsi8OIU" }, "outputs": [], "source": [ "dataset = tf.data.Dataset.range(10)\n", "dataset = dataset.window(5, shift=1, drop_remainder=True)\n", "for window_dataset in dataset:\n", " for val in window_dataset:\n", " print(val.numpy(), end=\" \")\n", " print()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kTRHiWxi8OIW" }, "outputs": [], "source": [ "dataset = tf.data.Dataset.range(10)\n", "dataset = dataset.window(5, shift=1, drop_remainder=True)\n", "dataset = dataset.flat_map(lambda window: window.batch(5))\n", "for window in dataset:\n", " print(window.numpy())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "iPsQbWHb8OIX" }, "outputs": [], "source": [ "dataset = tf.data.Dataset.range(10)\n", "dataset = dataset.window(5, shift=1, drop_remainder=True)\n", "dataset = dataset.flat_map(lambda window: window.batch(5))\n", "dataset = dataset.map(lambda window: (window[:-1], window[-1:]))\n", "for x, y in dataset:\n", " print(x.numpy(), y.numpy())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "hzp7RD6_8OIY" }, "outputs": [], "source": [ "dataset = tf.data.Dataset.range(10)\n", "dataset = dataset.window(5, shift=1, drop_remainder=True)\n", "dataset = dataset.flat_map(lambda window: window.batch(5))\n", "dataset = dataset.map(lambda window: (window[:-1], window[-1:]))\n", "dataset = dataset.shuffle(buffer_size=10)\n", "for x, y in dataset:\n", " print(x.numpy(), y.numpy())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "y70nV0EI8OIZ" }, "outputs": [], "source": [ "dataset = tf.data.Dataset.range(10)\n", "dataset = dataset.window(5, shift=1, drop_remainder=True)\n", "dataset = dataset.flat_map(lambda window: window.batch(5))\n", "dataset = dataset.map(lambda window: (window[:-1], window[-1:]))\n", "dataset = dataset.shuffle(buffer_size=10)\n", "dataset = dataset.batch(2).prefetch(1)\n", "for x, y in dataset:\n", " print(\"x =\", x.numpy())\n", " print(\"y =\", y.numpy())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1tl-0BOKkEtk" }, "outputs": [], "source": [ "def window_dataset(series, window_size, batch_size=32,\n", " shuffle_buffer=1000):\n", " dataset = tf.data.Dataset.from_tensor_slices(series)\n", " dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)\n", " dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))\n", " dataset = dataset.shuffle(shuffle_buffer)\n", " dataset = dataset.map(lambda window: (window[:-1], window[-1]))\n", " dataset = dataset.batch(batch_size).prefetch(1)\n", " return dataset" ] } ], "metadata": { "colab": { "collapsed_sections": [], "name": "l08c04_time_windows.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l08c05_forecasting_with_machine_learning.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "yHG6wUbvifuX" }, "source": [ "# Forecasting with machine learning" ] }, { "cell_type": "markdown", "metadata": { "id": "bQNmSYMPh1TB" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "vidayERjaO5q" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gqWabzlJ63nL" }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import tensorflow as tf\n", "\n", "keras = tf.keras" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cg1hfKCPldZG" }, "outputs": [], "source": [ "def plot_series(time, series, format=\"-\", start=0, end=None, label=None):\n", " plt.plot(time[start:end], series[start:end], format, label=label)\n", " plt.xlabel(\"Time\")\n", " plt.ylabel(\"Value\")\n", " if label:\n", " plt.legend(fontsize=14)\n", " plt.grid(True)\n", " \n", "def trend(time, slope=0):\n", " return slope * time\n", " \n", " \n", "def seasonal_pattern(season_time):\n", " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", " return np.where(season_time < 0.4,\n", " np.cos(season_time * 2 * np.pi),\n", " 1 / np.exp(3 * season_time))\n", "\n", " \n", "def seasonality(time, period, amplitude=1, phase=0):\n", " \"\"\"Repeats the same pattern at each period\"\"\"\n", " season_time = ((time + phase) % period) / period\n", " return amplitude * seasonal_pattern(season_time)\n", " \n", " \n", "def white_noise(time, noise_level=1, seed=None):\n", " rnd = np.random.RandomState(seed)\n", " return rnd.randn(len(time)) * noise_level" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "iL2DDjV3lel6" }, "outputs": [], "source": [ "time = np.arange(4 * 365 + 1)\n", "\n", "slope = 0.05\n", "baseline = 10\n", "amplitude = 40\n", "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", "\n", "noise_level = 5\n", "noise = white_noise(time, noise_level, seed=42)\n", "\n", "series += noise\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "ViWVB9qd8OIR" }, "source": [ "## Forecasting with Machine Learning\n", "\n", "First, we will train a model to forecast the next step given the previous 30 steps, therefore, we need to create a dataset of 30-step windows for training." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1tl-0BOKkEtk" }, "outputs": [], "source": [ "def window_dataset(series, window_size, batch_size=32,\n", " shuffle_buffer=1000):\n", " dataset = tf.data.Dataset.from_tensor_slices(series)\n", " dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)\n", " dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))\n", " dataset = dataset.shuffle(shuffle_buffer)\n", " dataset = dataset.map(lambda window: (window[:-1], window[-1]))\n", " dataset = dataset.batch(batch_size).prefetch(1)\n", " return dataset" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Zmp1JXKxk9Vb" }, "outputs": [], "source": [ "split_time = 1000\n", "time_train = time[:split_time]\n", "x_train = series[:split_time]\n", "time_valid = time[split_time:]\n", "x_valid = series[split_time:]" ] }, { "cell_type": "markdown", "metadata": { "id": "T1IvwAFn8OIc" }, "source": [ "### Linear Model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ieOKdcEQ0A6k" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = window_dataset(x_train, window_size)\n", "valid_set = window_dataset(x_valid, window_size)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.Dense(1, input_shape=[window_size])\n", "])\n", "optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "model.fit(train_set, epochs=100, validation_data=valid_set)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "N3N8AGRM8OIc" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = window_dataset(x_train, window_size)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.Dense(1, input_shape=[window_size])\n", "])\n", "\n", "lr_schedule = keras.callbacks.LearningRateScheduler(\n", " lambda epoch: 1e-6 * 10**(epoch / 30))\n", "optimizer = keras.optimizers.SGD(lr=1e-6, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "PF9e7IDm8OId" }, "outputs": [], "source": [ "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n", "plt.axis([1e-6, 1e-3, 0, 20])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "uMNwyIFE8OIf" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = window_dataset(x_train, window_size)\n", "valid_set = window_dataset(x_valid, window_size)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.Dense(1, input_shape=[window_size])\n", "])\n", "optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "early_stopping = keras.callbacks.EarlyStopping(patience=10)\n", "model.fit(train_set, epochs=500,\n", " validation_data=valid_set,\n", " callbacks=[early_stopping])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "_eaAX9g_jS5W" }, "outputs": [], "source": [ "def model_forecast(model, series, window_size):\n", " ds = tf.data.Dataset.from_tensor_slices(series)\n", " ds = ds.window(window_size, shift=1, drop_remainder=True)\n", " ds = ds.flat_map(lambda w: w.batch(window_size))\n", " ds = ds.batch(32).prefetch(1)\n", " forecast = model.predict(ds)\n", " return forecast" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FnIWROQ08OIj" }, "outputs": [], "source": [ "lin_forecast = model_forecast(model, series[split_time - window_size:-1], window_size)[:, 0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "xd7Tj_fA8OIk" }, "outputs": [], "source": [ "lin_forecast.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "F-nftslfgQJs" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid)\n", "plot_series(time_valid, lin_forecast)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W4E_jXktf7iv" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, lin_forecast).numpy()" ] }, { "cell_type": "markdown", "metadata": { "id": "9nEM33dZ8OIp" }, "source": [ "### Dense Model Forecasting" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "RhGTv4G_8OIp" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = window_dataset(x_train, window_size)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.Dense(10, activation=\"relu\", input_shape=[window_size]),\n", " keras.layers.Dense(10, activation=\"relu\"),\n", " keras.layers.Dense(1)\n", "])\n", "\n", "lr_schedule = keras.callbacks.LearningRateScheduler(\n", " lambda epoch: 1e-7 * 10**(epoch / 20))\n", "optimizer = keras.optimizers.SGD(lr=1e-7, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5g-nC_em8OIq" }, "outputs": [], "source": [ "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n", "plt.axis([1e-7, 5e-3, 0, 30])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "B7t0VrCH8OIr" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = window_dataset(x_train, window_size)\n", "valid_set = window_dataset(x_valid, window_size)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.Dense(10, activation=\"relu\", input_shape=[window_size]),\n", " keras.layers.Dense(10, activation=\"relu\"),\n", " keras.layers.Dense(1)\n", "])\n", "\n", "optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "early_stopping = keras.callbacks.EarlyStopping(patience=10)\n", "model.fit(train_set, epochs=500,\n", " validation_data=valid_set,\n", " callbacks=[early_stopping])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "RqQbX6DZ8OIu" }, "outputs": [], "source": [ "dense_forecast = model_forecast(\n", " model,\n", " series[split_time - window_size:-1],\n", " window_size)[:, 0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "98zwAuIo8OIv" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid)\n", "plot_series(time_valid, dense_forecast)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "EgkELN-58OIw" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, dense_forecast).numpy()" ] } ], "metadata": { "colab": { "collapsed_sections": [ "vidayERjaO5q" ], "name": "l08c05_forecasting_with_machine_learning.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l08c06_forecasting_with_rnn.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "nuRx7K-sirJr" }, "source": [ "# Forecasting with an RNN" ] }, { "cell_type": "markdown", "metadata": { "id": "97jsq1rHh2Ds" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "vidayERjaO5q" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gqWabzlJ63nL" }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import tensorflow as tf\n", "\n", "keras = tf.keras" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cg1hfKCPldZG" }, "outputs": [], "source": [ "def plot_series(time, series, format=\"-\", start=0, end=None, label=None):\n", " plt.plot(time[start:end], series[start:end], format, label=label)\n", " plt.xlabel(\"Time\")\n", " plt.ylabel(\"Value\")\n", " if label:\n", " plt.legend(fontsize=14)\n", " plt.grid(True)\n", " \n", "def trend(time, slope=0):\n", " return slope * time\n", " \n", " \n", "def seasonal_pattern(season_time):\n", " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", " return np.where(season_time < 0.4,\n", " np.cos(season_time * 2 * np.pi),\n", " 1 / np.exp(3 * season_time))\n", "\n", " \n", "def seasonality(time, period, amplitude=1, phase=0):\n", " \"\"\"Repeats the same pattern at each period\"\"\"\n", " season_time = ((time + phase) % period) / period\n", " return amplitude * seasonal_pattern(season_time)\n", " \n", " \n", "def white_noise(time, noise_level=1, seed=None):\n", " rnd = np.random.RandomState(seed)\n", " return rnd.randn(len(time)) * noise_level\n", " \n", " \n", "def window_dataset(series, window_size, batch_size=32,\n", " shuffle_buffer=1000):\n", " dataset = tf.data.Dataset.from_tensor_slices(series)\n", " dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)\n", " dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))\n", " dataset = dataset.shuffle(shuffle_buffer)\n", " dataset = dataset.map(lambda window: (window[:-1], window[-1]))\n", " dataset = dataset.batch(batch_size).prefetch(1)\n", " return dataset\n", " \n", "def model_forecast(model, series, window_size):\n", " ds = tf.data.Dataset.from_tensor_slices(series)\n", " ds = ds.window(window_size, shift=1, drop_remainder=True)\n", " ds = ds.flat_map(lambda w: w.batch(window_size))\n", " ds = ds.batch(32).prefetch(1)\n", " forecast = model.predict(ds)\n", " return forecast" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "iL2DDjV3lel6" }, "outputs": [], "source": [ "time = np.arange(4 * 365 + 1)\n", "\n", "slope = 0.05\n", "baseline = 10\n", "amplitude = 40\n", "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", "\n", "noise_level = 5\n", "noise = white_noise(time, noise_level, seed=42)\n", "\n", "series += noise\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Zmp1JXKxk9Vb" }, "outputs": [], "source": [ "split_time = 1000\n", "time_train = time[:split_time]\n", "x_train = series[:split_time]\n", "time_valid = time[split_time:]\n", "x_valid = series[split_time:]" ] }, { "cell_type": "markdown", "metadata": { "id": "vDs_w3kZ8OIw" }, "source": [ "## Simple RNN Forecasting" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "YU4xRp9G8OIx" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = window_dataset(x_train, window_size, batch_size=128)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=-1),\n", " input_shape=[None]),\n", " keras.layers.SimpleRNN(100, return_sequences=True),\n", " keras.layers.SimpleRNN(100),\n", " keras.layers.Dense(1),\n", " keras.layers.Lambda(lambda x: x * 200.0)\n", "])\n", "lr_schedule = keras.callbacks.LearningRateScheduler(\n", " lambda epoch: 1e-7 * 10**(epoch / 20))\n", "optimizer = keras.optimizers.SGD(lr=1e-7, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "YJTlFAXF8OIy" }, "outputs": [], "source": [ "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n", "plt.axis([1e-7, 1e-4, 0, 30])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "T3yNjxWE8OIz" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = window_dataset(x_train, window_size, batch_size=128)\n", "valid_set = window_dataset(x_valid, window_size, batch_size=128)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=-1),\n", " input_shape=[None]),\n", " keras.layers.SimpleRNN(100, return_sequences=True),\n", " keras.layers.SimpleRNN(100),\n", " keras.layers.Dense(1),\n", " keras.layers.Lambda(lambda x: x * 200.0)\n", "])\n", "optimizer = keras.optimizers.SGD(lr=1.5e-6, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "early_stopping = keras.callbacks.EarlyStopping(patience=50)\n", "model_checkpoint = keras.callbacks.ModelCheckpoint(\n", " \"my_checkpoint\", save_best_only=True)\n", "model.fit(train_set, epochs=500,\n", " validation_data=valid_set,\n", " callbacks=[early_stopping, model_checkpoint])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4KuPtKFe8OI0" }, "outputs": [], "source": [ "model = keras.models.load_model(\"my_checkpoint\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cxq09qOg8OI1" }, "outputs": [], "source": [ "rnn_forecast = model_forecast(\n", " model,\n", " series[split_time - window_size:-1],\n", " window_size)[:, 0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "PkC_JssS8OI2" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid)\n", "plot_series(time_valid, rnn_forecast)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1mwfgEK08OI3" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()" ] }, { "cell_type": "markdown", "metadata": { "id": "KNG7s8jt8OI4" }, "source": [ "## Sequence-to-Sequence Forecasting" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "bsKGxfiE8OI4" }, "outputs": [], "source": [ "def seq2seq_window_dataset(series, window_size, batch_size=32,\n", " shuffle_buffer=1000):\n", " series = tf.expand_dims(series, axis=-1)\n", " ds = tf.data.Dataset.from_tensor_slices(series)\n", " ds = ds.window(window_size + 1, shift=1, drop_remainder=True)\n", " ds = ds.flat_map(lambda w: w.batch(window_size + 1))\n", " ds = ds.shuffle(shuffle_buffer)\n", " ds = ds.map(lambda w: (w[:-1], w[1:]))\n", " return ds.batch(batch_size).prefetch(1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "5Nk2C7WP8OI5" }, "outputs": [], "source": [ "for X_batch, Y_batch in seq2seq_window_dataset(tf.range(10), 3,\n", " batch_size=1):\n", " print(\"X:\", X_batch.numpy())\n", " print(\"Y:\", Y_batch.numpy())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4JSc-Btk8OI7" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = seq2seq_window_dataset(x_train, window_size,\n", " batch_size=128)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.SimpleRNN(100, return_sequences=True,\n", " input_shape=[None, 1]),\n", " keras.layers.SimpleRNN(100, return_sequences=True),\n", " keras.layers.Dense(1),\n", " keras.layers.Lambda(lambda x: x * 200)\n", "])\n", "lr_schedule = keras.callbacks.LearningRateScheduler(\n", " lambda epoch: 1e-7 * 10**(epoch / 30))\n", "optimizer = keras.optimizers.SGD(lr=1e-7, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "YGNsWceq8OI8" }, "outputs": [], "source": [ "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n", "plt.axis([1e-7, 1e-4, 0, 30])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "G9lDnb0X8OI9" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = seq2seq_window_dataset(x_train, window_size,\n", " batch_size=128)\n", "valid_set = seq2seq_window_dataset(x_valid, window_size,\n", " batch_size=128)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.SimpleRNN(100, return_sequences=True,\n", " input_shape=[None, 1]),\n", " keras.layers.SimpleRNN(100, return_sequences=True),\n", " keras.layers.Dense(1),\n", " keras.layers.Lambda(lambda x: x * 200.0)\n", "])\n", "optimizer = keras.optimizers.SGD(lr=1e-6, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "early_stopping = keras.callbacks.EarlyStopping(patience=10)\n", "model.fit(train_set, epochs=500,\n", " validation_data=valid_set,\n", " callbacks=[early_stopping])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4mglBRex8OI_" }, "outputs": [], "source": [ "rnn_forecast = model_forecast(model, series[..., np.newaxis], window_size)\n", "rnn_forecast = rnn_forecast[split_time - window_size:-1, -1, 0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Zl_FkcdI8OJA" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid)\n", "plot_series(time_valid, rnn_forecast)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cznEtSVK8OJB" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()" ] } ], "metadata": { "colab": { "collapsed_sections": [ "vidayERjaO5q" ], "name": "l08c06_forecasting_with_rnn.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l08c07_forecasting_with_stateful_rnn.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "eJoEl2FhixRp" }, "source": [ "# Forecasting with a stateful RNN" ] }, { "cell_type": "markdown", "metadata": { "id": "w3Hibo73h236" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "vidayERjaO5q" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gqWabzlJ63nL" }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import tensorflow as tf\n", "\n", "keras = tf.keras" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cg1hfKCPldZG" }, "outputs": [], "source": [ "def plot_series(time, series, format=\"-\", start=0, end=None, label=None):\n", " plt.plot(time[start:end], series[start:end], format, label=label)\n", " plt.xlabel(\"Time\")\n", " plt.ylabel(\"Value\")\n", " if label:\n", " plt.legend(fontsize=14)\n", " plt.grid(True)\n", " \n", "def trend(time, slope=0):\n", " return slope * time\n", " \n", " \n", "def seasonal_pattern(season_time):\n", " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", " return np.where(season_time < 0.4,\n", " np.cos(season_time * 2 * np.pi),\n", " 1 / np.exp(3 * season_time))\n", "\n", " \n", "def seasonality(time, period, amplitude=1, phase=0):\n", " \"\"\"Repeats the same pattern at each period\"\"\"\n", " season_time = ((time + phase) % period) / period\n", " return amplitude * seasonal_pattern(season_time)\n", " \n", " \n", "def white_noise(time, noise_level=1, seed=None):\n", " rnd = np.random.RandomState(seed)\n", " return rnd.randn(len(time)) * noise_level" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "iL2DDjV3lel6" }, "outputs": [], "source": [ "time = np.arange(4 * 365 + 1)\n", "\n", "slope = 0.05\n", "baseline = 10\n", "amplitude = 40\n", "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", "\n", "noise_level = 5\n", "noise = white_noise(time, noise_level, seed=42)\n", "\n", "series += noise\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Zmp1JXKxk9Vb" }, "outputs": [], "source": [ "split_time = 1000\n", "time_train = time[:split_time]\n", "x_train = series[:split_time]\n", "time_valid = time[split_time:]\n", "x_valid = series[split_time:]" ] }, { "cell_type": "markdown", "metadata": { "id": "43Entmgk8OJC" }, "source": [ "## Stateful RNN Forecasting" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ezzIGpsq8OJC" }, "outputs": [], "source": [ "def sequential_window_dataset(series, window_size):\n", " series = tf.expand_dims(series, axis=-1)\n", " ds = tf.data.Dataset.from_tensor_slices(series)\n", " ds = ds.window(window_size + 1, shift=window_size, drop_remainder=True)\n", " ds = ds.flat_map(lambda window: window.batch(window_size + 1))\n", " ds = ds.map(lambda window: (window[:-1], window[1:]))\n", " return ds.batch(1).prefetch(1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cC2hh-Dn8OJD" }, "outputs": [], "source": [ "for X_batch, y_batch in sequential_window_dataset(tf.range(10), 3):\n", " print(X_batch.numpy(), y_batch.numpy())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "CHCYhVvl8OJE" }, "outputs": [], "source": [ "class ResetStatesCallback(keras.callbacks.Callback):\n", " def on_epoch_begin(self, epoch, logs):\n", " self.model.reset_states()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "g03h-KGJ8OJF" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = sequential_window_dataset(x_train, window_size)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.SimpleRNN(100, return_sequences=True, stateful=True,\n", " batch_input_shape=[1, None, 1]),\n", " keras.layers.SimpleRNN(100, return_sequences=True, stateful=True),\n", " keras.layers.Dense(1),\n", " keras.layers.Lambda(lambda x: x * 200.0)\n", "])\n", "lr_schedule = keras.callbacks.LearningRateScheduler(\n", " lambda epoch: 1e-8 * 10**(epoch / 30))\n", "reset_states = ResetStatesCallback()\n", "optimizer = keras.optimizers.SGD(lr=1e-8, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "history = model.fit(train_set, epochs=100,\n", " callbacks=[lr_schedule, reset_states])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "rl_s5X448OJG" }, "outputs": [], "source": [ "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n", "plt.axis([1e-8, 1e-4, 0, 30])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Gi3suj8F8OJH" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = sequential_window_dataset(x_train, window_size)\n", "valid_set = sequential_window_dataset(x_valid, window_size)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.SimpleRNN(100, return_sequences=True, stateful=True,\n", " batch_input_shape=[1, None, 1]),\n", " keras.layers.SimpleRNN(100, return_sequences=True, stateful=True),\n", " keras.layers.Dense(1),\n", " keras.layers.Lambda(lambda x: x * 200.0)\n", "])\n", "optimizer = keras.optimizers.SGD(lr=1e-7, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "reset_states = ResetStatesCallback()\n", "model_checkpoint = keras.callbacks.ModelCheckpoint(\n", " \"my_checkpoint.h5\", save_best_only=True)\n", "early_stopping = keras.callbacks.EarlyStopping(patience=50)\n", "model.fit(train_set, epochs=500,\n", " validation_data=valid_set,\n", " callbacks=[early_stopping, model_checkpoint, reset_states])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lfW42jg-8OJI" }, "outputs": [], "source": [ "model = keras.models.load_model(\"my_checkpoint.h5\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JC3XHLxd8OJJ" }, "outputs": [], "source": [ "model.reset_states()\n", "rnn_forecast = model.predict(series[np.newaxis, :, np.newaxis])\n", "rnn_forecast = rnn_forecast[0, split_time - 1:-1, 0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Cwsvi0cxmwBZ" }, "outputs": [], "source": [ "rnn_forecast.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "s6eZmrn48OJK" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid)\n", "plot_series(time_valid, rnn_forecast)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "8z6Py2Lx8OJL" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()" ] } ], "metadata": { "colab": { "collapsed_sections": [ "vidayERjaO5q" ], "name": "l08c07_forecasting_with_stateful_rnn.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l08c08_forecasting_with_lstm.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "E5VI4y76i14x" }, "source": [ "# Forecasting with an LSTM" ] }, { "cell_type": "markdown", "metadata": { "id": "-Da7hZzJh3mg" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "vidayERjaO5q" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gqWabzlJ63nL" }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import tensorflow as tf\n", "\n", "keras = tf.keras" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cg1hfKCPldZG" }, "outputs": [], "source": [ "def plot_series(time, series, format=\"-\", start=0, end=None, label=None):\n", " plt.plot(time[start:end], series[start:end], format, label=label)\n", " plt.xlabel(\"Time\")\n", " plt.ylabel(\"Value\")\n", " if label:\n", " plt.legend(fontsize=14)\n", " plt.grid(True)\n", " \n", "def trend(time, slope=0):\n", " return slope * time\n", " \n", " \n", "def seasonal_pattern(season_time):\n", " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", " return np.where(season_time < 0.4,\n", " np.cos(season_time * 2 * np.pi),\n", " 1 / np.exp(3 * season_time))\n", "\n", " \n", "def seasonality(time, period, amplitude=1, phase=0):\n", " \"\"\"Repeats the same pattern at each period\"\"\"\n", " season_time = ((time + phase) % period) / period\n", " return amplitude * seasonal_pattern(season_time)\n", " \n", " \n", "def white_noise(time, noise_level=1, seed=None):\n", " rnd = np.random.RandomState(seed)\n", " return rnd.randn(len(time)) * noise_level\n", " \n", "\n", "def sequential_window_dataset(series, window_size):\n", " series = tf.expand_dims(series, axis=-1)\n", " ds = tf.data.Dataset.from_tensor_slices(series)\n", " ds = ds.window(window_size + 1, shift=window_size, drop_remainder=True)\n", " ds = ds.flat_map(lambda window: window.batch(window_size + 1))\n", " ds = ds.map(lambda window: (window[:-1], window[1:]))\n", " return ds.batch(1).prefetch(1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "iL2DDjV3lel6" }, "outputs": [], "source": [ "time = np.arange(4 * 365 + 1)\n", "\n", "slope = 0.05\n", "baseline = 10\n", "amplitude = 40\n", "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", "\n", "noise_level = 5\n", "noise = white_noise(time, noise_level, seed=42)\n", "\n", "series += noise\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Zmp1JXKxk9Vb" }, "outputs": [], "source": [ "split_time = 1000\n", "time_train = time[:split_time]\n", "x_train = series[:split_time]\n", "time_valid = time[split_time:]\n", "x_valid = series[split_time:]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9fPenJpTtuDE" }, "outputs": [], "source": [ "class ResetStatesCallback(keras.callbacks.Callback):\n", " def on_epoch_begin(self, epoch, logs):\n", " self.model.reset_states()" ] }, { "cell_type": "markdown", "metadata": { "id": "EPjK0l9P8OJM" }, "source": [ "## LSTM RNN Forecasting" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cSoUmW-x8OJN" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = sequential_window_dataset(x_train, window_size)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.LSTM(100, return_sequences=True, stateful=True,\n", " batch_input_shape=[1, None, 1]),\n", " keras.layers.LSTM(100, return_sequences=True, stateful=True),\n", " keras.layers.Dense(1),\n", " keras.layers.Lambda(lambda x: x * 200.0)\n", "])\n", "lr_schedule = keras.callbacks.LearningRateScheduler(\n", " lambda epoch: 1e-8 * 10**(epoch / 20))\n", "reset_states = ResetStatesCallback()\n", "optimizer = keras.optimizers.SGD(lr=1e-8, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "history = model.fit(train_set, epochs=100,\n", " callbacks=[lr_schedule, reset_states])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "KA0GM9sQ8OJO" }, "outputs": [], "source": [ "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n", "plt.axis([1e-8, 1e-4, 0, 30])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "hiHR5pPL8OJP" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = sequential_window_dataset(x_train, window_size)\n", "valid_set = sequential_window_dataset(x_valid, window_size)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.LSTM(100, return_sequences=True, stateful=True,\n", " batch_input_shape=[1, None, 1]),\n", " keras.layers.LSTM(100, return_sequences=True, stateful=True),\n", " keras.layers.Dense(1),\n", " keras.layers.Lambda(lambda x: x * 200.0)\n", "])\n", "optimizer = keras.optimizers.SGD(lr=5e-7, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "reset_states = ResetStatesCallback()\n", "model_checkpoint = keras.callbacks.ModelCheckpoint(\n", " \"my_checkpoint.h5\", save_best_only=True)\n", "early_stopping = keras.callbacks.EarlyStopping(patience=50)\n", "model.fit(train_set, epochs=500,\n", " validation_data=valid_set,\n", " callbacks=[early_stopping, model_checkpoint, reset_states])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nPeZUfQy8OJQ" }, "outputs": [], "source": [ "model = keras.models.load_model(\"my_checkpoint.h5\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4tFrq5uW8OJR" }, "outputs": [], "source": [ "rnn_forecast = model.predict(series[np.newaxis, :, np.newaxis])\n", "rnn_forecast = rnn_forecast[0, split_time - 1:-1, 0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ZfaR6nqj8OJT" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid)\n", "plot_series(time_valid, rnn_forecast)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Wgf2u2Tp8OJV" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [ "vidayERjaO5q" ], "name": "l08c08_forecasting_with_lstm.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l08c09_forecasting_with_cnn.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "Tt5S6SiPi7ze" }, "source": [ "# Forecasting with a CNN" ] }, { "cell_type": "markdown", "metadata": { "id": "W2iENc3Nh6g7" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "vidayERjaO5q" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gqWabzlJ63nL" }, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import tensorflow as tf\n", "\n", "keras = tf.keras" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cg1hfKCPldZG" }, "outputs": [], "source": [ "def plot_series(time, series, format=\"-\", start=0, end=None, label=None):\n", " plt.plot(time[start:end], series[start:end], format, label=label)\n", " plt.xlabel(\"Time\")\n", " plt.ylabel(\"Value\")\n", " if label:\n", " plt.legend(fontsize=14)\n", " plt.grid(True)\n", "\n", "\n", "def trend(time, slope=0):\n", " return slope * time\n", " \n", " \n", "def seasonal_pattern(season_time):\n", " \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n", " return np.where(season_time < 0.4,\n", " np.cos(season_time * 2 * np.pi),\n", " 1 / np.exp(3 * season_time))\n", "\n", " \n", "def seasonality(time, period, amplitude=1, phase=0):\n", " \"\"\"Repeats the same pattern at each period\"\"\"\n", " season_time = ((time + phase) % period) / period\n", " return amplitude * seasonal_pattern(season_time)\n", " \n", " \n", "def white_noise(time, noise_level=1, seed=None):\n", " rnd = np.random.RandomState(seed)\n", " return rnd.randn(len(time)) * noise_level\n", " \n", "\n", "def seq2seq_window_dataset(series, window_size, batch_size=32,\n", " shuffle_buffer=1000):\n", " series = tf.expand_dims(series, axis=-1)\n", " ds = tf.data.Dataset.from_tensor_slices(series)\n", " ds = ds.window(window_size + 1, shift=1, drop_remainder=True)\n", " ds = ds.flat_map(lambda w: w.batch(window_size + 1))\n", " ds = ds.shuffle(shuffle_buffer)\n", " ds = ds.map(lambda w: (w[:-1], w[1:]))\n", " return ds.batch(batch_size).prefetch(1)\n", " \n", "\n", "def model_forecast(model, series, window_size):\n", " ds = tf.data.Dataset.from_tensor_slices(series)\n", " ds = ds.window(window_size, shift=1, drop_remainder=True)\n", " ds = ds.flat_map(lambda w: w.batch(window_size))\n", " ds = ds.batch(32).prefetch(1)\n", " forecast = model.predict(ds)\n", " return forecast" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "iL2DDjV3lel6" }, "outputs": [], "source": [ "time = np.arange(4 * 365 + 1)\n", "\n", "slope = 0.05\n", "baseline = 10\n", "amplitude = 40\n", "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n", "\n", "noise_level = 5\n", "noise = white_noise(time, noise_level, seed=42)\n", "\n", "series += noise\n", "\n", "plt.figure(figsize=(10, 6))\n", "plot_series(time, series)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Zmp1JXKxk9Vb" }, "outputs": [], "source": [ "split_time = 1000\n", "time_train = time[:split_time]\n", "x_train = series[:split_time]\n", "time_valid = time[split_time:]\n", "x_valid = series[split_time:]" ] }, { "cell_type": "markdown", "metadata": { "id": "DI2GlupZ8OJW" }, "source": [ "## Preprocessing With 1D-Convolutional Layers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6GFE82ci8OJW" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = seq2seq_window_dataset(x_train, window_size,\n", " batch_size=128)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.Conv1D(filters=32, kernel_size=5,\n", " strides=1, padding=\"causal\",\n", " activation=\"relu\",\n", " input_shape=[None, 1]),\n", " keras.layers.LSTM(32, return_sequences=True),\n", " keras.layers.LSTM(32, return_sequences=True),\n", " keras.layers.Dense(1),\n", " keras.layers.Lambda(lambda x: x * 200)\n", "])\n", "lr_schedule = keras.callbacks.LearningRateScheduler(\n", " lambda epoch: 1e-8 * 10**(epoch / 20))\n", "optimizer = keras.optimizers.SGD(lr=1e-8, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Uungw0H58OJX" }, "outputs": [], "source": [ "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n", "plt.axis([1e-8, 1e-4, 0, 30])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "UG7cO0yr8OJY" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 30\n", "train_set = seq2seq_window_dataset(x_train, window_size,\n", " batch_size=128)\n", "valid_set = seq2seq_window_dataset(x_valid, window_size,\n", " batch_size=128)\n", "\n", "model = keras.models.Sequential([\n", " keras.layers.Conv1D(filters=32, kernel_size=5,\n", " strides=1, padding=\"causal\",\n", " activation=\"relu\",\n", " input_shape=[None, 1]),\n", " keras.layers.LSTM(32, return_sequences=True),\n", " keras.layers.LSTM(32, return_sequences=True),\n", " keras.layers.Dense(1),\n", " keras.layers.Lambda(lambda x: x * 200)\n", "])\n", "optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "\n", "model_checkpoint = keras.callbacks.ModelCheckpoint(\n", " \"my_checkpoint.h5\", save_best_only=True)\n", "early_stopping = keras.callbacks.EarlyStopping(patience=50)\n", "model.fit(train_set, epochs=500,\n", " validation_data=valid_set,\n", " callbacks=[early_stopping, model_checkpoint])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "BlqzLwfn8OJa" }, "outputs": [], "source": [ "model = keras.models.load_model(\"my_checkpoint.h5\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Pj0rpT-48OJc" }, "outputs": [], "source": [ "rnn_forecast = model_forecast(model, series[:, np.newaxis], window_size)\n", "rnn_forecast = rnn_forecast[split_time - window_size:-1, -1, 0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3vnDU8wm8OJd" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid)\n", "plot_series(time_valid, rnn_forecast)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "W6tWOoE88OJe" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()" ] }, { "cell_type": "markdown", "metadata": { "id": "kfPTmghd8OJe" }, "source": [ "## Fully Convolutional Forecasting" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4-cPF5CX8OJf" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 64\n", "train_set = seq2seq_window_dataset(x_train, window_size,\n", " batch_size=128)\n", "\n", "model = keras.models.Sequential()\n", "model.add(keras.layers.InputLayer(input_shape=[None, 1]))\n", "for dilation_rate in (1, 2, 4, 8, 16, 32):\n", " model.add(\n", " keras.layers.Conv1D(filters=32,\n", " kernel_size=2,\n", " strides=1,\n", " dilation_rate=dilation_rate,\n", " padding=\"causal\",\n", " activation=\"relu\")\n", " )\n", "model.add(keras.layers.Conv1D(filters=1, kernel_size=1))\n", "lr_schedule = keras.callbacks.LearningRateScheduler(\n", " lambda epoch: 1e-4 * 10**(epoch / 30))\n", "optimizer = keras.optimizers.Adam(lr=1e-4)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "GfWVZ8k-8OJf" }, "outputs": [], "source": [ "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n", "plt.axis([1e-4, 1e-1, 0, 30])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "WVrxlzbk8OJg" }, "outputs": [], "source": [ "keras.backend.clear_session()\n", "tf.random.set_seed(42)\n", "np.random.seed(42)\n", "\n", "window_size = 64\n", "train_set = seq2seq_window_dataset(x_train, window_size,\n", " batch_size=128)\n", "valid_set = seq2seq_window_dataset(x_valid, window_size,\n", " batch_size=128)\n", "\n", "model = keras.models.Sequential()\n", "model.add(keras.layers.InputLayer(input_shape=[None, 1]))\n", "for dilation_rate in (1, 2, 4, 8, 16, 32):\n", " model.add(\n", " keras.layers.Conv1D(filters=32,\n", " kernel_size=2,\n", " strides=1,\n", " dilation_rate=dilation_rate,\n", " padding=\"causal\",\n", " activation=\"relu\")\n", " )\n", "model.add(keras.layers.Conv1D(filters=1, kernel_size=1))\n", "optimizer = keras.optimizers.Adam(lr=3e-4)\n", "model.compile(loss=keras.losses.Huber(),\n", " optimizer=optimizer,\n", " metrics=[\"mae\"])\n", "\n", "model_checkpoint = keras.callbacks.ModelCheckpoint(\n", " \"my_checkpoint.h5\", save_best_only=True)\n", "early_stopping = keras.callbacks.EarlyStopping(patience=50)\n", "history = model.fit(train_set, epochs=500,\n", " validation_data=valid_set,\n", " callbacks=[early_stopping, model_checkpoint])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "eNwWZB0d8OJh" }, "outputs": [], "source": [ "model = keras.models.load_model(\"my_checkpoint.h5\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "PgYwn9VM8OJi" }, "outputs": [], "source": [ "cnn_forecast = model_forecast(model, series[..., np.newaxis], window_size)\n", "cnn_forecast = cnn_forecast[split_time - window_size:-1, -1, 0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "MCgshNPx8OJi" }, "outputs": [], "source": [ "plt.figure(figsize=(10, 6))\n", "plot_series(time_valid, x_valid)\n", "plot_series(time_valid, cnn_forecast)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "epK1gFEN8OJj" }, "outputs": [], "source": [ "keras.metrics.mean_absolute_error(x_valid, cnn_forecast).numpy()" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [ "vidayERjaO5q" ], "name": "l08c09_forecasting_with_cnn.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l09c01_nlp_turn_words_into_tokens.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "fFv-USWkhQKA" }, "source": [ "# Tokenizing text and creating sequences for sentences" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "7hMGfCIDPnm8" }, "source": [ "This colab shows you how to tokenize text and create sequences for sentences as the first stage of preparing text for use with TensorFlow models." ] }, { "cell_type": "markdown", "metadata": { "id": "4mGaRDFcSamt" }, "source": [ "## Import the Tokenizer" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "EN1-FZodOuPl" }, "outputs": [], "source": [ "# Import the Tokenizer\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n" ] }, { "cell_type": "markdown", "metadata": { "id": "C5Qwn_7FSXW-" }, "source": [ "## Write some sentences\n", "\n", "Feel free to change and add sentences as you like" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "RMiq8BpWVVRa" }, "outputs": [], "source": [ "sentences = [\n", " 'My favorite food is ice cream',\n", " 'do you like ice cream too?',\n", " 'My dog likes ice cream!',\n", " \"your favorite flavor of icecream is chocolate\",\n", " \"chocolate isn't good for dogs\",\n", " \"your dog, your cat, and your parrot prefer broccoli\"\n", "]" ] }, { "cell_type": "markdown", "metadata": { "id": "wz845OtfRBCM" }, "source": [ "## Tokenize the words\n", "\n", "The first step to preparing text to be used in a machine learning model is to tokenize the text, in other words, to generate numbers for the words." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ZHTK1DAlQ1zO" }, "outputs": [], "source": [ "# Optionally set the max number of words to tokenize.\n", "# The out of vocabulary (OOV) token represents words that are not in the index.\n", "# Call fit_on_text() on the tokenizer to generate unique numbers for each word\n", "tokenizer = Tokenizer(num_words = 100, oov_token=\"\")\n", "tokenizer.fit_on_texts(sentences)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "Mylv-WuiRzd0" }, "source": [ "## View the word index\n", "After you tokenize the text, the tokenizer has a word index that contains key-value pairs for all the words and their numbers.\n", "\n", "The word is the key, and the number is the value.\n", "\n", "Notice that the OOV token is the first entry.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kX4VvsLySC7Z" }, "outputs": [], "source": [ "# Examine the word index\n", "word_index = tokenizer.word_index\n", "print(word_index)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JXKrGxsIVtLo" }, "outputs": [], "source": [ "# Get the number for a given word\n", "print(word_index['favorite'])" ] }, { "cell_type": "markdown", "metadata": { "id": "kcN_yM8O1oSX" }, "source": [ "# Create sequences for the sentences\n", "\n", "After you tokenize the words, the word index contains a unique number for each word. However, the numbers in the word index are not ordered. Words in a sentence have an order. So after tokenizing the words, the next step is to generate sequences for the sentences." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "QlUL6Ybf1sso" }, "outputs": [], "source": [ "sequences = tokenizer.texts_to_sequences(sentences)\n", "print (sequences)" ] }, { "cell_type": "markdown", "metadata": { "id": "AswZPbuW8f-f" }, "source": [ "# Sequence sentences that contain words that are not in the word index\n", "\n", "Let's take a look at what happens if the sentence being sequenced contains words that are not in the word index.\n", "\n", "The Out of Vocabluary (OOV) token is the first entry in the word index. You will see it shows up in the sequences in place of any word that is not in the word index." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Fir7qd6X8eZc" }, "outputs": [], "source": [ "sentences2 = [\"I like hot chocolate\", \"My dogs and my hedgehog like kibble but my squirrel prefers grapes and my chickens like ice cream, preferably vanilla\"]\n", "\n", "sequences2 = tokenizer.texts_to_sequences(sentences2)\n", "print(sequences2)" ] } ], "metadata": { "colab": { "collapsed_sections": [], "name": "l09c01_nlp_turn_words_into_tokens.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l09c02_nlp_padding.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "RX9Yx50TUies" }, "source": [ "# Preparing text to use with TensorFlow models\n" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "5_MCdtjT-bly" }, "source": [ "The high level steps to prepare text to be used in a machine learning model are:\n", "\n", "1. Tokenize the words to get numerical values for them\n", "2. Create numerical sequences of the sentences\n", "3. Adjust the sequences to all be the same length.\n", "\n", "In this colab, you learn how to use padding to make the sequences all be the same length." ] }, { "cell_type": "markdown", "metadata": { "id": "qJsd8KslUn7j" }, "source": [ "## Import the classes you need" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "_ZxQf11OUtQI" }, "outputs": [], "source": [ "# Import Tokenizer and pad_sequences\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences" ] }, { "cell_type": "markdown", "metadata": { "id": "1MeEgRq4WX0v" }, "source": [ "## Write some sentences\n", "\n", "Feel free to write your own sentences here.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "PwM7IP2lTr7T" }, "outputs": [], "source": [ "sentences = [\n", " 'My favorite food is ice cream',\n", " 'do you like ice cream too?',\n", " 'My dog likes ice cream!',\n", " \"your favorite flavor of icecream is chocolate\",\n", " \"chocolate isn't good for dogs\",\n", " \"your dog, your cat, and your parrot prefer broccoli\"\n", "]\n", "print(sentences)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "jaRa9opNWmA7" }, "source": [ "## Create the Tokenizer and define an out of vocabulary token\n", "When creating the Tokenizer, you can specify the max number of words in the dictionary. You can also specify a token to represent words that are out of the vocabulary (OOV), in other words, that are not in the dictionary. This OOV token will be used when you create sequences for sentences that contain words that are not in the word index." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "P7wuOJaBWiHZ" }, "outputs": [], "source": [ "tokenizer = Tokenizer(num_words = 100, oov_token=\"\")" ] }, { "cell_type": "markdown", "metadata": { "id": "r7nILrhKXPge" }, "source": [ "## Tokenize the words" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "YXooiuwrXROU" }, "outputs": [], "source": [ "tokenizer.fit_on_texts(sentences)\n", "word_index = tokenizer.word_index\n", "print(word_index)" ] }, { "cell_type": "markdown", "metadata": { "id": "0U-oe201Xm7T" }, "source": [ "## Turn sentences into sequences \n", "\n", "Each word now has a unique number in the word index. However, words in a sentence are in a specific order. You can't just randomly mix up words and have the outcome be a sentence.\n", "\n", "For example, although \"chocolate isn't good for dogs\" is a perfectly fine sentence, \"dogs isn't for chocolate good\" does not make sense as a sentence.\n", "\n", "So the next step to representing text in a way that can be meaningfully used by machine learning programs is to create numerical sequences that represent the sentences in the text.\n", "\n", "Each sentence will be converted into a sequence where each word is replaced by its number in the word index. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "70l5x1XRXoV4" }, "outputs": [], "source": [ "sequences = tokenizer.texts_to_sequences(sentences)\n", "print (sequences)" ] }, { "cell_type": "markdown", "metadata": { "id": "tcFghvQ34cZK" }, "source": [ "## Make the sequences all the same length\n", "\n", "Later, when you feed the sequences into a neural network to train a model, the sequences all need to be uniform in size. Currently the sequences have varied lengths, so the next step is to make them all be the same size, either by padding them with zeros and/or truncating them.\n", "\n", "Use f.keras.preprocessing.sequence.pad_sequences to add zeros to the sequences to make them all be the same length. By default, the padding goes at the start of the sequences, but you can specify to pad at the end.\n", "\n", "You can optionally specify the maximum length to pad the sequences to. Sequences that are longer than the specified max length will be truncated. By default, sequences are truncated from the beginning of the sequence, but you can specify to truncate from the end.\n", "\n", "If you don't provide the max length, then the sequences are padded to match the length of the longest sentence.\n", "\n", "For all the options when padding and truncating sequences, see https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences\n", "\n", " " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "r0m_nqHg4gqu" }, "outputs": [], "source": [ "padded = pad_sequences(sequences)\n", "print(\"\\nWord Index = \" , word_index)\n", "print(\"\\nSequences = \" , sequences)\n", "print(\"\\nPadded Sequences:\")\n", "print(padded)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "VzbGtYWQ6Ofo" }, "outputs": [], "source": [ "# Specify a max length for the padded sequences\n", "padded = pad_sequences(sequences, maxlen=15)\n", "print(padded)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "HzkbHi0B64w8" }, "outputs": [], "source": [ "# Put the padding at the end of the sequences\n", "padded = pad_sequences(sequences, maxlen=15, padding=\"post\")\n", "print(padded)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "OLHheI477okX" }, "outputs": [], "source": [ "# Limit the length of the sequences, you will see some sequences get truncated\n", "padded = pad_sequences(sequences, maxlen=3)\n", "print(padded)" ] }, { "cell_type": "markdown", "metadata": { "id": "OnRKDsR197-J" }, "source": [ "## What happens if some of the sentences contain words that are not in the word index?\n", "\n", "Here's where the \"out of vocabulary\" token is used. Try generating sequences for some sentences that have words that are not in the word index." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "iqodOpn64c2U" }, "outputs": [], "source": [ "# Try turning sentences that contain words that \n", "# aren't in the word index into sequences.\n", "# Add your own sentences to the test_data\n", "test_data = [\n", " \"my best friend's favorite ice cream flavor is strawberry\",\n", " \"my dog's best friend is a manatee\"\n", "]\n", "print (test_data)\n", "\n", "# Remind ourselves which number corresponds to the\n", "# out of vocabulary token in the word index\n", "print(\" has the number\", word_index[''], \"in the word index.\")\n", "\n", "# Convert the test sentences to sequences\n", "test_seq = tokenizer.texts_to_sequences(test_data)\n", "print(\"\\nTest Sequence = \", test_seq)\n", "\n", "# Pad the new sequences\n", "padded = pad_sequences(test_seq, maxlen=10)\n", "print(\"\\nPadded Test Sequence: \")\n", "\n", "# Notice that \"1\" appears in the sequence wherever there's a word \n", "# that's not in the word index\n", "print(padded)" ] } ], "metadata": { "colab": { "collapsed_sections": [], "name": "l09c02_nlp_padding.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l09c03_nlp_prepare_larger_text_corpus.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "qzwilbae73N4" }, "source": [ "# Tokenize and sequence a bigger corpus of text\n" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "VcB-N6WrAT9q" }, "source": [ "So far, you have written some test sentences and generated a word index and then created sequences for the sentences. \n", "\n", "Now you will tokenize and sequence a larger body of text, specifically reviews from Amazon and Yelp. \n", "\n", "## About the dataset\n", "\n", "You will use a dataset containing Amazon and Yelp reviews of products and restaurants. This dataset was originally extracted from [Kaggle](https://www.kaggle.com/marklvl/sentiment-labelled-sentences-data-set).\n", "\n", "The dataset includes reviews, and each review is labelled as 0 (bad) or 1 (good). However, in this exercise, you will only work with the reviews, not the labels, to practice tokenizing and sequencing the text. \n", "\n", "### Example good reviews:\n", "\n", "* This is hands down the best phone I've ever had.\n", "* Four stars for the food & the guy in the blue shirt for his great vibe & still letting us in to eat !\n", "\n", "### Example bad reviews: \n", "\n", "* A lady at the table next to us found a live green caterpillar In her salad\n", "* If you plan to use this in a car forget about it.\n", "\n", "### See more reviews\n", "Feel free to [download the dataset](https://drive.google.com/uc?id=13ySLC_ue6Umt9RJYSeM2t-V0kCv-4C-P) from a drive folder belonging to Udacity and open it on your local machine to see more reviews." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "wr21SvWhQhvN" }, "outputs": [], "source": [ "# Import Tokenizer and pad_sequences\n", "import tensorflow as tf\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", "\n", "# Import numpy and pandas\n", "import numpy as np\n", "import pandas as pd\n" ] }, { "cell_type": "markdown", "metadata": { "id": "cJOCSbdERsdc" }, "source": [ "# Get the corpus of text\n", "\n", "The combined dataset of reviews has been saved in a Google drive belonging to Udacity. You can download it from there." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kBpFip-X69Hf" }, "outputs": [], "source": [ "path = tf.keras.utils.get_file('reviews.csv', \n", " 'https://drive.google.com/uc?id=13ySLC_ue6Umt9RJYSeM2t-V0kCv-4C-P')\n", "print (path)" ] }, { "cell_type": "markdown", "metadata": { "id": "ZCT57MVGTENX" }, "source": [ "# Get the dataset\n", "\n", "Each row in the csv file is a separate review.\n", "\n", "The csv file has 2 columns:\n", "\n", "* **text** (the review)\n", "* **sentiment** (0 or 1 indicating a bad or good review)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "TlyreClyS7H3" }, "outputs": [], "source": [ "# Read the csv file\n", "dataset = pd.read_csv(path)\n", "\n", "# Review the first few entries in the dataset\n", "dataset.head()" ] }, { "cell_type": "markdown", "metadata": { "id": "Fk5uzq4Oco7h" }, "source": [ "# Get the reviews from the csv file" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "u7uCBlAqdEzK" }, "outputs": [], "source": [ "# Get the reviews from the text column\n", "reviews = dataset['text'].tolist()" ] }, { "cell_type": "markdown", "metadata": { "id": "OS0mg5yoVzQL" }, "source": [ "# Tokenize the text\n", "Create the tokenizer, specify the OOV token, tokenize the text, then inspect the word index." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "atgLJzAiVwqB" }, "outputs": [], "source": [ "tokenizer = Tokenizer(oov_token=\"\")\n", "tokenizer.fit_on_texts(reviews)\n", "\n", "word_index = tokenizer.word_index\n", "print(len(word_index))\n", "print(word_index)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "Vfh0WGmKWyjI" }, "source": [ "# Generate sequences for the reviews\n", "Generate a sequence for each review. Set the max length to match the longest review. Add the padding zeros at the end of the review for reviews that are not as long as the longest one." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "VwyqBS2nV53o" }, "outputs": [], "source": [ "sequences = tokenizer.texts_to_sequences(reviews)\n", "padded_sequences = pad_sequences(sequences, padding='post')\n", "\n", "# What is the shape of the vector containing the padded sequences?\n", "# The shape shows the number of sequences and the length of each one.\n", "print(padded_sequences.shape)\n", "\n", "# What is the first review?\n", "print (reviews[0])\n", "\n", "# Show the sequence for the first review\n", "print(padded_sequences[0])\n", "\n", "# Try printing the review and padded sequence for other elements." ] } ], "metadata": { "colab": { "name": "l09c03_nlp_prepare_larger_text_corpus.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l09c04_nlp_embeddings_and_sentiment.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "9uBA1i1BbiJn" }, "source": [ "# Word Embeddings and Sentiment" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "1iLe4E0dB7tj" }, "source": [ "In this colab, you'll work with word embeddings and train a basic neural network to predict text sentiment. At the end, you'll be able to visualize how the network sees the related sentiment of each word in the dataset." ] }, { "cell_type": "markdown", "metadata": { "id": "wqvz1jVgbwIN" }, "source": [ "## Import TensorFlow and related functions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XIG52aKPdpux" }, "outputs": [], "source": [ "import tensorflow as tf\n", "\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences" ] }, { "cell_type": "markdown", "metadata": { "id": "pazU5OmxehIA" }, "source": [ "## Get the dataset\n", "\n", "We're going to use a dataset containing Amazon and Yelp reviews, with their related sentiment (1 for positive, 0 for negative). This dataset was originally extracted from [here](https://www.kaggle.com/marklvl/sentiment-labelled-sentences-data-set)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "qpwQT2E9ez5B" }, "outputs": [], "source": [ "!wget --no-check-certificate \\\n", " -O /tmp/sentiment.csv https://drive.google.com/uc?id=13ySLC_ue6Umt9RJYSeM2t-V0kCv-4C-P" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6Zvp9NScfMnw" }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "\n", "dataset = pd.read_csv('/tmp/sentiment.csv')\n", "\n", "sentences = dataset['text'].tolist()\n", "labels = dataset['sentiment'].tolist()\n", "\n", "# Separate out the sentences and labels into training and test sets\n", "training_size = int(len(sentences) * 0.8)\n", "\n", "training_sentences = sentences[0:training_size]\n", "testing_sentences = sentences[training_size:]\n", "training_labels = labels[0:training_size]\n", "testing_labels = labels[training_size:]\n", "\n", "# Make labels into numpy arrays for use with the network later\n", "training_labels_final = np.array(training_labels)\n", "testing_labels_final = np.array(testing_labels)" ] }, { "cell_type": "markdown", "metadata": { "id": "NHpvqaSigcST" }, "source": [ "## Tokenize the dataset\n", "\n", "Tokenize the dataset, including padding and OOV" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "78icewYRgfxh" }, "outputs": [], "source": [ "vocab_size = 1000\n", "embedding_dim = 16\n", "max_length = 100\n", "trunc_type='post'\n", "padding_type='post'\n", "oov_tok = \"\"\n", "\n", "\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", "\n", "tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)\n", "tokenizer.fit_on_texts(training_sentences)\n", "word_index = tokenizer.word_index\n", "sequences = tokenizer.texts_to_sequences(training_sentences)\n", "padded = pad_sequences(sequences,maxlen=max_length, padding=padding_type, \n", " truncating=trunc_type)\n", "\n", "testing_sequences = tokenizer.texts_to_sequences(testing_sentences)\n", "testing_padded = pad_sequences(testing_sequences,maxlen=max_length, \n", " padding=padding_type, truncating=trunc_type)" ] }, { "cell_type": "markdown", "metadata": { "id": "q4yIEk_8kszh" }, "source": [ "## Review a Sequence\n", "\n", "Let's quickly take a look at one of the padded sequences to ensure everything above worked appropriately." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JTU3FmVGk100" }, "outputs": [], "source": [ "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", "\n", "def decode_review(text):\n", " return ' '.join([reverse_word_index.get(i, '?') for i in text])\n", "\n", "print(decode_review(padded[1]))\n", "print(training_sentences[1])" ] }, { "cell_type": "markdown", "metadata": { "id": "RI91liJnlA92" }, "source": [ "## Train a Basic Sentiment Model with Embeddings" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "bBMgzp-_lMTp" }, "outputs": [], "source": [ "# Build a basic sentiment network\n", "# Note the embedding layer is first, \n", "# and the output is only 1 node as it is either 0 or 1 (negative or positive)\n", "model = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.Flatten(),\n", " tf.keras.layers.Dense(6, activation='relu'),\n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", "model.summary()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Pfl1W-zVldpn" }, "outputs": [], "source": [ "num_epochs = 10\n", "model.fit(padded, training_labels_final, epochs=num_epochs, validation_data=(testing_padded, testing_labels_final))" ] }, { "cell_type": "markdown", "metadata": { "id": "GjMZ4ZFQl_48" }, "source": [ "## Get files for visualizing the network\n", "\n", "The code below will download two files for visualizing how your network \"sees\" the sentiment related to each word. Head to http://projector.tensorflow.org/ and load these files, then click the \"Sphereize\" checkbox." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "S2lB46FirAVx" }, "outputs": [], "source": [ "# First get the weights of the embedding layer\n", "e = model.layers[0]\n", "weights = e.get_weights()[0]\n", "print(weights.shape) # shape: (vocab_size, embedding_dim)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Xcha0oGemHX2" }, "outputs": [], "source": [ "import io\n", "\n", "# Write out the embedding vectors and metadata\n", "out_v = io.open('vecs.tsv', 'w', encoding='utf-8')\n", "out_m = io.open('meta.tsv', 'w', encoding='utf-8')\n", "for word_num in range(1, vocab_size):\n", " word = reverse_word_index[word_num]\n", " embeddings = weights[word_num]\n", " out_m.write(word + \"\\n\")\n", " out_v.write('\\t'.join([str(x) for x in embeddings]) + \"\\n\")\n", "out_v.close()\n", "out_m.close()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "g-Q6ALywmWVz" }, "outputs": [], "source": [ "# Download the files\n", "try:\n", " from google.colab import files\n", "except ImportError:\n", " pass\n", "else:\n", " files.download('vecs.tsv')\n", " files.download('meta.tsv')" ] }, { "cell_type": "markdown", "metadata": { "id": "GNoxfY-i3Ir1" }, "source": [ "## Predicting Sentiment in New Reviews\n", "\n", "Now that you've trained and visualized your network, take a look below at how we can predict sentiment in new reviews the network has never seen before." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "QXtfw-OY3WoZ" }, "outputs": [], "source": [ "# Use the model to predict a review \n", "fake_reviews = ['I love this phone', 'I hate spaghetti', \n", " 'Everything was cold',\n", " 'Everything was hot exactly as I wanted', \n", " 'Everything was green', \n", " 'the host seated us immediately',\n", " 'they gave us free chocolate cake', \n", " 'not sure about the wilted flowers on the table',\n", " 'only works when I stand on tippy toes', \n", " 'does not work when I stand on my head']\n", "\n", "print(fake_reviews) \n", "\n", "# Create the sequences\n", "padding_type='post'\n", "sample_sequences = tokenizer.texts_to_sequences(fake_reviews)\n", "fakes_padded = pad_sequences(sample_sequences, padding=padding_type, maxlen=max_length) \n", "\n", "print('\\nHOT OFF THE PRESS! HERE ARE SOME NEWLY MINTED, ABSOLUTELY GENUINE REVIEWS!\\n') \n", "\n", "classes = model.predict(fakes_padded)\n", "\n", "# The closer the class is to 1, the more positive the review is deemed to be\n", "for x in range(len(fake_reviews)):\n", " print(fake_reviews[x])\n", " print(classes[x])\n", " print('\\n')\n", "\n", "# Try adding reviews of your own\n", "# Add some negative words (such as \"not\") to the good reviews and see what happens\n", "# For example:\n", "# they gave us free chocolate cake and did not charge us" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l09c04_nlp_embeddings_and_sentiment.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l09c05_nlp_tweaking_the_model.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "QrxSyyyhygUR" }, "source": [ "# Tweaking the Model" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "xiWacy71Cu54" }, "source": [ "In this colab, you'll investigate how various tweaks to data processing and the model itself can impact results. At the end, you'll once again be able to visualize how the network sees the related sentiment of each word in the dataset." ] }, { "cell_type": "markdown", "metadata": { "id": "hY-fjvwfy2P9" }, "source": [ "## Import TensorFlow and related functions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "drsUfVVXyxJl" }, "outputs": [], "source": [ "import tensorflow as tf\n", "\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences" ] }, { "cell_type": "markdown", "metadata": { "id": "ZIf1N46jy6Ed" }, "source": [ "## Get the dataset\n", "\n", "We'll once again use the dataset containing Amazon and Yelp reviews. This dataset was originally extracted from [here](https://www.kaggle.com/marklvl/sentiment-labelled-sentences-data-set)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "m83g42sJzGO0" }, "outputs": [], "source": [ "!wget --no-check-certificate \\\n", " https://drive.google.com/uc?id=13ySLC_ue6Umt9RJYSeM2t-V0kCv-4C-P \\\n", " -O /tmp/sentiment.csv" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "y4e6GG2CzJUq" }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "\n", "dataset = pd.read_csv('/tmp/sentiment.csv')\n", "\n", "sentences = dataset['text'].tolist()\n", "labels = dataset['sentiment'].tolist()\n", "\n", "# Separate out the sentences and labels into training and test sets\n", "training_size = int(len(sentences) * 0.8)\n", "\n", "training_sentences = sentences[0:training_size]\n", "testing_sentences = sentences[training_size:]\n", "training_labels = labels[0:training_size]\n", "testing_labels = labels[training_size:]\n", "\n", "# Make labels into numpy arrays for use with the network later\n", "training_labels_final = np.array(training_labels)\n", "testing_labels_final = np.array(testing_labels)" ] }, { "cell_type": "markdown", "metadata": { "id": "drDkTFMuzW6N" }, "source": [ "## Tokenize the dataset (with tweaks!)\n", "\n", "Now, we'll tokenize the dataset, but we can make some changes to this from before. Previously, we used: \n", "```\n", "vocab_size = 1000\n", "embedding_dim = 16\n", "max_length = 100\n", "trunc_type='post'\n", "padding_type='post'\n", "```\n", "\n", "How might changing the `vocab_size`, `embedding_dim` or `max_length` affect how the model performs?" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "hjPUJFhQzuee" }, "outputs": [], "source": [ "vocab_size = 500\n", "embedding_dim = 16\n", "max_length = 50\n", "trunc_type='post'\n", "padding_type='post'\n", "oov_tok = \"\"\n", "\n", "tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)\n", "tokenizer.fit_on_texts(training_sentences)\n", "word_index = tokenizer.word_index\n", "training_sequences = tokenizer.texts_to_sequences(training_sentences)\n", "training_padded = pad_sequences(training_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)\n", "\n", "testing_sequences = tokenizer.texts_to_sequences(testing_sentences)\n", "testing_padded = pad_sequences(testing_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)" ] }, { "cell_type": "markdown", "metadata": { "id": "FwFjO1kg0UUK" }, "source": [ "## Train a Sentiment Model (with tweaks!)\n", "\n", "We'll use a slightly different model here, using `GlobalAveragePooling1D` instead of `Flatten()`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ectP92fl0dFO" }, "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.GlobalAveragePooling1D(),\n", " tf.keras.layers.Dense(6, activation='relu'),\n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", "model.summary()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "7TQIaGjs073w" }, "outputs": [], "source": [ "num_epochs = 30\n", "history = model.fit(training_padded, training_labels_final, epochs=num_epochs, validation_data=(testing_padded, testing_labels_final))" ] }, { "cell_type": "markdown", "metadata": { "id": "alAlYort7gWV" }, "source": [ "## Visualize the training graph\n", "\n", "You can use the code below to visualize the training and validation accuracy while you try out different tweaks to the hyperparameters and model." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "o9l5vBeU71vH" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "\n", "def plot_graphs(history, string):\n", " plt.plot(history.history[string])\n", " plt.plot(history.history['val_'+string])\n", " plt.xlabel(\"Epochs\")\n", " plt.ylabel(string)\n", " plt.legend([string, 'val_'+string])\n", " plt.show()\n", " \n", "plot_graphs(history, \"accuracy\")\n", "plot_graphs(history, \"loss\")" ] }, { "cell_type": "markdown", "metadata": { "id": "SZzXE-pT8K57" }, "source": [ "## Get files for visualizing the network\n", "\n", "The code below will download two files for visualizing how your network \"sees\" the sentiment related to each word. Head to http://projector.tensorflow.org/ and load these files, then click the checkbox to \"sphereize\" the data.\n", "\n", "Note: You may run into errors with the projection if your `vocab_size` earlier was larger than the actual number of words in the vocabulary, in which case you'll need to decrease this variable and re-train in order to visualize." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2Ex4o7Lc8Njl" }, "outputs": [], "source": [ "# First get the weights of the embedding layer\n", "e = model.layers[0]\n", "weights = e.get_weights()[0]\n", "print(weights.shape) # shape: (vocab_size, embedding_dim)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "bUL1zk5p8WIV" }, "outputs": [], "source": [ "import io\n", "\n", "# Create the reverse word index\n", "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", "\n", "# Write out the embedding vectors and metadata\n", "out_v = io.open('vecs.tsv', 'w', encoding='utf-8')\n", "out_m = io.open('meta.tsv', 'w', encoding='utf-8')\n", "for word_num in range(1, vocab_size):\n", " word = reverse_word_index[word_num]\n", " embeddings = weights[word_num]\n", " out_m.write(word + \"\\n\")\n", " out_v.write('\\t'.join([str(x) for x in embeddings]) + \"\\n\")\n", "out_v.close()\n", "out_m.close()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lqyV8QYnD46U" }, "outputs": [], "source": [ "# Download the files\n", "try:\n", " from google.colab import files\n", "except ImportError:\n", " pass\n", "else:\n", " files.download('vecs.tsv')\n", " files.download('meta.tsv')" ] }, { "cell_type": "markdown", "metadata": { "id": "XUXAlNNk59gG" }, "source": [ "## Predicting Sentiment in New Reviews\n", "\n", "Below, we've again included some example new reviews you can test your results on." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JbFTTcaK6Dan" }, "outputs": [], "source": [ "# Use the model to predict a review \n", "fake_reviews = ['I love this phone', 'I hate spaghetti', \n", " 'Everything was cold',\n", " 'Everything was hot exactly as I wanted', \n", " 'Everything was green', \n", " 'the host seated us immediately',\n", " 'they gave us free chocolate cake', \n", " 'not sure about the wilted flowers on the table',\n", " 'only works when I stand on tippy toes', \n", " 'does not work when I stand on my head']\n", "\n", "print(fake_reviews) \n", "\n", "# Create the sequences\n", "padding_type='post'\n", "sample_sequences = tokenizer.texts_to_sequences(fake_reviews)\n", "fakes_padded = pad_sequences(sample_sequences, padding=padding_type, maxlen=max_length) \n", "\n", "print('\\nHOT OFF THE PRESS! HERE ARE SOME NEWLY MINTED, ABSOLUTELY GENUINE REVIEWS!\\n') \n", "\n", "classes = model.predict(fakes_padded)\n", "\n", "# The closer the class is to 1, the more positive the review is deemed to be\n", "for x in range(len(fake_reviews)):\n", " print(fake_reviews[x])\n", " print(classes[x])\n", " print('\\n')\n", "\n", "# Try adding reviews of your own\n", "# Add some negative words (such as \"not\") to the good reviews and see what happens\n", "# For example:\n", "# they gave us free chocolate cake and did not charge us" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l09c05_nlp_tweaking_the_model.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l09c06_nlp_subwords.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "CH5gnvxl-N3U" }, "source": [ "# What's in a (sub)word?" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "ykxAKKa1Dl0s" }, "source": [ "In this colab, we'll work with subwords, or words made up of the pieces of larger words, and see how that impacts our network and related embeddings." ] }, { "cell_type": "markdown", "metadata": { "id": "QQCr_NAT-g5w" }, "source": [ "## Import TensorFlow and related functions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Q8Wa_ZlX-mPH" }, "outputs": [], "source": [ "import tensorflow as tf\n", "\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences" ] }, { "cell_type": "markdown", "metadata": { "id": "MRHk-4Te-yLJ" }, "source": [ "## Get the original dataset\n", "\n", "We'll once again use the dataset containing Amazon and Yelp reviews. This dataset was originally extracted from [here](https://www.kaggle.com/marklvl/sentiment-labelled-sentences-data-set)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XJAxrOLi-02C" }, "outputs": [], "source": [ "!wget --no-check-certificate \\\n", " https://drive.google.com/uc?id=13ySLC_ue6Umt9RJYSeM2t-V0kCv-4C-P \\\n", " -O /tmp/sentiment.csv" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Dr-EDUKP_HBl" }, "outputs": [], "source": [ "import pandas as pd\n", "\n", "dataset = pd.read_csv('/tmp/sentiment.csv')\n", "\n", "# Just extract out sentences and labels first - we will create subwords here\n", "sentences = dataset['text'].tolist()\n", "labels = dataset['sentiment'].tolist()" ] }, { "cell_type": "markdown", "metadata": { "id": "8zut9Wng_R3B" }, "source": [ "## Create a subwords dataset\n", "\n", "We can use the existing Amazon and Yelp reviews dataset with `tensorflow_datasets`'s `SubwordTextEncoder` functionality. `SubwordTextEncoder.build_from_corpus()` will create a tokenizer for us. You could also use this functionality to get subwords from a much larger corpus of text as well, but we'll just use our existing dataset here.\n", "\n", "The Amazon and Yelp dataset we are using isn't super large, so we'll create a subword `vocab_size` of only the 1,000 most common words, as well as cutting off each subword to be at most 5 characters.\n", "\n", "Check out the related documentation [here](https://www.tensorflow.org/datasets/api_docs/python/tfds/features/text/SubwordTextEncoder#build_from_corpus)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "aElsgxia_43g" }, "outputs": [], "source": [ "import tensorflow_datasets as tfds\n", "\n", "vocab_size = 1000\n", "tokenizer = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(sentences, vocab_size, max_subword_length=5)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "0XNZWGKqBDc3" }, "outputs": [], "source": [ "# Check that the tokenizer works appropriately\n", "num = 5\n", "print(sentences[num])\n", "encoded = tokenizer.encode(sentences[num])\n", "print(encoded)\n", "# Separately print out each subword, decoded\n", "for i in encoded:\n", " print(tokenizer.decode([i]))" ] }, { "cell_type": "markdown", "metadata": { "id": "gYnbqctXGKcC" }, "source": [ "## Replace sentence data with encoded subwords\n", "\n", "Now, we'll re-create the dataset to be used for training by actually encoding each of the individual sentences. This is equivalent to `text_to_sequences` with the `Tokenizer` we used in earlier exercises." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "rAmql34aGfeV" }, "outputs": [], "source": [ "for i, sentence in enumerate(sentences):\n", " sentences[i] = tokenizer.encode(sentence)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jNnee_csG5Iz" }, "outputs": [], "source": [ "# Check the sentences are appropriately replaced\n", "print(sentences[1])" ] }, { "cell_type": "markdown", "metadata": { "id": "zpIigjecHVkF" }, "source": [ "## Final pre-processing\n", "\n", "Before training, we still need to pad the sequences, as well as split into training and test sets." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "INIFSAcEHool" }, "outputs": [], "source": [ "import numpy as np\n", "\n", "max_length = 50\n", "trunc_type='post'\n", "padding_type='post'\n", "\n", "# Pad all sentences\n", "sentences_padded = pad_sequences(sentences, maxlen=max_length, \n", " padding=padding_type, truncating=trunc_type)\n", "\n", "# Separate out the sentences and labels into training and test sets\n", "training_size = int(len(sentences) * 0.8)\n", "\n", "training_sentences = sentences_padded[0:training_size]\n", "testing_sentences = sentences_padded[training_size:]\n", "training_labels = labels[0:training_size]\n", "testing_labels = labels[training_size:]\n", "\n", "# Make labels into numpy arrays for use with the network later\n", "training_labels_final = np.array(training_labels)\n", "testing_labels_final = np.array(testing_labels)" ] }, { "cell_type": "markdown", "metadata": { "id": "QC9A-sTpPPiL" }, "source": [ "## Train a Sentiment Model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "eDKcL64IPcfy" }, "outputs": [], "source": [ "embedding_dim = 16\n", "model = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.GlobalAveragePooling1D(),\n", " tf.keras.layers.Dense(6, activation='relu'),\n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "\n", "model.summary()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "VqkMNtIeP3oz" }, "outputs": [], "source": [ "num_epochs = 30\n", "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", "history = model.fit(training_sentences, training_labels_final, epochs=num_epochs, \n", " validation_data=(testing_sentences, testing_labels_final))" ] }, { "cell_type": "markdown", "metadata": { "id": "sj18M42kQkCi" }, "source": [ "## Visualize the Training Graph\n", "\n", "We can visualize the training graph below again. Does there appear to be a difference in how validation accuracy and loss is trending compared to with full words?" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "uy8KIMPIQlvH" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "\n", "def plot_graphs(history, string):\n", " plt.plot(history.history[string])\n", " plt.plot(history.history['val_'+string])\n", " plt.xlabel(\"Epochs\")\n", " plt.ylabel(string)\n", " plt.legend([string, 'val_'+string])\n", " plt.show()\n", " \n", "plot_graphs(history, \"accuracy\")\n", "plot_graphs(history, \"loss\")" ] }, { "cell_type": "markdown", "metadata": { "id": "_m7QzouQQ1Rs" }, "source": [ "## Get files for visualizing the network\n", "\n", "Once again, you can visualize the sentiment related to all of the subwords using the below code and by heading to http://projector.tensorflow.org/ to upload and view the data.\n", "\n", "Note that the below code does have a few small changes to handle the different way text is encoded in our dataset compared to before with the built in `Tokenizer`.\n", "\n", "You may get an error like \"Number of tensors (999) do not match the number of lines in metadata (992).\" As long as you load the vectors first without error and wait a few seconds after this pops up, you will be able to click outside the file load menu and still view the visualization." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dezs4wE5RMQu" }, "outputs": [], "source": [ "# First get the weights of the embedding layer\n", "e = model.layers[0]\n", "weights = e.get_weights()[0]\n", "print(weights.shape) # shape: (vocab_size, embedding_dim)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "LXKqy9Z1RSmt" }, "outputs": [], "source": [ "import io\n", "\n", "# Write out the embedding vectors and metadata\n", "out_v = io.open('vecs.tsv', 'w', encoding='utf-8')\n", "out_m = io.open('meta.tsv', 'w', encoding='utf-8')\n", "for word_num in range(0, vocab_size - 1):\n", " word = tokenizer.decode([word_num])\n", " embeddings = weights[word_num]\n", " out_m.write(word + \"\\n\")\n", " out_v.write('\\t'.join([str(x) for x in embeddings]) + \"\\n\")\n", "out_v.close()\n", "out_m.close()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "v04wBMybRoGx" }, "outputs": [], "source": [ "# Download the files\n", "try:\n", " from google.colab import files\n", "except ImportError:\n", " pass\n", "else:\n", " files.download('vecs.tsv')\n", " files.download('meta.tsv')" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l09c06_nlp_subwords.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l10c01_nlp_lstms_with_reviews_subwords_dataset.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "hAclqSm3OOml" }, "source": [ "# Using LSTMs with the subwords dataset\n" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "KTVx8__oGR9J" }, "source": [ "In this colab, you'll compare the results of using a model with an Embedding layer and then adding bidirectional LSTM layers.\n", "\n", "You'll work with the dataset of subwords for the combined Yelp and Amazon reviews.\n", "\n", "You'll use your models to predict the sentiment of new reviews." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "L62G7LTwNzoD" }, "outputs": [], "source": [ "import tensorflow as tf\n", "\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences" ] }, { "cell_type": "markdown", "metadata": { "id": "hLcl0QHvDjTV" }, "source": [ "# Get the dataset\n", "\n", "Start by getting the dataset containing Amazon and Yelp reviews, with their related sentiment (1 for positive, 0 for negative). This dataset was originally extracted from [here](https://www.kaggle.com/marklvl/sentiment-labelled-sentences-data-set).\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nCOtiRJZbxCH" }, "outputs": [], "source": [ "!wget --no-check-certificate \\\n", " https://drive.google.com/uc?id=13ySLC_ue6Umt9RJYSeM2t-V0kCv-4C-P -O /tmp/sentiment.csv" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XuqER_KMD-xS" }, "outputs": [], "source": [ "import pandas as pd\n", "\n", "dataset = pd.read_csv('/tmp/sentiment.csv')\n", "\n", "# Extract out sentences and labels\n", "sentences = dataset['text'].tolist()\n", "labels = dataset['sentiment'].tolist()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Tbsx1T2CXPNO" }, "outputs": [], "source": [ "# Print some example sentences and labels\n", "for x in range(2):\n", " print(sentences[x])\n", " print(labels[x])\n", " print(\"\\n\")" ] }, { "cell_type": "markdown", "metadata": { "id": "33AthPiALFZK" }, "source": [ "#Create a subwords dataset\n", "\n", "We will use the Amazon and Yelp reviews dataset with tensorflow_datasets's SubwordTextEncoder functionality. \n", "\n", "SubwordTextEncoder.build_from_corpus() will create a tokenizer for us. You could also use this functionality to get subwords from a much larger corpus of text as well, but we'll just use our existing dataset here.\n", "\n", "We'll create a subword vocab_size of only the 1,000 most common subwords, as well as cutting off each subword to be at most 5 characters.\n", "\n", "Check out the related documentation for the the subword text encoder [here](https://www.tensorflow.org/datasets/api_docs/python/tfds/features/text/SubwordTextEncoder#build_from_corpus)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6NaicNCcLYyf" }, "outputs": [], "source": [ "import tensorflow_datasets as tfds\n", "\n", "vocab_size = 1000\n", "tokenizer = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(sentences, vocab_size, max_subword_length=5)\n", "\n", "# How big is the vocab size?\n", "print(\"Vocab size is \", tokenizer.vocab_size)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "xvRVoeIVLevh" }, "outputs": [], "source": [ "# Check that the tokenizer works appropriately\n", "num = 5\n", "print(sentences[num])\n", "encoded = tokenizer.encode(sentences[num])\n", "print(encoded)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "G_vacTCifklV" }, "outputs": [], "source": [ "# Separately print out each subword, decoded\n", "for i in encoded:\n", " print(tokenizer.decode([i]))" ] }, { "cell_type": "markdown", "metadata": { "id": "cT528cptLupl" }, "source": [ "## Replace sentence data with encoded subwords\n", "\n", "Now, we'll create the sequences to be used for training by actually encoding each of the individual sentences. This is equivalent to `text_to_sequences` with the `Tokenizer` we used in earlier exercises." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lkseMhxjL09F" }, "outputs": [], "source": [ "for i, sentence in enumerate(sentences):\n", " sentences[i] = tokenizer.encode(sentence)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "y21yRuzmL43U" }, "outputs": [], "source": [ "# Check the sentences are appropriately replaced\n", "print(sentences[5])" ] }, { "cell_type": "markdown", "metadata": { "id": "8HrcPHESMBMs" }, "source": [ "## Final pre-processing\n", "\n", "Before training, we still need to pad the sequences, as well as split into training and test sets." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "50-hTsogLSL-" }, "outputs": [], "source": [ "import numpy as np\n", "\n", "max_length = 50\n", "trunc_type='post'\n", "padding_type='post'\n", "\n", "# Pad all sequences\n", "sequences_padded = pad_sequences(sentences, maxlen=max_length, \n", " padding=padding_type, truncating=trunc_type)\n", "\n", "# Separate out the sentences and labels into training and test sets\n", "training_size = int(len(sentences) * 0.8)\n", "\n", "training_sequences = sequences_padded[0:training_size]\n", "testing_sequences = sequences_padded[training_size:]\n", "training_labels = labels[0:training_size]\n", "testing_labels = labels[training_size:]\n", "\n", "# Make labels into numpy arrays for use with the network later\n", "training_labels_final = np.array(training_labels)\n", "testing_labels_final = np.array(testing_labels)" ] }, { "cell_type": "markdown", "metadata": { "id": "PahZm7YEQ8EI" }, "source": [ "# Create the model using an Embedding" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "c_nyQeI0RCCv" }, "outputs": [], "source": [ "embedding_dim = 16\n", "\n", "model = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.GlobalAveragePooling1D(), \n", " tf.keras.layers.Dense(6, activation='relu'),\n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "\n", "model.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "3WRXrx8BRO2L" }, "source": [ "# Train the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oBKyVYvxRQ_9" }, "outputs": [], "source": [ "num_epochs = 30\n", "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", "history = model.fit(training_sequences, training_labels_final, epochs=num_epochs, validation_data=(testing_sequences, testing_labels_final))\n" ] }, { "cell_type": "markdown", "metadata": { "id": "HhLPbUl2AZ0y" }, "source": [ "# Plot the accuracy and loss" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jzBM1PpJAYfD" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "\n", "def plot_graphs(history, string):\n", " plt.plot(history.history[string])\n", " plt.plot(history.history['val_'+string])\n", " plt.xlabel(\"Epochs\")\n", " plt.ylabel(string)\n", " plt.legend([string, 'val_'+string])\n", " plt.show()\n", " \n", "plot_graphs(history, \"accuracy\")\n", "plot_graphs(history, \"loss\")" ] }, { "cell_type": "markdown", "metadata": { "id": "Fwr5inBiWffb" }, "source": [ "# Define a function to predict the sentiment of reviews\n", "\n", "We'll be creating models with some differences and will use each model to predict the sentiment of some new reviews.\n", "\n", "To save time, create a function that will take in a model and some new reviews, and print out the sentiment of each reviews.\n", "\n", "The higher the sentiment value is to 1, the more positive the review is." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "aPNOYiiaha2y" }, "outputs": [], "source": [ "# Define a function to take a series of reviews\n", "# and predict whether each one is a positive or negative review\n", "\n", "# max_length = 100 # previously defined\n", "\n", "def predict_review(model, new_sentences, maxlen=max_length, show_padded_sequence=True ):\n", " # Keep the original sentences so that we can keep using them later\n", " # Create an array to hold the encoded sequences\n", " new_sequences = []\n", "\n", " # Convert the new reviews to sequences\n", " for i, frvw in enumerate(new_sentences):\n", " new_sequences.append(tokenizer.encode(frvw))\n", "\n", " trunc_type='post' \n", " padding_type='post'\n", "\n", " # Pad all sequences for the new reviews\n", " new_reviews_padded = pad_sequences(new_sequences, maxlen=max_length, \n", " padding=padding_type, truncating=trunc_type) \n", "\n", " classes = model.predict(new_reviews_padded)\n", "\n", " # The closer the class is to 1, the more positive the review is\n", " for x in range(len(new_sentences)):\n", " \n", " # We can see the padded sequence if desired\n", " # Print the sequence\n", " if (show_padded_sequence):\n", " print(new_reviews_padded[x])\n", " # Print the review as text\n", " print(new_sentences[x])\n", " # Print its predicted class\n", " print(classes[x])\n", " print(\"\\n\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Qg-maex27KPW" }, "outputs": [], "source": [ "# Use the model to predict some reviews \n", "fake_reviews = [\"I love this phone\", \n", " \"Everything was cold\",\n", " \"Everything was hot exactly as I wanted\", \n", " \"Everything was green\", \n", " \"the host seated us immediately\",\n", " \"they gave us free chocolate cake\", \n", " \"we couldn't hear each other talk because of the shouting in the kitchen\"\n", " ]\n", "\n", "predict_review(model, fake_reviews)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "ycJKbMq3K4iy" }, "source": [ "# Define a function to train and show the results of models with different layers\n", "\n", "In the rest of this colab, we will define models, and then see the results. \n", "\n", "Define a function that will take the model, compile it, train it, graph the accuracy and loss, and then predict some results." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "PevUcINXK3gn" }, "outputs": [], "source": [ "def fit_model_now (model, sentences) :\n", " model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", " model.summary()\n", " history = model.fit(training_sequences, training_labels_final, epochs=num_epochs, \n", " validation_data=(testing_sequences, testing_labels_final))\n", " return history\n", "\n", "def plot_results (history):\n", " plot_graphs(history, \"accuracy\")\n", " plot_graphs(history, \"loss\")\n", "\n", "def fit_model_and_show_results (model, sentences):\n", " history = fit_model_now(model, sentences)\n", " plot_results(history)\n", " predict_review(model, sentences)" ] }, { "cell_type": "markdown", "metadata": { "id": "U13JBiJUG1oq" }, "source": [ "# Add a bidirectional LSTM\n", "\n", "Create a new model that uses a bidirectional LSTM.\n", "\n", "Then use the function we have already defined to compile the model, train it, graph the accuracy and loss, then predict some results." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "scTUsFPAG4zP" }, "outputs": [], "source": [ "# Define the model\n", "model_bidi_lstm = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(embedding_dim)), \n", " tf.keras.layers.Dense(6, activation='relu'), \n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "\n", "# Compile and train the model and then show the predictions for our extra sentences\n", "fit_model_and_show_results(model_bidi_lstm, fake_reviews)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "QsxKPbCnPJTj" }, "source": [ "# Use multiple bidirectional layers\n", "\n", "Now let's see if we get any improvements from adding another Bidirectional LSTM layer to the model.\n", "\n", "Notice that the first Bidirectionl LSTM layer returns a sequence." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3N6Zul47PMED" }, "outputs": [], "source": [ "model_multiple_bidi_lstm = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(embedding_dim, \n", " return_sequences=True)), \n", " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(embedding_dim)),\n", " tf.keras.layers.Dense(6, activation='relu'),\n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "\n", "fit_model_and_show_results(model_multiple_bidi_lstm, fake_reviews)" ] }, { "cell_type": "markdown", "metadata": { "id": "ABVYYPwba8Hx" }, "source": [ "# Compare predictions for all the models\n", "\n", "It can be hard to see which model gives a better prediction for different reviews when you examine each model separately. So for comparison purposes, here we define some more reviews and print out the predictions that each of the three models gives for each review:\n", "\n", "* Embeddings and a Global Average Pooling layer\n", "* Embeddings and a Bidirectional LSTM layer\n", "* Embeddings and two Bidirectional LSTM layers\n", "\n", "The results are not always what you might expect. The input dataset is fairly small, it has less than 2000 reviews. Some of the reviews are fairly short, and some of the short ones are fairly repetitive which reduces their impact on improving the model, such as these two reviews:\n", "\n", "* Bad Quality.\n", "* Low Quality.\n", "\n", "Feel free to add more reviews of your own, or change the reviews. The results will depend on the combination of words in the reviews, and how well they match to reviews in the training set. \n", "\n", "How do the different models handle things like \"wasn't good\" which contains a positive word (good) but is a poor review?\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6XebrXt0jtOy" }, "outputs": [], "source": [ "my_reviews =[\"lovely\", \"dreadful\", \"stay away\",\n", " \"everything was hot exactly as I wanted\",\n", " \"everything was not exactly as I wanted\",\n", " \"they gave us free chocolate cake\",\n", " \"I've never eaten anything so spicy in my life, my throat burned for hours\",\n", " \"for a phone that is as expensive as this one I expect it to be much easier to use than this thing is\",\n", " \"we left there very full for a low price so I'd say you just can't go wrong at this place\",\n", " \"that place does not have quality meals and it isn't a good place to go for dinner\",\n", " ]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tRWGjkJLkY2y" }, "outputs": [], "source": [ "print(\"===================================\\n\",\"Embeddings only:\\n\", \"===================================\",)\n", "predict_review(model, my_reviews, show_padded_sequence=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "G2FJR3IVBt30" }, "outputs": [], "source": [ "print(\"===================================\\n\", \"With a single bidirectional LSTM:\\n\", \"===================================\")\n", "predict_review(model_bidi_lstm, my_reviews, show_padded_sequence=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "81v1r3Y2BwvC" }, "outputs": [], "source": [ "print(\"===================================\\n\",\"With two bidirectional LSTMs:\\n\", \"===================================\")\n", "predict_review(model_multiple_bidi_lstm, my_reviews, show_padded_sequence=False)" ] } ], "metadata": { "accelerator": "GPU", "colab": { "name": "l10c01_nlp_lstms_with_reviews_subwords_dataset.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l10c02_nlp_multiple_models_for_predicting_sentiment.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "hAclqSm3OOml" }, "source": [ "# Using LSTMs, CNNs, GRUs with a larger dataset" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "fentd-GnIj-j" }, "source": [ "In this colab, you use different kinds of layers to see how they affect the model.\n", "\n", "You will use the glue/sst2 dataset, which is available through tensorflow_datasets. \n", "\n", "The General Language Understanding Evaluation (GLUE) benchmark (https://gluebenchmark.com/) is a collection of resources for training, evaluating, and analyzing natural language understanding systems.\n", "\n", "These resources include the Stanford Sentiment Treebank (SST) dataset that consists of sentences from movie reviews and human annotations of their sentiment. This colab uses version 2 of the SST dataset.\n", "\n", "The splits are:\n", "\n", "* train\t67,349\n", "* validation\t872\n", "\n", "\n", "and the column headings are:\n", "\n", "* sentence\n", "* label\n", "\n", "\n", "For more information about the dataset, see [https://www.tensorflow.org/datasets/catalog/glue#gluesst2](https://www.tensorflow.org/datasets/catalog/glue#gluesst2)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "L62G7LTwNzoD" }, "outputs": [], "source": [ "import tensorflow as tf\n", "import tensorflow_datasets as tfds\n", "\n", "import numpy as np" ] }, { "cell_type": "markdown", "metadata": { "id": "o2P6xdtIJMyc" }, "source": [ "# Get the dataset\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nCOtiRJZbxCH" }, "outputs": [], "source": [ "# Get the dataset.\n", "# It has 70000 items, so might take a while to download\n", "dataset, info = tfds.load('glue/sst2', with_info=True)\n", "print(info.features)\n", "print(info.features[\"label\"].num_classes)\n", "print(info.features[\"label\"].names)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "yBMPhIdStAe2" }, "outputs": [], "source": [ "# Get the training and validation datasets\n", "dataset_train, dataset_validation = dataset['train'], dataset['validation']\n", "dataset_train" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jbZ-faiNWu1U" }, "outputs": [], "source": [ "# Print some of the entries\n", "for example in dataset_train.take(2): \n", " review, label = example[\"sentence\"], example[\"label\"]\n", " print(\"Review:\", review)\n", " print(\"Label: %d \\n\" % label.numpy())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "_fVZItTeZSbL" }, "outputs": [], "source": [ "# Get the sentences and the labels\n", "# for both the training and the validation sets\n", "training_reviews = []\n", "training_labels = []\n", " \n", "validation_reviews = []\n", "validation_labels = []\n", "\n", "# The dataset has 67,000 training entries, but that's a lot to process here!\n", "\n", "# If you want to take the entire dataset: WARNING: takes longer!!\n", "# for item in dataset_train.take(-1):\n", "\n", "# Take 10,000 reviews\n", "for item in dataset_train.take(10000):\n", " review, label = item[\"sentence\"], item[\"label\"]\n", " training_reviews.append(str(review.numpy()))\n", " training_labels.append(label.numpy())\n", "\n", "print (\"\\nNumber of training reviews is: \", len(training_reviews))\n", "\n", "# print some of the reviews and labels\n", "for i in range(0, 2):\n", " print (training_reviews[i])\n", " print (training_labels[i])\n", "\n", "# Get the validation data\n", "# there's only about 800 items, so take them all\n", "for item in dataset_validation.take(-1): \n", " review, label = item[\"sentence\"], item[\"label\"]\n", " validation_reviews.append(str(review.numpy()))\n", " validation_labels.append(label.numpy())\n", "\n", "print (\"\\nNumber of validation reviews is: \", len(validation_reviews))\n", "\n", "# Print some of the validation reviews and labels\n", "for i in range(0, 2):\n", " print (validation_reviews[i])\n", " print (validation_labels[i])\n" ] }, { "cell_type": "markdown", "metadata": { "id": "BY4ZoptJO55o" }, "source": [ "# Tokenize the words and sequence the sentences\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "0TWLvXA1Oa_W" }, "outputs": [], "source": [ "# There's a total of 21224 words in the reviews\n", "# but many of them are irrelevant like with, it, of, on.\n", "# If we take a subset of the training data, then the vocab\n", "# will be smaller.\n", "\n", "# A reasonable review might have about 50 words or so,\n", "# so we can set max_length to 50 (but feel free to change it as you like)\n", "\n", "vocab_size = 4000\n", "embedding_dim = 16\n", "max_length = 50\n", "trunc_type='post'\n", "pad_type='post'\n", "oov_tok = \"\"\n", "\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", "\n", "tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)\n", "tokenizer.fit_on_texts(training_reviews)\n", "word_index = tokenizer.word_index\n" ] }, { "cell_type": "markdown", "metadata": { "id": "JV-Ff5N0ryWv" }, "source": [ "# Pad the sequences" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "B-3scEznH2Va" }, "outputs": [], "source": [ "# Pad the sequences so that they are all the same length\n", "training_sequences = tokenizer.texts_to_sequences(training_reviews)\n", "training_padded = pad_sequences(training_sequences,maxlen=max_length, \n", " truncating=trunc_type, padding=pad_type)\n", "\n", "validation_sequences = tokenizer.texts_to_sequences(validation_reviews)\n", "validation_padded = pad_sequences(validation_sequences,maxlen=max_length)\n", "\n", "training_labels_final = np.array(training_labels)\n", "validation_labels_final = np.array(validation_labels)" ] }, { "cell_type": "markdown", "metadata": { "id": "PahZm7YEQ8EI" }, "source": [ "# Create the model using an Embedding" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "c_nyQeI0RCCv" }, "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.GlobalAveragePooling1D(), \n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n", "model.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "3WRXrx8BRO2L" }, "source": [ "# Train the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oBKyVYvxRQ_9" }, "outputs": [], "source": [ "num_epochs = 20\n", "history = model.fit(training_padded, training_labels_final, epochs=num_epochs, \n", " validation_data=(validation_padded, validation_labels_final))\n" ] }, { "cell_type": "markdown", "metadata": { "id": "HhLPbUl2AZ0y" }, "source": [ "# Plot the accurracy and loss" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jzBM1PpJAYfD" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "\n", "def plot_graphs(history, string):\n", " plt.plot(history.history[string])\n", " plt.plot(history.history['val_'+string])\n", " plt.xlabel(\"Epochs\")\n", " plt.ylabel(string)\n", " plt.legend([string, 'val_'+string])\n", " plt.show()\n", " \n", "plot_graphs(history, \"accuracy\")\n", "plot_graphs(history, \"loss\")" ] }, { "cell_type": "markdown", "metadata": { "id": "HEbcMCVEKToB" }, "source": [ "# Write a function to predict the sentiment of reviews" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "K0nKY9M4xzWE" }, "outputs": [], "source": [ "# Write some new reviews \n", "\n", "review1 = \"\"\"I loved this movie\"\"\"\n", "\n", "review2 = \"\"\"that was the worst movie I've ever seen\"\"\"\n", "\n", "review3 = \"\"\"too much violence even for a Bond film\"\"\"\n", "\n", "review4 = \"\"\"a captivating recounting of a cherished myth\"\"\"\n", "\n", "new_reviews = [review1, review2, review3, review4]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Qg-maex27KPW" }, "outputs": [], "source": [ "# Define a function to prepare the new reviews for use with a model\n", "# and then use the model to predict the sentiment of the new reviews \n", "\n", "def predict_review(model, reviews):\n", " # Create the sequences\n", " padding_type='post'\n", " sample_sequences = tokenizer.texts_to_sequences(reviews)\n", " reviews_padded = pad_sequences(sample_sequences, padding=padding_type, \n", " maxlen=max_length) \n", " classes = model.predict(reviews_padded)\n", " for x in range(len(reviews_padded)):\n", " print(reviews[x])\n", " print(classes[x])\n", " print('\\n')\n", "\n", "predict_review(model, new_reviews)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "ycJKbMq3K4iy" }, "source": [ "# Define a function to train and show the results of models with different layers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "PevUcINXK3gn" }, "outputs": [], "source": [ "def fit_model_and_show_results (model, reviews):\n", " model.summary()\n", " history = model.fit(training_padded, training_labels_final, epochs=num_epochs, \n", " validation_data=(validation_padded, validation_labels_final))\n", " plot_graphs(history, \"accuracy\")\n", " plot_graphs(history, \"loss\")\n", " predict_review(model, reviews)" ] }, { "cell_type": "markdown", "metadata": { "id": "W8jW-OLfTrDM" }, "source": [ "# Use a CNN" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "merAu9T3TtmQ" }, "outputs": [], "source": [ "num_epochs = 30\n", "\n", "model_cnn = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.Conv1D(16, 5, activation='relu'),\n", " tf.keras.layers.GlobalMaxPooling1D(),\n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "\n", "# Default learning rate for the Adam optimizer is 0.001\n", "# Let's slow down the learning rate by 10.\n", "learning_rate = 0.0001\n", "model_cnn.compile(loss='binary_crossentropy',\n", " optimizer=tf.keras.optimizers.Adam(learning_rate), \n", " metrics=['accuracy'])\n", "\n", "fit_model_and_show_results(model_cnn, new_reviews)" ] }, { "cell_type": "markdown", "metadata": { "id": "tXnoq0zITmSM" }, "source": [ "# Use a GRU" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "6jP6KAzZTpQ6" }, "outputs": [], "source": [ "num_epochs = 30\n", "\n", "model_gru = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.Bidirectional(tf.keras.layers.GRU(32)),\n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "\n", "learning_rate = 0.00003 # slower than the default learning rate\n", "model_gru.compile(loss='binary_crossentropy',\n", " optimizer=tf.keras.optimizers.Adam(learning_rate),\n", " metrics=['accuracy'])\n", "\n", "fit_model_and_show_results(model_gru, new_reviews)" ] }, { "cell_type": "markdown", "metadata": { "id": "U13JBiJUG1oq" }, "source": [ "# Add a bidirectional LSTM" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "scTUsFPAG4zP" }, "outputs": [], "source": [ "num_epochs = 30\n", "\n", "model_bidi_lstm = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(embedding_dim)), \n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "\n", "learning_rate = 0.00003\n", "model_bidi_lstm.compile(loss='binary_crossentropy',\n", " optimizer=tf.keras.optimizers.Adam(learning_rate),\n", " metrics=['accuracy'])\n", "fit_model_and_show_results(model_bidi_lstm, new_reviews)" ] }, { "cell_type": "markdown", "metadata": { "id": "QsxKPbCnPJTj" }, "source": [ "# Use multiple bidirectional LSTMs" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "3N6Zul47PMED" }, "outputs": [], "source": [ "num_epochs = 30\n", "\n", "model_multiple_bidi_lstm = tf.keras.Sequential([\n", " tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n", " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(embedding_dim, \n", " return_sequences=True)),\n", " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(embedding_dim)),\n", " tf.keras.layers.Dense(1, activation='sigmoid')\n", "])\n", "\n", "learning_rate = 0.0003\n", "model_multiple_bidi_lstm.compile(loss='binary_crossentropy',\n", " optimizer=tf.keras.optimizers.Adam(learning_rate),\n", " metrics=['accuracy'])\n", "fit_model_and_show_results(model_multiple_bidi_lstm, new_reviews)" ] }, { "cell_type": "markdown", "metadata": { "id": "gdN2tdW4YYJ1" }, "source": [ "# Try some more reviews" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "e45UQxQl_QAI" }, "outputs": [], "source": [ "# Write some new reviews \n", "\n", "review1 = \"\"\"I loved this movie\"\"\"\n", "\n", "review2 = \"\"\"that was the worst movie I've ever seen\"\"\"\n", "\n", "review3 = \"\"\"too much violence even for a Bond film\"\"\"\n", "\n", "review4 = \"\"\"a captivating recounting of a cherished myth\"\"\"\n", "\n", "review5 = \"\"\"I saw this movie yesterday and I was feeling low to start with,\n", " but it was such a wonderful movie that it lifted my spirits and brightened \n", " my day, you can\\'t go wrong with a movie with Whoopi Goldberg in it.\"\"\"\n", "\n", "review6 = \"\"\"I don\\'t understand why it received an oscar recommendation\n", " for best movie, it was long and boring\"\"\"\n", "\n", "review7 = \"\"\"the scenery was magnificent, the CGI of the dogs was so realistic I\n", " thought they were played by real dogs even though they talked!\"\"\"\n", "\n", "review8 = \"\"\"The ending was so sad and yet so uplifting at the same time. \n", " I'm looking for an excuse to see it again\"\"\"\n", "\n", "review9 = \"\"\"I had expected so much more from a movie made by the director \n", " who made my most favorite movie ever, I was very disappointed in the tedious \n", " story\"\"\"\n", "\n", "review10 = \"I wish I could watch this movie every day for the rest of my life\"\n", "\n", "more_reviews = [review1, review2, review3, review4, review5, review6, review7, \n", " review8, review9, review10]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "cQ4YZHOjYXQ2" }, "outputs": [], "source": [ "print(\"============================\\n\",\"Embeddings only:\\n\", \"============================\")\n", "predict_review(model, more_reviews)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "NI04noTAHK5t" }, "outputs": [], "source": [ "print(\"============================\\n\",\"With CNN\\n\", \"============================\")\n", "predict_review(model_cnn, more_reviews)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "vGJ32sRUHNu6" }, "outputs": [], "source": [ "print(\"===========================\\n\",\"With bidirectional GRU\\n\", \"============================\")\n", "predict_review(model_gru, more_reviews)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "IFw9Q0iQHP2P" }, "outputs": [], "source": [ "print(\"===========================\\n\", \"With a single bidirectional LSTM:\\n\", \"===========================\")\n", "predict_review(model_bidi_lstm, more_reviews)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zaw7fEVQHSWK" }, "outputs": [], "source": [ "print(\"===========================\\n\", \"With multiple bidirectional LSTM:\\n\", \"==========================\")\n", "predict_review(model_multiple_bidi_lstm, more_reviews)" ] } ], "metadata": { "accelerator": "GPU", "colab": { "name": "l10c02_nlp_multiple_models_for_predicting_sentiment.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l10c03_nlp_constructing_text_generation_model.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "Ph5eir3Pf-3z" }, "source": [ "# Constructing a Text Generation Model\n" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "7GbGfr_oLCat" }, "source": [ "Using most of the techniques you've already learned, it's now possible to generate new text by predicting the next word that follows a given seed word. To practice this method, we'll use the [Kaggle Song Lyrics Dataset](https://www.kaggle.com/mousehead/songlyrics)." ] }, { "cell_type": "markdown", "metadata": { "id": "4aHK2CYygXom" }, "source": [ "## Import TensorFlow and related functions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2LmLTREBf5ng" }, "outputs": [], "source": [ "import tensorflow as tf\n", "\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", "\n", "# Other imports for processing data\n", "import string\n", "import numpy as np\n", "import pandas as pd" ] }, { "cell_type": "markdown", "metadata": { "id": "GmLTO_dpgge9" }, "source": [ "## Get the Dataset\n", "\n", "As noted above, we'll utilize the [Song Lyrics dataset](https://www.kaggle.com/mousehead/songlyrics) on Kaggle." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4Bf5FVHfganK" }, "outputs": [], "source": [ "!wget --no-check-certificate \\\n", " https://drive.google.com/uc?id=1LiJFZd41ofrWoBtW-pMYsfz1w8Ny0Bj8 \\\n", " -O /tmp/songdata.csv" ] }, { "cell_type": "markdown", "metadata": { "id": "gu1BTzMIS1oy" }, "source": [ "## **First 10 Songs**\n", "\n", "Let's first look at just 10 songs from the dataset, and see how things perform." ] }, { "cell_type": "markdown", "metadata": { "id": "fmb9rGaAUDO-" }, "source": [ "### Preprocessing\n", "\n", "Let's perform some basic preprocessing to get rid of punctuation and make everything lowercase. We'll then split the lyrics up by line and tokenize the lyrics." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2AVAvyF_Vuh5" }, "outputs": [], "source": [ "def tokenize_corpus(corpus, num_words=-1):\n", " # Fit a Tokenizer on the corpus\n", " if num_words > -1:\n", " tokenizer = Tokenizer(num_words=num_words)\n", " else:\n", " tokenizer = Tokenizer()\n", " tokenizer.fit_on_texts(corpus)\n", " return tokenizer\n", "\n", "def create_lyrics_corpus(dataset, field):\n", " # Remove all other punctuation\n", " dataset[field] = dataset[field].str.replace('[{}]'.format(string.punctuation), '')\n", " # Make it lowercase\n", " dataset[field] = dataset[field].str.lower()\n", " # Make it one long string to split by line\n", " lyrics = dataset[field].str.cat()\n", " corpus = lyrics.split('\\n')\n", " # Remove any trailing whitespace\n", " for l in range(len(corpus)):\n", " corpus[l] = corpus[l].rstrip()\n", " # Remove any empty lines\n", " corpus = [l for l in corpus if l != '']\n", "\n", " return corpus" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "apcEXp7WhVBs" }, "outputs": [], "source": [ "# Read the dataset from csv - just first 10 songs for now\n", "dataset = pd.read_csv('/tmp/songdata.csv', dtype=str)[:10]\n", "# Create the corpus using the 'text' column containing lyrics\n", "corpus = create_lyrics_corpus(dataset, 'text')\n", "# Tokenize the corpus\n", "tokenizer = tokenize_corpus(corpus)\n", "\n", "total_words = len(tokenizer.word_index) + 1\n", "\n", "print(tokenizer.word_index)\n", "print(total_words)" ] }, { "cell_type": "markdown", "metadata": { "id": "v9x68iN_X6FK" }, "source": [ "### Create Sequences and Labels\n", "\n", "After preprocessing, we next need to create sequences and labels. Creating the sequences themselves is similar to before with `texts_to_sequences`, but also including the use of [N-Grams](https://towardsdatascience.com/introduction-to-language-models-n-gram-e323081503d9); creating the labels will now utilize those sequences as well as utilize one-hot encoding over all potential output words." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "QmlTsUqfikVO" }, "outputs": [], "source": [ "sequences = []\n", "for line in corpus:\n", "\ttoken_list = tokenizer.texts_to_sequences([line])[0]\n", "\tfor i in range(1, len(token_list)):\n", "\t\tn_gram_sequence = token_list[:i+1]\n", "\t\tsequences.append(n_gram_sequence)\n", "\n", "# Pad sequences for equal input length \n", "max_sequence_len = max([len(seq) for seq in sequences])\n", "sequences = np.array(pad_sequences(sequences, maxlen=max_sequence_len, padding='pre'))\n", "\n", "# Split sequences between the \"input\" sequence and \"output\" predicted word\n", "input_sequences, labels = sequences[:,:-1], sequences[:,-1]\n", "# One-hot encode the labels\n", "one_hot_labels = tf.keras.utils.to_categorical(labels, num_classes=total_words)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Zsmu3aEId49i" }, "outputs": [], "source": [ "# Check out how some of our data is being stored\n", "# The Tokenizer has just a single index per word\n", "print(tokenizer.word_index['know'])\n", "print(tokenizer.word_index['feeling'])\n", "# Input sequences will have multiple indexes\n", "print(input_sequences[5])\n", "print(input_sequences[6])\n", "# And the one hot labels will be as long as the full spread of tokenized words\n", "print(one_hot_labels[5])\n", "print(one_hot_labels[6])" ] }, { "cell_type": "markdown", "metadata": { "id": "-1TAJMlmfO8r" }, "source": [ "### Train a Text Generation Model\n", "\n", "Building an RNN to train our text generation model will be very similar to the sentiment models you've built previously. The only real change necessary is to make sure to use Categorical instead of Binary Cross Entropy as the loss function - we could use Binary before since the sentiment was only 0 or 1, but now there are hundreds of categories.\n", "\n", "From there, we should also consider using *more* epochs than before, as text generation can take a little longer to converge than sentiment analysis, *and* we aren't working with all that much data yet. I'll set it at 200 epochs here since we're only use part of the dataset, and training will tail off quite a bit over that many epochs." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "G1YXuxIqfygN" }, "outputs": [], "source": [ "from tensorflow.keras.models import Sequential\n", "from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional\n", "\n", "model = Sequential()\n", "model.add(Embedding(total_words, 64, input_length=max_sequence_len-1))\n", "model.add(Bidirectional(LSTM(20)))\n", "model.add(Dense(total_words, activation='softmax'))\n", "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n", "history = model.fit(input_sequences, one_hot_labels, epochs=200, verbose=1)" ] }, { "cell_type": "markdown", "metadata": { "id": "AXVFpoREhV6Y" }, "source": [ "### View the Training Graph" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "aeSNfS7uhch0" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "def plot_graphs(history, string):\n", " plt.plot(history.history[string])\n", " plt.xlabel(\"Epochs\")\n", " plt.ylabel(string)\n", " plt.show()\n", "\n", "plot_graphs(history, 'accuracy')" ] }, { "cell_type": "markdown", "metadata": { "id": "1rAgRpxYhjpB" }, "source": [ "### Generate new lyrics!\n", "\n", "It's finally time to generate some new lyrics from the trained model, and see what we get. To do so, we'll provide some \"seed text\", or an input sequence for the model to start with. We'll also decide just how long of an output sequence we want - this could essentially be infinite, as the input plus the previous output will be continuously fed in for a new output word (at least up to our max sequence length)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DC7zfcgviDTp" }, "outputs": [], "source": [ "seed_text = \"im feeling chills\"\n", "next_words = 100\n", " \n", "for _ in range(next_words):\n", "\ttoken_list = tokenizer.texts_to_sequences([seed_text])[0]\n", "\ttoken_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n", "\tpredicted = np.argmax(model.predict(token_list), axis=-1)\n", "\toutput_word = \"\"\n", "\tfor word, index in tokenizer.word_index.items():\n", "\t\tif index == predicted:\n", "\t\t\toutput_word = word\n", "\t\t\tbreak\n", "\tseed_text += \" \" + output_word\n", "print(seed_text)" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l10c03_nlp_constructing_text_generation_model.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_for_deep_learning/l10c04_nlp_optimizing_the_text_generation_model.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "punL79CN7Ox6" }, "source": [ "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "_ckMIh7O7s6D" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "Ph5eir3Pf-3z" }, "source": [ "# Optimizing the Text Generation Model" ] }, { "cell_type": "markdown", "metadata": { "id": "S5Uhzt6vVIB2" }, "source": [ "\n", " \n", " \n", "
\n", " Run in Google Colab\n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "dCxhW3mtLmfb" }, "source": [ "You've already done some amazing work with generating new songs, but so far we've seen some issues with repetition and a fair amount of incoherence. By using more data and further tweaking the model, you'll be able to get improved results. We'll once again use the [Kaggle Song Lyrics Dataset](https://www.kaggle.com/mousehead/songlyrics) here." ] }, { "cell_type": "markdown", "metadata": { "id": "4aHK2CYygXom" }, "source": [ "## Import TensorFlow and related functions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "2LmLTREBf5ng" }, "outputs": [], "source": [ "import tensorflow as tf\n", "\n", "from tensorflow.keras.preprocessing.text import Tokenizer\n", "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", "\n", "# Other imports for processing data\n", "import string\n", "import numpy as np\n", "import pandas as pd" ] }, { "cell_type": "markdown", "metadata": { "id": "GmLTO_dpgge9" }, "source": [ "## Get the Dataset\n", "\n", "As noted above, we'll utilize the [Song Lyrics dataset](https://www.kaggle.com/mousehead/songlyrics) on Kaggle again." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4Bf5FVHfganK" }, "outputs": [], "source": [ "!wget --no-check-certificate \\\n", " https://drive.google.com/uc?id=1LiJFZd41ofrWoBtW-pMYsfz1w8Ny0Bj8 \\\n", " -O /tmp/songdata.csv" ] }, { "cell_type": "markdown", "metadata": { "id": "Jz9x-7dWihxx" }, "source": [ "## 250 Songs\n", "\n", "Now we've seen a model trained on just a small sample of songs, and how this often leads to repetition as you get further along in trying to generate new text. Let's switch to using the 250 songs instead, and see if our output improves. This will actually be nearly 10K lines of lyrics, which should be sufficient.\n", "\n", "Note that we won't use the full dataset here as it will take up quite a bit of RAM and processing time, but you're welcome to try doing so on your own later. If interested, you'll likely want to use only some of the more common words for the Tokenizer, which will help shrink processing time and memory needed (or else you'd have an output array hundreds of thousands of words long)." ] }, { "cell_type": "markdown", "metadata": { "id": "nWbMN_19jfRT" }, "source": [ "### Preprocessing" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "LRmPPJegovBe" }, "outputs": [], "source": [ "def tokenize_corpus(corpus, num_words=-1):\n", " # Fit a Tokenizer on the corpus\n", " if num_words > -1:\n", " tokenizer = Tokenizer(num_words=num_words)\n", " else:\n", " tokenizer = Tokenizer()\n", " tokenizer.fit_on_texts(corpus)\n", " return tokenizer\n", "\n", "def create_lyrics_corpus(dataset, field):\n", " # Remove all other punctuation\n", " dataset[field] = dataset[field].str.replace('[{}]'.format(string.punctuation), '')\n", " # Make it lowercase\n", " dataset[field] = dataset[field].str.lower()\n", " # Make it one long string to split by line\n", " lyrics = dataset[field].str.cat()\n", " corpus = lyrics.split('\\n')\n", " # Remove any trailing whitespace\n", " for l in range(len(corpus)):\n", " corpus[l] = corpus[l].rstrip()\n", " # Remove any empty lines\n", " corpus = [l for l in corpus if l != '']\n", "\n", " return corpus" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kIGedF3XjHj4" }, "outputs": [], "source": [ "def tokenize_corpus(corpus, num_words=-1):\n", " # Fit a Tokenizer on the corpus\n", " if num_words > -1:\n", " tokenizer = Tokenizer(num_words=num_words)\n", " else:\n", " tokenizer = Tokenizer()\n", " tokenizer.fit_on_texts(corpus)\n", " return tokenizer\n", "\n", "# Read the dataset from csv - this time with 250 songs\n", "dataset = pd.read_csv('/tmp/songdata.csv', dtype=str)[:250]\n", "# Create the corpus using the 'text' column containing lyrics\n", "corpus = create_lyrics_corpus(dataset, 'text')\n", "# Tokenize the corpus\n", "tokenizer = tokenize_corpus(corpus, num_words=2000)\n", "total_words = tokenizer.num_words\n", "\n", "# There should be a lot more words now\n", "print(total_words)" ] }, { "cell_type": "markdown", "metadata": { "id": "quoDmw_FkNBA" }, "source": [ "### Create Sequences and Labels" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kkLAf3HmkPSo" }, "outputs": [], "source": [ "sequences = []\n", "for line in corpus:\n", "\ttoken_list = tokenizer.texts_to_sequences([line])[0]\n", "\tfor i in range(1, len(token_list)):\n", "\t\tn_gram_sequence = token_list[:i+1]\n", "\t\tsequences.append(n_gram_sequence)\n", "\n", "# Pad sequences for equal input length \n", "max_sequence_len = max([len(seq) for seq in sequences])\n", "sequences = np.array(pad_sequences(sequences, maxlen=max_sequence_len, padding='pre'))\n", "\n", "# Split sequences between the \"input\" sequence and \"output\" predicted word\n", "input_sequences, labels = sequences[:,:-1], sequences[:,-1]\n", "# One-hot encode the labels\n", "one_hot_labels = tf.keras.utils.to_categorical(labels, num_classes=total_words)" ] }, { "cell_type": "markdown", "metadata": { "id": "cECbqT-blMk-" }, "source": [ "### Train a (Better) Text Generation Model\n", "\n", "With more data, we'll cut off after 100 epochs to avoid keeping you here all day. You'll also want to change your runtime type to GPU if you haven't already (you'll need to re-run the above cells if you change runtimes)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "7nHOp6uWlP_P" }, "outputs": [], "source": [ "from tensorflow.keras.models import Sequential\n", "from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional\n", "\n", "model = Sequential()\n", "model.add(Embedding(total_words, 64, input_length=max_sequence_len-1))\n", "model.add(Bidirectional(LSTM(20)))\n", "model.add(Dense(total_words, activation='softmax'))\n", "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n", "history = model.fit(input_sequences, one_hot_labels, epochs=100, verbose=1)" ] }, { "cell_type": "markdown", "metadata": { "id": "MgvIz20nlQcq" }, "source": [ "### View the Training Graph" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "rOqmmarvlSLh" }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", "def plot_graphs(history, string):\n", " plt.plot(history.history[string])\n", " plt.xlabel(\"Epochs\")\n", " plt.ylabel(string)\n", " plt.show()\n", "\n", "plot_graphs(history, 'accuracy')" ] }, { "cell_type": "markdown", "metadata": { "id": "ISLZZGlQlSxh" }, "source": [ "### Generate better lyrics!\n", "\n", "This time around, we should be able to get a more interesting output with less repetition." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "P96oVMk3lU7y" }, "outputs": [], "source": [ "seed_text = \"im feeling chills\"\n", "next_words = 100\n", " \n", "for _ in range(next_words):\n", "\ttoken_list = tokenizer.texts_to_sequences([seed_text])[0]\n", "\ttoken_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n", "\tpredicted = np.argmax(model.predict(token_list), axis=-1)\n", "\toutput_word = \"\"\n", "\tfor word, index in tokenizer.word_index.items():\n", "\t\tif index == predicted:\n", "\t\t\toutput_word = word\n", "\t\t\tbreak\n", "\tseed_text += \" \" + output_word\n", "print(seed_text)" ] }, { "cell_type": "markdown", "metadata": { "id": "upgJKV8_oRU9" }, "source": [ "### Varying the Possible Outputs\n", "\n", "In running the above, you may notice that the same seed text will generate similar outputs. This is because the code is currently always choosing the top predicted class as the next word. What if you wanted more variance in the output? \n", "\n", "Switching from `model.predict_classes` to `model.predict_proba` will get us all of the class probabilities. We can combine this with `np.random.choice` to select a given predicted output based on a probability, thereby giving a bit more randomness to our outputs." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "lZe9gaJeoGVP" }, "outputs": [], "source": [ "# Test the method with just the first word after the seed text\n", "seed_text = \"im feeling chills\"\n", "next_words = 100\n", " \n", "token_list = tokenizer.texts_to_sequences([seed_text])[0]\n", "token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n", "predicted_probs = model.predict(token_list)[0]\n", "predicted = np.random.choice([x for x in range(len(predicted_probs))], \n", " p=predicted_probs)\n", "# Running this cell multiple times should get you some variance in output\n", "print(predicted)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ee7WKgRGrJy1" }, "outputs": [], "source": [ "# Use this process for the full output generation\n", "seed_text = \"im feeling chills\"\n", "next_words = 100\n", " \n", "for _ in range(next_words):\n", " token_list = tokenizer.texts_to_sequences([seed_text])[0]\n", " token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n", " predicted_probs = model.predict(token_list)[0]\n", " predicted = np.random.choice([x for x in range(len(predicted_probs))],\n", " p=predicted_probs)\n", " output_word = \"\"\n", " for word, index in tokenizer.word_index.items():\n", " if index == predicted:\n", " output_word = word\n", " break\n", " seed_text += \" \" + output_word\n", "print(seed_text)" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "l10c04_nlp_optimizing_the_text_generation_model.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_lite/tflite_c01_linear_regression.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "UysiGN3tGQHY" }, "source": [ "# Running TFLite models" ] }, { "cell_type": "markdown", "metadata": { "id": "2hOrvdmswy5O" }, "source": [ "\n", " \n", " \n", "
\n", " \n", " \n", " Run in Google Colab\n", " \n", " \n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "W-VhTkyTGcaQ" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dy4BcTjBFTWx" }, "outputs": [], "source": [ "import tensorflow as tf\n", "\n", "import pathlib\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "from tensorflow.keras.models import Model\n", "from tensorflow.keras.layers import Input" ] }, { "cell_type": "markdown", "metadata": { "id": "ceibQLDeGhI4" }, "source": [ "## Create a basic model of the form y = mx + c" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "YIBCsjQNF46Z" }, "outputs": [], "source": [ "# Create a simple Keras model.\n", "x = [-1, 0, 1, 2, 3, 4]\n", "y = [-3, -1, 1, 3, 5, 7]\n", "\n", "model = tf.keras.models.Sequential([\n", " tf.keras.layers.Dense(units=1, input_shape=[1])\n", "])\n", "model.compile(optimizer='sgd', loss='mean_squared_error')\n", "model.fit(x, y, epochs=200, verbose=1)" ] }, { "cell_type": "markdown", "metadata": { "id": "EjsB-QICGt6L" }, "source": [ "## Generate a SavedModel" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "a9xcbK7QHOfm" }, "outputs": [], "source": [ "export_dir = 'saved_model/1'\n", "tf.saved_model.save(model, export_dir)" ] }, { "cell_type": "markdown", "metadata": { "id": "RRtsNwkiGxcO" }, "source": [ "## Convert the SavedModel to TFLite" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "TtM8yKTVTpD3" }, "outputs": [], "source": [ "# Convert the model.\n", "converter = tf.lite.TFLiteConverter.from_saved_model(export_dir)\n", "tflite_model = converter.convert()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4idYulcNHTdO" }, "outputs": [], "source": [ "tflite_model_file = pathlib.Path('model.tflite')\n", "tflite_model_file.write_bytes(tflite_model)" ] }, { "cell_type": "markdown", "metadata": { "id": "HgGvp2yBG25Q" }, "source": [ "## Initialize the TFLite interpreter to try it out" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DOt94wIWF8m7" }, "outputs": [], "source": [ "# Load TFLite model and allocate tensors.\n", "interpreter = tf.lite.Interpreter(model_content=tflite_model)\n", "interpreter.allocate_tensors()\n", "\n", "# Get input and output tensors.\n", "input_details = interpreter.get_input_details()\n", "output_details = interpreter.get_output_details()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JGYkEK08F8qK" }, "outputs": [], "source": [ "# Test the TensorFlow Lite model on random input data.\n", "input_shape = input_details[0]['shape']\n", "inputs, outputs = [], []\n", "for _ in range(100):\n", " input_data = np.array(np.random.random_sample(input_shape), dtype=np.float32)\n", " interpreter.set_tensor(input_details[0]['index'], input_data)\n", "\n", " interpreter.invoke()\n", " tflite_results = interpreter.get_tensor(output_details[0]['index'])\n", "\n", " # Test the TensorFlow model on random input data.\n", " tf_results = model(tf.constant(input_data))\n", " output_data = np.array(tf_results)\n", " \n", " inputs.append(input_data[0][0])\n", " outputs.append(output_data[0][0])" ] }, { "cell_type": "markdown", "metadata": { "id": "t1gQGH1KWAgW" }, "source": [ "## Visualize the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "ccvQ1mEJVrqo" }, "outputs": [], "source": [ "plt.plot(inputs, outputs, 'r')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "WbugMH6yKvtd" }, "source": [ "## Download the TFLite model file" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FOAIMETeJmkc" }, "outputs": [], "source": [ "try:\n", " from google.colab import files\n", " files.download(tflite_model_file)\n", "except:\n", " pass" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "tflite_c01_linear_regression.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_lite/tflite_c02_transfer_learning.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "oYM61xrTsP5d" }, "source": [ "# Transfer Learning with TensorFlow Hub for TFLite" ] }, { "cell_type": "markdown", "metadata": { "id": "aFNhz34Svuhe" }, "source": [ "\n", " \n", " \n", "
\n", " \n", " \n", " Run in Google Colab\n", " \n", " \n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "bL54LWCHt5q5" }, "source": [ "## Setup " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dlauq-4FWGZM" }, "outputs": [], "source": [ "import os\n", "\n", "import matplotlib.pylab as plt\n", "import numpy as np\n", "\n", "import tensorflow as tf\n", "import tensorflow_hub as hub\n", "\n", "print(\"Version: \", tf.__version__)\n", "print(\"Eager mode: \", tf.executing_eagerly())\n", "print(\"Hub version: \", hub.__version__)\n", "print(\"GPU is\", \"available\" if tf.config.list_physical_devices('GPU') else \"NOT AVAILABLE\")" ] }, { "cell_type": "markdown", "metadata": { "id": "mmaHHH7Pvmth" }, "source": [ "## Select the Hub/TF2 module to use\n", "\n", "Hub modules for TF 1.x won't work here, please use one of the selections provided." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "FlsEcKVeuCnf" }, "outputs": [], "source": [ "module_selection = (\"mobilenet_v2\", 224, 1280) #@param [\"(\\\"mobilenet_v2\\\", 224, 1280)\", \"(\\\"inception_v3\\\", 299, 2048)\"] {type:\"raw\", allow-input: true}\n", "handle_base, pixels, FV_SIZE = module_selection\n", "MODULE_HANDLE =\"https://tfhub.dev/google/tf2-preview/{}/feature_vector/4\".format(handle_base)\n", "IMAGE_SIZE = (pixels, pixels)\n", "print(\"Using {} with input size {} and output dimension {}\".format(\n", " MODULE_HANDLE, IMAGE_SIZE, FV_SIZE))" ] }, { "cell_type": "markdown", "metadata": { "id": "sYUsgwCBv87A" }, "source": [ "## Data preprocessing" ] }, { "cell_type": "markdown", "metadata": { "id": "8nqVX3KYwGPh" }, "source": [ "Use [TensorFlow Datasets](http://tensorflow.org/datasets) to load the cats and dogs dataset.\n", "\n", "This `tfds` package is the easiest way to load pre-defined data. If you have your own data, and are interested in importing using it with TensorFlow see [loading image data](../load_data/images.ipynb)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jGvpkDj4wBup" }, "outputs": [], "source": [ "import tensorflow_datasets as tfds\n", "tfds.disable_progress_bar()" ] }, { "cell_type": "markdown", "metadata": { "id": "YkF4Boe5wN7N" }, "source": [ "The `tfds.load` method downloads and caches the data, and returns a `tf.data.Dataset` object. These objects provide powerful, efficient methods for manipulating data and piping it into your model.\n", "\n", "Since `\"cats_vs_dogs\"` doesn't define standard splits, use the subsplit feature to divide it into (train, validation, test) with 80%, 10%, 10% of the data respectively." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "SQ9xK9F2wGD8" }, "outputs": [], "source": [ "(train_examples, validation_examples, test_examples), info = tfds.load(\n", " 'cats_vs_dogs',\n", " split=['train[80%:]', 'train[80%:90%]', 'train[90%:]'],\n", " with_info=True, \n", " as_supervised=True, \n", ")\n", "\n", "num_examples = info.splits['train'].num_examples\n", "num_classes = info.features['label'].num_classes" ] }, { "cell_type": "markdown", "metadata": { "id": "pmXQYXNWwf19" }, "source": [ "### Format the Data\n", "\n", "Use the `tf.image` module to format the images for the task.\n", "\n", "Resize the images to a fixes input size, and rescale the input channels" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "y7UyXblSwkUS" }, "outputs": [], "source": [ "def format_image(image, label):\n", " image = tf.image.resize(image, IMAGE_SIZE) / 255.0\n", " return image, label" ] }, { "cell_type": "markdown", "metadata": { "id": "1nrDR8CnwrVk" }, "source": [ "Now shuffle and batch the data\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "zAEUG7vawxLm" }, "outputs": [], "source": [ "BATCH_SIZE = 32 #@param {type:\"integer\"}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fHEC9mbswxvM" }, "outputs": [], "source": [ "train_batches = train_examples.shuffle(num_examples // 4).map(format_image).batch(BATCH_SIZE).prefetch(1)\n", "validation_batches = validation_examples.map(format_image).batch(BATCH_SIZE).prefetch(1)\n", "test_batches = test_examples.map(format_image).batch(1)" ] }, { "cell_type": "markdown", "metadata": { "id": "ghQhZjgEw1cK" }, "source": [ "Inspect a batch" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gz0xsMCjwx54" }, "outputs": [], "source": [ "for image_batch, label_batch in train_batches.take(1):\n", " pass\n", "\n", "image_batch.shape" ] }, { "cell_type": "markdown", "metadata": { "id": "FS_gVStowW3G" }, "source": [ "## Defining the model\n", "\n", "All it takes is to put a linear classifier on top of the `feature_extractor_layer` with the Hub module.\n", "\n", "For speed, we start out with a non-trainable `feature_extractor_layer`, but you can also enable fine-tuning for greater accuracy." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "RaJW3XrPyFiF" }, "outputs": [], "source": [ "do_fine_tuning = False #@param {type:\"boolean\"}" ] }, { "cell_type": "markdown", "metadata": { "id": "wd0KfstqaUmE" }, "source": [ "Load TFHub Module" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "svvDrt3WUrrm" }, "outputs": [], "source": [ "feature_extractor = hub.KerasLayer(MODULE_HANDLE,\n", " input_shape=IMAGE_SIZE + (3,), \n", " output_shape=[FV_SIZE],\n", " trainable=do_fine_tuning)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "50FYNIb1dmJH" }, "outputs": [], "source": [ "print(\"Building model with\", MODULE_HANDLE)\n", "model = tf.keras.Sequential([\n", " feature_extractor,\n", " tf.keras.layers.Dense(num_classes)\n", "])\n", "model.summary()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "1PzLoQK0Zadv" }, "outputs": [], "source": [ "#@title (Optional) Unfreeze some layers\n", "NUM_LAYERS = 7 #@param {type:\"slider\", min:1, max:50, step:1}\n", " \n", "if do_fine_tuning:\n", " feature_extractor.trainable = True\n", " \n", " for layer in model.layers[-NUM_LAYERS:]:\n", " layer.trainable = True\n", "\n", "else:\n", " feature_extractor.trainable = False" ] }, { "cell_type": "markdown", "metadata": { "id": "u2e5WupIw2N2" }, "source": [ "## Training the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9f3yBUvkd_VJ" }, "outputs": [], "source": [ "if do_fine_tuning:\n", " model.compile(\n", " optimizer=tf.keras.optimizers.SGD(lr=0.002, momentum=0.9), \n", " loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])\n", "else:\n", " model.compile(\n", " optimizer='adam', \n", " loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "w_YKX2Qnfg6x" }, "outputs": [], "source": [ "EPOCHS = 5\n", "hist = model.fit(train_batches,\n", " epochs=EPOCHS,\n", " validation_data=validation_batches)" ] }, { "cell_type": "markdown", "metadata": { "id": "u_psFoTeLpHU" }, "source": [ "## Export the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XaSb5nVzHcVv" }, "outputs": [], "source": [ "CATS_VS_DOGS_SAVED_MODEL = \"exp_saved_model\"" ] }, { "cell_type": "markdown", "metadata": { "id": "fZqRAg1uz1Nu" }, "source": [ "Export the SavedModel" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "yJMue5YgnwtN" }, "outputs": [], "source": [ "tf.saved_model.save(model, CATS_VS_DOGS_SAVED_MODEL)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "SOQF4cOan0SY" }, "outputs": [], "source": [ "%%bash -s $CATS_VS_DOGS_SAVED_MODEL\n", "saved_model_cli show --dir $1 --tag_set serve --signature_def serving_default" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FY7QGBgBytwX" }, "outputs": [], "source": [ "loaded = tf.saved_model.load(CATS_VS_DOGS_SAVED_MODEL)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tIhPyMISz952" }, "outputs": [], "source": [ "print(list(loaded.signatures.keys()))\n", "infer = loaded.signatures[\"serving_default\"]\n", "print(infer.structured_input_signature)\n", "print(infer.structured_outputs)" ] }, { "cell_type": "markdown", "metadata": { "id": "XxLiLC8n0H16" }, "source": [ "## Convert using TFLite's Converter" ] }, { "cell_type": "markdown", "metadata": { "id": "1aUYvCpfWmrQ" }, "source": [ "Load the TFLiteConverter with the SavedModel" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dqJRyIg8Wl1n" }, "outputs": [], "source": [ "converter = tf.lite.TFLiteConverter.from_saved_model(CATS_VS_DOGS_SAVED_MODEL)" ] }, { "cell_type": "markdown", "metadata": { "id": "AudcNjT0UtfF" }, "source": [ "### Post-training quantization\n", "The simplest form of post-training quantization quantizes weights from floating point to 8-bits of precision. This technique is enabled as an option in the TensorFlow Lite converter. At inference, weights are converted from 8-bits of precision to floating point and computed using floating-point kernels. This conversion is done once and cached to reduce latency.\n", "\n", "To further improve latency, hybrid operators dynamically quantize activations to 8-bits and perform computations with 8-bit weights and activations. This optimization provides latencies close to fully fixed-point inference. However, the outputs are still stored using floating point, so that the speedup with hybrid ops is less than a full fixed-point computation." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "WmSr2-yZoUhz" }, "outputs": [], "source": [ "converter.optimizations = [tf.lite.Optimize.DEFAULT]" ] }, { "cell_type": "markdown", "metadata": { "id": "YpCijI08UxP0" }, "source": [ "### Post-training integer quantization\n", "We can get further latency improvements, reductions in peak memory usage, and access to integer only hardware accelerators by making sure all model math is quantized. To do this, we need to measure the dynamic range of activations and inputs with a representative data set. You can simply create an input data generator and provide it to our converter." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "clM_dTIkWdIa" }, "outputs": [], "source": [ "def representative_data_gen():\n", " for input_value, _ in test_batches.take(100):\n", " yield [input_value]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "0oPkAxDvUias" }, "outputs": [], "source": [ "converter.representative_dataset = representative_data_gen" ] }, { "cell_type": "markdown", "metadata": { "id": "IGUAVTqXVfnu" }, "source": [ "The resulting model will be fully quantized but still take float input and output for convenience.\n", "\n", "Ops that do not have quantized implementations will automatically be left in floating point. This allows conversion to occur smoothly but may restrict deployment to accelerators that support float. " ] }, { "cell_type": "markdown", "metadata": { "id": "cPVdjaEJVkHy" }, "source": [ "### Full integer quantization\n", "\n", "To require the converter to only output integer operations, one can specify:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "eQi1aO2cVhoL" }, "outputs": [], "source": [ "converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]" ] }, { "cell_type": "markdown", "metadata": { "id": "snwssESbVtFw" }, "source": [ "### Finally convert the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tUEgr46WVsqd" }, "outputs": [], "source": [ "tflite_model = converter.convert()\n", "tflite_model_file = 'converted_model.tflite'\n", "\n", "with open(tflite_model_file, \"wb\") as f:\n", " f.write(tflite_model)" ] }, { "cell_type": "markdown", "metadata": { "id": "BbTF6nd1KG2o" }, "source": [ "##Test the TFLite model using the Python Interpreter" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dg2NkVTmLUdJ" }, "outputs": [], "source": [ "# Load TFLite model and allocate tensors.\n", " \n", "interpreter = tf.lite.Interpreter(model_path=tflite_model_file)\n", "interpreter.allocate_tensors()\n", "\n", "input_index = interpreter.get_input_details()[0][\"index\"]\n", "output_index = interpreter.get_output_details()[0][\"index\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "snJQVs9JNglv" }, "outputs": [], "source": [ "from tqdm import tqdm\n", "\n", "# Gather results for the randomly sampled test images\n", "predictions = []\n", "\n", "test_labels, test_imgs = [], []\n", "for img, label in tqdm(test_batches.take(10)):\n", " interpreter.set_tensor(input_index, img)\n", " interpreter.invoke()\n", " predictions.append(interpreter.get_tensor(output_index))\n", " \n", " test_labels.append(label.numpy()[0])\n", " test_imgs.append(img)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "YMTWNqPpNiAI" }, "outputs": [], "source": [ "#@title Utility functions for plotting\n", "# Utilities for plotting\n", "\n", "class_names = ['cat', 'dog']\n", "\n", "def plot_image(i, predictions_array, true_label, img):\n", " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", " plt.grid(False)\n", " plt.xticks([])\n", " plt.yticks([])\n", " \n", " img = np.squeeze(img)\n", "\n", " plt.imshow(img, cmap=plt.cm.binary)\n", "\n", " predicted_label = np.argmax(predictions_array)\n", " if predicted_label == true_label:\n", " color = 'green'\n", " else:\n", " color = 'red'\n", " \n", " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", " 100*np.max(predictions_array),\n", " class_names[true_label]),\n", " color=color)\n" ] }, { "cell_type": "markdown", "metadata": { "id": "fK_CTyL3XQt1" }, "source": [ "NOTE: Colab runs on server CPUs. At the time of writing this, TensorFlow Lite doesn't have super optimized server CPU kernels. For this reason post-training full-integer quantized models may be slower here than the other kinds of optimized models. But for mobile CPUs, considerable speedup can be observed." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "1-lbnicPNkZs" }, "outputs": [], "source": [ "#@title Visualize the outputs { run: \"auto\" }\n", "index = 0 #@param {type:\"slider\", min:0, max:9, step:1}\n", "plt.figure(figsize=(6,3))\n", "plt.subplot(1,2,1)\n", "plot_image(index, predictions, test_labels, test_imgs)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "PmZRieHmKLY5" }, "source": [ "Download the model.\n", "\n", "**NOTE: You might have to run to the cell below twice**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "0jJAxrQB2VFw" }, "outputs": [], "source": [ "labels = ['cat', 'dog']\n", "\n", "with open('labels.txt', 'w') as f:\n", " f.write('\\n'.join(labels))\n", "\n", "try:\n", " from google.colab import files\n", " files.download('converted_model.tflite')\n", " files.download('labels.txt')\n", "except:\n", " pass" ] }, { "cell_type": "markdown", "metadata": { "id": "BDlmpjC6VnFZ" }, "source": [ "# Prepare the test images for download (Optional)" ] }, { "cell_type": "markdown", "metadata": { "id": "_1ja_WA0WZOH" }, "source": [ "This part involves downloading additional test images for the Mobile Apps only in case you need to try out more samples" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fzLKEBrfTREA" }, "outputs": [], "source": [ "!mkdir -p test_images" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Qn7ukNQCSewb" }, "outputs": [], "source": [ "from PIL import Image\n", "\n", "for index, (image, label) in enumerate(test_batches.take(50)):\n", " image = tf.cast(image * 255.0, tf.uint8)\n", " image = tf.squeeze(image).numpy()\n", " pil_image = Image.fromarray(image)\n", " pil_image.save('test_images/{}_{}.jpg'.format(class_names[label[0]], index))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "xVKKWUG8UMO5" }, "outputs": [], "source": [ "!ls test_images" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "l_w_-UdlS9Vi" }, "outputs": [], "source": [ "!zip -qq cats_vs_dogs_test_images.zip -r test_images/" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Giva6EHwWm6Y" }, "outputs": [], "source": [ "try:\n", " files.download('cats_vs_dogs_test_images.zip')\n", "except:\n", " pass" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "tflite_c02_transfer_learning.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_lite/tflite_c03_exercise_convert_model_to_tflite.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "KlUrRaN4w3ct" }, "source": [ "# Train Your Own Model and Convert It to TFLite" ] }, { "cell_type": "markdown", "metadata": { "id": "H3UojxdNw8J1" }, "source": [ "\n", " \n", " \n", "
\n", " \n", " \n", " Run in Google Colab\n", " \n", " \n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "pXX-pi1r6NfG" }, "source": [ "This notebook uses the [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) dataset which contains 70,000 grayscale images in 10 categories. The images show individual articles of clothing at low resolution (28 by 28 pixels), as seen here:\n", "\n", "\n", " \n", " \n", "
\n", " \"Fashion\n", "
\n", " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
 \n", "
\n", "\n", "Fashion MNIST is intended as a drop-in replacement for the classic [MNIST](http://yann.lecun.com/exdb/mnist/) dataset—often used as the \"Hello, World\" of machine learning programs for computer vision. The MNIST dataset contains images of handwritten digits (0, 1, 2, etc.) in a format identical to that of the articles of clothing we'll use here.\n", "\n", "This uses Fashion MNIST for variety, and because it's a slightly more challenging problem than regular MNIST. Both datasets are relatively small and are used to verify that an algorithm works as expected. They're good starting points to test and debug code.\n", "\n", "We will use 60,000 images to train the network and 10,000 images to evaluate how accurately the network learned to classify images. You can access the Fashion MNIST directly from TensorFlow. Import and load the Fashion MNIST data directly from TensorFlow:" ] }, { "cell_type": "markdown", "metadata": { "id": "rjOAfhgd__Sp" }, "source": [ "# Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "pfyZKowNAQ4j" }, "outputs": [], "source": [ "# TensorFlow and tf.keras\n", "import tensorflow as tf\n", "from tensorflow import keras\n", "\n", "import tensorflow_datasets as tfds\n", "tfds.disable_progress_bar()\n", "\n", "# Helper libraries\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import pathlib\n", "\n", "print(tf.__version__)" ] }, { "cell_type": "markdown", "metadata": { "id": "tadPBTEiAprt" }, "source": [ "# Download Fashion MNIST Dataset\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Ds9gfZKzAnkX" }, "outputs": [], "source": [ "splits = tfds.Split.ALL.subsplit(weighted=(80, 10, 10))\n", "\n", "splits, info = tfds.load('fashion_mnist', with_info=True, as_supervised=True, split=splits)\n", "\n", "(train_examples, validation_examples, test_examples) = splits\n", "\n", "num_examples = info.splits['train'].num_examples\n", "num_classes = info.features['label'].num_classes" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-eAv71FRm4JE" }, "outputs": [], "source": [ "class_names = ['T-shirt_top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "hXe6jNokqX3_" }, "outputs": [], "source": [ "with open('labels.txt', 'w') as f:\n", " f.write('\\n'.join(class_names))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "q0RxpwTmQN-y" }, "outputs": [], "source": [ "IMG_SIZE = 28" ] }, { "cell_type": "markdown", "metadata": { "id": "ZAkuq0V0Aw2X" }, "source": [ "# Preprocessing data" ] }, { "cell_type": "markdown", "metadata": { "id": "_5SIivkunKCC" }, "source": [ "## Preprocess" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nQMIkJf9AvJ4" }, "outputs": [], "source": [ "# Write a function to normalize and resize the images\n", "\n", "def format_example(image, label):\n", " # Cast image to float32\n", " image = # YOUR CODE HERE\n", " # Resize the image if necessary\n", " image = # YOUR CODE HERE\n", " # Normalize the image in the range [0, 1]\n", " image = # YOUR CODE HERE\n", " return image, label" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "oEQP743aMv4C" }, "outputs": [], "source": [ "# Set the batch size to 32\n", "\n", "BATCH_SIZE = 32" ] }, { "cell_type": "markdown", "metadata": { "id": "JM4HfIJtnNEk" }, "source": [ "## Create a Dataset from images and labels" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zOL4gSUARFjM" }, "outputs": [], "source": [ "# Prepare the examples by preprocessing the them and then batching them (and optionally prefetching them)\n", "\n", "# If you wish you can shuffle train set here\n", "train_batches = # YOUR CODE HERE\n", "\n", "validation_batches = # YOUR CODE HERE\n", "test_batches = # YOUR CODE HERE" ] }, { "cell_type": "markdown", "metadata": { "id": "M-topQaOm_LM" }, "source": [ "# Building the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "4gsYqdIlEFVg" }, "outputs": [], "source": [ "\"\"\"\n", "Model: \"sequential\"\n", "_________________________________________________________________\n", "Layer (type) Output Shape Param # \n", "=================================================================\n", "conv2d (Conv2D) (None, 26, 26, 16) 160 \n", "_________________________________________________________________\n", "max_pooling2d (MaxPooling2D) (None, 13, 13, 16) 0 \n", "_________________________________________________________________\n", "conv2d_1 (Conv2D) (None, 11, 11, 32) 4640 \n", "_________________________________________________________________\n", "flatten (Flatten) (None, 3872) 0 \n", "_________________________________________________________________\n", "dense (Dense) (None, 64) 247872 \n", "_________________________________________________________________\n", "dense_1 (Dense) (None, 10) 650 \n", "=================================================================\n", "Total params: 253,322\n", "Trainable params: 253,322\n", "Non-trainable params: 0\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kDqcwksFB1bh" }, "outputs": [], "source": [ "# Build the model shown in the previous cell\n", "\n", "\n", "model = tf.keras.Sequential([\n", " # Set the input shape to (28, 28, 1), kernel size=3, filters=16 and use ReLU activation, \n", " tf.keras.layers.Conv2D(# YOUR CODE HERE), \n", " tf.keras.layers.MaxPooling2D(),\n", " # Set the number of filters to 32, kernel size to 3 and use ReLU activation \n", " tf.keras.layers.Conv2D(# YOUR CODE HERE),\n", " # Flatten the output layer to 1 dimension\n", " tf.keras.layers.Flatten(),\n", " # Add a fully connected layer with 64 hidden units and ReLU activation\n", " tf.keras.layers.Dense(# YOUR CODE HERE),\n", " # Attach a final softmax classification head\n", " tf.keras.layers.Dense(# YOUR CODE HERE)])\n", "\n", "# Set the loss and accuracy metrics\n", "model.compile(\n", " optimizer='adam', \n", " loss=# YOUR CODE HERE, \n", " metrics=# YOUR CODE HERE)\n", " " ] }, { "cell_type": "markdown", "metadata": { "id": "zEMOz-LDnxgD" }, "source": [ "## Train" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JGlNoRtzCP4_" }, "outputs": [], "source": [ "model.fit(train_batches, \n", " epochs=10,\n", " validation_data=validation_batches)" ] }, { "cell_type": "markdown", "metadata": { "id": "TZT9-7w9n4YO" }, "source": [ "# Exporting to TFLite" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9dq78KBkCV2_" }, "outputs": [], "source": [ "export_dir = 'saved_model/1'\n", "\n", "# Use the tf.saved_model API to export the SavedModel\n", "\n", "# Your Code Here" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "EDGiYrBdE6fl" }, "outputs": [], "source": [ "optimization = tf.lite.Optimize.DEFAULT" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "RbcS9C00CzGe" }, "outputs": [], "source": [ "# Use the TFLiteConverter SavedModel API to initialize the converter\n", "converter = # YOUR CODE HERE\n", "\n", "# Set the optimzations\n", "converter.optimizations = # YOUR CODE HERE\n", "\n", "# Invoke the converter to finally generate the TFLite model\n", "tflite_model = # YOUR CODE HERE" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "q5PWCDsTC3El" }, "outputs": [], "source": [ "tflite_model_file = 'model.tflite'\n", "\n", "with open(tflite_model_file, \"wb\") as f:\n", " f.write(tflite_model)" ] }, { "cell_type": "markdown", "metadata": { "id": "SR6wFcQ1Fglm" }, "source": [ "# Test if your model is working" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "O3IFOcUEIzQx" }, "outputs": [], "source": [ "# Load TFLite model and allocate tensors.\n", "interpreter = tf.lite.Interpreter(model_content=tflite_model)\n", "interpreter.allocate_tensors()\n", "\n", "input_index = interpreter.get_input_details()[0][\"index\"]\n", "output_index = interpreter.get_output_details()[0][\"index\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "rKcToCBEC-Bu" }, "outputs": [], "source": [ "# Gather results for the randomly sampled test images\n", "predictions = []\n", "test_labels = []\n", "test_images = []\n", "\n", "for img, label in test_batches.take(50):\n", " interpreter.set_tensor(input_index, img)\n", " interpreter.invoke()\n", " predictions.append(interpreter.get_tensor(output_index))\n", " test_labels.append(label[0])\n", " test_images.append(np.array(img))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "kSjTmi05Tyod" }, "outputs": [], "source": [ "#@title Utility functions for plotting\n", "# Utilities for plotting\n", "\n", "def plot_image(i, predictions_array, true_label, img):\n", " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", " plt.grid(False)\n", " plt.xticks([])\n", " plt.yticks([])\n", " \n", " img = np.squeeze(img)\n", "\n", " plt.imshow(img, cmap=plt.cm.binary)\n", "\n", " predicted_label = np.argmax(predictions_array)\n", " if predicted_label == true_label.numpy():\n", " color = 'green'\n", " else:\n", " color = 'red'\n", " \n", " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", " 100*np.max(predictions_array),\n", " class_names[true_label]),\n", " color=color)\n", "\n", "def plot_value_array(i, predictions_array, true_label):\n", " predictions_array, true_label = predictions_array[i], true_label[i]\n", " plt.grid(False)\n", " plt.xticks(list(range(10)), class_names, rotation='vertical')\n", " plt.yticks([])\n", " thisplot = plt.bar(range(10), predictions_array[0], color=\"#777777\")\n", " plt.ylim([0, 1])\n", " predicted_label = np.argmax(predictions_array[0])\n", "\n", " thisplot[predicted_label].set_color('red')\n", " thisplot[true_label].set_color('green')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "ZZwg0wFaVXhZ" }, "outputs": [], "source": [ "#@title Visualize the outputs { run: \"auto\" }\n", "index = 49 #@param {type:\"slider\", min:1, max:50, step:1}\n", "plt.figure(figsize=(6,3))\n", "plt.subplot(1,2,1)\n", "plot_image(index, predictions, test_labels, test_images)\n", "plt.show()\n", "plot_value_array(index, predictions, test_labels)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "076bo3FMpRDb" }, "source": [ "# Download TFLite model and assets\n", "\n", "**NOTE: You might have to run to the cell below twice**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XsPXqPlgZPjE" }, "outputs": [], "source": [ "try:\n", " from google.colab import files\n", " files.download(tflite_model_file)\n", " files.download('labels.txt')\n", "except:\n", " pass" ] }, { "cell_type": "markdown", "metadata": { "id": "VyBVNwAzH3Oe" }, "source": [ "# Deploying TFLite model" ] }, { "cell_type": "markdown", "metadata": { "id": "pdfa5L6wH87u" }, "source": [ "Now once you've the trained TFLite model downloaded, you can ahead and deploy this on an Android/iOS application by placing the model assets in the appropriate location." ] }, { "cell_type": "markdown", "metadata": { "id": "iLY6X8P90L0P" }, "source": [ "# Prepare the test images for download (Optional)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "G3bjzLj10OJv" }, "outputs": [], "source": [ "!mkdir -p test_images" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "pVrBZv1-0Py-" }, "outputs": [], "source": [ "from PIL import Image\n", "\n", "for index, (image, label) in enumerate(test_batches.take(50)):\n", " image = tf.cast(image * 255.0, tf.uint8)\n", " image = tf.squeeze(image).numpy()\n", " pil_image = Image.fromarray(image)\n", " pil_image.save('test_images/{}_{}.jpg'.format(class_names[label[0]].lower(), index))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "nX0N0M8u0R2s" }, "outputs": [], "source": [ "!ls test_images" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "LvLht1QM0W8k" }, "outputs": [], "source": [ "!zip -qq fmnist_test_images.zip -r test_images/" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FdOq-4sT0X95" }, "outputs": [], "source": [ "try:\n", " files.download('fmnist_test_images.zip')\n", "except:\n", " pass" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "tflite_c03_exercise_convert_model_to_tflite.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_lite/tflite_c04_exercise_convert_model_to_tflite_solution.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "06ndLauQxiQm" }, "source": [ "# Train Your Own Model and Convert It to TFLite" ] }, { "cell_type": "markdown", "metadata": { "id": "Dtav_aq2xh6n" }, "source": [ "\n", " \n", " \n", "
\n", " \n", " \n", " Run in Google Colab\n", " \n", " \n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "Ka96-ajYzxVU" }, "source": [ "This notebook uses the [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) dataset which contains 70,000 grayscale images in 10 categories. The images show individual articles of clothing at low resolution (28 by 28 pixels), as seen here:\n", "\n", "\n", " \n", " \n", "
\n", " \"Fashion\n", "
\n", " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
 \n", "
\n", "\n", "Fashion MNIST is intended as a drop-in replacement for the classic [MNIST](http://yann.lecun.com/exdb/mnist/) dataset—often used as the \"Hello, World\" of machine learning programs for computer vision. The MNIST dataset contains images of handwritten digits (0, 1, 2, etc.) in a format identical to that of the articles of clothing we'll use here.\n", "\n", "This uses Fashion MNIST for variety, and because it's a slightly more challenging problem than regular MNIST. Both datasets are relatively small and are used to verify that an algorithm works as expected. They're good starting points to test and debug code.\n", "\n", "We will use 60,000 images to train the network and 10,000 images to evaluate how accurately the network learned to classify images. You can access the Fashion MNIST directly from TensorFlow. Import and load the Fashion MNIST data directly from TensorFlow:" ] }, { "cell_type": "markdown", "metadata": { "id": "rjOAfhgd__Sp" }, "source": [ "# Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "pfyZKowNAQ4j" }, "outputs": [], "source": [ "# TensorFlow and tf.keras\n", "import tensorflow as tf\n", "from tensorflow import keras\n", "\n", "# Helper libraries\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import pathlib\n", "\n", "\n", "print(tf.__version__)" ] }, { "cell_type": "markdown", "metadata": { "id": "tadPBTEiAprt" }, "source": [ "# Download Fashion MNIST Dataset" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jmSkLCyRKqKB" }, "outputs": [], "source": [ "import tensorflow_datasets as tfds\n", "tfds.disable_progress_bar()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XcNwi6nFKneZ" }, "outputs": [], "source": [ "splits, info = tfds.load('fashion_mnist', with_info=True, as_supervised=True, \n", " split=['train[:80%]', 'train[80%:90%]', 'train[90%:]'])\n", "\n", "(train_examples, validation_examples, test_examples) = splits\n", "\n", "num_examples = info.splits['train'].num_examples\n", "num_classes = info.features['label'].num_classes" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "-eAv71FRm4JE" }, "outputs": [], "source": [ "class_names = ['T-shirt_top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "hXe6jNokqX3_" }, "outputs": [], "source": [ "with open('labels.txt', 'w') as f:\n", " f.write('\\n'.join(class_names))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "iubWCThbdN8K" }, "outputs": [], "source": [ "IMG_SIZE = 28" ] }, { "cell_type": "markdown", "metadata": { "id": "ZAkuq0V0Aw2X" }, "source": [ "# Preprocessing data" ] }, { "cell_type": "markdown", "metadata": { "id": "_5SIivkunKCC" }, "source": [ "## Preprocess" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "BwyhsyGydHDl" }, "outputs": [], "source": [ "def format_example(image, label):\n", " image = tf.cast(image, tf.float32)\n", " image = tf.image.resize(image, (IMG_SIZE, IMG_SIZE))\n", " image = image / 255.0\n", " return image, label" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "HAlBlXOUMwqe" }, "outputs": [], "source": [ "BATCH_SIZE = 32" ] }, { "cell_type": "markdown", "metadata": { "id": "JM4HfIJtnNEk" }, "source": [ "## Create a Dataset from images and labels" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "uxe2I3oxLDhq" }, "outputs": [], "source": [ "train_batches = train_examples.cache().shuffle(num_examples//4).batch(BATCH_SIZE).map(format_example).prefetch(1)\n", "validation_batches = validation_examples.cache().batch(BATCH_SIZE).map(format_example).prefetch(1)\n", "test_batches = test_examples.cache().batch(1).map(format_example)" ] }, { "cell_type": "markdown", "metadata": { "id": "M-topQaOm_LM" }, "source": [ "# Building the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "kDqcwksFB1bh" }, "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " tf.keras.layers.Conv2D(16, 3, activation='relu', input_shape=(IMG_SIZE, IMG_SIZE, 1)),\n", " tf.keras.layers.MaxPooling2D(),\n", " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", " tf.keras.layers.Flatten(),\n", " tf.keras.layers.Dense(64, activation='relu'),\n", " tf.keras.layers.Dense(10)\n", "])\n", "\n", "model.compile(\n", " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " optimizer='adam',\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { "id": "zEMOz-LDnxgD" }, "source": [ "## Train" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "1fk2faPsjqfU" }, "outputs": [], "source": [ "validation_batches" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "DGJe_CNvjnhT" }, "outputs": [], "source": [ "train_batches, " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "JGlNoRtzCP4_" }, "outputs": [], "source": [ "model.fit(train_batches, \n", " epochs=10,\n", " validation_data=validation_batches)" ] }, { "cell_type": "markdown", "metadata": { "id": "TZT9-7w9n4YO" }, "source": [ "# Exporting to TFLite" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9dq78KBkCV2_" }, "outputs": [], "source": [ "export_dir = 'saved_model/1'\n", "tf.saved_model.save(model, export_dir)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "EDGiYrBdE6fl" }, "outputs": [], "source": [ "optimization = tf.lite.Optimize.DEFAULT" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "RbcS9C00CzGe" }, "outputs": [], "source": [ "# Convert the model.\n", "converter = tf.lite.TFLiteConverter.from_saved_model(export_dir)\n", "converter.optimizations = [optimization]\n", "tflite_model = converter.convert()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "q5PWCDsTC3El" }, "outputs": [], "source": [ "tflite_model_file = 'model.tflite'\n", "\n", "with open(tflite_model_file, \"wb\") as f:\n", " f.write(tflite_model)" ] }, { "cell_type": "markdown", "metadata": { "id": "SR6wFcQ1Fglm" }, "source": [ "# Test the model with TFLite interpreter " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "rKcToCBEC-Bu" }, "outputs": [], "source": [ "# Load TFLite model and allocate tensors.\n", "interpreter = tf.lite.Interpreter(model_content=tflite_model)\n", "interpreter.allocate_tensors()\n", "\n", "input_index = interpreter.get_input_details()[0][\"index\"]\n", "output_index = interpreter.get_output_details()[0][\"index\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "E8EpFpIBFkq8" }, "outputs": [], "source": [ "# Gather results for the randomly sampled test images\n", "predictions = []\n", "test_labels = []\n", "test_images = []\n", "\n", "for img, label in test_batches.take(50):\n", " interpreter.set_tensor(input_index, img)\n", " interpreter.invoke()\n", " predictions.append(interpreter.get_tensor(output_index))\n", " test_labels.append(label[0])\n", " test_images.append(np.array(img))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "kSjTmi05Tyod" }, "outputs": [], "source": [ "#@title Utility functions for plotting\n", "# Utilities for plotting\n", "\n", "def plot_image(i, predictions_array, true_label, img):\n", " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", " plt.grid(False)\n", " plt.xticks([])\n", " plt.yticks([])\n", " \n", " img = np.squeeze(img)\n", "\n", " plt.imshow(img, cmap=plt.cm.binary)\n", "\n", " predicted_label = np.argmax(predictions_array)\n", " if predicted_label == true_label.numpy():\n", " color = 'green'\n", " else:\n", " color = 'red'\n", " \n", " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", " 100*np.max(predictions_array),\n", " class_names[true_label]),\n", " color=color)\n", "\n", "def plot_value_array(i, predictions_array, true_label):\n", " predictions_array, true_label = predictions_array[i], true_label[i]\n", " plt.grid(False)\n", " plt.xticks(list(range(10)), class_names, rotation='vertical')\n", " plt.yticks([])\n", " thisplot = plt.bar(range(10), predictions_array[0], color=\"#777777\")\n", " plt.ylim([0, 1])\n", " predicted_label = np.argmax(predictions_array[0])\n", "\n", " thisplot[predicted_label].set_color('red')\n", " thisplot[true_label].set_color('green')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "ZZwg0wFaVXhZ" }, "outputs": [], "source": [ "#@title Visualize the outputs { run: \"auto\" }\n", "index = 12 #@param {type:\"slider\", min:1, max:50, step:1}\n", "plt.figure(figsize=(6,3))\n", "plt.subplot(1,2,1)\n", "plot_image(index, predictions, test_labels, test_images)\n", "plt.show()\n", "plot_value_array(index, predictions, test_labels)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "076bo3FMpRDb" }, "source": [ "# Download TFLite model and assets\n", "\n", "**NOTE: You might have to run to the cell below twice**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XsPXqPlgZPjE" }, "outputs": [], "source": [ "try:\n", " from google.colab import files\n", "\n", " files.download(tflite_model_file)\n", " files.download('labels.txt')\n", "except:\n", " pass" ] }, { "cell_type": "markdown", "metadata": { "id": "H8t7_jRiz9Vw" }, "source": [ "# Prepare the test images for download (Optional)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Fi09nIps0gBu" }, "outputs": [], "source": [ "!mkdir -p test_images" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "sF7EZ63J0hZs" }, "outputs": [], "source": [ "from PIL import Image\n", "\n", "for index, (image, label) in enumerate(test_batches.take(50)):\n", " image = tf.cast(image * 255.0, tf.uint8)\n", " image = tf.squeeze(image).numpy()\n", " pil_image = Image.fromarray(image)\n", " pil_image.save('test_images/{}_{}.jpg'.format(class_names[label[0]].lower(), index))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "uM35O-uv0iWS" }, "outputs": [], "source": [ "!ls test_images" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "aR20r4qW0jVm" }, "outputs": [], "source": [ "!zip -qq fmnist_test_images.zip -r test_images/" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tjk4537X0kWN" }, "outputs": [], "source": [ "try:\n", " files.download('fmnist_test_images.zip')\n", "except:\n", " pass" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "tflite_c04_exercise_convert_model_to_tflite_solution.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_lite/tflite_c05_exercise_rock_paper_scissors.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "oYM61xrTsP5d" }, "source": [ "# Rock, Paper & Scissors with TensorFlow Hub - TFLite" ] }, { "cell_type": "markdown", "metadata": { "id": "pAI9358VyAsE" }, "source": [ "\n", " \n", " \n", "
\n", " \n", " \n", " Run in Google Colab\n", " \n", " \n", " \n", " View source on GitHub\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "bL54LWCHt5q5" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dlauq-4FWGZM" }, "outputs": [], "source": [ "import os\n", "\n", "import matplotlib.pylab as plt\n", "import numpy as np\n", "\n", "import tensorflow as tf\n", "import tensorflow_hub as hub\n", "\n", "print(\"Version: \", tf.__version__)\n", "print(\"Eager mode: \", tf.executing_eagerly())\n", "print(\"Hub version: \", hub.__version__)\n", "print(\"GPU is\", \"available\" if tf.config.list_physical_devices('GPU') else \"NOT AVAILABLE\")" ] }, { "cell_type": "markdown", "metadata": { "id": "mmaHHH7Pvmth" }, "source": [ "## Select the Hub/TF2 module to use\n", "\n", "Hub modules for TF 1.x won't work here, please use one of the selections provided." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FlsEcKVeuCnf" }, "outputs": [], "source": [ "module_selection = (\"mobilenet_v2\", 224, 1280) #@param [\"(\\\"mobilenet_v2\\\", 224, 1280)\", \"(\\\"inception_v3\\\", 299, 2048)\"] {type:\"raw\", allow-input: true}\n", "handle_base, pixels, FV_SIZE = module_selection\n", "MODULE_HANDLE =\"https://tfhub.dev/google/tf2-preview/{}/feature_vector/4\".format(handle_base)\n", "IMAGE_SIZE = (pixels, pixels)\n", "print(\"Using {} with input size {} and output dimension {}\".format(\n", " MODULE_HANDLE, IMAGE_SIZE, FV_SIZE))" ] }, { "cell_type": "markdown", "metadata": { "id": "sYUsgwCBv87A" }, "source": [ "## Data preprocessing" ] }, { "cell_type": "markdown", "metadata": { "id": "8nqVX3KYwGPh" }, "source": [ "Use [TensorFlow Datasets](http://tensorflow.org/datasets) to load the rock, paper and scissors dataset.\n", "\n", "This `tfds` package is the easiest way to load pre-defined data. If you have your own data, and are interested in importing using it with TensorFlow see [loading image data](../load_data/images.ipynb)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jGvpkDj4wBup" }, "outputs": [], "source": [ "import tensorflow_datasets as tfds\n", "tfds.disable_progress_bar()" ] }, { "cell_type": "markdown", "metadata": { "id": "YkF4Boe5wN7N" }, "source": [ "The `tfds.load` method downloads and caches the data, and returns a `tf.data.Dataset` object. These objects provide powerful, efficient methods for manipulating data and piping it into your model.\n", "\n", "Since `\"rock_paper_scissors\"` doesn't define standard splits, use the subsplit feature to divide it into (train, validation, test) with 80%, 10%, 10% of the data respectively." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "SQ9xK9F2wGD8" }, "outputs": [], "source": [ "splits = tfds.Split.ALL.subsplit(weighted=(80, 10, 10))\n", "\n", "# Go to the TensorFlow Dataset's website and search for the Rock, Paper, Scissors dataset and load it here\n", "splits, info = tfds.load( # YOUR CODE HERE )\n", " \n", "# Save the dataset splits in a tuple\n", "(train_examples, validation_examples, test_examples) = splits\n", "\n", "num_examples = info.splits['train'].num_examples\n", "num_classes = info.features['label'].num_classes" ] }, { "cell_type": "markdown", "metadata": { "id": "pmXQYXNWwf19" }, "source": [ "### Format the Data\n", "\n", "Use the `tf.image` module to format the images for the task.\n", "\n", "Resize the images to a fixes input size, and rescale the input channels" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "y7UyXblSwkUS" }, "outputs": [], "source": [ "def format_image(image, label):\n", " image = tf.image.resize(image, IMAGE_SIZE) / 255.0\n", " return image, label\n" ] }, { "cell_type": "markdown", "metadata": { "id": "1nrDR8CnwrVk" }, "source": [ "Now shuffle and batch the data\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zAEUG7vawxLm" }, "outputs": [], "source": [ "BATCH_SIZE = 32 #@param {type:\"integer\"}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fHEC9mbswxvM" }, "outputs": [], "source": [ "# Prepare the examples by preprocessing them and then batching them (and optionally prefetching them)\n", "\n", "# If you wish you can shuffle train set here\n", "train_batches = # YOUR CODE HERE\n", "\n", "validation_batches = # YOUR CODE HERE\n", "test_batches = # YOUR CODE HERE" ] }, { "cell_type": "markdown", "metadata": { "id": "ghQhZjgEw1cK" }, "source": [ "Inspect a batch" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gz0xsMCjwx54" }, "outputs": [], "source": [ "for image_batch, label_batch in train_batches.take(1):\n", " pass\n", "\n", "image_batch.shape" ] }, { "cell_type": "markdown", "metadata": { "id": "FS_gVStowW3G" }, "source": [ "## Defining the model\n", "\n", "All it takes is to put a linear classifier on top of the `feature_extractor_layer` with the Hub module.\n", "\n", "For speed, we start out with a non-trainable `feature_extractor_layer`, but you can also enable fine-tuning for greater accuracy." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "RaJW3XrPyFiF" }, "outputs": [], "source": [ "do_fine_tuning = False #@param {type:\"boolean\"}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "50FYNIb1dmJH" }, "outputs": [], "source": [ "# Build the model with a TFHub KerasLayer and attach a classification head to it\n", "print(\"Building model with\", MODULE_HANDLE)\n", "model = tf.keras.Sequential([\n", " hub.KerasLayer(MODULE_HANDLE,\n", " input_shape=IMAGE_SIZE + (3, ), \n", " output_shape=[FV_SIZE],\n", " trainable=do_fine_tuning),\n", " tf.keras.layers.Dense(num_classes)\n", "])\n", "model.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "u2e5WupIw2N2" }, "source": [ "## Training the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9f3yBUvkd_VJ" }, "outputs": [], "source": [ "if do_fine_tuning:\n", " model.compile(\n", " optimizer=tf.keras.optimizers.SGD(lr=0.002, momentum=0.9), \n", " loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])\n", "else:\n", " model.compile(\n", " optimizer='adam', \n", " loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "w_YKX2Qnfg6x" }, "outputs": [], "source": [ "EPOCHS = 3\n", "hist = model.fit(train_batches,\n", " epochs=EPOCHS,\n", " validation_data=validation_batches)" ] }, { "cell_type": "markdown", "metadata": { "id": "u_psFoTeLpHU" }, "source": [ "## Export the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XaSb5nVzHcVv" }, "outputs": [], "source": [ "RPS_SAVED_MODEL = \"exp_saved_model\"" ] }, { "cell_type": "markdown", "metadata": { "id": "fZqRAg1uz1Nu" }, "source": [ "Export the SavedModel" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "yJMue5YgnwtN" }, "outputs": [], "source": [ "# Use TensorFlow's SavedModel API to export the SavedModel from the trained Keras model\n", "# YOUR CODE HERE" ] }, { "cell_type": "markdown", "metadata": { "id": "uVrnAWjqCMxs" }, "source": [ "Here you can verify the default signature of your exported SavedModel" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "SOQF4cOan0SY" }, "outputs": [], "source": [ "%%bash -s $RPS_SAVED_MODEL\n", "saved_model_cli show --dir $1 --tag_set serve --signature_def serving_default" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FY7QGBgBytwX" }, "outputs": [], "source": [ "loaded = tf.saved_model.load(RPS_SAVED_MODEL)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tIhPyMISz952" }, "outputs": [], "source": [ "print(list(loaded.signatures.keys()))\n", "infer = loaded.signatures[\"serving_default\"]\n", "print(infer.structured_input_signature)\n", "print(infer.structured_outputs)" ] }, { "cell_type": "markdown", "metadata": { "id": "XxLiLC8n0H16" }, "source": [ "## Convert with TFLiteConverter" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "WmSr2-yZoUhz" }, "outputs": [], "source": [ "# Intialize the TFLite converter to load the SavedModel\n", "converter = # YOUR CODE HERE\n", "# Set the optimization strategy for 'size' in the converter \n", "converter.optimizations = [# YOUR CODE HERE]\n", "\n", "# Use the tool to finally convert the model\n", "tflite_model = # YOUR CODE HERE\n", " \n", "with open(\"converted_model.tflite\", \"wb\") as f:\n", " f.write(tflite_model)" ] }, { "cell_type": "markdown", "metadata": { "id": "BbTF6nd1KG2o" }, "source": [ "Run the following cells to check whether your TFLite model is working using the Python Interpreter" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "dg2NkVTmLUdJ" }, "outputs": [], "source": [ "#@title Loading the converted TFLite model...\n", "# Load TFLite model and allocate tensors.\n", "tflite_model_file = 'converted_model.tflite'\n", "with open(tflite_model_file, 'rb') as fid:\n", " tflite_model = fid.read()\n", " \n", "interpreter = tf.lite.Interpreter(model_content=tflite_model)\n", "interpreter.allocate_tensors()\n", "\n", "input_index = interpreter.get_input_details()[0][\"index\"]\n", "output_index = interpreter.get_output_details()[0][\"index\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "snJQVs9JNglv" }, "outputs": [], "source": [ "#@title Testing on random test examples...\n", "from tqdm import tqdm\n", "\n", "# Gather results for the randomly sampled test images\n", "predictions = []\n", "\n", "test_labels, test_imgs = [], []\n", "for img, label in tqdm(test_batches.take(10)):\n", " interpreter.set_tensor(input_index, img)\n", " interpreter.invoke()\n", " predictions.append(interpreter.get_tensor(output_index))\n", " \n", " test_labels.append(label.numpy()[0])\n", " test_imgs.append(img)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "YMTWNqPpNiAI" }, "outputs": [], "source": [ "#@title Utility functions for plotting\n", "# Utilities for plotting\n", "\n", "class_names = ['rock', 'paper', 'scissors']\n", "\n", "def plot_image(i, predictions_array, true_label, img):\n", " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", " plt.grid(False)\n", " plt.xticks([])\n", " plt.yticks([])\n", " \n", " img = np.squeeze(img)\n", "\n", " plt.imshow(img, cmap=plt.cm.binary)\n", "\n", " predicted_label = np.argmax(predictions_array)\n", " if predicted_label == true_label:\n", " color = 'green'\n", " else:\n", " color = 'red'\n", " \n", " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", " 100*np.max(predictions_array),\n", " class_names[true_label]),\n", " color=color)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "1-lbnicPNkZs" }, "outputs": [], "source": [ "#@title Visualize the outputs { run: \"auto\" }\n", "index = 9 #@param {type:\"slider\", min:0, max:9, step:1}\n", "plt.figure(figsize=(6,3))\n", "plt.subplot(1,2,1)\n", "plot_image(index, predictions, test_labels, test_imgs)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "PmZRieHmKLY5" }, "source": [ "Download the model\n", "\n", "**NOTE: You might have to run to the cell below twice**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "0jJAxrQB2VFw" }, "outputs": [], "source": [ "with open('labels.txt', 'w') as f:\n", " f.write('\\n'.join(class_names))\n", "\n", "\n", "try:\n", " from google.colab import files\n", " files.download('converted_model.tflite') \n", " files.download('labels.txt')\n", "except:\n", " pass" ] }, { "cell_type": "markdown", "metadata": { "id": "BDlmpjC6VnFZ" }, "source": [ "# Prepare the test images for download (Optional)" ] }, { "cell_type": "markdown", "metadata": { "id": "_1ja_WA0WZOH" }, "source": [ "This part involves downloading additional test images for the Mobile Apps only in case you need to try out more samples" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fzLKEBrfTREA" }, "outputs": [], "source": [ "!mkdir -p test_images" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Qn7ukNQCSewb" }, "outputs": [], "source": [ "from PIL import Image\n", "\n", "for index, (image, label) in enumerate(test_batches.take(50)):\n", " image = tf.cast(image * 255.0, tf.uint8)\n", " image = tf.squeeze(image).numpy()\n", " pil_image = Image.fromarray(image)\n", " pil_image.save('test_images/{}_{}.jpg'.format(class_names[label[0]], index))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "xVKKWUG8UMO5" }, "outputs": [], "source": [ "!ls test_images" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "l_w_-UdlS9Vi" }, "outputs": [], "source": [ "!zip -qq rps_test_images.zip -r test_images/" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Giva6EHwWm6Y" }, "outputs": [], "source": [ "try:\n", " files.download('rps_test_images.zip')\n", "except:\n", " pass" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "tflite_c05_exercise_rock_paper_scissors.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: courses/udacity_intro_to_tensorflow_lite/tflite_c06_exercise_rock_paper_scissors_solution.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "id": "Za8-Nr5k11fh" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "Eq10uEbw0E4l" }, "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ] }, { "cell_type": "markdown", "metadata": { "id": "oYM61xrTsP5d" }, "source": [ "# Rock, Paper & Scissors with TensorFlow Hub - TFLite" ] }, { "cell_type": "markdown", "metadata": { "id": "xWFpUd1yy3gt" }, "source": [ "\n", " \n", " \n", " \n", "
\n", " \n", " \n", " Run in Google Colab\n", " \n", " \n", " \n", " View source on GitHub\n", " \n", " See TF Hub model\n", "
" ] }, { "cell_type": "markdown", "metadata": { "id": "bL54LWCHt5q5" }, "source": [ "## Setup" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dlauq-4FWGZM" }, "outputs": [], "source": [ "import os\n", "\n", "import matplotlib.pylab as plt\n", "import numpy as np\n", "\n", "import tensorflow as tf\n", "import tensorflow_hub as hub\n", "\n", "print(\"Version: \", tf.__version__)\n", "print(\"Eager mode: \", tf.executing_eagerly())\n", "print(\"Hub version: \", hub.__version__)\n", "print(\"GPU is\", \"available\" if tf.config.list_physical_devices('GPU') else \"NOT AVAILABLE\")" ] }, { "cell_type": "markdown", "metadata": { "id": "mmaHHH7Pvmth" }, "source": [ "## Select the Hub/TF2 module to use\n", "\n", "Hub modules for TF 1.x won't work here, please use one of the selections provided." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FlsEcKVeuCnf" }, "outputs": [], "source": [ "module_selection = (\"mobilenet_v2\", 224, 1280) #@param [\"(\\\"mobilenet_v2\\\", 224, 1280)\", \"(\\\"inception_v3\\\", 299, 2048)\"] {type:\"raw\", allow-input: true}\n", "handle_base, pixels, FV_SIZE = module_selection\n", "MODULE_HANDLE = \"https://tfhub.dev/google/tf2-preview/{}/feature_vector/4\".format(handle_base)\n", "IMAGE_SIZE = (pixels, pixels)\n", "print(\"Using {} with input size {} and output dimension {}\".format(\n", " MODULE_HANDLE, IMAGE_SIZE, FV_SIZE))" ] }, { "cell_type": "markdown", "metadata": { "id": "sYUsgwCBv87A" }, "source": [ "## Data preprocessing" ] }, { "cell_type": "markdown", "metadata": { "id": "8nqVX3KYwGPh" }, "source": [ "Use [TensorFlow Datasets](http://tensorflow.org/datasets) to load the rock, paper and scissors dataset.\n", "\n", "This `tfds` package is the easiest way to load pre-defined data. If you have your own data, and are interested in importing using it with TensorFlow see [loading image data](../load_data/images.ipynb)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "jGvpkDj4wBup" }, "outputs": [], "source": [ "import tensorflow_datasets as tfds\n", "tfds.disable_progress_bar()" ] }, { "cell_type": "markdown", "metadata": { "id": "YkF4Boe5wN7N" }, "source": [ "The `tfds.load` method downloads and caches the data, and returns a `tf.data.Dataset` object. These objects provide powerful, efficient methods for manipulating data and piping it into your model.\n", "\n", "Since `\"rock_paper_scissors\"` doesn't define standard splits, use the subsplit feature to divide it into (train, validation, test) with 80%, 10%, 10% of the data respectively." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "SQ9xK9F2wGD8" }, "outputs": [], "source": [ "splits = tfds.Split.ALL.subsplit(weighted=(80, 10, 10))\n", "\n", "splits, info = tfds.load('rock_paper_scissors', with_info=True, as_supervised=True, split = splits)\n", "\n", "(train_examples, validation_examples, test_examples) = splits\n", "\n", "num_examples = info.splits['train'].num_examples\n", "num_classes = info.features['label'].num_classes" ] }, { "cell_type": "markdown", "metadata": { "id": "pmXQYXNWwf19" }, "source": [ "### Format the Data\n", "\n", "Use the `tf.image` module to format the images for the task.\n", "\n", "Resize the images to a fixes input size, and rescale the input channels" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "y7UyXblSwkUS" }, "outputs": [], "source": [ "def format_image(image, label):\n", " image = tf.image.resize(image, IMAGE_SIZE) / 255.0\n", " return image, label\n" ] }, { "cell_type": "markdown", "metadata": { "id": "1nrDR8CnwrVk" }, "source": [ "Now shuffle and batch the data\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "zAEUG7vawxLm" }, "outputs": [], "source": [ "BATCH_SIZE = 32 #@param {type:\"integer\"}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fHEC9mbswxvM" }, "outputs": [], "source": [ "train_batches = train_examples.shuffle(num_examples // 4).batch(BATCH_SIZE).map(format_image).prefetch(1)\n", "validation_batches = validation_examples.batch(BATCH_SIZE).map(format_image).prefetch(1)\n", "test_batches = test_examples.batch(1).map(format_image)" ] }, { "cell_type": "markdown", "metadata": { "id": "ghQhZjgEw1cK" }, "source": [ "Inspect a batch" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "gz0xsMCjwx54" }, "outputs": [], "source": [ "for image_batch, label_batch in train_batches.take(1):\n", " pass\n", "\n", "image_batch.shape" ] }, { "cell_type": "markdown", "metadata": { "id": "FS_gVStowW3G" }, "source": [ "## Defining the model\n", "\n", "All it takes is to put a linear classifier on top of the `feature_extractor_layer` with the Hub module.\n", "\n", "For speed, we start out with a non-trainable `feature_extractor_layer`, but you can also enable fine-tuning for greater accuracy." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "RaJW3XrPyFiF" }, "outputs": [], "source": [ "do_fine_tuning = False #@param {type:\"boolean\"}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "50FYNIb1dmJH" }, "outputs": [], "source": [ "print(\"Building model with\", MODULE_HANDLE)\n", "model = tf.keras.Sequential([\n", " hub.KerasLayer(MODULE_HANDLE,\n", " input_shape=IMAGE_SIZE + (3, ), \n", " output_shape=[FV_SIZE],\n", " trainable=do_fine_tuning),\n", " tf.keras.layers.Dense(num_classes)\n", "])\n", "model.summary()" ] }, { "cell_type": "markdown", "metadata": { "id": "u2e5WupIw2N2" }, "source": [ "## Training the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "9f3yBUvkd_VJ" }, "outputs": [], "source": [ "if do_fine_tuning:\n", " model.compile(\n", " optimizer=tf.keras.optimizers.SGD(lr=0.002, momentum=0.9), \n", " loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])\n", "else:\n", " model.compile(\n", " optimizer='adam', \n", " loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "w_YKX2Qnfg6x" }, "outputs": [], "source": [ "EPOCHS = 5\n", "hist = model.fit(train_batches,\n", " epochs=EPOCHS,\n", " validation_data=validation_batches)" ] }, { "cell_type": "markdown", "metadata": { "id": "u_psFoTeLpHU" }, "source": [ "## Export the model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "XaSb5nVzHcVv" }, "outputs": [], "source": [ "RPS_SAVED_MODEL = \"rps_saved_model\"" ] }, { "cell_type": "markdown", "metadata": { "id": "fZqRAg1uz1Nu" }, "source": [ "Export the SavedModel" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "yJMue5YgnwtN" }, "outputs": [], "source": [ "tf.saved_model.save(model, RPS_SAVED_MODEL)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "SOQF4cOan0SY" }, "outputs": [], "source": [ "%%bash -s $RPS_SAVED_MODEL\n", "saved_model_cli show --dir $1 --tag_set serve --signature_def serving_default" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "FY7QGBgBytwX" }, "outputs": [], "source": [ "loaded = tf.saved_model.load(RPS_SAVED_MODEL)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "tIhPyMISz952" }, "outputs": [], "source": [ "print(list(loaded.signatures.keys()))\n", "infer = loaded.signatures[\"serving_default\"]\n", "print(infer.structured_input_signature)\n", "print(infer.structured_outputs)" ] }, { "cell_type": "markdown", "metadata": { "id": "XxLiLC8n0H16" }, "source": [ "## Convert with TFLiteConverter" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "WmSr2-yZoUhz" }, "outputs": [], "source": [ "converter = tf.lite.TFLiteConverter.from_saved_model(RPS_SAVED_MODEL)\n", "converter.optimizations = [tf.lite.Optimize.DEFAULT]\n", "\n", "\n", "tflite_model = converter.convert()\n", "with open(\"converted_model.tflite\", \"wb\") as f:\n", " f.write(tflite_model)" ] }, { "cell_type": "markdown", "metadata": { "id": "BbTF6nd1KG2o" }, "source": [ "Test the TFLite model using the Python Interpreter" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "dg2NkVTmLUdJ" }, "outputs": [], "source": [ "# Load TFLite model and allocate tensors.\n", "tflite_model_file = 'converted_model.tflite'\n", "with open(tflite_model_file, 'rb') as fid:\n", " tflite_model = fid.read()\n", " \n", "interpreter = tf.lite.Interpreter(model_content=tflite_model)\n", "interpreter.allocate_tensors()\n", "\n", "input_index = interpreter.get_input_details()[0][\"index\"]\n", "output_index = interpreter.get_output_details()[0][\"index\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "snJQVs9JNglv" }, "outputs": [], "source": [ "from tqdm import tqdm\n", "\n", "# Gather results for the randomly sampled test images\n", "predictions = []\n", "\n", "test_labels, test_imgs = [], []\n", "for img, label in tqdm(test_batches.take(10)):\n", " interpreter.set_tensor(input_index, img)\n", " interpreter.invoke()\n", " predictions.append(interpreter.get_tensor(output_index))\n", " \n", " test_labels.append(label.numpy()[0])\n", " test_imgs.append(img)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", "id": "YMTWNqPpNiAI" }, "outputs": [], "source": [ "#@title Utility functions for plotting\n", "# Utilities for plotting\n", "\n", "class_names = ['rock', 'paper', 'scissors']\n", "\n", "def plot_image(i, predictions_array, true_label, img):\n", " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", " plt.grid(False)\n", " plt.xticks([])\n", " plt.yticks([])\n", " \n", " img = np.squeeze(img)\n", "\n", " plt.imshow(img, cmap=plt.cm.binary)\n", "\n", " predicted_label = np.argmax(predictions_array)\n", " print(type(predicted_label), type(true_label))\n", " if predicted_label == true_label:\n", " color = 'green'\n", " else:\n", " color = 'red'\n", " \n", " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", " 100*np.max(predictions_array),\n", " class_names[true_label]),\n", " color=color)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "both", "id": "1-lbnicPNkZs" }, "outputs": [], "source": [ "#@title Visualize the outputs { run: \"auto\" }\n", "index = 0 #@param {type:\"slider\", min:0, max:9, step:1}\n", "plt.figure(figsize=(6,3))\n", "plt.subplot(1,2,1)\n", "plot_image(index, predictions, test_labels, test_imgs)\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "id": "PmZRieHmKLY5" }, "source": [ "Download the model\n", "\n", "**NOTE: You might have to run to the cell below twice**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "0jJAxrQB2VFw" }, "outputs": [], "source": [ "with open('labels.txt', 'w') as f:\n", " f.write('\\n'.join(class_names))\n", "\n", "try:\n", " from google.colab import files\n", " files.download('converted_model.tflite')\n", " files.download('labels.txt')\n", "except:\n", " pass" ] }, { "cell_type": "markdown", "metadata": { "id": "BDlmpjC6VnFZ" }, "source": [ "# Prepare the test images for download (Optional)" ] }, { "cell_type": "markdown", "metadata": { "id": "_1ja_WA0WZOH" }, "source": [ "This part involves downloading additional test images for the Mobile Apps only in case you need to try out more samples" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fzLKEBrfTREA" }, "outputs": [], "source": [ "!mkdir -p test_images" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Qn7ukNQCSewb" }, "outputs": [], "source": [ "from PIL import Image\n", "\n", "for index, (image, label) in enumerate(test_batches.take(50)):\n", " image = tf.cast(image * 255.0, tf.uint8)\n", " image = tf.squeeze(image).numpy()\n", " pil_image = Image.fromarray(image)\n", " pil_image.save('test_images/{}_{}.jpg'.format(class_names[label[0]], index))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "xVKKWUG8UMO5" }, "outputs": [], "source": [ "!ls test_images" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "l_w_-UdlS9Vi" }, "outputs": [], "source": [ "!zip -qq rps_test_images.zip -r test_images/" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Giva6EHwWm6Y" }, "outputs": [], "source": [ "try:\n", " files.download('rps_test_images.zip')\n", "except:\n", " pass" ] } ], "metadata": { "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "tflite_c06_exercise_rock_paper_scissors_solution.ipynb", "toc_visible": true }, "kernelspec": { "display_name": "Python 3", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: lite/.gitignore ================================================ # TensorFlow Lite model *.tflite # Swift ## Build generated build/ DerivedData/ ## Various settings *.pbxuser !default.pbxuser *.mode1v3 !default.mode1v3 *.mode2v3 !default.mode2v3 *.perspectivev3 !default.perspectivev3 xcuserdata/ ## Other *.moved-aside *.xccheckout *.xcscmblueprint ## Obj-C/Swift specific *.hmap *.ipa *.dSYM.zip *.dSYM # CocoaPods Pods/ *.xcworkspace # mkdocs ignore generated site site/ # Android # Built application files *.apk *.aar *.ap_ *.aab # Files for the ART/Dalvik VM *.dex # Java class files *.class # Generated files bin/ gen/ out/ # Uncomment the following line in case you need and you don't have the release build type files in your app # release/ # Gradle files .gradle/ build/ # Local configuration file (sdk path, etc) local.properties # Proguard folder generated by Eclipse proguard/ # Log Files *.log # Android Studio Navigation editor temp files .navigation/ # Android Studio captures folder captures/ # IntelliJ *.iml .idea/workspace.xml .idea/tasks.xml .idea/gradle.xml .idea/assetWizardSettings.xml .idea/dictionaries .idea/libraries # Android Studio 3 in .gitignore file. .idea/caches .idea/modules.xml .idea/navEditor.xml # Keystore files # Uncomment the following lines if you do not want to check your keystore files in. #*.jks #*.keystore # External native build folder generated in Android Studio 2.2 and later .externalNativeBuild .cxx/ # Google Services (e.g. APIs or Firebase) # google-services.json # Version control vcs.xml # lint lint/intermediates/ lint/generated/ lint/outputs/ lint/tmp/ # lint/reports/ ================================================ FILE: lite/README.md ================================================ # TensorFlow Lite sample applications The following samples demonstrate the use of TensorFlow Lite in mobile applications. Each sample is written for both Android and iOS. ## Image classification This app performs image classification on a live camera feed and displays the inference output in realtime on the screen. ### Samples [Android image classification](examples/image_classification/android/README.md) [iOS image classification](examples/image_classification/ios/README.md) [Raspberry Pi image classification](examples/image_classification/raspberry_pi/README.md) ## Object detection This app performs object detection on a live camera feed and displays the results in realtime on the screen. The app displays the confidence scores, classes and detected bounding boxes for multiple objects. A detected object is only displayed if the confidence score is greater than a defined threshold. ### Samples [Android object detection](examples/object_detection/android/README.md) [iOS object detection](examples/object_detection/ios/README.md) [Raspberry Pi object detection](examples/object_detection/raspberry_pi/README.md) ## Speech command recognition This application recognizes a set of voice commands using the device's microphone input. When a command is spoken, the corresponding class in the app is highlighted. ### Samples [iOS speech commands](examples/speech_commands/ios/README.md) ## Gesture classification This app uses a model to classify and recognize different gestures. A model is trained on webcam data captured using a web interface. The model is then converted to a TensorFlow Lite model and used to classify gestures in a mobile application. ![Gesture components](https://tensorflow.org/images/lite/screenshots/gesture_components.png) ### Web app First, we use TensorFlow.js embedded in a web interface to collect the data required to train the model. We then use TensorFlow.js to train the model. [Web gesture classification](examples/gesture_classification/web/README.md) ### Conversion script The model downloaded from the web interface is converted to a TensorFlow Lite model. [Conversion script (available as a Colab notebook)](examples/gesture_classification/ml/README.md). ### Mobile apps Once we have the TensorFlow Lite model, the implementation is very similar to the [Image classification](#image-classification) sample. #### Samples [Android gesture classification](examples/gesture_classification/android/README.md) [iOS gesture classification](examples/gesture_classification/ios/README.md) ## Model personalization This app performs model personalization on a live camera feed and displays the results in realtime on the screen. The app displays the confidence scores, classes and detected bounding boxes for multiple objects that were trained in realtime. ### Samples [Android Model Personalization](examples/model_personalization/README.md) ================================================ FILE: lite/codelabs/digit_classifier/README.md ================================================ # Codelabs for TensorFlow Lite This repository contains the code for the a TensorFlow Lite codelab: * [Build a handwritten digit classifier app with TensorFlow Lite](https://codelabs.developers.google.com/codelabs/digit-classifier-tflite) ## Introduction In these codelabs, you will learn: * How to train a digit classifer model model with TensorFlow. * How to convert a TensorFlow model to a TensorFlow Lite format. * How to deploy a TensorFlow Lite model to a mobile app. ## Pre-requisites None. ## Getting Started Visit the Google codelabs site to follow along the guided steps. ## Screenshots Screenshot ## Support - Stack Overflow: https://stackoverflow.com/questions/tagged/tensorflow-lite Patches are encouraged, and may be submitted by forking this project and submitting a pull request through GitHub. ## License Copyright 2019 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: lite/codelabs/digit_classifier/android/finish/app/build.gradle ================================================ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' android { compileSdkVersion 30 defaultConfig { applicationId "org.tensorflow.lite.codelabs.digitclassifier" minSdkVersion 21 targetSdkVersion 30 versionCode 1 versionName "1.0" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } // TODO: Add an option to avoid compressing TF Lite model file aaptOptions { noCompress "tflite" } buildTypes { release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' } } } // Sanity check if you have trained and downloaded TF Lite model. preBuild.doFirst { assert file("./src/main/assets/mnist.tflite").exists() : "mnist.tflite not found. Make sure you have trained and " + "downloaded your TensorFlow Lite model to assets/ folder" } dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" // Support Libraries implementation 'androidx.appcompat:appcompat:1.3.0' implementation 'androidx.core:core-ktx:1.5.0' implementation 'androidx.constraintlayout:constraintlayout:2.0.4' // AndroidDraw Library implementation 'com.github.divyanshub024:AndroidDraw:v0.1' // Task API implementation "com.google.android.gms:play-services-tasks:17.2.1" //TODO: Add TF Lite implementation 'org.tensorflow:tensorflow-lite:2.5.0' testImplementation 'junit:junit:4.13.2' androidTestImplementation 'androidx.test:runner:1.3.0' androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' } ================================================ FILE: lite/codelabs/digit_classifier/android/finish/app/proguard-rules.pro ================================================ # Add project specific ProGuard rules here. # You can control the set of applied configuration files using the # proguardFiles setting in build.gradle. # # For more details, see # http://developer.android.com/guide/developing/tools/proguard.html # If your project uses WebView with JS, uncomment the following # and specify the fully qualified class name to the JavaScript interface # class: #-keepclassmembers class fqcn.of.javascript.interface.for.webview { # public *; #} # Uncomment this to preserve the line number information for # debugging stack traces. #-keepattributes SourceFile,LineNumberTable # If you keep the line number information, uncomment this to # hide the original source file name. #-renamesourcefileattribute SourceFile ================================================ FILE: lite/codelabs/digit_classifier/android/finish/app/src/main/AndroidManifest.xml ================================================ ================================================ FILE: lite/codelabs/digit_classifier/android/finish/app/src/main/assets/ADD_TFLITE_MODEL_HERE ================================================ Place the TensorFlow Lite model file (mnist.tflite) you generated from step 2: "Train a machine learning model" here. ================================================ FILE: lite/codelabs/digit_classifier/android/finish/app/src/main/java/org/tensorflow/lite/codelabs/digitclassifier/DigitClassifier.kt ================================================ /* Copyright 2019 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ package org.tensorflow.lite.codelabs.digitclassifier import android.content.Context import android.content.res.AssetManager import android.graphics.Bitmap import android.util.Log import com.google.android.gms.tasks.Task import com.google.android.gms.tasks.TaskCompletionSource import java.io.FileInputStream import java.io.IOException import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.channels.FileChannel import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import org.tensorflow.lite.Interpreter class DigitClassifier(private val context: Context) { // TODO: Add a TF Lite interpreter as a field. private var interpreter: Interpreter? = null var isInitialized = false private set /** Executor to run inference task in the background. */ private val executorService: ExecutorService = Executors.newCachedThreadPool() private var inputImageWidth: Int = 0 // will be inferred from TF Lite model. private var inputImageHeight: Int = 0 // will be inferred from TF Lite model. private var modelInputSize: Int = 0 // will be inferred from TF Lite model. fun initialize(): Task { val task = TaskCompletionSource() executorService.execute { try { initializeInterpreter() task.setResult(null) } catch (e: IOException) { task.setException(e) } } return task.task } @Throws(IOException::class) private fun initializeInterpreter() { // TODO: Load the TF Lite model from file and initialize an interpreter. // Load the TF Lite model from asset folder and initialize TF Lite Interpreter with NNAPI enabled. val assetManager = context.assets val model = loadModelFile(assetManager, "mnist.tflite") val interpreter = Interpreter(model) // TODO: Read the model input shape from model file. // Read input shape from model file. val inputShape = interpreter.getInputTensor(0).shape() inputImageWidth = inputShape[1] inputImageHeight = inputShape[2] modelInputSize = FLOAT_TYPE_SIZE * inputImageWidth * inputImageHeight * PIXEL_SIZE // Finish interpreter initialization. this.interpreter = interpreter isInitialized = true Log.d(TAG, "Initialized TFLite interpreter.") } @Throws(IOException::class) private fun loadModelFile(assetManager: AssetManager, filename: String): ByteBuffer { val fileDescriptor = assetManager.openFd(filename) val inputStream = FileInputStream(fileDescriptor.fileDescriptor) val fileChannel = inputStream.channel val startOffset = fileDescriptor.startOffset val declaredLength = fileDescriptor.declaredLength return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength) } private fun classify(bitmap: Bitmap): String { check(isInitialized) { "TF Lite Interpreter is not initialized yet." } // TODO: Add code to run inference with TF Lite. // Pre-processing: resize the input image to match the model input shape. val resizedImage = Bitmap.createScaledBitmap( bitmap, inputImageWidth, inputImageHeight, true ) val byteBuffer = convertBitmapToByteBuffer(resizedImage) // Define an array to store the model output. val output = Array(1) { FloatArray(OUTPUT_CLASSES_COUNT) } // Run inference with the input data. interpreter?.run(byteBuffer, output) // Post-processing: find the digit that has the highest probability // and return it a human-readable string. val result = output[0] val maxIndex = result.indices.maxByOrNull { result[it] } ?: -1 val resultString = "Prediction Result: %d\nConfidence: %2f" .format(maxIndex, result[maxIndex]) return resultString } fun classifyAsync(bitmap: Bitmap): Task { val task = TaskCompletionSource() executorService.execute { val result = classify(bitmap) task.setResult(result) } return task.task } fun close() { executorService.execute { interpreter?.close() Log.d(TAG, "Closed TFLite interpreter.") } } private fun convertBitmapToByteBuffer(bitmap: Bitmap): ByteBuffer { val byteBuffer = ByteBuffer.allocateDirect(modelInputSize) byteBuffer.order(ByteOrder.nativeOrder()) val pixels = IntArray(inputImageWidth * inputImageHeight) bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height) for (pixelValue in pixels) { val r = (pixelValue shr 16 and 0xFF) val g = (pixelValue shr 8 and 0xFF) val b = (pixelValue and 0xFF) // Convert RGB to grayscale and normalize pixel value to [0..1]. val normalizedPixelValue = (r + g + b) / 3.0f / 255.0f byteBuffer.putFloat(normalizedPixelValue) } return byteBuffer } companion object { private const val TAG = "DigitClassifier" private const val FLOAT_TYPE_SIZE = 4 private const val PIXEL_SIZE = 1 private const val OUTPUT_CLASSES_COUNT = 10 } } ================================================ FILE: lite/codelabs/digit_classifier/android/finish/app/src/main/java/org/tensorflow/lite/codelabs/digitclassifier/MainActivity.kt ================================================ /* Copyright 2019 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ package org.tensorflow.lite.codelabs.digitclassifier import android.annotation.SuppressLint import android.graphics.Color import android.os.Bundle import androidx.appcompat.app.AppCompatActivity import android.util.Log import android.view.MotionEvent import android.widget.Button import android.widget.TextView import com.divyanshu.draw.widget.DrawView class MainActivity : AppCompatActivity() { private var drawView: DrawView? = null private var clearButton: Button? = null private var predictedTextView: TextView? = null private var digitClassifier = DigitClassifier(this) @SuppressLint("ClickableViewAccessibility") override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) // Setup view instances. drawView = findViewById(R.id.draw_view) drawView?.setStrokeWidth(70.0f) drawView?.setColor(Color.WHITE) drawView?.setBackgroundColor(Color.BLACK) clearButton = findViewById(R.id.clear_button) predictedTextView = findViewById(R.id.predicted_text) // Setup clear drawing button. clearButton?.setOnClickListener { drawView?.clearCanvas() predictedTextView?.text = getString(R.string.prediction_text_placeholder) } // Setup classification trigger so that it classify after every stroke drew. drawView?.setOnTouchListener { _, event -> // As we have interrupted DrawView's touch event, // we first need to pass touch events through to the instance for the drawing to show up. drawView?.onTouchEvent(event) // Then if user finished a touch event, run classification if (event.action == MotionEvent.ACTION_UP) { classifyDrawing() } true } // Setup digit classifier. digitClassifier .initialize() .addOnFailureListener { e -> Log.e(TAG, "Error to setting up digit classifier.", e) } } override fun onDestroy() { // Sync DigitClassifier instance lifecycle with MainActivity lifecycle, // and free up resources (e.g. TF Lite instance) once the activity is destroyed. digitClassifier.close() super.onDestroy() } private fun classifyDrawing() { val bitmap = drawView?.getBitmap() if ((bitmap != null) && (digitClassifier.isInitialized)) { digitClassifier .classifyAsync(bitmap) .addOnSuccessListener { resultText -> predictedTextView?.text = resultText } .addOnFailureListener { e -> predictedTextView?.text = getString( R.string.classification_error_message, e.localizedMessage ) Log.e(TAG, "Error classifying drawing.", e) } } } companion object { private const val TAG = "MainActivity" } } ================================================ FILE: lite/codelabs/digit_classifier/android/finish/app/src/main/res/drawable/ic_launcher_background.xml ================================================ ================================================ FILE: lite/codelabs/digit_classifier/android/finish/app/src/main/res/drawable-v24/ic_launcher_foreground.xml ================================================ ================================================ FILE: lite/codelabs/digit_classifier/android/finish/app/src/main/res/layout/activity_main.xml ================================================ ================================================ FILE: lite/examples/audio_classification/ios/AudioClassification/HomeViewController.swift ================================================ // Copyright 2022 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import AVFoundation import TensorFlowLiteTaskAudio import UIKit /// The sample app's home screen. class HomeViewController: UIViewController { // MARK: - Variables @IBOutlet private weak var tableView: UITableView! @IBOutlet private weak var inferenceView: InferenceView! private var inferenceResults: [ClassificationCategory] = [] private var modelType: ModelType = .Yamnet private var overLap = 0.5 private var maxResults = 3 private var threshold: Float = 0.0 private var threadCount = 2 private var audioClassificationHelper: AudioClassificationHelper? // MARK: - View controller lifecycle methods override func viewDidLoad() { super.viewDidLoad() inferenceView.setDefault( model: modelType, overLab: overLap, maxResult: maxResults, threshold: threshold, threads: threadCount) inferenceView.delegate = self startAudioClassification() } // MARK: - Private Methods /// Request permission and start audio classification if granted. private func startAudioClassification() { AVAudioSession.sharedInstance().requestRecordPermission { [weak self] granted in if granted { DispatchQueue.main.async { self?.restartClassifier() } } else { self?.checkPermissions() } } } /// Check permission and show error if user denied permission. private func checkPermissions() { switch AVAudioSession.sharedInstance().recordPermission { case .granted, .undetermined: startAudioClassification() case .denied: showPermissionsErrorAlert() @unknown default: fatalError("Microphone permission check returned unexpected result.") } } /// Start a new audio classification routine. private func restartClassifier() { // Stop the existing classifier if one is running. audioClassificationHelper?.stopClassifier() // Create a new classifier instance. audioClassificationHelper = AudioClassificationHelper( modelType: modelType, threadCount: threadCount, scoreThreshold: threshold, maxResults: maxResults) // Start the new classification routine. audioClassificationHelper?.delegate = self audioClassificationHelper?.startClassifier(overlap: overLap) } } extension HomeViewController { private func showPermissionsErrorAlert() { let alertController = UIAlertController( title: "Microphone Permissions Denied", message: "Microphone permissions have been denied for this app. You can change this by going to Settings", preferredStyle: .alert ) let cancelAction = UIAlertAction(title: "Cancel", style: .cancel, handler: nil) let settingsAction = UIAlertAction(title: "Settings", style: .default) { _ in UIApplication.shared.open( URL(string: UIApplication.openSettingsURLString)!, options: [:], completionHandler: nil ) } alertController.addAction(cancelAction) alertController.addAction(settingsAction) present(alertController, animated: true, completion: nil) } } enum ModelType: String { case Yamnet = "YAMNet" case speechCommandModel = "Speech Command" var fileName: String { switch self { case .Yamnet: return "yamnet" case .speechCommandModel: return "speech_commands" } } } extension HomeViewController: UITableViewDataSource, UITableViewDelegate { func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell { guard let cell = tableView.dequeueReusableCell(withIdentifier: "ResultCell") as? ResultTableViewCell else { fatalError() } cell.setData(inferenceResults[indexPath.row]) return cell } func tableView(_ tableView: UITableView, numberOfRowsInSection section: Int) -> Int { return inferenceResults.count } } extension HomeViewController: AudioClassificationHelperDelegate { func audioClassificationHelper(_ helper: AudioClassificationHelper, didSucceed result: Result) { inferenceResults = result.categories tableView.reloadData() inferenceView.setInferenceTime(result.inferenceTime) } func audioClassificationHelper(_ helper: AudioClassificationHelper, didFail error: Error) { let errorMessage = "An error occured while running audio classification: \(error.localizedDescription)" let alert = UIAlertController( title: "Error", message: errorMessage, preferredStyle: UIAlertController.Style.alert) alert.addAction(UIAlertAction(title: "OK", style: UIAlertAction.Style.default, handler: nil)) present(alert, animated: true, completion: nil) } } extension HomeViewController: InferenceViewDelegate { func view(_ view: InferenceView, needPerformActions action: InferenceView.Action) { switch action { case .changeModel(let modelType): self.modelType = modelType case .changeOverlap(let overLap): self.overLap = overLap case .changeMaxResults(let maxResults): self.maxResults = maxResults case .changeScoreThreshold(let threshold): self.threshold = threshold case .changeThreadCount(let threadCount): self.threadCount = threadCount } // Restart the audio classifier as the config as changed. restartClassifier() } } ================================================ FILE: lite/examples/audio_classification/ios/AudioClassification/InferenceView.swift ================================================ // Copyright 2022 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit protocol InferenceViewDelegate: AnyObject { /// This method is called when the user changes the value to update model used for inference. func view(_ view: InferenceView, needPerformActions action: InferenceView.Action) } /// View to allows users changing the inference configs. class InferenceView: UIView { enum Action { case changeModel(ModelType) case changeOverlap(Double) case changeMaxResults(Int) case changeScoreThreshold(Float) case changeThreadCount(Int) } weak var delegate: InferenceViewDelegate? @IBOutlet private weak var inferenceTimeLabel: UILabel! @IBOutlet private weak var overlabLabel: UILabel! @IBOutlet private weak var maxResulteLabel: UILabel! @IBOutlet private weak var thresholdLabel: UILabel! @IBOutlet private weak var threadsLabel: UILabel! @IBOutlet private weak var overLapStepper: UIStepper! @IBOutlet private weak var maxResultsStepper: UIStepper! @IBOutlet private weak var thresholdStepper: UIStepper! @IBOutlet private weak var threadsStepper: UIStepper! @IBOutlet private weak var modelSegmentedControl: UISegmentedControl! @IBOutlet private weak var showHidenButtonLayoutConstraint: NSLayoutConstraint! @IBOutlet private weak var showHidenButton: UIButton! /// Set the default settings. func setDefault(model: ModelType, overLab: Double, maxResult: Int, threshold: Float, threads: Int) { modelSegmentedControl.selectedSegmentIndex = model == .Yamnet ? 0 : 1 overlabLabel.text = "\(Int(overLab * 100))%" overLapStepper.value = overLab maxResulteLabel.text = "\(maxResult)" maxResultsStepper.value = Double(maxResult) thresholdLabel.text = String(format: "%.1f", threshold) thresholdStepper.value = Double(threshold) threadsLabel.text = "\(threads)" threadsStepper.value = Double(threads) } func setInferenceTime(_ inferenceTime: TimeInterval) { inferenceTimeLabel.text = "\(Int(inferenceTime * 1000)) ms" } @IBAction func modelSegmentedValueChanged(_ sender: UISegmentedControl) { let modelSelect: ModelType = sender.selectedSegmentIndex == 0 ? .Yamnet : .speechCommandModel delegate?.view(self, needPerformActions: .changeModel(modelSelect)) } @IBAction func overlapStepperValueChanged(_ sender: UIStepper) { overlabLabel.text = String(format: "%.0f", sender.value * 100) + "%" delegate?.view(self, needPerformActions: .changeOverlap(sender.value)) } @IBAction func maxResultsStepperValueChanged(_ sender: UIStepper) { maxResulteLabel.text = "\(Int(sender.value))" delegate?.view(self, needPerformActions: .changeMaxResults(Int(sender.value))) } @IBAction func thresholdStepperValueChanged(_ sender: UIStepper) { thresholdLabel.text = String(format: "%.1f", sender.value) delegate?.view(self, needPerformActions: .changeScoreThreshold(Float(sender.value))) } @IBAction func threadsStepperValueChanged(_ sender: UIStepper) { threadsLabel.text = "\(Int(sender.value))" delegate?.view(self, needPerformActions: .changeThreadCount(Int(sender.value))) } @IBAction func showHidenButtonTouchUpInside(_ sender: UIButton) { sender.isSelected.toggle() showHidenButtonLayoutConstraint.constant = sender.isSelected ? 300 : 40 UIView.animate( withDuration: 0.3, animations: { self.superview?.layoutIfNeeded() }, completion: nil) } } ================================================ FILE: lite/examples/audio_classification/ios/AudioClassification/Info.plist ================================================ ================================================ FILE: lite/examples/audio_classification/ios/AudioClassification/ResultTableViewCell.swift ================================================ // Copyright 2022 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import TensorFlowLiteTaskAudio import UIKit /// TableViewCell to display the inference results. Each cell corresponds to a single category. class ResultTableViewCell: UITableViewCell { @IBOutlet weak var nameLabel: UILabel! @IBOutlet weak var scoreWidthLayoutConstraint: NSLayoutConstraint! func setData(_ data: ClassificationCategory) { nameLabel.text = data.label if !data.score.isNaN { // score view width is equal 1/4 screen with scoreWidthLayoutConstraint.constant = UIScreen.main.bounds.width / 4 * CGFloat(data.score) } else { scoreWidthLayoutConstraint.constant = 0 } } } ================================================ FILE: lite/examples/audio_classification/ios/AudioClassification/TFLite/AudioClassificationHelper.swift ================================================ // Copyright 2022 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import TensorFlowLiteTaskAudio /// Delegate to returns the classification results. protocol AudioClassificationHelperDelegate: AnyObject { func audioClassificationHelper(_ helper: AudioClassificationHelper, didSucceed result: Result) func audioClassificationHelper(_ helper: AudioClassificationHelper, didFail error: Error) } private let errorDomain = "org.tensorflow.lite.examples" /// Stores results for a particular audio snipprt that was successfully classified. struct Result { let inferenceTime: Double let categories: [ClassificationCategory] } /// Information about a model file. typealias FileInfo = (name: String, extension: String) /// This class handles all data preprocessing and makes calls to run inference on a audio snippet /// by invoking the Task Library's `AudioClassifier`. class AudioClassificationHelper { // MARK: Public properties weak var delegate: AudioClassificationHelperDelegate? // MARK: Private properties /// An `AudioClassifier` object for performing audio classification using a given model. private let classifier: AudioClassifier /// An object to continously record audio using the device's microphone. private let audioRecord: AudioRecord /// A tensor to store the input audio for the model. private let inputAudioTensor: AudioTensor /// A timer to schedule classification routine to run periodically. private var timer: Timer? /// A queue to offload the classification routine to a background thread. private let processQueue = DispatchQueue(label: "processQueue") // MARK: - Initialization /// A failable initializer for `AudioClassificationHelper`. /// /// A new instance is created if the model is successfully loaded from the app's main bundle. init?(modelType: ModelType, threadCount: Int, scoreThreshold: Float, maxResults: Int) { let modelFilename = modelType.fileName // Construct the path to the model file. guard let modelPath = Bundle.main.path( forResource: modelFilename, ofType: "tflite" ) else { print("Failed to load the model file \(modelFilename).tflite.") return nil } // Specify the options for the classifier. let classifierOptions = AudioClassifierOptions(modelPath: modelPath) classifierOptions.baseOptions.computeSettings.cpuSettings.numThreads = threadCount classifierOptions.classificationOptions.maxResults = maxResults classifierOptions.classificationOptions.scoreThreshold = scoreThreshold do { // Create the classifier. classifier = try AudioClassifier.classifier(options: classifierOptions) // Create an `AudioRecord` instance to record input audio that satisfies // the model's requirements. audioRecord = try classifier.createAudioRecord() inputAudioTensor = classifier.createInputAudioTensor() } catch { print("Failed to create the classifier with error: \(error.localizedDescription)") return nil } } func stopClassifier() { audioRecord.stop() timer?.invalidate() timer = nil } /// Start the audio classification routine in the background. /// /// Classification results are periodically returned to the delegate. /// - Parameters overlap: Overlapping factor between consecutive audio snippet to be classified. /// Value must be >= 0 and < 1. func startClassifier(overlap: Double) { if overlap < 0 { let error = NSError( domain: errorDomain, code: 0, userInfo: [NSLocalizedDescriptionKey: "overlap must be equal or larger than 0."]) delegate?.audioClassificationHelper(self, didFail: error) } if overlap >= 1 { let error = NSError( domain: errorDomain, code: 0, userInfo: [NSLocalizedDescriptionKey: "overlap must be smaller than 1."]) delegate?.audioClassificationHelper(self, didFail: error) } do { // Start recording audio. try audioRecord.startRecording() // Calculate interval between sampling based on overlap. let audioFormat = inputAudioTensor.audioFormat let lengthInMilliSeconds = Double(inputAudioTensor.bufferSize) / Double(audioFormat.sampleRate) let interval = lengthInMilliSeconds * Double(1 - overlap) timer?.invalidate() // Schedule the classification routine to run every fixed interval. timer = Timer.scheduledTimer(withTimeInterval: interval, repeats: true) { [weak self] _ in self?.processQueue.async { self?.runClassification() } } } catch { delegate?.audioClassificationHelper(self, didFail: error) } } /// Run the classification routine with the latest audio stored in the `AudioRecord` instance's /// buffer. private func runClassification() { let startTime = Date().timeIntervalSince1970 do { // Grab the latest audio chunk in the audio record and run classification. try inputAudioTensor.load(audioRecord: audioRecord) let results = try classifier.classify(audioTensor: inputAudioTensor) let inferenceTime = Date().timeIntervalSince1970 - startTime // Return the classification result to the delegate. DispatchQueue.main.async { let result = Result( inferenceTime: inferenceTime, categories: results.classifications[0].categories) // Send classification result to the delegate. self.delegate?.audioClassificationHelper(self, didSucceed: result) } } catch { delegate?.audioClassificationHelper(self, didFail: error) } } } ================================================ FILE: lite/examples/audio_classification/ios/AudioClassification.xcodeproj/project.pbxproj ================================================ // !$*UTF8*$! { archiveVersion = 1; classes = { }; objectVersion = 55; objects = { /* Begin PBXBuildFile section */ 9E2DC7A5D9D560890EC07075 /* Pods_AudioClassification.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 8CB90CD43F9B8F013EA69E62 /* Pods_AudioClassification.framework */; }; BF646A7528A4DBD7006741D8 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF646A7428A4DBD7006741D8 /* AppDelegate.swift */; }; BF646A7928A4DBD7006741D8 /* HomeViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF646A7828A4DBD7006741D8 /* HomeViewController.swift */; }; BF646A7C28A4DBD7006741D8 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = BF646A7A28A4DBD7006741D8 /* Main.storyboard */; }; BF646A8128A4DBD9006741D8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = BF646A7F28A4DBD9006741D8 /* LaunchScreen.storyboard */; }; BF646A9E28A4DDEA006741D8 /* InferenceView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF646A9728A4DDEA006741D8 /* InferenceView.swift */; }; BF646A9F28A4DDEA006741D8 /* AudioClassificationHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF646A9928A4DDEA006741D8 /* AudioClassificationHelper.swift */; }; BF646AA028A4DDEA006741D8 /* yamnet.tflite in Resources */ = {isa = PBXBuildFile; fileRef = BF646A9A28A4DDEA006741D8 /* yamnet.tflite */; }; BF646AA128A4DDEA006741D8 /* speech_commands.tflite in Resources */ = {isa = PBXBuildFile; fileRef = BF646A9B28A4DDEA006741D8 /* speech_commands.tflite */; }; BF646AA228A4DDEA006741D8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = BF646A9C28A4DDEA006741D8 /* Assets.xcassets */; }; BF646AA328A4DDEA006741D8 /* ResultTableViewCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF646A9D28A4DDEA006741D8 /* ResultTableViewCell.swift */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ 816D964E1C100FB246C3D6A2 /* Pods-AudioClassification.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-AudioClassification.release.xcconfig"; path = "Target Support Files/Pods-AudioClassification/Pods-AudioClassification.release.xcconfig"; sourceTree = ""; }; 8CB90CD43F9B8F013EA69E62 /* Pods_AudioClassification.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_AudioClassification.framework; sourceTree = BUILT_PRODUCTS_DIR; }; BF646A7128A4DBD7006741D8 /* AudioClassification.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = AudioClassification.app; sourceTree = BUILT_PRODUCTS_DIR; }; BF646A7428A4DBD7006741D8 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; BF646A7828A4DBD7006741D8 /* HomeViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HomeViewController.swift; sourceTree = ""; }; BF646A7B28A4DBD7006741D8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; BF646A8028A4DBD9006741D8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; BF646A8228A4DBD9006741D8 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; BF646A9728A4DDEA006741D8 /* InferenceView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = InferenceView.swift; sourceTree = ""; }; BF646A9928A4DDEA006741D8 /* AudioClassificationHelper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioClassificationHelper.swift; sourceTree = ""; }; BF646A9A28A4DDEA006741D8 /* yamnet.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = yamnet.tflite; sourceTree = ""; }; BF646A9B28A4DDEA006741D8 /* speech_commands.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = speech_commands.tflite; sourceTree = ""; }; BF646A9C28A4DDEA006741D8 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; BF646A9D28A4DDEA006741D8 /* ResultTableViewCell.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ResultTableViewCell.swift; sourceTree = ""; }; DA7425FEF4C4974DA750B8B2 /* Pods-AudioClassification.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-AudioClassification.debug.xcconfig"; path = "Target Support Files/Pods-AudioClassification/Pods-AudioClassification.debug.xcconfig"; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ BF646A6E28A4DBD7006741D8 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( 9E2DC7A5D9D560890EC07075 /* Pods_AudioClassification.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ 5D022BF2F241C787DF24AA65 /* Frameworks */ = { isa = PBXGroup; children = ( 8CB90CD43F9B8F013EA69E62 /* Pods_AudioClassification.framework */, ); name = Frameworks; sourceTree = ""; }; BF646A6828A4DBD7006741D8 = { isa = PBXGroup; children = ( BF646A7328A4DBD7006741D8 /* AudioClassification */, BF646A7228A4DBD7006741D8 /* Products */, F700903006C2F56FB9F961AD /* Pods */, 5D022BF2F241C787DF24AA65 /* Frameworks */, ); sourceTree = ""; }; BF646A7228A4DBD7006741D8 /* Products */ = { isa = PBXGroup; children = ( BF646A7128A4DBD7006741D8 /* AudioClassification.app */, ); name = Products; sourceTree = ""; }; BF646A7328A4DBD7006741D8 /* AudioClassification */ = { isa = PBXGroup; children = ( BF646A9828A4DDEA006741D8 /* TFLite */, BF646A7828A4DBD7006741D8 /* HomeViewController.swift */, BF646A9728A4DDEA006741D8 /* InferenceView.swift */, BF646A9D28A4DDEA006741D8 /* ResultTableViewCell.swift */, BF646A7428A4DBD7006741D8 /* AppDelegate.swift */, BF646A7A28A4DBD7006741D8 /* Main.storyboard */, BF646A7F28A4DBD9006741D8 /* LaunchScreen.storyboard */, BF646A8228A4DBD9006741D8 /* Info.plist */, BF646A9C28A4DDEA006741D8 /* Assets.xcassets */, ); path = AudioClassification; sourceTree = ""; }; BF646A9828A4DDEA006741D8 /* TFLite */ = { isa = PBXGroup; children = ( BF646A9928A4DDEA006741D8 /* AudioClassificationHelper.swift */, BF646A9A28A4DDEA006741D8 /* yamnet.tflite */, BF646A9B28A4DDEA006741D8 /* speech_commands.tflite */, ); path = TFLite; sourceTree = ""; }; F700903006C2F56FB9F961AD /* Pods */ = { isa = PBXGroup; children = ( DA7425FEF4C4974DA750B8B2 /* Pods-AudioClassification.debug.xcconfig */, 816D964E1C100FB246C3D6A2 /* Pods-AudioClassification.release.xcconfig */, ); path = Pods; sourceTree = ""; }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ BF646A7028A4DBD7006741D8 /* AudioClassification */ = { isa = PBXNativeTarget; buildConfigurationList = BF646A8528A4DBD9006741D8 /* Build configuration list for PBXNativeTarget "AudioClassification" */; buildPhases = ( C98995D72BE0711017D783BE /* [CP] Check Pods Manifest.lock */, BF2F021F28A4E08500B948E3 /* ShellScript */, BF646A6D28A4DBD7006741D8 /* Sources */, BF646A6E28A4DBD7006741D8 /* Frameworks */, BF646A6F28A4DBD7006741D8 /* Resources */, ); buildRules = ( ); dependencies = ( ); name = AudioClassification; productName = AudioClassification; productReference = BF646A7128A4DBD7006741D8 /* AudioClassification.app */; productType = "com.apple.product-type.application"; }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ BF646A6928A4DBD7006741D8 /* Project object */ = { isa = PBXProject; attributes = { BuildIndependentTargetsInParallel = 1; LastSwiftUpdateCheck = 1340; LastUpgradeCheck = 1340; TargetAttributes = { BF646A7028A4DBD7006741D8 = { CreatedOnToolsVersion = 13.4.1; }; }; }; buildConfigurationList = BF646A6C28A4DBD7006741D8 /* Build configuration list for PBXProject "AudioClassification" */; compatibilityVersion = "Xcode 13.0"; developmentRegion = en; hasScannedForEncodings = 0; knownRegions = ( en, Base, ); mainGroup = BF646A6828A4DBD7006741D8; productRefGroup = BF646A7228A4DBD7006741D8 /* Products */; projectDirPath = ""; projectRoot = ""; targets = ( BF646A7028A4DBD7006741D8 /* AudioClassification */, ); }; /* End PBXProject section */ /* Begin PBXResourcesBuildPhase section */ BF646A6F28A4DBD7006741D8 /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( BF646A8128A4DBD9006741D8 /* LaunchScreen.storyboard in Resources */, BF646AA128A4DDEA006741D8 /* speech_commands.tflite in Resources */, BF646AA028A4DDEA006741D8 /* yamnet.tflite in Resources */, BF646A7C28A4DBD7006741D8 /* Main.storyboard in Resources */, BF646AA228A4DDEA006741D8 /* Assets.xcassets in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ BF2F021F28A4E08500B948E3 /* ShellScript */ = { isa = PBXShellScriptBuildPhase; alwaysOutOfDate = 1; buildActionMask = 2147483647; files = ( ); inputFileListPaths = ( ); inputPaths = ( ); outputFileListPaths = ( ); outputPaths = ( ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "# Type a script or drag a script file from your workspace to insert its path.\n\"$SRCROOT/RunScripts/download_models.sh\"\n"; }; C98995D72BE0711017D783BE /* [CP] Check Pods Manifest.lock */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; files = ( ); inputFileListPaths = ( ); inputPaths = ( "${PODS_PODFILE_DIR_PATH}/Podfile.lock", "${PODS_ROOT}/Manifest.lock", ); name = "[CP] Check Pods Manifest.lock"; outputFileListPaths = ( ); outputPaths = ( "$(DERIVED_FILE_DIR)/Pods-AudioClassification-checkManifestLockResult.txt", ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ BF646A6D28A4DBD7006741D8 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( BF646A9E28A4DDEA006741D8 /* InferenceView.swift in Sources */, BF646A7928A4DBD7006741D8 /* HomeViewController.swift in Sources */, BF646A7528A4DBD7006741D8 /* AppDelegate.swift in Sources */, BF646AA328A4DDEA006741D8 /* ResultTableViewCell.swift in Sources */, BF646A9F28A4DDEA006741D8 /* AudioClassificationHelper.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXSourcesBuildPhase section */ /* Begin PBXVariantGroup section */ BF646A7A28A4DBD7006741D8 /* Main.storyboard */ = { isa = PBXVariantGroup; children = ( BF646A7B28A4DBD7006741D8 /* Base */, ); name = Main.storyboard; sourceTree = ""; }; BF646A7F28A4DBD9006741D8 /* LaunchScreen.storyboard */ = { isa = PBXVariantGroup; children = ( BF646A8028A4DBD9006741D8 /* Base */, ); name = LaunchScreen.storyboard; sourceTree = ""; }; /* End PBXVariantGroup section */ /* Begin XCBuildConfiguration section */ BF646A8328A4DBD9006741D8 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; CLANG_ANALYZER_NONNULL = YES; CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; CLANG_ENABLE_MODULES = YES; CLANG_ENABLE_OBJC_ARC = YES; CLANG_ENABLE_OBJC_WEAK = YES; CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; CLANG_WARN_BOOL_CONVERSION = YES; CLANG_WARN_COMMA = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; CLANG_WARN_DOCUMENTATION_COMMENTS = YES; CLANG_WARN_EMPTY_BODY = YES; CLANG_WARN_ENUM_CONVERSION = YES; CLANG_WARN_INFINITE_RECURSION = YES; CLANG_WARN_INT_CONVERSION = YES; CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; CLANG_WARN_STRICT_PROTOTYPES = YES; CLANG_WARN_SUSPICIOUS_MOVE = YES; CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; COPY_PHASE_STRIP = NO; DEBUG_INFORMATION_FORMAT = dwarf; ENABLE_STRICT_OBJC_MSGSEND = YES; ENABLE_TESTABILITY = YES; GCC_C_LANGUAGE_STANDARD = gnu11; GCC_DYNAMIC_NO_PIC = NO; GCC_NO_COMMON_BLOCKS = YES; GCC_OPTIMIZATION_LEVEL = 0; GCC_PREPROCESSOR_DEFINITIONS = ( "DEBUG=1", "$(inherited)", ); GCC_WARN_64_TO_32_BIT_CONVERSION = YES; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; GCC_WARN_UNDECLARED_SELECTOR = YES; GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; IPHONEOS_DEPLOYMENT_TARGET = 15.5; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; SDKROOT = iphoneos; SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; }; name = Debug; }; BF646A8428A4DBD9006741D8 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; CLANG_ANALYZER_NONNULL = YES; CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; CLANG_ENABLE_MODULES = YES; CLANG_ENABLE_OBJC_ARC = YES; CLANG_ENABLE_OBJC_WEAK = YES; CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; CLANG_WARN_BOOL_CONVERSION = YES; CLANG_WARN_COMMA = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; CLANG_WARN_DOCUMENTATION_COMMENTS = YES; CLANG_WARN_EMPTY_BODY = YES; CLANG_WARN_ENUM_CONVERSION = YES; CLANG_WARN_INFINITE_RECURSION = YES; CLANG_WARN_INT_CONVERSION = YES; CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; CLANG_WARN_STRICT_PROTOTYPES = YES; CLANG_WARN_SUSPICIOUS_MOVE = YES; CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; COPY_PHASE_STRIP = NO; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; ENABLE_NS_ASSERTIONS = NO; ENABLE_STRICT_OBJC_MSGSEND = YES; GCC_C_LANGUAGE_STANDARD = gnu11; GCC_NO_COMMON_BLOCKS = YES; GCC_WARN_64_TO_32_BIT_CONVERSION = YES; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; GCC_WARN_UNDECLARED_SELECTOR = YES; GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; IPHONEOS_DEPLOYMENT_TARGET = 15.5; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; SDKROOT = iphoneos; SWIFT_COMPILATION_MODE = wholemodule; SWIFT_OPTIMIZATION_LEVEL = "-O"; VALIDATE_PRODUCT = YES; }; name = Release; }; BF646A8628A4DBD9006741D8 /* Debug */ = { isa = XCBuildConfiguration; baseConfigurationReference = DA7425FEF4C4974DA750B8B2 /* Pods-AudioClassification.debug.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = ""; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = AudioClassification/Info.plist; INFOPLIST_KEY_NSMicrophoneUsageDescription = "This app will use the microphone to get audio input in order to classify the sound made by user."; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = org.tensorflow.lite.examples.AudioClassification; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; }; name = Debug; }; BF646A8728A4DBD9006741D8 /* Release */ = { isa = XCBuildConfiguration; baseConfigurationReference = 816D964E1C100FB246C3D6A2 /* Pods-AudioClassification.release.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = ""; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_FILE = AudioClassification/Info.plist; INFOPLIST_KEY_NSMicrophoneUsageDescription = "This app will use the microphone to get audio input in order to classify the sound made by user."; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchStoryboardName = LaunchScreen; INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = org.tensorflow.lite.examples.AudioClassification; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; }; name = Release; }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ BF646A6C28A4DBD7006741D8 /* Build configuration list for PBXProject "AudioClassification" */ = { isa = XCConfigurationList; buildConfigurations = ( BF646A8328A4DBD9006741D8 /* Debug */, BF646A8428A4DBD9006741D8 /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; BF646A8528A4DBD9006741D8 /* Build configuration list for PBXNativeTarget "AudioClassification" */ = { isa = XCConfigurationList; buildConfigurations = ( BF646A8628A4DBD9006741D8 /* Debug */, BF646A8728A4DBD9006741D8 /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; /* End XCConfigurationList section */ }; rootObject = BF646A6928A4DBD7006741D8 /* Project object */; } ================================================ FILE: lite/examples/audio_classification/ios/Podfile ================================================ # Uncomment the next line to define a global platform for your project # platform :ios, '9.0' target 'AudioClassification' do # Comment the next line if you don't want to use dynamic frameworks use_frameworks! pod 'TensorFlowLiteTaskAudio', '~> 0.0.1-nightly' # Pods for AudioClassification end ================================================ FILE: lite/examples/audio_classification/ios/README.md ================================================ # TensorFlow Lite audio classification iOS example application ## Overview This is an example application for [TensorFlow Lite](https://tensorflow.org/lite) on iOS. ### Model The model will be downloaded as part of the build process. There are two models included in this sample app: * [YAMNet](https://tfhub.dev/google/lite-model/yamnet/classification/tflite/1) is a general purpose audio classification model that can detects 521 different type of sounds. * [Speech command](https://www.tensorflow.org/lite/models/modify/model_maker/speech_recognition) is a demonstrative model that can recognize a handful of single-word audio command. Also, you can use your own model generated on [Teachable Machine](https://teachablemachine.withgoogle.com/train/audio) or [Model Maker](https://www.tensorflow.org/lite/models/modify/model_maker/audio_classification). ### iOS app details The app is written entirely in Swift and uses the TensorFlow Lite [Swift library](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/swift) for performing sound classification. ## Requirements * Device with iOS 12.0 or above * Xcode 13.0 or above * Xcode command-line tools (run `xcode-select --install`) * [CocoaPods](https://cocoapods.org/) (run `sudo gem install cocoapods`) * (Optional) Valid Apple Developer ID. If you don't have one, you can run the sample app on an iOS Simulator. If this is a new install, you will need to run the Xcode application once to agree to the license before continuing. ## Build and run 1. Clone this GitHub repository to your workstation. `git clone https://github.com/tensorflow/examples.git` 2. Install the pod to generate the workspace file: `cd examples/lite/examples/sound_classification/ios && pod install` Note: If you have installed this pod before and that command doesn't work, try `pod update`. At the end of this step you should have a directory called `AudioClassification.xcworkspace`. 1. Open the project in Xcode with the following command: `open AudioClassification.xcworkspace` This launches Xcode and opens the `AudioClassification` project. You can run the app on an iOS Sumi Follow these steps to run the sample app on a physical device. 1. Select the `AudioClassification` project in the left hand navigation to open the project configuration. In the **Signing** section of the **General** tab, select your development team from the dropdown. 2. In order to build the project, you must modify the **Bundle Identifier** in the **Identity** section so that it is unique across all Xcode projects. To create a unique identifier, try adding your initials and a number to the end of the string. 3. With an iOS device connected, build and run the app in Xcode. You'll have to grant permissions for the app to use the device's camera. Point the camera at various objects and enjoy seeing how the model classifies things! ## Model references *Do not delete the empty references* to the .tflite files after you clone the repo and open the project. These references will be fulfilled once the model and label files are downloaded when the application is built and run for the first time. If you delete the references to them, you can still find that the .tflite and .txt files are downloaded to the Model folder, the next time you build the application. You will have to add the references to these files in the bundle separately in that case. ================================================ FILE: lite/examples/audio_classification/ios/RunScripts/download_models.sh ================================================ #!/bin/bash # Copyright 2019 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== set -ex SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" YAMNET_URL="https://storage.googleapis.com/download.tensorflow.org/models/tflite/task_library/audio_classification/ios/lite-model_yamnet_classification_tflite_1.tflite" SPEECH_COMMANDS_URL="https://storage.googleapis.com/download.tensorflow.org/models/tflite/task_library/audio_classification/ios/speech_commands.tflite" YAMNET_NAME="yamnet.tflite" SPEECH_COMMANDS_NAME="speech_commands.tflite" DOWNLOADS_DIR=$(mktemp -d) cd "$SCRIPT_DIR" download() { local usage="Usage: download_and_extract URL DIR" local url="${1:?${usage}}" local dir="${2:?${usage}}" local name="${3:?${usage}}" echo "downloading ${url}" >&2 mkdir -p "${dir}" tempdir=$(mktemp -d) curl -L ${url} > ${tempdir}/${name} cp -R ${tempdir}/* ${dir}/ rm -rf ${tempdir} } has_download=false if [ -f ../AudioClassification/TFLite/${YAMNET_NAME} ] then echo "File ${YAMNET_NAME} exists." else has_download=true download "${YAMNET_URL}" "${DOWNLOADS_DIR}/models" "${YAMNET_NAME}" file ${DOWNLOADS_DIR}/models fi if [ -f ../AudioClassification/TFLite/${SPEECH_COMMANDS_NAME} ] then echo "File ${SPEECH_COMMANDS_NAME} exists." else has_download=true download "${SPEECH_COMMANDS_URL}" "${DOWNLOADS_DIR}/models" "${SPEECH_COMMANDS_NAME}" file ${DOWNLOADS_DIR}/models fi if ${has_download} then cp ${DOWNLOADS_DIR}/models/* ../AudioClassification/TFLite rm -rf ${DOWNLOADS_DIR} fi ================================================ FILE: lite/examples/audio_classification/raspberry_pi/README.md ================================================ # TensorFlow Lite Python audio classification example with Raspberry Pi. This example uses [TensorFlow Lite](https://tensorflow.org/lite) with Python on a Raspberry Pi to perform real-time audio classification using audio streamed from the microphone. At the end of this page, there are extra steps to accelerate the example using the Coral USB Accelerator to increase inference speed. ## Set up your hardware Before you begin, you need to [set up your Raspberry Pi](https://projects.raspberrypi.org/en/projects/raspberry-pi-setting-up) with Raspberry Pi OS (preferably updated to Buster). Raspberry Pi doesn't have a microphone integrated on its board, so you need to plug in a USB microphone to record audio. ## Install the TensorFlow Lite runtime In this project, all you need from the TensorFlow Lite API is the `Interpreter` class. So instead of installing the large `tensorflow` package, we're using the much smaller `tflite_runtime` package. To install this on your Raspberry Pi, follow the instructions in the [Python quickstart](https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python). You can install the TFLite runtime using this script. ``` sh setup.sh ``` ## Download the example files First, clone this Git repo onto your Raspberry Pi like this: ``` git clone https://github.com/tensorflow/examples.git --depth 1 ``` Then use our script to install a couple Python packages, and download the TFLite model: ``` cd lite/examples/audio_classification/raspberry_pi # Run this script to install the required dependencies and download the TFLite models. sh setup.sh ``` ## Run the example ``` python3 classify.py ``` * You can optionally specify the `model` parameter to set the TensorFlow Lite model to be used: * The default value is `yamnet.tflite` * You can optionally specify the `maxResults` parameter to limit the list of classification results: * Supported value: A positive integer. * Default value: `5`. * Example usage: ``` python3 classify.py \ --model yamnet.tflite \ --maxResults 5 ``` ## Speed up the inference time (optional) If you want to speed up the inference time, you can attach an ML accelerator such as the [Coral USB Accelerator](https://coral.withgoogle.com/products/accelerator)—a USB accessory that adds the [Edge TPU ML accelerator](https://coral.withgoogle.com/docs/edgetpu/faq/) to any Linux-based system. If you have a Coral USB Accelerator, you can run the sample with it enabled: 1. First, be sure you have completed the [USB Accelerator setup instructions](https://coral.withgoogle.com/docs/accelerator/get-started/). 2. Run the audio classification script using the Edge TPU TFLite model and enable the Edge TPU option. ``` python3 classify.py \ --model yamnet_edgetpu.tflite \ --enableEdgeTPU ``` For more information about creating and running TensorFlow Lite models with Coral devices, read [TensorFlow models on the Edge TPU](https://coral.withgoogle.com/docs/edgetpu/models-intro/). For more information about executing inferences with TensorFlow Lite, read [TensorFlow Lite inference](https://www.tensorflow.org/lite/guide/inference). ================================================ FILE: lite/examples/audio_classification/raspberry_pi/classify.py ================================================ # Copyright 2021 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Main scripts to run audio classification.""" import argparse import time from tflite_support.task import audio from tflite_support.task import core from tflite_support.task import processor from utils import Plotter def run(model: str, max_results: int, score_threshold: float, overlapping_factor: float, num_threads: int, enable_edgetpu: bool) -> None: """Continuously run inference on audio data acquired from the device. Args: model: Name of the TFLite audio classification model. max_results: Maximum number of classification results to display. score_threshold: The score threshold of classification results. overlapping_factor: Target overlapping between adjacent inferences. num_threads: Number of CPU threads to run the model. enable_edgetpu: Whether to run the model on EdgeTPU. """ if (overlapping_factor <= 0) or (overlapping_factor >= 1.0): raise ValueError('Overlapping factor must be between 0 and 1.') if (score_threshold < 0) or (score_threshold > 1.0): raise ValueError('Score threshold must be between (inclusive) 0 and 1.') # Initialize the audio classification model. base_options = core.BaseOptions( file_name=model, use_coral=enable_edgetpu, num_threads=num_threads) classification_options = processor.ClassificationOptions( max_results=max_results, score_threshold=score_threshold) options = audio.AudioClassifierOptions( base_options=base_options, classification_options=classification_options) classifier = audio.AudioClassifier.create_from_options(options) # Initialize the audio recorder and a tensor to store the audio input. audio_record = classifier.create_audio_record() tensor_audio = classifier.create_input_tensor_audio() # We'll try to run inference every interval_between_inference seconds. # This is usually half of the model's input length to create an overlapping # between incoming audio segments to improve classification accuracy. input_length_in_second = float(len( tensor_audio.buffer)) / tensor_audio.format.sample_rate interval_between_inference = input_length_in_second * (1 - overlapping_factor) pause_time = interval_between_inference * 0.1 last_inference_time = time.time() # Initialize a plotter instance to display the classification results. plotter = Plotter() # Start audio recording in the background. audio_record.start_recording() # Loop until the user close the classification results plot. while True: # Wait until at least interval_between_inference seconds has passed since # the last inference. now = time.time() diff = now - last_inference_time if diff < interval_between_inference: time.sleep(pause_time) continue last_inference_time = now # Load the input audio and run classify. tensor_audio.load_from_audio_record(audio_record) result = classifier.classify(tensor_audio) # Plot the classification results. plotter.plot(result) def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--model', help='Name of the audio classification model.', required=False, default='yamnet.tflite') parser.add_argument( '--maxResults', help='Maximum number of results to show.', required=False, default=5) parser.add_argument( '--overlappingFactor', help='Target overlapping between adjacent inferences. Value must be in (0, 1)', required=False, default=0.5) parser.add_argument( '--scoreThreshold', help='The score threshold of classification results.', required=False, default=0.0) parser.add_argument( '--numThreads', help='Number of CPU threads to run the model.', required=False, default=4) parser.add_argument( '--enableEdgeTPU', help='Whether to run the model on EdgeTPU.', action='store_true', required=False, default=False) args = parser.parse_args() run(args.model, int(args.maxResults), float(args.scoreThreshold), float(args.overlappingFactor), int(args.numThreads), bool(args.enableEdgeTPU)) if __name__ == '__main__': main() ================================================ FILE: lite/examples/audio_classification/raspberry_pi/requirements.txt ================================================ argparse matplotlib>=3.2.1 sounddevice>=0.4.1 scipy>=1.7.3 # Only required for test code. tflite-support>=0.4.2 ================================================ FILE: lite/examples/audio_classification/raspberry_pi/setup.sh ================================================ #!/bin/bash if [ $# -eq 0 ]; then DATA_DIR="./" else DATA_DIR="$1" fi # Install Python dependencies python3 -m pip install -r requirements.txt # Download TF Lite models with metadata. FILE=${DATA_DIR}/yamnet.tflite if [ ! -f "$FILE" ]; then curl \ -L 'https://storage.googleapis.com/download.tensorflow.org/models/tflite/task_library/audio_classification/rpi/lite-model_yamnet_classification_tflite_1.tflite' \ -o ${FILE} fi FILE=${DATA_DIR}/yamnet_edgetpu.tflite if [ ! -f "$FILE" ]; then curl \ -L 'https://storage.googleapis.com/download.tensorflow.org/models/tflite/task_library/audio_classification/rpi/coral-model_yamnet_classification_coral_1.tflite' \ -o ${FILE} fi echo -e "Downloaded files are in ${DATA_DIR}" ================================================ FILE: lite/examples/audio_classification/raspberry_pi/test_data/ground_truth.csv ================================================ label,score Cat,0.73828125 Animal,0.66796875 "Domestic animals, pets",0.66796875 Meow,0.4140625 Caterwaul,0.33203125 ================================================ FILE: lite/examples/audio_classification/raspberry_pi/utils.py ================================================ # Copyright 2021 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """A module with util functions.""" import sys from matplotlib import rcParams import matplotlib.pyplot as plt rcParams.update({ # Set the plot left margin so that the labels are visible. 'figure.subplot.left': 0.3, # Hide the bottom toolbar. 'toolbar': 'None' }) class Plotter(object): """An util class to display the classification results.""" _PAUSE_TIME = 0.05 """Time for matplotlib to wait for UI event.""" def __init__(self) -> None: fig, self._axes = plt.subplots() fig.canvas.manager.set_window_title('Audio classification') # Stop the program when the ESC key is pressed. def event_callback(event): if event.key == 'escape': sys.exit(0) fig.canvas.mpl_connect('key_press_event', event_callback) plt.show(block=False) # TODO(khanhlvg): Add type hint for result once ClassificationResult added # to tflite_support.task.processor module def plot(self, result) -> None: """Plot the audio classification result. Args: result: Classification results returned by an audio classification model. """ # Clear the axes self._axes.cla() self._axes.set_title('Press ESC to exit.') self._axes.set_xlim((0, 1)) # Plot the results so that the most probable category comes at the top. classification = result.classifications[0] label_list = [ category.category_name for category in classification.categories ] score_list = [category.score for category in classification.categories] self._axes.barh(label_list[::-1], score_list[::-1]) # Wait for the UI event. plt.pause(self._PAUSE_TIME) ================================================ FILE: lite/examples/bert_qa/android/README.md ================================================ # TensorFlow Lite BERT Question & Answer Demo Application ### Overview This is an end-to-end example of BERT Question & Answer application built with TensorFlow 2.0, and tested on SQuAD dataset. The demo app provides 48 passages from the dataset for users to choose from, and gives 5 most possible answers corresponding to the input passage and query. These instructions walk you through building and running the demo on an Android device. The model files are downloaded via Gradle scripts when you build and run the app. You don't need to do any steps to download TFLite models into the project explicitly. This application should be run on a physical Android device. ![App example UI.](screenshot.png?raw=true "Screenshot QA screen") ## Build the demo using Android Studio ### Prerequisites * The **[Android Studio](https://developer.android.com/studio/index.html)** IDE (Android Studio 2021.2.1 or newer). This sample has been tested on Android Studio Chipmunk. * A physical Android device with a minimum OS version of SDK 23 (Android 6.0 - Marshmallow) with developer mode enabled. The process of enabling developer mode may vary by device. ### Building * Open Android Studio. From the Welcome screen, select Open an existing Android Studio project. * From the Open File or Project window that appears, navigate to and select the tensorflow-lite/examples/bert_qa/android directory. Click OK. * If it asks you to do a Gradle Sync, click OK. * With your Android device connected to your computer and developer mode enabled, click on the green Run arrow in Android Studio. ### Models used Downloading, extraction, and placing the models into the assets folder is managed automatically by the download.gradle file. ================================================ FILE: lite/examples/bert_qa/android/app/build.gradle ================================================ plugins { id 'com.android.application' id 'org.jetbrains.kotlin.android' id 'kotlin-kapt' id 'androidx.navigation.safeargs' id 'de.undercouch.download' } android { compileSdk 32 defaultConfig { applicationId "org.tensorflow.lite.examples.bertqa" minSdk 23 targetSdk 32 versionCode 1 versionName "1.0" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } buildTypes { release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' } } compileOptions { sourceCompatibility JavaVersion.VERSION_1_8 targetCompatibility JavaVersion.VERSION_1_8 } kotlinOptions { jvmTarget = '1.8' } buildFeatures { viewBinding true } androidResources { noCompress 'tflite' } } // import DownloadModels task project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets' project.ext.TEST_ASSETS_DIR = projectDir.toString() + '/src/androidTest/assets' // Download default models; if you wish to use your own models then // place them in the "assets" directory and comment out this line. apply from: 'download_models.gradle' dependencies { // Kotlin lang implementation 'androidx.core:core-ktx:1.15.0' // App compat and UI things implementation 'androidx.appcompat:appcompat:1.4.2' implementation 'com.google.android.material:material:1.6.1' implementation 'androidx.constraintlayout:constraintlayout:2.1.4' // Navigation library def nav_version = "2.5.0" implementation "androidx.navigation:navigation-fragment-ktx:$nav_version" implementation "androidx.navigation:navigation-ui-ktx:$nav_version" // CameraX core library def camerax_version = '1.2.0-alpha03' implementation "androidx.camera:camera-core:$camerax_version" // CameraX Camera2 extensions implementation "androidx.camera:camera-camera2:$camerax_version" // CameraX Lifecycle library implementation "androidx.camera:camera-lifecycle:$camerax_version" // CameraX View class implementation "androidx.camera:camera-view:$camerax_version" //WindowManager implementation 'androidx.window:window:1.1.0-alpha02' // Unit testing testImplementation 'junit:junit:4.13.2' // Instrumented testing androidTestImplementation 'androidx.test.ext:junit:1.1.3' androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0' // Gson library implementation 'com.google.code.gson:gson:2.9.0' implementation 'com.google.guava:guava:28.1-android' // Import tensorflow library implementation 'org.tensorflow:tensorflow-lite-task-text:0.3.0' // Import the GPU delegate plugin Library for GPU inference implementation 'org.tensorflow:tensorflow-lite-gpu-delegate-plugin:0.4.0' implementation 'org.tensorflow:tensorflow-lite-gpu:2.9.0' } ================================================ FILE: lite/examples/bert_qa/android/app/download_models.gradle ================================================ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ task downloadModelFile(type: Download) { src 'https://storage.googleapis.com/download.tensorflow.org/models/tflite/task_library/bert_qa/android/models_tflite_task_library_bert_qa_lite-model_mobilebert_1_metadata_1.tflite' dest project.ext.ASSET_DIR + '/mobilebert.tflite' overwrite false } task downloadQAJson(type: Download) { src 'https://storage.googleapis.com/download.tensorflow.org/models/tflite/bert_qa/contents_from_squad.json' dest project.ext.ASSET_DIR + '/qa.json' overwrite false } task copyTestModel(type: Copy, dependsOn: downloadModelFile) { from project.ext.ASSET_DIR + '/mobilebert.tflite' into project.ext.TEST_ASSETS_DIR } preBuild.dependsOn downloadModelFile, downloadQAJson, copyTestModel ================================================ FILE: lite/examples/bert_qa/android/app/proguard-rules.pro ================================================ # Add project specific ProGuard rules here. # You can control the set of applied configuration files using the # proguardFiles setting in build.gradle. # # For more details, see # http://developer.android.com/guide/developing/tools/proguard.html # If your project uses WebView with JS, uncomment the following # and specify the fully qualified class name to the JavaScript interface # class: #-keepclassmembers class fqcn.of.javascript.interface.for.webview { # public *; #} # Uncomment this to preserve the line number information for # debugging stack traces. #-keepattributes SourceFile,LineNumberTable # If you keep the line number information, uncomment this to # hide the original source file name. #-renamesourcefileattribute SourceFile ================================================ FILE: lite/examples/bert_qa/android/app/src/androidTest/java/org/tensorflow/lite/examples/bertqa/BertQaHelperTest.kt ================================================ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.tensorflow.lite.examples.bertqa import androidx.test.platform.app.InstrumentationRegistry import androidx.test.ext.junit.runners.AndroidJUnit4 import org.junit.Test import org.junit.runner.RunWith import org.junit.Assert.* import org.junit.Before import org.tensorflow.lite.task.text.qa.QaAnswer /** * Instrumented test, which will execute on an Android device. * * See [testing documentation](http://d.android.com/tools/testing). */ @RunWith(AndroidJUnit4::class) class BertQaHelperTest { private val content = "Doraemon is a blue cat automaton corresponding (tints of pink-orange in earlier comic chapters and media) from the 22nd century, who weighs 129.3kg (285.05lbs) and measures at 129.3cm (4'3\") tall." private val question = "What color is Doraemon?" private val answer = "blue" private lateinit var helper: BertQaHelper @Test fun bertAnswersShouldNotChange() { helper = BertQaHelper(context = InstrumentationRegistry.getInstrumentation().context, answererListener = object : BertQaHelper.AnswererListener { override fun onError(error: String) { // no op } override fun onResults(results: List?, inferenceTime: Long) { assert(!results.isNullOrEmpty()) // verify bert qa answer and expected answer. assertEquals(answer, results!![0].text) } }) helper.answer(content, question) } } ================================================ FILE: lite/examples/bert_qa/android/app/src/main/AndroidManifest.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/java/org/tensorflow/lite/examples/bertqa/BertQaHelper.kt ================================================ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.tensorflow.lite.examples.bertqa import android.content.Context import android.os.SystemClock import android.util.Log import org.tensorflow.lite.gpu.CompatibilityList import org.tensorflow.lite.task.core.BaseOptions import org.tensorflow.lite.task.text.qa.BertQuestionAnswerer import org.tensorflow.lite.task.text.qa.BertQuestionAnswerer.BertQuestionAnswererOptions import org.tensorflow.lite.task.text.qa.QaAnswer import java.lang.IllegalStateException class BertQaHelper( val context: Context, var numThreads: Int = 2, var currentDelegate: Int = 0, val answererListener: AnswererListener? ) { private var bertQuestionAnswerer: BertQuestionAnswerer? = null init { setupBertQuestionAnswerer() } fun clearBertQuestionAnswerer() { bertQuestionAnswerer = null } private fun setupBertQuestionAnswerer() { val baseOptionsBuilder = BaseOptions.builder().setNumThreads(numThreads) when (currentDelegate) { DELEGATE_CPU -> { // Default } DELEGATE_GPU -> { if (CompatibilityList().isDelegateSupportedOnThisDevice) { baseOptionsBuilder.useGpu() } else { answererListener?.onError("GPU is not supported on this device") } } DELEGATE_NNAPI -> { baseOptionsBuilder.useNnapi() } } val options = BertQuestionAnswererOptions.builder() .setBaseOptions(baseOptionsBuilder.build()) .build() try { bertQuestionAnswerer = BertQuestionAnswerer.createFromFileAndOptions(context, BERT_QA_MODEL, options) } catch (e: IllegalStateException) { answererListener ?.onError("Bert Question Answerer failed to initialize. See error logs for details") Log.e(TAG, "TFLite failed to load model with error: " + e.message) } } fun answer(contextOfQuestion: String, question: String) { if (bertQuestionAnswerer == null) { setupBertQuestionAnswerer() } // Inference time is the difference between the system time at the start and finish of the // process var inferenceTime = SystemClock.uptimeMillis() val answers = bertQuestionAnswerer?.answer(contextOfQuestion, question) inferenceTime = SystemClock.uptimeMillis() - inferenceTime answererListener?.onResults(answers, inferenceTime) } interface AnswererListener { fun onError(error: String) fun onResults( results: List?, inferenceTime: Long ) } companion object { private const val BERT_QA_MODEL = "mobilebert.tflite" private const val TAG = "BertQaHelper" const val DELEGATE_CPU = 0 const val DELEGATE_GPU = 1 const val DELEGATE_NNAPI = 2 } } ================================================ FILE: lite/examples/bert_qa/android/app/src/main/java/org/tensorflow/lite/examples/bertqa/MainActivity.kt ================================================ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.tensorflow.lite.examples.bertqa import android.os.Bundle import androidx.appcompat.app.AppCompatActivity import androidx.navigation.findNavController import androidx.navigation.fragment.NavHostFragment import androidx.navigation.ui.AppBarConfiguration import androidx.navigation.ui.navigateUp import androidx.navigation.ui.setupActionBarWithNavController import org.tensorflow.lite.examples.bertqa.databinding.ActivityMainBinding class MainActivity : AppCompatActivity() { private lateinit var binding: ActivityMainBinding private lateinit var appBarConfiguration: AppBarConfiguration override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) binding = ActivityMainBinding.inflate(layoutInflater) setContentView(binding.root) // setup toolbar setSupportActionBar(binding.toolbar) val navHostFragment = (supportFragmentManager.findFragmentById(R.id.fragment_container) as NavHostFragment) val navController = navHostFragment.navController appBarConfiguration = AppBarConfiguration(navController.graph) setupActionBarWithNavController(navController, appBarConfiguration) supportActionBar?.setDisplayShowTitleEnabled(false) } override fun onSupportNavigateUp(): Boolean { val navController = findNavController(R.id.fragment_container) return navController.navigateUp(appBarConfiguration) || super.onSupportNavigateUp() } } ================================================ FILE: lite/examples/bert_qa/android/app/src/main/java/org/tensorflow/lite/examples/bertqa/dataset/DataSet.kt ================================================ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.tensorflow.lite.examples.bertqa.dataset import com.google.gson.annotations.SerializedName data class DataSet( @SerializedName("titles") private val titles: List>, @SerializedName("contents") private val contents: List>, @SerializedName("questions") val questions: List> ) { fun getTitles(): List { return titles.map { it[0] } } fun getContents(): List { return contents.map { it[0] } } } ================================================ FILE: lite/examples/bert_qa/android/app/src/main/java/org/tensorflow/lite/examples/bertqa/dataset/LoadDataSetClient.kt ================================================ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.tensorflow.lite.examples.bertqa.dataset import android.content.Context import android.util.Log import com.google.gson.Gson import com.google.gson.reflect.TypeToken import java.io.IOException import java.io.InputStream class LoadDataSetClient(private val context: Context) { private companion object { private const val TAG = "BertAppDemo" private const val JSON_DIR = "qa.json" } // Load json file into a data object. fun loadJson(): DataSet? { var dataSet: DataSet? = null try { val inputStream: InputStream = context.assets.open(JSON_DIR) val bufferReader = inputStream.bufferedReader() val stringJson: String = bufferReader.use { it.readText() } val datasetType = object : TypeToken() {}.type dataSet = Gson().fromJson(stringJson, datasetType) } catch (e: IOException) { Log.e(TAG, e.message.toString()) } return dataSet } } ================================================ FILE: lite/examples/bert_qa/android/app/src/main/java/org/tensorflow/lite/examples/bertqa/fragments/DatasetAdapter.kt ================================================ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.tensorflow.lite.examples.bertqa.fragments import android.view.LayoutInflater import android.view.ViewGroup import androidx.recyclerview.widget.RecyclerView import org.tensorflow.lite.examples.bertqa.databinding.ItemDatasetBinding class DatasetAdapter( private val dataSetTitle: List, private val onSelected: (Int) -> Unit ) : RecyclerView.Adapter() { inner class ViewHolder(private val binding: ItemDatasetBinding) : RecyclerView.ViewHolder(binding.root) { init { binding.tvDataTitle.setOnClickListener { onSelected.invoke(adapterPosition) } } fun bind(title: String) { with(binding) { tvDataTitle.text = title } } } override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { val binding = ItemDatasetBinding.inflate( LayoutInflater.from(parent.context), parent, false ) return ViewHolder(binding) } override fun onBindViewHolder(holder: ViewHolder, position: Int) { holder.bind(dataSetTitle[position]) } override fun getItemCount() = dataSetTitle.size } ================================================ FILE: lite/examples/bert_qa/android/app/src/main/java/org/tensorflow/lite/examples/bertqa/fragments/DatasetFragment.kt ================================================ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.tensorflow.lite.examples.bertqa.fragments import android.os.Bundle import android.view.LayoutInflater import android.view.View import android.view.ViewGroup import androidx.fragment.app.Fragment import androidx.navigation.Navigation import androidx.recyclerview.widget.DividerItemDecoration import androidx.recyclerview.widget.LinearLayoutManager import org.tensorflow.lite.examples.bertqa.R import org.tensorflow.lite.examples.bertqa.databinding.FragmentDatasetBinding import org.tensorflow.lite.examples.bertqa.dataset.LoadDataSetClient class DatasetFragment : Fragment() { private var _fragmentDatasetList: FragmentDatasetBinding? = null private val fragmentDatasetList get() = _fragmentDatasetList!! private var titles : List = emptyList() override fun onCreateView( inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle? ): View { _fragmentDatasetList = FragmentDatasetBinding.inflate(inflater, container, false) return fragmentDatasetList.root } override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) val client = LoadDataSetClient(requireActivity()) client.loadJson()?.let { titles = it.getTitles() } initRecyclerView() } private fun initRecyclerView() { val dataSetAdapter = DatasetAdapter(titles) { startQaScreen(it) } val linearLayoutManager = LinearLayoutManager(requireContext()) val decoration = DividerItemDecoration( fragmentDatasetList.recyclerView.context, linearLayoutManager.orientation ) with(fragmentDatasetList.recyclerView) { layoutManager = linearLayoutManager adapter = dataSetAdapter addItemDecoration(decoration) } } private fun startQaScreen(position: Int) { val action = DatasetFragmentDirections.actionDatasetFragmentToQaFragment(position) Navigation.findNavController(requireActivity(), R.id.fragment_container) .navigate(action) } } ================================================ FILE: lite/examples/bert_qa/android/app/src/main/java/org/tensorflow/lite/examples/bertqa/fragments/QaAdapter.kt ================================================ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.tensorflow.lite.examples.bertqa.fragments import android.view.LayoutInflater import android.view.ViewGroup import androidx.recyclerview.widget.RecyclerView import org.tensorflow.lite.examples.bertqa.databinding.ItemQuestionBinding class QaAdapter(private val question: List, private val select: (Int) -> Unit) : RecyclerView.Adapter() { inner class ViewHolder(private val binding: ItemQuestionBinding) : RecyclerView.ViewHolder(binding.root) { init { binding.tvQuestionSuggestion.setOnClickListener { select.invoke(adapterPosition) } } fun bind(question: String) { binding.tvQuestionSuggestion.text = question } } override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { val binding = ItemQuestionBinding.inflate( LayoutInflater.from(parent.context), parent, false ) return ViewHolder(binding) } override fun onBindViewHolder(holder: ViewHolder, position: Int) { holder.bind(question[position]) } override fun getItemCount() = question.size } ================================================ FILE: lite/examples/bert_qa/android/app/src/main/java/org/tensorflow/lite/examples/bertqa/fragments/QaFragment.kt ================================================ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.tensorflow.lite.examples.bertqa.fragments import android.os.Bundle import android.text.Editable import android.text.Spannable import android.text.SpannableString import android.text.TextWatcher import android.text.style.BackgroundColorSpan import android.text.style.ForegroundColorSpan import android.view.LayoutInflater import android.view.View import android.view.ViewGroup import android.widget.AdapterView import android.widget.Toast import androidx.fragment.app.Fragment import androidx.navigation.fragment.navArgs import androidx.recyclerview.widget.DividerItemDecoration import androidx.recyclerview.widget.LinearLayoutManager import org.tensorflow.lite.examples.bertqa.BertQaHelper import org.tensorflow.lite.examples.bertqa.R import org.tensorflow.lite.examples.bertqa.databinding.FragmentQaBinding import org.tensorflow.lite.examples.bertqa.dataset.LoadDataSetClient import org.tensorflow.lite.task.text.qa.QaAnswer class QaFragment : Fragment(), BertQaHelper.AnswererListener { private var _fragmentQaBinding: FragmentQaBinding? = null private val fragmentQaBinding get() = _fragmentQaBinding!! private lateinit var bertQaHelper: BertQaHelper private val args: QaFragmentArgs by navArgs() private var content: String = "" private var questions: List = emptyList() override fun onCreateView( inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle? ): View { _fragmentQaBinding = FragmentQaBinding.inflate(inflater, container, false) return fragmentQaBinding.root } override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) bertQaHelper = BertQaHelper(context = requireContext(), answererListener = this) val client = LoadDataSetClient(requireActivity()) client.loadJson()?.let { content = it.getContents()[args.datasetPosition] questions = it.questions[args.datasetPosition] } fragmentQaBinding.tvDatasetContent.text = content initRecyclerView() handleListener() } private fun initRecyclerView() { val decoration = DividerItemDecoration(requireContext(), DividerItemDecoration.VERTICAL) with(fragmentQaBinding.recyclerView) { adapter = QaAdapter(questions) { setQuestion(it) } layoutManager = LinearLayoutManager(requireContext(), LinearLayoutManager.VERTICAL, false) addItemDecoration(decoration) } } private fun handleListener() { fragmentQaBinding.edtQuestion.addTextChangedListener(object : TextWatcher { override fun beforeTextChanged( charSequence: CharSequence?, start: Int, count: Int, after: Int ) { // no op } override fun onTextChanged( charSequence: CharSequence?, start: Int, before: Int, count: Int ) { // Only allow clicking Ask button if there is a question. val shouldAskButtonActive: Boolean = charSequence.toString().isNotEmpty() fragmentQaBinding.imgBtnAsk.isClickable = shouldAskButtonActive fragmentQaBinding.imgBtnAsk.setImageResource( if (shouldAskButtonActive) R.drawable.ic_ask_active else R.drawable.ic_ask_inactive ) } override fun afterTextChanged(editable: Editable?) { // no op } }) // When clicked, add the question in suggestion question list to query edittext. fragmentQaBinding.imgBtnAsk.setOnClickListener { answerQuestion(fragmentQaBinding.edtQuestion.text.toString()) } // When clicked, decrease the number of threads used for answer fragmentQaBinding.bottomSheetLayout.threadsMinus.setOnClickListener { if (bertQaHelper.numThreads > 1) { bertQaHelper.numThreads-- updateControlsUi() } } // When clicked, increase the number of threads used for answer fragmentQaBinding.bottomSheetLayout.threadsPlus.setOnClickListener { if (bertQaHelper.numThreads < 4) { bertQaHelper.numThreads++ updateControlsUi() } } // When clicked, change the underlying hardware used for inference. Current options are CPU // GPU, and NNAPI fragmentQaBinding.bottomSheetLayout.spinnerDelegate.setSelection(0, false) fragmentQaBinding.bottomSheetLayout.spinnerDelegate.onItemSelectedListener = object : AdapterView.OnItemSelectedListener { override fun onItemSelected( parent: AdapterView<*>?, view: View?, position: Int, id: Long ) { bertQaHelper.currentDelegate = position updateControlsUi() } override fun onNothingSelected(parent: AdapterView<*>?) { /* no op */ } } } // Update the values displayed in the bottom sheet. Reset answerer. private fun updateControlsUi() { fragmentQaBinding.bottomSheetLayout.threadsValue.text = bertQaHelper.numThreads.toString() // Needs to be cleared instead of reinitialized because the GPU // delegate needs to be initialized on the thread using it when applicable bertQaHelper.clearBertQuestionAnswerer() } private fun setQuestion(position: Int) { fragmentQaBinding.edtQuestion.setText(questions[position]) } private fun answerQuestion(question: String) { bertQaHelper.answer(content, question) } // Highlight the answer private fun highlightAnswer(answer: String) { val start = content.indexOf(answer) val end = start + answer.length val str = SpannableString(content) str.setSpan( BackgroundColorSpan(requireActivity().getColor(R.color.highlight_background_color)), start, end, Spannable.SPAN_EXCLUSIVE_EXCLUSIVE ) str.setSpan( ForegroundColorSpan(requireActivity().getColor(R.color.highlight_text_color)), start, end, Spannable.SPAN_EXCLUSIVE_EXCLUSIVE ) fragmentQaBinding.tvDatasetContent.text = str } override fun onError(error: String) { Toast.makeText(requireContext(), error, Toast.LENGTH_SHORT).show() } override fun onResults(results: List?, inferenceTime: Long) { results?.first()?.let { highlightAnswer(it.text) } fragmentQaBinding.tvInferenceTime.text = String.format( requireActivity().getString(R.string.bottom_view_inference_time), inferenceTime ) } override fun onDestroyView() { fragmentQaBinding.edtQuestion.addTextChangedListener(null) super.onDestroyView() } override fun onDestroy() { bertQaHelper.clearBertQuestionAnswerer() super.onDestroy() } } ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/drawable/bg_question_items.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/drawable/ic_ask_active.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/drawable/ic_ask_inactive.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/drawable/ic_launcher_background.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/drawable/ic_minus.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/drawable/ic_plus.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/layout/activity_main.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/layout/fragment_dataset.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/layout/fragment_qa.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/layout/info_bottom_sheet.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/layout/item_dataset.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/layout/item_question.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/navigation/nav_graph.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/values/colors.xml ================================================ #FF6F00 #EEEEEE #FFFFFF @android:color/black #FF6F00 #FFFFFF #FFFFFF #EEEEEE ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/values/dimens.xml ================================================ 16dp 10dp 5dp 15dp 8dp 10dp 20dp 10dp 10dp 3 16dp 50dp 16dp 48dp 10dp 160dp 240dp 20sp 200dp 3 ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/values/strings.xml ================================================ TFLite Bert QA Demo App Please select an article below. You may want to ask… Model options Bottom sheet expandable indicator Decrease the number of threads used Increase the number of threads used 2 Number of Threads Delegate Inference Time: %d ms CPU GPU NNAPI ================================================ FILE: lite/examples/bert_qa/android/app/src/main/res/values/styles.xml ================================================ ================================================ FILE: lite/examples/bert_qa/android/build.gradle ================================================ // Top-level build file where you can add configuration options common to all sub-projects/modules. buildscript { dependencies { classpath 'androidx.navigation:navigation-safe-args-gradle-plugin:2.5.0' classpath 'de.undercouch:gradle-download-task:4.1.2' } } plugins { id 'com.android.application' version '7.2.0' apply false id 'com.android.library' version '7.2.0' apply false id 'org.jetbrains.kotlin.android' version '1.6.21' apply false } task clean(type: Delete) { delete rootProject.buildDir } ================================================ FILE: lite/examples/bert_qa/android/gradle/wrapper/gradle-wrapper.properties ================================================ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.4-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists ================================================ FILE: lite/examples/bert_qa/android/gradle.properties ================================================ # Project-wide Gradle settings. # IDE (e.g. Android Studio) users: # Gradle settings configured through the IDE *will override* # any settings specified in this file. # For more details on how to configure your build environment visit # http://www.gradle.org/docs/current/userguide/build_environment.html # Specifies the JVM arguments used for the daemon process. # The setting is particularly useful for tweaking memory settings. org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 # When configured, Gradle will run in incubating parallel mode. # This option should only be used with decoupled projects. More details, visit # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects # org.gradle.parallel=true # AndroidX package structure to make it clearer which packages are bundled with the # Android operating system, and which are packaged with your app"s APK # https://developer.android.com/topic/libraries/support-library/androidx-rn android.useAndroidX=true # Kotlin code style for this project: "official" or "obsolete": kotlin.code.style=official # Enables namespacing of each library's R class so that its R class includes only the # resources declared in the library itself and none from the library's dependencies, # thereby reducing the size of the R class for that library android.nonTransitiveRClass=true ================================================ FILE: lite/examples/bert_qa/android/gradlew ================================================ #!/bin/sh # # Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # SPDX-License-Identifier: Apache-2.0 # ############################################################################## # # Gradle start up script for POSIX generated by Gradle. # # Important for running: # # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is # noncompliant, but you have some other compliant shell such as ksh or # bash, then to run this script, type that shell name before the whole # command line, like: # # ksh Gradle # # Busybox and similar reduced shells will NOT work, because this script # requires all of these POSIX shell features: # * functions; # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», # «${var#prefix}», «${var%suffix}», and «$( cmd )»; # * compound commands having a testable exit status, especially «case»; # * various built-in commands including «command», «set», and «ulimit». # # Important for patching: # # (2) This script targets any POSIX shell, so it avoids extensions provided # by Bash, Ksh, etc; in particular arrays are avoided. # # The "traditional" practice of packing multiple parameters into a # space-separated string is a well documented source of bugs and security # problems, so this is (mostly) avoided, by progressively accumulating # options in "$@", and eventually passing that to Java. # # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; # see the in-line comments for details. # # There are tweaks for specific operating systems such as AIX, CygWin, # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template # https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. # ############################################################################## # Attempt to set APP_HOME # Resolve links: $0 may be a link app_path=$0 # Need this for daisy-chained symlinks. while APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path [ -h "$app_path" ] do ls=$( ls -ld "$app_path" ) link=${ls#*' -> '} case $link in #( /*) app_path=$link ;; #( *) app_path=$APP_HOME$link ;; esac done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum warn () { echo "$*" } >&2 die () { echo echo "$*" echo exit 1 } >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false case "$( uname )" in #( CYGWIN* ) cygwin=true ;; #( Darwin* ) darwin=true ;; #( MSYS* | MINGW* ) msys=true ;; #( NONSTOP* ) nonstop=true ;; esac CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD=$JAVA_HOME/jre/sh/java else JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else JAVACMD=java if ! command -v java >/dev/null 2>&1 then die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac fi # Collect all arguments for the java command, stacking in reverse order: # * args from the command line # * the main class name # * -classpath # * -D...appname settings # * --module-path (only if needed) # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java if "$cygwin" || "$msys" ; then APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) JAVACMD=$( cygpath --unix "$JAVACMD" ) # Now convert the arguments - kludge to limit ourselves to /bin/sh for arg do if case $arg in #( -*) false ;; # don't mess with options #( /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath [ -e "$t" ] ;; #( *) false ;; esac then arg=$( cygpath --path --ignore --mixed "$arg" ) fi # Roll the args list around exactly as many times as the number of # args, so each arg winds up back in the position where it started, but # possibly modified. # # NB: a `for` loop captures its iteration list before it begins, so # changing the positional parameters here affects neither the number of # iterations, nor the values presented in `arg`. shift # remove old arg set -- "$@" "$arg" # push replacement arg done fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: # * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" # Stop when "xargs" is not available. if ! command -v xargs >/dev/null 2>&1 then die "xargs is not available" fi # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. # # In Bash we could simply go: # # readarray ARGS < <( xargs -n1 <<<"$var" ) && # set -- "${ARGS[@]}" "$@" # # but POSIX shell has neither arrays nor command substitution, so instead we # post-process each arg (as a line of input to sed) to backslash-escape any # character that might be a shell metacharacter, then use eval to reverse # that process (while maintaining the separation between arguments), and wrap # the whole thing up as a single "set" statement. # # This will of course break if any of these variables contains a newline or # an unmatched quote. # eval "set -- $( printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | xargs -n1 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | tr '\n' ' ' )" '"$@"' exec "$JAVACMD" "$@" ================================================ FILE: lite/examples/bert_qa/android/gradlew.bat ================================================ @rem @rem Copyright 2015 the original author or authors. @rem @rem Licensed under the Apache License, Version 2.0 (the "License"); @rem you may not use this file except in compliance with the License. @rem You may obtain a copy of the License at @rem @rem https://www.apache.org/licenses/LICENSE-2.0 @rem @rem Unless required by applicable law or agreed to in writing, software @rem distributed under the License is distributed on an "AS IS" BASIS, @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem @rem SPDX-License-Identifier: Apache-2.0 @rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @rem @rem ########################################################################## @rem Set local scope for the variables with windows NT shell if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. @rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Resolve any "." and ".." in APP_HOME to make it shorter. for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute echo. 1>&2 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 echo. 1>&2 echo Please set the JAVA_HOME variable in your environment to match the 1>&2 echo location of your Java installation. 1>&2 goto fail :findJavaFromJavaHome set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute echo. 1>&2 echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 echo. 1>&2 echo Please set the JAVA_HOME variable in your environment to match the 1>&2 echo location of your Java installation. 1>&2 goto fail :execute @rem Setup the command line set CLASSPATH= @rem Execute Gradle "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* :end @rem End local scope for the variables with windows NT shell if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! set EXIT_CODE=%ERRORLEVEL% if %EXIT_CODE% equ 0 set EXIT_CODE=1 if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal :omega ================================================ FILE: lite/examples/bert_qa/android/settings.gradle ================================================ pluginManagement { repositories { gradlePluginPortal() google() mavenCentral() } } dependencyResolutionManagement { repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) repositories { google() mavenCentral() } } rootProject.name = "TFLite Bert QA Demo App" include ':app' ================================================ FILE: lite/examples/bert_qa/ios/.gitignore ================================================ vocab.txt contents_from_squad_dict_format.json ================================================ FILE: lite/examples/bert_qa/ios/Assets.xcassets/AppIcon.appiconset/Contents.json ================================================ { "images": [ { "size": "60x60", "expected-size": "180", "filename": "180.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "3x" }, { "size": "40x40", "expected-size": "80", "filename": "80.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "2x" }, { "size": "40x40", "expected-size": "120", "filename": "120.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "3x" }, { "size": "60x60", "expected-size": "120", "filename": "120.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "2x" }, { "size": "29x29", "expected-size": "58", "filename": "58.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "2x" }, { "size": "29x29", "expected-size": "29", "filename": "29.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "1x" }, { "size": "29x29", "expected-size": "87", "filename": "87.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "3x" }, { "size": "20x20", "expected-size": "40", "filename": "40.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "2x" }, { "size": "20x20", "expected-size": "60", "filename": "60.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "3x" }, { "size": "1024x1024", "filename": "1024.png", "expected-size": "1024", "idiom": "ios-marketing", "folder": "Assets.xcassets/AppIcon.appiconset/", "scale": "1x" }, { "size": "40x40", "expected-size": "80", "filename": "80.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "72x72", "expected-size": "72", "filename": "72.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "76x76", "expected-size": "152", "filename": "152.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "50x50", "expected-size": "100", "filename": "100.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "29x29", "expected-size": "58", "filename": "58.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "76x76", "expected-size": "76", "filename": "76.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "29x29", "expected-size": "29", "filename": "29.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "50x50", "expected-size": "50", "filename": "50.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "72x72", "expected-size": "144", "filename": "144.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "40x40", "expected-size": "40", "filename": "40.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "83.5x83.5", "expected-size": "167", "filename": "167.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "20x20", "expected-size": "20", "filename": "20.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "20x20", "expected-size": "40", "filename": "40.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" } ] } ================================================ FILE: lite/examples/bert_qa/ios/Assets.xcassets/Contents.json ================================================ { "info" : { "version" : 1, "author" : "xcode" } } ================================================ FILE: lite/examples/bert_qa/ios/BertQA.xcodeproj/project.pbxproj ================================================ // !$*UTF8*$! { archiveVersion = 1; classes = { }; objectVersion = 50; objects = { /* Begin PBXBuildFile section */ 3A4D441F43E606AFFBEDC0F4 /* Pods_BertQA_UIKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 64C806A982D14ED6BC003A9A /* Pods_BertQA_UIKit.framework */; }; 8073F32F9E31519DA1C60690 /* Pods_BertQA_SwiftUI.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 2D0CCBAA9B671F431851D96A /* Pods_BertQA_SwiftUI.framework */; }; 8402440423D9BFDE00704ABD /* FullTokenizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8402440323D9BFDE00704ABD /* FullTokenizer.swift */; }; 8402440623D9C18600704ABD /* WordpieceTokenizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8402440523D9C18600704ABD /* WordpieceTokenizer.swift */; }; 8402440823D9C1FB00704ABD /* BasicTokenizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8402440723D9C1FB00704ABD /* BasicTokenizer.swift */; }; 840245262424CB63002D1DAD /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840245252424CB63002D1DAD /* AppDelegate.swift */; }; 840245282424CB63002D1DAD /* SceneDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840245272424CB63002D1DAD /* SceneDelegate.swift */; }; 840245322424CB64002D1DAD /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 840245302424CB64002D1DAD /* LaunchScreen.storyboard */; }; 840C6F3F2403A39E00E6905E /* DataExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840C6F3B2403A39E00E6905E /* DataExtension.swift */; }; 840C6F402403A39E00E6905E /* InputFeatures.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840C6F3C2403A39E00E6905E /* InputFeatures.swift */; }; 840C6F412403A39E00E6905E /* BertQAHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840C6F3D2403A39E00E6905E /* BertQAHandler.swift */; }; 840C6F422403A39E00E6905E /* Result.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840C6F3E2403A39E00E6905E /* Result.swift */; }; 840C6F442403A5C900E6905E /* mobilebert_float_20191023.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 840C6F432403A5C900E6905E /* mobilebert_float_20191023.tflite */; }; 842E65D123CEDD6F00AE9416 /* Constants.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842E65D023CEDD6F00AE9416 /* Constants.swift */; }; 842E65D423CEDF7200AE9416 /* DatasetTitleCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842E65D323CEDF7200AE9416 /* DatasetTitleCell.swift */; }; 8433178E23E933AC001B0AE8 /* StringExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8433178D23E933AC001B0AE8 /* StringExtension.swift */; }; 84433D4D240EA8EF00C983EA /* ContentData.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84433D4C240EA8EF00C983EA /* ContentData.swift */; }; 844C87012424CE29006DA707 /* DatasetListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 844C86FC2424CD7F006DA707 /* DatasetListView.swift */; }; 844C87022424CE2E006DA707 /* DatasetDetailView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 844C86FE2424CD97006DA707 /* DatasetDetailView.swift */; }; 844C87032424CFD0006DA707 /* contents_from_squad_dict_format.json in Resources */ = {isa = PBXBuildFile; fileRef = 8498AA6C240FA5DF003ABA5E /* contents_from_squad_dict_format.json */; }; 844C87072424D019006DA707 /* KeyboardHeightObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 844C87062424D019006DA707 /* KeyboardHeightObserver.swift */; }; 844C87082424D03E006DA707 /* Constants.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842E65D023CEDD6F00AE9416 /* Constants.swift */; }; 84551A6C23F2710100C71C16 /* vocab.txt in Resources */ = {isa = PBXBuildFile; fileRef = 84551A6B23F2710100C71C16 /* vocab.txt */; }; 845B4FD42434856500943980 /* Dataset.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845B4FD32434856500943980 /* Dataset.swift */; }; 845B4FD52434856A00943980 /* Dataset.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845B4FD32434856500943980 /* Dataset.swift */; }; 846918EB2425212B00A90808 /* DataExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840C6F3B2403A39E00E6905E /* DataExtension.swift */; }; 846918EC2425212F00A90808 /* UnicodeScalarExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84C62F6123F6BD3A00E4BB77 /* UnicodeScalarExtension.swift */; }; 846918ED2425213100A90808 /* StringExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8433178D23E933AC001B0AE8 /* StringExtension.swift */; }; 846918EF2425213800A90808 /* FileLoader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84E81C6023FFC2B400D50692 /* FileLoader.swift */; }; 846918F12425213B00A90808 /* FullTokenizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8402440323D9BFDE00704ABD /* FullTokenizer.swift */; }; 846918F32425213E00A90808 /* BasicTokenizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8402440723D9C1FB00704ABD /* BasicTokenizer.swift */; }; 846918F52425214200A90808 /* WordpieceTokenizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8402440523D9C18600704ABD /* WordpieceTokenizer.swift */; }; 846918F82425214B00A90808 /* mobilebert_float_20191023.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 840C6F432403A5C900E6905E /* mobilebert_float_20191023.tflite */; }; 846918FA2425214D00A90808 /* vocab.txt in Resources */ = {isa = PBXBuildFile; fileRef = 84551A6B23F2710100C71C16 /* vocab.txt */; }; 8475814E23CDC1660003472A /* README.md in Resources */ = {isa = PBXBuildFile; fileRef = 8475814D23CDC1660003472A /* README.md */; }; 8475815023CDC24D0003472A /* DataSetDetailViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8475814F23CDC24D0003472A /* DataSetDetailViewController.swift */; }; 8475815223CDC28C0003472A /* DataSetsTableViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8475815123CDC28C0003472A /* DataSetsTableViewController.swift */; }; 849401F824447883005F0C25 /* StringExtensionTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849401F024447883005F0C25 /* StringExtensionTest.swift */; }; 849401F924447883005F0C25 /* UnicodeScalarExtensionTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849401F124447883005F0C25 /* UnicodeScalarExtensionTest.swift */; }; 849401FA24447883005F0C25 /* WordpieceTokenizerTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849401F324447883005F0C25 /* WordpieceTokenizerTest.swift */; }; 849401FB24447883005F0C25 /* FullTokenizerTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849401F424447883005F0C25 /* FullTokenizerTest.swift */; }; 849401FC24447883005F0C25 /* BasicTokenizerTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849401F524447883005F0C25 /* BasicTokenizerTest.swift */; }; 849401FD24447883005F0C25 /* BertQAHandlerTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 849401F724447883005F0C25 /* BertQAHandlerTest.swift */; }; 8496E7E123CF019C00386218 /* SuggestedQuestionButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8496E7E023CF019C00386218 /* SuggestedQuestionButton.swift */; }; 8498AA6D240FA5DF003ABA5E /* contents_from_squad_dict_format.json in Resources */ = {isa = PBXBuildFile; fileRef = 8498AA6C240FA5DF003ABA5E /* contents_from_squad_dict_format.json */; }; 84C3284F23B49E2E00A7980D /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84C3284E23B49E2E00A7980D /* AppDelegate.swift */; }; 84C3285423B49E2E00A7980D /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 84C3285223B49E2E00A7980D /* Main.storyboard */; }; 84C3285623B49E2F00A7980D /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 84C3285523B49E2F00A7980D /* Assets.xcassets */; }; 84C3285923B49E2F00A7980D /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 84C3285723B49E2F00A7980D /* LaunchScreen.storyboard */; }; 84C62F6223F6BD3A00E4BB77 /* UnicodeScalarExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84C62F6123F6BD3A00E4BB77 /* UnicodeScalarExtension.swift */; }; 84CBD8F92424DCBF00DC20AA /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 84C3285523B49E2F00A7980D /* Assets.xcassets */; }; 84D84026243C4DFB00E0B769 /* BertQAHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840C6F3D2403A39E00E6905E /* BertQAHandler.swift */; }; 84D84027243C4E0100E0B769 /* InputFeatures.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840C6F3C2403A39E00E6905E /* InputFeatures.swift */; }; 84D84028243C4E0300E0B769 /* ContentData.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84433D4C240EA8EF00C983EA /* ContentData.swift */; }; 84D84029243C4E0600E0B769 /* Result.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840C6F3E2403A39E00E6905E /* Result.swift */; }; 84D8402E243C4E3E00E0B769 /* StatusView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84D8402B243C4E3E00E0B769 /* StatusView.swift */; }; 84D8402F243C4E3E00E0B769 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84D8402C243C4E3E00E0B769 /* ContentView.swift */; }; 84D84030243C4E3E00E0B769 /* SuggestedQuestionsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84D8402D243C4E3E00E0B769 /* SuggestedQuestionsView.swift */; }; 84E81C6123FFC2B400D50692 /* FileLoader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84E81C6023FFC2B400D50692 /* FileLoader.swift */; }; 84F0354E2418B7380039B5D5 /* OptionViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84F0354D2418B7380039B5D5 /* OptionViewController.swift */; }; 84F035502418B7800039B5D5 /* OptionTableViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84F0354F2418B7800039B5D5 /* OptionTableViewController.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ 849401DB244462AB005F0C25 /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; containerPortal = 84C3284323B49E2E00A7980D /* Project object */; proxyType = 1; remoteGlobalIDString = 84C3284A23B49E2E00A7980D; remoteInfo = "BertQA-UIKit"; }; /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ 20076681A0E239448C8D4B05 /* Pods-BertQA.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-BertQA.release.xcconfig"; path = "Target Support Files/Pods-BertQA/Pods-BertQA.release.xcconfig"; sourceTree = ""; }; 2D0CCBAA9B671F431851D96A /* Pods_BertQA_SwiftUI.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_BertQA_SwiftUI.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 37F95CAA32D15283DD33CF2F /* Pods-BertQA.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-BertQA.debug.xcconfig"; path = "Target Support Files/Pods-BertQA/Pods-BertQA.debug.xcconfig"; sourceTree = ""; }; 64C806A982D14ED6BC003A9A /* Pods_BertQA_UIKit.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_BertQA_UIKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 67EE9FD8BFDA0BBBC2A8C538 /* Pods-BertQA-UIKit.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-BertQA-UIKit.release.xcconfig"; path = "Target Support Files/Pods-BertQA-UIKit/Pods-BertQA-UIKit.release.xcconfig"; sourceTree = ""; }; 6D06722FADA8CA7F3CCC3492 /* Pods-BertQA-SwiftUI.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-BertQA-SwiftUI.release.xcconfig"; path = "Target Support Files/Pods-BertQA-SwiftUI/Pods-BertQA-SwiftUI.release.xcconfig"; sourceTree = ""; }; 7595E796644E0735F1BD68A4 /* Pods-BertQA-SwiftUI.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-BertQA-SwiftUI.debug.xcconfig"; path = "Target Support Files/Pods-BertQA-SwiftUI/Pods-BertQA-SwiftUI.debug.xcconfig"; sourceTree = ""; }; 7E5B19F92467606B003BC3C2 /* download_resources.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = download_resources.sh; sourceTree = ""; }; 8402440323D9BFDE00704ABD /* FullTokenizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FullTokenizer.swift; sourceTree = ""; }; 8402440523D9C18600704ABD /* WordpieceTokenizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WordpieceTokenizer.swift; sourceTree = ""; }; 8402440723D9C1FB00704ABD /* BasicTokenizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BasicTokenizer.swift; sourceTree = ""; }; 840245232424CB63002D1DAD /* BertQA-SwiftUI.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "BertQA-SwiftUI.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 840245252424CB63002D1DAD /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 840245272424CB63002D1DAD /* SceneDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SceneDelegate.swift; sourceTree = ""; }; 840245312424CB64002D1DAD /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 840245332424CB64002D1DAD /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 840C6F3B2403A39E00E6905E /* DataExtension.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DataExtension.swift; sourceTree = ""; }; 840C6F3C2403A39E00E6905E /* InputFeatures.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = InputFeatures.swift; sourceTree = ""; }; 840C6F3D2403A39E00E6905E /* BertQAHandler.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = BertQAHandler.swift; sourceTree = ""; }; 840C6F3E2403A39E00E6905E /* Result.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Result.swift; sourceTree = ""; }; 840C6F432403A5C900E6905E /* mobilebert_float_20191023.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = mobilebert_float_20191023.tflite; sourceTree = ""; }; 84126CCB24111D5E00349A2A /* ControlPanelView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ControlPanelView.swift; sourceTree = ""; }; 842E65D023CEDD6F00AE9416 /* Constants.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Constants.swift; sourceTree = ""; }; 842E65D323CEDF7200AE9416 /* DatasetTitleCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DatasetTitleCell.swift; sourceTree = ""; }; 8433178D23E933AC001B0AE8 /* StringExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StringExtension.swift; sourceTree = ""; }; 84433D4C240EA8EF00C983EA /* ContentData.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContentData.swift; sourceTree = ""; }; 844C86FC2424CD7F006DA707 /* DatasetListView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DatasetListView.swift; sourceTree = ""; }; 844C86FE2424CD97006DA707 /* DatasetDetailView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DatasetDetailView.swift; sourceTree = ""; }; 844C87062424D019006DA707 /* KeyboardHeightObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyboardHeightObserver.swift; sourceTree = ""; }; 84551A6B23F2710100C71C16 /* vocab.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = vocab.txt; sourceTree = ""; }; 845B4FD32434856500943980 /* Dataset.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Dataset.swift; sourceTree = ""; }; 8475814D23CDC1660003472A /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; 8475814F23CDC24D0003472A /* DataSetDetailViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataSetDetailViewController.swift; sourceTree = ""; }; 8475815123CDC28C0003472A /* DataSetsTableViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DataSetsTableViewController.swift; sourceTree = ""; }; 848BC39924002091009434DE /* BertQATests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = BertQATests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 849401D6244462AB005F0C25 /* BertQA-Tests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "BertQA-Tests.xctest"; sourceTree = BUILT_PRODUCTS_DIR; }; 849401DA244462AB005F0C25 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 849401F024447883005F0C25 /* StringExtensionTest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = StringExtensionTest.swift; sourceTree = ""; }; 849401F124447883005F0C25 /* UnicodeScalarExtensionTest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnicodeScalarExtensionTest.swift; sourceTree = ""; }; 849401F324447883005F0C25 /* WordpieceTokenizerTest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = WordpieceTokenizerTest.swift; sourceTree = ""; }; 849401F424447883005F0C25 /* FullTokenizerTest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FullTokenizerTest.swift; sourceTree = ""; }; 849401F524447883005F0C25 /* BasicTokenizerTest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = BasicTokenizerTest.swift; sourceTree = ""; }; 849401F724447883005F0C25 /* BertQAHandlerTest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = BertQAHandlerTest.swift; sourceTree = ""; }; 8496E7E023CF019C00386218 /* SuggestedQuestionButton.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SuggestedQuestionButton.swift; sourceTree = ""; }; 8498AA6C240FA5DF003ABA5E /* contents_from_squad_dict_format.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = contents_from_squad_dict_format.json; sourceTree = ""; }; 84C3284B23B49E2E00A7980D /* BertQA-UIKit.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "BertQA-UIKit.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 84C3284E23B49E2E00A7980D /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 84C3285323B49E2E00A7980D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; 84C3285523B49E2F00A7980D /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 84C3285823B49E2F00A7980D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 84C3285A23B49E2F00A7980D /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 84C62F6123F6BD3A00E4BB77 /* UnicodeScalarExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UnicodeScalarExtension.swift; sourceTree = ""; }; 84D8402B243C4E3E00E0B769 /* StatusView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = StatusView.swift; sourceTree = ""; }; 84D8402C243C4E3E00E0B769 /* ContentView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; 84D8402D243C4E3E00E0B769 /* SuggestedQuestionsView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SuggestedQuestionsView.swift; sourceTree = ""; }; 84E81C6023FFC2B400D50692 /* FileLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileLoader.swift; sourceTree = ""; }; 84F0354D2418B7380039B5D5 /* OptionViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OptionViewController.swift; sourceTree = ""; }; 84F0354F2418B7800039B5D5 /* OptionTableViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OptionTableViewController.swift; sourceTree = ""; }; FA8F6BA054A264013FFDBD91 /* Pods-BertQA-UIKit.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-BertQA-UIKit.debug.xcconfig"; path = "Target Support Files/Pods-BertQA-UIKit/Pods-BertQA-UIKit.debug.xcconfig"; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ 840245202424CB63002D1DAD /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( 8073F32F9E31519DA1C60690 /* Pods_BertQA_SwiftUI.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; 849401D3244462AB005F0C25 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( ); runOnlyForDeploymentPostprocessing = 0; }; 84C3284823B49E2E00A7980D /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( 3A4D441F43E606AFFBEDC0F4 /* Pods_BertQA_UIKit.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ 5A57E91BD94E10B803571A3B /* Frameworks */ = { isa = PBXGroup; children = ( 2D0CCBAA9B671F431851D96A /* Pods_BertQA_SwiftUI.framework */, 64C806A982D14ED6BC003A9A /* Pods_BertQA_UIKit.framework */, ); name = Frameworks; sourceTree = ""; }; 781038914BC670D9D4E03C4B /* Pods */ = { isa = PBXGroup; children = ( 37F95CAA32D15283DD33CF2F /* Pods-BertQA.debug.xcconfig */, 20076681A0E239448C8D4B05 /* Pods-BertQA.release.xcconfig */, 7595E796644E0735F1BD68A4 /* Pods-BertQA-SwiftUI.debug.xcconfig */, 6D06722FADA8CA7F3CCC3492 /* Pods-BertQA-SwiftUI.release.xcconfig */, FA8F6BA054A264013FFDBD91 /* Pods-BertQA-UIKit.debug.xcconfig */, 67EE9FD8BFDA0BBBC2A8C538 /* Pods-BertQA-UIKit.release.xcconfig */, ); path = Pods; sourceTree = ""; }; 8402440223D9BC0600704ABD /* Tokenizers */ = { isa = PBXGroup; children = ( 8402440323D9BFDE00704ABD /* FullTokenizer.swift */, 8402440723D9C1FB00704ABD /* BasicTokenizer.swift */, 8402440523D9C18600704ABD /* WordpieceTokenizer.swift */, ); path = Tokenizers; sourceTree = ""; }; 840245242424CB63002D1DAD /* ViewInSwiftUI */ = { isa = PBXGroup; children = ( 844C86FB2424CD67006DA707 /* Views */, 840245252424CB63002D1DAD /* AppDelegate.swift */, 840245272424CB63002D1DAD /* SceneDelegate.swift */, 844C87062424D019006DA707 /* KeyboardHeightObserver.swift */, 840245302424CB64002D1DAD /* LaunchScreen.storyboard */, 840245332424CB64002D1DAD /* Info.plist */, ); path = ViewInSwiftUI; sourceTree = ""; }; 840C6F322403A36000E6905E /* ML */ = { isa = PBXGroup; children = ( 840C6F3C2403A39E00E6905E /* InputFeatures.swift */, 84433D4C240EA8EF00C983EA /* ContentData.swift */, 840C6F3E2403A39E00E6905E /* Result.swift */, 840C6F3D2403A39E00E6905E /* BertQAHandler.swift */, ); path = ML; sourceTree = ""; }; 841BD7242422698B0050D71C /* BertQACore */ = { isa = PBXGroup; children = ( 842E65CD23CEDBA900AE9416 /* Resources */, 8475815523CDC64B0003472A /* Models */, 84E81C6223FFC2F600D50692 /* Extensions */, 842E65D023CEDD6F00AE9416 /* Constants.swift */, ); path = BertQACore; sourceTree = ""; }; 841BD7262422698B0050D71C /* ViewInUIKit */ = { isa = PBXGroup; children = ( 842E65D223CEDF4700AE9416 /* Views */, 84F96F0023CDACA0009FBDF5 /* Controllers */, 84C3284E23B49E2E00A7980D /* AppDelegate.swift */, 84C3285223B49E2E00A7980D /* Main.storyboard */, 84C3285723B49E2F00A7980D /* LaunchScreen.storyboard */, 84C3285A23B49E2F00A7980D /* Info.plist */, ); path = ViewInUIKit; sourceTree = ""; }; 842E65CD23CEDBA900AE9416 /* Resources */ = { isa = PBXGroup; children = ( 8498AA6C240FA5DF003ABA5E /* contents_from_squad_dict_format.json */, 840C6F432403A5C900E6905E /* mobilebert_float_20191023.tflite */, 84551A6B23F2710100C71C16 /* vocab.txt */, ); path = Resources; sourceTree = ""; }; 842E65D223CEDF4700AE9416 /* Views */ = { isa = PBXGroup; children = ( 842E65D323CEDF7200AE9416 /* DatasetTitleCell.swift */, 8496E7E023CF019C00386218 /* SuggestedQuestionButton.swift */, 84126CCB24111D5E00349A2A /* ControlPanelView.swift */, ); path = Views; sourceTree = ""; }; 844C86FB2424CD67006DA707 /* Views */ = { isa = PBXGroup; children = ( 84D8402C243C4E3E00E0B769 /* ContentView.swift */, 84D8402B243C4E3E00E0B769 /* StatusView.swift */, 84D8402D243C4E3E00E0B769 /* SuggestedQuestionsView.swift */, 844C86FC2424CD7F006DA707 /* DatasetListView.swift */, 844C86FE2424CD97006DA707 /* DatasetDetailView.swift */, ); path = Views; sourceTree = ""; }; 8475815523CDC64B0003472A /* Models */ = { isa = PBXGroup; children = ( 845B4FD32434856500943980 /* Dataset.swift */, 84E81C6023FFC2B400D50692 /* FileLoader.swift */, 8402440223D9BC0600704ABD /* Tokenizers */, 840C6F322403A36000E6905E /* ML */, ); path = Models; sourceTree = ""; }; 849401D7244462AB005F0C25 /* BertQATests */ = { isa = PBXGroup; children = ( 849401EF24447883005F0C25 /* ExtensionTest */, 849401F624447883005F0C25 /* MLTest */, 849401F224447883005F0C25 /* TokenizerTest */, 849401DA244462AB005F0C25 /* Info.plist */, ); path = BertQATests; sourceTree = ""; }; 849401EF24447883005F0C25 /* ExtensionTest */ = { isa = PBXGroup; children = ( 849401F024447883005F0C25 /* StringExtensionTest.swift */, 849401F124447883005F0C25 /* UnicodeScalarExtensionTest.swift */, ); path = ExtensionTest; sourceTree = ""; }; 849401F224447883005F0C25 /* TokenizerTest */ = { isa = PBXGroup; children = ( 849401F324447883005F0C25 /* WordpieceTokenizerTest.swift */, 849401F424447883005F0C25 /* FullTokenizerTest.swift */, 849401F524447883005F0C25 /* BasicTokenizerTest.swift */, ); path = TokenizerTest; sourceTree = ""; }; 849401F624447883005F0C25 /* MLTest */ = { isa = PBXGroup; children = ( 849401F724447883005F0C25 /* BertQAHandlerTest.swift */, ); path = MLTest; sourceTree = ""; }; 84B3C6A9240F74EA00BA6F5E /* RunScripts */ = { isa = PBXGroup; children = ( 7E5B19F92467606B003BC3C2 /* download_resources.sh */, ); path = RunScripts; sourceTree = ""; }; 84C3284223B49E2E00A7980D = { isa = PBXGroup; children = ( 8475814D23CDC1660003472A /* README.md */, 841BD7242422698B0050D71C /* BertQACore */, 841BD7262422698B0050D71C /* ViewInUIKit */, 840245242424CB63002D1DAD /* ViewInSwiftUI */, 849401D7244462AB005F0C25 /* BertQATests */, 84C3285523B49E2F00A7980D /* Assets.xcassets */, 84B3C6A9240F74EA00BA6F5E /* RunScripts */, 84C3284C23B49E2E00A7980D /* Products */, 781038914BC670D9D4E03C4B /* Pods */, 5A57E91BD94E10B803571A3B /* Frameworks */, ); sourceTree = ""; }; 84C3284C23B49E2E00A7980D /* Products */ = { isa = PBXGroup; children = ( 84C3284B23B49E2E00A7980D /* BertQA-UIKit.app */, 848BC39924002091009434DE /* BertQATests.xctest */, 840245232424CB63002D1DAD /* BertQA-SwiftUI.app */, 849401D6244462AB005F0C25 /* BertQA-Tests.xctest */, ); name = Products; sourceTree = ""; }; 84E81C6223FFC2F600D50692 /* Extensions */ = { isa = PBXGroup; children = ( 8433178D23E933AC001B0AE8 /* StringExtension.swift */, 84C62F6123F6BD3A00E4BB77 /* UnicodeScalarExtension.swift */, 840C6F3B2403A39E00E6905E /* DataExtension.swift */, ); path = Extensions; sourceTree = ""; }; 84F96F0023CDACA0009FBDF5 /* Controllers */ = { isa = PBXGroup; children = ( 8475815123CDC28C0003472A /* DataSetsTableViewController.swift */, 8475814F23CDC24D0003472A /* DataSetDetailViewController.swift */, 84F0354D2418B7380039B5D5 /* OptionViewController.swift */, 84F0354F2418B7800039B5D5 /* OptionTableViewController.swift */, ); path = Controllers; sourceTree = ""; }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ 840245222424CB63002D1DAD /* BertQA-SwiftUI */ = { isa = PBXNativeTarget; buildConfigurationList = 840245342424CB64002D1DAD /* Build configuration list for PBXNativeTarget "BertQA-SwiftUI" */; buildPhases = ( F746FAE7EA2F78CAB43DFA3B /* [CP] Check Pods Manifest.lock */, 846918AB2425200800A90808 /* ShellScript */, 8402451F2424CB63002D1DAD /* Sources */, 840245202424CB63002D1DAD /* Frameworks */, 840245212424CB63002D1DAD /* Resources */, ); buildRules = ( ); dependencies = ( ); name = "BertQA-SwiftUI"; productName = ViewInSwiftUI; productReference = 840245232424CB63002D1DAD /* BertQA-SwiftUI.app */; productType = "com.apple.product-type.application"; }; 849401D5244462AB005F0C25 /* BertQA-Tests */ = { isa = PBXNativeTarget; buildConfigurationList = 849401DD244462AB005F0C25 /* Build configuration list for PBXNativeTarget "BertQA-Tests" */; buildPhases = ( 849401D2244462AB005F0C25 /* Sources */, 849401D3244462AB005F0C25 /* Frameworks */, 849401D4244462AB005F0C25 /* Resources */, ); buildRules = ( ); dependencies = ( 849401DC244462AB005F0C25 /* PBXTargetDependency */, ); name = "BertQA-Tests"; productName = "BertQA-Tests"; productReference = 849401D6244462AB005F0C25 /* BertQA-Tests.xctest */; productType = "com.apple.product-type.bundle.unit-test"; }; 84C3284A23B49E2E00A7980D /* BertQA-UIKit */ = { isa = PBXNativeTarget; buildConfigurationList = 84C3285D23B49E2F00A7980D /* Build configuration list for PBXNativeTarget "BertQA-UIKit" */; buildPhases = ( F18482CEB971E4CB207BA1C9 /* [CP] Check Pods Manifest.lock */, 84B3C6AC240F758B00BA6F5E /* ShellScript */, 84C3284723B49E2E00A7980D /* Sources */, 84C3284823B49E2E00A7980D /* Frameworks */, 84C3284923B49E2E00A7980D /* Resources */, ); buildRules = ( ); dependencies = ( ); name = "BertQA-UIKit"; productName = BertQA; productReference = 84C3284B23B49E2E00A7980D /* BertQA-UIKit.app */; productType = "com.apple.product-type.application"; }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ 84C3284323B49E2E00A7980D /* Project object */ = { isa = PBXProject; attributes = { LastSwiftUpdateCheck = 1120; LastUpgradeCheck = 1030; ORGANIZATIONNAME = tensorflow; TargetAttributes = { 840245222424CB63002D1DAD = { CreatedOnToolsVersion = 11.2.1; }; 849401D5244462AB005F0C25 = { CreatedOnToolsVersion = 11.2.1; TestTargetID = 84C3284A23B49E2E00A7980D; }; 84C3284A23B49E2E00A7980D = { CreatedOnToolsVersion = 10.3; }; }; }; buildConfigurationList = 84C3284623B49E2E00A7980D /* Build configuration list for PBXProject "BertQA" */; compatibilityVersion = "Xcode 9.3"; developmentRegion = en; hasScannedForEncodings = 0; knownRegions = ( en, Base, ); mainGroup = 84C3284223B49E2E00A7980D; productRefGroup = 84C3284C23B49E2E00A7980D /* Products */; projectDirPath = ""; projectRoot = ""; targets = ( 84C3284A23B49E2E00A7980D /* BertQA-UIKit */, 840245222424CB63002D1DAD /* BertQA-SwiftUI */, 849401D5244462AB005F0C25 /* BertQA-Tests */, ); }; /* End PBXProject section */ /* Begin PBXResourcesBuildPhase section */ 840245212424CB63002D1DAD /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( 846918FA2425214D00A90808 /* vocab.txt in Resources */, 840245322424CB64002D1DAD /* LaunchScreen.storyboard in Resources */, 844C87032424CFD0006DA707 /* contents_from_squad_dict_format.json in Resources */, 846918F82425214B00A90808 /* mobilebert_float_20191023.tflite in Resources */, 84CBD8F92424DCBF00DC20AA /* Assets.xcassets in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; 849401D4244462AB005F0C25 /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( ); runOnlyForDeploymentPostprocessing = 0; }; 84C3284923B49E2E00A7980D /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( 84551A6C23F2710100C71C16 /* vocab.txt in Resources */, 84C3285923B49E2F00A7980D /* LaunchScreen.storyboard in Resources */, 84C3285623B49E2F00A7980D /* Assets.xcassets in Resources */, 840C6F442403A5C900E6905E /* mobilebert_float_20191023.tflite in Resources */, 8475814E23CDC1660003472A /* README.md in Resources */, 8498AA6D240FA5DF003ABA5E /* contents_from_squad_dict_format.json in Resources */, 84C3285423B49E2E00A7980D /* Main.storyboard in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ 846918AB2425200800A90808 /* ShellScript */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; files = ( ); inputFileListPaths = ( ); inputPaths = ( ); outputFileListPaths = ( ); outputPaths = ( ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "$SRCROOT/RunScripts/download_resources.sh\n"; }; 84B3C6AC240F758B00BA6F5E /* ShellScript */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; files = ( ); inputFileListPaths = ( ); inputPaths = ( ); outputFileListPaths = ( ); outputPaths = ( ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "$SRCROOT/RunScripts/download_resources.sh\n"; }; F18482CEB971E4CB207BA1C9 /* [CP] Check Pods Manifest.lock */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; files = ( ); inputFileListPaths = ( ); inputPaths = ( "${PODS_PODFILE_DIR_PATH}/Podfile.lock", "${PODS_ROOT}/Manifest.lock", ); name = "[CP] Check Pods Manifest.lock"; outputFileListPaths = ( ); outputPaths = ( "$(DERIVED_FILE_DIR)/Pods-BertQA-UIKit-checkManifestLockResult.txt", ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; showEnvVarsInLog = 0; }; F746FAE7EA2F78CAB43DFA3B /* [CP] Check Pods Manifest.lock */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; files = ( ); inputFileListPaths = ( ); inputPaths = ( "${PODS_PODFILE_DIR_PATH}/Podfile.lock", "${PODS_ROOT}/Manifest.lock", ); name = "[CP] Check Pods Manifest.lock"; outputFileListPaths = ( ); outputPaths = ( "$(DERIVED_FILE_DIR)/Pods-BertQA-SwiftUI-checkManifestLockResult.txt", ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ 8402451F2424CB63002D1DAD /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( 84D84029243C4E0600E0B769 /* Result.swift in Sources */, 846918F32425213E00A90808 /* BasicTokenizer.swift in Sources */, 84D8402F243C4E3E00E0B769 /* ContentView.swift in Sources */, 846918F12425213B00A90808 /* FullTokenizer.swift in Sources */, 84D84030243C4E3E00E0B769 /* SuggestedQuestionsView.swift in Sources */, 840245262424CB63002D1DAD /* AppDelegate.swift in Sources */, 846918EC2425212F00A90808 /* UnicodeScalarExtension.swift in Sources */, 844C87072424D019006DA707 /* KeyboardHeightObserver.swift in Sources */, 844C87012424CE29006DA707 /* DatasetListView.swift in Sources */, 84D84026243C4DFB00E0B769 /* BertQAHandler.swift in Sources */, 846918EF2425213800A90808 /* FileLoader.swift in Sources */, 844C87022424CE2E006DA707 /* DatasetDetailView.swift in Sources */, 840245282424CB63002D1DAD /* SceneDelegate.swift in Sources */, 846918EB2425212B00A90808 /* DataExtension.swift in Sources */, 846918ED2425213100A90808 /* StringExtension.swift in Sources */, 84D8402E243C4E3E00E0B769 /* StatusView.swift in Sources */, 844C87082424D03E006DA707 /* Constants.swift in Sources */, 84D84027243C4E0100E0B769 /* InputFeatures.swift in Sources */, 84D84028243C4E0300E0B769 /* ContentData.swift in Sources */, 845B4FD52434856A00943980 /* Dataset.swift in Sources */, 846918F52425214200A90808 /* WordpieceTokenizer.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; 849401D2244462AB005F0C25 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( 849401FB24447883005F0C25 /* FullTokenizerTest.swift in Sources */, 849401F924447883005F0C25 /* UnicodeScalarExtensionTest.swift in Sources */, 849401F824447883005F0C25 /* StringExtensionTest.swift in Sources */, 849401FC24447883005F0C25 /* BasicTokenizerTest.swift in Sources */, 849401FA24447883005F0C25 /* WordpieceTokenizerTest.swift in Sources */, 849401FD24447883005F0C25 /* BertQAHandlerTest.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; 84C3284723B49E2E00A7980D /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( 845B4FD42434856500943980 /* Dataset.swift in Sources */, 840C6F402403A39E00E6905E /* InputFeatures.swift in Sources */, 84F0354E2418B7380039B5D5 /* OptionViewController.swift in Sources */, 842E65D423CEDF7200AE9416 /* DatasetTitleCell.swift in Sources */, 842E65D123CEDD6F00AE9416 /* Constants.swift in Sources */, 84433D4D240EA8EF00C983EA /* ContentData.swift in Sources */, 8475815223CDC28C0003472A /* DataSetsTableViewController.swift in Sources */, 8402440623D9C18600704ABD /* WordpieceTokenizer.swift in Sources */, 8433178E23E933AC001B0AE8 /* StringExtension.swift in Sources */, 8402440823D9C1FB00704ABD /* BasicTokenizer.swift in Sources */, 840C6F422403A39E00E6905E /* Result.swift in Sources */, 84C3284F23B49E2E00A7980D /* AppDelegate.swift in Sources */, 840C6F3F2403A39E00E6905E /* DataExtension.swift in Sources */, 84C62F6223F6BD3A00E4BB77 /* UnicodeScalarExtension.swift in Sources */, 8402440423D9BFDE00704ABD /* FullTokenizer.swift in Sources */, 840C6F412403A39E00E6905E /* BertQAHandler.swift in Sources */, 84E81C6123FFC2B400D50692 /* FileLoader.swift in Sources */, 84F035502418B7800039B5D5 /* OptionTableViewController.swift in Sources */, 8475815023CDC24D0003472A /* DataSetDetailViewController.swift in Sources */, 8496E7E123CF019C00386218 /* SuggestedQuestionButton.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ 849401DC244462AB005F0C25 /* PBXTargetDependency */ = { isa = PBXTargetDependency; target = 84C3284A23B49E2E00A7980D /* BertQA-UIKit */; targetProxy = 849401DB244462AB005F0C25 /* PBXContainerItemProxy */; }; /* End PBXTargetDependency section */ /* Begin PBXVariantGroup section */ 840245302424CB64002D1DAD /* LaunchScreen.storyboard */ = { isa = PBXVariantGroup; children = ( 840245312424CB64002D1DAD /* Base */, ); name = LaunchScreen.storyboard; sourceTree = ""; }; 84C3285223B49E2E00A7980D /* Main.storyboard */ = { isa = PBXVariantGroup; children = ( 84C3285323B49E2E00A7980D /* Base */, ); name = Main.storyboard; sourceTree = ""; }; 84C3285723B49E2F00A7980D /* LaunchScreen.storyboard */ = { isa = PBXVariantGroup; children = ( 84C3285823B49E2F00A7980D /* Base */, ); name = LaunchScreen.storyboard; sourceTree = ""; }; /* End PBXVariantGroup section */ /* Begin XCBuildConfiguration section */ 840245352424CB64002D1DAD /* Debug */ = { isa = XCBuildConfiguration; baseConfigurationReference = 7595E796644E0735F1BD68A4 /* Pods-BertQA-SwiftUI.debug.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; ENABLE_PREVIEWS = YES; INFOPLIST_FILE = ViewInSwiftUI/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); PRODUCT_BUNDLE_IDENTIFIER = "org.tensorflow.BertQA-SwiftUI"; PRODUCT_NAME = "BertQA-SwiftUI"; PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; }; name = Debug; }; 840245362424CB64002D1DAD /* Release */ = { isa = XCBuildConfiguration; baseConfigurationReference = 6D06722FADA8CA7F3CCC3492 /* Pods-BertQA-SwiftUI.release.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; ENABLE_PREVIEWS = YES; INFOPLIST_FILE = ViewInSwiftUI/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); PRODUCT_BUNDLE_IDENTIFIER = "org.tensorflow.BertQA-SwiftUI"; PRODUCT_NAME = "BertQA-SwiftUI"; PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; }; name = Release; }; 849401DE244462AB005F0C25 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CODE_SIGN_STYLE = Automatic; INFOPLIST_FILE = BertQATests/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.2; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", "@loader_path/Frameworks", ); PRODUCT_BUNDLE_IDENTIFIER = "org.tensorflow.BertQA.BertQA-Tests"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; TEST_HOST = "$(BUILT_PRODUCTS_DIR)/BertQA-UIKit.app/BertQA-UIKit"; }; name = Debug; }; 849401DF244462AB005F0C25 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CODE_SIGN_STYLE = Automatic; INFOPLIST_FILE = BertQATests/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 13.2; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", "@loader_path/Frameworks", ); PRODUCT_BUNDLE_IDENTIFIER = "org.tensorflow.BertQA.BertQA-Tests"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; TEST_HOST = "$(BUILT_PRODUCTS_DIR)/BertQA-UIKit.app/BertQA-UIKit"; }; name = Release; }; 84C3285B23B49E2F00A7980D /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; CLANG_ANALYZER_NONNULL = YES; CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; CLANG_CXX_LIBRARY = "libc++"; CLANG_ENABLE_MODULES = YES; CLANG_ENABLE_OBJC_ARC = YES; CLANG_ENABLE_OBJC_WEAK = YES; CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; CLANG_WARN_BOOL_CONVERSION = YES; CLANG_WARN_COMMA = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; CLANG_WARN_DOCUMENTATION_COMMENTS = YES; CLANG_WARN_EMPTY_BODY = YES; CLANG_WARN_ENUM_CONVERSION = YES; CLANG_WARN_INFINITE_RECURSION = YES; CLANG_WARN_INT_CONVERSION = YES; CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; CLANG_WARN_STRICT_PROTOTYPES = YES; CLANG_WARN_SUSPICIOUS_MOVE = YES; CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_IDENTITY = "iPhone Developer"; COPY_PHASE_STRIP = NO; DEBUG_INFORMATION_FORMAT = dwarf; ENABLE_STRICT_OBJC_MSGSEND = YES; ENABLE_TESTABILITY = YES; GCC_C_LANGUAGE_STANDARD = gnu11; GCC_DYNAMIC_NO_PIC = NO; GCC_NO_COMMON_BLOCKS = YES; GCC_OPTIMIZATION_LEVEL = 0; GCC_PREPROCESSOR_DEFINITIONS = ( "DEBUG=1", "$(inherited)", ); GCC_WARN_64_TO_32_BIT_CONVERSION = YES; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; GCC_WARN_UNDECLARED_SELECTOR = YES; GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; IPHONEOS_DEPLOYMENT_TARGET = 12.4; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; SDKROOT = iphoneos; SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; }; name = Debug; }; 84C3285C23B49E2F00A7980D /* Release */ = { isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; CLANG_ANALYZER_NONNULL = YES; CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; CLANG_CXX_LIBRARY = "libc++"; CLANG_ENABLE_MODULES = YES; CLANG_ENABLE_OBJC_ARC = YES; CLANG_ENABLE_OBJC_WEAK = YES; CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; CLANG_WARN_BOOL_CONVERSION = YES; CLANG_WARN_COMMA = YES; CLANG_WARN_CONSTANT_CONVERSION = YES; CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; CLANG_WARN_DOCUMENTATION_COMMENTS = YES; CLANG_WARN_EMPTY_BODY = YES; CLANG_WARN_ENUM_CONVERSION = YES; CLANG_WARN_INFINITE_RECURSION = YES; CLANG_WARN_INT_CONVERSION = YES; CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; CLANG_WARN_STRICT_PROTOTYPES = YES; CLANG_WARN_SUSPICIOUS_MOVE = YES; CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_IDENTITY = "iPhone Developer"; COPY_PHASE_STRIP = NO; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; ENABLE_NS_ASSERTIONS = NO; ENABLE_STRICT_OBJC_MSGSEND = YES; GCC_C_LANGUAGE_STANDARD = gnu11; GCC_NO_COMMON_BLOCKS = YES; GCC_WARN_64_TO_32_BIT_CONVERSION = YES; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; GCC_WARN_UNDECLARED_SELECTOR = YES; GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; IPHONEOS_DEPLOYMENT_TARGET = 12.4; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; SDKROOT = iphoneos; SWIFT_COMPILATION_MODE = wholemodule; SWIFT_OPTIMIZATION_LEVEL = "-O"; VALIDATE_PRODUCT = YES; }; name = Release; }; 84C3285E23B49E2F00A7980D /* Debug */ = { isa = XCBuildConfiguration; baseConfigurationReference = FA8F6BA054A264013FFDBD91 /* Pods-BertQA-UIKit.debug.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_IDENTITY = "iPhone Developer"; CODE_SIGN_STYLE = Automatic; INFOPLIST_FILE = ViewInUIKit/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); PRODUCT_BUNDLE_IDENTIFIER = "org.tensorflow.BertQA-UIKit"; PRODUCT_NAME = "BertQA-UIKit"; PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; }; name = Debug; }; 84C3285F23B49E2F00A7980D /* Release */ = { isa = XCBuildConfiguration; baseConfigurationReference = 67EE9FD8BFDA0BBBC2A8C538 /* Pods-BertQA-UIKit.release.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_IDENTITY = "iPhone Developer"; CODE_SIGN_STYLE = Automatic; INFOPLIST_FILE = ViewInUIKit/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); PRODUCT_BUNDLE_IDENTIFIER = "org.tensorflow.BertQA-UIKit"; PRODUCT_NAME = "BertQA-UIKit"; PROVISIONING_PROFILE_SPECIFIER = ""; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; }; name = Release; }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ 840245342424CB64002D1DAD /* Build configuration list for PBXNativeTarget "BertQA-SwiftUI" */ = { isa = XCConfigurationList; buildConfigurations = ( 840245352424CB64002D1DAD /* Debug */, 840245362424CB64002D1DAD /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; 849401DD244462AB005F0C25 /* Build configuration list for PBXNativeTarget "BertQA-Tests" */ = { isa = XCConfigurationList; buildConfigurations = ( 849401DE244462AB005F0C25 /* Debug */, 849401DF244462AB005F0C25 /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; 84C3284623B49E2E00A7980D /* Build configuration list for PBXProject "BertQA" */ = { isa = XCConfigurationList; buildConfigurations = ( 84C3285B23B49E2F00A7980D /* Debug */, 84C3285C23B49E2F00A7980D /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; 84C3285D23B49E2F00A7980D /* Build configuration list for PBXNativeTarget "BertQA-UIKit" */ = { isa = XCConfigurationList; buildConfigurations = ( 84C3285E23B49E2F00A7980D /* Debug */, 84C3285F23B49E2F00A7980D /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; /* End XCConfigurationList section */ }; rootObject = 84C3284323B49E2E00A7980D /* Project object */; } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Constants.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit enum InterpreterOptions { // Default thread count is 2, unless maximum thread count is 1. static let threadCount = ( defaultValue: 2, minimumValue: 1, maximumValue: Int(ProcessInfo.processInfo.activeProcessorCount), id: "threadCount" ) } enum MobileBERT { static let maxAnsLen = 32 static let maxQueryLen = 64 static let maxSeqLen = 384 static let predictAnsNum = 5 static let outputOffset = 1 // Need to shift 1 for outputs ([CLS]) static let doLowerCase = true static let inputDimension = [1, MobileBERT.maxSeqLen] static let outputDimension = [1, MobileBERT.maxSeqLen] static let dataset = File(name: "contents_from_squad_dict_format", ext: "json") static let vocabulary = File(name: "vocab", ext: "txt") static let model = File(name: "mobilebert_float_20191023", ext: "tflite") } struct File { let name: String let ext: String let description: String init(name: String, ext: String) { self.name = name self.ext = ext self.description = "\(name).\(ext)" } } enum CustomUI { static let textHighlightColor = UIColor(red: 1.0, green: 0.7, blue: 0.0, alpha: 0.3) static let runButtonOpacity = 0.8 static let statusTextViewCornerRadius = CGFloat(7) static let suggestedQuestionCornerRadius = CGFloat(10) static let keyboardAnimationDuration = 0.23 static let stackSpacing = CGFloat(5) static let padding = CGFloat(5) static let contentViewPadding = CGFloat(7) static let controlViewPadding = CGFloat(10) static let textSidePadding = CGFloat(4) static let textPadding = CGFloat(3) static let statusFontSize = CGFloat(14) } enum StatusMessage { static let askRun = "Tap ▶︎ button to get the answer." static let warnEmptyQuery = "⚠️Got empty question.\nPlease enter non-empty question." static let inferenceFailError = "❗️Failed to inference the answer." } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Extensions/DataExtension.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // ============================================================================= import Foundation import TensorFlowLite // MARK: - Data extension extension Data { /// Creates a new buffer by copying the buffer pointer of the given array. /// /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit /// for bit with no indirection or reference-counting operations; otherwise, reinterpreting /// data from the resulting buffer has undefined behavior. /// - Parameter array: An array with elements of type `T`. init(copyingBufferOf array: [T]) { self = array.withUnsafeBufferPointer(Data.init) } /// Convert a Data instance to Array representation. func toArray(type: T.Type) -> [T] where T: AdditiveArithmetic { var array = [T](repeating: T.zero, count: self.count / MemoryLayout.stride) _ = array.withUnsafeMutableBytes { self.copyBytes(to: $0) } return array } } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Extensions/StringExtension.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation import os /// Helper functions used for tokenizing. extension String { /// Performs invalid character removal and whitespace cleanup on text. /// /// Replaces all whitespace code points with spaces and control characters including \t, \n, \r. /// /// - Returns: Cleaned text. func cleaned() -> String { return String( // Normalize string to NFC(Normalization Form Canonical Composition). self.precomposedStringWithCanonicalMapping .unicodeScalars.compactMap { unicodeScalar in if unicodeScalar.isWhitespaceForBert { return " " } else if !unicodeScalar.isControlForBert && !unicodeScalar.shouldBeRemovedForBert { return Character(unicodeScalar) } return nil }) } /// Splits this string on whitespace. func splitByWhitespace() -> [String] { // Normalize string to NFC(Normalization Form Canonical Composition). return self.precomposedStringWithCanonicalMapping .unicodeScalars.split { $0.isWhitespaceForBert }.map { String($0) } } /// Tokenizes this string into word and punctuation tokens. /// /// For example: /// ``` /// input: "Hi,there." /// output: ["Hi", ",", "there", "."] /// ``` /// ``` /// input: "Hi, there.\n" /// output: ["Hi", ",", " there", ".", "\n"] /// ``` func tokenizedWithPunctuation() -> [String] { var tokens = [String]() var currentToken = "" // Normalize string to NFC(Normalization Form Canonical Composition). self.precomposedStringWithCanonicalMapping .unicodeScalars.forEach { unicode in if unicode.isPunctuationForBert { if !currentToken.isEmpty { // Add current token before the punctuation mark to the list of tokens. tokens.append(currentToken) } tokens.append(String(unicode)) currentToken = "" } else { // As it is not a punctuation mark, keep building current token. currentToken += String(unicode) } } if !currentToken.isEmpty { tokens.append(currentToken) } return tokens } } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Extensions/UnicodeScalarExtension.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation /// Provides some functions that make it easy to classify the character. extension UnicodeScalar { /// Whether `self` is a whitespace character. /// /// \t, \n, and \r are technically control characters but we treat them as whitespace since they /// are generally considered as such. var isWhitespaceForBert: Bool { switch self { case " ", "\t", "\n", "\r": return true default: return properties.generalCategory == .spaceSeparator } } /// Whether `self` is a control character. var isControlForBert: Bool { // These are technically control characters but we count them as whitespace characters. if isWhitespaceForBert { return false } switch properties.generalCategory { case .control, .format: return true default: return false } } /// Whether `self` should be removed for Bert tokenization. var shouldBeRemovedForBert: Bool { return self == UnicodeScalar(0) || self == UnicodeScalar(0xfffd) } /// Whether `self` is a punctuation character. /// /// We treat all non-letter/number ASCII as punctuation, except ASCII character 0 to 32. /// Characters such as "^", "$", and "`" are not in the Unicode Punctuation class but we treat /// them as punctuation anyways, for consistency. var isPunctuationForBert: Bool { if isASCII && value > 32 && !properties.isAlphabetic && properties.numericType == nil { return true } switch properties.generalCategory { case .closePunctuation, .connectorPunctuation, .dashPunctuation, .finalPunctuation, .initialPunctuation, .openPunctuation, .otherPunctuation: return true default: return false } } } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Models/Dataset.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit /// Data set to run the TensorFlow Lite model. struct Dataset: Decodable { let title: String let content: String let questions: [String] /// Wrapper to decode json file into `Decodable` struct. static func load(_ file: File = MobileBERT.dataset) -> T { let data: Data guard let fileUrl = Bundle.main.url(forResource: file.name, withExtension: file.ext) else { fatalError("Couldn't find \(file.description) in main bundle.") } do { data = try Data(contentsOf: fileUrl) } catch { fatalError("Couldn't load \(file.description) from main bundle:\n\(error)") } do { let decoder = JSONDecoder() return try decoder.decode(T.self, from: data) } catch { fatalError("Couldn't parse \(file.description) as \(T.self):\n\(error)") } } } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Models/FileLoader.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation import os class FileLoader { /// Loads a vocabulary file into a dictionary of vocabulary to its ID. /// /// - Parameter file: `File` of a vocabulary. /// - Returns: Vocabulary IDs from given `file` data. static func loadVocabularies(from file: File) -> [String: Int32] { guard let path = Bundle(for: FileLoader.self).path(forResource: file.name, ofType: file.ext) else { fatalError("Cannot read the file: \(file.description)") } var vocabularyIDs = [String: Int32]() do { let data = try String(contentsOfFile: path, encoding: .utf8) for (index, string) in data.components(separatedBy: .newlines).enumerated() { vocabularyIDs[string] = Int32(index) } } catch { os_log("%s", type: .error, error.localizedDescription) } return vocabularyIDs } } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Models/ML/BertQAHandler.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation import TensorFlowLite import os /// Handles Bert model with TensorFlow Lite. final class BertQAHandler { private let interpreter: Interpreter private let dic: [String: Int32] private let tokenizer: FullTokenizer init( with modelFile: File = MobileBERT.model, threadCount: Int = InterpreterOptions.threadCount.defaultValue ) throws { os_log( "[BertQAHandler] Initialize interpreter with %d thread(s).", threadCount) // Load dictionary from vocabulary file. dic = FileLoader.loadVocabularies(from: MobileBERT.vocabulary) // Initialize `FullTokenizer` with given `dic`. tokenizer = FullTokenizer(with: dic, isCaseInsensitive: MobileBERT.doLowerCase) // Construct the path to the model file. guard let modelPath = Bundle(for: type(of: self)).path( forResource: modelFile.name, ofType: modelFile.ext) else { fatalError("Failed to load the model file: \(modelFile.description)") } // Specify the options for the `Interpreter`. var options = Interpreter.Options() options.threadCount = threadCount // Create the `Interpreter`. interpreter = try Interpreter(modelPath: modelPath, options: options) // Initialize input and output `Tensor`s. try interpreter.allocateTensors() // Get allocated input `Tensor`s. let inputIdsTensor: Tensor let inputMaskTensor: Tensor let segmentIdsTensor: Tensor inputIdsTensor = try interpreter.input(at: 0) inputMaskTensor = try interpreter.input(at: 1) segmentIdsTensor = try interpreter.input(at: 2) // Get allocated output `Tensor`s. let endLogitsTensor: Tensor let startLogitsTensor: Tensor endLogitsTensor = try interpreter.output(at: 0) startLogitsTensor = try interpreter.output(at: 1) // Check if input and output `Tensor`s are in the expected formats. guard inputIdsTensor.shape.dimensions == MobileBERT.inputDimension && inputMaskTensor.shape.dimensions == MobileBERT.inputDimension && segmentIdsTensor.shape.dimensions == MobileBERT.inputDimension else { fatalError("Unexpected model: Input Tensor shape") } guard endLogitsTensor.shape.dimensions == MobileBERT.outputDimension && startLogitsTensor.shape.dimensions == MobileBERT.outputDimension else { fatalError("Unexpected model: Output Tensor shape") } } func run(query: String, content: String) -> Result? { // MARK: - Preprocessing let (features, contentData) = preprocessing(query: query, content: content) // Convert input arrays to `Data` type. os_log("[BertQAHandler] Setting inputs..") let inputIdsData = Data(copyingBufferOf: features.inputIds) let inputMaskData = Data(copyingBufferOf: features.inputMask) let segmentIdsData = Data(copyingBufferOf: features.segmentIds) // MARK: - Inferencing let inferenceStartTime = Date() let endLogitsTensor: Tensor let startLogitsTensor: Tensor do { // Assign input `Data` to the `interpreter`. try interpreter.copy(inputIdsData, toInputAt: 0) try interpreter.copy(inputMaskData, toInputAt: 1) try interpreter.copy(segmentIdsData, toInputAt: 2) // Run inference by invoking the `Interpreter`. os_log("[BertQAHandler] Runing inference..") try interpreter.invoke() // Get the output `Tensor` to process the inference results endLogitsTensor = try interpreter.output(at: 0) startLogitsTensor = try interpreter.output(at: 1) } catch let error { os_log( "[BertQAHandler] Failed to invoke the interpreter with error: %s", type: .error, error.localizedDescription) return nil } let inferenceTime = Date().timeIntervalSince(inferenceStartTime) * 1000 // MARK: - Postprocessing os_log("[BertQAHandler] Getting answer..") let answers = postprocessing( startLogits: startLogitsTensor.data.toArray(type: Float32.self), endLogits: endLogitsTensor.data.toArray(type: Float32.self), contentData: contentData) os_log("[BertQAHandler] Finished.") guard let answer = answers.first else { return nil } return Result(answer: answer, inferenceTime: inferenceTime) } // MARK: - Private functions /// Tokenizes input query and content to `InputFeatures` and make `ContentData` to find the /// original string in the content. /// /// - Parameters: /// - query: Input query to run the model. /// - content: Input content to run the model. /// - Returns: A tuple of `InputFeatures` and `ContentData`. private func preprocessing(query: String, content: String) -> (InputFeatures, ContentData) { var queryTokens = tokenizer.tokenize(query) queryTokens = Array(queryTokens.prefix(MobileBERT.maxQueryLen)) let contentWords = content.splitByWhitespace() var contentTokenIdxToWordIdxMapping = [Int]() var contentTokens = [String]() for (i, token) in contentWords.enumerated() { tokenizer.tokenize(token).forEach { subToken in contentTokenIdxToWordIdxMapping.append(i) contentTokens.append(subToken) } } // -3 accounts for [CLS], [SEP] and [SEP]. let maxContentLen = MobileBERT.maxSeqLen - queryTokens.count - 3 contentTokens = Array(contentTokens.prefix(maxContentLen)) var tokens = [String]() var segmentIds = [Int32]() // Map token index to original index (in feature.origTokens). var tokenIdxToWordIdxMapping = [Int: Int]() // Start of generating the `InputFeatures`. tokens.append("[CLS]") segmentIds.append(0) // For query input. queryTokens.forEach { tokens.append($0) segmentIds.append(0) } // For separation. tokens.append("[SEP]") segmentIds.append(0) // For text input. for (i, docToken) in contentTokens.enumerated() { tokens.append(docToken) segmentIds.append(1) tokenIdxToWordIdxMapping[tokens.count] = contentTokenIdxToWordIdxMapping[i] } // For ending mark. tokens.append("[SEP]") segmentIds.append(1) var inputIds = tokenizer.convertToIDs(tokens: tokens) var inputMask = [Int32](repeating: 1, count: inputIds.count) while inputIds.count < MobileBERT.maxSeqLen { inputIds.append(0) inputMask.append(0) segmentIds.append(0) } let inputFeatures = InputFeatures( inputIds: inputIds, inputMask: inputMask, segmentIds: segmentIds) let contentData = ContentData( contentWords: contentWords, tokenIdxToWordIdxMapping: tokenIdxToWordIdxMapping, originalContent: content) return (inputFeatures, contentData) } /// Get a list of most possible `Answer`s up to `Model.predictAnsNum`. /// /// - Parameters: /// - startLogits: List of `Logit` if the index can be a start token index of an answer. /// - endLogits: List of `Logit` if the index can be a end token index of an answer. /// - features: `InputFeatures` used to run the model. /// - Returns: List of `Answer`s. private func postprocessing( startLogits: [Float], endLogits: [Float], contentData: ContentData ) -> [Answer] { // Get the candidate start/end indexes of answer from `startLogits` and `endLogits`. let startIndexes = candidateAnswerIndexes(from: startLogits) let endIndexes = candidateAnswerIndexes(from: endLogits) // Make list which stores prediction and its range to find original results and filter invalid // pairs. let candidates: [Prediction] = startIndexes.flatMap { start in endIndexes.compactMap { end -> Prediction? in // Initialize logit struct with given indexes. guard let prediction = Prediction( logit: startLogits[start] + endLogits[end], start: start, end: end, tokenIdxToWordIdxMapping: contentData.tokenIdxToWordIdxMapping) else { return nil } return prediction } }.sorted { $0.logit > $1.logit } // Extract firstmost `Model.predictAnsNum` of predictions and calculate score from logits array // with softmax. let scores = softmaxed(Array(candidates.prefix(MobileBERT.predictAnsNum))) // Return answer list. return scores.compactMap { score in guard let excerpted = contentData.excerptWords(from: score.range) else { return nil } return Answer(text: excerpted, score: score) } } /// Get the `Model.prediectAnsNum` number of indexes of the candidate answers from given logit /// list. /// /// - Parameter from: The array of logits. /// - Returns: `Model.predictAnsNum` number of indexes. private func candidateAnswerIndexes(from logits: [Float]) -> [Int] { return logits.prefix(MobileBERT.maxSeqLen) .enumerated() .sorted { $0.element > $1.element } .prefix(MobileBERT.predictAnsNum) .map { $0.offset } } /// Compute softmax probability score over raw logits. /// /// - Parameter predictions: Array of logit and it range sorted by the logit value in decreasing /// order. private func softmaxed(_ predictions: [Prediction]) -> [Score] { // Find maximum logit value. guard let maximumLogit = predictions.first?.logit else { return [] } // Calculate numerator array of the softmaxed values and its sum. let numerators: [(Float, Prediction)] = predictions.map { prediction in let numerator = exp(prediction.logit - maximumLogit) return (numerator, prediction) } let sum: Float = numerators.reduce(0) { $0 + $1.0 } return numerators.compactMap { (numerator, prediction) in Score(value: numerator / sum, range: prediction.wordRange, logit: prediction.logit) } } } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Models/ML/ContentData.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation /// Manages content data to excerpt answer string with a token index. struct ContentData { let contentWords: [String] let tokenIdxToWordIdxMapping: [Int: Int] let originalContent: String /// Find and excerpt the original string with given index. /// /// - Parameters: /// - feature: `InputFeature` of the query. /// - range: Range of token ID of the string to find. /// - Returns: Returns the original string with given range. `nil` if the index is not correct. func excerptWords(from range: ClosedRange) -> Excerpt? { let pattern: String = contentWords[range].map { NSRegularExpression.escapedPattern(for: $0) }.joined(separator: "\\s+") let compareOptions: NSString.CompareOptions if range.count == 1 { compareOptions = [] } else { compareOptions = .regularExpression } guard let range = originalContent.range(of: pattern, options: compareOptions) else { return nil } return Excerpt(value: String(originalContent[range]), range: range) } } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Models/ML/InputFeatures.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation /// Stores input features for BERT model. struct InputFeatures { let inputIds: [Int32] let inputMask: [Int32] let segmentIds: [Int32] } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Models/ML/Result.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation struct Result { let answer: Answer let inferenceTime: Double var description: String { """ Inference time: \(String(format: "%.2lf ms", inferenceTime)) Score: \(String(format: "%.2lf", answer.score.value)) """ } } struct Answer { let text: Excerpt let score: Score } /// Stores exceprted text and its range from the original text. struct Excerpt { let value: String let range: Range } /// Stores probability score of given range of words in the original content. struct Score { let value: Float /// Score's range of original word. let range: ClosedRange /// Logit value of this score. let logit: Float } /// Stores logit value and its range in token and word list. struct Prediction { /// Logit value. let logit: Float /// Logit's range of result token. let tokenRange: ClosedRange /// Logit's range of original word. let wordRange: ClosedRange init?(logit: Float, start: Int, end: Int, tokenIdxToWordIdxMapping: [Int: Int]) { self.logit = logit guard start <= end else { return nil } self.tokenRange = start...end guard let wordRange = Prediction.convert(from: tokenRange, with: tokenIdxToWordIdxMapping) else { return nil } self.wordRange = wordRange } private static func convert(from tokenRange: ClosedRange, with map: [Int: Int]) -> ClosedRange? { guard tokenRange.count <= MobileBERT.maxAnsLen, let start = tokenRange.first, let end = tokenRange.last, let startIndex = map[start + MobileBERT.outputOffset], let endIndex = map[end + MobileBERT.outputOffset] else { return nil } guard startIndex <= endIndex else { return nil } return startIndex...endIndex } } extension Prediction: Equatable { static func == (lhs: Prediction, rhs: Prediction) -> Bool { return lhs.logit == rhs.logit && lhs.tokenRange == rhs.tokenRange } } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Models/Tokenizers/BasicTokenizer.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation /// Runs basic tokenization such as punctuation spliting, lower casing. /// /// Name of functions and variables are from /// [google-research/bert]( https://github.com/google-research/bert/blob/d66a146741588fb208450bde15aa7db143baaa69/tokenization.py#L185). struct BasicTokenizer { let isCaseInsensitive: Bool /// Constructs a BasicTokenizer. /// /// - Parameter isCaseInsensitive: Whether to lower case the input. init(isCaseInsensitive: Bool) { self.isCaseInsensitive = isCaseInsensitive } /// Tokenizes a piece of text. /// /// - Parameter text: Text to be tokenized. func tokenize(_ text: String) -> [String] { var cleanedText = text.cleaned() if isCaseInsensitive { cleanedText = cleanedText.lowercased() } return cleanedText.splitByWhitespace().flatMap { $0.tokenizedWithPunctuation() } } } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Models/Tokenizers/FullTokenizer.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation /// Runs end-to-end tokenization. /// /// Name of functions and variables are from /// [google-research/bert](https://github.com/google-research/bert/blob/d66a146741588fb208450bde15aa7db143baaa69/tokenization.py#L161). class FullTokenizer { let basicTokenizer: BasicTokenizer let wordpieceTokenizer: WordpieceTokenizer /// A mapping of strings to IDs, where the IDs come from let vocabularyIDs: [String: Int32] /// Initialize `Fulltokenizer`. /// - Parameters /// - vocabularyIDs: A mapping of strings to IDs, where the IDs come from the number order of /// vocabulary from the vocabulary file. /// - isCaseInsensitive: `true` if the tokenizer ignores the case. init(with vocabularyIDs: [String: Int32], isCaseInsensitive: Bool) { self.vocabularyIDs = vocabularyIDs basicTokenizer = BasicTokenizer(isCaseInsensitive: isCaseInsensitive) wordpieceTokenizer = WordpieceTokenizer(with: vocabularyIDs) } func tokenize(_ text: String) -> [String] { return basicTokenizer.tokenize(text).flatMap { wordpieceTokenizer.tokenize($0) } } func convertToIDs(tokens: [String]) -> [Int32] { return tokens.compactMap { vocabularyIDs[$0] } } } ================================================ FILE: lite/examples/bert_qa/ios/BertQACore/Models/Tokenizers/WordpieceTokenizer.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation /// Runs WordPiece tokenziation. /// /// Name of functions and variables are from /// [google-research/bert](https://github.com/google-research/bert/blob/d66a146741588fb208450bde15aa7db143baaa69/tokenization.py#L300). struct WordpieceTokenizer { let vocabularyIDs: [String: Int32] private static let UNKNOWN_TOKEN = "[UNK]" // For unknown words. private static let MAX_INPUT_CHARS_PER_WORD = 200 init(with vocaburalyID: [String: Int32]) { self.vocabularyIDs = vocaburalyID } /// Tokenizes a piece of text into its word pieces. /// /// This uses a greedy longest-match-first algorithm to perform tokenization using the given /// vocabulary. /// /// For example: /// ``` /// input = "unaffable" /// output = ["un", "##aff", "##able"] /// ``` /// /// ``` /// input = "unaffableX" /// output = ["[UNK]"] /// ``` /// /// - Parameter text: A single token or whitespace separated tokens. This should have already been /// passed through `BasicTokenizer. /// - Returns: A list of wordpiece tokens. func tokenize(_ text: String) -> [String] { var outputTokens = [String]() text.splitByWhitespace().forEach { token in if token.count > WordpieceTokenizer.MAX_INPUT_CHARS_PER_WORD { outputTokens.append(WordpieceTokenizer.UNKNOWN_TOKEN) return } var start = token.startIndex var subWords = [String]() // Find all subwords in `token`. while start < token.endIndex { var end = token.endIndex var hasFound = false // Find longest known subword in the `token` from its `start` index. while start < end { var subStr = String(token[start.. token.startIndex { subStr = "##" + subStr } if vocabularyIDs[subStr] != nil { // `subStr` is the longest subword that can be found. hasFound = true subWords.append(subStr) break } else { end = token.index(before: end) } } if hasFound { // Proceed to tokenize the residual string. start = end } else { // The `token` contains unknown subwords. It can't be tokenized into subwords. subWords = [WordpieceTokenizer.UNKNOWN_TOKEN] break } } outputTokens.append(contentsOf: subWords) } return outputTokens } } ================================================ FILE: lite/examples/bert_qa/ios/BertQATests/ExtensionTest/StringExtensionTest.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import XCTest @testable import BertQA_UIKit class StringExtensionTest: XCTestCase { func testCleaned() { let testExample1 = "This is an\rexample.\n" let testExample2 = testExample1 + "\u{0}" let testExample3 = testExample1 + "\u{fffd}" let expectedResult = "This is an example. " XCTAssertEqual(testExample1.cleaned(), expectedResult) XCTAssertEqual(testExample2.cleaned(), expectedResult) XCTAssertEqual(testExample3.cleaned(), expectedResult) } func testSplitByWhitespace() { let testExample = "Hi ,\n This is an example. " let expectedResult = ["Hi", ",", "This", "is", "an", "example."] let reversedResult = [String](expectedResult.reversed()) XCTAssertNotEqual(testExample.splitByWhitespace(), reversedResult) XCTAssertEqual(testExample.splitByWhitespace(), expectedResult) XCTAssertEqual(" ".splitByWhitespace(), []) XCTAssertEqual("".splitByWhitespace(), []) } func testTokenizedWithPunctuation() { let testExample1 = "Hi,there." let expectedResult1 = ["Hi", ",", "there", "."] let reversedResult1 = [String](expectedResult1.reversed()) XCTAssertEqual(testExample1.tokenizedWithPunctuation(), expectedResult1) XCTAssertNotEqual(testExample1.tokenizedWithPunctuation(), reversedResult1) let testExample2 = "I\'m \"Spider-Man\"" // Input: I'm "Spider-Man" let expectedResult2 = ["I", "\'", "m ", "\"", "Spider", "-", "Man", "\""] let reversedResult2 = [String](expectedResult2.reversed()) XCTAssertNotEqual(testExample2.tokenizedWithPunctuation(), reversedResult2) XCTAssertEqual(testExample2.tokenizedWithPunctuation(), expectedResult2) XCTAssertEqual("".tokenizedWithPunctuation(), []) } } ================================================ FILE: lite/examples/bert_qa/ios/BertQATests/ExtensionTest/UnicodeScalarExtensionTest.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import XCTest @testable import BertQA_UIKit class UnicodeScalarExtensionTest: XCTestCase { func testIsWhitespaceForBert() { XCTAssertTrue(UnicodeScalar(" ").isWhitespaceForBert) XCTAssertTrue(UnicodeScalar("\t").isWhitespaceForBert) XCTAssertTrue(UnicodeScalar("\r").isWhitespaceForBert) XCTAssertTrue(UnicodeScalar("\n").isWhitespaceForBert) XCTAssertTrue(UnicodeScalar(0x00A0).isWhitespaceForBert) XCTAssertFalse(UnicodeScalar("A").isWhitespaceForBert) XCTAssertFalse(UnicodeScalar("-").isWhitespaceForBert) } func testIsControlForBert() { XCTAssertTrue(UnicodeScalar(0x0005).isControlForBert) XCTAssertFalse(UnicodeScalar("A").isControlForBert) XCTAssertFalse(UnicodeScalar(" ").isControlForBert) XCTAssertFalse(UnicodeScalar("\t").isControlForBert) XCTAssertFalse(UnicodeScalar("\r").isControlForBert) XCTAssertFalse(UnicodeScalar("\u{1F4A9}").isControlForBert) } func testIsPunctuationForBert() { XCTAssertTrue(UnicodeScalar("-").isPunctuationForBert) XCTAssertTrue(UnicodeScalar("$").isPunctuationForBert) XCTAssertTrue(UnicodeScalar("`").isPunctuationForBert) XCTAssertTrue(UnicodeScalar(".").isPunctuationForBert) XCTAssertFalse(UnicodeScalar("A").isPunctuationForBert) XCTAssertFalse(UnicodeScalar(" ").isPunctuationForBert) } } ================================================ FILE: lite/examples/bert_qa/ios/BertQATests/Info.plist ================================================ CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleExecutable $(EXECUTABLE_NAME) CFBundleIdentifier $(PRODUCT_BUNDLE_IDENTIFIER) CFBundleInfoDictionaryVersion 6.0 CFBundleName $(PRODUCT_NAME) CFBundlePackageType $(PRODUCT_BUNDLE_PACKAGE_TYPE) CFBundleShortVersionString 1.0 CFBundleVersion 1 ================================================ FILE: lite/examples/bert_qa/ios/BertQATests/MLTest/BertQAHandlerTest.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import XCTest @testable import BertQA_UIKit class BertQAHandlerTest: XCTestCase { /// Test BertQA handler with ASCII code only content & question. func testBertQAHandler() { let bertQA: BertQAHandler do { bertQA = try BertQAHandler() let content = "TensorFlow is a free and open-source software library for dataflow and " + "differentiable programming across a range of tasks. It is a symbolic math library, and " + "is also used for machine learning applications such as neural networks. It is used for " + "both research and production at Google. TensorFlow was developed by the Google Brain " + "team for internal Google use. It was released under the Apache License 2.0 on November " + "9, 2015." let question1 = "What is TensorFlow" let answer1 = "a free and open-source software library for dataflow and differentiable " + "programming across a range of tasks" if let result1 = bertQA.run(query: question1, content: content) { XCTAssert(result1.answer.text.value.contains(answer1)) } else { XCTFail("Failed to run BertQA with \(question1).") } let question2 = "Who developed TensorFlow?" let answer2 = "Google Brain team" if let result2 = bertQA.run(query: question2, content: content) { XCTAssert(result2.answer.text.value.contains(answer2)) } else { XCTFail("Failed to run BertQA with \(question2).") } let question3 = "When was TensorFlow released?" let answer3 = "November 9, 2015" if let result3 = bertQA.run(query: question3, content: content) { XCTAssert(result3.answer.text.value.contains(answer3)) } else { XCTFail("Failed to run BertQA with \(question3).") } let question4 = "What is TensorFlow used for?" let answer4 = "symbolic math library, and is also used for machine learning applications such as neural " + "networks" if let result4 = bertQA.run(query: question4, content: content) { XCTAssert(result4.answer.text.value.contains(answer4)) } else { XCTFail("Failed to run BertQA with \(question4).") } let question5 = "How is TensorFlow used in Google?" let answer5 = "both research and production" if let result5 = bertQA.run(query: question5, content: content) { XCTAssert(result5.answer.text.value.contains(answer5)) } else { XCTFail("Failed to run BertQA with \(question5).") } let question6 = "Which license does TensorFlow use?" let answer6 = "Apache License 2.0" if let result6 = bertQA.run(query: question6, content: content) { XCTAssert(result6.answer.text.value.contains(answer6)) } else { XCTFail("Failed to run BertQA with \(question6).") } } catch let error { XCTFail(error.localizedDescription) } } /// Test BertQA handler with a content & question including unicode. func testBertQAHandlerWithUnicode() { let bertQA: BertQAHandler do { bertQA = try BertQAHandler() let content = "Nikola Tesla (Serbian Cyrillic: \u{041d}\u{0438}\u{043a}\u{043e}\u{043b}" + "\u{0430} \u{0422}\u{0435}\u{0441}\u{043b}\u{0430}; 10 July 1856 \u{2013} 7 January 1943) " + "was a Serbian American inventor, electrical engineer, mechanical engineer, physicist, and " + "futurist best known for his contributions to the design of the modern alternating current " + "(AC) electricity supply system." let question1 = "What is Tesla's home country?" let answer1 = "Serbian" if let result1 = bertQA.run(query: question1, content: content) { XCTAssert(result1.answer.text.value.contains(answer1)) } else { XCTFail() XCTFail("Failed to run BertQA with \(question1).") } let question2 = "What was Nikola Tesla's ethnicity?" let answer2 = "Serbian" if let result2 = bertQA.run(query: question2, content: content) { XCTAssert(result2.answer.text.value.contains(answer2)) } else { XCTFail("Failed to run BertQA with \(question2).") } let question3 = "What does AC stand for?" let answer3 = "alternating current" if let result3 = bertQA.run(query: question3, content: content) { XCTAssert(result3.answer.text.value.contains(answer3)) } else { XCTFail("Failed to run BertQA with \(question3).") } let question4 = "When was Tesla born?" let answer4 = "10 July 1856" if let result4 = bertQA.run(query: question4, content: content) { XCTAssert(result4.answer.text.value.contains(answer4)) } else { XCTFail("Failed to run BertQA with \(question4).") } let question5 = "In what year did Tesla die?" let answer5 = "1943" if let result5 = bertQA.run(query: question5, content: content) { XCTAssert(result5.answer.text.value.contains(answer5)) } else { XCTFail("Failed to run BertQA with \(question5).") } } catch let error { XCTFail(error.localizedDescription) } } } ================================================ FILE: lite/examples/bert_qa/ios/BertQATests/TokenizerTest/BasicTokenizerTest.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import XCTest @testable import BertQA_UIKit class BasicTokenizerTest: XCTestCase { func testTokenize() { let tokenizer = BasicTokenizer(isCaseInsensitive: false) let testInput1 = " Hi, This\tis an example.\n" let expectedResult1 = ["Hi", ",", "This", "is", "an", "example", "."] XCTAssertEqual(tokenizer.tokenize(testInput1), expectedResult1) let testInput2 = "Hello,How are you?" let expectedResult2 = ["Hello", ",", "How", "are", "you", "?"] XCTAssertEqual(tokenizer.tokenize(testInput2), expectedResult2) } } ================================================ FILE: lite/examples/bert_qa/ios/BertQATests/TokenizerTest/FullTokenizerTest.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import XCTest import os @testable import BertQA_UIKit class FullTokenizerTest: XCTestCase { func testTokenize() { let vocabularyIDs = FileLoader.loadVocabularies(from: MobileBERT.vocabulary) let tokenizer = FullTokenizer(with: vocabularyIDs, isCaseInsensitive: true) XCTAssertEqual(tokenizer.tokenize(""), []) let testInput1 = "Good morning, I'm your teacher.\n" let expectedResult1 = ["good", "morning", ",", "i", "'", "m", "your", "teacher", "."] XCTAssertEqual(tokenizer.tokenize(testInput1), expectedResult1) let testInput2 = "Nikola Tesla\t(Serbian Cyrillic: 10 July 1856 ~ 7 January 1943)" let expectedResult2 = [ "nikola", "tesla", "(", "serbian", "cyrillic", ":", "10", "july", "1856", "~", "7", "january", "1943", ")", ] XCTAssertEqual(tokenizer.tokenize(testInput2), expectedResult2) } func testConvertTokensToIds() { let vocabularyIDs = FileLoader.loadVocabularies(from: MobileBERT.vocabulary) let tokenizer = FullTokenizer(with: vocabularyIDs, isCaseInsensitive: true) let testInput = ["good", "morning", ",", "i", "'", "m", "your", "teacher", "."] let expectedResult: [Int32] = [2204, 2851, 1010, 1045, 1005, 1049, 2115, 3836, 1012] XCTAssertEqual(tokenizer.convertToIDs(tokens: testInput), expectedResult) } } ================================================ FILE: lite/examples/bert_qa/ios/BertQATests/TokenizerTest/WordpieceTokenizerTest.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import XCTest @testable import BertQA_UIKit class WordpieceTokenizerTest: XCTestCase { func testTokenize() { let vocabularyIDs = FileLoader.loadVocabularies(from: MobileBERT.vocabulary) let tokenizer = WordpieceTokenizer(with: vocabularyIDs) XCTAssertEqual(tokenizer.tokenize("meaningfully"), ["meaningful", "##ly"]) XCTAssertEqual(tokenizer.tokenize("teacher"), ["teacher"]) } func testTokenizerWithCustomvocabularyIDs() { let vocab = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"] var vocabularyIDs = [String: Int32]() for (index, string) in vocab.enumerated() { vocabularyIDs[string] = Int32(index) } let tokenizer = WordpieceTokenizer(with: vocabularyIDs) XCTAssertEqual(tokenizer.tokenize(""), []) XCTAssertEqual( tokenizer.tokenize("unwanted running"), ["un", "##want", "##ed", "runn", "##ing"]) XCTAssertEqual(tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) } } ================================================ FILE: lite/examples/bert_qa/ios/Podfile ================================================ # Uncomment the next line to define a global platform for your project platform :ios, '12.0' target 'BertQA-UIKit' do # Comment the next line if you don't want to use dynamic frameworks use_frameworks! # Pods for BertQA-UIKit pod 'TensorFlowLiteSwift', '~> 0.0.1-nightly' end target 'BertQA-SwiftUI' do # Comment the next line if you don't want to use dynamic frameworks use_frameworks! # Pods for BertQA-SwiftUI pod 'TensorFlowLiteSwift', '~> 0.0.1-nightly' end ================================================ FILE: lite/examples/bert_qa/ios/README.md ================================================ # TensorFlow Lite BERT QA iOS Example Application ![UIKit screencast] | ![SwiftUI screencast] :-----------------------: | :-------------------------: UIKit version screen cast | SwiftUI version screen cast ## Overview This is an end-to-end example of [BERT] Question & Answer application built with TensorFlow 2.0, and tested on [SQuAD] dataset version 1.1. The demo app provides 48 passages from the dataset for users to choose from, and gives 5 most possible answers corresponding to the input passage and query. This example includes two types of application and a test set, one application is developed with [UIKit], and the other is developed with [SwiftUI]. Each application can be run with this XCode project, by choosing the target to build. Each application shares the core logic needed to run the BertQA model. The test set tests this core logic. Question input to the application is discarded after inference. ### Model used [BERT], or Bidirectional Encoder Representations from Transformers, is a method of pre-training language representations which obtains state-of-the-art results on a wide array of Natural Language Processing tasks. This app uses MobileBERT, a compressed version of [BERT] that runs 4x faster and has 4x smaller model size. For more information, refer to the [BERT github page][BERT]. ### Preprocessing While preprocessing, it tokenizes input query and content with given vocabulary data. Tokenization process mostly followed the [bert tokenization], except handling Chinese characters as we don’t have passages written in Chinese. It hands over the IDs of tokens with two other data. One is its segment ID to verify whether it is a question or a content. The other is a mask, which indicates if the given token is valid input to be processed or just a padding to fit the input tensor. As the model requires a fixed size of token ID array, some entries of the array could have invalid data. ### Inference Assign preprocessed data to the input tensor and run the model. Output data is assigned to the output tensors as a result. ### Postprocessing While postprocessing, it retrieves original string from the arrays of start & end logits in the output tensor. A logit of a token from `start` to `end` is derived from a summation of start logit array’s `start`th value and end logit array’s `end`th value. The higher sum of two logits, the more likely the token between starting point and end point could be an answer. After finding an answer, it retrieves the original string in the given range and calculates the score. The score of the answer is calculated by the softmax function. ## Requirements * Xcode 11.0 or above * Valid Apple Developer ID * Real iOS device Note: You can also use an iOS emulator, but some of the functionality may not be fully supported. * iOS version 12.0 or above * Xcode command line tools (to install, `run xcode-select --install`) * CocoaPods (to install, `run sudo gem install cocoapods`) ## Build and run 1. Clone the TensorFlow examples GitHub repository to your computer to get the demo application: `git clone https://github.com/tensorflow/examples` 1. Install the pod to generate the workspace file: `cd examples/lite/examples/bert_qa/ios && pod install` Note: If you have installed this pod before and that command doesn't work, try `pod update`. At the end of this step you should have a directory called `BertQA.xcworkspace`. 1. Open the project in Xcode with the following command: `open BertQA.xcworkspace` This launches Xcode and opens the BertQA project. 1. In the Menu bar, select `Product` → `Destination` and choose your device. 1. Follow the direction below if you want to: * Run the application: 1. In the Menu bar, select `Product` → `Scheme` and choose `BertQA-UIKit` or `BertQA-SwiftUI`. 1. In the Menu bar, select `Product` → `Run` to install the app on your device. * Test the core logic: 1. In the Menu bar, select `Product` --> `Scheme` and choose `BertQA-UIKit`. 1. In the Menu bar, select `Product` --> `Test`. [UIKit screencast]: https://storage.googleapis.com/download.tensorflow.org/models/tflite/screenshots/bertqa_ios_uikit_demo.gif [SwiftUI screencast]: https://storage.googleapis.com/download.tensorflow.org/models/tflite/screenshots/bertqa_ios_swiftui_demo.gif [BERT]: https://github.com/google-research/bert [SQuAD]: https://rajpurkar.github.io/SQuAD-explorer/ [UIKit]: https://developer.apple.com/documentation/uikit [SwiftUI]: https://developer.apple.com/documentation/swiftui [bert tokenization]: https://github.com/google-research/bert#tokenization ================================================ FILE: lite/examples/bert_qa/ios/RunScripts/download_resources.sh ================================================ #!/bin/bash # Copyright 2020 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== # Download TF Lite model from the internet if it does not exist. TFLITE_RESOURCES="mobilebert_qa_vocab.zip" TFLITE_MODEL="mobilebert_float_20191023.tflite" TFLITE_VOCA="vocab.txt" TFLITE_DIC="contents_from_squad_dict_format.json" TFLITE_URL="https://storage.googleapis.com/download.tensorflow.org/models/tflite/bert_qa" TFLITE_MODEL_REMOTE_PATH="https://storage.googleapis.com/download.tensorflow.org/models/tflite/task_library/bert_qa/ios/models_tflite_bert_qa_mobilebert_float_20191023.tflite" RESOURCES_DIR="BertQACore/Resources" RESOURCES_ZIP_PATH="${RESOURCES_DIR}/${TFLITE_RESOURCES}" MODEL_PATH="${RESOURCES_DIR}/${TFLITE_MODEL}" VOCA_PATH="${RESOURCES_DIR}/${TFLITE_VOCA}" DIC_PATH="${RESOURCES_DIR}/${TFLITE_DIC}" RESOURCES=($MODEL_PATH $VOCA_PATH) function is_all_existing() { local bool="true" local files=("$@") for file in ${files[@]} do if [ ! -f "${file}" ]; then bool="false" fi done echo "${bool}" } tf_resource_exists=$( is_all_existing "${RESOURCES[@]}" ) if [ "${tf_resource_exists}" = "false" ]; then # Download zipped resources. curl --create-dirs -o "${RESOURCES_ZIP_PATH}" "${TFLITE_URL}/${TFLITE_RESOURCES}" unzip -n "${RESOURCES_ZIP_PATH}" -d "${RESOURCES_DIR}" rm "${RESOURCES_ZIP_PATH}" # Remove old tflite model. rm "${MODEL_PATH}" # Download new tflite model. curl --create-dirs -o "${MODEL_PATH}" "${TFLITE_MODEL_REMOTE_PATH}" echo "INFO: Downloaded TensorFlow Lite resources to ${RESOURCES_DIR}." fi if [ ! -f "${DIC_PATH}" ]; then # Donwload content data. curl --create-dirs -o "${DIC_PATH}" "${TFLITE_URL}/${TFLITE_DIC}" echo "INFO: Downloaded content and question data to ${RESOURCES_DIR}." fi echo "INFO: All resources are prepared." ================================================ FILE: lite/examples/bert_qa/ios/ViewInSwiftUI/AppDelegate.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit @UIApplicationMain class AppDelegate: UIResponder, UIApplicationDelegate { var window: UIWindow? func application( _ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? ) -> Bool { return true } // MARK: UISceneSession Lifecycle func application( _ application: UIApplication, configurationForConnecting connectingSceneSession: UISceneSession, options: UIScene.ConnectionOptions ) -> UISceneConfiguration { return UISceneConfiguration( name: "Default Configuration", sessionRole: connectingSceneSession.role) } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInSwiftUI/Base.lproj/LaunchScreen.storyboard ================================================ ================================================ FILE: lite/examples/bert_qa/ios/ViewInSwiftUI/Info.plist ================================================ CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleExecutable $(EXECUTABLE_NAME) CFBundleIdentifier $(PRODUCT_BUNDLE_IDENTIFIER) CFBundleInfoDictionaryVersion 6.0 CFBundleName $(PRODUCT_NAME) CFBundlePackageType APPL CFBundleShortVersionString 1.0 CFBundleVersion 1 LSRequiresIPhoneOS UIApplicationSceneManifest UIApplicationSupportsMultipleScenes UISceneConfigurations UIWindowSceneSessionRoleApplication UISceneConfigurationName Default Configuration UISceneDelegateClassName BertQA_SwiftUI.SceneDelegate UILaunchStoryboardName LaunchScreen UIRequiredDeviceCapabilities armv7 UISupportedInterfaceOrientations UIInterfaceOrientationPortrait UISupportedInterfaceOrientations~ipad UIInterfaceOrientationLandscapeRight UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationPortrait UIUserInterfaceStyle Light ================================================ FILE: lite/examples/bert_qa/ios/ViewInSwiftUI/KeyboardHeightObserver.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import SwiftUI /// Observes keyboard height events. final class KeyboardHeightObserver: ObservableObject { @Published private(set) var height: CGFloat = 0 init() { NotificationCenter.default.addObserver( self, selector: #selector(keyBoardWillShow(notification:)), name: UIResponder.keyboardWillShowNotification, object: nil) NotificationCenter.default.addObserver( self, selector: #selector(keyBoardWillHide(notification:)), name: UIResponder.keyboardWillHideNotification, object: nil) } deinit { NotificationCenter.default.removeObserver(self) } @objc func keyBoardWillShow(notification: Notification) { if let keyboardFrame = (notification.userInfo?[UIResponder.keyboardFrameEndUserInfoKey] as? NSValue)? .cgRectValue { height = keyboardFrame.height } else { height = 0 } } @objc func keyBoardWillHide(notification: Notification) { height = 0 } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInSwiftUI/SceneDelegate.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import SwiftUI import UIKit class SceneDelegate: UIResponder, UIWindowSceneDelegate { var window: UIWindow? func scene( _ scene: UIScene, willConnectTo session: UISceneSession, options connectionOptions: UIScene.ConnectionOptions ) { // Use a UIHostingController as window root view controller if let windowScene = scene as? UIWindowScene { let bertQA: BertQAHandler do { bertQA = try BertQAHandler() } catch let error { fatalError(error.localizedDescription) } let window = UIWindow(windowScene: windowScene) window.rootViewController = UIHostingController( rootView: DatasetListView(datasets: Dataset.load(), bertQA: bertQA)) self.window = window window.makeKeyAndVisible() } } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInSwiftUI/Views/ContentView.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import SwiftUI /// Content view to show dataset's content and highlight answer of the question if exists. struct ContentView: View { var highlightRange: Range? var content: String var body: some View { guard let range = highlightRange else { return Text(content) } // If there's range to highlight, change the color of that range. return Text(content[..? = nil @EnvironmentObject var keyboard: KeyboardHeightObserver var body: some View { GeometryReader { geometry in VStack { Group { ScrollView(.vertical, showsIndicators: true) { ContentView(highlightRange: self.highlightRange, content: self.dataset.content) } } .padding(CustomUI.contentViewPadding) .onTapGesture { self.hideKeyboard() } Spacer() VStack(spacing: CustomUI.stackSpacing) { StatusView(statusMessage: self.$statusMessage) HStack { Text("You might want to ask:") .font(.footnote) Spacer() } SuggestedQuestionsView(questions: self.dataset.questions, toFill: self.$question) HStack { TextField( "Enter question", text: self.$question, onEditingChanged: { _ in } ) .textFieldStyle(RoundedBorderTextFieldStyle()) .animation(.easeOut(duration: CustomUI.keyboardAnimationDuration)) Button(action: self.tapRunButton) { Text("▶︎").font(.title) } .foregroundColor(Color.orange.opacity(CustomUI.runButtonOpacity)) } } .navigationBarTitle(Text(self.dataset.title), displayMode: .inline) .padding([.leading, .trailing, .bottom], CGFloat(CustomUI.controlViewPadding)) .padding(.all, CustomUI.padding) .padding(.bottom, max(0, self.keyboard.height - geometry.safeAreaInsets.bottom)) } } } func tapRunButton() { // Clean up previous result. highlightRange = nil // Hide keyboard. self.hideKeyboard() // Trim the whitespaces and newlines in the first and end. var query = question.trimmingCharacters(in: .whitespacesAndNewlines) guard !query.isEmpty else { os_log("Textfield failed to filter the empty query.") statusMessage = StatusMessage.warnEmptyQuery return } // A query must end with question mark. if query.last != "?" { query.append("?") } // Inference the answer with BertQA model. guard let result = bertQA.run(query: query, content: dataset.content) else { os_log("Failed to inference the answer.") statusMessage = StatusMessage.inferenceFailError return } statusMessage = result.description // Render the answer in the `contentView`. highlightRange = result.answer.text.range } func hideKeyboard() { UIApplication.shared.sendAction( #selector(UIResponder.resignFirstResponder), to: nil, from: nil, for: nil) } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInSwiftUI/Views/DatasetListView.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import SwiftUI /// List of Data Sets. struct DatasetListView: View { let datasets: [Dataset] let bertQA: BertQAHandler var keyboardHeightObserver = KeyboardHeightObserver() var body: some View { NavigationView { List(datasets, id: \.title) { dataset in NavigationLink( destination: DatasetDetailView(dataset: dataset, bertQA: self.bertQA) .environmentObject(self.keyboardHeightObserver) ) { DatasetRow(dataset: dataset) } } .navigationBarTitle(Text("Question and Answer")) } } } /// Row for each data set. struct DatasetRow: View { var dataset: Dataset var body: some View { HStack { Text(dataset.title) Spacer() } } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInSwiftUI/Views/StatusView.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import SwiftUI /// Status View to inform the result of inference or error. struct StatusView: View { @Binding var statusMessage: String var body: some View { Text(self.statusMessage) .frame(maxWidth: .infinity, minHeight: 40, maxHeight: 40, alignment: .topLeading) .padding(.all, CustomUI.padding) .font(.system(size: CustomUI.statusFontSize, weight: .semibold)) .lineLimit(2) .background(Color(red: 255 / 255, green: 244 / 255, blue: 229 / 255)) .cornerRadius(CustomUI.statusTextViewCornerRadius) } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInSwiftUI/Views/SuggestedQuestionsView.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import SwiftUI /// A container view of a list of suggested question buttons. struct SuggestedQuestionsView: View { var questions: [String] @Binding var toFill: String var body: some View { ScrollView(.horizontal, showsIndicators: false) { HStack { ForEach(questions, id: \.self) { question in Button(action: { self.toFill = question }) { SuggestedQuestionText(text: question) } .buttonStyle(SuggestedQuestionStyle()) } } } } } /// Text on a suggested question button. struct SuggestedQuestionText: View { var text: String var body: some View { Text(text) .font(.caption) .fontWeight(.medium) .foregroundColor(.black) .padding(CustomUI.textPadding) .padding([.leading, .trailing], CustomUI.textSidePadding) } } /// Style of a suggested question button. struct SuggestedQuestionStyle: ButtonStyle { func makeBody(configuration: Self.Configuration) -> some View { configuration.label.background( RoundedRectangle(cornerRadius: CustomUI.suggestedQuestionCornerRadius) .stroke( LinearGradient( gradient: Gradient(colors: [Color.orange, Color.orange.opacity(0.7), Color.yellow]), startPoint: .topLeading, endPoint: .bottomTrailing), lineWidth: 1 ) .padding(1) ).scaleEffect(configuration.isPressed ? 0.95 : 1.0) } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/AppDelegate.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit @UIApplicationMain class AppDelegate: UIResponder, UIApplicationDelegate { var window: UIWindow? func application( _ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? ) -> Bool { // Set default value of the application options. UserDefaults.standard.register(defaults: [ InterpreterOptions.threadCount.id: InterpreterOptions.threadCount.defaultValue, ]) return true } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/Base.lproj/LaunchScreen.storyboard ================================================ ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/Base.lproj/Main.storyboard ================================================ Lorem ipsum dolor sit er elit lamet, consectetaur cillium adipisicing pecu, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Nam liber te conscient to factor tum poen legum odioque civiuda. ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/Controllers/DataSetDetailViewController.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit import os /// Controller for detailed view of each `Dataset`. class DatasetDetailViewController: UIViewController { /// Selected `Dataset` to run the Bert model. var dataset: Dataset? var bertQA: BertQAHandler? // MARK: Views created at Storyboard @IBOutlet weak var contentView: UITextView! @IBOutlet weak var statusTextView: UITextView! @IBOutlet weak var questionField: UITextField! @IBOutlet weak var runButton: UIButton! // MARK: View created programmatically @IBOutlet weak var suggestedQuestionsStackView: UIStackView! // MARK: Custom init and deinit required init?(coder: NSCoder) { super.init(coder: coder) // Add obserber for keyboard notification to adjust view size. NotificationCenter.default.addObserver( self, selector: #selector(keyboardWillShow(notification:)), name: UIResponder.keyboardWillShowNotification, object: nil ) NotificationCenter.default.addObserver( self, selector: #selector(keyboardWillHide(notification:)), name: UIResponder.keyboardWillHideNotification, object: nil ) } deinit { // Remove added observer. NotificationCenter.default.removeObserver(self) } // MARK: - View handling methods override func viewDidLoad() { super.viewDidLoad() guard let dataset = dataset else { fatalError("Data set was not passed correctly.") } navigationItem.title = dataset.title // Make status text view to have rounded border. statusTextView.layer.cornerRadius = CustomUI.statusTextViewCornerRadius // Add content of the data set to the view. contentView.text = dataset.content contentView.font = .systemFont(ofSize: 17) // Add suggested questions as buttons to the stack view. for question in dataset.questions { let button = SuggestedQuestionButton(of: question) suggestedQuestionsStackView.addArrangedSubview(button) button.addTarget( self, action: #selector(tapSuggestedQuestionButton(of:)), for: .touchUpInside) } // Disable run button at the begining as it is empty. runButton.isEnabled = false } // MARK: Action for keyboard notification @objc func keyboardWillShow(notification: Notification) { // Get keyboard height. let keyboardHeight: CGFloat if let keyboardFrame = (notification.userInfo?[UIResponder.keyboardFrameEndUserInfoKey] as? NSValue)? .cgRectValue { keyboardHeight = keyboardFrame.height } else { keyboardHeight = 0 } // Resize the view adjusting to the keyboard height. view.frame.size.height = UIScreen.main.bounds.height - keyboardHeight } @objc func keyboardWillHide(notification: Notification) { // Resize the view to fil the screen. view.frame.size.height = UIScreen.main.bounds.height } // MARK: - Custom actions @IBAction func textFieldDidChange(textField: UITextField) { // Drop empty question. guard let query = textField.text?.trimmingCharacters(in: .whitespacesAndNewlines), !query.isEmpty else { os_log("Got empty query.") statusTextView.text = StatusMessage.warnEmptyQuery runButton.isEnabled = false return } // Ask to run if query is not empty. statusTextView.text = StatusMessage.askRun runButton.isEnabled = true } @IBAction func tapContentView() { // Hide keyboard. questionField.resignFirstResponder() } @IBAction func tapSuggestedQuestionButton(of sender: SuggestedQuestionButton!) { // Fill the `questionField` in the view with selected question. questionField.text = sender.title(for: .normal) // Activate the button and ask to run. statusTextView.text = StatusMessage.askRun runButton.isEnabled = true } @IBAction func tapRunButton() { // Disable run button until getting the answer. runButton.isEnabled = false // Clean up previous result. contentView.textStorage .removeAttribute( .backgroundColor, range: NSRange(location: 0, length: contentView.text.count)) // Hide keyboard. questionField.resignFirstResponder() // Trim the whitespaces and newlines in the first and end. guard var query = questionField.text?.trimmingCharacters(in: .whitespacesAndNewlines), !query.isEmpty else { os_log("Textfield failed to filter the empty query.") statusTextView.text = StatusMessage.warnEmptyQuery return } // A query must end with question mark. if query.last != "?" { query.append("?") } // Inference the answer with BertQA model. guard let result = bertQA?.run(query: query, content: contentView.text) else { os_log("Failed to inference the answer.") statusTextView.text = StatusMessage.inferenceFailError return } statusTextView.text = result.description // Render the answer in the `contentView`. contentView.textStorage .addAttribute( .backgroundColor, value: CustomUI.textHighlightColor, range: NSRange(result.answer.text.range, in: contentView.text) ) // Enable button as the process is finished. runButton.isEnabled = true } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/Controllers/DataSetsTableViewController.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit import os /// Controller for a table for the list of `Dataset`s. class DatasetsTableViewController: UITableViewController { /// List of `Dataset`s to run the Bert model. var datasets: [Dataset] = Dataset.load() var bertQA: BertQAHandler? required init?(coder: NSCoder) { super.init(coder: coder) // Add obserber for interpreter option changing. NotificationCenter.default.addObserver( self, selector: #selector(interpreterOptionDidChange(notification:)), name: UserDefaults.didChangeNotification, object: nil ) } deinit { // Remove added observer. NotificationCenter.default.removeObserver(self) } // MARK: - View handling methods override func viewDidLoad() { super.viewDidLoad() updateBertQA() } // MARK: - Table view data source override func numberOfSections(in tableView: UITableView) -> Int { return 1 } override func tableView(_ tableView: UITableView, numberOfRowsInSection section: Int) -> Int { return datasets.count } override func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell { let cell = tableView.dequeueReusableCell(withIdentifier: "DatasetTitleCell") as! DatasetTitleCell cell.titleLabel.text = datasets[indexPath.row].title return cell } // MARK: - Navigation override func prepare(for segue: UIStoryboardSegue, sender: Any?) { // Hand over `Dataset` of the selected title to the `DatasetDetailViewController`. guard segue.identifier == "ChooseDataset", let datasetDetailViewController = segue.destination as? DatasetDetailViewController, let selectedIndex = tableView.indexPathForSelectedRow?.row else { return } datasetDetailViewController.dataset = datasets[selectedIndex] datasetDetailViewController.bertQA = bertQA } // MARK: Update BertQA on changing interpreter option @objc func interpreterOptionDidChange(notification: Notification) { updateBertQA() } private func updateBertQA() { let threadCount = UserDefaults.standard.integer(forKey: InterpreterOptions.threadCount.id) do { bertQA = try BertQAHandler(threadCount: threadCount) } catch let error { fatalError(error.localizedDescription) } } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/Controllers/OptionTableViewController.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit class OptionTableViewController: UITableViewController { // MARK: Storyboard Connections @IBOutlet weak var threadCountLabel: UILabel! @IBOutlet weak var threadCountStepper: UIStepper! // MARK: - View handling methods override func viewDidLoad() { // Initialize UI properties with user default value. let threadCount = UserDefaults.standard.integer(forKey: InterpreterOptions.threadCount.id) threadCountLabel.text = threadCount.description threadCountStepper.value = Double(threadCount) // Set thread count limits. threadCountStepper.maximumValue = Double(InterpreterOptions.threadCount.maximumValue) threadCountStepper.minimumValue = Double(InterpreterOptions.threadCount.minimumValue) } // MARK: Button Actions @IBAction func didChangeThreadCount(_ sender: UIStepper) { threadCountLabel.text = Int(sender.value).description } @IBAction func didResetOptions() { threadCountLabel.text = InterpreterOptions.threadCount.defaultValue.description threadCountStepper.value = Double(InterpreterOptions.threadCount.defaultValue) } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/Controllers/OptionViewController.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit import os class OptionViewController: UIViewController { var optionTableView: OptionTableViewController? // MARK: Storyboard Connections @IBOutlet weak var cancelButton: UIButton! @IBOutlet weak var doneButton: UIButton! // MARK: - Custom actions @IBAction func tapCancelButton() { self.dismiss(animated: true, completion: nil) } @IBAction func tapDoneButton() { guard let newThreadCount = optionTableView?.threadCountStepper.value else { os_log("[Option View]: Cannot get the option value", type: .error) self.dismiss(animated: true, completion: nil) return } let currentThreadCount = Double( UserDefaults.standard.integer(forKey: InterpreterOptions.threadCount.id)) if currentThreadCount != newThreadCount { UserDefaults.standard.set(Int(newThreadCount), forKey: InterpreterOptions.threadCount.id) } self.dismiss(animated: true, completion: nil) } // MARK: - Navigation override func prepare(for segue: UIStoryboardSegue, sender: Any?) { if segue.identifier == "optionTableEmbedingSegue", let destination = segue.destination as? OptionTableViewController { optionTableView = destination } } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/Info.plist ================================================ CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleExecutable $(EXECUTABLE_NAME) CFBundleIdentifier $(PRODUCT_BUNDLE_IDENTIFIER) CFBundleInfoDictionaryVersion 6.0 CFBundleName $(PRODUCT_NAME) CFBundlePackageType APPL CFBundleShortVersionString 1.0 CFBundleVersion 1 LSRequiresIPhoneOS UILaunchStoryboardName LaunchScreen UIMainStoryboardFile Main UIRequiredDeviceCapabilities armv7 UISupportedInterfaceOrientations UIInterfaceOrientationLandscapeRight UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationPortrait UISupportedInterfaceOrientations~ipad UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight UIUserInterfaceStyle Light ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/Views/ControlPanelView.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit /// Custom view for control panel which has a separating line at the top of it. class ControlPanelView: UIView { lazy var separatingLine = CALayer() required init?(coder: NSCoder) { super.init(coder: coder) addSeparatingLine() } override init(frame: CGRect) { super.init(frame: frame) addSeparatingLine() } override func layoutSubviews() { super.layoutSubviews() updateSeparatingLine() } /// Add seperating line at the top of the view. private func addSeparatingLine() { separatingLine.backgroundColor = UIColor.lightGray.cgColor layer.addSublayer(separatingLine) } /// Update separating line. private func updateSeparatingLine() { separatingLine.frame = getSeparatingLineFrame() } /// Get border of separating line to fill the width of the view. private func getSeparatingLineFrame() -> CGRect { var width = frame.width let left = safeAreaInsets.right let right = safeAreaInsets.right width = width + left + right return CGRect(x: -left, y: 0, width: width, height: 0.7) } } ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/Views/DataSetTitleCell.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit /// Table cell for a title of `Dataset`. class DatasetTitleCell: UITableViewCell { @IBOutlet weak var titleLabel: UILabel! } ================================================ FILE: lite/examples/bert_qa/ios/ViewInUIKit/Views/SuggestedQuestionButton.swift ================================================ // Copyright 2020 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import UIKit /// Button for Sugessted Questions. @IBDesignable class SuggestedQuestionButton: UIButton { var question: String? lazy var border = CAShapeLayer() lazy var gradient = CAGradientLayer() lazy var fill = CAShapeLayer() lazy var borderPath = CGPath(rect: CGRect(), transform: nil) override var bounds: CGRect { didSet { borderPath = UIBezierPath( roundedRect: self.bounds.insetBy(dx: 1, dy: 6), cornerRadius: 10.0 ).cgPath } } required init?(coder: NSCoder) { super.init(coder: coder) addSublayers() } override init(frame: CGRect) { super.init(frame: frame) addSublayers() } convenience init(of question: String) { self.init(frame: CGRect.zero) self.question = question } override func draw(_ rect: CGRect) { setTitle(question, for: .normal) setTitleColor(.black, for: .normal) titleLabel?.font = .systemFont(ofSize: 11, weight: .medium) contentEdgeInsets = UIEdgeInsets(top: 0, left: 4, bottom: 0, right: 4) } override func layoutSubviews() { super.layoutSubviews() updateSublayers() } private func addSublayers() { // Add border layer to the button. border.lineWidth = 1.0 border.fillColor = UIColor.clear.cgColor border.strokeColor = UIColor.black.cgColor gradient.colors = [ UIColor.orange.cgColor, UIColor.orange.withAlphaComponent(0.7).cgColor, UIColor.yellow.cgColor, ] gradient.startPoint = CGPoint(x: 0, y: 0) gradient.endPoint = CGPoint(x: 1, y: 1) layer.addSublayer(gradient) // Add color fill to the button. fill.fillColor = UIColor.white.cgColor layer.addSublayer(fill) } private func updateSublayers() { // Update customized border along to the size of the button. border.frame = bounds border.path = borderPath gradient.frame = border.bounds gradient.mask = border // Update color fill along to the size of the button. fill.path = borderPath } } ================================================ FILE: lite/examples/classification_by_retrieval/.bazelrc ================================================ # Align with TensorFlow .bazelrc to build from the source. # Please refer to TensorFlow .bazelrc for more details: # https://github.com/tensorflow/tensorflow/blob/master/.bazelrc # Android configs. Bazel needs to have --cpu and --fat_apk_cpu both set to the # target CPU to build transient dependencies correctly. See # https://docs.bazel.build/versions/master/user-manual.html#flag--fat_apk_cpu build:android --crosstool_top=//external:android/crosstool build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain build:android_arm --config=android build:android_arm --cpu=armeabi-v7a build:android_arm --fat_apk_cpu=armeabi-v7a build:android_arm64 --config=android build:android_arm64 --cpu=arm64-v8a build:android_arm64 --fat_apk_cpu=arm64-v8a build:android_x86 --config=android build:android_x86 --cpu=x86 build:android_x86 --fat_apk_cpu=x86 build:android_x86_64 --config=android build:android_x86_64 --cpu=x86_64 build:android_x86_64 --fat_apk_cpu=x86_64 # Sets the default Apple platform to macOS. build --apple_platform_type=macos # iOS configs for each architecture and the fat binary builds. build:ios --apple_platform_type=ios build:ios --apple_bitcode=embedded --copt=-fembed-bitcode build:ios_armv7 --config=ios build:ios_armv7 --cpu=ios_armv7 build:ios_armv7 --cops -Wno-c++11-narrowing build:ios_arm64 --config=ios build:ios_arm64 --cpu=ios_arm64 build:ios_x86_64 --config=ios build:ios_x86_64 --cpu=ios_x86_64 build:ios_fat --config=ios build:ios_fat --ios_multi_cpus=armv7,arm64 build:ios_fat --copt -Wno-c++11-narrowing # BEGIN OF REMOTE BUILD EXECUTION OPTIONS # Options when using remote execution # WARNING: THESE OPTIONS WONT WORK IF YOU DO NOT HAVE PROPER AUTHENTICATION AND PERMISSIONS # Flag to enable remote config common --experimental_repo_remote_exec # Config to use a mostly-static build and disable modular op registration # support (this will revert to loading TensorFlow with RTLD_GLOBAL in Python). # By default, TensorFlow will build with a dependence on # //tensorflow:libtensorflow_framework.so. build:monolithic --define framework_shared_object=false # Flags for open source build, always set to be true. build --define open_source_build=true test --define open_source_build=true # Flags for fat_apk_cpu build, always set to multiple cpus. build --fat_apk_cpu=x86,x86_64,arm64-v8a,armeabi-v7a test --fat_apk_cpu=x86,x86_64,arm64-v8a,armeabi-v7a # Other build flags. build --define=grpc_no_ares=true # See https://github.com/bazelbuild/bazel/issues/7362 for information on what # --incompatible_remove_legacy_whole_archive flag does. # This flag is set to true in Bazel 1.0 and newer versions. We tried to migrate # Tensorflow to the default, however test coverage wasn't enough to catch the # errors. # There is ongoing work on Bazel team's side to provide support for transitive # shared libraries. As part of migrating to transitive shared libraries, we # hope to provide a better mechanism for control over symbol exporting, and # then tackle this issue again. # # TODO: Remove this line once TF doesn't depend on Bazel wrapping all library # archives in -whole_archive -no_whole_archive. build --noincompatible_remove_legacy_whole_archive # These are bazel 2.0's incompatible flags. Tensorflow needs to use bazel 2.0.0 # to use cc_shared_library, as part of the Tensorflow Build Improvements RFC: # https://github.com/tensorflow/community/pull/179 build --noincompatible_prohibit_aapt1 # Modular TF build options build:dynamic_kernels --define=dynamic_loaded_kernels=true build:dynamic_kernels --copt=-DAUTOLOAD_DYNAMIC_KERNELS # Build TF with C++ 17 features. build:c++17 --cxxopt=-std=c++1z build:c++17 --cxxopt=-stdlib=libc++ build:c++1z --config=c++17 # Enable using platform specific build settings build --enable_platform_specific_config # Suppress C++ compiler warnings, otherwise build logs become 10s of MBs. build:linux --copt=-w build:macos --copt=-w build:windows --copt=/w # Tensorflow uses M_* math constants that only get defined by MSVC headers if # _USE_MATH_DEFINES is defined. build:windows --copt=/D_USE_MATH_DEFINES build:windows --host_copt=/D_USE_MATH_DEFINES # Default paths for TF_SYSTEM_LIBS build:linux --define=PREFIX=/usr build:linux --define=LIBDIR=$(PREFIX)/lib build:linux --define=INCLUDEDIR=$(PREFIX)/include build:macos --define=PREFIX=/usr build:macos --define=LIBDIR=$(PREFIX)/lib build:macos --define=INCLUDEDIR=$(PREFIX)/include # TF_SYSTEM_LIBS do not work on windows. # By default, build TF in C++ 14 mode. build:linux --cxxopt=-std=c++14 build:linux --host_cxxopt=-std=c++14 build:macos --cxxopt=-std=c++14 build:macos --host_cxxopt=-std=c++14 build:windows --cxxopt=/std:c++14 build:windows --host_cxxopt=/std:c++14 # On windows, we still link everything into a single DLL. build:windows --config=monolithic # On linux, we dynamically link small amount of kernels build:linux --config=dynamic_kernels # Make sure to include as little of windows.h as possible build:windows --copt=-DWIN32_LEAN_AND_MEAN build:windows --host_copt=-DWIN32_LEAN_AND_MEAN build:windows --copt=-DNOGDI build:windows --host_copt=-DNOGDI # Misc build options we need for windows. build:windows --linkopt=/DEBUG build:windows --host_linkopt=/DEBUG build:windows --linkopt=/OPT:REF build:windows --host_linkopt=/OPT:REF build:windows --linkopt=/OPT:ICF build:windows --host_linkopt=/OPT:ICF build:windows --experimental_strict_action_env=true # Verbose failure logs when something goes wrong build:windows --verbose_failures # Suppress all warning messages. build:short_logs --output_filter=DONT_MATCH_ANYTHING # Instruction set optimizations # TODO(gunan): Create a feature in toolchains for avx/avx2 to # avoid having to define linux/win separately. build:avx_linux --copt=-mavx build:avx2_linux --copt=-mavx2 build:native_arch_linux --copt=-march=native build:avx_win --copt=/arch=AVX build:avx2_win --copt=/arch=AVX2 # Options to build TensorFlow 1.x or 2.x. build:v1 --define=tf_api_version=1 build:v2 --define=tf_api_version=2 build:v1 --action_env=TF2_BEHAVIOR=0 build:v2 --action_env=TF2_BEHAVIOR=1 build --config=v2 test --config=v2 # Enable XLA build:xla --action_env=TF_ENABLE_XLA=1 build:xla --define=with_xla_support=true ================================================ FILE: lite/examples/classification_by_retrieval/.bazelversion ================================================ 3.7.2 ================================================ FILE: lite/examples/classification_by_retrieval/README.md ================================================ # CbR: Classification-by-Retrieval Classification-by-retrieval provides an easy way to create a neural network-based classifier without computationally expensive backpropagation training. Using this technology, you can create a lightweight mobile model with as little as [one image per class](#model-accuracy-comparison-with-few-shot-learning), or you can create an on-device model that can classify as many as tens of thousands of classes. For example, we created [mobile models that can recognize tens of thousands of landmarks](https://tfhub.dev/google/collections/landmarks/1) with the classification-by-retrieval technology. We provide an [iOS app](ios/README.md), where you can choose images from the photo library and label them to create a TfLite classifier within seconds (if the number of images are small), and test the created model right away. We also provide a C++ command line tool (in lib/tests) to build a classifier. There are many use-cases for classification-by-retrieval, including: * Machine learning education (e.g., an educational hackathon event). * Easily prototyping, or demonstrating ML classification. * Custom product recognition (e.g., developing a product recognition app for a small/medium business without the need to gather extensive training data or write lots of code). ## Technical background Classification and retrieval are two distinct methods of image recognition. A typical object recognition approach is to build a neural network classifier and train it with a large amount of training data (often thousands of images, or more). On the contrary, the retrieval approach uses a pre-trained feature extractor (e.g., an image embedding model) with feature matching based on a nearest neighbor search algorithm. The retrieval approach is scalable and flexible. For example, it can handle a large number of classes (say, > 1 million), and adding or removing classes does not require extra training. One would need as little as a single training data per class, which makes it effectively few-shot learning. A downside of the retrieval approach is that it requires extra infrastructure, and is less intuitive to use than a classification model. Classification-by-retrieval (CbR) is a neural network model with image retrieval layers baked into it. With the CbR technology, you can easily create a TensorFlow classification model without any training. ![Conventional Approaches](docs/images/conventional-approaches.svg) ![Classification-by-Retrieval](docs/images/classification-by-retrieval.svg) ## How do the retrieval layers work? A classification-by-retrieval model is an extension of an embedding model with extra retrieval layers. The retrieval layers are computed (not trained) from the training data, i.e., the index data. The retrieval layers consists of two components: * Nearest neighbor matching component * Result aggregation component The nearest neighbor matching component is essentially a fully connected layer where its weights are the normalized embeddings of the index data. Note that a dot-product of two normalized vectors (cos similarity) is linear (with a negative coefficient) to the squared L2 distance. Therefore, the output of the fully connected layer is effectively identical to the nearest neighbor matching result. The retrieval result is given for each training instance, not for each class. Therefore, we add another *result aggregation component* on top of the nearest neighbor matching layer. The aggregation component consists of a selection layer for each class followed by an aggregation (e.g., max) layer for each of them. Finally, the results are concatenated to form a single output vector. ## Base Embedding Model One may choose a base embedding model that best fits the domain. There are many embedding models available, for example, in [TensorFlow hub](http://tfhub.dev), for various domains. The provided [iOS demo](ios/README.md) uses a [MobileNet V3 trained with ImageNet](https://tfhub.dev/google/lite-model/imagenet/mobilenet_v3_small_100_224/feature_vector/5/default/1), which is a generic and efficient on-device model. ## Model Accuracy: Comparison with Few Shot Learning In some sense, CbR (indexing) can be considered as a few-shot learning approach without training. Although it is not apples to apples to compare CbR with an arbitrary pre-trained base embedding model with a typical few-shot learning approach where the whole model trained with given training data, there is a [research](https://arxiv.org/pdf/1911.04623.pdf) that compares nearest neighbor retrieval (which is equivalent to CbR) with few-shot learning approaches. It shows that nearest neighbor retrieval can be comparable or even better than many few-shot learning approaches. ## Responsible Model Building We encourage you to build a model that is fair and responsible. To learn more about building a responsible model: * https://www.tensorflow.org/responsible_ai * https://design.google/library/fair-not-default/ * https://developers.google.com/machine-learning/crash-course/fairness/video-lecture ================================================ FILE: lite/examples/classification_by_retrieval/WORKSPACE ================================================ """Classification-by-Retrieval Workspace""" workspace(name = "org_tensorflow_lite_examples_classificationbyretrieval") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") # TFLite Support http_archive( name = "org_tensorflow_lite_support", strip_prefix = "tflite-support-f5dadc83b36d863700f080ca45877f8b3b3d7f47", sha256 = "c07c1e54eced03d568168f03ecb3cde34c6f8b8fad7d29d2b09c39c14d2f0899", urls = ["https://github.com/tensorflow/tflite-support/archive/f5dadc83b36d863700f080ca45877f8b3b3d7f47.zip"], ) # TFLite Support dependencies. http_archive( name = "com_google_glog", sha256 = "1ee310e5d0a19b9d584a855000434bb724aa744745d5b8ab1855c85bff8a8e21", strip_prefix = "glog-028d37889a1e80e8a07da1b8945ac706259e5fd8", urls = [ "https://mirror.bazel.build/github.com/google/glog/archive/028d37889a1e80e8a07da1b8945ac706259e5fd8.tar.gz", "https://github.com/google/glog/archive/028d37889a1e80e8a07da1b8945ac706259e5fd8.tar.gz", ], ) http_archive( name = "zlib", build_file = "@org_tensorflow_lite_support//third_party:zlib.BUILD", sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", strip_prefix = "zlib-1.2.11", urls = [ "http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz", "http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15 ], ) http_archive( name = "org_libzip", build_file = "@org_tensorflow_lite_support//third_party:libzip.BUILD", sha256 = "a5d22f0c87a2625450eaa5e10db18b8ee4ef17042102d04c62e311993a2ba363", strip_prefix = "libzip-rel-1-5-1", urls = [ # Bazel does not like the official download link at libzip.org, # so use the GitHub release tag. "https://mirror.bazel.build/github.com/nih-at/libzip/archive/rel-1-5-1.zip", "https://github.com/nih-at/libzip/archive/rel-1-5-1.zip", ], ) http_archive( name = "libyuv", urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/39240f7149cffde62e3620344d222c8ab2c21178.tar.gz"], # Adding the constrain of sha256 and strip_prefix will cause failure as of # Jan 2021. It seems that the downloaded libyuv was different every time, # so that the specified sha256 and strip_prefix cannot match. # sha256 = "01c2e30eb8e83880f9ba382f6bece9c38cd5b07f9cadae46ef1d5a69e07fafaf", # strip_prefix = "libyuv-39240f7149cffde62e3620344d222c8ab2c21178", build_file = "@org_tensorflow_lite_support//third_party:libyuv.BUILD", ) http_archive( name = "com_google_absl", strip_prefix = "abseil-cpp-20210324.2", sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f", urls = ["https://github.com/abseil/abseil-cpp/archive/20210324.2.tar.gz"], ) # TF on 2021-09-29. http_archive( name = "org_tensorflow", strip_prefix = "tensorflow-a221f72e69fea7a46977e35961e5cdb1e51fec36", sha256 = "d0e57bcf455df772cfdf65fdf59a94dfef7547c6aafd50b382dab3a182b0c5b3", urls = ["https://github.com/tensorflow/tensorflow/archive/a221f72e69fea7a46977e35961e5cdb1e51fec36.tar.gz"], ) # Set up TF. load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3") tf_workspace3() load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2") tf_workspace2() load("@org_tensorflow//tensorflow:workspace1.bzl", "tf_workspace1") tf_workspace1() load("@org_tensorflow//tensorflow:workspace0.bzl", "tf_workspace0") tf_workspace0() # Download the model file. http_file( name = "imagenet-mobilenet_v3_small_100_224-feature_vector", downloaded_file_path = "imagenet-mobilenet_v3_small_100_224-feature_vector.tflite", sha256 = "383220188d049b60b044da89b1a1e9eacb676c63562867875aceaf6885a8c761", urls = ["https://tfhub.dev/google/lite-model/imagenet/mobilenet_v3_small_100_224/feature_vector/5/metadata/1?lite-format=tflite"], ) ================================================ FILE: lite/examples/classification_by_retrieval/ios/BUILD ================================================ load("//third_party/bazel_rules/rules_cc/cc:objc_library.bzl", "objc_library") load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application") load("//tools/build_defs/swift:swift_library.bzl", "swift_library") package( default_applicable_licenses = ["//third_party/py/tensorflow_examples:license"], default_visibility = ["//visibility:private"], ) licenses(["notice"]) MINIMUM_OS_VERSION = "15.0" ios_application( name = "ImageClassifierBuilder", app_icons = glob(["ImageClassifierBuilder/Assets.xcassets/AppIcon.appiconset/**"]), bundle_id = "com.tensorflow.lite.swift.ImageClassifierBuilder", families = ["iphone"], infoplists = ["ImageClassifierBuilder/Info.plist"], minimum_os_version = MINIMUM_OS_VERSION, provisioning_profile = ":ProvisioningProfile.mobileprovision", deps = [ ":ImageClassifierBuilderLib", ], ) # Swift files target. swift_library( name = "ImageClassifierBuilderLib", srcs = glob(["ImageClassifierBuilder/*.swift"]), data = glob(["ImageClassifierBuilder/TFLiteDocumentIcons/**"]), deps = [ ":ImageClassifierBuilderLibObjC", ], ) # Hide header files that contain C++, as otherwise the `objc_library` can't be used in a # `swift_library`. PRIVATE_HEADERS = [ "ImageClassifierBuilder/NSString+AbseilStringView.h", ] # Objective-C(++) files target. objc_library( name = "ImageClassifierBuilderLibObjC", srcs = glob([ "ImageClassifierBuilder/*.m", "ImageClassifierBuilder/*.mm", ]) + PRIVATE_HEADERS, hdrs = glob( ["ImageClassifierBuilder/*.h"], exclude = PRIVATE_HEADERS, ), data = [ "@imagenet-mobilenet_v3_small_100_224-feature_vector//file", ], deps = [ "//third_party/apple_frameworks:Accelerate", "//third_party/apple_frameworks:CoreGraphics", "//third_party/apple_frameworks:CoreVideo", "//third_party/apple_frameworks:Foundation", "//third_party/apple_frameworks:UIKit", "//lib:model_builder", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@flatbuffers//:runtime_cc", "@org_tensorflow_lite_support//tensorflow_lite_support/cc/port:statusor", "@org_tensorflow_lite_support//tensorflow_lite_support/cc/task/vision:image_classifier", "@org_tensorflow_lite_support//tensorflow_lite_support/cc/task/vision/core:frame_buffer", "@org_tensorflow_lite_support//tensorflow_lite_support/cc/task/vision/proto:classifications_proto_inc", "@org_tensorflow_lite_support//tensorflow_lite_support/cc/task/vision/proto:image_classifier_options_proto_inc", "@org_tensorflow_lite_support//tensorflow_lite_support/cc/task/vision/proto:image_embedder_options_proto_inc", "@org_tensorflow_lite_support//tensorflow_lite_support/cc/task/vision/utils:frame_buffer_common_utils", ], ) exports_files(["ProvisioningProfile.mobileprovision"]) ================================================ FILE: lite/examples/classification_by_retrieval/ios/ImageClassifierBuilder/Album.swift ================================================ // Copyright 2021 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Foundation import Photos /// Describes an album from the user Photos Library within the AlbumSelector UI. struct Album { /// The handle to the underlying Photos Library album. let collection: PHAssetCollection /// Whether the album is selected in the AlbumSelector UI. var selected = false } ================================================ FILE: lite/examples/classification_by_retrieval/ios/ImageClassifierBuilder/AlbumSelector.swift ================================================ // Copyright 2021 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import Photos import SwiftUI /// Displays the list of Photos Library albums. /// /// The user can then make a selection before proceeding to the next step. /// /// This screen is part of the model creation UX. struct AlbumSelector: View { /// The metadata of the model to create. let modelMetadata: ModelMetadata /// The closure to call once the Model object will be created. let completion: (Model?) -> Void /// The list of albums from the Photos Library. @State private var albums: [Album] = [] var body: some View { VStack { if PHPhotoLibrary.authorizationStatus(for: .readWrite) == .authorized { Text( "The model classes will be named after the Photos Library albums you select. Make sure " + "an album contains images of only one class." ) .padding() List { Section(header: Text("Albums")) { ForEach(albums.indices, id: \.self) { index in Row(album: albums[index]) .onTapGesture { albums[index].selected.toggle() } } } } .listStyle(GroupedListStyle()) } else { Text( "Classification by Retrieval lets you select albums from your library to train the " + "model. Please allow access in Settings." ) .padding() Button("Settings") { UIApplication.shared.open(URL(string: UIApplication.openSettingsURLString)!) } } } .onAppear { fetchAlbums() } .navigationBarTitle(Text("Select Albums"), displayMode: .inline) .toolbar { ToolbarItem(placement: .primaryAction) { NavigationLink( "Next", destination: ModelTrainerView( modelMetadata: modelMetadata, albums: albums.filter(\.selected), completion: completion) ) .accessibilityHint( "Proceeds to the next model creation step. Dimmed until at least two albums are selected." ) .disabled(albums.filter(\.selected).count < 2) } } } /// Requests authorization to access the library and fetches the albums if authorized. func fetchAlbums() { PHPhotoLibrary.requestAuthorization(for: .readWrite) { status in switch status { case .notDetermined: print("Photo Library authorization not determined.") case .restricted: print("Photo Library authorization restricted.") case .denied: print("Photo Library authorization denied.") case .authorized: self.albums = PHCollectionList.fetchAlbums() case .limited: print("Photo Library authorization limited.") @unknown default: fatalError() } } } /// Displays an album in the list. struct Row: View { /// The represented album. let album: Album var body: some View { HStack { Image(systemName: "photo") HStack(alignment: .lastTextBaseline) { Text(album.collection.localizedTitle!) Text( "\(album.collection.estimatedAssetCount) " + "\(album.collection.estimatedAssetCount > 1 ? "photos" : "photo")" ) .font(.caption) .foregroundColor(.secondary) } Spacer() Image(systemName: album.selected ? "checkmark.circle.fill" : "circle") } .contentShape(Rectangle()) .accessibilityAddTraits(album.selected ? [.isSelected] : []) .accessibilityElement(children: .combine) } } } extension PHCollectionList { /// Fetches the Photos Library albums. static func fetchAlbums() -> [Album] { var albums = [Album]() let userCollections = fetchTopLevelUserCollections(with: nil) userCollections.enumerateObjects { (collection, index, stop) in if let collection = collection as? PHAssetCollection, collection.localizedTitle != nil { albums.append(Album(collection: collection)) } } return albums } } ================================================ FILE: lite/examples/classification_by_retrieval/ios/ImageClassifierBuilder/Assets.xcassets/AppIcon.appiconset/Contents.json ================================================ { "images": [ { "size": "60x60", "expected-size": "180", "filename": "180.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "3x" }, { "size": "40x40", "expected-size": "80", "filename": "80.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "2x" }, { "size": "40x40", "expected-size": "120", "filename": "120.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "3x" }, { "size": "60x60", "expected-size": "120", "filename": "120.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "2x" }, { "size": "29x29", "expected-size": "58", "filename": "58.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "2x" }, { "size": "29x29", "expected-size": "29", "filename": "29.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "1x" }, { "size": "29x29", "expected-size": "87", "filename": "87.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "3x" }, { "size": "20x20", "expected-size": "40", "filename": "40.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "2x" }, { "size": "20x20", "expected-size": "60", "filename": "60.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "iphone", "scale": "3x" }, { "size": "1024x1024", "filename": "1024.png", "expected-size": "1024", "idiom": "ios-marketing", "folder": "Assets.xcassets/AppIcon.appiconset/", "scale": "1x" }, { "size": "40x40", "expected-size": "80", "filename": "80.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "72x72", "expected-size": "72", "filename": "72.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "76x76", "expected-size": "152", "filename": "152.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "50x50", "expected-size": "100", "filename": "100.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "29x29", "expected-size": "58", "filename": "58.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "76x76", "expected-size": "76", "filename": "76.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "29x29", "expected-size": "29", "filename": "29.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "50x50", "expected-size": "50", "filename": "50.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "72x72", "expected-size": "144", "filename": "144.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "40x40", "expected-size": "40", "filename": "40.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "83.5x83.5", "expected-size": "167", "filename": "167.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" }, { "size": "20x20", "expected-size": "20", "filename": "20.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "1x" }, { "size": "20x20", "expected-size": "40", "filename": "40.png", "folder": "Assets.xcassets/AppIcon.appiconset/", "idiom": "ipad", "scale": "2x" } ] } ================================================ FILE: lite/examples/classification_by_retrieval/ios/ImageClassifierBuilder/Assets.xcassets/Contents.json ================================================ { "info" : { "version" : 1, "author" : "xcode" } } ================================================ FILE: lite/examples/classification_by_retrieval/ios/ImageClassifierBuilder/CameraView.swift ================================================ // Copyright 2021 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import AVFoundation import CoreVideo import SwiftUI /// Shows the live feed from the camera and vends individual frames are accessible via the `onFrame` /// callback. struct CameraView: UIViewControllerRepresentable { /// The queue on which to call `onFrame`. let queue: DispatchQueue /// The callback providing individual frames as image buffer. /// /// This is called on `queue`. let onFrame: (CVImageBuffer) -> Void func makeUIViewController(context: Context) -> CameraViewController { CameraViewController(sampleBufferDelegate: context.coordinator, queue: queue) } func updateUIViewController(_ uiViewController: CameraViewController, context: Context) { } func makeCoordinator() -> Coordinator { Coordinator(self) } /// Registers as `CameraViewController` delegate to receive frames, then calls the `CameraView`'s /// `onFrame` callback. final class Coordinator: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate { private let parent: CameraView fileprivate init(_ parent: CameraView) { self.parent = parent } func captureOutput( _ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection ) { if let imageBuffer = sampleBuffer.imageBuffer { parent.onFrame(imageBuffer) } } } } ================================================ FILE: lite/examples/classification_by_retrieval/ios/ImageClassifierBuilder/CameraViewController.swift ================================================ // Copyright 2021 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import AVFoundation import CoreVideo import UIKit /// Shows the live feed from the camera and vends individual frames are provided to the /// `sampleBufferDelegate` on `queue`. class CameraViewController: UIViewController { /// The delegate receiving the sample buffers. let sampleBufferDelegate: AVCaptureVideoDataOutputSampleBufferDelegate /// The queue on which the delegate is receiving the sample buffers. let queue: DispatchQueue /// The underlying capture session. private let captureSession = AVCaptureSession() /// The underlying layer displaying the live camera feed. private var videoPreviewLayer: AVCaptureVideoPreviewLayer? /// Initializes a `CameraViewController` with the details to receive camera frames. /// /// - Parameters: /// - sampleBufferDelegate: The delegate receiving the sample buffers. /// - queue: The queue on which the delegate is receiving the sample buffers. init(sampleBufferDelegate: AVCaptureVideoDataOutputSampleBufferDelegate, queue: DispatchQueue) { self.sampleBufferDelegate = sampleBufferDelegate self.queue = queue super.init(nibName: nil, bundle: nil) } required init(coder: NSCoder) { fatalError() } override func viewDidLoad() { super.viewDidLoad() view.backgroundColor = .black switch AVCaptureDevice.authorizationStatus(for: .video) { case .authorized: setupCaptureSession() case .notDetermined: AVCaptureDevice.requestAccess(for: .video) { granted in if granted { DispatchQueue.main.async { [weak self] in self?.setupCaptureSession() } } } default: print("Camera access not authorized.") } } override func viewWillAppear(_ animated: Bool) { super.viewWillAppear(animated) captureSession.startRunning() } override func viewDidDisappear(_ animated: Bool) { super.viewDidDisappear(animated) captureSession.stopRunning() } override func viewDidLayoutSubviews() { super.viewDidLayoutSubviews() if let videoPreviewLayer = videoPreviewLayer { videoPreviewLayer.frame = view.bounds } } override func viewWillTransition( to size: CGSize, with coordinator: UIViewControllerTransitionCoordinator ) { super.viewWillTransition(to: size, with: coordinator) if let connection = videoPreviewLayer?.connection { switch UIDevice.current.orientation { case .portrait, .unknown, .faceUp, .faceDown: connection.videoOrientation = .portrait case .portraitUpsideDown: connection.videoOrientation = .portraitUpsideDown case .landscapeLeft: connection.videoOrientation = .landscapeRight case .landscapeRight: connection.videoOrientation = .landscapeLeft @unknown default: connection.videoOrientation = .portrait } } } /// Configures the capture session inputs and outputs, as well as the layer displaying the live /// camera feed. private func setupCaptureSession() { guard let camera = AVCaptureDevice.default(for: .video) else { print("Unable to access camera.") return } do { let input = try AVCaptureDeviceInput(device: camera) captureSession.addInput(input) let output = AVCaptureVideoDataOutput() output.videoSettings = [kCVPixelBufferPixelFormatTypeKey: Int(kCVPixelFormatType_32BGRA)] as [String: Any] output.alwaysDiscardsLateVideoFrames = true output.setSampleBufferDelegate(sampleBufferDelegate, queue: queue) captureSession.addOutput(output) videoPreviewLayer = AVCaptureVideoPreviewLayer(session: captureSession) if let videoPreviewLayer = videoPreviewLayer { videoPreviewLayer.videoGravity = .resizeAspectFill videoPreviewLayer.connection?.videoOrientation = .portrait view.layer.addSublayer(videoPreviewLayer) } } catch let error { print("Error Unable to initialize camera: \(error.localizedDescription)") } } } ================================================ FILE: lite/examples/classification_by_retrieval/ios/ImageClassifierBuilder/Classifier.h ================================================ // Copyright 2021 The TensorFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #import #import NS_ASSUME_NONNULL_BEGIN @class Label; /// A classifier infers labels from a given image. This wraps the C++ Image Classifier TFLite Task /// API. @interface Classifier : NSObject /// Initializes a classifier, loading the given model. /// @param URL The URL of the model file on disk. - (instancetype)initWithModelURL:(NSURL *)modelURL NS_SWIFT_NAME(init(modelURL:)); /// MARK: Inference /// Infers labels from the given image. /// @param pixelBuffer The image, as a Core Video Pixel Buffer. /// @return The list of labels. - (NSArray