Repository: googlesamples/mlkit Branch: master Commit: 3bdf01826657 Files: 867 Total size: 3.9 MB Directory structure: gitextract_e0cwqdyw/ ├── .github/ │ └── ISSUE_TEMPLATE/ │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── LICENSE ├── README.md ├── android/ │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── android-snippets/ │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ ├── proguard-rules.pro │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── example/ │ │ │ │ └── mlkit/ │ │ │ │ ├── BarcodeScanningActivity.java │ │ │ │ ├── FaceDetectionActivity.java │ │ │ │ ├── ImageLabelingActivity.java │ │ │ │ ├── LanguageIdentificationActivity.java │ │ │ │ ├── MLKitVisionImage.java │ │ │ │ ├── MainActivity.java │ │ │ │ ├── ObjectDetectionActivity.java │ │ │ │ ├── TextRecognitionActivity.java │ │ │ │ └── kotlin/ │ │ │ │ ├── BarcodeScanningActivity.kt │ │ │ │ ├── FaceDetectionActivity.kt │ │ │ │ ├── ImageLabelingActivity.kt │ │ │ │ ├── LanguageIdentificationActivity.kt │ │ │ │ ├── MLKitVisionImage.kt │ │ │ │ ├── MainActivity.kt │ │ │ │ ├── ObjectDetectionActivity.kt │ │ │ │ └── TextRecognitionActivity.kt │ │ │ └── res/ │ │ │ ├── layout/ │ │ │ │ └── activity_main.xml │ │ │ ├── values/ │ │ │ │ ├── colors.xml │ │ │ │ ├── dimens.xml │ │ │ │ ├── strings.xml │ │ │ │ └── styles.xml │ │ │ └── values-w820dp/ │ │ │ └── dimens.xml │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradle.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle │ ├── automl/ │ │ ├── LICENSE │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ ├── proguard-rules.pro │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── vision/ │ │ │ │ └── automl/ │ │ │ │ └── demo/ │ │ │ │ ├── BitmapUtils.java │ │ │ │ ├── CameraImageGraphic.java │ │ │ │ ├── CameraSource.java │ │ │ │ ├── CameraSourcePreview.java │ │ │ │ ├── CameraXLivePreviewActivity.java │ │ │ │ ├── CameraXViewModel.java │ │ │ │ ├── ChooserActivity.java │ │ │ │ ├── FrameMetadata.java │ │ │ │ ├── GraphicOverlay.java │ │ │ │ ├── InferenceInfoGraphic.java │ │ │ │ ├── LivePreviewActivity.java │ │ │ │ ├── ScopedExecutor.java │ │ │ │ ├── StillImageActivity.java │ │ │ │ ├── VisionImageProcessor.java │ │ │ │ ├── VisionProcessorBase.java │ │ │ │ ├── automl/ │ │ │ │ │ ├── AutoMLImageLabelerProcessor.java │ │ │ │ │ └── LabelGraphic.java │ │ │ │ ├── object/ │ │ │ │ │ ├── ObjectDetectorProcessor.java │ │ │ │ │ └── ObjectGraphic.java │ │ │ │ └── preference/ │ │ │ │ ├── CameraXLivePreviewPreferenceFragment.java │ │ │ │ ├── LivePreviewPreferenceFragment.java │ │ │ │ ├── PreferenceUtils.java │ │ │ │ ├── SettingsActivity.java │ │ │ │ └── StillImagePreferenceFragment.java │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ └── logo_mlkit.xml │ │ │ ├── drawable-hdpi/ │ │ │ │ └── ic_switch_camera_white_48dp.xml │ │ │ ├── drawable-mdpi/ │ │ │ │ └── ic_switch_camera_white_48dp.xml │ │ │ ├── drawable-xhdpi/ │ │ │ │ └── ic_switch_camera_white_48dp.xml │ │ │ ├── drawable-xxhdpi/ │ │ │ │ └── ic_switch_camera_white_48dp.xml │ │ │ ├── drawable-xxxhdpi/ │ │ │ │ └── ic_switch_camera_white_48dp.xml │ │ │ ├── layout/ │ │ │ │ ├── activity_camerax_live_preview.xml │ │ │ │ ├── activity_chooser.xml │ │ │ │ ├── activity_live_preview.xml │ │ │ │ ├── activity_settings.xml │ │ │ │ ├── activity_still_image.xml │ │ │ │ ├── settings_style.xml │ │ │ │ ├── spinner_style.xml │ │ │ │ └── toggle_style.xml │ │ │ ├── layout-land/ │ │ │ │ ├── activity_camerax_live_preview.xml │ │ │ │ └── activity_live_preview.xml │ │ │ ├── menu/ │ │ │ │ └── camera_button_menu.xml │ │ │ ├── values/ │ │ │ │ ├── colors.xml │ │ │ │ ├── dimens.xml │ │ │ │ ├── strings.xml │ │ │ │ └── styles.xml │ │ │ └── xml/ │ │ │ ├── preference_live_preview_automl.xml │ │ │ └── preference_still_image.xml │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradle.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle │ ├── codescanner/ │ │ ├── LICENSE │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── samples/ │ │ │ │ └── codescanner/ │ │ │ │ ├── EntryChoiceActivity.kt │ │ │ │ ├── java/ │ │ │ │ │ └── MainActivity.java │ │ │ │ └── kotlin/ │ │ │ │ └── MainActivity.kt │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ └── logo_mlkit.xml │ │ │ ├── layout/ │ │ │ │ └── activity_main.xml │ │ │ └── values/ │ │ │ ├── colors.xml │ │ │ ├── strings.xml │ │ │ └── styles.xml │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradle.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle │ ├── digitalink/ │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ ├── proguard-rules.pro │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── samples/ │ │ │ │ └── vision/ │ │ │ │ └── digitalink/ │ │ │ │ └── recognition/ │ │ │ │ ├── DigitalInkMainActivity.java │ │ │ │ ├── DrawingView.java │ │ │ │ ├── ModelManager.java │ │ │ │ ├── RecognitionTask.java │ │ │ │ ├── StatusTextView.java │ │ │ │ ├── StrokeManager.java │ │ │ │ └── kotlin/ │ │ │ │ ├── DigitalInkMainActivity.kt │ │ │ │ ├── DrawingView.kt │ │ │ │ ├── ModelManager.kt │ │ │ │ ├── RecognitionTask.kt │ │ │ │ ├── StatusTextView.kt │ │ │ │ └── StrokeManager.kt │ │ │ └── res/ │ │ │ ├── layout/ │ │ │ │ ├── activity_digital_ink_main.xml │ │ │ │ └── activity_digital_ink_main_kotlin.xml │ │ │ └── values/ │ │ │ ├── colors.xml │ │ │ ├── strings.xml │ │ │ └── styles.xml │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradle.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle │ ├── documentscanner/ │ │ ├── LICENSE │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── samples/ │ │ │ │ └── documentscanner/ │ │ │ │ ├── EntryChoiceActivity.kt │ │ │ │ ├── java/ │ │ │ │ │ └── MainActivity.java │ │ │ │ └── kotlin/ │ │ │ │ └── MainActivity.kt │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ └── logo_mlkit.xml │ │ │ ├── layout/ │ │ │ │ └── activity_main.xml │ │ │ ├── values/ │ │ │ │ ├── colors.xml │ │ │ │ ├── strings.xml │ │ │ │ └── styles.xml │ │ │ └── xml/ │ │ │ └── file_paths.xml │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradle.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle │ ├── entityextraction/ │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── samples/ │ │ │ │ └── nl/ │ │ │ │ └── entityextraction/ │ │ │ │ ├── EntryChoiceActivity.kt │ │ │ │ ├── java/ │ │ │ │ │ ├── MainActivityJava.java │ │ │ │ │ └── ModelsActivityJava.java │ │ │ │ └── kotlin/ │ │ │ │ ├── MainActivityKotlin.kt │ │ │ │ └── ModelsActivityKotlin.kt │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ ├── ic_baseline_delete_gray_32.xml │ │ │ │ ├── ic_baseline_get_app_gray_32.xml │ │ │ │ ├── ic_baseline_get_app_white_32.xml │ │ │ │ └── logo_mlkit.xml │ │ │ ├── layout/ │ │ │ │ ├── activity_main.xml │ │ │ │ ├── activity_models.xml │ │ │ │ └── list_item.xml │ │ │ ├── menu/ │ │ │ │ └── menu_choose_model.xml │ │ │ └── values/ │ │ │ ├── colors.xml │ │ │ ├── strings.xml │ │ │ └── styles.xml │ │ ├── build.gradle │ │ ├── gradle.properties │ │ └── settings.gradle │ ├── genai/ │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── genai/ │ │ │ │ └── demo/ │ │ │ │ ├── ContentAdapter.kt │ │ │ │ ├── ContentItem.kt │ │ │ │ ├── EntryChoiceActivity.kt │ │ │ │ ├── GenerationConfigDialog.kt │ │ │ │ ├── GenerationConfigUtils.kt │ │ │ │ ├── java/ │ │ │ │ │ ├── BaseActivity.java │ │ │ │ │ ├── ImageDescriptionActivity.java │ │ │ │ │ ├── OpenPromptActivity.java │ │ │ │ │ ├── ProofreadingActivity.java │ │ │ │ │ ├── RewritingActivity.java │ │ │ │ │ ├── SummarizationActivity.java │ │ │ │ │ └── TextInputBaseActivity.java │ │ │ │ └── kotlin/ │ │ │ │ ├── BaseActivity.kt │ │ │ │ ├── ImageDescriptionActivity.kt │ │ │ │ ├── OpenPromptActivity.kt │ │ │ │ ├── ProofreadingActivity.kt │ │ │ │ ├── RewritingActivity.kt │ │ │ │ ├── SummarizationActivity.kt │ │ │ │ └── TextInputBaseActivity.kt │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ ├── edit_text_background_chat.xml │ │ │ │ ├── ic_add_photo.xml │ │ │ │ ├── ic_send.xml │ │ │ │ ├── list_item_background.xml │ │ │ │ ├── request_item_background.xml │ │ │ │ └── response_item_background.xml │ │ │ ├── layout/ │ │ │ │ ├── activity_entry_choice.xml │ │ │ │ ├── activity_image_description.xml │ │ │ │ ├── activity_openprompt.xml │ │ │ │ ├── activity_proofreading.xml │ │ │ │ ├── activity_rewrite.xml │ │ │ │ ├── activity_summarization.xml │ │ │ │ ├── dialog_generation_config.xml │ │ │ │ ├── entry_choice_item.xml │ │ │ │ ├── row_item_request_image.xml │ │ │ │ ├── row_item_request_text.xml │ │ │ │ ├── row_item_request_text_and_images.xml │ │ │ │ └── row_item_response.xml │ │ │ ├── menu/ │ │ │ │ └── menu_main.xml │ │ │ └── values/ │ │ │ ├── colors.xml │ │ │ ├── strings.xml │ │ │ └── themes.xml │ │ ├── build.gradle │ │ ├── gradle.properties │ │ └── settings.gradle │ ├── internal/ │ │ ├── chooserx/ │ │ │ ├── build.gradle │ │ │ ├── proguard-rules.pro │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── mlkit/ │ │ │ │ └── example/ │ │ │ │ └── internal/ │ │ │ │ ├── BaseEntryChoiceActivity.java │ │ │ │ ├── Choice.java │ │ │ │ └── ChoiceAdapter.java │ │ │ └── res/ │ │ │ └── layout/ │ │ │ ├── activity_entry_choice.xml │ │ │ └── item_choice.xml │ │ ├── lint/ │ │ │ ├── build.gradle │ │ │ ├── gradle/ │ │ │ │ └── wrapper/ │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ │ ├── gradlew │ │ │ ├── gradlew.bat │ │ │ └── src/ │ │ │ ├── main/ │ │ │ │ └── java/ │ │ │ │ └── com/ │ │ │ │ └── mlkit/ │ │ │ │ └── lint/ │ │ │ │ ├── HungarianNotationDetector.kt │ │ │ │ ├── InvalidImportDetector.kt │ │ │ │ └── QuickstartIssueRegistry.kt │ │ │ └── test/ │ │ │ └── java/ │ │ │ └── com/ │ │ │ └── mlkit/ │ │ │ └── lint/ │ │ │ └── InvalidImportDetectorTest.kt │ │ └── lintchecks/ │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ ├── proguard-rules.pro │ │ └── src/ │ │ └── main/ │ │ └── AndroidManifest.xml │ ├── langid/ │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ ├── proguard-rules.pro │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── samples/ │ │ │ │ └── nl/ │ │ │ │ └── languageid/ │ │ │ │ ├── EntryChoiceActivity.kt │ │ │ │ ├── java/ │ │ │ │ │ └── MainActivityJava.java │ │ │ │ └── kotlin/ │ │ │ │ └── MainActivityKotlin.kt │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ └── logo_mlkit.xml │ │ │ ├── layout/ │ │ │ │ └── activity_main.xml │ │ │ └── values/ │ │ │ ├── colors.xml │ │ │ ├── strings.xml │ │ │ └── styles.xml │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradle.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle │ ├── material-showcase/ │ │ ├── .editorconfig │ │ ├── README.md │ │ ├── app/ │ │ │ ├── assets/ │ │ │ │ └── custom_models/ │ │ │ │ └── bird_classifier.tflite │ │ │ ├── build.gradle │ │ │ ├── proguard-rules.pro │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── md/ │ │ │ │ ├── CustomModelObjectDetectionActivity.kt │ │ │ │ ├── InputInfo.kt │ │ │ │ ├── LiveBarcodeScanningActivity.kt │ │ │ │ ├── LiveObjectDetectionActivity.kt │ │ │ │ ├── MainActivity.kt │ │ │ │ ├── ScopedExecutor.kt │ │ │ │ ├── StaticObjectDetectionActivity.kt │ │ │ │ ├── TaskExt.kt │ │ │ │ ├── Utils.kt │ │ │ │ ├── barcodedetection/ │ │ │ │ │ ├── BarcodeConfirmingGraphic.kt │ │ │ │ │ ├── BarcodeField.kt │ │ │ │ │ ├── BarcodeFieldAdapter.kt │ │ │ │ │ ├── BarcodeGraphicBase.kt │ │ │ │ │ ├── BarcodeLoadingGraphic.kt │ │ │ │ │ ├── BarcodeProcessor.kt │ │ │ │ │ ├── BarcodeResultFragment.kt │ │ │ │ │ └── BarcodeReticleGraphic.kt │ │ │ │ ├── camera/ │ │ │ │ │ ├── CameraReticleAnimator.kt │ │ │ │ │ ├── CameraSizePair.kt │ │ │ │ │ ├── CameraSource.kt │ │ │ │ │ ├── CameraSourcePreview.kt │ │ │ │ │ ├── FrameMetadata.kt │ │ │ │ │ ├── FrameProcessor.kt │ │ │ │ │ ├── FrameProcessorBase.kt │ │ │ │ │ ├── GraphicOverlay.kt │ │ │ │ │ └── WorkflowModel.kt │ │ │ │ ├── objectdetection/ │ │ │ │ │ ├── DetectedObjectInfo.kt │ │ │ │ │ ├── MultiObjectProcessor.kt │ │ │ │ │ ├── ObjectConfirmationController.kt │ │ │ │ │ ├── ObjectConfirmationGraphic.kt │ │ │ │ │ ├── ObjectDotAnimator.kt │ │ │ │ │ ├── ObjectDotGraphic.kt │ │ │ │ │ ├── ObjectGraphicInMultiMode.kt │ │ │ │ │ ├── ObjectGraphicInProminentMode.kt │ │ │ │ │ ├── ObjectReticleGraphic.kt │ │ │ │ │ ├── ProminentObjectProcessor.kt │ │ │ │ │ └── StaticObjectDotView.kt │ │ │ │ ├── productsearch/ │ │ │ │ │ ├── BottomSheetScrimView.kt │ │ │ │ │ ├── ImageDownloadTask.kt │ │ │ │ │ ├── PreviewCardAdapter.kt │ │ │ │ │ ├── Product.kt │ │ │ │ │ ├── ProductAdapter.kt │ │ │ │ │ ├── SearchEngine.kt │ │ │ │ │ └── SearchedObject.kt │ │ │ │ └── settings/ │ │ │ │ ├── PreferenceUtils.kt │ │ │ │ ├── SettingsActivity.kt │ │ │ │ └── SettingsFragment.kt │ │ │ └── res/ │ │ │ ├── animator/ │ │ │ │ ├── bottom_prompt_chip_enter.xml │ │ │ │ ├── search_button_enter.xml │ │ │ │ └── static_image_dot_enter.xml │ │ │ ├── drawable/ │ │ │ │ ├── barcode_field_box.xml │ │ │ │ ├── camera_flash.xml │ │ │ │ ├── ic_close_vd_white_24.xml │ │ │ │ ├── ic_flash_off_vd_white_24.xml │ │ │ │ ├── ic_flash_on_vd_white_24.xml │ │ │ │ ├── ic_image_search_vd_black_24.xml │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ ├── ic_photo_library_vd_white_24.xml │ │ │ │ ├── ic_settings_vd_white_24.xml │ │ │ │ ├── logo_mlkit.xml │ │ │ │ └── top_action_bar_scrim.xml │ │ │ ├── layout/ │ │ │ │ ├── activity_live_barcode.xml │ │ │ │ ├── activity_live_object.xml │ │ │ │ ├── activity_main.xml │ │ │ │ ├── activity_settings.xml │ │ │ │ ├── activity_static_object.xml │ │ │ │ ├── barcode_bottom_sheet.xml │ │ │ │ ├── barcode_field.xml │ │ │ │ ├── camera_preview_overlay.xml │ │ │ │ ├── detection_mode_item.xml │ │ │ │ ├── product_bottom_sheet.xml │ │ │ │ ├── product_item.xml │ │ │ │ ├── products_preview_card.xml │ │ │ │ ├── top_action_bar_in_live_camera.xml │ │ │ │ └── top_action_bar_in_static_image.xml │ │ │ ├── values/ │ │ │ │ ├── colors.xml │ │ │ │ ├── dimens.xml │ │ │ │ ├── strings.xml │ │ │ │ └── styles.xml │ │ │ ├── values-v21/ │ │ │ │ └── styles.xml │ │ │ └── xml/ │ │ │ └── preferences.xml │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradle.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle │ ├── smartreply/ │ │ ├── LICENSE │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ ├── proguard-rules.pro │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── samples/ │ │ │ │ └── nl/ │ │ │ │ └── smartreply/ │ │ │ │ ├── EntryChoiceActivity.kt │ │ │ │ ├── java/ │ │ │ │ │ ├── MainActivityJava.java │ │ │ │ │ ├── chat/ │ │ │ │ │ │ ├── ChatFragment.java │ │ │ │ │ │ ├── ChatViewModel.java │ │ │ │ │ │ ├── MessageListAdapter.java │ │ │ │ │ │ └── ReplyChipAdapter.java │ │ │ │ │ └── model/ │ │ │ │ │ └── Message.java │ │ │ │ └── kotlin/ │ │ │ │ ├── MainActivityKotlin.kt │ │ │ │ ├── chat/ │ │ │ │ │ ├── ChatFragment.kt │ │ │ │ │ ├── ChatViewModel.kt │ │ │ │ │ ├── MessageListAdapter.kt │ │ │ │ │ └── ReplyChipAdapter.kt │ │ │ │ └── model/ │ │ │ │ └── Message.kt │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ ├── ic_camera_front_black_24dp.xml │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ ├── ic_tag_faces_black_24dp.xml │ │ │ │ ├── logo_mlkit.xml │ │ │ │ ├── rounded_rectangle_blue.xml │ │ │ │ ├── rounded_rectangle_gray.xml │ │ │ │ └── rounded_rectangle_light_gray.xml │ │ │ ├── drawable-v24/ │ │ │ │ └── ic_launcher_foreground.xml │ │ │ ├── layout/ │ │ │ │ ├── chat_fragment.xml │ │ │ │ ├── item_message_local.xml │ │ │ │ ├── item_message_remote.xml │ │ │ │ ├── main_smartreply_activity.xml │ │ │ │ └── smart_reply_chip.xml │ │ │ ├── menu/ │ │ │ │ └── chat_fragment_actions.xml │ │ │ └── values/ │ │ │ ├── colors.xml │ │ │ ├── dimens.xml │ │ │ ├── strings.xml │ │ │ └── styles.xml │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradle.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle │ ├── speech/ │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── genai/ │ │ │ │ └── speech/ │ │ │ │ └── demo/ │ │ │ │ └── SpeechRecognitionActivity.kt │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ ├── ic_launcher_background.xml │ │ │ │ └── ic_launcher_foreground.xml │ │ │ ├── layout/ │ │ │ │ └── activity_speech_recognition.xml │ │ │ ├── mipmap-anydpi/ │ │ │ │ ├── ic_launcher.xml │ │ │ │ └── ic_launcher_round.xml │ │ │ └── values/ │ │ │ ├── arrays.xml │ │ │ ├── colors.xml │ │ │ ├── strings.xml │ │ │ └── themes.xml │ │ ├── build.gradle │ │ ├── gradle.properties │ │ └── settings.gradle │ ├── translate/ │ │ ├── LICENSE │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ ├── proguard-rules.pro │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── samples/ │ │ │ │ └── nl/ │ │ │ │ └── translate/ │ │ │ │ ├── EntryChoiceActivity.kt │ │ │ │ ├── java/ │ │ │ │ │ ├── MainActivity.java │ │ │ │ │ ├── TranslateFragment.java │ │ │ │ │ └── TranslateViewModel.java │ │ │ │ └── kotlin/ │ │ │ │ ├── MainActivity.kt │ │ │ │ ├── TranslateFragment.kt │ │ │ │ └── TranslateViewModel.kt │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ ├── ic_baseline_delete_24.xml │ │ │ │ ├── ic_baseline_translate_32.xml │ │ │ │ ├── ic_compare_arrows_black_24dp.xml │ │ │ │ ├── ic_file_download_white_24dp.xml │ │ │ │ └── logo_mlkit.xml │ │ │ ├── layout/ │ │ │ │ ├── activity_translate_main.xml │ │ │ │ └── translate_fragment.xml │ │ │ └── values/ │ │ │ ├── colors.xml │ │ │ ├── dimens.xml │ │ │ ├── strings.xml │ │ │ └── styles.xml │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradle.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle │ ├── translate-showcase/ │ │ ├── README.md │ │ ├── app/ │ │ │ ├── build.gradle │ │ │ ├── proguard-rules.pro │ │ │ └── src/ │ │ │ └── main/ │ │ │ ├── AndroidManifest.xml │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── google/ │ │ │ │ └── mlkit/ │ │ │ │ └── showcase/ │ │ │ │ └── translate/ │ │ │ │ ├── MainActivity.kt │ │ │ │ ├── analyzer/ │ │ │ │ │ └── TextAnalyzer.kt │ │ │ │ ├── main/ │ │ │ │ │ ├── MainFragment.kt │ │ │ │ │ └── MainViewModel.kt │ │ │ │ └── util/ │ │ │ │ ├── ImageUtils.kt │ │ │ │ ├── Language.kt │ │ │ │ ├── ResultOrError.kt │ │ │ │ ├── ScopedExecutor.kt │ │ │ │ └── SmoothedMutableLiveData.kt │ │ │ └── res/ │ │ │ ├── drawable/ │ │ │ │ └── logo_mlkit.xml │ │ │ ├── layout/ │ │ │ │ ├── main_fragment.xml │ │ │ │ └── main_translateshowcase_activity.xml │ │ │ └── values/ │ │ │ ├── colors.xml │ │ │ ├── strings.xml │ │ │ └── styles.xml │ │ ├── build.gradle │ │ ├── gradle/ │ │ │ └── wrapper/ │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── gradle.properties │ │ ├── gradlew │ │ ├── gradlew.bat │ │ └── settings.gradle │ └── vision-quickstart/ │ ├── LICENSE │ ├── README.md │ ├── app/ │ │ ├── build.gradle │ │ └── src/ │ │ └── main/ │ │ ├── AndroidManifest.xml │ │ ├── assets/ │ │ │ ├── automl/ │ │ │ │ ├── dict.txt │ │ │ │ ├── manifest.json │ │ │ │ └── model.tflite │ │ │ ├── custom_models/ │ │ │ │ ├── bird_classifier.tflite │ │ │ │ └── object_labeler.tflite │ │ │ └── pose/ │ │ │ └── fitness_pose_samples.csv │ │ ├── java/ │ │ │ └── com/ │ │ │ └── google/ │ │ │ └── mlkit/ │ │ │ └── vision/ │ │ │ └── demo/ │ │ │ ├── BitmapUtils.java │ │ │ ├── CameraImageGraphic.java │ │ │ ├── CameraSource.java │ │ │ ├── CameraSourcePreview.java │ │ │ ├── CameraXViewModel.java │ │ │ ├── EntryChoiceActivity.kt │ │ │ ├── FrameMetadata.java │ │ │ ├── GraphicOverlay.java │ │ │ ├── InferenceInfoGraphic.java │ │ │ ├── ScopedExecutor.java │ │ │ ├── TemperatureMonitor.java │ │ │ ├── VisionImageProcessor.java │ │ │ ├── java/ │ │ │ │ ├── CameraXLivePreviewActivity.java │ │ │ │ ├── CameraXSourceDemoActivity.java │ │ │ │ ├── ChooserActivity.java │ │ │ │ ├── LivePreviewActivity.java │ │ │ │ ├── StillImageActivity.java │ │ │ │ ├── VisionProcessorBase.java │ │ │ │ ├── barcodescanner/ │ │ │ │ │ ├── BarcodeGraphic.java │ │ │ │ │ └── BarcodeScannerProcessor.java │ │ │ │ ├── facedetector/ │ │ │ │ │ ├── FaceDetectorProcessor.java │ │ │ │ │ └── FaceGraphic.java │ │ │ │ ├── facemeshdetector/ │ │ │ │ │ ├── FaceMeshDetectorProcessor.java │ │ │ │ │ └── FaceMeshGraphic.java │ │ │ │ ├── labeldetector/ │ │ │ │ │ ├── LabelDetectorProcessor.java │ │ │ │ │ └── LabelGraphic.java │ │ │ │ ├── objectdetector/ │ │ │ │ │ ├── ObjectDetectorProcessor.java │ │ │ │ │ └── ObjectGraphic.java │ │ │ │ ├── posedetector/ │ │ │ │ │ ├── PoseDetectorProcessor.java │ │ │ │ │ ├── PoseGraphic.java │ │ │ │ │ └── classification/ │ │ │ │ │ ├── ClassificationResult.java │ │ │ │ │ ├── EMASmoothing.java │ │ │ │ │ ├── PoseClassifier.java │ │ │ │ │ ├── PoseClassifierProcessor.java │ │ │ │ │ ├── PoseEmbedding.java │ │ │ │ │ ├── PoseSample.java │ │ │ │ │ ├── RepetitionCounter.java │ │ │ │ │ └── Utils.java │ │ │ │ ├── segmenter/ │ │ │ │ │ ├── SegmentationGraphic.java │ │ │ │ │ └── SegmenterProcessor.java │ │ │ │ ├── subjectsegmenter/ │ │ │ │ │ ├── SubjectSegmentationGraphic.java │ │ │ │ │ └── SubjectSegmenterProcessor.java │ │ │ │ └── textdetector/ │ │ │ │ ├── TextGraphic.java │ │ │ │ └── TextRecognitionProcessor.java │ │ │ ├── kotlin/ │ │ │ │ ├── CameraXLivePreviewActivity.kt │ │ │ │ ├── CameraXSourceDemoActivity.kt │ │ │ │ ├── ChooserActivity.kt │ │ │ │ ├── LivePreviewActivity.kt │ │ │ │ ├── StillImageActivity.kt │ │ │ │ ├── TaskExt.kt │ │ │ │ ├── VisionProcessorBase.kt │ │ │ │ ├── barcodescanner/ │ │ │ │ │ ├── BarcodeGraphic.kt │ │ │ │ │ └── BarcodeScannerProcessor.kt │ │ │ │ ├── facedetector/ │ │ │ │ │ ├── FaceDetectorProcessor.kt │ │ │ │ │ └── FaceGraphic.kt │ │ │ │ ├── facemeshdetector/ │ │ │ │ │ ├── FaceMeshDetectorProcessor.kt │ │ │ │ │ └── FaceMeshGraphic.kt │ │ │ │ ├── labeldetector/ │ │ │ │ │ ├── LabelDetectorProcessor.kt │ │ │ │ │ └── LabelGraphic.kt │ │ │ │ ├── objectdetector/ │ │ │ │ │ ├── ObjectDetectorProcessor.kt │ │ │ │ │ └── ObjectGraphic.kt │ │ │ │ ├── posedetector/ │ │ │ │ │ ├── PoseDetectorProcessor.kt │ │ │ │ │ └── PoseGraphic.kt │ │ │ │ ├── segmenter/ │ │ │ │ │ ├── SegmentationGraphic.kt │ │ │ │ │ └── SegmenterProcessor.kt │ │ │ │ ├── subjectsegmenter/ │ │ │ │ │ ├── SubjectSegmentationGraphic.kt │ │ │ │ │ └── SubjectSegmenterProcessor.kt │ │ │ │ └── textdetector/ │ │ │ │ ├── TextGraphic.kt │ │ │ │ └── TextRecognitionProcessor.kt │ │ │ └── preference/ │ │ │ ├── CameraXLivePreviewPreferenceFragment.java │ │ │ ├── CameraXSourceDemoPreferenceFragment.java │ │ │ ├── LivePreviewPreferenceFragment.java │ │ │ ├── PreferenceUtils.java │ │ │ ├── SettingsActivity.java │ │ │ └── StillImagePreferenceFragment.java │ │ └── res/ │ │ ├── drawable/ │ │ │ ├── list_item_background.xml │ │ │ └── logo_mlkit.xml │ │ ├── drawable-hdpi/ │ │ │ └── ic_switch_camera_white_48dp.xml │ │ ├── drawable-mdpi/ │ │ │ └── ic_switch_camera_white_48dp.xml │ │ ├── drawable-xhdpi/ │ │ │ └── ic_switch_camera_white_48dp.xml │ │ ├── drawable-xxhdpi/ │ │ │ └── ic_switch_camera_white_48dp.xml │ │ ├── drawable-xxxhdpi/ │ │ │ └── ic_switch_camera_white_48dp.xml │ │ ├── layout/ │ │ │ ├── activity_chooser.xml │ │ │ ├── activity_settings.xml │ │ │ ├── activity_still_image.xml │ │ │ ├── activity_vision_camerax_live_preview.xml │ │ │ ├── activity_vision_cameraxsource_demo.xml │ │ │ ├── activity_vision_entry_choice.xml │ │ │ ├── activity_vision_live_preview.xml │ │ │ ├── settings_style.xml │ │ │ ├── spinner_style.xml │ │ │ └── toggle_style.xml │ │ ├── layout-land/ │ │ │ ├── activity_vision_camerax_live_preview.xml │ │ │ ├── activity_vision_cameraxsource_demo.xml │ │ │ └── activity_vision_live_preview.xml │ │ ├── menu/ │ │ │ └── camera_button_menu.xml │ │ ├── values/ │ │ │ ├── arrays.xml │ │ │ ├── colors.xml │ │ │ ├── dimens.xml │ │ │ ├── strings.xml │ │ │ └── styles.xml │ │ └── xml/ │ │ ├── preference_live_preview_quickstart.xml │ │ └── preference_still_image.xml │ ├── build.gradle │ ├── gradle/ │ │ └── wrapper/ │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties │ ├── gradle.properties │ ├── gradlew │ ├── gradlew.bat │ └── settings.gradle ├── ios/ │ ├── ios-snippets/ │ │ ├── Podfile │ │ ├── mlkit-snippets.xcodeproj/ │ │ │ ├── project.pbxproj │ │ │ ├── project.xcworkspace/ │ │ │ │ ├── contents.xcworkspacedata │ │ │ │ └── xcshareddata/ │ │ │ │ └── IDEWorkspaceChecks.plist │ │ │ └── xcuserdata/ │ │ │ └── dfurlong.xcuserdatad/ │ │ │ └── xcschemes/ │ │ │ └── xcschememanagement.plist │ │ ├── mlkit-snippets.xcworkspace/ │ │ │ ├── contents.xcworkspacedata │ │ │ ├── xcshareddata/ │ │ │ │ └── IDEWorkspaceChecks.plist │ │ │ └── xcuserdata/ │ │ │ └── dfurlong.xcuserdatad/ │ │ │ └── xcdebugger/ │ │ │ └── Breakpoints_v2.xcbkptlist │ │ ├── objc-snippets/ │ │ │ ├── AppDelegate.h │ │ │ ├── AppDelegate.m │ │ │ ├── Assets.xcassets/ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ └── Contents.json │ │ │ │ └── Contents.json │ │ │ ├── AutoMLVision.m │ │ │ ├── Base.lproj/ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ └── Main.storyboard │ │ │ ├── ImagePreparation.m │ │ │ ├── Info.plist │ │ │ ├── ModelManagement.m │ │ │ ├── ViewController.h │ │ │ ├── ViewController.m │ │ │ └── main.m │ │ └── swift-snippets/ │ │ ├── AppDelegate.swift │ │ ├── Assets.xcassets/ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ └── Contents.json │ │ │ └── Contents.json │ │ ├── AutoMLVision.swift │ │ ├── Base.lproj/ │ │ │ ├── LaunchScreen.storyboard │ │ │ └── Main.storyboard │ │ ├── ImagePreparation.swift │ │ ├── Info.plist │ │ ├── ModelManagement.swift │ │ └── ViewController.swift │ ├── quickstarts/ │ │ ├── automl/ │ │ │ ├── Assets.xcassets/ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── Contents.json │ │ │ │ ├── cloud_download.imageset/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── delete.imageset/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── photo_camera.imageset/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── photo_library.imageset/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── switch_camera.imageset/ │ │ │ │ │ └── Contents.json │ │ │ │ └── video_camera.imageset/ │ │ │ │ └── Contents.json │ │ │ ├── AutoMLExample/ │ │ │ │ ├── AppDelegate.swift │ │ │ │ ├── Base.lproj/ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ └── Main.storyboard │ │ │ │ ├── CameraViewController.swift │ │ │ │ ├── Info.plist │ │ │ │ ├── MLKitExtensions.swift │ │ │ │ ├── UIUtilities.swift │ │ │ │ └── ViewController.swift │ │ │ ├── AutoMLExample.xcodeproj/ │ │ │ │ └── project.pbxproj │ │ │ ├── AutoMLExampleObjC/ │ │ │ │ ├── AppDelegate.h │ │ │ │ ├── AppDelegate.m │ │ │ │ ├── CameraViewController.h │ │ │ │ ├── CameraViewController.m │ │ │ │ ├── UIImage+VisionDetection.h │ │ │ │ ├── UIImage+VisionDetection.m │ │ │ │ ├── UIUtilities.h │ │ │ │ ├── UIUtilities.m │ │ │ │ ├── ViewController.h │ │ │ │ ├── ViewController.m │ │ │ │ └── main.m │ │ │ ├── Podfile │ │ │ ├── README.md │ │ │ └── Resources/ │ │ │ ├── automl_labeler_labels.txt │ │ │ ├── automl_labeler_manifest.json │ │ │ └── automl_labeler_model.tflite │ │ ├── digitalinkrecognition/ │ │ │ ├── DigitalInkRecognitionExample/ │ │ │ │ ├── AppDelegate.swift │ │ │ │ ├── Info.plist │ │ │ │ ├── RecognizedInk.swift │ │ │ │ ├── SceneDelegate.swift │ │ │ │ ├── StrokeManager.swift │ │ │ │ └── ViewController.swift │ │ │ ├── DigitalInkRecognitionExample.xcodeproj/ │ │ │ │ └── project.pbxproj │ │ │ ├── DigitalInkRecognitionExampleObjC/ │ │ │ │ ├── AppDelegate.h │ │ │ │ ├── AppDelegate.m │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ └── Contents.json │ │ │ │ ├── Base.lproj/ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ └── Main.storyboard │ │ │ │ ├── RecognizedInk.h │ │ │ │ ├── RecognizedInk.m │ │ │ │ ├── SceneDelegate.h │ │ │ │ ├── SceneDelegate.m │ │ │ │ ├── StrokeManager.h │ │ │ │ ├── StrokeManager.m │ │ │ │ ├── ViewController.h │ │ │ │ ├── ViewController.m │ │ │ │ └── main.m │ │ │ ├── Podfile │ │ │ └── README.md │ │ ├── entityextraction/ │ │ │ ├── Assets.xcassets/ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── cloud_download_24pt.imageset/ │ │ │ │ │ └── Contents.json │ │ │ │ └── delete_24pt.imageset/ │ │ │ │ └── Contents.json │ │ │ ├── EntityExtractionExample/ │ │ │ │ ├── AppDelegate.swift │ │ │ │ ├── Base.lproj/ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ └── Main.storyboard │ │ │ │ ├── EntityExtractionModelIdentifierExtensions.swift │ │ │ │ ├── EntityViewController.swift │ │ │ │ ├── Info.plist │ │ │ │ └── ModelManagementViewController.swift │ │ │ ├── EntityExtractionExample.xcodeproj/ │ │ │ │ └── project.pbxproj │ │ │ ├── EntityExtractionExampleObjC/ │ │ │ │ ├── AppDelegate.h │ │ │ │ ├── AppDelegate.m │ │ │ │ ├── EntityViewController.h │ │ │ │ ├── EntityViewController.m │ │ │ │ ├── MLKEntityExtractionModelIdentifier+Extensions.h │ │ │ │ ├── MLKEntityExtractionModelIdentifier+Extensions.m │ │ │ │ ├── ModelManagementViewController.h │ │ │ │ ├── ModelManagementViewController.m │ │ │ │ └── main.m │ │ │ ├── Podfile │ │ │ └── README.md │ │ ├── languageid/ │ │ │ ├── LanguageIDExample/ │ │ │ │ ├── AppDelegate.swift │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ └── Contents.json │ │ │ │ ├── Base.lproj/ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ └── Main.storyboard │ │ │ │ ├── Info.plist │ │ │ │ └── ViewController.swift │ │ │ ├── LanguageIDExample.xcodeproj/ │ │ │ │ └── project.pbxproj │ │ │ ├── LanguageIDExampleObjC/ │ │ │ │ ├── AppDelegate.h │ │ │ │ ├── AppDelegate.m │ │ │ │ ├── ViewController.h │ │ │ │ ├── ViewController.m │ │ │ │ └── main.m │ │ │ ├── Podfile │ │ │ └── README.md │ │ ├── smartreply/ │ │ │ ├── Podfile │ │ │ ├── README.md │ │ │ ├── SmartReplyExample/ │ │ │ │ ├── AppDelegate.swift │ │ │ │ ├── DateExtension.swift │ │ │ │ ├── MainViewController.swift │ │ │ │ └── UITextViewPlaceholder.swift │ │ │ ├── SmartReplyExample.xcodeproj/ │ │ │ │ └── project.pbxproj │ │ │ └── SmartReplyExampleObjC/ │ │ │ ├── AppDelegate.h │ │ │ ├── AppDelegate.m │ │ │ ├── Assets.xcassets/ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── LaunchImage.launchimage/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── ic_account_circle_36pt.imageset/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── ic_more_vert_white.imageset/ │ │ │ │ │ └── Contents.json │ │ │ │ └── ic_send.imageset/ │ │ │ │ └── Contents.json │ │ │ ├── Base.lproj/ │ │ │ │ └── Main.storyboard │ │ │ ├── Info.plist │ │ │ ├── LaunchScreen.xib │ │ │ ├── MainViewController.h │ │ │ ├── MainViewController.m │ │ │ ├── NSDate+Format.h │ │ │ ├── NSDate+Format.m │ │ │ ├── UITextView+Placeholder.h │ │ │ ├── UITextView+Placeholder.m │ │ │ └── main.m │ │ ├── translate/ │ │ │ ├── Podfile │ │ │ ├── README.md │ │ │ ├── TranslateExample/ │ │ │ │ ├── AppDelegate.swift │ │ │ │ ├── Assets.xcassets/ │ │ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ │ │ └── Contents.json │ │ │ │ │ ├── Contents.json │ │ │ │ │ └── baseline_swap_horiz_black_48pt.imageset/ │ │ │ │ │ └── Contents.json │ │ │ │ ├── Base.lproj/ │ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ │ └── Main.storyboard │ │ │ │ ├── Info.plist │ │ │ │ └── ViewController.swift │ │ │ ├── TranslateExample.xcodeproj/ │ │ │ │ └── project.pbxproj │ │ │ └── TranslateExampleObjC/ │ │ │ ├── AppDelegate.h │ │ │ ├── AppDelegate.m │ │ │ ├── ViewController.h │ │ │ ├── ViewController.m │ │ │ └── main.m │ │ └── vision/ │ │ ├── Assets.xcassets/ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ └── Contents.json │ │ │ ├── Contents.json │ │ │ ├── photo_camera.imageset/ │ │ │ │ └── Contents.json │ │ │ ├── photo_library.imageset/ │ │ │ │ └── Contents.json │ │ │ ├── switch_camera.imageset/ │ │ │ │ └── Contents.json │ │ │ └── video_camera.imageset/ │ │ │ └── Contents.json │ │ ├── Podfile │ │ ├── README.md │ │ ├── Resources/ │ │ │ └── bird.tflite │ │ ├── VisionExample/ │ │ │ ├── AppDelegate.swift │ │ │ ├── Base.lproj/ │ │ │ │ ├── LaunchScreen.storyboard │ │ │ │ └── Main.storyboard │ │ │ ├── CameraViewController.swift │ │ │ ├── Info.plist │ │ │ ├── MLKitExtensions.swift │ │ │ ├── UIUtilities.swift │ │ │ └── ViewController.swift │ │ ├── VisionExample.xcodeproj/ │ │ │ └── project.pbxproj │ │ └── VisionExampleObjC/ │ │ ├── AppDelegate.h │ │ ├── AppDelegate.m │ │ ├── CameraViewController.h │ │ ├── CameraViewController.m │ │ ├── UIImage+VisionDetection.h │ │ ├── UIImage+VisionDetection.m │ │ ├── UIUtilities.h │ │ ├── UIUtilities.m │ │ ├── ViewController.h │ │ ├── ViewController.m │ │ └── main.m │ └── showcase/ │ ├── translate-showcase/ │ │ ├── Assets.xcassets/ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ └── Contents.json │ │ │ ├── Contents.json │ │ │ ├── Image.imageset/ │ │ │ │ └── Contents.json │ │ │ ├── baseline_check_black_24pt.imageset/ │ │ │ │ └── Contents.json │ │ │ ├── ic_arrow_back.imageset/ │ │ │ │ └── Contents.json │ │ │ └── ic_close.imageset/ │ │ │ └── Contents.json │ │ ├── Podfile │ │ ├── README.md │ │ ├── TranslateDemo/ │ │ │ ├── AppDelegate.swift │ │ │ ├── Base.lproj/ │ │ │ │ └── Main.storyboard │ │ │ ├── CVPixelBuffer+Helpers.swift │ │ │ ├── CameraOverlayview.swift │ │ │ ├── CameraViewController.swift │ │ │ ├── Info.plist │ │ │ ├── MLKitExtensions.swift │ │ │ ├── SearchViewController.swift │ │ │ ├── TranslateLanguage+Helpers.swift │ │ │ └── UIUtilities.swift │ │ ├── TranslateDemo.xcodeproj/ │ │ │ └── project.pbxproj │ │ └── TranslateDemoTests/ │ │ ├── Info.plist │ │ └── TranslateDemoTests.swift │ └── vision-showcase/ │ ├── Podfile │ ├── README.md │ ├── ShowcaseApp/ │ │ ├── AppDelegate.h │ │ ├── AppDelegate.m │ │ ├── Base.lproj/ │ │ │ └── LaunchScreen.xib │ │ ├── Common/ │ │ │ ├── MLKImageUtilities.h │ │ │ ├── MLKImageUtilities.m │ │ │ ├── MLKUIUtilities.h │ │ │ ├── MLKUIUtilities.m │ │ │ ├── UIImage+MLKShowcase.h │ │ │ └── UIImage+MLKShowcase.m │ │ ├── Controllers/ │ │ │ ├── MLKLiveObjectDetectionViewController.h │ │ │ ├── MLKLiveObjectDetectionViewController.m │ │ │ ├── MLKResultListViewController.h │ │ │ ├── MLKResultListViewController.m │ │ │ ├── MLKStartPageViewController.h │ │ │ └── MLKStartPageViewController.m │ │ ├── Images.xcassets/ │ │ │ ├── AppIcon.appiconset/ │ │ │ │ └── Contents.json │ │ │ └── ic_arrow_back_ios.imageset/ │ │ │ └── Contents.json │ │ ├── Info.plist │ │ ├── Models/ │ │ │ ├── MLKODTStatus.h │ │ │ ├── MLKProductSearchRequest.h │ │ │ ├── MLKProductSearchRequest.m │ │ │ ├── MLKResult.h │ │ │ └── MLKResult.m │ │ ├── Views/ │ │ │ ├── MLKCameraReticle.h │ │ │ ├── MLKCameraReticle.m │ │ │ ├── MLKConfirmationSpinner.h │ │ │ ├── MLKConfirmationSpinner.m │ │ │ ├── MLKDetectionOverlayView.h │ │ │ ├── MLKDetectionOverlayView.m │ │ │ ├── MLKResultCell.h │ │ │ ├── MLKResultCell.m │ │ │ ├── MLKResultListHeaderView.h │ │ │ ├── MLKResultListHeaderView.m │ │ │ ├── MLKStartPageCell.h │ │ │ ├── MLKStartPageCell.m │ │ │ ├── MLKStartPageHeaderView.h │ │ │ └── MLKStartPageHeaderView.m │ │ ├── bird.tflite │ │ └── main.m │ └── ShowcaseApp.xcodeproj/ │ └── project.pbxproj └── tutorials/ ├── README.md └── mlkit_image_labeling_model_maker.ipynb ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Create a report to help us improve title: "[Bug report] Title for the bug" labels: '' assignees: '' --- **Describe the bug** A clear and concise description of what the bug is. **To Reproduce** Example Steps to reproduce the behavior in sample app: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error If applicable, please include `code snippet` and `sample input(image, video, text, etc)` **Expected behavior** A clear and concise description of what you expected to happen. **SDK Info:** - SDK Name & Version [e.g. com.google.mlkit:face-detection:16.1.3] **Smartphone:** - Device/Simulator: [e.g. iPhone6, Pixel 5] - Device/Simulator OS: [e.g. iOS8.1, Android API 30] **Development Environment:** *(For Android issue feel free to skip this section)* - IDE Eversion: [e.g. Xcode 13.1, Android Studio 4.2] - Laptop/Desktop: [e.g. iMac Pro (2017), HP Z840 Workstation] - Laptop/Desktop OS/version: [e.g. macOS 12.0.1] ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Suggest an idea for this project title: "[Feature request] Title for the request" labels: '' assignees: '' --- **What's your feature request? Please describe.** A clear and concise description of what the request is. Ex. I would like to have X language support in text recognition[...] **Mobile environment** Android, IOS or both **Additional context** Add any other context or screenshots about the feature request here. ================================================ FILE: .gitignore ================================================ *.DS_Store *.idea *.iml .gradle local.properties .project .settings .classpath **/Pods/ **/*.xcuserstate ios-snippets/Podfile.lock ios-snippets/mlkit-snippets.xcworkspace/xcuserdata/dfurlong.xcuserdatad/UserInterfaceState.xcuserstate ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # MLKit Samples A collection of quickstart samples demonstrating the [ML Kit](https://developers.google.com/ml-kit) APIs on Android and iOS. Note: due to how this repo works, we no longer accept pull requests directly. Instead, we'll patch them internally and then sync them out. ================================================ FILE: android/.gitignore ================================================ .gradle build/ .DS_Store *.iml *.apk *.aar *.zip google-services.json .project .settings .classpath local.properties .idea/* !idea/codeStyles captures .externalNativeBuild .cxx ================================================ FILE: android/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2020 Google Inc Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: android/README.md ================================================ # MLKit Android Samples A collection of quickstart samples demonstrating the MLKit APIs on Android. [See a full list of Quickstart apps here.](https://developers.google.com/ml-kit/samples) ## Introduction [Read more about ML Kit](https://developers.google.com/ml-kit) ================================================ FILE: android/android-snippets/README.md ================================================ # ML Kit Android Snippets This is a collection of Android code snippets seen on https://developers.google.com/ml-kit. This project is not meant to be run as a standalone app to demo ML Kit APIs. ## License Copyright 2020 Google, Inc. Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: android/android-snippets/app/build.gradle ================================================ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' apply plugin: 'kotlin-android-extensions' android { compileSdkVersion 31 defaultConfig { applicationId "com.google.example.mlkit" minSdkVersion 19 targetSdkVersion 31 versionCode 1 versionName "1.0" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" multiDexEnabled true } buildTypes { release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' } } packagingOptions { exclude 'META-INF/androidx.exifinterface_exifinterface.version' exclude 'META-INF/proguard/androidx-annotations.pro' } } dependencies { implementation 'androidx.legacy:legacy-support-v4:1.0.0' implementation 'androidx.appcompat:appcompat:1.4.2' implementation 'androidx.exifinterface:exifinterface:1.3.3' implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" // Add this dependency and multiDexEnabled = true in your defaultConfig to solve the error: // Cannot fit requested classes in a single dex file implementation 'androidx.multidex:multidex:2.0.1' // ML Kit related dependencies // Barcode model implementation 'com.google.mlkit:barcode-scanning:17.0.2' // Object detection and tracking features implementation 'com.google.mlkit:object-detection:17.0.0' implementation 'com.google.mlkit:object-detection-custom:17.0.0' // Face features implementation 'com.google.android.gms:play-services-mlkit-face-detection:17.0.1' // Text features implementation 'com.google.android.gms:play-services-mlkit-text-recognition:18.0.0' // Image labeling implementation 'com.google.mlkit:image-labeling:17.0.7' // Language Identification implementation 'com.google.mlkit:language-id:17.0.3' implementation 'androidx.appcompat:appcompat:1.4.2' implementation 'androidx.constraintlayout:constraintlayout:2.1.4' } ================================================ FILE: android/android-snippets/app/proguard-rules.pro ================================================ # Add project specific ProGuard rules here. # By default, the flags in this file are appended to flags specified # in /Users/ianbarber/Library/Android/sdk/tools/proguard/proguard-android.txt # You can edit the include path and order by changing the proguardFiles # directive in build.gradle. # # For more details, see # http://developer.android.com/guide/developing/tools/proguard.html # Add any project specific keep options here: # If your project uses WebView with JS, uncomment the following # and specify the fully qualified class name to the JavaScript interface # class: #-keepclassmembers class fqcn.of.javascript.interface.for.webview { # public *; #} ================================================ FILE: android/android-snippets/app/src/main/AndroidManifest.xml ================================================ ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/BarcodeScanningActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit; import android.graphics.Point; import android.graphics.Rect; import android.os.Bundle; import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.android.gms.tasks.Task; import com.google.mlkit.vision.barcode.common.Barcode; import com.google.mlkit.vision.barcode.BarcodeScanner; import com.google.mlkit.vision.barcode.BarcodeScannerOptions; import com.google.mlkit.vision.barcode.BarcodeScanning; import com.google.mlkit.vision.common.InputImage; import java.util.List; public class BarcodeScanningActivity extends AppCompatActivity { @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); } private void scanBarcodes(InputImage image) { // [START set_detector_options] BarcodeScannerOptions options = new BarcodeScannerOptions.Builder() .setBarcodeFormats( Barcode.FORMAT_QR_CODE, Barcode.FORMAT_AZTEC) .build(); // [END set_detector_options] // [START get_detector] BarcodeScanner scanner = BarcodeScanning.getClient(); // Or, to specify the formats to recognize: // BarcodeScanner scanner = BarcodeScanning.getClient(options); // [END get_detector] // [START run_detector] Task> result = scanner.process(image) .addOnSuccessListener(new OnSuccessListener>() { @Override public void onSuccess(List barcodes) { // Task completed successfully // [START_EXCLUDE] // [START get_barcodes] for (Barcode barcode: barcodes) { Rect bounds = barcode.getBoundingBox(); Point[] corners = barcode.getCornerPoints(); String rawValue = barcode.getRawValue(); int valueType = barcode.getValueType(); // See API reference for complete list of supported types switch (valueType) { case Barcode.TYPE_WIFI: String ssid = barcode.getWifi().getSsid(); String password = barcode.getWifi().getPassword(); int type = barcode.getWifi().getEncryptionType(); break; case Barcode.TYPE_URL: String title = barcode.getUrl().getTitle(); String url = barcode.getUrl().getUrl(); break; } } // [END get_barcodes] // [END_EXCLUDE] } }) .addOnFailureListener(new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { // Task failed with an exception // ... } }); // [END run_detector] } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/FaceDetectionActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit; import android.graphics.PointF; import android.graphics.Rect; import android.os.Bundle; import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.android.gms.tasks.Task; import com.google.mlkit.vision.common.InputImage; import com.google.mlkit.vision.face.Face; import com.google.mlkit.vision.face.FaceContour; import com.google.mlkit.vision.face.FaceDetection; import com.google.mlkit.vision.face.FaceDetector; import com.google.mlkit.vision.face.FaceDetectorOptions; import com.google.mlkit.vision.face.FaceLandmark; import java.util.List; public class FaceDetectionActivity extends AppCompatActivity { @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); } private void detectFaces(InputImage image) { // [START set_detector_options] FaceDetectorOptions options = new FaceDetectorOptions.Builder() .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE) .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) .setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_ALL) .setMinFaceSize(0.15f) .enableTracking() .build(); // [END set_detector_options] // [START get_detector] FaceDetector detector = FaceDetection.getClient(options); // Or use the default options: // FaceDetector detector = FaceDetection.getClient(); // [END get_detector] // [START run_detector] Task> result = detector.process(image) .addOnSuccessListener( new OnSuccessListener>() { @Override public void onSuccess(List faces) { // Task completed successfully // [START_EXCLUDE] // [START get_face_info] for (Face face : faces) { Rect bounds = face.getBoundingBox(); float rotY = face.getHeadEulerAngleY(); // Head is rotated to the right rotY degrees float rotZ = face.getHeadEulerAngleZ(); // Head is tilted sideways rotZ degrees // If landmark detection was enabled (mouth, ears, eyes, cheeks, and // nose available): FaceLandmark leftEar = face.getLandmark(FaceLandmark.LEFT_EAR); if (leftEar != null) { PointF leftEarPos = leftEar.getPosition(); } // If classification was enabled: if (face.getSmilingProbability() != null) { float smileProb = face.getSmilingProbability(); } if (face.getRightEyeOpenProbability() != null) { float rightEyeOpenProb = face.getRightEyeOpenProbability(); } // If face tracking was enabled: if (face.getTrackingId() != null) { int id = face.getTrackingId(); } } // [END get_face_info] // [END_EXCLUDE] } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { // Task failed with an exception // ... } }); // [END run_detector] } private void faceOptionsExamples() { // [START mlkit_face_options_examples] // High-accuracy landmark detection and face classification FaceDetectorOptions highAccuracyOpts = new FaceDetectorOptions.Builder() .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE) .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) .setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_ALL) .build(); // Real-time contour detection FaceDetectorOptions realTimeOpts = new FaceDetectorOptions.Builder() .setContourMode(FaceDetectorOptions.CONTOUR_MODE_ALL) .build(); // [END mlkit_face_options_examples] } private void processFaceList(List faces) { // [START mlkit_face_list] for (Face face : faces) { Rect bounds = face.getBoundingBox(); float rotY = face.getHeadEulerAngleY(); // Head is rotated to the right rotY degrees float rotZ = face.getHeadEulerAngleZ(); // Head is tilted sideways rotZ degrees // If landmark detection was enabled (mouth, ears, eyes, cheeks, and // nose available): FaceLandmark leftEar = face.getLandmark(FaceLandmark.LEFT_EAR); if (leftEar != null) { PointF leftEarPos = leftEar.getPosition(); } // If contour detection was enabled: List leftEyeContour = face.getContour(FaceContour.LEFT_EYE).getPoints(); List upperLipBottomContour = face.getContour(FaceContour.UPPER_LIP_BOTTOM).getPoints(); // If classification was enabled: if (face.getSmilingProbability() != null) { float smileProb = face.getSmilingProbability(); } if (face.getRightEyeOpenProbability() != null) { float rightEyeOpenProb = face.getRightEyeOpenProbability(); } // If face tracking was enabled: if (face.getTrackingId() != null) { int id = face.getTrackingId(); } } // [END mlkit_face_list] } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/ImageLabelingActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit; import android.os.Bundle; import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.android.gms.tasks.Task; import com.google.mlkit.vision.common.InputImage; import com.google.mlkit.vision.label.ImageLabel; import com.google.mlkit.vision.label.ImageLabeler; import com.google.mlkit.vision.label.ImageLabeling; import com.google.mlkit.vision.label.defaults.ImageLabelerOptions; import java.util.List; public class ImageLabelingActivity extends AppCompatActivity { @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); } private void labelImages(InputImage image) { ImageLabelerOptions options = new ImageLabelerOptions.Builder() .setConfidenceThreshold(0.8f) .build(); // [START get_detector_options] ImageLabeler labeler = ImageLabeling.getClient(options); // [END get_detector_options] /* // [START get_detector_default] // Or use the default options: ImageLabeler detector = ImageLabeling.getClient(ImageLabelerOptions.DEFAULT_OPTIONS); // [END get_detector_default] */ // [START run_detector] Task> result = labeler.process(image) .addOnSuccessListener( new OnSuccessListener>() { @Override public void onSuccess(List labels) { // Task completed successfully // [START_EXCLUDE] // [START get_labels] for (ImageLabel label : labels) { String text = label.getText(); float confidence = label.getConfidence(); } // [END get_labels] // [END_EXCLUDE] } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { // Task failed with an exception // ... } }); // [END run_detector] } private void configureAndRunImageLabeler(InputImage image) { // [START on_device_image_labeler] // To use default options: ImageLabeler labeler = ImageLabeling.getClient(ImageLabelerOptions.DEFAULT_OPTIONS); // Or, to set the minimum confidence required: // ImageLabelerOptions options = // new ImageLabelerOptions.Builder() // .setConfidenceThreshold(0.7f) // .build(); // ImageLabeler labeler = ImageLabeling.getClient(options); // [END on_device_image_labeler] // Process image with custom onSuccess() example // [START process_image] labeler.process(image) .addOnSuccessListener(new OnSuccessListener>() { @Override public void onSuccess(List labels) { // Task completed successfully // ... } }) .addOnFailureListener(new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { // Task failed with an exception // ... } }); // [END process_image] // Process image with example onSuccess() labeler.process(image) .addOnSuccessListener(new OnSuccessListener>() { @Override public void onSuccess(List labels) { // [START get_image_label_info] for (ImageLabel label : labels) { String text = label.getText(); float confidence = label.getConfidence(); int index = label.getIndex(); } // [END get_image_label_info] } }); } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/LanguageIdentificationActivity.java ================================================ package com.google.example.mlkit; import android.os.Bundle; import android.util.Log; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.appcompat.app.AppCompatActivity; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.mlkit.nl.languageid.IdentifiedLanguage; import com.google.mlkit.nl.languageid.LanguageIdentification; import com.google.mlkit.nl.languageid.LanguageIdentificationOptions; import com.google.mlkit.nl.languageid.LanguageIdentifier; import java.util.List; public class LanguageIdentificationActivity extends AppCompatActivity { private static final String TAG = "LangID"; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); } private void identifyLanguageWithStringInput(String text) { // [START identify_languages] LanguageIdentifier languageIdentifier = LanguageIdentification.getClient(); languageIdentifier.identifyLanguage(text) .addOnSuccessListener( new OnSuccessListener() { @Override public void onSuccess(@Nullable String languageCode) { if (languageCode.equals("und")) { Log.i(TAG, "Can't identify language."); } else { Log.i(TAG, "Language: " + languageCode); } } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { // Model couldn’t be loaded or other internal error. // ... } }); // [END identify_languages] } private void setConfidence() { // [START set_confidence] LanguageIdentifier languageIdentifier = LanguageIdentification.getClient( new LanguageIdentificationOptions.Builder() .setConfidenceThreshold(0.34f) .build()); // [END set_confidence] } private void getPossibleLanguuages(String text) { // [START get_possible_languages] LanguageIdentifier languageIdentifier = LanguageIdentification.getClient(); languageIdentifier.identifyPossibleLanguages(text) .addOnSuccessListener(new OnSuccessListener>() { @Override public void onSuccess(List identifiedLanguages) { for (IdentifiedLanguage identifiedLanguage : identifiedLanguages) { String language = identifiedLanguage.getLanguageTag(); float confidence = identifiedLanguage.getConfidence(); Log.i(TAG, language + " (" + confidence + ")"); } } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { // Model couldn’t be loaded or other internal error. // ... } }); // [END get_possible_languages] } private void setConfidenceThreshold() { // [START set_confidence_threshold] LanguageIdentificationOptions identifierOptions = new LanguageIdentificationOptions.Builder() .setConfidenceThreshold(0.5f) .build(); LanguageIdentifier languageIdentifier = LanguageIdentification .getClient(identifierOptions); // [END set_confidence_threshold] } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/MLKitVisionImage.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit; import android.app.Activity; import android.content.Context; import android.graphics.Bitmap; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraManager; import android.media.Image; import android.net.Uri; import android.os.Build; import androidx.annotation.RequiresApi; import android.util.SparseIntArray; import android.view.Surface; import com.google.mlkit.vision.common.InputImage; import java.io.IOException; import java.nio.ByteBuffer; import static android.content.Context.CAMERA_SERVICE; public class MLKitVisionImage { private static final String TAG = "MLKIT"; private static final String MY_CAMERA_ID = "my_camera_id"; private void imageFromBitmap(Bitmap bitmap) { int rotationDegree = 0; // [START image_from_bitmap] InputImage image = InputImage.fromBitmap(bitmap, rotationDegree); // [END image_from_bitmap] } @RequiresApi(api = Build.VERSION_CODES.KITKAT) private void imageFromMediaImage(Image mediaImage, int rotation) { // [START image_from_media_image] InputImage image = InputImage.fromMediaImage(mediaImage, rotation); // [END image_from_media_image] } private void imageFromBuffer(ByteBuffer byteBuffer, int rotationDegrees) { // [START set_metadata] // TODO How do we document the FrameMetadata developers need to implement? // [END set_metadata] // [START image_from_buffer] InputImage image = InputImage.fromByteBuffer(byteBuffer, /* image width */ 480, /* image height */ 360, rotationDegrees, InputImage.IMAGE_FORMAT_NV21 // or IMAGE_FORMAT_YV12 ); // [END image_from_buffer] } private void imageFromArray(byte[] byteArray, int rotation) { // [START image_from_array] InputImage image = InputImage.fromByteArray( byteArray, /* image width */480, /* image height */360, rotation, InputImage.IMAGE_FORMAT_NV21 // or IMAGE_FORMAT_YV12 ); // [END image_from_array] } private void imageFromPath(Context context, Uri uri) { // [START image_from_path] InputImage image; try { image = InputImage.fromFilePath(context, uri); } catch (IOException e) { e.printStackTrace(); } // [END image_from_path] } // [START get_rotation] private static final SparseIntArray ORIENTATIONS = new SparseIntArray(); static { ORIENTATIONS.append(Surface.ROTATION_0, 0); ORIENTATIONS.append(Surface.ROTATION_90, 90); ORIENTATIONS.append(Surface.ROTATION_180, 180); ORIENTATIONS.append(Surface.ROTATION_270, 270); } /** * Get the angle by which an image must be rotated given the device's current * orientation. */ @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) private int getRotationCompensation(String cameraId, Activity activity, boolean isFrontFacing) throws CameraAccessException { // Get the device's current rotation relative to its "native" orientation. // Then, from the ORIENTATIONS table, look up the angle the image must be // rotated to compensate for the device's rotation. int deviceRotation = activity.getWindowManager().getDefaultDisplay().getRotation(); int rotationCompensation = ORIENTATIONS.get(deviceRotation); // Get the device's sensor orientation. CameraManager cameraManager = (CameraManager) activity.getSystemService(CAMERA_SERVICE); int sensorOrientation = cameraManager .getCameraCharacteristics(cameraId) .get(CameraCharacteristics.SENSOR_ORIENTATION); if (isFrontFacing) { rotationCompensation = (sensorOrientation + rotationCompensation) % 360; } else { // back-facing rotationCompensation = (sensorOrientation - rotationCompensation + 360) % 360; } return rotationCompensation; } // [END get_rotation] @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) private void getCompensation(Activity activity, boolean isFrontFacing) throws CameraAccessException { // Get the ID of the camera using CameraManager. Then: int rotation = getRotationCompensation(MY_CAMERA_ID, activity, isFrontFacing); } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/MainActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit; import androidx.appcompat.app.AppCompatActivity; public class MainActivity extends AppCompatActivity { } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/ObjectDetectionActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit; import android.graphics.Bitmap; import android.graphics.Rect; import android.os.Bundle; import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.mlkit.common.model.LocalModel; import com.google.mlkit.vision.common.InputImage; import com.google.mlkit.vision.objects.DetectedObject; import com.google.mlkit.vision.objects.ObjectDetection; import com.google.mlkit.vision.objects.ObjectDetector; import com.google.mlkit.vision.objects.custom.CustomObjectDetectorOptions; import com.google.mlkit.vision.objects.defaults.ObjectDetectorOptions; import com.google.mlkit.vision.objects.defaults.PredefinedCategory; import java.util.ArrayList; import java.util.List; public class ObjectDetectionActivity extends AppCompatActivity { @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); } private void useDefaultObjectDetector() { // [START create_default_options] // Live detection and tracking ObjectDetectorOptions options = new ObjectDetectorOptions.Builder() .setDetectorMode(ObjectDetectorOptions.STREAM_MODE) .enableClassification() // Optional .build(); // Multiple object detection in static images options = new ObjectDetectorOptions.Builder() .setDetectorMode(ObjectDetectorOptions.SINGLE_IMAGE_MODE) .enableMultipleObjects() .enableClassification() // Optional .build(); // [END create_default_options] // [START create_detector] ObjectDetector objectDetector = ObjectDetection.getClient(options); // [END create_detector] InputImage image = InputImage.fromBitmap( Bitmap.createBitmap(new int[100 * 100], 100, 100, Bitmap.Config.ARGB_8888), 0); // [START process_image] objectDetector.process(image) .addOnSuccessListener( new OnSuccessListener>() { @Override public void onSuccess(List detectedObjects) { // Task completed successfully // ... } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { // Task failed with an exception // ... } }); // [END process_image] List results = new ArrayList<>(); // [START read_results_default] // The list of detected objects contains one item if multiple // object detection wasn't enabled. for (DetectedObject detectedObject : results) { Rect boundingBox = detectedObject.getBoundingBox(); Integer trackingId = detectedObject.getTrackingId(); for (DetectedObject.Label label : detectedObject.getLabels()) { String text = label.getText(); if (PredefinedCategory.FOOD.equals(text)) { // ... } int index = label.getIndex(); if (PredefinedCategory.FOOD_INDEX == index) { // ... } float confidence = label.getConfidence(); } } // [END read_results_default] } private void useCustomObjectDetector() { InputImage image = InputImage.fromBitmap( Bitmap.createBitmap(new int[100 * 100], 100, 100, Bitmap.Config.ARGB_8888), 0); // [START create_local_model] LocalModel localModel = new LocalModel.Builder() .setAssetFilePath("asset_file_path_to_tflite_model") // or .setAbsoluteFilePath("absolute_file_path_to_tflite_model") .build(); // [END create_local_model] // [START create_custom_options] // Live detection and tracking CustomObjectDetectorOptions options = new CustomObjectDetectorOptions.Builder(localModel) .setDetectorMode(CustomObjectDetectorOptions.STREAM_MODE) .enableClassification() .setClassificationConfidenceThreshold(0.5f) .setMaxPerObjectLabelCount(3) .build(); // Multiple object detection in static images options = new CustomObjectDetectorOptions.Builder(localModel) .setDetectorMode(CustomObjectDetectorOptions.SINGLE_IMAGE_MODE) .enableMultipleObjects() .enableClassification() .setClassificationConfidenceThreshold(0.5f) .setMaxPerObjectLabelCount(3) .build(); // [END create_custom_options] List results = new ArrayList<>(); // [START read_results_custom] // The list of detected objects contains one item if multiple // object detection wasn't enabled. for (DetectedObject detectedObject : results) { Rect boundingBox = detectedObject.getBoundingBox(); Integer trackingId = detectedObject.getTrackingId(); for (DetectedObject.Label label : detectedObject.getLabels()) { String text = label.getText(); int index = label.getIndex(); float confidence = label.getConfidence(); } } // [END read_results_custom] } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/TextRecognitionActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit; import android.graphics.Point; import android.graphics.Rect; import android.os.Bundle; import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; import com.google.android.gms.tasks.Task; import com.google.mlkit.vision.common.InputImage; import com.google.mlkit.vision.text.Text; import com.google.mlkit.vision.text.TextRecognition; import com.google.mlkit.vision.text.TextRecognizer; import com.google.mlkit.vision.text.latin.TextRecognizerOptions; public class TextRecognitionActivity extends AppCompatActivity { @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); } private void recognizeText(InputImage image) { // [START get_detector_default] TextRecognizer recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS); // [END get_detector_default] // [START run_detector] Task result = recognizer.process(image) .addOnSuccessListener(new OnSuccessListener() { @Override public void onSuccess(Text visionText) { // Task completed successfully // [START_EXCLUDE] // [START get_text] for (Text.TextBlock block : visionText.getTextBlocks()) { Rect boundingBox = block.getBoundingBox(); Point[] cornerPoints = block.getCornerPoints(); String text = block.getText(); for (Text.Line line: block.getLines()) { // ... for (Text.Element element: line.getElements()) { // ... for (Text.Symbol symbol: element.getSymbols()) { // ... } } } } // [END get_text] // [END_EXCLUDE] } }) .addOnFailureListener( new OnFailureListener() { @Override public void onFailure(@NonNull Exception e) { // Task failed with an exception // ... } }); // [END run_detector] } private void processTextBlock(Text result) { // [START mlkit_process_text_block] String resultText = result.getText(); for (Text.TextBlock block : result.getTextBlocks()) { String blockText = block.getText(); Point[] blockCornerPoints = block.getCornerPoints(); Rect blockFrame = block.getBoundingBox(); for (Text.Line line : block.getLines()) { String lineText = line.getText(); Point[] lineCornerPoints = line.getCornerPoints(); Rect lineFrame = line.getBoundingBox(); for (Text.Element element : line.getElements()) { String elementText = element.getText(); Point[] elementCornerPoints = element.getCornerPoints(); Rect elementFrame = element.getBoundingBox(); for (Text.Symbol symbol : element.getSymbols()) { String symbolText = symbol.getText(); Point[] symbolCornerPoints = symbol.getCornerPoints(); Rect symbolFrame = symbol.getBoundingBox(); } } } } // [END mlkit_process_text_block] } private TextRecognizer getTextRecognizer() { // [START mlkit_local_doc_recognizer] TextRecognizer detector = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS); // [END mlkit_local_doc_recognizer] return detector; } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/kotlin/BarcodeScanningActivity.kt ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit.kotlin import androidx.appcompat.app.AppCompatActivity import com.google.mlkit.vision.barcode.BarcodeScannerOptions import com.google.mlkit.vision.barcode.BarcodeScanning import com.google.mlkit.vision.barcode.common.Barcode import com.google.mlkit.vision.common.InputImage class BarcodeScanningActivity : AppCompatActivity() { private fun scanBarcodes(image: InputImage) { // [START set_detector_options] val options = BarcodeScannerOptions.Builder() .setBarcodeFormats( Barcode.FORMAT_QR_CODE, Barcode.FORMAT_AZTEC) .build() // [END set_detector_options] // [START get_detector] val scanner = BarcodeScanning.getClient() // Or, to specify the formats to recognize: // val scanner = BarcodeScanning.getClient(options) // [END get_detector] // [START run_detector] val result = scanner.process(image) .addOnSuccessListener { barcodes -> // Task completed successfully // [START_EXCLUDE] // [START get_barcodes] for (barcode in barcodes) { val bounds = barcode.boundingBox val corners = barcode.cornerPoints val rawValue = barcode.rawValue val valueType = barcode.valueType // See API reference for complete list of supported types when (valueType) { Barcode.TYPE_WIFI -> { val ssid = barcode.wifi!!.ssid val password = barcode.wifi!!.password val type = barcode.wifi!!.encryptionType } Barcode.TYPE_URL -> { val title = barcode.url!!.title val url = barcode.url!!.url } } } // [END get_barcodes] // [END_EXCLUDE] } .addOnFailureListener { // Task failed with an exception // ... } // [END run_detector] } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/kotlin/FaceDetectionActivity.kt ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit.kotlin import androidx.appcompat.app.AppCompatActivity import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.face.* class FaceDetectionActivity : AppCompatActivity() { private fun detectFaces(image: InputImage) { // [START set_detector_options] val options = FaceDetectorOptions.Builder() .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE) .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) .setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_ALL) .setMinFaceSize(0.15f) .enableTracking() .build() // [END set_detector_options] // [START get_detector] val detector = FaceDetection.getClient(options) // Or, to use the default option: // val detector = FaceDetection.getClient(); // [END get_detector] // [START run_detector] val result = detector.process(image) .addOnSuccessListener { faces -> // Task completed successfully // [START_EXCLUDE] // [START get_face_info] for (face in faces) { val bounds = face.boundingBox val rotY = face.headEulerAngleY // Head is rotated to the right rotY degrees val rotZ = face.headEulerAngleZ // Head is tilted sideways rotZ degrees // If landmark detection was enabled (mouth, ears, eyes, cheeks, and // nose available): val leftEar = face.getLandmark(FaceLandmark.LEFT_EAR) leftEar?.let { val leftEarPos = leftEar.position } // If classification was enabled: if (face.smilingProbability != null) { val smileProb = face.smilingProbability } if (face.rightEyeOpenProbability != null) { val rightEyeOpenProb = face.rightEyeOpenProbability } // If face tracking was enabled: if (face.trackingId != null) { val id = face.trackingId } } // [END get_face_info] // [END_EXCLUDE] } .addOnFailureListener { e -> // Task failed with an exception // ... } // [END run_detector] } private fun faceOptionsExamples() { // [START mlkit_face_options_examples] // High-accuracy landmark detection and face classification val highAccuracyOpts = FaceDetectorOptions.Builder() .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE) .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) .setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_ALL) .build() // Real-time contour detection val realTimeOpts = FaceDetectorOptions.Builder() .setContourMode(FaceDetectorOptions.CONTOUR_MODE_ALL) .build() // [END mlkit_face_options_examples] } private fun processFaceList(faces: List) { // [START mlkit_face_list] for (face in faces) { val bounds = face.boundingBox val rotY = face.headEulerAngleY // Head is rotated to the right rotY degrees val rotZ = face.headEulerAngleZ // Head is tilted sideways rotZ degrees // If landmark detection was enabled (mouth, ears, eyes, cheeks, and // nose available): val leftEar = face.getLandmark(FaceLandmark.LEFT_EAR) leftEar?.let { val leftEarPos = leftEar.position } // If contour detection was enabled: val leftEyeContour = face.getContour(FaceContour.LEFT_EYE)?.points val upperLipBottomContour = face.getContour(FaceContour.UPPER_LIP_BOTTOM)?.points // If classification was enabled: if (face.smilingProbability != null) { val smileProb = face.smilingProbability } if (face.rightEyeOpenProbability != null) { val rightEyeOpenProb = face.rightEyeOpenProbability } // If face tracking was enabled: if (face.trackingId != null) { val id = face.trackingId } } // [END mlkit_face_list] } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/kotlin/ImageLabelingActivity.kt ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit.kotlin import androidx.appcompat.app.AppCompatActivity import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.label.ImageLabeler import com.google.mlkit.vision.label.ImageLabeling import com.google.mlkit.vision.label.defaults.ImageLabelerOptions class ImageLabelingActivity : AppCompatActivity() { private fun labelImages(image: InputImage) { val options = ImageLabelerOptions.Builder() .setConfidenceThreshold(0.8f) .build() val labeler = ImageLabeling.getClient(options) // [START run_detector] val result = labeler.process(image) .addOnSuccessListener { labels -> // Task completed successfully // [START_EXCLUDE] // [START get_labels] for (label in labels) { val text = label.text val confidence = label.confidence } // [END get_labels] // [END_EXCLUDE] } .addOnFailureListener { e -> // Task failed with an exception // ... } // [END run_detector] } private fun configureAndRunImageLabeler(image: InputImage) { // [START on_device_image_labeler] // To use default options: val labeler = ImageLabeling.getClient(ImageLabelerOptions.DEFAULT_OPTIONS) // Or, to set the minimum confidence required: // val options = ImageLabelerOptions.Builder() // .setConfidenceThreshold(0.7f) // .build() // val labeler = ImageLabeling.getClient(options) // [END on_device_image_labeler] // Process image with custom onSuccess() example // [START process_image] labeler.process(image) .addOnSuccessListener { labels -> // Task completed successfully // ... } .addOnFailureListener { e -> // Task failed with an exception // ... } // [END process_image] // Process image with example onSuccess() labeler.process(image) .addOnSuccessListener { labels -> // [START get_image_label_info] for (label in labels) { val text = label.text val confidence = label.confidence val index = label.index } // [END get_image_label_info] } } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/kotlin/LanguageIdentificationActivity.kt ================================================ package com.google.example.mlkit.kotlin import android.os.Bundle import android.util.Log import androidx.appcompat.app.AppCompatActivity import com.google.mlkit.nl.languageid.LanguageIdentification import com.google.mlkit.nl.languageid.LanguageIdentificationOptions private const val TAG = "LangIDActivity" class LanguageIdentificationActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) } fun identifyLanguageWithStringInput(text: String) { // [START identify_languages] val languageIdentifier = LanguageIdentification.getClient() languageIdentifier.identifyLanguage(text) .addOnSuccessListener { languageCode -> if (languageCode == "und") { Log.i(TAG, "Can't identify language.") } else { Log.i(TAG, "Language: $languageCode") } } .addOnFailureListener { // Model couldn’t be loaded or other internal error. // ... } // [END identify_languages] } fun setConfidence() { // [START set_confidence] val languageIdentifier = LanguageIdentification .getClient(LanguageIdentificationOptions.Builder() .setConfidenceThreshold(0.34f) .build()) // [END set_confidence] } fun getPossibleLanguuages(text: String) { // [START get_possible_languages] val languageIdentifier = LanguageIdentification.getClient() languageIdentifier.identifyPossibleLanguages(text) .addOnSuccessListener { identifiedLanguages -> for (identifiedLanguage in identifiedLanguages) { val language = identifiedLanguage.languageTag val confidence = identifiedLanguage.confidence Log.i(TAG, "$language $confidence") } } .addOnFailureListener { // Model couldn’t be loaded or other internal error. // ... } // [END get_possible_languages] } private fun setConfidenceThreshold() { // [START set_confidence_threshold] val identifierOptions = LanguageIdentificationOptions.Builder() .setConfidenceThreshold(0.5f) .build() val languageIdentifier = LanguageIdentification .getClient(identifierOptions) // [END set_confidence_threshold] } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/kotlin/MLKitVisionImage.kt ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit.kotlin import android.app.Activity import android.content.Context import android.content.Context.CAMERA_SERVICE import android.graphics.Bitmap import android.hardware.camera2.CameraAccessException import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraManager import android.media.Image import android.net.Uri import android.os.Build import android.util.SparseIntArray import android.view.Surface import androidx.annotation.RequiresApi import com.google.mlkit.vision.common.InputImage import java.io.IOException import java.nio.ByteBuffer class MLKitVisionImage { private fun imageFromBitmap(bitmap: Bitmap) { val rotationDegrees = 0 // [START image_from_bitmap] val image = InputImage.fromBitmap(bitmap, 0) // [END image_from_bitmap] } @RequiresApi(api = Build.VERSION_CODES.KITKAT) private fun imageFromMediaImage(mediaImage: Image, rotation: Int) { // [START image_from_media_image] val image = InputImage.fromMediaImage(mediaImage, rotation) // [END image_from_media_image] } private fun imageFromBuffer(byteBuffer: ByteBuffer, rotationDegrees: Int) { // [START set_metadata] // TODO How do we document the FrameMetadata developers need to implement? // [END set_metadata] // [START image_from_buffer] val image = InputImage.fromByteBuffer( byteBuffer, /* image width */ 480, /* image height */ 360, rotationDegrees, InputImage.IMAGE_FORMAT_NV21 // or IMAGE_FORMAT_YV12 ) // [END image_from_buffer] } private fun imageFromArray(byteArray: ByteArray, rotationDegrees: Int) { // [START image_from_array] val image = InputImage.fromByteArray( byteArray, /* image width */ 480, /* image height */ 360, rotationDegrees, InputImage.IMAGE_FORMAT_NV21 // or IMAGE_FORMAT_YV12 ) // [END image_from_array] } private fun imageFromPath(context: Context, uri: Uri) { // [START image_from_path] val image: InputImage try { image = InputImage.fromFilePath(context, uri) } catch (e: IOException) { e.printStackTrace() } // [END image_from_path] } // [START get_rotation] /** * Get the angle by which an image must be rotated given the device's current * orientation. */ @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) @Throws(CameraAccessException::class) private fun getRotationCompensation(cameraId: String, activity: Activity, isFrontFacing: Boolean): Int { // Get the device's current rotation relative to its "native" orientation. // Then, from the ORIENTATIONS table, look up the angle the image must be // rotated to compensate for the device's rotation. val deviceRotation = activity.windowManager.defaultDisplay.rotation var rotationCompensation = ORIENTATIONS.get(deviceRotation) // Get the device's sensor orientation. val cameraManager = activity.getSystemService(CAMERA_SERVICE) as CameraManager val sensorOrientation = cameraManager .getCameraCharacteristics(cameraId) .get(CameraCharacteristics.SENSOR_ORIENTATION)!! if (isFrontFacing) { rotationCompensation = (sensorOrientation + rotationCompensation) % 360 } else { // back-facing rotationCompensation = (sensorOrientation - rotationCompensation + 360) % 360 } return rotationCompensation } // [END get_rotation] @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) @Throws(CameraAccessException::class) private fun getCompensation(activity: Activity, context: Context, isFrontFacing: Boolean) { // Get the ID of the camera using CameraManager. Then: val rotation = getRotationCompensation(MY_CAMERA_ID, activity, isFrontFacing) } companion object { private val TAG = "MLKIT" private val MY_CAMERA_ID = "my_camera_id" // [START camera_orientations] private val ORIENTATIONS = SparseIntArray() init { ORIENTATIONS.append(Surface.ROTATION_0, 0) ORIENTATIONS.append(Surface.ROTATION_90, 90) ORIENTATIONS.append(Surface.ROTATION_180, 180) ORIENTATIONS.append(Surface.ROTATION_270, 270) } // [END camera_orientations] } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/kotlin/MainActivity.kt ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit.kotlin import androidx.appcompat.app.AppCompatActivity class MainActivity : AppCompatActivity() ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/kotlin/ObjectDetectionActivity.kt ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit.kotlin import android.graphics.Bitmap import androidx.appcompat.app.AppCompatActivity import com.google.mlkit.common.model.LocalModel import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.objects.DetectedObject import com.google.mlkit.vision.objects.ObjectDetection import com.google.mlkit.vision.objects.custom.CustomObjectDetectorOptions import com.google.mlkit.vision.objects.defaults.ObjectDetectorOptions import com.google.mlkit.vision.objects.defaults.PredefinedCategory class ObjectDetectionActivity : AppCompatActivity() { private fun useDefaultObjectDetector() { // [START create_default_options] // Live detection and tracking var options = ObjectDetectorOptions.Builder() .setDetectorMode(ObjectDetectorOptions.STREAM_MODE) .enableClassification() // Optional .build() // Multiple object detection in static images options = ObjectDetectorOptions.Builder() .setDetectorMode(ObjectDetectorOptions.SINGLE_IMAGE_MODE) .enableMultipleObjects() .enableClassification() // Optional .build() // [END create_default_options] // [START create_detector] val objectDetector = ObjectDetection.getClient(options) // [END create_detector] val image = InputImage.fromBitmap( Bitmap.createBitmap(IntArray(100 * 100), 100, 100, Bitmap.Config.ARGB_8888), 0) // [START process_image] objectDetector.process(image) .addOnSuccessListener { results -> // Task completed successfully // ... } .addOnFailureListener { e -> // Task failed with an exception // ... } // [END process_image] val results = listOf() // [START read_results_default] for (detectedObject in results) { val boundingBox = detectedObject.boundingBox val trackingId = detectedObject.trackingId for (label in detectedObject.labels) { val text = label.text if (PredefinedCategory.FOOD == text) { // ... } val index = label.index if (PredefinedCategory.FOOD_INDEX == index) { // ... } val confidence = label.confidence } } // [END read_results_default] } private fun useCustomObjectDetector() { val image = InputImage.fromBitmap( Bitmap.createBitmap(IntArray(100 * 100), 100, 100, Bitmap.Config.ARGB_8888), 0) // [START create_local_model] val localModel = LocalModel.Builder() .setAssetFilePath("asset_file_path_to_tflite_model") // or .setAbsoluteFilePath("absolute_file_path_to_tflite_model") .build() // [END create_local_model] // [START create_custom_options] // Live detection and tracking var options = CustomObjectDetectorOptions.Builder(localModel) .setDetectorMode(CustomObjectDetectorOptions.STREAM_MODE) .enableClassification() .setClassificationConfidenceThreshold(0.5f) .setMaxPerObjectLabelCount(3) .build() // Multiple object detection in static images options = CustomObjectDetectorOptions.Builder(localModel) .setDetectorMode(CustomObjectDetectorOptions.SINGLE_IMAGE_MODE) .enableMultipleObjects() .enableClassification() .setClassificationConfidenceThreshold(0.5f) .setMaxPerObjectLabelCount(3) .build() // [END create_custom_options] val results = listOf() // [START read_results_custom] for (detectedObject in results) { val boundingBox = detectedObject.boundingBox val trackingId = detectedObject.trackingId for (label in detectedObject.labels) { val text = label.text val index = label.index val confidence = label.confidence } } // [END read_results_custom] } } ================================================ FILE: android/android-snippets/app/src/main/java/com/google/example/mlkit/kotlin/TextRecognitionActivity.kt ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.example.mlkit.kotlin import androidx.appcompat.app.AppCompatActivity import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.text.Text import com.google.mlkit.vision.text.TextRecognition import com.google.mlkit.vision.text.TextRecognizer import com.google.mlkit.vision.text.latin.TextRecognizerOptions class TextRecognitionActivity : AppCompatActivity() { private fun recognizeText(image: InputImage) { // [START get_detector_default] val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) // [END get_detector_default] // [START run_detector] val result = recognizer.process(image) .addOnSuccessListener { visionText -> // Task completed successfully // [START_EXCLUDE] // [START get_text] for (block in visionText.textBlocks) { val boundingBox = block.boundingBox val cornerPoints = block.cornerPoints val text = block.text for (line in block.lines) { // ... for (element in line.elements) { // ... } } } // [END get_text] // [END_EXCLUDE] } .addOnFailureListener { e -> // Task failed with an exception // ... } // [END run_detector] } private fun processTextBlock(result: Text) { // [START mlkit_process_text_block] val resultText = result.text for (block in result.textBlocks) { val blockText = block.text val blockCornerPoints = block.cornerPoints val blockFrame = block.boundingBox for (line in block.lines) { val lineText = line.text val lineCornerPoints = line.cornerPoints val lineFrame = line.boundingBox for (element in line.elements) { val elementText = element.text val elementCornerPoints = element.cornerPoints val elementFrame = element.boundingBox } } } // [END mlkit_process_text_block] } private fun getTextRecognizer(): TextRecognizer { // [START mlkit_local_doc_recognizer] return TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) // [END mlkit_local_doc_recognizer] } } ================================================ FILE: android/android-snippets/app/src/main/res/layout/activity_main.xml ================================================ ================================================ FILE: android/android-snippets/app/src/main/res/values/colors.xml ================================================ #3F51B5 #303F9F #FF4081 ================================================ FILE: android/android-snippets/app/src/main/res/values/dimens.xml ================================================ 16dp 16dp ================================================ FILE: android/android-snippets/app/src/main/res/values/strings.xml ================================================ MLKit Code Snippets ================================================ FILE: android/android-snippets/app/src/main/res/values/styles.xml ================================================ ================================================ FILE: android/android-snippets/app/src/main/res/values-w820dp/dimens.xml ================================================ 64dp ================================================ FILE: android/android-snippets/build.gradle ================================================ // Top-level build file where you can add configuration options common to all sub-projects/modules. buildscript { ext.kotlin_version = '1.4.31' repositories { mavenLocal() google() jcenter() } dependencies { classpath 'com.android.tools.build:gradle:7.2.1' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" } } allprojects { repositories { google() mavenLocal() jcenter() } } task clean(type: Delete) { delete rootProject.buildDir } ================================================ FILE: android/android-snippets/gradle/wrapper/gradle-wrapper.properties ================================================ #Mon Aug 24 22:29:37 PDT 2020 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-all.zip ================================================ FILE: android/android-snippets/gradle.properties ================================================ # Project-wide Gradle settings. # IDE (e.g. Android Studio) users: # Gradle settings configured through the IDE *will override* # any settings specified in this file. # For more details on how to configure your build environment visit # http://www.gradle.org/docs/current/userguide/build_environment.html # Specifies the JVM arguments used for the daemon process. # The setting is particularly useful for tweaking memory settings. android.enableJetifier=true android.useAndroidX=true org.gradle.jvmargs=-Xmx1536m # When configured, Gradle will run in incubating parallel mode. # This option should only be used with decoupled projects. More details, visit # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects # org.gradle.parallel=true ================================================ FILE: android/android-snippets/gradlew ================================================ #!/usr/bin/env sh # # Copyright 2015 the original author or authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ############################################################################## ## ## Gradle start up script for UN*X ## ############################################################################## # Attempt to set APP_HOME # Resolve links: $0 may be a link PRG="$0" # Need this for relative symlinks. while [ -h "$PRG" ] ; do ls=`ls -ld "$PRG"` link=`expr "$ls" : '.*-> \(.*\)$'` if expr "$link" : '/.*' > /dev/null; then PRG="$link" else PRG=`dirname "$PRG"`"/$link" fi done SAVED="`pwd`" cd "`dirname \"$PRG\"`/" >/dev/null APP_HOME="`pwd -P`" cd "$SAVED" >/dev/null APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" warn () { echo "$*" } die () { echo echo "$*" echo exit 1 } # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false case "`uname`" in CYGWIN* ) cygwin=true ;; Darwin* ) darwin=true ;; MINGW* ) msys=true ;; NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD="$JAVA_HOME/jre/sh/java" else JAVACMD="$JAVA_HOME/bin/java" fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else JAVACMD="java" which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi # Increase the maximum file descriptors if we can. if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then MAX_FD_LIMIT=`ulimit -H -n` if [ $? -eq 0 ] ; then if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then MAX_FD="$MAX_FD_LIMIT" fi ulimit -n $MAX_FD if [ $? -ne 0 ] ; then warn "Could not set maximum file descriptor limit: $MAX_FD" fi else warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" fi fi # For Darwin, add options to specify how the application appears in the dock if $darwin; then GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" fi # For Cygwin, switch paths to Windows format before running java if $cygwin ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` SEP="" for dir in $ROOTDIRSRAW ; do ROOTDIRS="$ROOTDIRS$SEP$dir" SEP="|" done OURCYGPATTERN="(^($ROOTDIRS))" # Add a user-defined pattern to the cygpath arguments if [ "$GRADLE_CYGPATTERN" != "" ] ; then OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" fi # Now convert the arguments - kludge to limit ourselves to /bin/sh i=0 for arg in "$@" ; do CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` else eval `echo args$i`="\"$arg\"" fi i=$((i+1)) done case $i in (0) set -- ;; (1) set -- "$args0" ;; (2) set -- "$args0" "$args1" ;; (3) set -- "$args0" "$args1" "$args2" ;; (4) set -- "$args0" "$args1" "$args2" "$args3" ;; (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; esac fi # Escape application args save () { for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done echo " " } APP_ARGS=$(save "$@") # Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then cd "$(dirname "$0")" fi exec "$JAVACMD" "$@" ================================================ FILE: android/android-snippets/gradlew.bat ================================================ @rem @rem Copyright 2015 the original author or authors. @rem @rem Licensed under the Apache License, Version 2.0 (the "License"); @rem you may not use this file except in compliance with the License. @rem You may obtain a copy of the License at @rem @rem http://www.apache.org/licenses/LICENSE-2.0 @rem @rem Unless required by applicable law or agreed to in writing, software @rem distributed under the License is distributed on an "AS IS" BASIS, @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem @if "%DEBUG%" == "" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @rem @rem ########################################################################## @rem Set local scope for the variables with windows NT shell if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 if "%DIRNAME%" == "" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if "%ERRORLEVEL%" == "0" goto init echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. echo. echo Please set the JAVA_HOME variable in your environment to match the echo location of your Java installation. goto fail :findJavaFromJavaHome set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto init echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% echo. echo Please set the JAVA_HOME variable in your environment to match the echo location of your Java installation. goto fail :init @rem Get command-line arguments, handling Windows variants if not "%OS%" == "Windows_NT" goto win9xME_args :win9xME_args @rem Slurp the command line arguments. set CMD_LINE_ARGS= set _SKIP=2 :win9xME_args_slurp if "x%~1" == "x" goto execute set CMD_LINE_ARGS=%* :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% :end @rem End local scope for the variables with windows NT shell if "%ERRORLEVEL%"=="0" goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 exit /b 1 :mainEnd if "%OS%"=="Windows_NT" endlocal :omega ================================================ FILE: android/android-snippets/settings.gradle ================================================ include ':app' ================================================ FILE: android/automl/LICENSE ================================================ Copyright 2020 Google LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS ================================================ FILE: android/automl/README.md ================================================ # ML Kit AutoML Remote Model Quickstart Sample App * [Read more about training an image classification model using AutoML Vision Edge model](https://firebase.google.com/docs/ml/automl-image-labeling) * [Read more about hosting models with Firebase Model deployment](https://firebase.google.com/docs/ml/manage-hosted-models) ## Introduction This ML Kit Quickstart App demonstrates how to label images and objects with an AutoML Vision Edge trained model, and also how to download models that are hosted within Firebase. ## Feature List Features that are included in this demo app: * [Custom Image labeling](https://developers.google.com/ml-kit/vision/image-labeling/custom-models/android) - Label images using a custom AutoML VisionEdge trained model that is downloaded from Firebase. * [Custom Object Detection and Tracking](https://developers.google.com/ml-kit/vision/object-detection/custom-models/android) - Detect and label object using a custom AutoML VisionEdge trained model that is downloaded from Firebase. ## Getting Started * In order to setup this app to run, you need to create a Firebase project as outlined [here](https://firebase.google.com/docs/android/setup). * Run the sample code on your Android device or emulator * Try extending the code to add new features and functionality * Ensure your device is connected to the Internet to download the remote model ## How to use the app This app supports three usage scenarios: Live Camera, Static Image, and CameraX enabled live camera. ### Live Camera Scenario This mode uses the camera preview as input and allows you to label images using a remotely downloaded AutoML Vision Edge model. There is also a settings page that allows you to configure several options: * Camera * Preview Size -- Specify the preview size of rear camera manually (Default size is chose appropriately based on screen size) * Enable live viewport -- Prevent the live camera preview from being blocked by API rendering speed. * AutoML Image Labeling * AutoML Remote Model Name -- Allows you to specify an AutoML VisionEdge model to remotely download from the Firebase Console ### Static Image Scenario The static image scenario is identical to the live camera scenario, but instead relies on images fed into the app through the gallery. ### CameraX Live Preview scenario The CameraX live preview scenario is very similar to the native live camera scenario, but instead relies on CameraX live preview instead of the Camera2 live preview. Note: CameraX is only supported on API level 21+. ## Support * [Stack Overflow](https://stackoverflow.com/questions/tagged/google-mlkit) ## License Copyright 2020 Google, Inc. Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: android/automl/app/build.gradle ================================================ apply plugin: 'com.android.application' android { compileSdkVersion 34 defaultConfig { applicationId "com.google.mlkit.vision.automl.demo" minSdkVersion 21 multiDexEnabled true targetSdkVersion 34 // Version code for next release (version 9 is now in Play Store) versionCode 11 versionName "1.11" vectorDrawables.useSupportLibrary = true setProperty("archivesBaseName", "vision-automl-remote-model-quickstart") } buildTypes { proguard { debuggable false minifyEnabled true shrinkResources true proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro', 'proguard.cfg' testProguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguardTest-rules.pro', 'proguard.cfg' } testBuildType "proguard" release { minifyEnabled true proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' } debug { minifyEnabled false proguardFiles 'proguard-rules.pro' } } compileOptions { sourceCompatibility JavaVersion.VERSION_1_8 targetCompatibility JavaVersion.VERSION_1_8 } // Do NOT compress tflite model files (need to call out to developers!) aaptOptions { noCompress "tflite" } dexOptions { incremental true javaMaxHeapSize "4g" } } repositories { // Depending on AndroidX Snapshot Builds to get the latest CameraX libs. maven { url 'https://androidx.dev/snapshots/builds/6787662/artifacts/repository/' } } dependencies { // Image labeling with AutoML model support implementation 'com.google.mlkit:image-labeling-custom:17.0.3' // Or comment the dependency above and uncomment the dependency below to // use unbundled model that depends on Google Play Services // implementation 'com.google.android.gms:play-services-mlkit-image-labeling-custom:16.0.0-beta5' implementation 'com.google.mlkit:object-detection-custom:17.0.2' implementation 'com.google.mlkit:linkfirebase:17.0.0' implementation 'androidx.multidex:multidex:2.0.1' // ------------------------------------------------------- implementation 'com.google.code.gson:gson:2.8.6' implementation 'com.google.guava:guava:27.1-android' // ViewModel and LiveData implementation "androidx.lifecycle:lifecycle-livedata:2.3.1" implementation "androidx.lifecycle:lifecycle-viewmodel:2.3.1" implementation 'androidx.appcompat:appcompat:1.2.0' implementation 'androidx.annotation:annotation:1.2.0' implementation 'androidx.constraintlayout:constraintlayout:2.0.4' // CameraX implementation "androidx.camera:camera-camera2:1.0.0-SNAPSHOT" implementation "androidx.camera:camera-lifecycle:1.0.0-SNAPSHOT" implementation "androidx.camera:camera-view:1.0.0-SNAPSHOT" } configurations { // Resolves dependency conflict caused by some dependencies use // com.google.guava:guava and com.google.guava:listenablefuture together. all*.exclude group: 'com.google.guava', module: 'listenablefuture' } ================================================ FILE: android/automl/app/proguard-rules.pro ================================================ # Add project specific ProGuard rules here. # You can control the set of applied configuration files using the # proguardFiles setting in build.gradle. # # For more details, see # http://developer.android.com/guide/developing/tools/proguard.html # If your project uses WebView with JS, uncomment the following # and specify the fully qualified class name to the JavaScript interface # class: #-keepclassmembers class fqcn.of.javascript.interface.for.webview { # public *; #} # Uncomment this to preserve the line number information for # debugging stack traces. #-keepattributes SourceFile,LineNumberTable # If you keep the line number information, uncomment this to # hide the original source file name. #-renamesourcefileattribute SourceFile ================================================ FILE: android/automl/app/src/main/AndroidManifest.xml ================================================ ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/BitmapUtils.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import static java.lang.Math.min; import android.content.ContentResolver; import android.content.Context; import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.graphics.ImageFormat; import android.graphics.Matrix; import android.graphics.Rect; import android.graphics.YuvImage; import android.media.Image; import android.media.Image.Plane; import android.net.Uri; import android.os.Build.VERSION_CODES; import android.provider.MediaStore; import android.util.Log; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.camera.core.ExperimentalGetImage; import androidx.camera.core.ImageProxy; import androidx.exifinterface.media.ExifInterface; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; /** Utils functions for bitmap conversions. */ public final class BitmapUtils { private static final String TAG = "BitmapUtils"; /** Converts NV21 format byte buffer to bitmap. */ @Nullable public static Bitmap getBitmap(ByteBuffer data, FrameMetadata metadata) { data.rewind(); byte[] imageInBuffer = new byte[data.limit()]; data.get(imageInBuffer, 0, imageInBuffer.length); try { YuvImage image = new YuvImage( imageInBuffer, ImageFormat.NV21, metadata.getWidth(), metadata.getHeight(), null); ByteArrayOutputStream stream = new ByteArrayOutputStream(); image.compressToJpeg(new Rect(0, 0, metadata.getWidth(), metadata.getHeight()), 80, stream); Bitmap bmp = BitmapFactory.decodeByteArray(stream.toByteArray(), 0, stream.size()); stream.close(); return rotateBitmap(bmp, metadata.getRotation(), false, false); } catch (Exception e) { Log.e("VisionProcessorBase", "Error: " + e.getMessage()); } return null; } /** Converts a YUV_420_888 image from CameraX API to a bitmap. */ @RequiresApi(VERSION_CODES.LOLLIPOP) @Nullable @ExperimentalGetImage public static Bitmap getBitmap(ImageProxy image) { FrameMetadata frameMetadata = new FrameMetadata.Builder() .setWidth(image.getWidth()) .setHeight(image.getHeight()) .setRotation(image.getImageInfo().getRotationDegrees()) .build(); ByteBuffer nv21Buffer = yuv420ThreePlanesToNV21(image.getImage().getPlanes(), image.getWidth(), image.getHeight()); return getBitmap(nv21Buffer, frameMetadata); } /** Rotates a bitmap if it is converted from a bytebuffer. */ private static Bitmap rotateBitmap( Bitmap bitmap, int rotationDegrees, boolean flipX, boolean flipY) { Matrix matrix = new Matrix(); // Rotate the image back to straight. matrix.postRotate(rotationDegrees); // Mirror the image along the X or Y axis. matrix.postScale(flipX ? -1.0f : 1.0f, flipY ? -1.0f : 1.0f); Bitmap rotatedBitmap = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true); // Recycle the old bitmap if it has changed. if (rotatedBitmap != bitmap) { bitmap.recycle(); } return rotatedBitmap; } @Nullable public static Bitmap getBitmapFromAsset(Context context, String fileName) { InputStream inputStream = null; try { inputStream = context.getAssets().open(fileName); return BitmapFactory.decodeStream(inputStream); } catch (IOException e) { Log.e(TAG, "Error reading asset: " + fileName, e); } finally { if (inputStream != null) { try { inputStream.close(); } catch (IOException e) { Log.e(TAG, "Failed to close input stream: ", e); } } } return null; } @Nullable public static Bitmap getBitmapFromContentUri(ContentResolver contentResolver, Uri imageUri) throws IOException { Bitmap decodedBitmap = MediaStore.Images.Media.getBitmap(contentResolver, imageUri); if (decodedBitmap == null) { return null; } int orientation = getExifOrientationTag(contentResolver, imageUri); int rotationDegrees = 0; boolean flipX = false; boolean flipY = false; // See e.g. https://magnushoff.com/articles/jpeg-orientation/ for a detailed explanation on each // orientation. switch (orientation) { case ExifInterface.ORIENTATION_FLIP_HORIZONTAL: flipX = true; break; case ExifInterface.ORIENTATION_ROTATE_90: rotationDegrees = 90; break; case ExifInterface.ORIENTATION_TRANSPOSE: rotationDegrees = 90; flipX = true; break; case ExifInterface.ORIENTATION_ROTATE_180: rotationDegrees = 180; break; case ExifInterface.ORIENTATION_FLIP_VERTICAL: flipY = true; break; case ExifInterface.ORIENTATION_ROTATE_270: rotationDegrees = -90; break; case ExifInterface.ORIENTATION_TRANSVERSE: rotationDegrees = -90; flipX = true; break; case ExifInterface.ORIENTATION_UNDEFINED: case ExifInterface.ORIENTATION_NORMAL: default: // No transformations necessary in this case. } return rotateBitmap(decodedBitmap, rotationDegrees, flipX, flipY); } private static int getExifOrientationTag(ContentResolver resolver, Uri imageUri) { // We only support parsing EXIF orientation tag from local file on the device. // See also: // https://android-developers.googleblog.com/2016/12/introducing-the-exifinterface-support-library.html if (!ContentResolver.SCHEME_CONTENT.equals(imageUri.getScheme()) && !ContentResolver.SCHEME_FILE.equals(imageUri.getScheme())) { return 0; } ExifInterface exif; try (InputStream inputStream = resolver.openInputStream(imageUri)) { if (inputStream == null) { return 0; } exif = new ExifInterface(inputStream); } catch (IOException e) { Log.e(TAG, "failed to open file to read rotation meta data: " + imageUri, e); return 0; } return exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL); } public static ByteBuffer convertBitmapToNv21Buffer(Bitmap bitmap) { return ByteBuffer.wrap(convertBitmapToNv21Bytes(bitmap)); } public static byte[] convertBitmapToNv21Bytes(Bitmap bitmap) { int inputWidth = bitmap.getWidth(); int inputHeight = bitmap.getHeight(); int[] argb = new int[inputWidth * inputHeight]; bitmap.getPixels(argb, 0, inputWidth, 0, 0, inputWidth, inputHeight); byte[] nv21Bytes = new byte [inputHeight * inputWidth + 2 * (int) Math.ceil(inputHeight / 2.0) * (int) Math.ceil(inputWidth / 2.0)]; encodeToNv21(nv21Bytes, argb, inputWidth, inputHeight); return nv21Bytes; } private static void encodeToNv21(byte[] nv21Bytes, int[] argb, int width, int height) { int frameSize = width * height; int yIndex = 0; int uvIndex = frameSize; int red; int green; int blue; int y; int u; int v; int index = 0; for (int j = 0; j < height; j++) { for (int i = 0; i < width; i++) { // first byte is alpha, but is unused red = (argb[index] & 0xff0000) >> 16; green = (argb[index] & 0xff00) >> 8; blue = (argb[index] & 0xff) >> 0; // well known RGB to YUV algorithm y = ((66 * red + 129 * green + 25 * blue + 128) >> 8) + 16; u = ((-38 * red - 74 * green + 112 * blue + 128) >> 8) + 128; v = ((112 * red - 94 * green - 18 * blue + 128) >> 8) + 128; // NV21 has a plane of Y and interleaved planes of VU each sampled by a factor of 2 // meaning for every 4 Y pixels there are 1 V and 1 U. Note the sampling is every other // pixel AND every other scanline. nv21Bytes[yIndex++] = (byte) ((y < 0) ? 0 : min(255, y)); if (j % 2 == 0 && index % 2 == 0) { nv21Bytes[uvIndex++] = (byte) ((v < 0) ? 0 : min(255, v)); nv21Bytes[uvIndex++] = (byte) ((u < 0) ? 0 : min(255, u)); } index++; } } } public static ByteBuffer convertBitmapToYv12Buffer(Bitmap bitmap) { return ByteBuffer.wrap(convertBitmapToYv12Bytes(bitmap)); } public static byte[] convertBitmapToYv12Bytes(Bitmap bitmap) { byte[] nv21Bytes = convertBitmapToNv21Bytes(bitmap); return nv21Toyv12(nv21Bytes); } /** * Converts nv21 byte[] to yv12 byte[]. * *

NV21 (4:2:0) Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y V U V U V U V U V U V U * *

YV12 (4:2:0) Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y V V V V V V U U U U U U */ private static byte[] nv21Toyv12(byte[] nv21Bytes) { int totalBytes = nv21Bytes.length; int rowSize = totalBytes / 6; // 4+2+0 byte[] yv12Bytes = new byte[totalBytes]; System.arraycopy(nv21Bytes, 0, yv12Bytes, 0, rowSize * 4); int offSet = totalBytes / 6 * 4; for (int i = 0; i < rowSize; i++) { yv12Bytes[offSet + i] = nv21Bytes[offSet + 2 * i]; // V yv12Bytes[offSet + rowSize + i] = nv21Bytes[offSet + 2 * i + 1]; // U } return yv12Bytes; } /** * Converts YUV_420_888 to NV21 bytebuffer. * *

The NV21 format consists of a single byte array containing the Y, U and V values. For an * image of size S, the first S positions of the array contain all the Y values. The remaining * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU * *

YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into * the first part of the NV21 array. The U and V planes may already have the representation in the * NV21 format. This happens if the planes share the same buffer, the V buffer is one position * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy * them to the NV21 array. */ private static ByteBuffer yuv420ThreePlanesToNV21( Plane[] yuv420888planes, int width, int height) { int imageSize = width * height; byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; if (areUVPlanesNV21(yuv420888planes, width, height)) { // Copy the Y values. yuv420888planes[0].getBuffer().get(out, 0, imageSize); ByteBuffer uBuffer = yuv420888planes[1].getBuffer(); ByteBuffer vBuffer = yuv420888planes[2].getBuffer(); // Get the first V value from the V buffer, since the U buffer does not contain it. vBuffer.get(out, imageSize, 1); // Copy the first U value and the remaining VU values from the U buffer. uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); } else { // Fallback to copying the UV values one by one, which is slower but also works. // Unpack Y. unpackPlane(yuv420888planes[0], width, height, out, 0, 1); // Unpack U. unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2); // Unpack V. unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2); } return ByteBuffer.wrap(out); } /** Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. */ private static boolean areUVPlanesNV21(Plane[] planes, int width, int height) { int imageSize = width * height; ByteBuffer uBuffer = planes[1].getBuffer(); ByteBuffer vBuffer = planes[2].getBuffer(); // Backup buffer properties. int vBufferPosition = vBuffer.position(); int uBufferLimit = uBuffer.limit(); // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value. vBuffer.position(vBufferPosition + 1); // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value. uBuffer.limit(uBufferLimit - 1); // Check that the buffers are equal and have the expected number of elements. boolean areNV21 = (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); // Restore buffers to their initial state. vBuffer.position(vBufferPosition); uBuffer.limit(uBufferLimit); return areNV21; } /** * Unpack an image plane into a byte array. * *

The input plane data will be copied in 'out', starting at 'offset' and every pixel will be * spaced by 'pixelStride'. Note that there is no row padding on the output. */ private static void unpackPlane( Plane plane, int width, int height, byte[] out, int offset, int pixelStride) { ByteBuffer buffer = plane.getBuffer(); buffer.rewind(); // Compute the size of the current plane. // We assume that it has the aspect ratio as the original image. int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride(); if (numRow == 0) { return; } int scaleFactor = height / numRow; int numCol = width / scaleFactor; // Extract the data in the output buffer. int outputPos = offset; int rowStart = 0; for (int row = 0; row < numRow; row++) { int inputPos = rowStart; for (int col = 0; col < numCol; col++) { out[outputPos] = buffer.get(inputPos); outputPos += pixelStride; inputPos += plane.getPixelStride(); } rowStart += plane.getRowStride(); } } private BitmapUtils() {} } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/CameraImageGraphic.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.graphics.Bitmap; import android.graphics.Canvas; import com.google.mlkit.vision.automl.demo.GraphicOverlay.Graphic; /** Draw camera image to background. */ public class CameraImageGraphic extends Graphic { private final Bitmap bitmap; public CameraImageGraphic(GraphicOverlay overlay, Bitmap bitmap) { super(overlay); this.bitmap = bitmap; } @Override public void draw(Canvas canvas) { canvas.drawBitmap(bitmap, getTransformationMatrix(), null); } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/CameraSource.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.Manifest; import android.annotation.SuppressLint; import android.app.Activity; import android.content.Context; import android.graphics.ImageFormat; import android.graphics.SurfaceTexture; import android.hardware.Camera; import android.hardware.Camera.CameraInfo; import android.util.Log; import android.view.Surface; import android.view.SurfaceHolder; import android.view.WindowManager; import androidx.annotation.Nullable; import androidx.annotation.RequiresPermission; import com.google.android.gms.common.images.Size; import com.google.mlkit.vision.automl.demo.preference.PreferenceUtils; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.List; /** * Manages the camera and allows UI updates on top of it (e.g. overlaying extra Graphics or * displaying extra information). This receives preview frames from the camera at a specified rate, * sending those frames to child classes' detectors / classifiers as fast as it is able to process. */ public class CameraSource { @SuppressLint("InlinedApi") public static final int CAMERA_FACING_BACK = CameraInfo.CAMERA_FACING_BACK; @SuppressLint("InlinedApi") public static final int CAMERA_FACING_FRONT = CameraInfo.CAMERA_FACING_FRONT; public static final int IMAGE_FORMAT = ImageFormat.NV21; public static final int DEFAULT_REQUESTED_CAMERA_PREVIEW_WIDTH = 480; public static final int DEFAULT_REQUESTED_CAMERA_PREVIEW_HEIGHT = 360; private static final String TAG = "MIDemoApp:CameraSource"; /** * The dummy surface texture must be assigned a chosen name. Since we never use an OpenGL context, * we can choose any ID we want here. The dummy surface texture is not a crazy hack - it is * actually how the camera team recommends using the camera without a preview. */ private static final int DUMMY_TEXTURE_NAME = 100; /** * If the absolute difference between a preview size aspect ratio and a picture size aspect ratio * is less than this tolerance, they are considered to be the same aspect ratio. */ private static final float ASPECT_RATIO_TOLERANCE = 0.01f; private static final float REQUESTED_FPS = 30.0f; private static final boolean REQUESTED_AUTO_FOCUS = true; protected Activity activity; private Camera camera; private int facing = CAMERA_FACING_BACK; /** Rotation of the device, and thus the associated preview images captured from the device. */ private int rotationDegrees; private Size previewSize; // These instances need to be held onto to avoid GC of their underlying resources. Even though // these aren't used outside of the method that creates them, they still must have hard // references maintained to them. private SurfaceTexture dummySurfaceTexture; private final GraphicOverlay graphicOverlay; // True if a SurfaceTexture is being used for the preview, false if a SurfaceHolder is being // used for the preview. We want to be compatible back to Gingerbread, but SurfaceTexture // wasn't introduced until Honeycomb. Since the interface cannot use a SurfaceTexture, if the // developer wants to display a preview we must use a SurfaceHolder. If the developer doesn't // want to display a preview we use a SurfaceTexture if we are running at least Honeycomb. private boolean usingSurfaceTexture; /** * Dedicated thread and associated runnable for calling into the detector with frames, as the * frames become available from the camera. */ private Thread processingThread; private final FrameProcessingRunnable processingRunnable; private final Object processorLock = new Object(); // TODO(b/74400062) Re-enable the annotaion // @GuardedBy("processorLock") private VisionImageProcessor frameProcessor; /** * Map to convert between a byte array, received from the camera, and its associated byte buffer. * We use byte buffers internally because this is a more efficient way to call into native code * later (avoids a potential copy). * *

Note: uses IdentityHashMap here instead of HashMap because the behavior of an array's * equals, hashCode and toString methods is both useless and unexpected. IdentityHashMap enforces * identity ('==') check on the keys. */ private final IdentityHashMap bytesToByteBuffer = new IdentityHashMap<>(); public CameraSource(Activity activity, GraphicOverlay overlay) { this.activity = activity; graphicOverlay = overlay; graphicOverlay.clear(); processingRunnable = new FrameProcessingRunnable(); } // ============================================================================================== // Public // ============================================================================================== /** Stops the camera and releases the resources of the camera and underlying detector. */ public void release() { synchronized (processorLock) { stop(); cleanScreen(); if (frameProcessor != null) { frameProcessor.stop(); } } } /** * Opens the camera and starts sending preview frames to the underlying detector. The preview * frames are not displayed. * * @throws IOException if the camera's preview texture or display could not be initialized */ @RequiresPermission(Manifest.permission.CAMERA) public synchronized CameraSource start() throws IOException { if (camera != null) { return this; } camera = createCamera(); dummySurfaceTexture = new SurfaceTexture(DUMMY_TEXTURE_NAME); camera.setPreviewTexture(dummySurfaceTexture); usingSurfaceTexture = true; camera.startPreview(); processingThread = new Thread(processingRunnable); processingRunnable.setActive(true); processingThread.start(); return this; } /** * Opens the camera and starts sending preview frames to the underlying detector. The supplied * surface holder is used for the preview so frames can be displayed to the user. * * @param surfaceHolder the surface holder to use for the preview frames * @throws IOException if the supplied surface holder could not be used as the preview display */ @RequiresPermission(Manifest.permission.CAMERA) public synchronized CameraSource start(SurfaceHolder surfaceHolder) throws IOException { if (camera != null) { return this; } camera = createCamera(); camera.setPreviewDisplay(surfaceHolder); camera.startPreview(); processingThread = new Thread(processingRunnable); processingRunnable.setActive(true); processingThread.start(); usingSurfaceTexture = false; return this; } /** * Closes the camera and stops sending frames to the underlying frame detector. * *

This camera source may be restarted again by calling {@link #start()} or {@link * #start(SurfaceHolder)}. * *

Call {@link #release()} instead to completely shut down this camera source and release the * resources of the underlying detector. */ public synchronized void stop() { processingRunnable.setActive(false); if (processingThread != null) { try { // Wait for the thread to complete to ensure that we can't have multiple threads // executing at the same time (i.e., which would happen if we called start too // quickly after stop). processingThread.join(); } catch (InterruptedException e) { Log.d(TAG, "Frame processing thread interrupted on release."); } processingThread = null; } if (camera != null) { camera.stopPreview(); camera.setPreviewCallbackWithBuffer(null); try { if (usingSurfaceTexture) { camera.setPreviewTexture(null); } else { camera.setPreviewDisplay(null); } } catch (Exception e) { Log.e(TAG, "Failed to clear camera preview: " + e); } camera.release(); camera = null; } // Release the reference to any image buffers, since these will no longer be in use. bytesToByteBuffer.clear(); } /** Changes the facing of the camera. */ public synchronized void setFacing(int facing) { if ((facing != CAMERA_FACING_BACK) && (facing != CAMERA_FACING_FRONT)) { throw new IllegalArgumentException("Invalid camera: " + facing); } this.facing = facing; } /** Returns the preview size that is currently in use by the underlying camera. */ public Size getPreviewSize() { return previewSize; } /** * Returns the selected camera; one of {@link #CAMERA_FACING_BACK} or {@link * #CAMERA_FACING_FRONT}. */ public int getCameraFacing() { return facing; } /** * Opens the camera and applies the user settings. * * @throws IOException if camera cannot be found or preview cannot be processed */ @SuppressLint("InlinedApi") private Camera createCamera() throws IOException { int requestedCameraId = getIdForRequestedCamera(facing); if (requestedCameraId == -1) { throw new IOException("Could not find requested camera."); } Camera camera = Camera.open(requestedCameraId); SizePair sizePair = PreferenceUtils.getCameraPreviewSizePair(activity, requestedCameraId); if (sizePair == null) { sizePair = selectSizePair( camera, DEFAULT_REQUESTED_CAMERA_PREVIEW_WIDTH, DEFAULT_REQUESTED_CAMERA_PREVIEW_HEIGHT); } if (sizePair == null) { throw new IOException("Could not find suitable preview size."); } previewSize = sizePair.preview; Log.v(TAG, "Camera preview size: " + previewSize); int[] previewFpsRange = selectPreviewFpsRange(camera, REQUESTED_FPS); if (previewFpsRange == null) { throw new IOException("Could not find suitable preview frames per second range."); } Camera.Parameters parameters = camera.getParameters(); Size pictureSize = sizePair.picture; if (pictureSize != null) { Log.v(TAG, "Camera picture size: " + pictureSize); parameters.setPictureSize(pictureSize.getWidth(), pictureSize.getHeight()); } parameters.setPreviewSize(previewSize.getWidth(), previewSize.getHeight()); parameters.setPreviewFpsRange( previewFpsRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], previewFpsRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); // Use YV12 so that we can exercise YV12->NV21 auto-conversion logic for OCR detection parameters.setPreviewFormat(IMAGE_FORMAT); setRotation(camera, parameters, requestedCameraId); if (REQUESTED_AUTO_FOCUS) { if (parameters .getSupportedFocusModes() .contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO)) { parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO); } else { Log.i(TAG, "Camera auto focus is not supported on this device."); } } camera.setParameters(parameters); // Four frame buffers are needed for working with the camera: // // one for the frame that is currently being executed upon in doing detection // one for the next pending frame to process immediately upon completing detection // two for the frames that the camera uses to populate future preview images // // Through trial and error it appears that two free buffers, in addition to the two buffers // used in this code, are needed for the camera to work properly. Perhaps the camera has // one thread for acquiring images, and another thread for calling into user code. If only // three buffers are used, then the camera will spew thousands of warning messages when // detection takes a non-trivial amount of time. camera.setPreviewCallbackWithBuffer(new CameraPreviewCallback()); camera.addCallbackBuffer(createPreviewBuffer(previewSize)); camera.addCallbackBuffer(createPreviewBuffer(previewSize)); camera.addCallbackBuffer(createPreviewBuffer(previewSize)); camera.addCallbackBuffer(createPreviewBuffer(previewSize)); return camera; } /** * Gets the id for the camera specified by the direction it is facing. Returns -1 if no such * camera was found. * * @param facing the desired camera (front-facing or rear-facing) */ private static int getIdForRequestedCamera(int facing) { CameraInfo cameraInfo = new CameraInfo(); for (int i = 0; i < Camera.getNumberOfCameras(); ++i) { Camera.getCameraInfo(i, cameraInfo); if (cameraInfo.facing == facing) { return i; } } return -1; } /** * Selects the most suitable preview and picture size, given the desired width and height. * *

Even though we only need to find the preview size, it's necessary to find both the preview * size and the picture size of the camera together, because these need to have the same aspect * ratio. On some hardware, if you would only set the preview size, you will get a distorted * image. * * @param camera the camera to select a preview size from * @param desiredWidth the desired width of the camera preview frames * @param desiredHeight the desired height of the camera preview frames * @return the selected preview and picture size pair */ public static SizePair selectSizePair(Camera camera, int desiredWidth, int desiredHeight) { List validPreviewSizes = generateValidPreviewSizeList(camera); // The method for selecting the best size is to minimize the sum of the differences between // the desired values and the actual values for width and height. This is certainly not the // only way to select the best size, but it provides a decent tradeoff between using the // closest aspect ratio vs. using the closest pixel area. SizePair selectedPair = null; int minDiff = Integer.MAX_VALUE; for (SizePair sizePair : validPreviewSizes) { Size size = sizePair.preview; int diff = Math.abs(size.getWidth() - desiredWidth) + Math.abs(size.getHeight() - desiredHeight); if (diff < minDiff) { selectedPair = sizePair; minDiff = diff; } } return selectedPair; } /** * Stores a preview size and a corresponding same-aspect-ratio picture size. To avoid distorted * preview images on some devices, the picture size must be set to a size that is the same aspect * ratio as the preview size or the preview may end up being distorted. If the picture size is * null, then there is no picture size with the same aspect ratio as the preview size. */ public static class SizePair { public final Size preview; @Nullable public final Size picture; SizePair( Camera.Size previewSize, @Nullable Camera.Size pictureSize) { preview = new Size(previewSize.width, previewSize.height); picture = pictureSize != null ? new Size(pictureSize.width, pictureSize.height) : null; } public SizePair(Size previewSize, @Nullable Size pictureSize) { preview = previewSize; picture = pictureSize; } } /** * Generates a list of acceptable preview sizes. Preview sizes are not acceptable if there is not * a corresponding picture size of the same aspect ratio. If there is a corresponding picture size * of the same aspect ratio, the picture size is paired up with the preview size. * *

This is necessary because even if we don't use still pictures, the still picture size must * be set to a size that is the same aspect ratio as the preview size we choose. Otherwise, the * preview images may be distorted on some devices. */ public static List generateValidPreviewSizeList(Camera camera) { Camera.Parameters parameters = camera.getParameters(); List supportedPreviewSizes = parameters.getSupportedPreviewSizes(); List supportedPictureSizes = parameters.getSupportedPictureSizes(); List validPreviewSizes = new ArrayList<>(); for (Camera.Size previewSize : supportedPreviewSizes) { float previewAspectRatio = (float) previewSize.width / (float) previewSize.height; // By looping through the picture sizes in order, we favor the higher resolutions. // We choose the highest resolution in order to support taking the full resolution // picture later. for (Camera.Size pictureSize : supportedPictureSizes) { float pictureAspectRatio = (float) pictureSize.width / (float) pictureSize.height; if (Math.abs(previewAspectRatio - pictureAspectRatio) < ASPECT_RATIO_TOLERANCE) { validPreviewSizes.add(new SizePair(previewSize, pictureSize)); break; } } } // If there are no picture sizes with the same aspect ratio as any preview sizes, allow all // of the preview sizes and hope that the camera can handle it. Probably unlikely, but we // still account for it. if (validPreviewSizes.isEmpty()) { Log.w(TAG, "No preview sizes have a corresponding same-aspect-ratio picture size"); for (Camera.Size previewSize : supportedPreviewSizes) { // The null picture size will let us know that we shouldn't set a picture size. validPreviewSizes.add(new SizePair(previewSize, null)); } } return validPreviewSizes; } /** * Selects the most suitable preview frames per second range, given the desired frames per second. * * @param camera the camera to select a frames per second range from * @param desiredPreviewFps the desired frames per second for the camera preview frames * @return the selected preview frames per second range */ @SuppressLint("InlinedApi") private static int[] selectPreviewFpsRange(Camera camera, float desiredPreviewFps) { // The camera API uses integers scaled by a factor of 1000 instead of floating-point frame // rates. int desiredPreviewFpsScaled = (int) (desiredPreviewFps * 1000.0f); // The method for selecting the best range is to minimize the sum of the differences between // the desired value and the upper and lower bounds of the range. This may select a range // that the desired value is outside of, but this is often preferred. For example, if the // desired frame rate is 29.97, the range (30, 30) is probably more desirable than the // range (15, 30). int[] selectedFpsRange = null; int minDiff = Integer.MAX_VALUE; List previewFpsRangeList = camera.getParameters().getSupportedPreviewFpsRange(); for (int[] range : previewFpsRangeList) { int deltaMin = desiredPreviewFpsScaled - range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX]; int deltaMax = desiredPreviewFpsScaled - range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]; int diff = Math.abs(deltaMin) + Math.abs(deltaMax); if (diff < minDiff) { selectedFpsRange = range; minDiff = diff; } } return selectedFpsRange; } /** * Calculates the correct rotation for the given camera id and sets the rotation in the * parameters. It also sets the camera's display orientation and rotation. * * @param parameters the camera parameters for which to set the rotation * @param cameraId the camera id to set rotation based on */ private void setRotation(Camera camera, Camera.Parameters parameters, int cameraId) { WindowManager windowManager = (WindowManager) activity.getSystemService(Context.WINDOW_SERVICE); int degrees = 0; int rotation = windowManager.getDefaultDisplay().getRotation(); switch (rotation) { case Surface.ROTATION_0: degrees = 0; break; case Surface.ROTATION_90: degrees = 90; break; case Surface.ROTATION_180: degrees = 180; break; case Surface.ROTATION_270: degrees = 270; break; default: Log.e(TAG, "Bad rotation value: " + rotation); } CameraInfo cameraInfo = new CameraInfo(); Camera.getCameraInfo(cameraId, cameraInfo); int displayAngle; if (cameraInfo.facing == CameraInfo.CAMERA_FACING_FRONT) { this.rotationDegrees = (cameraInfo.orientation + degrees) % 360; displayAngle = (360 - this.rotationDegrees) % 360; // compensate for it being mirrored } else { // back-facing this.rotationDegrees = (cameraInfo.orientation - degrees + 360) % 360; displayAngle = this.rotationDegrees; } Log.d(TAG, "Display rotation is: " + rotation); Log.d(TAG, "Camera face is: " + cameraInfo.facing); Log.d(TAG, "Camera rotation is: " + cameraInfo.orientation); // This value should be one of the degrees that ImageMetadata accepts: 0, 90, 180 or 270. Log.d(TAG, "RotationDegrees is: " + this.rotationDegrees); camera.setDisplayOrientation(displayAngle); parameters.setRotation(this.rotationDegrees); } /** * Creates one buffer for the camera preview callback. The size of the buffer is based off of the * camera preview size and the format of the camera image. * * @return a new preview buffer of the appropriate size for the current camera settings */ @SuppressLint("InlinedApi") private byte[] createPreviewBuffer(Size previewSize) { int bitsPerPixel = ImageFormat.getBitsPerPixel(IMAGE_FORMAT); long sizeInBits = (long) previewSize.getHeight() * previewSize.getWidth() * bitsPerPixel; int bufferSize = (int) Math.ceil(sizeInBits / 8.0d) + 1; // Creating the byte array this way and wrapping it, as opposed to using .allocate(), // should guarantee that there will be an array to work with. byte[] byteArray = new byte[bufferSize]; ByteBuffer buffer = ByteBuffer.wrap(byteArray); if (!buffer.hasArray() || (buffer.array() != byteArray)) { // I don't think that this will ever happen. But if it does, then we wouldn't be // passing the preview content to the underlying detector later. throw new IllegalStateException("Failed to create valid buffer for camera source."); } bytesToByteBuffer.put(byteArray, buffer); return byteArray; } // ============================================================================================== // Frame processing // ============================================================================================== /** Called when the camera has a new preview frame. */ private class CameraPreviewCallback implements Camera.PreviewCallback { @Override public void onPreviewFrame(byte[] data, Camera camera) { processingRunnable.setNextFrame(data, camera); } } void setMachineLearningFrameProcessor(VisionImageProcessor processor) { synchronized (processorLock) { cleanScreen(); if (frameProcessor != null) { frameProcessor.stop(); } frameProcessor = processor; } } /** * This runnable controls access to the underlying receiver, calling it to process frames when * available from the camera. This is designed to run detection on frames as fast as possible * (i.e., without unnecessary context switching or waiting on the next frame). * *

While detection is running on a frame, new frames may be received from the camera. As these * frames come in, the most recent frame is held onto as pending. As soon as detection and its * associated processing is done for the previous frame, detection on the mostly recently received * frame will immediately start on the same thread. */ private class FrameProcessingRunnable implements Runnable { // This lock guards all of the member variables below. private final Object lock = new Object(); private boolean active = true; // These pending variables hold the state associated with the new frame awaiting processing. private ByteBuffer pendingFrameData; FrameProcessingRunnable() {} /** Marks the runnable as active/not active. Signals any blocked threads to continue. */ void setActive(boolean active) { synchronized (lock) { this.active = active; lock.notifyAll(); } } /** * Sets the frame data received from the camera. This adds the previous unused frame buffer (if * present) back to the camera, and keeps a pending reference to the frame data for future use. */ @SuppressWarnings("ByteBufferBackingArray") void setNextFrame(byte[] data, Camera camera) { synchronized (lock) { if (pendingFrameData != null) { camera.addCallbackBuffer(pendingFrameData.array()); pendingFrameData = null; } if (!bytesToByteBuffer.containsKey(data)) { Log.d( TAG, "Skipping frame. Could not find ByteBuffer associated with the image " + "data from the camera."); return; } pendingFrameData = bytesToByteBuffer.get(data); // Notify the processor thread if it is waiting on the next frame (see below). lock.notifyAll(); } } /** * As long as the processing thread is active, this executes detection on frames continuously. * The next pending frame is either immediately available or hasn't been received yet. Once it * is available, we transfer the frame info to local variables and run detection on that frame. * It immediately loops back for the next frame without pausing. * *

If detection takes longer than the time in between new frames from the camera, this will * mean that this loop will run without ever waiting on a frame, avoiding any context switching * or frame acquisition time latency. * *

If you find that this is using more CPU than you'd like, you should probably decrease the * FPS setting above to allow for some idle time in between frames. */ @SuppressLint("InlinedApi") @SuppressWarnings({"GuardedBy", "ByteBufferBackingArray"}) @Override public void run() { ByteBuffer data; while (true) { synchronized (lock) { while (active && (pendingFrameData == null)) { try { // Wait for the next frame to be received from the camera, since we // don't have it yet. lock.wait(); } catch (InterruptedException e) { Log.d(TAG, "Frame processing loop terminated.", e); return; } } if (!active) { // Exit the loop once this camera source is stopped or released. We check // this here, immediately after the wait() above, to handle the case where // setActive(false) had been called, triggering the termination of this // loop. return; } // Hold onto the frame data locally, so that we can use this for detection // below. We need to clear pendingFrameData to ensure that this buffer isn't // recycled back to the camera before we are done using that data. data = pendingFrameData; pendingFrameData = null; } // The code below needs to run outside of synchronization, because this will allow // the camera to add pending frame(s) while we are running detection on the current // frame. try { synchronized (processorLock) { frameProcessor.processByteBuffer( data, new FrameMetadata.Builder() .setWidth(previewSize.getWidth()) .setHeight(previewSize.getHeight()) .setRotation(rotationDegrees) .build(), graphicOverlay); } } catch (Exception t) { Log.e(TAG, "Exception thrown from receiver.", t); } finally { camera.addCallbackBuffer(data.array()); } } } } /** Cleans up graphicOverlay and child classes can do their cleanups as well . */ private void cleanScreen() { graphicOverlay.clear(); } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/CameraSourcePreview.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.content.Context; import android.content.res.Configuration; import android.util.AttributeSet; import android.util.Log; import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.ViewGroup; import com.google.android.gms.common.images.Size; import com.google.mlkit.vision.automl.demo.preference.PreferenceUtils; import java.io.IOException; /** Preview the camera image in the screen. */ public class CameraSourcePreview extends ViewGroup { private static final String TAG = "MIDemoApp:Preview"; private final Context context; private final SurfaceView surfaceView; private boolean startRequested; private boolean surfaceAvailable; private CameraSource cameraSource; private GraphicOverlay overlay; public CameraSourcePreview(Context context, AttributeSet attrs) { super(context, attrs); this.context = context; startRequested = false; surfaceAvailable = false; surfaceView = new SurfaceView(context); surfaceView.getHolder().addCallback(new SurfaceCallback()); addView(surfaceView); } private void start(CameraSource cameraSource) throws IOException { this.cameraSource = cameraSource; if (this.cameraSource != null) { startRequested = true; startIfReady(); } } public void start(CameraSource cameraSource, GraphicOverlay overlay) throws IOException { this.overlay = overlay; start(cameraSource); } public void stop() { if (cameraSource != null) { cameraSource.stop(); } } public void release() { if (cameraSource != null) { cameraSource.release(); cameraSource = null; } surfaceView.getHolder().getSurface().release(); } private void startIfReady() throws IOException, SecurityException { if (startRequested && surfaceAvailable) { if (PreferenceUtils.isCameraLiveViewportEnabled(context)) { cameraSource.start(surfaceView.getHolder()); } else { cameraSource.start(); } requestLayout(); if (overlay != null) { Size size = cameraSource.getPreviewSize(); int min = Math.min(size.getWidth(), size.getHeight()); int max = Math.max(size.getWidth(), size.getHeight()); boolean isImageFlipped = cameraSource.getCameraFacing() == CameraSource.CAMERA_FACING_FRONT; if (isPortraitMode()) { // Swap width and height sizes when in portrait, since it will be rotated by 90 degrees. // The camera preview and the image being processed have the same size. overlay.setImageSourceInfo(min, max, isImageFlipped); } else { overlay.setImageSourceInfo(max, min, isImageFlipped); } overlay.clear(); } startRequested = false; } } private class SurfaceCallback implements SurfaceHolder.Callback { @Override public void surfaceCreated(SurfaceHolder surface) { surfaceAvailable = true; try { startIfReady(); } catch (IOException e) { Log.e(TAG, "Could not start camera source.", e); } } @Override public void surfaceDestroyed(SurfaceHolder surface) { surfaceAvailable = false; } @Override public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {} } @Override protected void onLayout(boolean changed, int left, int top, int right, int bottom) { int width = 320; int height = 240; if (cameraSource != null) { Size size = cameraSource.getPreviewSize(); if (size != null) { width = size.getWidth(); height = size.getHeight(); } } // Swap width and height sizes when in portrait, since it will be rotated 90 degrees if (isPortraitMode()) { int tmp = width; width = height; height = tmp; } final int layoutWidth = right - left; final int layoutHeight = bottom - top; // Computes height and width for potentially doing fit width. int childWidth = layoutWidth; int childHeight = (int) (((float) layoutWidth / (float) width) * height); // If height is too tall using fit width, does fit height instead. if (childHeight > layoutHeight) { childHeight = layoutHeight; childWidth = (int) (((float) layoutHeight / (float) height) * width); } for (int i = 0; i < getChildCount(); ++i) { getChildAt(i).layout(0, 0, childWidth, childHeight); Log.d(TAG, "Assigned view: " + i); } } private boolean isPortraitMode() { int orientation = context.getResources().getConfiguration().orientation; if (orientation == Configuration.ORIENTATION_LANDSCAPE) { return false; } if (orientation == Configuration.ORIENTATION_PORTRAIT) { return true; } Log.d(TAG, "isPortraitMode returning false by default"); return false; } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/CameraXLivePreviewActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.content.Intent; import android.os.Build.VERSION_CODES; import android.os.Bundle; import androidx.appcompat.app.AppCompatActivity; import android.util.Log; import android.util.Size; import android.view.View; import android.widget.AdapterView; import android.widget.AdapterView.OnItemSelectedListener; import android.widget.ArrayAdapter; import android.widget.CompoundButton; import android.widget.ImageView; import android.widget.Spinner; import android.widget.Toast; import android.widget.ToggleButton; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.camera.core.CameraInfoUnavailableException; import androidx.camera.core.CameraSelector; import androidx.camera.core.ImageAnalysis; import androidx.camera.core.Preview; import androidx.camera.lifecycle.ProcessCameraProvider; import androidx.camera.view.PreviewView; import androidx.core.content.ContextCompat; import androidx.lifecycle.ViewModelProvider; import androidx.lifecycle.ViewModelProvider.AndroidViewModelFactory; import com.google.android.gms.common.annotation.KeepName; import com.google.mlkit.common.MlKitException; import com.google.mlkit.common.model.CustomRemoteModel; import com.google.mlkit.linkfirebase.FirebaseModelSource; import com.google.mlkit.vision.automl.demo.automl.AutoMLImageLabelerProcessor; import com.google.mlkit.vision.automl.demo.automl.AutoMLImageLabelerProcessor.Mode; import com.google.mlkit.vision.automl.demo.object.ObjectDetectorProcessor; import com.google.mlkit.vision.automl.demo.preference.PreferenceUtils; import com.google.mlkit.vision.automl.demo.preference.SettingsActivity; import com.google.mlkit.vision.label.custom.CustomImageLabelerOptions; import com.google.mlkit.vision.objects.custom.CustomObjectDetectorOptions; import java.util.ArrayList; import java.util.List; /** Live preview demo app for ML Kit APIs using CameraX. */ @KeepName @RequiresApi(VERSION_CODES.LOLLIPOP) public final class CameraXLivePreviewActivity extends AppCompatActivity implements OnItemSelectedListener, CompoundButton.OnCheckedChangeListener { private static final String TAG = "CameraXLivePreview"; private static final String CUSTOM_AUTOML_LABELING = "Custom AutoML Image Labeling"; private static final String CUSTOM_AUTOML_OBJECT_DETECTION = "Custom AutoML Object Detection"; private static final String STATE_SELECTED_MODEL = "selected_model"; private static final String STATE_LENS_FACING = "lens_facing"; private PreviewView previewView; private GraphicOverlay graphicOverlay; @Nullable private ProcessCameraProvider cameraProvider; @Nullable private Preview previewUseCase; @Nullable private ImageAnalysis analysisUseCase; @Nullable private VisionImageProcessor imageProcessor; private boolean needUpdateGraphicOverlayImageSourceInfo; private String selectedModel = CUSTOM_AUTOML_LABELING; private int lensFacing = CameraSelector.LENS_FACING_BACK; private CameraSelector cameraSelector; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); Log.d(TAG, "onCreate"); if (savedInstanceState != null) { selectedModel = savedInstanceState.getString(STATE_SELECTED_MODEL, CUSTOM_AUTOML_LABELING); lensFacing = savedInstanceState.getInt(STATE_LENS_FACING, CameraSelector.LENS_FACING_BACK); } cameraSelector = new CameraSelector.Builder().requireLensFacing(lensFacing).build(); setContentView(R.layout.activity_camerax_live_preview); previewView = findViewById(R.id.preview_view); if (previewView == null) { Log.d(TAG, "previewView is null"); } graphicOverlay = findViewById(R.id.graphic_overlay); if (graphicOverlay == null) { Log.d(TAG, "graphicOverlay is null"); } Spinner spinner = findViewById(R.id.spinner); List options = new ArrayList<>(); options.add(CUSTOM_AUTOML_LABELING); options.add(CUSTOM_AUTOML_OBJECT_DETECTION); // Creating adapter for spinner ArrayAdapter dataAdapter = new ArrayAdapter<>(this, R.layout.spinner_style, options); // Drop down layout style - list view with radio button dataAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); // attaching data adapter to spinner spinner.setAdapter(dataAdapter); spinner.setOnItemSelectedListener(this); ToggleButton facingSwitch = findViewById(R.id.facing_switch); facingSwitch.setOnCheckedChangeListener(this); new ViewModelProvider(this, AndroidViewModelFactory.getInstance(getApplication())) .get(CameraXViewModel.class) .getProcessCameraProvider() .observe( this, provider -> { cameraProvider = provider; bindAllCameraUseCases(); }); ImageView settingsButton = findViewById(R.id.settings_button); settingsButton.setOnClickListener( v -> { Intent intent = new Intent(getApplicationContext(), SettingsActivity.class); intent.putExtra( SettingsActivity.EXTRA_LAUNCH_SOURCE, SettingsActivity.LaunchSource.CAMERAX_LIVE_PREVIEW); startActivity(intent); }); } @Override protected void onSaveInstanceState(@NonNull Bundle bundle) { super.onSaveInstanceState(bundle); bundle.putString(STATE_SELECTED_MODEL, selectedModel); bundle.putInt(STATE_LENS_FACING, lensFacing); } @Override public synchronized void onItemSelected(AdapterView parent, View view, int pos, long id) { // An item was selected. You can retrieve the selected item using // parent.getItemAtPosition(pos) selectedModel = parent.getItemAtPosition(pos).toString(); Log.d(TAG, "Selected model: " + selectedModel); bindAnalysisUseCase(); } @Override public void onNothingSelected(AdapterView parent) { // Do nothing. } @Override public void onCheckedChanged(CompoundButton buttonView, boolean isChecked) { Log.d(TAG, "Set facing"); if (cameraProvider == null) { return; } int newLensFacing = lensFacing == CameraSelector.LENS_FACING_FRONT ? CameraSelector.LENS_FACING_BACK : CameraSelector.LENS_FACING_FRONT; CameraSelector newCameraSelector = new CameraSelector.Builder().requireLensFacing(newLensFacing).build(); try { if (cameraProvider.hasCamera(newCameraSelector)) { lensFacing = newLensFacing; cameraSelector = newCameraSelector; bindAllCameraUseCases(); return; } } catch (CameraInfoUnavailableException e) { // Falls through } Toast.makeText( getApplicationContext(), "This device does not have lens with facing: " + newLensFacing, Toast.LENGTH_SHORT) .show(); } @Override public void onResume() { super.onResume(); bindAllCameraUseCases(); } @Override protected void onPause() { super.onPause(); if (imageProcessor != null) { imageProcessor.stop(); } } @Override public void onDestroy() { super.onDestroy(); if (imageProcessor != null) { imageProcessor.stop(); } } private void bindAllCameraUseCases() { bindPreviewUseCase(); bindAnalysisUseCase(); } private void bindPreviewUseCase() { if (!PreferenceUtils.isCameraLiveViewportEnabled(this)) { return; } if (cameraProvider == null) { return; } if (previewUseCase != null) { cameraProvider.unbind(previewUseCase); } Preview.Builder builder = new Preview.Builder(); Size targetResolution = PreferenceUtils.getCameraXTargetResolution(this, lensFacing); if (targetResolution != null) { builder.setTargetResolution(targetResolution); } previewUseCase = builder.build(); previewUseCase.setSurfaceProvider(previewView.getSurfaceProvider()); cameraProvider.bindToLifecycle(/* lifecycleOwner= */ this, cameraSelector, previewUseCase); } private void bindAnalysisUseCase() { if (cameraProvider == null) { return; } if (analysisUseCase != null) { cameraProvider.unbind(analysisUseCase); } if (imageProcessor != null) { imageProcessor.stop(); } try { String autoMLRemoteModelName = PreferenceUtils.getAutoMLRemoteModelName(this); switch (selectedModel) { case CUSTOM_AUTOML_LABELING: Log.i(TAG, "Create Custom AutoML Image Label Processor"); CustomRemoteModel customRemoteModel = new CustomRemoteModel.Builder( new FirebaseModelSource.Builder(autoMLRemoteModelName).build()) .build(); CustomImageLabelerOptions customImageLabelerOptions = new CustomImageLabelerOptions.Builder(customRemoteModel) .setConfidenceThreshold(0) .build(); imageProcessor = new AutoMLImageLabelerProcessor( this, customRemoteModel, customImageLabelerOptions, Mode.LIVE_PREVIEW); break; case CUSTOM_AUTOML_OBJECT_DETECTION: Log.i(TAG, "Using Custom AutoML Object Detector Processor"); CustomRemoteModel customODTRemoteModel = new CustomRemoteModel.Builder( new FirebaseModelSource.Builder(autoMLRemoteModelName).build()) .build(); CustomObjectDetectorOptions customAutoMLODTOptions = new CustomObjectDetectorOptions.Builder(customODTRemoteModel) .setDetectorMode(CustomObjectDetectorOptions.STREAM_MODE) .enableClassification() .setClassificationConfidenceThreshold(0) .setMaxPerObjectLabelCount(1) .build(); imageProcessor = new ObjectDetectorProcessor(this, customODTRemoteModel, customAutoMLODTOptions); break; default: throw new IllegalStateException("Invalid model name"); } } catch (RuntimeException e) { Log.e(TAG, "Can not create image processor: " + selectedModel, e); Toast.makeText( getApplicationContext(), "Can not create image processor: " + e.getLocalizedMessage(), Toast.LENGTH_LONG) .show(); return; } ImageAnalysis.Builder builder = new ImageAnalysis.Builder(); Size targetResolution = PreferenceUtils.getCameraXTargetResolution(this, lensFacing); if (targetResolution != null) { builder.setTargetResolution(targetResolution); } analysisUseCase = builder.build(); needUpdateGraphicOverlayImageSourceInfo = true; analysisUseCase.setAnalyzer( // imageProcessor.processImageProxy will use another thread to run the detection underneath, // thus we can just runs the analyzer itself on main thread. ContextCompat.getMainExecutor(this), imageProxy -> { if (needUpdateGraphicOverlayImageSourceInfo) { boolean isImageFlipped = lensFacing == CameraSelector.LENS_FACING_FRONT; int rotationDegrees = imageProxy.getImageInfo().getRotationDegrees(); if (rotationDegrees == 0 || rotationDegrees == 180) { graphicOverlay.setImageSourceInfo( imageProxy.getWidth(), imageProxy.getHeight(), isImageFlipped); } else { graphicOverlay.setImageSourceInfo( imageProxy.getHeight(), imageProxy.getWidth(), isImageFlipped); } needUpdateGraphicOverlayImageSourceInfo = false; } try { imageProcessor.processImageProxy(imageProxy, graphicOverlay); } catch (MlKitException e) { Log.e(TAG, "Failed to process image. Error: " + e.getLocalizedMessage()); Toast.makeText(getApplicationContext(), e.getLocalizedMessage(), Toast.LENGTH_SHORT) .show(); } }); cameraProvider.bindToLifecycle(/* lifecycleOwner= */ this, cameraSelector, analysisUseCase); } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/CameraXViewModel.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.app.Application; import android.os.Build.VERSION_CODES; import android.util.Log; import androidx.annotation.NonNull; import androidx.annotation.RequiresApi; import androidx.camera.lifecycle.ProcessCameraProvider; import androidx.core.content.ContextCompat; import androidx.lifecycle.AndroidViewModel; import androidx.lifecycle.LiveData; import androidx.lifecycle.MutableLiveData; import com.google.common.util.concurrent.ListenableFuture; import java.util.concurrent.ExecutionException; /** View model for interacting with CameraX. */ @RequiresApi(VERSION_CODES.LOLLIPOP) public final class CameraXViewModel extends AndroidViewModel { private static final String TAG = "CameraXViewModel"; private MutableLiveData cameraProviderLiveData; /** * Create an instance which interacts with the camera service via the given application context. */ public CameraXViewModel(@NonNull Application application) { super(application); } LiveData getProcessCameraProvider() { if (cameraProviderLiveData == null) { cameraProviderLiveData = new MutableLiveData<>(); ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(getApplication()); cameraProviderFuture.addListener( () -> { try { cameraProviderLiveData.setValue(cameraProviderFuture.get()); } catch (ExecutionException | InterruptedException e) { // Handle any errors (including cancellation) here. Log.e(TAG, "Unhandled exception", e); } }, ContextCompat.getMainExecutor(getApplication())); } return cameraProviderLiveData; } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/ChooserActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.Manifest; import android.content.Context; import android.content.Intent; import android.content.pm.PackageManager; import android.os.Build.VERSION; import android.os.Build.VERSION_CODES; import android.os.Bundle; import android.os.StrictMode; import androidx.appcompat.app.AppCompatActivity; import android.util.Log; import android.view.LayoutInflater; import android.view.View; import android.view.ViewGroup; import android.widget.AdapterView; import android.widget.ArrayAdapter; import android.widget.ListView; import android.widget.TextView; import androidx.core.app.ActivityCompat; import androidx.core.app.ActivityCompat.OnRequestPermissionsResultCallback; import androidx.core.content.ContextCompat; import java.util.ArrayList; import java.util.List; /** * Demo app chooser which takes care of runtime permission requesting and allow you pick from all * available testing Activities. */ public final class ChooserActivity extends AppCompatActivity implements OnRequestPermissionsResultCallback, AdapterView.OnItemClickListener { private static final String TAG = "ChooserActivity"; private static final int PERMISSION_REQUESTS = 1; private static final String[] REQUIRED_RUNTIME_PERMISSIONS = { Manifest.permission.CAMERA, Manifest.permission.WRITE_EXTERNAL_STORAGE, Manifest.permission.READ_EXTERNAL_STORAGE }; @SuppressWarnings("NewApi") // CameraX is only available on API 21+ private static final Class[] CLASSES = VERSION.SDK_INT < VERSION_CODES.LOLLIPOP ? new Class[] { LivePreviewActivity.class, StillImageActivity.class, } : new Class[] { LivePreviewActivity.class, StillImageActivity.class, CameraXLivePreviewActivity.class, }; private static final int[] DESCRIPTION_IDS = VERSION.SDK_INT < VERSION_CODES.LOLLIPOP ? new int[] { R.string.desc_camera_source_activity, R.string.desc_still_image_activity, } : new int[] { R.string.desc_camera_source_activity, R.string.desc_still_image_activity, R.string.desc_camerax_live_preview_activity, }; @Override protected void onCreate(Bundle savedInstanceState) { if (BuildConfig.DEBUG) { StrictMode.setThreadPolicy( new StrictMode.ThreadPolicy.Builder().detectAll().penaltyLog().build()); StrictMode.setVmPolicy( new StrictMode.VmPolicy.Builder() .detectLeakedSqlLiteObjects() .detectLeakedClosableObjects() .penaltyLog() .penaltyDeath() .build()); } super.onCreate(savedInstanceState); Log.d(TAG, "onCreate"); setContentView(R.layout.activity_chooser); // Set up ListView and Adapter ListView listView = findViewById(R.id.test_activity_list_view); MyArrayAdapter adapter = new MyArrayAdapter(this, android.R.layout.simple_list_item_2, CLASSES); adapter.setDescriptionIds(DESCRIPTION_IDS); listView.setAdapter(adapter); listView.setOnItemClickListener(this); if (!allPermissionsGranted()) { getRuntimePermissions(); } } @Override public void onItemClick(AdapterView parent, View view, int position, long id) { Class clicked = CLASSES[position]; startActivity(new Intent(this, clicked)); } private boolean allPermissionsGranted() { for (String permission : REQUIRED_RUNTIME_PERMISSIONS) { if (!isPermissionGranted(this, permission)) { return false; } } return true; } private void getRuntimePermissions() { List allNeededPermissions = new ArrayList<>(); for (String permission : REQUIRED_RUNTIME_PERMISSIONS) { if (!isPermissionGranted(this, permission)) { allNeededPermissions.add(permission); } } if (!allNeededPermissions.isEmpty()) { ActivityCompat.requestPermissions( this, allNeededPermissions.toArray(new String[0]), PERMISSION_REQUESTS); } } private static boolean isPermissionGranted(Context context, String permission) { if (ContextCompat.checkSelfPermission(context, permission) == PackageManager.PERMISSION_GRANTED) { Log.i(TAG, "Permission granted: " + permission); return true; } Log.i(TAG, "Permission NOT granted: " + permission); return false; } private static class MyArrayAdapter extends ArrayAdapter> { private final Context context; private final Class[] classes; private int[] descriptionIds; MyArrayAdapter(Context context, int resource, Class[] objects) { super(context, resource, objects); this.context = context; classes = objects; } @Override public View getView(int position, View convertView, ViewGroup parent) { View view = convertView; if (convertView == null) { LayoutInflater inflater = (LayoutInflater) context.getSystemService(LAYOUT_INFLATER_SERVICE); view = inflater.inflate(android.R.layout.simple_list_item_2, null); } ((TextView) view.findViewById(android.R.id.text1)).setText(classes[position].getSimpleName()); ((TextView) view.findViewById(android.R.id.text2)).setText(descriptionIds[position]); return view; } void setDescriptionIds(int[] descriptionIds) { this.descriptionIds = descriptionIds; } } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/FrameMetadata.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; /** Describing a frame info. */ public class FrameMetadata { private final int width; private final int height; private final int rotation; public int getWidth() { return width; } public int getHeight() { return height; } public int getRotation() { return rotation; } private FrameMetadata(int width, int height, int rotation) { this.width = width; this.height = height; this.rotation = rotation; } /** Builder of {@link FrameMetadata}. */ public static class Builder { private int width; private int height; private int rotation; public Builder setWidth(int width) { this.width = width; return this; } public Builder setHeight(int height) { this.height = height; return this; } public Builder setRotation(int rotation) { this.rotation = rotation; return this; } public FrameMetadata build() { return new FrameMetadata(width, height, rotation); } } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/GraphicOverlay.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.content.Context; import android.graphics.Canvas; import android.graphics.Matrix; import android.util.AttributeSet; import android.view.View; import com.google.common.base.Preconditions; import java.util.ArrayList; import java.util.List; /** * A view which renders a series of custom graphics to be overlayed on top of an associated preview * (i.e., the camera preview). The creator can add graphics objects, update the objects, and remove * them, triggering the appropriate drawing and invalidation within the view. * *

Supports scaling and mirroring of the graphics relative the camera's preview properties. The * idea is that detection items are expressed in terms of an image size, but need to be scaled up * to the full view size, and also mirrored in the case of the front-facing camera. * *

Associated {@link Graphic} items should use the following methods to convert to view * coordinates for the graphics that are drawn: * *

    *
  1. {@link Graphic#scale(float)} adjusts the size of the supplied value from the image scale * to the view scale. *
  2. {@link Graphic#translateX(float)} and {@link Graphic#translateY(float)} adjust the * coordinate from the image's coordinate system to the view coordinate system. *
*/ public class GraphicOverlay extends View { private final Object lock = new Object(); private final List graphics = new ArrayList<>(); // Matrix for transforming from image coordinates to overlay view coordinates. private final Matrix transformationMatrix = new Matrix(); private int imageWidth; private int imageHeight; // The factor of overlay View size to image size. Anything in the image coordinates need to be // scaled by this amount to fit with the area of overlay View. private float scaleFactor = 1.0f; // The number of horizontal pixels needed to be cropped on each side to fit the image with the // area of overlay View after scaling. private float postScaleWidthOffset; // The number of vertical pixels needed to be cropped on each side to fit the image with the // area of overlay View after scaling. private float postScaleHeightOffset; private boolean isImageFlipped; private boolean needUpdateTransformation = true; /** * Base class for a custom graphics object to be rendered within the graphic overlay. Subclass * this and implement the {@link Graphic#draw(Canvas)} method to define the graphics element. Add * instances to the overlay using {@link GraphicOverlay#add(Graphic)}. */ public abstract static class Graphic { private GraphicOverlay overlay; public Graphic(GraphicOverlay overlay) { this.overlay = overlay; } /** * Draw the graphic on the supplied canvas. Drawing should use the following methods to convert * to view coordinates for the graphics that are drawn: * *
    *
  1. {@link Graphic#scale(float)} adjusts the size of the supplied value from the image * scale to the view scale. *
  2. {@link Graphic#translateX(float)} and {@link Graphic#translateY(float)} adjust the * coordinate from the image's coordinate system to the view coordinate system. *
* * @param canvas drawing canvas */ public abstract void draw(Canvas canvas); /** * Adjusts the supplied value from the image scale to the view scale. */ public float scale(float imagePixel) { return imagePixel * overlay.scaleFactor; } /** Returns the application context of the app. */ public Context getApplicationContext() { return overlay.getContext().getApplicationContext(); } public boolean isImageFlipped() { return overlay.isImageFlipped; } /** * Adjusts the x coordinate from the image's coordinate system to the view coordinate system. */ public float translateX(float x) { if (overlay.isImageFlipped) { return overlay.getWidth() - (scale(x) - overlay.postScaleWidthOffset); } else { return scale(x) - overlay.postScaleWidthOffset; } } /** * Adjusts the y coordinate from the image's coordinate system to the view coordinate system. */ public float translateY(float y) { return scale(y) - overlay.postScaleHeightOffset; } /** * Returns a {@link Matrix} for transforming from image coordinates to overlay view coordinates. */ public Matrix getTransformationMatrix() { return overlay.transformationMatrix; } public void postInvalidate() { overlay.postInvalidate(); } } public GraphicOverlay(Context context, AttributeSet attrs) { super(context, attrs); addOnLayoutChangeListener( (view, left, top, right, bottom, oldLeft, oldTop, oldRight, oldBottom) -> needUpdateTransformation = true); } /** Removes all graphics from the overlay. */ public void clear() { synchronized (lock) { graphics.clear(); } postInvalidate(); } /** Adds a graphic to the overlay. */ public void add(Graphic graphic) { synchronized (lock) { graphics.add(graphic); } } /** Removes a graphic from the overlay. */ public void remove(Graphic graphic) { synchronized (lock) { graphics.remove(graphic); } postInvalidate(); } /** * Sets the source information of the image being processed by detectors, including size and * whether it is flipped, which informs how to transform image coordinates later. * * @param imageWidth the width of the image sent to ML Kit detectors * @param imageHeight the height of the image sent to ML Kit detectors * @param isFlipped whether the image is flipped. Should set it to true when the image is from the * front camera. */ public void setImageSourceInfo(int imageWidth, int imageHeight, boolean isFlipped) { Preconditions.checkState(imageWidth > 0, "image width must be positive"); Preconditions.checkState(imageHeight > 0, "image height must be positive"); synchronized (lock) { this.imageWidth = imageWidth; this.imageHeight = imageHeight; this.isImageFlipped = isFlipped; needUpdateTransformation = true; } postInvalidate(); } public int getImageWidth() { return imageWidth; } public int getImageHeight() { return imageHeight; } private void updateTransformationIfNeeded() { if (!needUpdateTransformation || imageWidth <= 0 || imageHeight <= 0) { return; } float viewAspectRatio = (float) getWidth() / getHeight(); float imageAspectRatio = (float) imageWidth / imageHeight; postScaleWidthOffset = 0; postScaleHeightOffset = 0; if (viewAspectRatio > imageAspectRatio) { // The image needs to be vertically cropped to be displayed in this view. scaleFactor = (float) getWidth() / imageWidth; postScaleHeightOffset = ((float) getWidth() / imageAspectRatio - getHeight()) / 2; } else { // The image needs to be horizontally cropped to be displayed in this view. scaleFactor = (float) getHeight() / imageHeight; postScaleWidthOffset = ((float) getHeight() * imageAspectRatio - getWidth()) / 2; } transformationMatrix.reset(); transformationMatrix.setScale(scaleFactor, scaleFactor); transformationMatrix.postTranslate(-postScaleWidthOffset, -postScaleHeightOffset); if (isImageFlipped) { transformationMatrix.postScale(-1f, 1f, getWidth() / 2f, getHeight() / 2f); } needUpdateTransformation = false; } /** Draws the overlay with its associated graphic objects. */ @Override protected void onDraw(Canvas canvas) { super.onDraw(canvas); synchronized (lock) { updateTransformationIfNeeded(); for (Graphic graphic : graphics) { graphic.draw(canvas); } } } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/InferenceInfoGraphic.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.graphics.Canvas; import android.graphics.Color; import android.graphics.Paint; import androidx.annotation.Nullable; /** Graphic instance for rendering inference info (latency, FPS, resolution) in an overlay view. */ public class InferenceInfoGraphic extends GraphicOverlay.Graphic { private static final int TEXT_COLOR = Color.WHITE; private static final float TEXT_SIZE = 60.0f; private final Paint textPaint; private final GraphicOverlay overlay; private final double latency; // Only valid when a stream of input images is being processed. Null for single image mode. @Nullable private final Integer framesPerSecond; public InferenceInfoGraphic( GraphicOverlay overlay, double latency, @Nullable Integer framesPerSecond) { super(overlay); this.overlay = overlay; this.latency = latency; this.framesPerSecond = framesPerSecond; textPaint = new Paint(); textPaint.setColor(TEXT_COLOR); textPaint.setTextSize(TEXT_SIZE); postInvalidate(); } @Override public synchronized void draw(Canvas canvas) { float x = TEXT_SIZE * 0.5f; float y = TEXT_SIZE * 1.5f; canvas.drawText( "InputImage size: " + overlay.getImageHeight() + "x" + overlay.getImageWidth(), x, y, textPaint); // Draw FPS (if valid) and inference latency if (framesPerSecond != null) { canvas.drawText( "FPS: " + framesPerSecond + ", latency: " + latency + " ms", x, y + TEXT_SIZE, textPaint); } else { canvas.drawText("Latency: " + latency + " ms", x, y + TEXT_SIZE, textPaint); } } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/LivePreviewActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.content.Intent; import android.os.Bundle; import androidx.appcompat.app.AppCompatActivity; import android.util.Log; import android.view.View; import android.widget.AdapterView; import android.widget.AdapterView.OnItemSelectedListener; import android.widget.ArrayAdapter; import android.widget.CompoundButton; import android.widget.ImageView; import android.widget.Spinner; import android.widget.Toast; import android.widget.ToggleButton; import com.google.android.gms.common.annotation.KeepName; import com.google.mlkit.common.model.CustomRemoteModel; import com.google.mlkit.linkfirebase.FirebaseModelSource; import com.google.mlkit.vision.automl.demo.automl.AutoMLImageLabelerProcessor; import com.google.mlkit.vision.automl.demo.automl.AutoMLImageLabelerProcessor.Mode; import com.google.mlkit.vision.automl.demo.object.ObjectDetectorProcessor; import com.google.mlkit.vision.automl.demo.preference.PreferenceUtils; import com.google.mlkit.vision.automl.demo.preference.SettingsActivity; import com.google.mlkit.vision.label.custom.CustomImageLabelerOptions; import com.google.mlkit.vision.objects.custom.CustomObjectDetectorOptions; import java.io.IOException; import java.util.ArrayList; import java.util.List; /** Live preview demo for ML Kit APIs. */ @KeepName public final class LivePreviewActivity extends AppCompatActivity implements OnItemSelectedListener, CompoundButton.OnCheckedChangeListener { private static final String CUSTOM_AUTOML_LABELING = "Custom AutoML Image Labeling"; private static final String CUSTOM_AUTOML_OBJECT_DETECTION = "Custom AutoML Object Detection"; private static final String TAG = "LivePreviewActivity"; private CameraSource cameraSource = null; private CameraSourcePreview preview; private GraphicOverlay graphicOverlay; private String selectedModel = CUSTOM_AUTOML_LABELING; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); Log.d(TAG, "onCreate"); setContentView(R.layout.activity_live_preview); preview = findViewById(R.id.preview); if (preview == null) { Log.d(TAG, "Preview is null"); } graphicOverlay = findViewById(R.id.graphic_overlay); if (graphicOverlay == null) { Log.d(TAG, "graphicOverlay is null"); } Spinner spinner = findViewById(R.id.spinner); List options = new ArrayList<>(); options.add(CUSTOM_AUTOML_LABELING); options.add(CUSTOM_AUTOML_OBJECT_DETECTION); // Creating adapter for spinner ArrayAdapter dataAdapter = new ArrayAdapter<>(this, R.layout.spinner_style, options); // Drop down layout style - list view with radio button dataAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); // attaching data adapter to spinner spinner.setAdapter(dataAdapter); spinner.setOnItemSelectedListener(this); ToggleButton facingSwitch = findViewById(R.id.facing_switch); facingSwitch.setOnCheckedChangeListener(this); ImageView settingsButton = findViewById(R.id.settings_button); settingsButton.setOnClickListener( v -> { Intent intent = new Intent(getApplicationContext(), SettingsActivity.class); intent.putExtra(SettingsActivity.EXTRA_LAUNCH_SOURCE, SettingsActivity.LaunchSource.LIVE_PREVIEW); startActivity(intent); }); createCameraSource(selectedModel); } @Override public synchronized void onItemSelected(AdapterView parent, View view, int pos, long id) { // An item was selected. You can retrieve the selected item using // parent.getItemAtPosition(pos) selectedModel = parent.getItemAtPosition(pos).toString(); Log.d(TAG, "Selected model: " + selectedModel); preview.stop(); createCameraSource(selectedModel); startCameraSource(); } @Override public void onNothingSelected(AdapterView parent) { // Do nothing. } @Override public void onCheckedChanged(CompoundButton buttonView, boolean isChecked) { Log.d(TAG, "Set facing"); if (cameraSource != null) { if (isChecked) { cameraSource.setFacing(CameraSource.CAMERA_FACING_FRONT); } else { cameraSource.setFacing(CameraSource.CAMERA_FACING_BACK); } } preview.stop(); startCameraSource(); } private void createCameraSource(String model) { // If there's no existing cameraSource, create one. if (cameraSource == null) { cameraSource = new CameraSource(this, graphicOverlay); } String autoMLRemoteModelName = PreferenceUtils.getAutoMLRemoteModelName(this); try { switch (model) { case CUSTOM_AUTOML_LABELING: Log.i(TAG, "Create Custom AutoML Image Label Processor"); CustomRemoteModel customRemoteModel = new CustomRemoteModel.Builder( new FirebaseModelSource.Builder(autoMLRemoteModelName).build()) .build(); CustomImageLabelerOptions customImageLabelerOptions = new CustomImageLabelerOptions.Builder(customRemoteModel) .setConfidenceThreshold(0) .build(); cameraSource.setMachineLearningFrameProcessor( new AutoMLImageLabelerProcessor( this, customRemoteModel, customImageLabelerOptions, Mode.LIVE_PREVIEW)); break; case CUSTOM_AUTOML_OBJECT_DETECTION: Log.i(TAG, "Using Custom AutoML Object Detector Processor"); CustomRemoteModel customODTRemoteModel = new CustomRemoteModel.Builder( new FirebaseModelSource.Builder(autoMLRemoteModelName).build()) .build(); CustomObjectDetectorOptions customAutoMLODTOptions = new CustomObjectDetectorOptions.Builder(customODTRemoteModel) .setDetectorMode(CustomObjectDetectorOptions.STREAM_MODE) .enableClassification() .setClassificationConfidenceThreshold(0) .setMaxPerObjectLabelCount(1) .build(); cameraSource.setMachineLearningFrameProcessor( new ObjectDetectorProcessor(this, customODTRemoteModel, customAutoMLODTOptions)); break; default: Log.e(TAG, "Unknown model: " + model); } } catch (RuntimeException e) { Log.e(TAG, "Can not create image processor: " + model, e); Toast.makeText( getApplicationContext(), "Can not create image processor: " + e.getMessage(), Toast.LENGTH_LONG) .show(); } } /** * Starts or restarts the camera source, if it exists. If the camera source doesn't exist yet * (e.g., because onResume was called before the camera source was created), this will be called * again when the camera source is created. */ private void startCameraSource() { if (cameraSource != null) { try { if (preview == null) { Log.d(TAG, "resume: Preview is null"); } if (graphicOverlay == null) { Log.d(TAG, "resume: graphOverlay is null"); } preview.start(cameraSource, graphicOverlay); } catch (IOException e) { Log.e(TAG, "Unable to start camera source.", e); cameraSource.release(); cameraSource = null; } } } @Override public void onResume() { super.onResume(); Log.d(TAG, "onResume"); createCameraSource(selectedModel); startCameraSource(); } /** Stops the camera. */ @Override protected void onPause() { super.onPause(); preview.stop(); } @Override public void onDestroy() { super.onDestroy(); if (cameraSource != null) { cameraSource.release(); } } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/ScopedExecutor.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import androidx.annotation.NonNull; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; /** * Wraps an existing executor to provide a {@link #shutdown} method that allows subsequent * cancellation of submitted runnables. */ public class ScopedExecutor implements Executor { private final Executor executor; private final AtomicBoolean shutdown = new AtomicBoolean(); public ScopedExecutor(@NonNull Executor executor) { this.executor = executor; } @Override public void execute(@NonNull Runnable command) { // Return early if this object has been shut down. if (shutdown.get()) { return; } executor.execute( () -> { // Check again in case it has been shut down in the mean time. if (shutdown.get()) { return; } command.run(); }); } /** * After this method is called, no runnables that have been submitted or are subsequently * submitted will start to execute, turning this executor into a no-op. * *

Runnables that have already started to execute will continue. */ public void shutdown() { shutdown.set(true); } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/StillImageActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import static java.lang.Math.max; import android.content.ContentValues; import android.content.Intent; import android.content.res.Configuration; import android.graphics.Bitmap; import android.net.Uri; import android.os.Bundle; import android.provider.MediaStore; import androidx.appcompat.app.AppCompatActivity; import android.util.Log; import android.util.Pair; import android.view.MenuInflater; import android.view.View; import android.view.ViewTreeObserver.OnGlobalLayoutListener; import android.widget.AdapterView; import android.widget.AdapterView.OnItemSelectedListener; import android.widget.ArrayAdapter; import android.widget.ImageView; import android.widget.PopupMenu; import android.widget.Spinner; import android.widget.Toast; import com.google.android.gms.common.annotation.KeepName; import com.google.mlkit.common.model.CustomRemoteModel; import com.google.mlkit.linkfirebase.FirebaseModelSource; import com.google.mlkit.vision.automl.demo.automl.AutoMLImageLabelerProcessor; import com.google.mlkit.vision.automl.demo.automl.AutoMLImageLabelerProcessor.Mode; import com.google.mlkit.vision.automl.demo.object.ObjectDetectorProcessor; import com.google.mlkit.vision.automl.demo.preference.PreferenceUtils; import com.google.mlkit.vision.automl.demo.preference.SettingsActivity; import com.google.mlkit.vision.automl.demo.preference.SettingsActivity.LaunchSource; import com.google.mlkit.vision.label.custom.CustomImageLabelerOptions; import com.google.mlkit.vision.objects.custom.CustomObjectDetectorOptions; import java.io.IOException; import java.util.ArrayList; import java.util.List; /** Activity demonstrating different image detector features with a still image from camera. */ @KeepName public final class StillImageActivity extends AppCompatActivity { private static final String TAG = "StillImageActivity"; private static final String CUSTOM_AUTOML_LABELING = "Custom AutoML Image Labeling"; private static final String CUSTOM_AUTOML_OBJECT_DETECTION = "Custom AutoML Object Detection"; private static final String SIZE_SCREEN = "w:screen"; // Match screen width private static final String SIZE_1024_768 = "w:1024"; // ~1024*768 in a normal ratio private static final String SIZE_640_480 = "w:640"; // ~640*480 in a normal ratio private static final String KEY_IMAGE_URI = "com.google.mlkit.vision.automl.demo.KEY_IMAGE_URI"; private static final String KEY_SELECTED_SIZE = "com.google.mlkit.vision.automl.demo.KEY_SELECTED_SIZE"; private static final int REQUEST_IMAGE_CAPTURE = 1001; private static final int REQUEST_CHOOSE_IMAGE = 1002; private ImageView preview; private GraphicOverlay graphicOverlay; private String selectedMode = CUSTOM_AUTOML_LABELING; private String selectedSize = SIZE_SCREEN; boolean isLandScape; private Uri imageUri; private int imageMaxWidth; private int imageMaxHeight; private VisionImageProcessor imageProcessor; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_still_image); findViewById(R.id.select_image_button) .setOnClickListener( view -> { // Menu for selecting either: a) take new photo b) select from existing PopupMenu popup = new PopupMenu(StillImageActivity.this, view); popup.setOnMenuItemClickListener( menuItem -> { int itemId = menuItem.getItemId(); if (itemId == R.id.select_images_from_local) { startChooseImageIntentForResult(); return true; } else if (itemId == R.id.take_photo_using_camera) { startCameraIntentForResult(); return true; } return false; }); MenuInflater inflater = popup.getMenuInflater(); inflater.inflate(R.menu.camera_button_menu, popup.getMenu()); popup.show(); }); preview = findViewById(R.id.preview); graphicOverlay = findViewById(R.id.graphic_overlay); populateFeatureSelector(); populateSizeSelector(); isLandScape = (getResources().getConfiguration().orientation == Configuration.ORIENTATION_LANDSCAPE); if (savedInstanceState != null) { imageUri = savedInstanceState.getParcelable(KEY_IMAGE_URI); selectedSize = savedInstanceState.getString(KEY_SELECTED_SIZE); } View rootView = findViewById(R.id.root); rootView.getViewTreeObserver().addOnGlobalLayoutListener(new OnGlobalLayoutListener() { @Override public void onGlobalLayout() { rootView.getViewTreeObserver().removeOnGlobalLayoutListener(this); imageMaxWidth = rootView.getWidth(); imageMaxHeight = rootView.getHeight() - findViewById(R.id.control).getHeight(); if (SIZE_SCREEN.equals(selectedSize)) { tryReloadAndDetectInImage(); } } }); ImageView settingsButton = findViewById(R.id.settings_button); settingsButton.setOnClickListener( v -> { Intent intent = new Intent(getApplicationContext(), SettingsActivity.class); intent.putExtra(SettingsActivity.EXTRA_LAUNCH_SOURCE, LaunchSource.STILL_IMAGE); startActivity(intent); }); } @Override public void onResume() { super.onResume(); Log.d(TAG, "onResume"); createImageProcessor(); tryReloadAndDetectInImage(); } private void populateFeatureSelector() { Spinner featureSpinner = findViewById(R.id.feature_selector); List options = new ArrayList<>(); options.add(CUSTOM_AUTOML_LABELING); options.add(CUSTOM_AUTOML_OBJECT_DETECTION); // Creating adapter for featureSpinner ArrayAdapter dataAdapter = new ArrayAdapter<>(this, R.layout.spinner_style, options); // Drop down layout style - list view with radio button dataAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); // attaching data adapter to spinner featureSpinner.setAdapter(dataAdapter); featureSpinner.setOnItemSelectedListener( new OnItemSelectedListener() { @Override public void onItemSelected( AdapterView parentView, View selectedItemView, int pos, long id) { selectedMode = parentView.getItemAtPosition(pos).toString(); createImageProcessor(); tryReloadAndDetectInImage(); } @Override public void onNothingSelected(AdapterView arg0) {} }); } private void populateSizeSelector() { Spinner sizeSpinner = findViewById(R.id.size_selector); List options = new ArrayList<>(); options.add(SIZE_SCREEN); options.add(SIZE_1024_768); options.add(SIZE_640_480); // Creating adapter for featureSpinner ArrayAdapter dataAdapter = new ArrayAdapter<>(this, R.layout.spinner_style, options); // Drop down layout style - list view with radio button dataAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); // attaching data adapter to spinner sizeSpinner.setAdapter(dataAdapter); sizeSpinner.setOnItemSelectedListener( new OnItemSelectedListener() { @Override public void onItemSelected( AdapterView parentView, View selectedItemView, int pos, long id) { selectedSize = parentView.getItemAtPosition(pos).toString(); createImageProcessor(); tryReloadAndDetectInImage(); } @Override public void onNothingSelected(AdapterView arg0) {} }); } @Override public void onSaveInstanceState(Bundle outState) { super.onSaveInstanceState(outState); outState.putParcelable(KEY_IMAGE_URI, imageUri); outState.putString(KEY_SELECTED_SIZE, selectedSize); } private void startCameraIntentForResult() { // Clean up last time's image imageUri = null; preview.setImageBitmap(null); Intent takePictureIntent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE); if (takePictureIntent.resolveActivity(getPackageManager()) != null) { ContentValues values = new ContentValues(); values.put(MediaStore.Images.Media.TITLE, "New Picture"); values.put(MediaStore.Images.Media.DESCRIPTION, "From Camera"); imageUri = getContentResolver().insert(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, values); takePictureIntent.putExtra(MediaStore.EXTRA_OUTPUT, imageUri); startActivityForResult(takePictureIntent, REQUEST_IMAGE_CAPTURE); } } private void startChooseImageIntentForResult() { Intent intent = new Intent(); intent.setType("image/*"); intent.setAction(Intent.ACTION_GET_CONTENT); startActivityForResult(Intent.createChooser(intent, "Select Picture"), REQUEST_CHOOSE_IMAGE); } @Override protected void onActivityResult(int requestCode, int resultCode, Intent data) { if (requestCode == REQUEST_IMAGE_CAPTURE && resultCode == RESULT_OK) { tryReloadAndDetectInImage(); } else if (requestCode == REQUEST_CHOOSE_IMAGE && resultCode == RESULT_OK) { // In this case, imageUri is returned by the chooser, save it. imageUri = data.getData(); tryReloadAndDetectInImage(); } else { super.onActivityResult(requestCode, resultCode, data); } } private void tryReloadAndDetectInImage() { Log.d(TAG, "Try reload and detect image"); try { if (imageUri == null) { return; } if (SIZE_SCREEN.equals(selectedSize) && imageMaxWidth == 0) { // UI layout has not finished yet, will reload once it's ready. return; } Bitmap imageBitmap = BitmapUtils.getBitmapFromContentUri(getContentResolver(), imageUri); if (imageBitmap == null) { return; } // Clear the overlay first graphicOverlay.clear(); // Get the dimensions of the image view Pair targetedSize = getTargetedWidthHeight(); // Determine how much to scale down the image float scaleFactor = max( (float) imageBitmap.getWidth() / (float) targetedSize.first, (float) imageBitmap.getHeight() / (float) targetedSize.second); Bitmap resizedBitmap = Bitmap.createScaledBitmap( imageBitmap, (int) (imageBitmap.getWidth() / scaleFactor), (int) (imageBitmap.getHeight() / scaleFactor), true); preview.setImageBitmap(resizedBitmap); if (imageProcessor != null) { graphicOverlay.setImageSourceInfo( resizedBitmap.getWidth(), resizedBitmap.getHeight(), /* isFlipped= */ false); imageProcessor.processBitmap(resizedBitmap, graphicOverlay); } else { Log.e(TAG, "Null imageProcessor, please check adb logs for imageProcessor creation error"); } } catch (IOException e) { Log.e(TAG, "Error retrieving saved image"); imageUri = null; } } private Pair getTargetedWidthHeight() { int targetWidth; int targetHeight; switch (selectedSize) { case SIZE_SCREEN: targetWidth = imageMaxWidth; targetHeight = imageMaxHeight; break; case SIZE_640_480: targetWidth = isLandScape ? 640 : 480; targetHeight = isLandScape ? 480 : 640; break; case SIZE_1024_768: targetWidth = isLandScape ? 1024 : 768; targetHeight = isLandScape ? 768 : 1024; break; default: throw new IllegalStateException("Unknown size"); } return new Pair<>(targetWidth, targetHeight); } private void createImageProcessor() { try { String autoMLRemoteModelName = PreferenceUtils.getAutoMLRemoteModelName(this); switch (selectedMode) { case CUSTOM_AUTOML_LABELING: Log.i(TAG, "Create Custom AutoML Image Label Processor"); CustomRemoteModel customRemoteModel = new CustomRemoteModel.Builder( new FirebaseModelSource.Builder(autoMLRemoteModelName).build()) .build(); CustomImageLabelerOptions customImageLabelerOptions = new CustomImageLabelerOptions.Builder(customRemoteModel) .setConfidenceThreshold(0) .build(); imageProcessor = new AutoMLImageLabelerProcessor( this, customRemoteModel, customImageLabelerOptions, Mode.STILL_IMAGE); break; case CUSTOM_AUTOML_OBJECT_DETECTION: Log.i(TAG, "Using Custom AutoML Object Detector Processor"); CustomRemoteModel customODTRemoteModel = new CustomRemoteModel.Builder( new FirebaseModelSource.Builder(autoMLRemoteModelName).build()) .build(); CustomObjectDetectorOptions customAutoMLODTOptions = new CustomObjectDetectorOptions.Builder(customODTRemoteModel) .setDetectorMode(CustomObjectDetectorOptions.SINGLE_IMAGE_MODE) .enableClassification() .setClassificationConfidenceThreshold(0) .setMaxPerObjectLabelCount(1) .build(); imageProcessor = new ObjectDetectorProcessor(this, customODTRemoteModel, customAutoMLODTOptions); break; default: Log.e(TAG, "Unknown selectedMode: " + selectedMode); } } catch (RuntimeException e) { Log.e(TAG, "Can not create image processor: " + selectedMode, e); Toast.makeText( getApplicationContext(), "Can not create image processor: " + e.getMessage(), Toast.LENGTH_LONG) .show(); } } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/VisionImageProcessor.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.graphics.Bitmap; import androidx.camera.core.ImageProxy; import com.google.mlkit.common.MlKitException; import java.nio.ByteBuffer; /** An interface to process the images with different vision detectors and custom image models. */ public interface VisionImageProcessor { /** Processes a bitmap image. */ void processBitmap(Bitmap bitmap, GraphicOverlay graphicOverlay); /** Processes ByteBuffer image data, e.g. used for Camera1 live preview case. */ void processByteBuffer( ByteBuffer data, FrameMetadata frameMetadata, GraphicOverlay graphicOverlay) throws MlKitException; /** Processes ImageProxy image data, e.g. used for CameraX live preview case. */ void processImageProxy(ImageProxy image, GraphicOverlay graphicOverlay) throws MlKitException; /** Stops the underlying machine learning model and release resources. */ void stop(); } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/VisionProcessorBase.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo; import android.app.ActivityManager; import android.app.ActivityManager.MemoryInfo; import android.content.Context; import android.graphics.Bitmap; import android.os.Build.VERSION_CODES; import android.os.SystemClock; import android.util.Log; import android.widget.Toast; import androidx.annotation.GuardedBy; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.camera.core.ExperimentalGetImage; import androidx.camera.core.ImageProxy; import com.google.android.gms.tasks.Task; import com.google.android.gms.tasks.TaskExecutors; import com.google.mlkit.vision.automl.demo.preference.PreferenceUtils; import com.google.mlkit.vision.common.InputImage; import java.nio.ByteBuffer; import java.util.Timer; import java.util.TimerTask; /** * Abstract base class for vision frame processors. Subclasses need to implement {@link * #onSuccess(Object, GraphicOverlay)} to define what they want to with the detection results and * {@link #detectInImage(InputImage)} to specify the detector object. * * @param The type of the detected feature. */ public abstract class VisionProcessorBase implements VisionImageProcessor { protected static final String MANUAL_TESTING_LOG = "LogTagForTest"; private static final String TAG = "VisionProcessorBase"; private final ActivityManager activityManager; private final Timer fpsTimer = new Timer(); private final ScopedExecutor executor; private final Toast toast; // Whether this processor is already shut down private boolean isShutdown; // Used to calculate latency, running in the same thread, no sync needed. private int numRuns = 0; private long totalRunMs = 0; private long maxRunMs = 0; private long minRunMs = Long.MAX_VALUE; // Frame count that have been processed so far in an one second interval to calculate FPS. private int frameProcessedInOneSecondInterval = 0; private int framesPerSecond = 0; // To keep the latest images and its metadata. @GuardedBy("this") private ByteBuffer latestImage; @GuardedBy("this") private FrameMetadata latestImageMetaData; // To keep the images and metadata in process. @GuardedBy("this") private ByteBuffer processingImage; @GuardedBy("this") private FrameMetadata processingMetaData; protected VisionProcessorBase(Context context) { activityManager = (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); executor = new ScopedExecutor(TaskExecutors.MAIN_THREAD); fpsTimer.scheduleAtFixedRate( new TimerTask() { @Override public void run() { framesPerSecond = frameProcessedInOneSecondInterval; frameProcessedInOneSecondInterval = 0; } }, /* delay= */ 0, /* period= */ 1000); toast = Toast.makeText(context, "", Toast.LENGTH_LONG); } // -----------------Code for processing single still image---------------------------------------- @Override public void processBitmap(Bitmap bitmap, final GraphicOverlay graphicOverlay) { requestDetectInImage( InputImage.fromBitmap(bitmap, 0), graphicOverlay, /* originalCameraImage= */ null, /* shouldShowFps= */ false); } // -----------------Code for processing live preview frame from Camera1 API----------------------- @Override public synchronized void processByteBuffer( ByteBuffer data, final FrameMetadata frameMetadata, final GraphicOverlay graphicOverlay) { latestImage = data; latestImageMetaData = frameMetadata; if (processingImage == null && processingMetaData == null) { processLatestImage(graphicOverlay); } } private synchronized void processLatestImage(final GraphicOverlay graphicOverlay) { processingImage = latestImage; processingMetaData = latestImageMetaData; latestImage = null; latestImageMetaData = null; if (processingImage != null && processingMetaData != null && !isShutdown) { processImage(processingImage, processingMetaData, graphicOverlay); } } private void processImage( ByteBuffer data, final FrameMetadata frameMetadata, final GraphicOverlay graphicOverlay) { // If live viewport is on (that is the underneath surface view takes care of the camera preview // drawing), skip the unnecessary bitmap creation that used for the manual preview drawing. Bitmap bitmap = PreferenceUtils.isCameraLiveViewportEnabled(graphicOverlay.getContext()) ? null : BitmapUtils.getBitmap(data, frameMetadata); requestDetectInImage( InputImage.fromByteBuffer( data, frameMetadata.getWidth(), frameMetadata.getHeight(), frameMetadata.getRotation(), InputImage.IMAGE_FORMAT_NV21), graphicOverlay, bitmap, /* shouldShowFps= */ true) .addOnSuccessListener(executor, results -> processLatestImage(graphicOverlay)); } // -----------------Code for processing live preview frame from CameraX API----------------------- @Override @RequiresApi(VERSION_CODES.LOLLIPOP) @ExperimentalGetImage public void processImageProxy(ImageProxy image, GraphicOverlay graphicOverlay) { if (isShutdown) { image.close(); return; } Bitmap bitmap = null; if (!PreferenceUtils.isCameraLiveViewportEnabled(graphicOverlay.getContext())) { bitmap = BitmapUtils.getBitmap(image); } requestDetectInImage( InputImage.fromMediaImage(image.getImage(), image.getImageInfo().getRotationDegrees()), graphicOverlay, /* originalCameraImage= */ bitmap, /* shouldShowFps= */ true) // When the image is from CameraX analysis use case, must call image.close() on received // images when finished using them. Otherwise, new images may not be received or the camera // may stall. .addOnCompleteListener(results -> image.close()); } // -----------------Common processing logic------------------------------------------------------- private Task requestDetectInImage( final InputImage image, final GraphicOverlay graphicOverlay, @Nullable final Bitmap originalCameraImage, boolean shouldShowFps) { final long startMs = SystemClock.elapsedRealtime(); return detectInImage(image) .addOnSuccessListener( executor, results -> { long currentLatencyMs = SystemClock.elapsedRealtime() - startMs; numRuns++; frameProcessedInOneSecondInterval++; totalRunMs += currentLatencyMs; maxRunMs = Math.max(currentLatencyMs, maxRunMs); minRunMs = Math.min(currentLatencyMs, minRunMs); // Only log inference info once per second. When frameProcessedInOneSecondInterval is // equal to 1, it means this is the first frame processed during the current second. if (frameProcessedInOneSecondInterval == 1) { Log.d(TAG, "Max latency is: " + maxRunMs); Log.d(TAG, "Min latency is: " + minRunMs); Log.d(TAG, "Num of Runs: " + numRuns + ", Avg latency is: " + totalRunMs / numRuns); MemoryInfo mi = new MemoryInfo(); activityManager.getMemoryInfo(mi); long availableMegs = mi.availMem / 0x100000L; Log.d(TAG, "Memory available in system: " + availableMegs + " MB"); } graphicOverlay.clear(); if (originalCameraImage != null) { graphicOverlay.add(new CameraImageGraphic(graphicOverlay, originalCameraImage)); } VisionProcessorBase.this.onSuccess(results, graphicOverlay); graphicOverlay.add( new InferenceInfoGraphic( graphicOverlay, currentLatencyMs, shouldShowFps ? framesPerSecond : null)); graphicOverlay.postInvalidate(); }) .addOnFailureListener( executor, e -> { graphicOverlay.clear(); graphicOverlay.postInvalidate(); String error = "Failed to process. Error: " + e.getLocalizedMessage(); toast.setText(error + "\nCause: " + e.getCause()); toast.show(); Log.d(TAG, error); e.printStackTrace(); VisionProcessorBase.this.onFailure(e); }); } @Override public void stop() { executor.shutdown(); isShutdown = true; numRuns = 0; totalRunMs = 0; fpsTimer.cancel(); } protected abstract Task detectInImage(InputImage image); protected abstract void onSuccess(@NonNull T results, @NonNull GraphicOverlay graphicOverlay); protected abstract void onFailure(@NonNull Exception e); } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/automl/AutoMLImageLabelerProcessor.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo.automl; import android.content.Context; import android.util.Log; import android.widget.Toast; import androidx.annotation.NonNull; import com.google.android.gms.tasks.Task; import com.google.android.gms.tasks.Tasks; import com.google.mlkit.common.model.DownloadConditions; import com.google.mlkit.common.model.RemoteModel; import com.google.mlkit.common.model.RemoteModelManager; import com.google.mlkit.vision.automl.demo.GraphicOverlay; import com.google.mlkit.vision.automl.demo.VisionProcessorBase; import com.google.mlkit.vision.common.InputImage; import com.google.mlkit.vision.label.ImageLabel; import com.google.mlkit.vision.label.ImageLabeler; import com.google.mlkit.vision.label.ImageLabelerOptionsBase; import com.google.mlkit.vision.label.ImageLabeling; import java.util.ArrayList; import java.util.List; /** AutoML image labeler demo. */ public class AutoMLImageLabelerProcessor extends VisionProcessorBase> { private static final String TAG = "AutoMLProcessor"; private final ImageLabeler imageLabeler; private final Context context; private final Task modelDownloadingTask; private final Mode mode; public AutoMLImageLabelerProcessor( Context context, RemoteModel remoteModel, ImageLabelerOptionsBase options, Mode mode) { super(context); this.mode = mode; this.context = context; imageLabeler = ImageLabeling.getClient(options); DownloadConditions downloadConditions = new DownloadConditions.Builder().requireWifi().build(); modelDownloadingTask = RemoteModelManager.getInstance() .download(remoteModel, downloadConditions) .addOnFailureListener( ignored -> Toast.makeText( context, "Model download failed for AutoMLImageLabelerImpl," + " please check your connection.", Toast.LENGTH_LONG) .show()); } @Override public void stop() { super.stop(); imageLabeler.close(); } @Override protected Task> detectInImage(InputImage image) { if (!modelDownloadingTask.isComplete()) { if (mode == Mode.LIVE_PREVIEW) { Log.i(TAG, "Model download is in progress. Skip detecting image."); return Tasks.forResult(new ArrayList<>()); } else { Log.i(TAG, "Model download is in progress. Waiting..."); return modelDownloadingTask.continueWithTask(task -> processImageOnDownloadComplete(image)); } } else { return processImageOnDownloadComplete(image); } } private Task> processImageOnDownloadComplete(InputImage image) { if (modelDownloadingTask != null && modelDownloadingTask.isSuccessful()) { if (imageLabeler == null) { Log.e(TAG, "image labeler has not been initialized; Skipped."); Toast.makeText(context, "no initialized Labeler.", Toast.LENGTH_SHORT).show(); } return imageLabeler.process(image); } else { String downloadingError = "Error downloading remote model."; Log.e(TAG, downloadingError, modelDownloadingTask.getException()); Toast.makeText(context, downloadingError, Toast.LENGTH_SHORT).show(); return Tasks.forException( new Exception("Failed to download remote model.", modelDownloadingTask.getException())); } } @Override protected void onSuccess( @NonNull List labels, @NonNull GraphicOverlay graphicOverlay) { graphicOverlay.add(new LabelGraphic(graphicOverlay, labels)); } @Override protected void onFailure(@NonNull Exception e) { Log.w(TAG, "Label detection failed.", e); } /** * The detection mode of the processor. Different modes will have different behavior on whether or * not waiting for the model download complete. */ public enum Mode { STILL_IMAGE, LIVE_PREVIEW } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/automl/LabelGraphic.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo.automl; import static com.google.common.primitives.Floats.max; import static java.lang.Math.max; import android.graphics.Canvas; import android.graphics.Color; import android.graphics.Paint; import com.google.mlkit.vision.automl.demo.GraphicOverlay; import com.google.mlkit.vision.label.ImageLabel; import java.util.List; import java.util.Locale; /** Graphic instance for rendering a label within an associated graphic overlay view. */ public class LabelGraphic extends GraphicOverlay.Graphic { private static final float TEXT_SIZE = 70.0f; private static final String LABEL_FORMAT = "%.2f%% confidence (index: %d)"; private final Paint textPaint; private final Paint labelPaint; private final GraphicOverlay overlay; private final List labels; public LabelGraphic(GraphicOverlay overlay, List labels) { super(overlay); this.overlay = overlay; this.labels = labels; textPaint = new Paint(); textPaint.setColor(Color.WHITE); textPaint.setTextSize(TEXT_SIZE); labelPaint = new Paint(); labelPaint.setColor(Color.BLACK); labelPaint.setStyle(Paint.Style.FILL); labelPaint.setAlpha(200); } @Override public synchronized void draw(Canvas canvas) { // First try to find maxWidth and totalHeight in order to draw to the center of the screen. float maxWidth = 0; float totalHeight = labels.size() * 2 * TEXT_SIZE; for (ImageLabel label : labels) { float line1Width = textPaint.measureText(label.getText()); float line2Width = textPaint.measureText( String.format(Locale.US, LABEL_FORMAT, label.getConfidence() * 100, label.getIndex())); maxWidth = max(maxWidth, line1Width, line2Width); } float x = max(0, overlay.getWidth() / 2.0f - maxWidth / 2.0f); float y = max(200, overlay.getHeight() / 2.0f - totalHeight / 2.0f); if (!labels.isEmpty()) { float padding = 20; canvas.drawRect(x - padding, y - padding, x + maxWidth + padding, y + totalHeight + padding, labelPaint); } for (ImageLabel label : labels) { if (y + TEXT_SIZE * 2 > overlay.getHeight()) { break; } canvas.drawText(label.getText(), x, y + TEXT_SIZE, textPaint); y += TEXT_SIZE; canvas.drawText( String.format(Locale.US, LABEL_FORMAT, label.getConfidence() * 100, label.getIndex()), x, y + TEXT_SIZE, textPaint); y += TEXT_SIZE; } } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/object/ObjectDetectorProcessor.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo.object; import android.content.Context; import android.util.Log; import android.widget.Toast; import androidx.annotation.NonNull; import com.google.android.gms.tasks.Task; import com.google.android.gms.tasks.Tasks; import com.google.mlkit.common.model.DownloadConditions; import com.google.mlkit.common.model.RemoteModel; import com.google.mlkit.common.model.RemoteModelManager; import com.google.mlkit.vision.automl.demo.GraphicOverlay; import com.google.mlkit.vision.automl.demo.VisionProcessorBase; import com.google.mlkit.vision.common.InputImage; import com.google.mlkit.vision.objects.DetectedObject; import com.google.mlkit.vision.objects.ObjectDetection; import com.google.mlkit.vision.objects.ObjectDetector; import com.google.mlkit.vision.objects.ObjectDetectorOptionsBase; import java.util.ArrayList; import java.util.List; /** A processor to run object detector. */ public class ObjectDetectorProcessor extends VisionProcessorBase> { private static final String TAG = "ObjectDetectorProcessor"; private final ObjectDetector detector; private final Context context; private final Task modelDownloadingTask; private final int detectorMode; public ObjectDetectorProcessor( Context context, RemoteModel remoteModel, ObjectDetectorOptionsBase options) { super(context); this.detectorMode = options.getDetectorMode(); this.context = context; detector = ObjectDetection.getClient(options); DownloadConditions downloadConditions = new DownloadConditions.Builder().requireWifi().build(); modelDownloadingTask = RemoteModelManager.getInstance() .download(remoteModel, downloadConditions) .addOnFailureListener( ignored -> Toast.makeText( context, "Model download failed, please check your connection.", Toast.LENGTH_LONG) .show()); } @Override public void stop() { super.stop(); detector.close(); } @Override protected Task> detectInImage(InputImage image) { if (!modelDownloadingTask.isComplete()) { if (detectorMode == ObjectDetectorOptionsBase.STREAM_MODE) { Log.i(TAG, "Model download is in progress. Skip detecting image."); return Tasks.forResult(new ArrayList<>()); } else { Log.i(TAG, "Model download is in progress. Waiting..."); return modelDownloadingTask.continueWithTask(task -> processImageOnDownloadComplete(image)); } } else { return processImageOnDownloadComplete(image); } } private Task> processImageOnDownloadComplete(InputImage image) { if (modelDownloadingTask != null && modelDownloadingTask.isSuccessful()) { if (detector == null) { Log.e(TAG, "object detector has not been initialized; Skipped."); Toast.makeText(context, "no initialized Detector.", Toast.LENGTH_SHORT).show(); } return detector.process(image); } else { String downloadingError = "Error downloading remote model."; Log.e(TAG, downloadingError, modelDownloadingTask.getException()); Toast.makeText(context, downloadingError, Toast.LENGTH_SHORT).show(); return Tasks.forException( new Exception("Failed to download remote model.", modelDownloadingTask.getException())); } } @Override protected void onSuccess( @NonNull List results, @NonNull GraphicOverlay graphicOverlay) { for (DetectedObject object : results) { graphicOverlay.add(new ObjectGraphic(graphicOverlay, object)); } } @Override protected void onFailure(@NonNull Exception e) { Log.e(TAG, "Object detection failed!", e); } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/object/ObjectGraphic.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo.object; import android.graphics.Canvas; import android.graphics.Color; import android.graphics.Paint; import android.graphics.RectF; import com.google.mlkit.vision.automl.demo.GraphicOverlay; import com.google.mlkit.vision.automl.demo.GraphicOverlay.Graphic; import com.google.mlkit.vision.objects.DetectedObject; import com.google.mlkit.vision.objects.DetectedObject.Label; import java.util.Locale; /** Draw the detected object info in preview. */ public class ObjectGraphic extends Graphic { private static final float TEXT_SIZE = 54.0f; private static final float STROKE_WIDTH = 4.0f; private static final int NUM_COLORS = 10; private static final int[][] COLORS = new int[][] { // {Text color, background color} {Color.BLACK, Color.WHITE}, {Color.WHITE, Color.MAGENTA}, {Color.BLACK, Color.LTGRAY}, {Color.WHITE, Color.RED}, {Color.WHITE, Color.BLUE}, {Color.WHITE, Color.DKGRAY}, {Color.BLACK, Color.CYAN}, {Color.BLACK, Color.YELLOW}, {Color.WHITE, Color.BLACK}, {Color.BLACK, Color.GREEN} }; private static final String LABEL_FORMAT = "%.2f%% confidence (index: %d)"; private final DetectedObject object; private final Paint[] boxPaints; private final Paint[] textPaints; private final Paint[] labelPaints; ObjectGraphic(GraphicOverlay overlay, DetectedObject object) { super(overlay); this.object = object; int numColors = COLORS.length; textPaints = new Paint[numColors]; boxPaints = new Paint[numColors]; labelPaints = new Paint[numColors]; for (int i = 0; i < numColors; i++) { textPaints[i] = new Paint(); textPaints[i].setColor(COLORS[i][0] /* text color */); textPaints[i].setTextSize(TEXT_SIZE); boxPaints[i] = new Paint(); boxPaints[i].setColor(COLORS[i][1] /* background color */); boxPaints[i].setStyle(Paint.Style.STROKE); boxPaints[i].setStrokeWidth(STROKE_WIDTH); labelPaints[i] = new Paint(); labelPaints[i].setColor(COLORS[i][1] /* background color */); labelPaints[i].setStyle(Paint.Style.FILL); } } @Override public void draw(Canvas canvas) { // Decide color based on object tracking ID int colorID = object.getTrackingId() == null ? 0 : Math.abs(object.getTrackingId() % NUM_COLORS); float textWidth = textPaints[colorID].measureText("Tracking ID: " + object.getTrackingId()); float lineHeight = TEXT_SIZE + STROKE_WIDTH; float yLabelOffset = -lineHeight; // Calculate width and height of label box for (Label label : object.getLabels()) { textWidth = Math.max(textWidth, textPaints[colorID].measureText(label.getText())); textWidth = Math.max( textWidth, textPaints[colorID].measureText( String.format( Locale.US, LABEL_FORMAT, label.getConfidence() * 100, label.getIndex()))); yLabelOffset -= 2 * lineHeight; } // Draws the bounding box. RectF rect = new RectF(object.getBoundingBox()); // If the image is flipped, the left will be translated to right, and the right to left. float x0 = translateX(rect.left); float x1 = translateX(rect.right); rect.left = Math.min(x0, x1); rect.right = Math.max(x0, x1); rect.top = translateY(rect.top); rect.bottom = translateY(rect.bottom); canvas.drawRect(rect, boxPaints[colorID]); // Draws other object info. canvas.drawRect( rect.left - STROKE_WIDTH, rect.top + yLabelOffset, rect.left + textWidth + (2 * STROKE_WIDTH), rect.top, labelPaints[colorID]); yLabelOffset += TEXT_SIZE; canvas.drawText( "Tracking ID: " + object.getTrackingId(), rect.left, rect.top + yLabelOffset, textPaints[colorID]); yLabelOffset += lineHeight; for (Label label : object.getLabels()) { canvas.drawText(label.getText(), rect.left, rect.top + yLabelOffset, textPaints[colorID]); yLabelOffset += lineHeight; canvas.drawText( String.format(Locale.US, LABEL_FORMAT, label.getConfidence() * 100, label.getIndex()), rect.left, rect.top + yLabelOffset, textPaints[colorID]); yLabelOffset += lineHeight; } } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/preference/CameraXLivePreviewPreferenceFragment.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo.preference; import android.content.Context; import android.graphics.SurfaceTexture; import android.hardware.camera2.CameraAccessException; import android.hardware.camera2.CameraCharacteristics; import android.hardware.camera2.CameraManager; import android.hardware.camera2.params.StreamConfigurationMap; import android.os.Build.VERSION_CODES; import android.preference.ListPreference; import android.preference.PreferenceCategory; import android.util.Size; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.annotation.StringRes; import androidx.camera.core.CameraSelector; import com.google.mlkit.vision.automl.demo.R; import java.util.Arrays; import java.util.List; /** Configures CameraX live preview demo settings. */ @RequiresApi(VERSION_CODES.LOLLIPOP) public class CameraXLivePreviewPreferenceFragment extends LivePreviewPreferenceFragment { @Override void setUpCameraPreferences() { PreferenceCategory cameraPreference = (PreferenceCategory) findPreference(getString(R.string.pref_category_key_camera)); cameraPreference.removePreference( findPreference(getString(R.string.pref_key_rear_camera_preview_size))); cameraPreference.removePreference( findPreference(getString(R.string.pref_key_front_camera_preview_size))); setUpCameraXTargetAnalysisSizePreference( R.string.pref_key_camerax_rear_camera_target_resolution, CameraSelector.LENS_FACING_BACK); setUpCameraXTargetAnalysisSizePreference( R.string.pref_key_camerax_front_camera_target_resolution, CameraSelector.LENS_FACING_FRONT); } private void setUpCameraXTargetAnalysisSizePreference( @StringRes int previewSizePrefKeyId, int lensFacing) { ListPreference pref = (ListPreference) findPreference(getString(previewSizePrefKeyId)); CameraCharacteristics cameraCharacteristics = getCameraCharacteristics(getActivity(), lensFacing); String[] entries; if (cameraCharacteristics != null) { StreamConfigurationMap map = cameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); Size[] outputSizes = map.getOutputSizes(SurfaceTexture.class); entries = new String[outputSizes.length]; for (int i = 0; i < outputSizes.length; i++) { entries[i] = outputSizes[i].toString(); } } else { entries = new String[] { "2000x2000", "1600x1600", "1200x1200", "1000x1000", "800x800", "600x600", "400x400", "200x200", "100x100", }; } pref.setEntries(entries); pref.setEntryValues(entries); pref.setSummary(pref.getEntry() == null ? "Default" : pref.getEntry()); pref.setOnPreferenceChangeListener( (preference, newValue) -> { String newStringValue = (String) newValue; pref.setSummary(newStringValue); PreferenceUtils.saveString(getActivity(), previewSizePrefKeyId, newStringValue); return true; }); } @Nullable public static CameraCharacteristics getCameraCharacteristics( Context context, Integer lensFacing) { CameraManager cameraManager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE); try { List cameraList = Arrays.asList(cameraManager.getCameraIdList()); for (String availableCameraId : cameraList) { CameraCharacteristics availableCameraCharacteristics = cameraManager.getCameraCharacteristics(availableCameraId); Integer availableLensFacing = availableCameraCharacteristics.get(CameraCharacteristics.LENS_FACING); if (availableLensFacing == null) { continue; } if (availableLensFacing.equals(lensFacing)) { return availableCameraCharacteristics; } } } catch (CameraAccessException e) { // Accessing camera ID info got error } return null; } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/preference/LivePreviewPreferenceFragment.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo.preference; import android.hardware.Camera; import android.os.Bundle; import android.preference.ListPreference; import android.preference.PreferenceCategory; import android.preference.PreferenceFragment; import androidx.annotation.StringRes; import com.google.mlkit.vision.automl.demo.CameraSource; import com.google.mlkit.vision.automl.demo.CameraSource.SizePair; import com.google.mlkit.vision.automl.demo.R; import java.util.HashMap; import java.util.List; import java.util.Map; /** Configures live preview demo settings. */ public class LivePreviewPreferenceFragment extends PreferenceFragment { @Override public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); addPreferencesFromResource(R.xml.preference_live_preview_automl); setUpCameraPreferences(); PreferenceUtils.setUpRemoteModelNamePreferences(this); } void setUpCameraPreferences() { PreferenceCategory cameraPreference = (PreferenceCategory) findPreference(getString(R.string.pref_category_key_camera)); cameraPreference.removePreference( findPreference(getString(R.string.pref_key_camerax_rear_camera_target_resolution))); cameraPreference.removePreference( findPreference(getString(R.string.pref_key_camerax_front_camera_target_resolution))); setUpCameraPreviewSizePreference( R.string.pref_key_rear_camera_preview_size, R.string.pref_key_rear_camera_picture_size, CameraSource.CAMERA_FACING_BACK); setUpCameraPreviewSizePreference( R.string.pref_key_front_camera_preview_size, R.string.pref_key_front_camera_picture_size, CameraSource.CAMERA_FACING_FRONT); } private void setUpCameraPreviewSizePreference( @StringRes int previewSizePrefKeyId, @StringRes int pictureSizePrefKeyId, int cameraId) { ListPreference previewSizePreference = (ListPreference) findPreference(getString(previewSizePrefKeyId)); Camera camera = null; try { camera = Camera.open(cameraId); List previewSizeList = CameraSource.generateValidPreviewSizeList(camera); String[] previewSizeStringValues = new String[previewSizeList.size()]; Map previewToPictureSizeStringMap = new HashMap<>(); for (int i = 0; i < previewSizeList.size(); i++) { SizePair sizePair = previewSizeList.get(i); previewSizeStringValues[i] = sizePair.preview.toString(); if (sizePair.picture != null) { previewToPictureSizeStringMap.put( sizePair.preview.toString(), sizePair.picture.toString()); } } previewSizePreference.setEntries(previewSizeStringValues); previewSizePreference.setEntryValues(previewSizeStringValues); if (previewSizePreference.getEntry() == null) { // First time of opening the Settings page. SizePair sizePair = CameraSource.selectSizePair( camera, CameraSource.DEFAULT_REQUESTED_CAMERA_PREVIEW_WIDTH, CameraSource.DEFAULT_REQUESTED_CAMERA_PREVIEW_HEIGHT); String previewSizeString = sizePair.preview.toString(); previewSizePreference.setValue(previewSizeString); previewSizePreference.setSummary(previewSizeString); PreferenceUtils.saveString( getActivity(), pictureSizePrefKeyId, sizePair.picture != null ? sizePair.picture.toString() : null); } else { previewSizePreference.setSummary(previewSizePreference.getEntry()); } previewSizePreference.setOnPreferenceChangeListener( (preference, newValue) -> { String newPreviewSizeStringValue = (String) newValue; previewSizePreference.setSummary(newPreviewSizeStringValue); PreferenceUtils.saveString( getActivity(), pictureSizePrefKeyId, previewToPictureSizeStringMap.get(newPreviewSizeStringValue)); return true; }); } catch (RuntimeException e) { // If there's no camera for the given camera id, hide the corresponding preference. ((PreferenceCategory) findPreference(getString(R.string.pref_category_key_camera))) .removePreference(previewSizePreference); } finally { if (camera != null) { camera.release(); } } } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/preference/PreferenceUtils.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo.preference; import android.content.Context; import android.content.SharedPreferences; import android.os.Build.VERSION_CODES; import android.preference.EditTextPreference; import android.preference.PreferenceFragment; import android.preference.PreferenceManager; import android.widget.Toast; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.annotation.StringRes; import androidx.camera.core.CameraSelector; import com.google.android.gms.common.images.Size; import com.google.common.base.Preconditions; import com.google.mlkit.vision.automl.demo.CameraSource; import com.google.mlkit.vision.automl.demo.CameraSource.SizePair; import com.google.mlkit.vision.automl.demo.R; /** Utility class to retrieve shared preferences. */ public final class PreferenceUtils { static void saveString(Context context, @StringRes int prefKeyId, @Nullable String value) { PreferenceManager.getDefaultSharedPreferences(context) .edit() .putString(context.getString(prefKeyId), value) .apply(); } @Nullable public static SizePair getCameraPreviewSizePair(Context context, int cameraId) { Preconditions.checkArgument( cameraId == CameraSource.CAMERA_FACING_BACK || cameraId == CameraSource.CAMERA_FACING_FRONT); String previewSizePrefKey; String pictureSizePrefKey; if (cameraId == CameraSource.CAMERA_FACING_BACK) { previewSizePrefKey = context.getString(R.string.pref_key_rear_camera_preview_size); pictureSizePrefKey = context.getString(R.string.pref_key_rear_camera_picture_size); } else { previewSizePrefKey = context.getString(R.string.pref_key_front_camera_preview_size); pictureSizePrefKey = context.getString(R.string.pref_key_front_camera_picture_size); } try { SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); return new SizePair( Size.parseSize(sharedPreferences.getString(previewSizePrefKey, null)), Size.parseSize(sharedPreferences.getString(pictureSizePrefKey, null))); } catch (RuntimeException e) { return null; } } @RequiresApi(VERSION_CODES.LOLLIPOP) @Nullable public static android.util.Size getCameraXTargetResolution(Context context, int lensfacing) { Preconditions.checkArgument( lensfacing == CameraSelector.LENS_FACING_BACK || lensfacing == CameraSelector.LENS_FACING_FRONT); String prefKey = lensfacing == CameraSelector.LENS_FACING_BACK ? context.getString(R.string.pref_key_camerax_rear_camera_target_resolution) : context.getString(R.string.pref_key_camerax_front_camera_target_resolution); SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); try { return android.util.Size.parseSize(sharedPreferences.getString(prefKey, null)); } catch (RuntimeException e) { return null; } } public static boolean isCameraLiveViewportEnabled(Context context) { SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); String prefKey = context.getString(R.string.pref_key_camera_live_viewport); return sharedPreferences.getBoolean(prefKey, false); } public static String getAutoMLRemoteModelName(Context context) { SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); String modelNamePrefKey = context.getString(R.string.pref_key_automl_remote_model_name); String defaultModelName = "mlkit_flowers"; String remoteModelName = sharedPreferences.getString(modelNamePrefKey, defaultModelName); if (remoteModelName.isEmpty()) { remoteModelName = defaultModelName; } return remoteModelName; } public static void setUpRemoteModelNamePreferences(PreferenceFragment preferenceFragment) { EditTextPreference autoMLRemoteModelNamePref = (EditTextPreference) preferenceFragment.findPreference( preferenceFragment.getString(R.string.pref_key_automl_remote_model_name)); autoMLRemoteModelNamePref.setSummary(autoMLRemoteModelNamePref.getText()); autoMLRemoteModelNamePref.setOnPreferenceChangeListener( (preference, newValue) -> { String modelName = (String) newValue; if (!modelName.isEmpty()) { autoMLRemoteModelNamePref.setSummary((String) newValue); return true; } Toast.makeText( preferenceFragment.getActivity(), R.string.pref_key_automl_remote_model_name, Toast.LENGTH_LONG) .show(); return false; }); } private PreferenceUtils() {} } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/preference/SettingsActivity.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo.preference; import android.os.Bundle; import android.preference.PreferenceFragment; import androidx.appcompat.app.ActionBar; import androidx.appcompat.app.AppCompatActivity; import com.google.mlkit.vision.automl.demo.R; /** * Hosts the preference fragment to configure settings for a demo activity that specified by the * {@link LaunchSource}. */ public class SettingsActivity extends AppCompatActivity { public static final String EXTRA_LAUNCH_SOURCE = "extra_launch_source"; /** Specifies where this activity is launched from. */ @SuppressWarnings("NewApi") // CameraX is only available on API 21+ public enum LaunchSource { LIVE_PREVIEW(R.string.pref_screen_title_live_preview, LivePreviewPreferenceFragment.class), STILL_IMAGE(R.string.pref_screen_title_still_image, StillImagePreferenceFragment.class), CAMERAX_LIVE_PREVIEW( R.string.pref_screen_title_camerax_live_preview, CameraXLivePreviewPreferenceFragment.class); private final int titleResId; private final Class prefFragmentClass; LaunchSource(int titleResId, Class prefFragmentClass) { this.titleResId = titleResId; this.prefFragmentClass = prefFragmentClass; } } @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_settings); LaunchSource launchSource = (LaunchSource) getIntent().getSerializableExtra(EXTRA_LAUNCH_SOURCE); ActionBar actionBar = getSupportActionBar(); if (actionBar != null) { actionBar.setTitle(launchSource.titleResId); } try { getFragmentManager() .beginTransaction() .replace( R.id.settings_container, launchSource.prefFragmentClass.getDeclaredConstructor().newInstance()) .commit(); } catch (Exception e) { throw new RuntimeException(e); } } } ================================================ FILE: android/automl/app/src/main/java/com/google/mlkit/vision/automl/demo/preference/StillImagePreferenceFragment.java ================================================ /* * Copyright 2020 Google LLC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.mlkit.vision.automl.demo.preference; import android.os.Bundle; import android.preference.PreferenceFragment; import com.google.mlkit.vision.automl.demo.R; /** Configures still image demo settings. */ public class StillImagePreferenceFragment extends PreferenceFragment { @Override public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); addPreferencesFromResource(R.xml.preference_still_image); PreferenceUtils.setUpRemoteModelNamePreferences(this); } } ================================================ FILE: android/automl/app/src/main/res/drawable/logo_mlkit.xml ================================================ ================================================ FILE: android/automl/app/src/main/res/drawable-hdpi/ic_switch_camera_white_48dp.xml ================================================ ================================================ FILE: android/automl/app/src/main/res/drawable-mdpi/ic_switch_camera_white_48dp.xml ================================================ ================================================ FILE: android/automl/app/src/main/res/drawable-xhdpi/ic_switch_camera_white_48dp.xml ================================================ ================================================ FILE: android/automl/app/src/main/res/drawable-xxhdpi/ic_switch_camera_white_48dp.xml ================================================ ================================================ FILE: android/automl/app/src/main/res/drawable-xxxhdpi/ic_switch_camera_white_48dp.xml ================================================ ================================================ FILE: android/automl/app/src/main/res/layout/activity_camerax_live_preview.xml ================================================ ================================================ FILE: android/automl/app/src/main/res/layout/activity_chooser.xml ================================================ ================================================ FILE: android/automl/app/src/main/res/layout/activity_live_preview.xml ================================================ ================================================ FILE: android/automl/app/src/main/res/layout/activity_settings.xml ================================================ ================================================ FILE: android/automl/app/src/main/res/layout/activity_still_image.xml ================================================