[ { "path": ".codecov.yml", "content": "codecov:\n notify:\n require_ci_to_pass: yes\n\ncoverage:\n precision: 2\n round: down\n range: \"30...100\"\n\n status:\n project: yes\n patch: yes\n changes: no\n\nparsers:\n gcov:\n branch_detection:\n conditional: yes\n loop: yes\n method: no\n macro: no\n\ncomment:\n layout: \"header, diff\"\n behavior: default\n require_changes: no\n" }, { "path": ".github/FUNDING.yml", "content": "github: OlafenwaMoses" }, { "path": ".github/workflows/build.yml", "content": "name: Build and Testing\n\non:\n push:\n branches: [master]\n pull_request:\n branches: [master]\n\njobs:\n UnitestPython37:\n \n name: Python3.7 Tests\n runs-on: ubuntu-latest\n # needs: None\n steps:\n - uses: actions/checkout@v3\n - uses: actions/setup-python@v4\n with:\n python-version: '3.7'\n cache: 'pip'\n - name: Install Dependencies\n run: |\n pip install -r requirements.txt\n pip install -r requirements_extra.txt\n - name: Download and Setup Resources\n env:\n CI: false\n run: |\n sudo apt-get update\n sudo apt-get install unzip -y\n\n mkdir test/data-models\n mkdir test/data-json\n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/densenet121-a639ec97.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/densenet121-idenprof-test_acc_0.82550_epoch-95.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/inception_v3-idenprof-test_acc_0.81050_epoch-92.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/inception_v3_google-1a9a5a14.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/mobilenet_v2-b0353104.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/mobilenet_v2-idenprof-test_acc_0.85300_epoch-92.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-19c8e357.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-idenprof-test_acc_0.78200_epoch-91.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/retinanet_resnet50_fpn_coco-eeacb38b.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/tiny-yolov3.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/tiny_yolov3_number-plate-dataset-imageai_mAP-0.22595_epoch-20.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_number-plate-dataset-imageai_mAP-0.57145_epoch-11.pt -P test/data-models\n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/idenprof.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/number-plate-dataset-imageai_tiny_yolov3_detection_config.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/number-plate-dataset-imageai_yolov3_detection_config.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/idenprof_model_classes.json -P test/data-json\n \n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-datasets.zip -P test\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-images.zip -P test\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-videos.zip -P test\n\n unzip test/data-datasets.zip -d test\n unzip test/data-images.zip -d test\n unzip test/data-videos.zip -d test\n - name: Run Unittest\n run: |\n pytest test -vvv \n UnitestPython38:\n \n name: Python3.8 Tests\n runs-on: ubuntu-latest\n # needs: None\n steps:\n - uses: actions/checkout@v3\n - uses: actions/setup-python@v4\n with:\n python-version: '3.8'\n cache: 'pip'\n - name: Install Dependencies\n run: |\n pip install -r requirements.txt\n pip install -r requirements_extra.txt\n - name: Download and Setup Resources\n env:\n CI: false\n run: |\n sudo apt-get update\n sudo apt-get install unzip -y\n\n mkdir test/data-models\n mkdir test/data-json\n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/densenet121-a639ec97.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/densenet121-idenprof-test_acc_0.82550_epoch-95.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/inception_v3-idenprof-test_acc_0.81050_epoch-92.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/inception_v3_google-1a9a5a14.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/mobilenet_v2-b0353104.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/mobilenet_v2-idenprof-test_acc_0.85300_epoch-92.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-19c8e357.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-idenprof-test_acc_0.78200_epoch-91.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/retinanet_resnet50_fpn_coco-eeacb38b.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/tiny-yolov3.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/tiny_yolov3_number-plate-dataset-imageai_mAP-0.22595_epoch-20.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_number-plate-dataset-imageai_mAP-0.57145_epoch-11.pt -P test/data-models\n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/idenprof.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/number-plate-dataset-imageai_tiny_yolov3_detection_config.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/number-plate-dataset-imageai_yolov3_detection_config.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/idenprof_model_classes.json -P test/data-json\n \n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-datasets.zip -P test\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-images.zip -P test\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-videos.zip -P test\n\n unzip test/data-datasets.zip -d test\n unzip test/data-images.zip -d test\n unzip test/data-videos.zip -d test\n - name: Run Unittest\n run: |\n pytest test -vvv\n \n UnitestPython39:\n \n name: Python3.9 Tests\n runs-on: ubuntu-latest\n # needs: None\n steps:\n - uses: actions/checkout@v3\n - uses: actions/setup-python@v4\n with:\n python-version: '3.9'\n cache: 'pip'\n - name: Install Dependencies\n run: |\n pip install -r requirements.txt\n pip install -r requirements_extra.txt\n - name: Download and Setup Resources\n env:\n CI: false\n run: |\n sudo apt-get update\n sudo apt-get install unzip -y\n\n mkdir test/data-models\n mkdir test/data-json\n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/densenet121-a639ec97.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/densenet121-idenprof-test_acc_0.82550_epoch-95.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/inception_v3-idenprof-test_acc_0.81050_epoch-92.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/inception_v3_google-1a9a5a14.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/mobilenet_v2-b0353104.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/mobilenet_v2-idenprof-test_acc_0.85300_epoch-92.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-19c8e357.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-idenprof-test_acc_0.78200_epoch-91.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/retinanet_resnet50_fpn_coco-eeacb38b.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/tiny-yolov3.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/tiny_yolov3_number-plate-dataset-imageai_mAP-0.22595_epoch-20.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_number-plate-dataset-imageai_mAP-0.57145_epoch-11.pt -P test/data-models\n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/idenprof.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/number-plate-dataset-imageai_tiny_yolov3_detection_config.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/number-plate-dataset-imageai_yolov3_detection_config.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/idenprof_model_classes.json -P test/data-json\n \n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-datasets.zip -P test\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-images.zip -P test\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-videos.zip -P test\n\n unzip test/data-datasets.zip -d test\n unzip test/data-images.zip -d test\n unzip test/data-videos.zip -d test\n - name: Run Unittest\n run: |\n pytest test -vvv\n \n UnitestPython310:\n \n name: Python3.10 Tests\n runs-on: ubuntu-latest\n # needs: None\n steps:\n - uses: actions/checkout@v3\n - uses: actions/setup-python@v4\n with:\n python-version: '3.10'\n cache: 'pip'\n - name: Install Dependencies\n run: |\n pip install -r requirements.txt\n pip install -r requirements_extra.txt\n - name: Download and Setup Resources\n env:\n CI: false\n run: |\n sudo apt-get update\n sudo apt-get install unzip -y\n\n mkdir test/data-models\n mkdir test/data-json\n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/densenet121-a639ec97.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/densenet121-idenprof-test_acc_0.82550_epoch-95.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/inception_v3-idenprof-test_acc_0.81050_epoch-92.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/inception_v3_google-1a9a5a14.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/mobilenet_v2-b0353104.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/mobilenet_v2-idenprof-test_acc_0.85300_epoch-92.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-19c8e357.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-idenprof-test_acc_0.78200_epoch-91.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/retinanet_resnet50_fpn_coco-eeacb38b.pth -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/tiny-yolov3.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/tiny_yolov3_number-plate-dataset-imageai_mAP-0.22595_epoch-20.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt -P test/data-models\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_number-plate-dataset-imageai_mAP-0.57145_epoch-11.pt -P test/data-models\n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/idenprof.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/number-plate-dataset-imageai_tiny_yolov3_detection_config.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/number-plate-dataset-imageai_yolov3_detection_config.json -P test/data-json\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/idenprof_model_classes.json -P test/data-json\n \n\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-datasets.zip -P test\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-images.zip -P test\n wget https://github.com/OlafenwaMoses/ImageAI/releases/download/test-resources-v3/data-videos.zip -P test\n\n unzip test/data-datasets.zip -d test\n unzip test/data-images.zip -d test\n unzip test/data-videos.zip -d test\n - name: Run Unittest\n run: |\n pytest test -vvv\n " }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\ncover/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\n.pybuilder/\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n# For a library or package, you might want to ignore these files since the code is\n# intended to run in multiple environments; otherwise, check them in:\n# .python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# poetry\n# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.\n# This is especially recommended for binary packages to ensure reproducibility, and is more\n# commonly ignored for libraries.\n# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control\n#poetry.lock\n\n# pdm\n# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.\n#pdm.lock\n# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it\n# in version control.\n# https://pdm.fming.dev/#use-with-ide\n.pdm.toml\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# pytype static type analyzer\n.pytype/\n\n# Cython debug symbols\ncython_debug/\n\n# PyCharm\n# JetBrains specific template is maintained in a separate JetBrains.gitignore that can\n# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore\n# and can be added to the global gitignore or merged into this file. For a more nuclear\n# option (not recommended) you can uncomment the following to ignore the entire idea folder.\n#.idea/\n\n\n# Other files and folders\ntest/data-models\ntest/data-images\ntest/data-json\ntest/data-videos\ntest/data-datasets\nexperiment" }, { "path": ".travis.yml", "content": "dist: xenial\nsudo: required\nlanguage: python\npython:\n - '3.7.6'\ninstall:\n - pip install -r requirements.txt\n - pip install pytest\n - pip install pytest-cov\nscript:\n - python setup.py install\n - cd test\n - mkdir data-models\n - mkdir data-temp\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/DenseNet-BC-121-32.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/inception_v3_weights_tf_dim_ordering_tf_kernels.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/essentials-v5/resnet50_imagenet_tf.2.0.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/essentials-v5/mobilenet_v2.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/models-v3/idenprof_densenet-0.763500.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/models-v3/idenprof_full_resnet_ex-001_acc-0.119792.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/essentials-v5/idenprof_resnet_ex-056_acc-0.993062.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/essentials-v5/resnet50_coco_best_v2.1.0.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/yolo.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/yolo-tiny.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/essential-v4/pretrained-yolov3.h5\n - wget -P data-models/ https://github.com/OlafenwaMoses/ImageAI/releases/download/essential-v4/hololens-ex-60--loss-2.76.h5\n - pytest -v --cov\nafter_script:\n - bash <(curl -s https://codecov.io/bash)\n\n\n\n\n" }, { "path": "BACKEND_MIGRATION.md", "content": "# Overview\n\nIn December 2022, ImageAI `3.0.2` was released which effected the change from Tensorflow backend to PyTorch backend. This change allows ImageAI to support `Python 3.7` up to `Python 3.10` for all its features and deprecates a number of functionalities for this and future versions of ImageAI.\n\n\n# Deprecated functionalities\n- Tensorflow backend no longer supported. Now replaced with PyTorch\n- All `.h5` pretrained models and custom trained `.h5` models no longer supported. If you still intend to use these models, see the `Using Tensorflow backend` section.\n- `Speed mode` have been removed from model loading\n- Custom detection model training dataset format changed to YOLO format from Pascal VOC. To convert your dataset to YOLO format, see the `Convert Pascal VOC dataset to YOLO format` section.\n- Enhance data for custom classification model training now removed\n- Detection model training standalone evaluation now removed\n\n# Using Tensorflow backend\nTo use Tensorflow backend, do the following\n\n- Install Python 3.7\n- Install Tensorflow \n - CPU: `pip install tensorflow==2.4.0`\n - GPU: `pip install tensorflow-gpu==2.4.0`\n- Install other dependencies: `pip install keras==2.4.3 numpy==1.19.3 pillow==7.0.0 scipy==1.4.1 h5py==2.10.0 matplotlib==3.3.2 opencv-python keras-resnet==0.2.0`\n- Install ImageAI **2.1.6**: `pip install imageai==2.1.6`\n- Download the Tensorflow models from the releases below\n - [Models for Image Recognition and Object Detection](https://github.com/OlafenwaMoses/ImageAI/releases/tag/1.0)\n - [TF2.x Models [ Exclusives ]](https://github.com/OlafenwaMoses/ImageAI/releases/tag/essentials-v5)\n\n\n\n# Convert Pascal VOC dataset to YOLO format\nBecause ImageAI now uses `YOLO format` for training custom object detection models; should you need to train a new model with the new ImageAI version, you will need to convert your `Pascal VOC` datasets to YOLO format by doing the following \n- Run the command below\n ```\n python scripts/pascal_voc_to_yolo.py --dataset_dir \n ```\n- Once completed, you will find the YOLO version of the dataset next to your Pascal VOC dataset.\n - E.g, if your dataset is in `C:/Users/Troublemaker/Documents/datasets/headset`, your conversion command will be\n ```\n python scripts/pascal_voc_to_yolo.py --dataset_dir C:/Users/Troublemaker/Documents/datasets/headset\n ```\n and once completed, the output will be in `C:/Users/Troublemaker/Documents/datasets/headset-yolo`\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2019 MOSES OLAFENWA\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "MANIFEST.in", "content": "recursive-include imageai/Detection *.txt\nrecursive-include imageai/Classification *.txt" }, { "path": "README.md", "content": "# ImageAI (v3.0.3)\n\n\n\n[![Build Status](https://travis-ci.com/OlafenwaMoses/ImageAI.svg?branch=master)](https://travis-ci.com/OlafenwaMoses/ImageAI) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/OlafenwaMoses/ImageAI/blob/master/LICENSE) [![PyPI version](https://badge.fury.io/py/imageai.svg)](https://badge.fury.io/py/imageai) [![Downloads](https://pepy.tech/badge/imageai/month)](https://pepy.tech/project/imageai) [![Downloads](https://pepy.tech/badge/imageai/week)](https://pepy.tech/project/imageai)\n\nAn open-source python library built to empower developers to build applications and systems with self-contained Deep Learning and Computer Vision capabilities using simple and few lines of code.\n \n If you will like to sponsor this project, kindly visit the [Github sponsor page](https://github.com/sponsors/OlafenwaMoses).\n \n \n## ---------------------------------------------------\n## Introducing Jarvis and TheiaEngine.\n\nWe the creators of ImageAI are glad to announce 2 new AI projects to provide state-of-the-art Generative AI, LLM and Image Understanding on your personal computer and servers. \n\n\n[![](jarvis.png)](https://jarvis.genxr.co)\n\nInstall Jarvis on PC/Mac to setup limitless access to LLM powered AI Chats for your every day work, research and generative AI needs with 100% privacy and full offline capability.\n\n\nVisit [https://jarvis.genxr.co](https://jarvis.genxr.co/) to get started.\n\n\n[![](theiaengine.png)](https://www.genxr.co/theia-engine)\n\n\n[TheiaEngine](https://www.genxr.co/theia-engine), the next-generation computer Vision AI API capable of all Generative and Understanding computer vision tasks in a single API call and available via REST API to all programming languages. Features include\n- **Detect 300+ objects** ( 220 more objects than ImageAI)\n- **Provide answers to any content or context questions** asked on an image\n - very useful to get information on any object, action or information without needing to train a new custom model for every tasks\n- **Generate scene description and summary**\n- **Convert 2D image to 3D pointcloud and triangular mesh**\n- **Semantic Scene mapping of objects, walls, floors, etc**\n- **Stateless Face recognition and emotion detection**\n- **Image generation and augmentation from prompt**\n- etc.\n\nVisit [https://www.genxr.co/theia-engine](https://www.genxr.co/theia-engine) to try the demo and join in the beta testing today.\n## ---------------------------------------------------\n \n![](logo1.png)\n\nDeveloped and maintained by [Moses Olafenwa](https://twitter.com/OlafenwaMoses)\n\n---\n\nBuilt with simplicity in mind, **ImageAI** \n supports a list of state-of-the-art Machine Learning algorithms for image prediction, custom image prediction, object detection, video detection, video object tracking\n and image predictions trainings. **ImageAI** currently supports image prediction and training using 4 different Machine Learning algorithms \n trained on the ImageNet-1000 dataset. **ImageAI** also supports object detection, video detection and object tracking using RetinaNet, YOLOv3 and TinyYOLOv3 trained on COCO dataset. Finally, **ImageAI** allows you to train custom models for performing detection and recognition of new objects. \n \nEventually, **ImageAI** will provide support for a wider and more specialized aspects of Computer Vision\n\n\n**New Release : ImageAI 3.0.2**\n\nWhat's new:\n- PyTorch backend\n- TinyYOLOv3 model training\n\n\n### TABLE OF CONTENTS\n- :white_square_button: Installation\n- :white_square_button: Features\n- :white_square_button: Documentation\n- :white_square_button: Sponsors\n- :white_square_button: Projects Built on ImageAI\n- :white_square_button: High Performance Implementation\n- :white_square_button: AI Practice Recommendations\n- :white_square_button: Contact Developers\n- :white_square_button: Citation\n- :white_square_button: References\n\n\n\n## Installation\n
\n \nTo install ImageAI, run the python installation instruction below in the command line:\n\n- [Download and Install](https://www.python.org/downloads/) **Python 3.7**, **Python 3.8**, **Python 3.9** or **Python 3.10**\n- Install dependencies\n - **CPU**: Download [requirements.txt](https://github.com/OlafenwaMoses/ImageAI/blob/master/requirements.txt) file and install via the command\n ```\n pip install -r requirements.txt\n ```\n or simply copy and run the command below\n\n ```\n pip install cython pillow>=7.0.0 numpy>=1.18.1 opencv-python>=4.1.2 torch>=1.9.0 --extra-index-url https://download.pytorch.org/whl/cpu torchvision>=0.10.0 --extra-index-url https://download.pytorch.org/whl/cpu pytest==7.1.3 tqdm==4.64.1 scipy>=1.7.3 matplotlib>=3.4.3 mock==4.0.3\n ```\n\n - **GPU/CUDA**: Download [requirements_gpu.txt](https://github.com/OlafenwaMoses/ImageAI/blob/master/requirements_gpu.txt) file and install via the command\n ```\n pip install -r requirements_gpu.txt\n ```\n or smiply copy and run the command below\n ```\n pip install cython pillow>=7.0.0 numpy>=1.18.1 opencv-python>=4.1.2 torch>=1.9.0 --extra-index-url https://download.pytorch.org/whl/cu102 torchvision>=0.10.0 --extra-index-url https://download.pytorch.org/whl/cu102 pytest==7.1.3 tqdm==4.64.1 scipy>=1.7.3 matplotlib>=3.4.3 mock==4.0.3\n ```\n- If you plan to train custom AI models, download [requirements_extra.txt](https://github.com/OlafenwaMoses/ImageAI/blob/master/requirements_extra.txt) file and install via the command\n \n ```\n pip install -r requirements_extra.txt\n ```\n or simply copy and run the command below\n ```\n pip install pycocotools@git+https://github.com/gautamchitnis/cocoapi.git@cocodataset-master#subdirectory=PythonAPI\n ```\n- Then run the command below to install ImageAI\n ```\n pip install imageai --upgrade\n ```\n\n## Features\n
\n\n \n \n \n \n \n \n \n

Image Classification

\n

ImageAI provides 4 different algorithms and model types to perform image prediction, trained on the ImageNet-1000 dataset. The 4 algorithms provided for image prediction include MobileNetV2, ResNet50, InceptionV3 and DenseNet121.\n Click the link below to see the full sample codes, explanations and best practices guide.

\n >>> Get Started\n
\n\n
\n\n \n \n \n \n \n \n \n

Object Detection

\n \n

ImageAI provides very convenient and powerful methods to perform object detection on images and extract each object from the image. The object detection class provides support for RetinaNet, YOLOv3 and TinyYOLOv3, with options to adjust for state of the art performance or real time processing. Click the link below to see the full sample codes, explanations and best practices guide.

\n >>> Get Started\n
\n\n\n\n \n \n \n \n \n \n \n

Video Object Detection & Analysis

\n

ImageAI provides very convenient and powerful methods to perform object detection in videos. The video object detection class provided only supports the current state-of-the-art RetinaNet. Click the link to see the full videos, sample codes, explanations and best practices guide.

\n >>> Get Started\n
\n\n\n \n \n \n \n \n \n \n \n

Custom Classification model training

\n \n

ImageAI provides classes and methods for you to train a new model that can be used to perform prediction on your own custom objects. You can train your custom models using MobileNetV2, ResNet50, InceptionV3 and DenseNet in 5 lines of code. Click the link below to see the guide to preparing training images, sample training codes, explanations and best practices.

\n >>> Get Started\n
\n\n \n \n \n \n \n \n \n \n

Custom Model Classification

\n

ImageAI provides classes and methods for you to run image prediction your own custom objects using your own model trained with ImageAI Model Training class. You can use your custom models trained with MobileNetV2, ResNet50, InceptionV3 and DenseNet and the JSON file containing the mapping of the custom object names. Click the link below to see the guide to sample training codes, explanations, and best practices guide.

\n >>> Get Started\n
\n\n \n \n \n \n \n \n \n \n

Custom Detection Model Training

\n \n

ImageAI provides classes and methods for you to train new YOLOv3 or TinyYOLOv3 object detection models on your custom dataset. This means you can train a model to detect literally any object of interest by providing the images, the annotations and training with ImageAI. Click the link below to see the guide to sample training codes, explanations, and best practices guide.

\n >>> Get Started\n
\n\n\n \n \n \n \n \n \n

Custom Object Detection

\n

ImageAI now provides classes and methods for you detect and recognize your own custom objects in images using your own model trained with the DetectionModelTrainer class. You can use your custom trained YOLOv3 or TinyYOLOv3 model and the **.json** file generated during the training. Click the link below to see the guide to sample training codes, explanations, and best practices guide.

\n >>> Get Started\n
\n\n\n\n \n \n \n \n \n \n

Custom Video Object Detection & Analysis

\n \n

ImageAI now provides classes and methods for you detect and recognize your own custom objects in images using your own model trained with the DetectionModelTrainer class. You can use your custom trained YOLOv3 or TinyYOLOv3 model and the **.json** file generated during the training. Click the link below to see the guide to sample training codes, explanations, and best practices guide.

\n >>> Get Started\n
\n\n## Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions. Visit the link below:\n\n- Documentation - **English Version** [https://imageai.readthedocs.io](https://imageai.readthedocs.io)\n\n\n## Sponsors\n
\n\n\n## Real-Time and High Performance Implementation\n
\n\n**ImageAI** provides abstracted and convenient implementations of state-of-the-art Computer Vision technologies. All of **ImageAI** implementations and code can work on any computer system with moderate CPU capacity. However, the speed of processing for operations like image prediction, object detection and others on CPU is slow and not suitable for real-time applications. To perform real-time Computer Vision operations with high performance, you need to use GPU enabled technologies.\n\n**ImageAI** uses the PyTorch backbone for it's Computer Vision operations. PyTorch supports both CPUs and GPUs ( Specifically NVIDIA GPUs. You can get one for your PC or get a PC that has one) for machine learning and artificial intelligence algorithms' implementations.\n\n\n\n## Projects Built on ImageAI\n
\n\n\n\n## AI Practice Recommendations\n
\n\nFor anyone interested in building AI systems and using them for business, economic, social and research purposes, it is critical that the person knows the likely positive, negative and unprecedented impacts the use of such technologies will have.\nThey must also be aware of approaches and practices recommended by experienced industry experts to ensure every use of AI brings overall benefit to mankind.\nWe therefore recommend to everyone that wishes to use ImageAI and other AI tools and resources to read Microsoft's January 2018 publication on AI titled \"The Future Computed : Artificial Intelligence and its role in society\".\nKindly follow the link below to download the publication.\n\n[https://blogs.microsoft.com/blog/2018/01/17/future-computed-artificial-intelligence-role-society](https://blogs.microsoft.com/blog/2018/01/17/future-computed-artificial-intelligence-role-society/)\n\n### Contact Developer\n
\n\n- **Moses Olafenwa**\n * _Email:_ guymodscientist@gmail.com\n * _Twitter:_ [@OlafenwaMoses](https://twitter.com/OlafenwaMoses)\n * _Medium:_ [@guymodscientist](https://medium.com/@guymodscientist)\n * _Facebook:_ [moses.olafenwa](https://facebook.com/moses.olafenwa)\n- **John Olafenwa**\n * _Email:_ johnolafenwa@gmail.com\n * _Website:_ [https://john.aicommons.science](https://john.aicommons.science)\n * _Twitter:_ [@johnolafenwa](https://twitter.com/johnolafenwa)\n * _Medium:_ [@johnolafenwa](https://medium.com/@johnolafenwa)\n * _Facebook:_ [olafenwajohn](https://facebook.com/olafenwajohn)\n\n\n### Citation\n
\n\nYou can cite **ImageAI** in your projects and research papers via the **BibTeX** entry below. \n \n```\n@misc {ImageAI,\n author = \"Moses\",\n title = \"ImageAI, an open source python library built to empower developers to build applications and systems with self-contained Computer Vision capabilities\",\n url = \"https://github.com/OlafenwaMoses/ImageAI\",\n month = \"mar\",\n year = \"2018--\"\n}\n```\n\n\n\n ### References\n
\n\n 1. Somshubra Majumdar, DenseNet Implementation of the paper, Densely Connected Convolutional Networks in Keras\n[https://github.com/titu1994/DenseNet](https://github.com/titu1994/DenseNet)\n 2. Broad Institute of MIT and Harvard, Keras package for deep residual networks\n[https://github.com/broadinstitute/keras-resnet](https://github.com/broadinstitute/keras-resnet)\n 3. Fizyr, Keras implementation of RetinaNet object detection\n[https://github.com/fizyr/keras-retinanet](https://github.com/fizyr/keras-retinanet)\n 4. Francois Chollet, Keras code and weights files for popular deeplearning models\n[https://github.com/fchollet/deep-learning-models](https://github.com/fchollet/deep-learning-models)\n 5. Forrest N. et al, SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and <0.5MB model size\n[https://arxiv.org/abs/1602.07360](https://arxiv.org/abs/1602.07360)\n 6. Kaiming H. et al, Deep Residual Learning for Image Recognition\n[https://arxiv.org/abs/1512.03385](https://arxiv.org/abs/1512.03385)\n 7. Szegedy. et al, Rethinking the Inception Architecture for Computer Vision\n[https://arxiv.org/abs/1512.00567](https://arxiv.org/abs/1512.00567)\n 8. Gao. et al, Densely Connected Convolutional Networks\n[https://arxiv.org/abs/1608.06993](https://arxiv.org/abs/1608.06993)\n 9. Tsung-Yi. et al, Focal Loss for Dense Object Detection\n[https://arxiv.org/abs/1708.02002](https://arxiv.org/abs/1708.02002)\n 10. O Russakovsky et al, ImageNet Large Scale Visual Recognition Challenge\n[https://arxiv.org/abs/1409.0575](https://arxiv.org/abs/1409.0575)\n 11. TY Lin et al, Microsoft COCO: Common Objects in Context\n[https://arxiv.org/abs/1405.0312](https://arxiv.org/abs/1405.0312)\n 12. Moses & John Olafenwa, A collection of images of identifiable professionals.\n[https://github.com/OlafenwaMoses/IdenProf](https://github.com/OlafenwaMoses/IdenProf)\n 13. Joseph Redmon and Ali Farhadi, YOLOv3: An Incremental Improvement.\n[https://arxiv.org/abs/1804.02767](https://arxiv.org/abs/1804.02767)\n 14. Experiencor, Training and Detecting Objects with YOLO3\n[https://github.com/experiencor/keras-yolo3](https://github.com/experiencor/keras-yolo3)\n 15. MobileNetV2: Inverted Residuals and Linear Bottlenecks\n[https://arxiv.org/abs/1801.04381](https://arxiv.org/abs/1801.04381)\n 16. YOLOv3 in PyTorch > ONNX > CoreML > TFLite [https://github.com/ultralytics/yolov3](https://github.com/ultralytics/yolov3)\n" }, { "path": "examples/camera_feed_detection.py", "content": "from imageai.Detection import VideoObjectDetection\nimport os\nimport cv2\n\nexecution_path = os.getcwd()\n\ncamera = cv2.VideoCapture(0)\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(os.path.join(execution_path , \"yolov3.pt\")) # Download the model via this link https://github.com/OlafenwaMoses/ImageAI/releases/tag/1.0\ndetector.loadModel()\n\nvideo_path = detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"camera_detected_video\")\n , frames_per_second=20, log_progress=True, minimum_percentage_probability=30)\nprint(video_path)" }, { "path": "examples/custom_detection.py", "content": "from imageai.Detection.Custom import CustomObjectDetection\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\ndetector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/hololens-yolo_yolov3_detection_config.json\ndetector.loadModel()\ndetections = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", output_image_path=\"holo2-detected.jpg\")\nfor detection in detections:\n print(detection[\"name\"], \" : \", detection[\"percentage_probability\"], \" : \", detection[\"box_points\"])\n\n\n\"\"\"\nEXAMPLE RESULT\n\nhololens : 39.69653248786926 : [611, 74, 751, 154]\nhololens : 87.6643180847168 : [23, 46, 90, 79]\nhololens : 89.25175070762634 : [191, 66, 243, 95]\nhololens : 64.49641585350037 : [437, 81, 514, 133]\nhololens : 91.78624749183655 : [380, 113, 423, 138]\n\n\"\"\"" }, { "path": "examples/custom_detection_array_input_output.py", "content": "from imageai.Detection.Custom import CustomObjectDetection\nimport cv2\n\nimage_array = cv2.imread(\"holo2.jpg\")\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\ndetector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/hololens-yolo_yolov3_detection_config.json\ndetector.loadModel()\ndetected_image, detections = detector.detectObjectsFromImage(input_image=image_array, input_type=\"array\", output_type=\"array\")\n\nfor eachObject in detections:\n print(eachObject[\"name\"], \" : \", eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"])\n\ncv2.imshow(\"Main Image\", detected_image)\ncv2.waitKey()\ncv2.destroyAllWindows()\n\n\n\"\"\"\nSAMPLE RESULT\n\nhololens : 39.69653248786926 : [611, 74, 751, 154]\nhololens : 87.6643180847168 : [23, 46, 90, 79]\nhololens : 89.25175070762634 : [191, 66, 243, 95]\nhololens : 64.49641585350037 : [437, 81, 514, 133]\nhololens : 91.78624749183655 : [380, 113, 423, 138]\n\"\"\"" }, { "path": "examples/custom_detection_extract_objects.py", "content": "from imageai.Detection.Custom import CustomObjectDetection\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\ndetector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/hololens-yolo_yolov3_detection_config.json\ndetector.loadModel()\ndetections, extracted_objects_array = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", output_image_path=\"holo2-detected.jpg\", extract_detected_objects=True)\n\nfor detection, object_path in zip(detections, extracted_objects_array):\n print(object_path)\n print(detection[\"name\"], \" : \", detection[\"percentage_probability\"], \" : \", detection[\"box_points\"])\n print(\"---------------\")\n\n\"\"\"\nSAMPLE RESULT\n\nholo2-detected-objects\\hololens-1.jpg\nhololens : 39.69653248786926 : [611, 74, 751, 154]\n---------------\n\nholo2-detected-objects\\hololens-1.jpg\nhololens : 87.6643180847168 : [23, 46, 90, 79]\n---------------\n\nholo2-detected-objects\\hololens-1.jpg\nhololens : 89.25175070762634 : [191, 66, 243, 95]\n---------------\n\nholo2-detected-objects\\hololens-1.jpg\nhololens : 64.49641585350037 : [437, 81, 514, 133]\n---------------\n\nholo2-detected-objects\\hololens-1.jpg\nhololens : 91.78624749183655 : [380, 113, 423, 138]\n---------------\n\"\"\"" }, { "path": "examples/custom_detection_from_array_extract_objects_array.py", "content": "from imageai.Detection.Custom import CustomObjectDetection\nimport cv2\n\nimage_array = cv2.imread(\"holo2.jpg\")\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\ndetector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/hololens-yolo_yolov3_detection_config.json\ndetector.loadModel()\ndetected_image, detections, extracted_objects = detector.detectObjectsFromImage(input_image=image_array, extract_detected_objects=True, input_type=\"array\", output_type=\"array\")\n\n\nfor eachObject in detections:\n print(eachObject[\"name\"], \" : \", eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"])\n\n\ncv2.imshow(\"Main Image\", detected_image)\ncount = 0\nfor img in extracted_objects:\n count += 1\n\n cv2.imshow(\"Window\" + str(count), img)\n\ncv2.waitKey()\ncv2.destroyAllWindows()\n\n\n\"\"\"\nSAMPLE RESULT\n\nhololens : 39.69653248786926 : [611, 74, 751, 154]\nhololens : 87.6643180847168 : [23, 46, 90, 79]\nhololens : 89.25175070762634 : [191, 66, 243, 95]\nhololens : 64.49641585350037 : [437, 81, 514, 133]\nhololens : 91.78624749183655 : [380, 113, 423, 138]\n\"\"\"" }, { "path": "examples/custom_detection_from_file_extract_objects_array.py", "content": "from imageai.Detection.Custom import CustomObjectDetection\nimport cv2\n\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\ndetector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/hololens-yolo_yolov3_detection_config.json\ndetector.loadModel()\ndetected_image, detections, extracted_objects = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", extract_detected_objects=True, output_type=\"array\")\n\n\nfor eachObject in detections:\n print(eachObject[\"name\"], \" : \", eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"])\n\ncv2.imshow(\"Main Image\", detected_image)\ncount = 0\nfor img in extracted_objects:\n count += 1\n\n cv2.imshow(\"Window\" + str(count), img)\n\ncv2.waitKey()\ncv2.destroyAllWindows()\n\n\n\"\"\"\nSAMPLE RESULT\n\nhololens : 39.69653248786926 : [611, 74, 751, 154]\nhololens : 87.6643180847168 : [23, 46, 90, 79]\nhololens : 89.25175070762634 : [191, 66, 243, 95]\nhololens : 64.49641585350037 : [437, 81, 514, 133]\nhololens : 91.78624749183655 : [380, 113, 423, 138]\n\n\"\"\"" }, { "path": "examples/custom_detection_train.py", "content": "from imageai.Detection.Custom import DetectionModelTrainer\n\ntrainer = DetectionModelTrainer()\ntrainer.setModelTypeAsYOLOv3()\ntrainer.setDataDirectory(data_directory=\"hololens\")\ntrainer.setTrainConfig(object_names_array=[\"hololens\"], batch_size=4, num_experiments=200, train_from_pretrained_model=\"yolov3.pt\")\n#download pre-trained model via https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt\n# If you are training to detect more than 1 object, set names of objects above like object_names_array=[\"hololens\", \"google-glass\", \"oculus\", \"magic-leap\"]\ntrainer.trainModel()\n\n\n\n\"\"\"\nSAMPLE RESULT\n\nGenerating anchor boxes for training images...\nthr=0.25: 1.0000 best possible recall, 6.93 anchors past thr\nn=9, img_size=416, metric_all=0.463/0.856-mean/best, past_thr=0.549-mean:\n====================\nPretrained YOLOv3 model loaded to initialize weights\n====================\nEpoch 1/100\n----------\nTrain:\n30it [00:14, 2.09it/s]\n box loss-> 0.09820, object loss-> 0.27985, class loss-> 0.00000\nValidation:\n15it [01:45, 7.05s/it]\n recall: 0.085714 precision: 0.000364 mAP@0.5: 0.000186, mAP@0.5-0.95: 0.000030\n\nEpoch 2/100\n----------\nTrain:\n30it [00:07, 4.25it/s]\n box loss-> 0.08691, object loss-> 0.07011, class loss-> 0.00000\nValidation:\n15it [01:37, 6.53s/it]\n recall: 0.214286 precision: 0.000854 mAP@0.5: 0.000516, mAP@0.5-0.95: 0.000111\n\"\"\"\n\n" }, { "path": "examples/custom_detection_video.py", "content": "from imageai.Detection.Custom import CustomVideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\nvideo_detector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\") # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/hololens-yolo_yolov3_detection_config.json\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(input_file_path=\"holo1.mp4\",\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20,\n minimum_percentage_probability=40,\n log_progress=True)" }, { "path": "examples/custom_model_prediction.py", "content": "from imageai.Classification.Custom import CustomImageClassification\nimport os\n\nexecution_path = os.getcwd()\n\nprediction = CustomImageClassification()\nprediction.setModelTypeAsResNet50()\nprediction.setModelPath(os.path.join(execution_path, \"resnet50-idenprof-test_acc_0.78200_epoch-91.pt\")) # Download the model via this link https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-idenprof-test_acc_0.78200_epoch-91.pt\nprediction.setJsonPath(os.path.join(execution_path, \"idenprof_model_classes.json\")) # Download from here https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/idenprof_model_classes.json\nprediction.loadModel(num_objects=10)\n\npredictions, probabilities = prediction.classifyImage(os.path.join(execution_path, \"9.jpg\"), result_count=5)\n\nfor eachPrediction, eachProbability in zip(predictions, probabilities):\n print(eachPrediction , \" : \" , eachProbability)" }, { "path": "examples/custom_model_training.py", "content": "from imageai.Classification.Custom import ClassificationModelTrainer\n\n\nmodel_trainer = ClassificationModelTrainer()\nmodel_trainer.setModelTypeAsResNet50()\nmodel_trainer.setDataDirectory(\"idenprof\")\nmodel_trainer.trainModel(num_experiments=200, batch_size=32)\n" }, { "path": "examples/image_custom_object_detection.py", "content": "from imageai.Detection import ObjectDetection\nimport os\nfrom time import time\n\nexecution_path = os.getcwd()\n\ndetector = ObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"yolov3.pt\")) # Download the model via this link https://github.com/OlafenwaMoses/ImageAI/releases/tag/1.0\ndetector.loadModel()\n\nour_time = time()\n\ncustom = detector.CustomObjects(bicycle=True, backpack=True)\n\ndetections = detector.detectCustomObjectsFromImage( custom_objects=custom, input_image=os.path.join(execution_path , \"7.jpg\"), output_image_path=os.path.join(execution_path , \"7-detected.jpg\"), minimum_percentage_probability=40)\nfor eachObject in detections:\n print(eachObject[\"name\"] , \" : \" , eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"--------------------------------\")\n" }, { "path": "examples/image_prediction.py", "content": "from imageai.Classification import ImageClassification\nimport os\n\nexecution_path = os.getcwd()\n\nprediction = ImageClassification()\nprediction.setModelTypeAsResNet50()\nprediction.setModelPath(os.path.join(execution_path, \"resnet50-19c8e357.pth\")) # Download the model via this link https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-19c8e357.pth\nprediction.loadModel()\n\npredictions, probabilities = prediction.classifyImage(os.path.join(execution_path, \"1.jpg\"), result_count=10)\nfor eachPrediction, eachProbability in zip(predictions, probabilities):\n print(eachPrediction , \" : \" , eachProbability)" }, { "path": "examples/object_detection.py", "content": "from imageai.Detection import ObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = ObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath( os.path.join(execution_path , \"retinanet_resnet50_fpn_coco-eeacb38b.pth\")) # Download the model via this link https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/retinanet_resnet50_fpn_coco-eeacb38b.pth\ndetector.loadModel()\ndetections = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"2.jpg\"), output_image_path=os.path.join(execution_path , \"2_detected.jpg\"), minimum_percentage_probability=40)\n\nfor eachObject in detections:\n print(eachObject[\"name\"] , \" : \", eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"--------------------------------\")" }, { "path": "examples/video_analysis_per_frame.py", "content": "from imageai.Detection import VideoObjectDetection\nimport os\nfrom matplotlib import pyplot as plt\n\n\nexecution_path = os.getcwd()\n\ncolor_index = {'bus': 'red', 'handbag': 'steelblue', 'giraffe': 'orange', 'spoon': 'gray', 'cup': 'yellow', 'chair': 'green', 'elephant': 'pink', 'truck': 'indigo', 'motorcycle': 'azure', 'refrigerator': 'gold', 'keyboard': 'violet', 'cow': 'magenta', 'mouse': 'crimson', 'sports ball': 'raspberry', 'horse': 'maroon', 'cat': 'orchid', 'boat': 'slateblue', 'hot dog': 'navy', 'apple': 'cobalt', 'parking meter': 'aliceblue', 'sandwich': 'skyblue', 'skis': 'deepskyblue', 'microwave': 'peacock', 'knife': 'cadetblue', 'baseball bat': 'cyan', 'oven': 'lightcyan', 'carrot': 'coldgrey', 'scissors': 'seagreen', 'sheep': 'deepgreen', 'toothbrush': 'cobaltgreen', 'fire hydrant': 'limegreen', 'remote': 'forestgreen', 'bicycle': 'olivedrab', 'toilet': 'ivory', 'tv': 'khaki', 'skateboard': 'palegoldenrod', 'train': 'cornsilk', 'zebra': 'wheat', 'tie': 'burlywood', 'orange': 'melon', 'bird': 'bisque', 'dining table': 'chocolate', 'hair drier': 'sandybrown', 'cell phone': 'sienna', 'sink': 'coral', 'bench': 'salmon', 'bottle': 'brown', 'car': 'silver', 'bowl': 'maroon', 'tennis racket': 'palevilotered', 'airplane': 'lavenderblush', 'pizza': 'hotpink', 'umbrella': 'deeppink', 'bear': 'plum', 'fork': 'purple', 'laptop': 'indigo', 'vase': 'mediumpurple', 'baseball glove': 'slateblue', 'traffic light': 'mediumblue', 'bed': 'navy', 'broccoli': 'royalblue', 'backpack': 'slategray', 'snowboard': 'skyblue', 'kite': 'cadetblue', 'teddy bear': 'peacock', 'clock': 'lightcyan', 'wine glass': 'teal', 'frisbee': 'aquamarine', 'donut': 'mincream', 'suitcase': 'seagreen', 'dog': 'springgreen', 'banana': 'emeraldgreen', 'person': 'honeydew', 'surfboard': 'palegreen', 'cake': 'sapgreen', 'book': 'lawngreen', 'potted plant': 'greenyellow', 'toaster': 'ivory', 'stop sign': 'beige', 'couch': 'khaki'}\n\n\nresized = False\n\ndef forFrame(frame_number, output_array, output_count, returned_frame):\n\n plt.clf()\n\n this_colors = []\n labels = []\n sizes = []\n\n counter = 0\n\n for eachItem in output_count:\n counter += 1\n labels.append(eachItem + \" = \" + str(output_count[eachItem]))\n sizes.append(output_count[eachItem])\n this_colors.append(color_index[eachItem])\n\n global resized\n\n if (resized == False):\n manager = plt.get_current_fig_manager()\n manager.resize(width=1000, height=500)\n resized = True\n\n plt.subplot(1, 2, 1)\n plt.title(\"Frame : \" + str(frame_number))\n plt.axis(\"off\")\n plt.imshow(returned_frame, interpolation=\"none\")\n\n plt.subplot(1, 2, 2)\n plt.title(\"Analysis: \" + str(frame_number))\n plt.pie(sizes, labels=labels, colors=this_colors, shadow=True, startangle=140, autopct=\"%1.1f%%\")\n\n plt.pause(0.01)\n\n\n\nvideo_detector = VideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(os.path.join(execution_path, \"yolov3.pt\")) # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt\nvideo_detector.loadModel()\n\nplt.show()\n\nvideo_detector.detectObjectsFromVideo(input_file_path=os.path.join(execution_path, \"traffic.mp4\"), output_file_path=os.path.join(execution_path, \"video_frame_analysis\") , frames_per_second=20, per_frame_function=forFrame, minimum_percentage_probability=30, return_detected_frame=True)\n" }, { "path": "examples/video_analysis_per_second.py", "content": "from imageai.Detection import VideoObjectDetection\nimport os\nfrom matplotlib import pyplot as plt\n\n\nexecution_path = os.getcwd()\n\ncolor_index = {'bus': 'red', 'handbag': 'steelblue', 'giraffe': 'orange', 'spoon': 'gray', 'cup': 'yellow', 'chair': 'green', 'elephant': 'pink', 'truck': 'indigo', 'motorcycle': 'azure', 'refrigerator': 'gold', 'keyboard': 'violet', 'cow': 'magenta', 'mouse': 'crimson', 'sports ball': 'raspberry', 'horse': 'maroon', 'cat': 'orchid', 'boat': 'slateblue', 'hot dog': 'navy', 'apple': 'cobalt', 'parking meter': 'aliceblue', 'sandwich': 'skyblue', 'skis': 'deepskyblue', 'microwave': 'peacock', 'knife': 'cadetblue', 'baseball bat': 'cyan', 'oven': 'lightcyan', 'carrot': 'coldgrey', 'scissors': 'seagreen', 'sheep': 'deepgreen', 'toothbrush': 'cobaltgreen', 'fire hydrant': 'limegreen', 'remote': 'forestgreen', 'bicycle': 'olivedrab', 'toilet': 'ivory', 'tv': 'khaki', 'skateboard': 'palegoldenrod', 'train': 'cornsilk', 'zebra': 'wheat', 'tie': 'burlywood', 'orange': 'melon', 'bird': 'bisque', 'dining table': 'chocolate', 'hair drier': 'sandybrown', 'cell phone': 'sienna', 'sink': 'coral', 'bench': 'salmon', 'bottle': 'brown', 'car': 'silver', 'bowl': 'maroon', 'tennis racket': 'palevilotered', 'airplane': 'lavenderblush', 'pizza': 'hotpink', 'umbrella': 'deeppink', 'bear': 'plum', 'fork': 'purple', 'laptop': 'indigo', 'vase': 'mediumpurple', 'baseball glove': 'slateblue', 'traffic light': 'mediumblue', 'bed': 'navy', 'broccoli': 'royalblue', 'backpack': 'slategray', 'snowboard': 'skyblue', 'kite': 'cadetblue', 'teddy bear': 'peacock', 'clock': 'lightcyan', 'wine glass': 'teal', 'frisbee': 'aquamarine', 'donut': 'mincream', 'suitcase': 'seagreen', 'dog': 'springgreen', 'banana': 'emeraldgreen', 'person': 'honeydew', 'surfboard': 'palegreen', 'cake': 'sapgreen', 'book': 'lawngreen', 'potted plant': 'greenyellow', 'toaster': 'ivory', 'stop sign': 'beige', 'couch': 'khaki'}\n\n\nresized = False\n\ndef forSecond(frame_number, output_arrays, count_arrays, average_count, returned_frame):\n\n plt.clf()\n\n this_colors = []\n labels = []\n sizes = []\n\n counter = 0\n\n for eachItem in average_count:\n counter += 1\n labels.append(eachItem + \" = \" + str(average_count[eachItem]))\n sizes.append(average_count[eachItem])\n this_colors.append(color_index[eachItem])\n\n global resized\n\n if (resized == False):\n manager = plt.get_current_fig_manager()\n manager.resize(width=1000, height=500)\n resized = True\n\n plt.subplot(1, 2, 1)\n plt.title(\"Second : \" + str(frame_number))\n plt.axis(\"off\")\n plt.imshow(returned_frame, interpolation=\"none\")\n\n plt.subplot(1, 2, 2)\n plt.title(\"Analysis: \" + str(frame_number))\n plt.pie(sizes, labels=labels, colors=this_colors, shadow=True, startangle=140, autopct=\"%1.1f%%\")\n\n plt.pause(0.01)\n\n\n\nvideo_detector = VideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(os.path.join(execution_path, \"yolov3.pt\")) # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt\nvideo_detector.loadModel()\n\nplt.show()\n\nvideo_detector.detectObjectsFromVideo(input_file_path=os.path.join(execution_path, \"traffic.mp4\"), output_file_path=os.path.join(execution_path, \"video_second_analysis\") , frames_per_second=20, per_second_function=forSecond, minimum_percentage_probability=30, return_detected_frame=True, log_progress=True)\n" }, { "path": "examples/video_custom_object_detection.py", "content": "from imageai.Detection import VideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(os.path.join(execution_path, \"yolov3.pt\")) # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt\ndetector.loadModel()\n\ncustom = detector.CustomObjects(person=True, motorcycle=True, bus=True)\n\nvideo_path = detector.detectCustomObjectsFromVideo(custom_objects=custom, input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected_custom\")\n , frames_per_second=20, log_progress=True)\nprint(video_path)" }, { "path": "examples/video_object_detection.py", "content": "from imageai.Detection import VideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(os.path.join(execution_path, \"yolov3.pt\")) # https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt\ndetector.loadModel()\n\nvideo_path = detector.detectObjectsFromVideo(input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\")\n , frames_per_second=20, log_progress=True)\nprint(video_path)" }, { "path": "imageai/Classification/CUSTOMCLASSIFICATION.md", "content": "# ImageAI : Custom Image Classification\n\nImageAI provides 4 different algorithms and model types to perform custom image prediction using your custom models.\nYou will be able to use your model trained with **ImageAI** and the corresponding model_class JSON file to predict custom objects\nthat you have trained the model on.\n\n### TABLE OF CONTENTS\n\n- :white_square_button: Custom Model Prediction\n- :white_square_button: Custom Model Prediction with Full Model (NEW)\n\n### Custom Model Prediction\n
\n\nIn this example, we will be using the model trained for 20 experiments on **IdenProf**, a dataset of uniformed professionals and achieved 65.17% accuracy on the test dataset.\n(You can use your own trained model and generated JSON file. This 'class' is provided mainly for the purpose to use your own custom models.)\nDownload the ResNet model of the model and JSON files in links below:\n\n- [**ResNet50**](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-idenprof-test_acc_0.78200_epoch-91.pt) _(Size = 90.4 mb)_\n- [**idenprof_model_class.json file**](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/idenprof_model_classes.json)\n\nGreat!\nOnce you have downloaded this model file and the JSON file, start a new python project, and then copy the model file and the JSON file to your project folder where your python files (.py files) will be.\nDownload the image below, or take any image on your computer that include any of the following professionals(Chef, Doctor, Engineer, Farmer, Fireman, Judge, Mechanic, Pilot, Police and Waiter) and copy it to your python project's folder.\nThen create a python file and give it a name; an example is **FirstCustomPrediction.py**.\nThen write the code below into the python file:\n\n### FirstCustomPrediction.py\n\n```python\nfrom imageai.Classification.Custom import CustomImageClassification\nimport os\n\nexecution_path = os.getcwd()\n\nprediction = CustomImageClassification()\nprediction.setModelTypeAsResNet50()\nprediction.setModelPath(os.path.join(execution_path, \"resnet50-idenprof-test_acc_0.78200_epoch-91.pt\"))\nprediction.setJsonPath(os.path.join(execution_path, \"idenprof_model_class.json\"))\nprediction.loadModel(num_objects=10)\n\npredictions, probabilities = prediction.classifyImage(os.path.join(execution_path, \"4.jpg\"), result_count=5)\n\nfor eachPrediction, eachProbability in zip(predictions, probabilities):\n print(eachPrediction + \" : \" + eachProbability)\n```\n\n**Sample Result:**\n\n![Sample Result](../../data-images/4.jpg)\n```\nmechanic : 76.82620286941528\nchef : 10.106072574853897\nwaiter : 4.036874696612358\npolice : 2.6663416996598244\npilot : 2.239348366856575\n```\n\nThe code above works as follows:\n```python\nfrom imageai.Classification.Custom import CustomImageClassification\nimport os\n```\nThe code above imports the **ImageAI** library for custom image prediction and the python **os** class.\n\n```python\nexecution_path = os.getcwd()\n```\n\nThe above line obtains the path to the folder that contains your python file (in this example, your FirstCustomPrediction.py).\n\n```python\nprediction = CustomImageClassification()\nprediction.setModelTypeAsResNet50()\nprediction.setModelPath(os.path.join(execution_path, \"resnet50-idenprof-test_acc_0.78200_epoch-91.pt\"))\nprediction.setJsonPath(os.path.join(execution_path, \"idenprof_model_class.json\"))\nprediction.loadModel(num_objects=10)\n```\n\nIn the lines above, we created and instance of the `CustomImageClassification()`\n class in the first line, then we set the model type of the prediction object to ResNet by caling the `.setModelTypeAsResNet50()`\n in the second line, we set the model path of the prediction object to the path of the custom model file (`resnet50-idenprof-test_acc_0.78200_epoch-91.pt`) we copied to the python file folder\n in the third line, we set the path to the idenprof_model_class.json of the model, we load the model and parse the number of objected that can be predicted in the model.\n\n```python\npredictions, probabilities = prediction.classifyImage(os.path.join(execution_path, \"4.jpg\"), result_count=5)\n```\n\nIn the above line, we defined 2 variables to be equal to the function called to predict an image, which is the `.classifyImage()` function, into which we parsed the path to our image and also state the number of prediction results we want to have (values from 1 to 10 in this case) parsing `result_count=5`. The `.classifyImage()` function will return 2 array objects with the first (**predictions**) being an array of predictions and the second (**percentage_probabilities**) being an array of the corresponding percentage probability for each prediction.\n\n```python\nfor eachPrediction, eachProbability in zip(predictions, probabilities):\n print(eachPrediction + \" : \" + eachProbability)\n```\n\nThe above line obtains each object in the **predictions** array, and also obtains the corresponding percentage probability from the **percentage_probabilities**, and finally prints the result of both to console.\n\n**CustomImageClassification** class also supports the multiple predictions, input types and prediction speeds that are contained\nin the **ImageClassification** class. Follow this [link](README.md) to see all the details.\n\n\n### Documentation\n\nWe have provided full documentation for all **ImageAI** classes and functions in 3 major languages. Find links below:**\n\n* Documentation - **English Version [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**\n" }, { "path": "imageai/Classification/CUSTOMTRAINING.md", "content": "# ImageAI : Custom Prediction Model Training \n\n## ---------------------------------------------------\n## Introducing Jarvis and TheiaEngine.\n\nWe the creators of ImageAI are glad to announce 2 new AI projects to provide state-of-the-art Generative AI, LLM and Image Understanding on your personal computer and servers. \n\n\n[![](../../jarvis.png)](https://jarvis.genxr.co)\n\nInstall Jarvis on PC/Mac to setup limitless access to LLM powered AI Chats for your every day work, research and generative AI needs with 100% privacy and full offline capability.\n\n\nVisit [https://jarvis.genxr.co](https://jarvis.genxr.co/) to get started.\n\n\n[![](../../theiaengine.png)]((https://www.genxr.co/theia-engine))\n\n\n[TheiaEngine](https://www.genxr.co/theia-engine), the next-generation computer Vision AI API capable of all Generative and Understanding computer vision tasks in a single API call and available via REST API to all programming languages. Features include\n- **Detect 300+ objects** ( 220 more objects than ImageAI)\n- **Provide answers to any content or context questions** asked on an image\n - very useful to get information on any object, action or information without needing to train a new custom model for every tasks\n- **Generate scene description and summary**\n- **Convert 2D image to 3D pointcloud and triangular mesh**\n- **Semantic Scene mapping of objects, walls, floors, etc**\n- **Stateless Face recognition and emotion detection**\n- **Image generation and augmentation from prompt**\n- etc.\n\nVisit [https://www.genxr.co/theia-engine](https://www.genxr.co/theia-engine) to try the demo and join in the beta testing today.\n## ---------------------------------------------------\n\n**ImageAI** provides the most simple and powerful approach to training custom image prediction models\nusing state-of-the-art SqueezeNet, ResNet50, InceptionV3 and DenseNet\nwhich you can load into the `imageai.Classification.Custom.CustomImageClassification` class. This allows\n you to train your own model on any set of images that corresponds to any type of objects/persons.\nThe training process generates a JSON file that maps the objects types in your image dataset\nand creates lots of models. You will then pick the model with the highest accuracy and perform custom\nimage prediction using the model and the JSON file generated.\n\n### TABLE OF CONTENTS\n- :white_square_button: Custom Model Training Prediction \n- :white_square_button: Saving Full Custom Model \n- :white_square_button: Training on the IdenProf Dataset \n- :white_square_button: Continuous Model Training \n- :white_square_button: Transfer Learning (Training from a pre-trained model)\n\n\n### Custom Model Training\n
\n\nBecause model training is a compute intensive tasks, we strongly advise you perform this experiment using a computer with a NVIDIA GPU and the GPU version of Tensorflow installed. Performing model training on CPU will my take hours or days. With NVIDIA GPU powered computer system, this will take a few hours. You can use Google Colab for this experiment as it has an NVIDIA K80 GPU available.\n\nTo train a custom prediction model, you need to prepare the images you want to use to train the model.\nYou will prepare the images as follows:\n\n1. Create a dataset folder with the name you will like your dataset to be called (e.g pets) \n2. In the dataset folder, create a folder by the name **train** \n3. In the dataset folder, create a folder by the name **test** \n4. In the train folder, create a folder for each object you want to the model to predict and give the folder a name that corresponds to the respective object name (e.g dog, cat, squirrel, snake) \n5. In the test folder, create a folder for each object you want to the model to predict and give\n the folder a name that corresponds to the respective object name (e.g dog, cat, squirrel, snake) \n6. In each folder present in the train folder, put the images of each object in its respective folder. This images are the ones to be used to train the model To produce a model that can perform well in practical applications, I recommend you about 500 or more images per object. 1000 images per object is just great \n7. In each folder present in the test folder, put about 100 to 200 images of each object in its respective folder. These images are the ones to be used to test the model as it trains \n8. Once you have done this, the structure of your image dataset folder should look like below: \n ```\n pets//train//dog//dog-train-images\n pets//train//cat//cat-train-images\n pets//train//squirrel//squirrel-train-images\n pets//train//snake//snake-train-images \n pets//test//dog//dog-test-images\n pets//test//cat//cat-test-images\n pets//test//squirrel//squirrel-test-images\n pets//test//snake//snake-test-images\n ```\n9. Then your training code goes as follows: \n ```python\n from imageai.Classification.Custom import ClassificationModelTrainer\n model_trainer = ClassificationModelTrainer()\n model_trainer.setModelTypeAsResNet50()\n model_trainer.setDataDirectory(\"pets\")\n model_trainer.trainModel(num_objects=4, num_experiments=100, enhance_data=True, batch_size=32, show_network_summary=True)\n ```\n\n Yes! Just 5 lines of code and you can train any of the available 4 state-of-the-art Deep Learning algorithms on your custom dataset.\nNow lets take a look at how the code above works.\n\n```python\nfrom imageai.Classification.Custom import ClassificationModelTrainer\nmodel_trainer = ClassificationModelTrainer()\nmodel_trainer.setModelTypeAsResNet50()\nmodel_trainer.setDataDirectory(\"pets\")\n```\n\nIn the first line, we import the **ImageAI** model training class, then we define the model trainer in the second line,\n we set the network type in the third line and set the path to the image dataset we want to train the network on.\n\n```python\nmodel_trainer.trainModel(num_experiments=100, batch_size=32)\n```\n\nIn the code above, we start the training process. The parameters stated in the function are as below:\n- **num_experiments** : this is to state the number of times the network will train over all the training images,\n which is also called epochs \n- **batch_size** : This is to state the number of images the network will process at ones. The images\n are processed in batches until they are exhausted per each experiment performed. \n\n\nWhen you start the training, you should see something like this in the console:\n\n```\n==================================================\nTraining with GPU\n==================================================\nEpoch 1/100\n----------\n100%|█████████████████████████████████████████████████████████████████████████████████| 282/282 [02:15<00:00, 2.08it/s]\ntrain Loss: 3.8062 Accuracy: 0.1178\n100%|███████████████████████████████████████████████████████████████████████████████████| 63/63 [00:26<00:00, 2.36it/s]\ntest Loss: 2.2829 Accuracy: 0.1215\nEpoch 2/100\n----------\n100%|█████████████████████████████████████████████████████████████████████████████████| 282/282 [01:57<00:00, 2.40it/s]\ntrain Loss: 2.2682 Accuracy: 0.1303\n100%|███████████████████████████████████████████████████████████████████████████████████| 63/63 [00:20<00:00, 3.07it/s]\ntest Loss: 2.2388 Accuracy: 0.1470\n```\n\nLet us explain the details shown above: \n1. The line **Epoch 1/100** means the network is training the first experiment of the targeted 100 \n2. The line `1/25 [>.............................] - ETA: 52s - loss: 2.3026 - acc: 0.2500` represents the number of batches that has been trained in the present experiment\n3. The best model is automatically saved to `/models>`\n \n Once you are done training your custom model, you can use the \"CustomImageClassification\" class to perform image prediction with your model. Simply follow the link below.\n[imageai/Classification/CUSTOMCLASSIFICATION.md](https://github.com/OlafenwaMoses/ImageAI/blob/master/imageai/Classification/CUSTOMCLASSIFICATION.md)\n\n\n\n### Documentation\n\nWe have provided full documentation for all **ImageAI** classes and functions. Find links below:\n\n* Documentation - **English Version [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**\n" }, { "path": "imageai/Classification/Custom/__init__.py", "content": "import time, warnings\nimport os\nimport copy\nimport re\nimport json\nfrom typing import List, Tuple, Union\nfrom PIL import Image\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nfrom torchvision import datasets\nfrom torchvision import transforms\nfrom torchvision.models import mobilenet_v2, inception_v3, resnet50, densenet121\nfrom torchvision.models.inception import InceptionOutputs\n\nfrom .data_transformation import data_transforms1, data_transforms2\nfrom .training_params import resnet50_train_params, densenet121_train_params, inception_v3_train_params, mobilenet_v2_train_params\nfrom tqdm import tqdm\n\nfrom ...backend_check.model_extension import extension_check\n\n\n\nclass ClassificationModelTrainer():\n \"\"\"\n This is the Classification Model training class, that allows you to define a deep learning network\n from the 4 available networks types supported by ImageAI which are MobileNetv2, ResNet50,\n InceptionV3 and DenseNet121 and then train on custom image data.\n \"\"\"\n\n def __init__(self) -> None:\n self.__model_type = \"\"\n self.__device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.__data_dir = \"\"\n self.__data_loaders = None\n self.__class_names = None\n self.__dataset_sizes = None\n self.__dataset_name = \"\"\n self.__model = None\n self.__optimizer = None\n self.__lr_scheduler = None\n self.__loss_fn = nn.CrossEntropyLoss()\n self.__transfer_learning_mode = \"fine_tune_all\"\n self.__model_path = \"\"\n self.__training_params = None\n\n def __set_training_param(self) -> None:\n if not self.__model_type:\n raise RuntimeError(\"The model type is not set!!!\")\n self.__model = self.__training_params[\"model\"]\n optimizer = self.__training_params[\"optimizer\"]\n lr_decay_rate = self.__training_params[\"lr_decay_rate\"]\n lr_step_size = self.__training_params[\"lr_step_size\"]\n lr = self.__training_params[\"lr\"]\n weight_decay = self.__training_params[\"weight_decay\"]\n\n if self.__model_path:\n self.__set_transfer_learning_mode()\n print(\"==> Transfer learning enabled\")\n \n # change the last linear layer to have output features of\n # same size as the number of unique classes in the new\n # dataset.\n if self.__model_type == \"mobilenet_v2\":\n in_features = self.__model.classifier[1].in_features\n self.__model.classifier[1] = nn.Linear(in_features, len(self.__class_names))\n elif self.__model_type == \"densenet121\":\n in_features = self.__model.classifier.in_features\n self.__model.classifier = nn.Linear(in_features, len(self.__class_names))\n else:\n in_features = self.__model.fc.in_features\n self.__model.fc = nn.Linear(in_features, len(self.__class_names))\n\n self.__model.to(self.__device)\n self.__optimizer = optimizer(\n self.__model.parameters(),\n lr=lr,\n momentum=0.9,\n weight_decay=weight_decay\n )\n if lr_decay_rate and lr_step_size:\n self.__lr_scheduler = lr_scheduler.StepLR(\n self.__optimizer,\n gamma=lr_decay_rate,\n step_size=lr_step_size\n )\n\n def __set_transfer_learning_mode(self) -> None:\n\n state_dict = torch.load(self.__model_path)\n if self.__model_type == \"densenet121\":\n # '.'s are no longer allowed in module names, but previous densenet layers\n # as provided by the pytorch organization has names that uses '.'s.\n pattern = re.compile(\n r\"^(.*denselayer\\d+\\.(?:norm|relu|conv))\\.((?:[12])\\.\"\n \"(?:weight|bias|running_mean|running_var))$\"\n )\n for key in list(state_dict.keys()):\n res = pattern.match(key)\n if res:\n new_key = res.group(1) + res.group(2)\n state_dict[new_key] = state_dict[key]\n del state_dict[key]\n\n self.__model.load_state_dict(state_dict)\n self.__model.to(self.__device)\n\n if self.__transfer_learning_mode == \"freeze_all\":\n for param in self.__model.parameters():\n param.requires_grad = False\n\n def __load_data(self, batch_size : int = 8) -> None:\n \n if not self.__data_dir:\n raise RuntimeError(\"The dataset directory not yet set.\")\n image_dataset = {\n x:datasets.ImageFolder(\n os.path.join(self.__data_dir, x),\n data_transforms2[x] if self.__model_type==\"inception_v3\" else data_transforms1[x]\n )\n for x in [\"train\", \"test\"]\n }\n self.__data_loaders = {\n x:torch.utils.data.DataLoader(\n image_dataset[x], batch_size=batch_size,\n shuffle=True\n )\n for x in [\"train\", \"test\"]\n }\n self.__dataset_sizes = {x:len(image_dataset[x]) for x in [\"train\", \"test\"]}\n self.__class_names = image_dataset[\"train\"].classes\n self.__dataset_name = os.path.basename(self.__data_dir.rstrip(os.path.sep))\n\n def setDataDirectory(self, data_directory : str = \"\") -> None:\n \"\"\"\n Sets the directory that contains the training and test dataset. The data directory should contain 'train' and 'test' subdirectories\n for the training and test datasets.\n\n In each of these subdirectories, each object must have a dedicated folder and the folder containing images for the object.\n\n The structure of the 'test' and 'train' folder must be as follows:\n \n >> train >> class1 >> class1_train_images\n >> class2 >> class2_train_images\n >> class3 >> class3_train_images\n >> class4 >> class4_train_images\n >> class5 >> class5_train_images\n >> test >> class1 >> class1_test_images\n >> class2 >> class2_test_images\n >> class3 >> class3_test_images\n >> class4 >> class4_test_images\n >> class5 >> class5_test_images\n\n \"\"\"\n if os.path.isdir(data_directory):\n self.__data_dir = data_directory\n return\n raise ValueError(\"expected a path to a directory\")\n\n def setModelTypeAsMobileNetV2(self) -> None:\n \"\"\"\n 'setModelTypeAsMobileNetV2()' is used to set the model type to the MobileNetV2 model.\n :return:\n \"\"\"\n self.__model_type = \"mobilenet_v2\"\n self.__training_params = mobilenet_v2_train_params()\n\n def setModelTypeAsResNet50(self) -> None:\n \"\"\"\n 'setModelTypeAsResNet50()' is used to set the model type to the ResNet50 model.\n :return:\n \"\"\"\n self.__model_type = \"resnet50\"\n self.__training_params = resnet50_train_params()\n\n def setModelTypeAsInceptionV3(self) -> None:\n \"\"\"\n 'setModelTypeAsInceptionV3()' is used to set the model type to the InceptionV3 model.\n :return:\n \"\"\"\n self.__model_type = \"inception_v3\"\n self.__training_params = inception_v3_train_params()\n\n def setModelTypeAsDenseNet121(self) -> None:\n \"\"\"\n 'setModelTypeAsDenseNet()' is used to set the model type to the DenseNet model.\n :return:\n \"\"\"\n self.__model_type = \"densenet121\"\n self.__training_params = densenet121_train_params()\n\n def freezeAllLayers(self) -> None:\n \"\"\"\n Set the transfer learning mode to freeze all layers.\n\n NOTE: The last layer (fully connected layer) is trainable.\n \"\"\"\n self.__transfer_learning_mode = \"freeze_all\"\n\n def fineTuneAllLayers(self) -> None:\n \"\"\"\n Sets the transfer learning mode to fine-tune the pretrained weights\n \"\"\"\n self.__transfer_learning_mode = \"fine_tune_all\"\n\n def trainModel(\n self,\n num_experiments : int = 100,\n batch_size : int = 8,\n model_directory : str = None,\n transfer_from_model: str = None,\n verbose : bool = True\n ) -> None:\n \n \"\"\"\n 'trainModel()' function starts the model actual training. It accepts the following values:\n - num_experiments: Also known as epochs, is the number of times the network will process all the images in the training dataset\n - batch_size: The number of image data that will be loaded into memory at once during training\n - model_directory: Location where json mapping and trained models will be saved\n - transfer_from_model: Path to a pre-trained imagenet model that corresponds to the training model type\n - verbose: Option to enable/disable training logs\n \n :param num_experiments:\n :param batch_size:\n :model_directory:\n :transfer_from_model:\n :verbose:\n :return:\n \"\"\"\n\n # Load dataset\n self.__load_data(batch_size)\n\n # Check and effect transfer learning if enabled\n if transfer_from_model:\n extension_check(transfer_from_model)\n self.__model_path = transfer_from_model\n\n # Load training parameters for the specified model type\n self.__set_training_param()\n\n \n # Create output directory to save trained models and json mappings\n if not model_directory:\n model_directory = os.path.join(self.__data_dir, \"models\")\n\n if not os.path.exists(model_directory):\n os.mkdir(model_directory)\n \n # Dump class mappings to json file\n with open(os.path.join(model_directory, f\"{self.__dataset_name}_model_classes.json\"), \"w\") as f:\n classes_dict = {}\n class_list = sorted(self.__class_names)\n for i in range(len(class_list)):\n classes_dict[str(i)] = class_list[i]\n json.dump(classes_dict, f)\n\n # Prep model weights for training\n since = time.time()\n\n best_model_weights = copy.deepcopy(self.__model.state_dict())\n best_acc = 0.0\n prev_save_name, recent_save_name = \"\", \"\"\n\n # Device check and log\n print(\"=\" * 50)\n print(\"Training with GPU\") if self.__device == \"cuda\" else print(\"Training with CPU. This might cause slower train.\")\n print(\"=\" * 50)\n\n\n for epoch in range(num_experiments):\n if verbose:\n print(f\"Epoch {epoch + 1}/{num_experiments}\", \"-\"*10, sep=\"\\n\")\n\n # each epoch has a training and test phase\n for phase in [\"train\", \"test\"]:\n if phase == \"train\":\n self.__model.train()\n else:\n self.__model.eval()\n\n running_loss = 0.0\n running_corrects = 0\n\n # Iterate on the dataset in batches\n for imgs, labels in tqdm(self.__data_loaders[phase]):\n imgs = imgs.to(self.__device)\n labels = labels.to(self.__device)\n\n self.__optimizer.zero_grad()\n\n with torch.set_grad_enabled(phase == \"train\"):\n output = self.__model(imgs)\n if self.__model_type == \"inception_v3\" and type(output) == InceptionOutputs:\n output = output[0]\n _, preds = torch.max(output, 1)\n loss = self.__loss_fn(output, labels)\n\n if phase==\"train\":\n loss.backward()\n self.__optimizer.step()\n running_loss += loss.item() * imgs.size(0)\n running_corrects += torch.sum(preds==labels.data)\n\n # Compute accuracy and loss metrics post epoch training\n if phase == \"train\" and isinstance(self.__lr_scheduler, torch.optim.lr_scheduler.StepLR):\n self.__lr_scheduler.step()\n\n epoch_loss = running_loss / self.__dataset_sizes[phase]\n epoch_acc = running_corrects.double() / self.__dataset_sizes[phase]\n\n if verbose:\n print(f\"{phase} Loss: {epoch_loss:.4f} Accuracy: {epoch_acc:.4f}\")\n if phase == \"test\" and epoch_acc > best_acc:\n best_acc = epoch_acc\n recent_save_name = self.__model_type+f\"-{self.__dataset_name}-test_acc_{best_acc:.5f}_epoch-{epoch}.pt\"\n if prev_save_name:\n os.remove(os.path.join(model_directory, prev_save_name))\n best_model_weights = copy.deepcopy(self.__model.state_dict())\n torch.save(\n best_model_weights, os.path.join(model_directory, recent_save_name)\n )\n prev_save_name = recent_save_name\n \n\n time_elapsed = time.time() - since\n print(f\"Training completed in {time_elapsed//60:.0f}m {time_elapsed % 60:.0f}s\")\n print(f\"Best test accuracy: {best_acc:.4f}\")\n\n\nclass CustomImageClassification:\n \"\"\"\n An implementation that allows for easy classification of images\n using the state of the art computer vision classification model\n trained on custom data.\n\n The class provides 4 different classification models which are ResNet50, DensesNet121, InceptionV3 and MobileNetV2.\n\n The following functions are required to be called before a classification can be made\n\n * At least of of the following and it must correspond to the model set in the setModelPath()\n [setModelTypeAsMobileNetV2(), setModelTypeAsResNet(), setModelTypeAsDenseNet, setModelTypeAsInceptionV3]\n\n * setModelPath: This is used to specify the absolute path to the trained model file.\n\n * setJsonPath: This is used to specify the absolute path to the\n json file saved during the training of the custom model.\n\n * useCPU (Optional): If you will like to force the image classification to be performed on CPU, call this function.\n\n * loadModel: Used to load the trained model weights and json data.\n\n * classifyImage(): Used for classifying an image.\n \"\"\"\n def __init__(self) -> None:\n self.__model = None\n self.__model_type = \"\"\n self.__model_loaded = False\n self.__device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.__json_path = None\n self.__class_names = None\n self.__model_loaded = False\n\n def __load_image(self, image_input: Union[str, np.ndarray, Image.Image]) -> torch.Tensor:\n images = []\n preprocess = transforms.Compose([\n transforms.Resize(256),\n transforms.CenterCrop(224),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n ])\n if type(image_input) == str:\n if os.path.isfile(image_input):\n img = Image.open(image_input).convert(\"RGB\")\n images.append(preprocess(img))\n else:\n raise ValueError(f\"image path '{image_input}' is not found or a valid file\")\n elif type(image_input) == np.ndarray:\n img = Image.fromarray(image_input).convert(\"RGB\")\n images.append(preprocess(img))\n elif \"PIL\" in str(type(image_input)):\n img = image_input.convert(\"RGB\")\n images.append(preprocess(img))\n else:\n raise ValueError(f\"Invalid image input format\")\n\n return torch.stack(images)\n \n def __load_classes(self):\n if self.__json_path:\n with open(self.__json_path, 'r') as f:\n self.__class_names = list(json.load(f).values())\n else:\n raise ValueError(\"Invalid json path. Set a valid json mapping path by calling the 'setJsonPath()' function\")\n\n def setModelPath(self, path : str) -> None:\n \"\"\"\n Sets the path to the pretrained weight.\n \"\"\"\n if os.path.isfile(path):\n extension_check(path)\n self.__model_path = path\n self.__model_loaded = False\n else:\n raise ValueError(\n f\"The path '{path}' isn't a valid file. Ensure you specify the path to a valid trained model file.\"\n )\n \n def setJsonPath(self, path : str) -> None:\n \"\"\"\n Sets the path to the pretrained weight.\n \"\"\"\n if os.path.isfile(path):\n self.__json_path = path\n else:\n raise ValueError(\n \"parameter path should be a valid path to the json mapping file.\"\n )\n\n def setModelTypeAsMobileNetV2(self) -> None:\n \"\"\"\n 'setModelTypeAsMobileNetV2()' is used to set the model type to the MobileNetV2 model.\n :return:\n \"\"\"\n self.__model_type = \"mobilenet_v2\"\n\n def setModelTypeAsResNet50(self) -> None:\n \"\"\"\n 'setModelTypeAsResNet50()' is used to set the model type to the ResNet50 model.\n :return:\n \"\"\"\n self.__model_type = \"resnet50\"\n\n def setModelTypeAsInceptionV3(self) -> None:\n \"\"\"\n 'setModelTypeAsInceptionV3()' is used to set the model type to the InceptionV3 model.\n :return:\n \"\"\"\n self.__model_type = \"inception_v3\"\n\n def setModelTypeAsDenseNet121(self) -> None:\n \"\"\"\n 'setModelTypeAsDenseNet121()' is used to set the model type to the DenseNet121 model.\n :return:\n \"\"\"\n self.__model_type = \"densenet121\"\n \n def useCPU(self):\n \"\"\"\n Used to force classification to be done on CPU.\n By default, classification will occur on GPU compute if available else CPU compute.\n \"\"\"\n self.__device = \"cpu\"\n if self.__model_loaded:\n self.__model_loaded = False\n self.loadModel()\n\n def loadModel(self) -> None:\n \"\"\"\n 'loadModel()' function is used to load the model weights into the model architecture from the file path defined\n in the setModelPath() function.\n :return:\n \"\"\"\n if not self.__model_loaded:\n self.__load_classes()\n try:\n # change the last layer of the networks to conform to the number\n # of unique classes in the custom dataset used to train the custom\n # model\n\n if self.__model_type == \"resnet50\":\n self.__model = resnet50(pretrained=False)\n in_features = self.__model.fc.in_features\n self.__model.fc = nn.Linear(in_features, len(self.__class_names))\n elif self.__model_type == \"mobilenet_v2\":\n self.__model = mobilenet_v2(pretrained=False)\n in_features = self.__model.classifier[1].in_features\n self.__model.classifier[1] = nn.Linear(in_features, len(self.__class_names))\n elif self.__model_type == \"inception_v3\":\n self.__model = inception_v3(pretrained=False)\n in_features = self.__model.fc.in_features\n self.__model.fc = nn.Linear(in_features, len(self.__class_names))\n elif self.__model_type == \"densenet121\":\n self.__model = densenet121(pretrained=False)\n in_features = self.__model.classifier.in_features\n self.__model.classifier = nn.Linear(in_features, len(self.__class_names))\n else:\n raise RuntimeError(\"Unknown model type.\\nEnsure the model type is properly set.\")\n\n state_dict = torch.load(self.__model_path, map_location=self.__device)\n\n if self.__model_type == \"densenet121\":\n # '.'s are no longer allowed in module names, but previous densenet layers\n # as provided by the pytorch organization has names that uses '.'s.\n pattern = re.compile(\n r\"^(.*denselayer\\d+\\.(?:norm|relu|conv))\\.((?:[12])\\.\"\n \"(?:weight|bias|running_mean|running_var))$\"\n )\n for key in list(state_dict.keys()):\n res = pattern.match(key)\n if res:\n new_key = res.group(1) + res.group(2)\n state_dict[new_key] = state_dict[key]\n del state_dict[key]\n\n self.__model.load_state_dict(state_dict)\n self.__model.to(self.__device).eval()\n self.__model_loaded = True\n\n except Exception as e:\n raise Exception(\"Weight loading failed.\\nEnsure the model path is\"\n \" set and the weight file is in the specified model path.\")\n\n def classifyImage(self, image_input: Union[str, np.ndarray, Image.Image], result_count: int) -> Tuple[List[str], List[float]]:\n \"\"\"\n 'classifyImage()' function is used to classify a given image by receiving the following arguments:\n * image_input: file path, numpy array or PIL image of the input image.\n * result_count (optional) , the number of classifications to be sent which must be whole numbers between 1 and total number of classes the model is trained to classify.\n\n This function returns 2 arrays namely 'classification_results' and 'classification_probabilities'. The 'classification_results'\n contains possible objects classes arranged in descending of their percentage probabilities. The 'classification_probabilities'\n contains the percentage probability of each object class. The position of each object class in the 'classification_results'\n array corresponds with the positions of the percentage probability in the 'classification_probabilities' array.\n \n :param image_input:\n :param result_count:\n :return classification_results, classification_probabilities:\n \"\"\"\n if not self.__model_loaded:\n raise RuntimeError(\n \"Model not yet loaded. You need to call '.loadModel()' before performing image classification\"\n )\n\n images = self.__load_image(image_input)\n images = images.to(self.__device)\n \n with torch.no_grad():\n output = self.__model(images)\n probabilities = torch.softmax(output, dim=1)\n topN_prob, topN_catid = torch.topk(probabilities, result_count)\n \n predictions = [\n [\n (self.__class_names[topN_catid[i][j]], topN_prob[i][j].item()*100)\n for j in range(topN_prob.shape[1])\n ]\n for i in range(topN_prob.shape[0])\n ]\n \n labels_pred = []\n probabilities_pred = []\n\n for idx, pred in enumerate(predictions):\n for label, score in pred:\n labels_pred.append(label)\n probabilities_pred.append(round(score, 4))\n \n return labels_pred, probabilities_pred" }, { "path": "imageai/Classification/Custom/data_transformation.py", "content": "from torchvision import transforms\n\ndata_transforms1 = {\n \"train\":transforms.Compose([\n transforms.RandomResizedCrop(224),\n transforms.RandomHorizontalFlip(),\n transforms.ToTensor(),\n transforms.Normalize(\n [0.485, 0.456, 0.406],\n [0.229, 0.224, 0.225]\n )\n ]),\n \"test\": transforms.Compose([\n transforms.Resize(256),\n transforms.CenterCrop(224),\n transforms.ToTensor(),\n transforms.Normalize(\n [0.485, 0.456, 0.406],\n [0.229, 0.224, 0.225]\n )\n ])\n }\n\ndata_transforms2 = {\n \"train\":transforms.Compose([\n transforms.RandomResizedCrop(299),\n transforms.RandomHorizontalFlip(),\n transforms.ToTensor(),\n transforms.Normalize(\n [0.485, 0.456, 0.406],\n [0.229, 0.224, 0.225]\n )\n ]),\n \"test\": transforms.Compose([\n transforms.Resize(299),\n transforms.CenterCrop(299),\n transforms.ToTensor(),\n transforms.Normalize(\n [0.485, 0.456, 0.406],\n [0.229, 0.224, 0.225]\n )\n ])\n }\n" }, { "path": "imageai/Classification/Custom/training_params.py", "content": "import torch\nfrom torch.optim import SGD\nfrom torchvision.models import resnet50, inception_v3, mobilenet_v2, densenet121\n\nmodel = resnet50(pretrained=False)\n\n\ndef resnet50_train_params():\n model = resnet50(pretrained=False)\n return {\n \"model\": model,\n \"optimizer\": SGD,\n \"weight_decay\":1e-4,\n \"lr\":0.1,\n \"lr_decay_rate\": None,\n \"lr_step_size\": None\n }\n\ndef inception_v3_train_params():\n model = inception_v3(pretrained=False, init_weights=False)\n\n return {\n \"model\": model,\n \"optimizer\": SGD,\n \"weight_decay\":0,\n \"lr\":0.045,\n \"lr_decay_rate\": 0.94,\n \"lr_step_size\":2\n }\n\ndef mobilenet_v2_train_params():\n model = mobilenet_v2(pretrained=False)\n \n return {\n \"model\": model,\n \"optimizer\": SGD,\n \"weight_decay\":4e-5,\n \"lr\":0.045,\n \"lr_decay_rate\": 0.98,\n \"lr_step_size\":1\n }\n\ndef densenet121_train_params():\n model = densenet121(pretrained=False)\n\n return {\n \"model\": model,\n \"optimizer\": SGD,\n \"weight_decay\":1e-4,\n \"lr\":0.1,\n \"lr_decay_rate\": None,\n \"lr_step_size\":None,\n }" }, { "path": "imageai/Classification/README.md", "content": "# ImageAI : Image Classification\n\n\n## ---------------------------------------------------\n## Introducing Jarvis and TheiaEngine.\n\nWe the creators of ImageAI are glad to announce 2 new AI projects to provide state-of-the-art Generative AI, LLM and Image Understanding on your personal computer and servers. \n\n\n[![](../../jarvis.png)](https://jarvis.genxr.co)\n\nInstall Jarvis on PC/Mac to setup limitless access to LLM powered AI Chats for your every day work, research and generative AI needs with 100% privacy and full offline capability.\n\n\nVisit [https://jarvis.genxr.co](https://jarvis.genxr.co/) to get started.\n\n\n[![](../../theiaengine.png)](https://www.genxr.co/theia-engine)\n\n\n[TheiaEngine](https://www.genxr.co/theia-engine), the next-generation computer Vision AI API capable of all Generative and Understanding computer vision tasks in a single API call and available via REST API to all programming languages. Features include\n- **Detect 300+ objects** ( 220 more objects than ImageAI)\n- **Provide answers to any content or context questions** asked on an image\n - very useful to get information on any object, action or information without needing to train a new custom model for every tasks\n- **Generate scene description and summary**\n- **Convert 2D image to 3D pointcloud and triangular mesh**\n- **Semantic Scene mapping of objects, walls, floors, etc**\n- **Stateless Face recognition and emotion detection**\n- **Image generation and augmentation from prompt**\n- etc.\n\nVisit [https://www.genxr.co/theia-engine](https://www.genxr.co/theia-engine) to try the demo and join in the beta testing today.\n## ---------------------------------------------------\n\n### TABLE OF CONTENTS\n- :white_square_button: First Prediction\n- :white_square_button: Documentation\n\nImageAI provides 4 different algorithms and model types to perform image prediction.\nTo perform image prediction on any picture, take the following simple steps. The 4 algorithms provided for\n image prediction include **MobileNetV2**, **ResNet50**, **InceptionV3** and **DenseNet121**. Each of these\n algorithms have individual model files which you must use depending on the choice of your algorithm. To download the\n model file for your choice of algorithm, click on any of the links below:\n \n- **[MobileNetV2](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/mobilenet_v2-b0353104.pth)** _(Size = 4.82 mb, fastest prediction time and moderate accuracy)_\n- **[ResNet50](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/resnet50-19c8e357.pth)** by Microsoft Research _(Size = 98 mb, fast prediction time and high accuracy)_\n - **[InceptionV3](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/inception_v3_google-1a9a5a14.pth)** by Google Brain team _(Size = 91.6 mb, slow prediction time and higher accuracy)_\n - **[DenseNet121](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/densenet121-a639ec97.pth)** by Facebook AI Research _(Size = 31.6 mb, slower prediction time and highest accuracy)_\n\n Great! Once you have downloaded this model file, start a new python project, and then copy the model file to your project\n folder where your python files (.py files) will be . Download the image below, or take any image on your computer\n and copy it to your python project's folder. Then create a python file and give it a name; an example is `FirstPrediction.py`.\n Then write the code below into the python file:\n \n### FirstPrediction.py\n
\n\n```python\nfrom imageai.Classification import ImageClassification\nimport os\n\nexecution_path = os.getcwd()\n\nprediction = ImageClassification()\nprediction.setModelTypeAsResNet50()\nprediction.setModelPath(os.path.join(execution_path, \"resnet50-19c8e357.pth\"))\nprediction.loadModel()\n\npredictions, probabilities = prediction.classifyImage(os.path.join(execution_path, \"1.jpg\"), result_count=5 )\nfor eachPrediction, eachProbability in zip(predictions, probabilities):\n print(eachPrediction , \" : \" , eachProbability)\n```\n\nSample Result:\n![](../../data-images/1.jpg)\n\n```\nconvertible : 52.459555864334106\nsports_car : 37.61284649372101\npickup : 3.1751200556755066\ncar_wheel : 1.817505806684494\nminivan : 1.7487050965428352\n```\n\nThe code above works as follows:\n```python\nfrom imageai.Classification import ImageClassification\nimport os\n```\nThe code above imports the `ImageAI` library and the python `os` class.\n```python\nexecution_path = os.getcwd()\n```\nThe above line obtains the path to the folder that contains your python file (in this example, your FirstPrediction.py).\n\n```python\nprediction = ImageClassification()\nprediction.setModelTypeAsResNet50()\nprediction.setModelPath(os.path.join(execution_path, \"resnet50-19c8e357.pth\"))\n```\nIn the lines above, we created and instance of the `ImagePrediction()` class in the first line, then we set the model type of the prediction object to ResNet by caling the `.setModelTypeAsResNet50()` in the second line and then we set the model path of the prediction object to the path of the model file (`resnet50-19c8e357.pth`) we copied to the python file folder in the third line.\n\n```python\npredictions, probabilities = prediction.classifyImage(os.path.join(execution_path, \"1.jpg\"), result_count=5 )\n```\n\nIn the above line, we defined 2 variables to be equal to the function called to predict an image, which is the `.classifyImage()` function, into which we parsed the path to our image and also state the number of prediction results we want to have (values from 1 to 1000) parsing `result_count=5`. The `.classifyImage()` function will return 2 array objects with the first (**predictions**) being an array of predictions and the second (**percentage_probabilities**) being an array of the corresponding percentage probability for each prediction.\n\n```python\nfor eachPrediction, eachProbability in zip(predictions, probabilities):\n print(eachPrediction, \" : \" , eachProbability)\n```\nThe above line obtains each object in the **predictions** array, and also obtains the corresponding percentage probability from the **percentage_probabilities**, and finally prints the result of both to console.\n\n\n\n\n### Documentation\n\nWe have provided full documentation for all **ImageAI** classes and functions. Find links below:**\n\n* Documentation - **English Version [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**\n\n" }, { "path": "imageai/Classification/__init__.py", "content": "import os, re\nfrom typing import Union\nfrom typing import List, Tuple\nimport numpy as np\nimport torch\nfrom torchvision.models import resnet50, densenet121, mobilenet_v2, inception_v3\nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom PIL import Image\nimport traceback\nfrom ..backend_check.model_extension import extension_check\n\nclassification_models = {\n \"resnet50\": {\n \"model\": resnet50(pretrained=False)\n },\n \"densenet121\": {\n \"model\": densenet121(pretrained=False)\n },\n \"inceptionv3\": {\n \"model\": inception_v3(pretrained=False)\n },\n \"mobilenetv2\": {\n \"model\": mobilenet_v2(pretrained=False)\n }\n}\n\nclass ImageClassification:\n \"\"\"\n This is the image classification class in the ImageAI library. It allows you to classify objects into all the 1000 different classes in the ImageNet dataset [ https://www.kaggle.com/c/imagenet-object-localization-challenge/overview/description ].\n\n The class provides 4 different classification models which are ResNet50, DensesNet121, InceptionV3 and MobileNetV2.\n\n The following functions are required to be called before a classification can be made\n\n * At least of of the following and it must correspond to the model set in the setModelPath()\n [setModelTypeAsMobileNetV2(), setModelTypeAsResNet(), setModelTypeAsDenseNet, setModelTypeAsInceptionV3]\n\n * setModelPath: This is used to specify the absolute path to a pretrained model file. Download any of the files in this release -> https://github.com/OlafenwaMoses/ImageAI/releases/tag/3.0.0-pretrained\n\n * useCPU (Optional): If you will like to force the image classification to be performed on CPU, call this function.\n\n * loadModel: Used to load the pretrained model weights\n\n * classifyImage(): Used for classifying an image.\n\n \"\"\"\n def __init__(self) -> None:\n self.__model_type:str = None\n self.__model:Union[resnet50, densenet121, mobilenet_v2, inception_v3] = None\n self.__model_path: str = None\n self.__classes_path: str = os.path.join(os.path.dirname(os.path.abspath(__file__)), \"imagenet_classes.txt\")\n self.__model_loaded: bool = False\n self.__device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.__classes: List[str] = []\n \n def setModelPath(self, path: str):\n \"\"\"\n 'setModelPath()' function is required and is used to set the file path to the model adopted from the list of the\n available 4 model types. The model path must correspond to the model type set for the classification instance object.\n :param model_path:\n :return:\n \"\"\"\n if os.path.isfile(path):\n extension_check(path)\n self.__model_path = path\n else:\n raise ValueError(\n f\"The path '{path}' isn't a valid file. Ensure you specify the path to a valid trained model file.\"\n )\n\n def __load_classes(self) -> List[str]:\n with open(self.__classes_path) as f:\n self.__classes = [c.strip() for c in f.readlines()]\n \n def __load_image(self, image_input: Union[str, np.ndarray, Image.Image]) -> torch.Tensor:\n images = []\n preprocess = transforms.Compose([\n transforms.Resize(256),\n transforms.CenterCrop(224),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n ])\n if type(image_input) == str:\n if os.path.isfile(image_input):\n img = Image.open(image_input).convert(\"RGB\")\n images.append(preprocess(img))\n else:\n raise ValueError(f\"image path '{image_input}' is not found or a valid file\")\n elif type(image_input) == np.ndarray:\n img = Image.fromarray(image_input).convert(\"RGB\")\n images.append(preprocess(img))\n elif \"PIL\" in str(type(image_input)):\n img = image_input.convert(\"RGB\")\n images.append(preprocess(img))\n else:\n raise ValueError(f\"Invalid image input format\")\n\n return torch.stack(images)\n\n def setModelTypeAsResNet50(self):\n \"\"\"\n 'setModelTypeAsResNet50()' is used to set the model type to the ResNet50 model.\n :return:\n \"\"\"\n if self.__model_type == None:\n self.__model_type = \"resnet50\"\n\n def setModelTypeAsDenseNet121(self):\n \"\"\"\n 'setModelTypeAsDenseNet121()' is used to set the model type to the DenseNet121 model.\n :return:\n \"\"\"\n if self.__model_type == None:\n self.__model_type = \"densenet121\"\n \n def setModelTypeAsInceptionV3(self):\n \"\"\"\n 'setModelTypeAsInceptionV3()' is used to set the model type to the InceptionV3 model.\n :return:\n \"\"\"\n if self.__model_type == None:\n self.__model_type = \"inceptionv3\"\n \n def setModelTypeAsMobileNetV2(self):\n \"\"\"\n 'setModelTypeAsMobileNetV2()' is used to set the model type to the MobileNetV2 model.\n :return:\n \"\"\"\n if self.__model_type == None:\n self.__model_type = \"mobilenetv2\"\n \n def useCPU(self):\n \"\"\"\n Used to force classification to be done on CPU.\n By default, classification will occur on GPU compute if available else CPU compute.\n \"\"\"\n self.__device = \"cpu\"\n if self.__model_loaded:\n self.__model_loaded = False\n self.loadModel()\n\n def loadModel(self):\n \"\"\"\n 'loadModel()' function is used to load the model weights into the model architecture from the file path defined\n in the setModelPath() function.\n :return:\n \"\"\"\n if not self.__model_loaded:\n try:\n if self.__model_path == None:\n raise ValueError(\n \"Model path not specified. Call '.setModelPath()' and parse the path to the model file before loading the model.\"\n )\n \n if self.__model_type in classification_models.keys():\n self.__model = classification_models[self.__model_type][\"model\"]\n else:\n raise ValueError(\n f\"Model type '{self.__model_type}' not supported.\"\n )\n state_dict = torch.load(self.__model_path)\n if self.__model_type == \"densenet121\":\n # '.'s are no longer allowed in module names, but previous densenet layers\n # as provided by the Pytorch's model zoon has names that uses '.'s.\n pattern = re.compile(\n r\"^(.*denselayer\\d+\\.(?:norm|relu|conv))\\.((?:[12])\\.\"\n \"(?:weight|bias|running_mean|running_var))$\"\n )\n for key in list(state_dict.keys()):\n res = pattern.match(key)\n if res:\n new_key = res.group(1) + res.group(2)\n state_dict[new_key] = state_dict[key]\n del state_dict[key]\n\n self.__model.load_state_dict(\n state_dict\n )\n self.__model.to(self.__device)\n self.__model_loaded = True\n self.__model.eval()\n self.__load_classes()\n except Exception:\n print(traceback.print_exc())\n print(\"Weight loading failed.\\nEnsure the model path is\"\n \" set and the weight file is in the specified model path.\")\n \n \n\n def classifyImage(self, image_input: Union[str, np.ndarray, Image.Image], result_count: int=5) -> Tuple[List[str], List[float]]:\n\n \"\"\"\n 'classifyImage()' function is used to classify a given image by receiving the following arguments:\n * image_input: file path, numpy array or PIL image of the input image.\n * result_count (optional) , the number of classifications to be sent which must be whole numbers between\n 1 and 1000. The default is 5.\n\n This function returns 2 arrays namely 'classification_results' and 'classification_probabilities'. The 'classification_results'\n contains possible objects classes arranged in descending of their percentage probabilities. The 'classification_probabilities'\n contains the percentage probability of each object class. The position of each object class in the 'classification_results'\n array corresponds with the positions of the percentage probability in the 'classification_probabilities' array.\n \n :param image_input:\n :param result_count:\n :return classification_results, classification_probabilities:\n \"\"\"\n\n if not self.__model_loaded:\n raise RuntimeError(\n \"Model not yet loaded. You need to call '.loadModel()' before performing image classification\"\n )\n\n images = self.__load_image(image_input)\n images = images.to(self.__device)\n \n with torch.no_grad():\n output = self.__model(images)\n probabilities = torch.softmax(output, dim=1)\n topN_prob, topN_catid = torch.topk(probabilities, result_count)\n \n predictions = [\n [\n (self.__classes[topN_catid[i][j]], topN_prob[i][j].item()*100)\n for j in range(topN_prob.shape[1])\n ]\n for i in range(topN_prob.shape[0])\n ]\n \n labels_pred = []\n probabilities_pred = []\n\n for idx, pred in enumerate(predictions):\n for label, score in pred:\n labels_pred.append(label)\n probabilities_pred.append(round(score, 4))\n \n return labels_pred, probabilities_pred\n \n\n \n " }, { "path": "imageai/Classification/imagenet_classes.txt", "content": "tench\ngoldfish\ngreat white shark\ntiger shark\nhammerhead\nelectric ray\nstingray\ncock\nhen\nostrich\nbrambling\ngoldfinch\nhouse finch\njunco\nindigo bunting\nrobin\nbulbul\njay\nmagpie\nchickadee\nwater ouzel\nkite\nbald eagle\nvulture\ngreat grey owl\nEuropean fire salamander\ncommon newt\neft\nspotted salamander\naxolotl\nbullfrog\ntree frog\ntailed frog\nloggerhead\nleatherback turtle\nmud turtle\nterrapin\nbox turtle\nbanded gecko\ncommon iguana\nAmerican chameleon\nwhiptail\nagama\nfrilled lizard\nalligator lizard\nGila monster\ngreen lizard\nAfrican chameleon\nKomodo dragon\nAfrican crocodile\nAmerican alligator\ntriceratops\nthunder snake\nringneck snake\nhognose snake\ngreen snake\nking snake\ngarter snake\nwater snake\nvine snake\nnight snake\nboa constrictor\nrock python\nIndian cobra\ngreen mamba\nsea snake\nhorned viper\ndiamondback\nsidewinder\ntrilobite\nharvestman\nscorpion\nblack and gold garden spider\nbarn spider\ngarden spider\nblack widow\ntarantula\nwolf spider\ntick\ncentipede\nblack grouse\nptarmigan\nruffed grouse\nprairie chicken\npeacock\nquail\npartridge\nAfrican grey\nmacaw\nsulphur-crested cockatoo\nlorikeet\ncoucal\nbee eater\nhornbill\nhummingbird\njacamar\ntoucan\ndrake\nred-breasted merganser\ngoose\nblack swan\ntusker\nechidna\nplatypus\nwallaby\nkoala\nwombat\njellyfish\nsea anemone\nbrain coral\nflatworm\nnematode\nconch\nsnail\nslug\nsea slug\nchiton\nchambered nautilus\nDungeness crab\nrock crab\nfiddler crab\nking crab\nAmerican lobster\nspiny lobster\ncrayfish\nhermit crab\nisopod\nwhite stork\nblack stork\nspoonbill\nflamingo\nlittle blue heron\nAmerican egret\nbittern\ncrane\nlimpkin\nEuropean gallinule\nAmerican coot\nbustard\nruddy turnstone\nred-backed sandpiper\nredshank\ndowitcher\noystercatcher\npelican\nking penguin\nalbatross\ngrey whale\nkiller whale\ndugong\nsea lion\nChihuahua\nJapanese spaniel\nMaltese dog\nPekinese\nShih-Tzu\nBlenheim spaniel\npapillon\ntoy terrier\nRhodesian ridgeback\nAfghan hound\nbasset\nbeagle\nbloodhound\nbluetick\nblack-and-tan coonhound\nWalker hound\nEnglish foxhound\nredbone\nborzoi\nIrish wolfhound\nItalian greyhound\nwhippet\nIbizan hound\nNorwegian elkhound\notterhound\nSaluki\nScottish deerhound\nWeimaraner\nStaffordshire bullterrier\nAmerican Staffordshire terrier\nBedlington terrier\nBorder terrier\nKerry blue terrier\nIrish terrier\nNorfolk terrier\nNorwich terrier\nYorkshire terrier\nwire-haired fox terrier\nLakeland terrier\nSealyham terrier\nAiredale\ncairn\nAustralian terrier\nDandie Dinmont\nBoston bull\nminiature schnauzer\ngiant schnauzer\nstandard schnauzer\nScotch terrier\nTibetan terrier\nsilky terrier\nsoft-coated wheaten terrier\nWest Highland white terrier\nLhasa\nflat-coated retriever\ncurly-coated retriever\ngolden retriever\nLabrador retriever\nChesapeake Bay retriever\nGerman short-haired pointer\nvizsla\nEnglish setter\nIrish setter\nGordon setter\nBrittany spaniel\nclumber\nEnglish springer\nWelsh springer spaniel\ncocker spaniel\nSussex spaniel\nIrish water spaniel\nkuvasz\nschipperke\ngroenendael\nmalinois\nbriard\nkelpie\nkomondor\nOld English sheepdog\nShetland sheepdog\ncollie\nBorder collie\nBouvier des Flandres\nRottweiler\nGerman shepherd\nDoberman\nminiature pinscher\nGreater Swiss Mountain dog\nBernese mountain dog\nAppenzeller\nEntleBucher\nboxer\nbull mastiff\nTibetan mastiff\nFrench bulldog\nGreat Dane\nSaint Bernard\nEskimo dog\nmalamute\nSiberian husky\ndalmatian\naffenpinscher\nbasenji\npug\nLeonberg\nNewfoundland\nGreat Pyrenees\nSamoyed\nPomeranian\nchow\nkeeshond\nBrabancon griffon\nPembroke\nCardigan\ntoy poodle\nminiature poodle\nstandard poodle\nMexican hairless\ntimber wolf\nwhite wolf\nred wolf\ncoyote\ndingo\ndhole\nAfrican hunting dog\nhyena\nred fox\nkit fox\nArctic fox\ngrey fox\ntabby\ntiger cat\nPersian cat\nSiamese cat\nEgyptian cat\ncougar\nlynx\nleopard\nsnow leopard\njaguar\nlion\ntiger\ncheetah\nbrown bear\nAmerican black bear\nice bear\nsloth bear\nmongoose\nmeerkat\ntiger beetle\nladybug\nground beetle\nlong-horned beetle\nleaf beetle\ndung beetle\nrhinoceros beetle\nweevil\nfly\nbee\nant\ngrasshopper\ncricket\nwalking stick\ncockroach\nmantis\ncicada\nleafhopper\nlacewing\ndragonfly\ndamselfly\nadmiral\nringlet\nmonarch\ncabbage butterfly\nsulphur butterfly\nlycaenid\nstarfish\nsea urchin\nsea cucumber\nwood rabbit\nhare\nAngora\nhamster\nporcupine\nfox squirrel\nmarmot\nbeaver\nguinea pig\nsorrel\nzebra\nhog\nwild boar\nwarthog\nhippopotamus\nox\nwater buffalo\nbison\nram\nbighorn\nibex\nhartebeest\nimpala\ngazelle\nArabian camel\nllama\nweasel\nmink\npolecat\nblack-footed ferret\notter\nskunk\nbadger\narmadillo\nthree-toed sloth\norangutan\ngorilla\nchimpanzee\ngibbon\nsiamang\nguenon\npatas\nbaboon\nmacaque\nlangur\ncolobus\nproboscis monkey\nmarmoset\ncapuchin\nhowler monkey\ntiti\nspider monkey\nsquirrel monkey\nMadagascar cat\nindri\nIndian elephant\nAfrican elephant\nlesser panda\ngiant panda\nbarracouta\neel\ncoho\nrock beauty\nanemone fish\nsturgeon\ngar\nlionfish\npuffer\nabacus\nabaya\nacademic gown\naccordion\nacoustic guitar\naircraft carrier\nairliner\nairship\naltar\nambulance\namphibian\nanalog clock\napiary\napron\nashcan\nassault rifle\nbackpack\nbakery\nbalance beam\nballoon\nballpoint\nBand Aid\nbanjo\nbannister\nbarbell\nbarber chair\nbarbershop\nbarn\nbarometer\nbarrel\nbarrow\nbaseball\nbasketball\nbassinet\nbassoon\nbathing cap\nbath towel\nbathtub\nbeach wagon\nbeacon\nbeaker\nbearskin\nbeer bottle\nbeer glass\nbell cote\nbib\nbicycle-built-for-two\nbikini\nbinder\nbinoculars\nbirdhouse\nboathouse\nbobsled\nbolo tie\nbonnet\nbookcase\nbookshop\nbottlecap\nbow\nbow tie\nbrass\nbrassiere\nbreakwater\nbreastplate\nbroom\nbucket\nbuckle\nbulletproof vest\nbullet train\nbutcher shop\ncab\ncaldron\ncandle\ncannon\ncanoe\ncan opener\ncardigan\ncar mirror\ncarousel\ncarpenter's kit\ncarton\ncar wheel\ncash machine\ncassette\ncassette player\ncastle\ncatamaran\nCD player\ncello\ncellular telephone\nchain\nchainlink fence\nchain mail\nchain saw\nchest\nchiffonier\nchime\nchina cabinet\nChristmas stocking\nchurch\ncinema\ncleaver\ncliff dwelling\ncloak\nclog\ncocktail shaker\ncoffee mug\ncoffeepot\ncoil\ncombination lock\ncomputer keyboard\nconfectionery\ncontainer ship\nconvertible\ncorkscrew\ncornet\ncowboy boot\ncowboy hat\ncradle\ncrane\ncrash helmet\ncrate\ncrib\nCrock Pot\ncroquet ball\ncrutch\ncuirass\ndam\ndesk\ndesktop computer\ndial telephone\ndiaper\ndigital clock\ndigital watch\ndining table\ndishrag\ndishwasher\ndisk brake\ndock\ndogsled\ndome\ndoormat\ndrilling platform\ndrum\ndrumstick\ndumbbell\nDutch oven\nelectric fan\nelectric guitar\nelectric locomotive\nentertainment center\nenvelope\nespresso maker\nface powder\nfeather boa\nfile\nfireboat\nfire engine\nfire screen\nflagpole\nflute\nfolding chair\nfootball helmet\nforklift\nfountain\nfountain pen\nfour-poster\nfreight car\nFrench horn\nfrying pan\nfur coat\ngarbage truck\ngasmask\ngas pump\ngoblet\ngo-kart\ngolf ball\ngolfcart\ngondola\ngong\ngown\ngrand piano\ngreenhouse\ngrille\ngrocery store\nguillotine\nhair slide\nhair spray\nhalf track\nhammer\nhamper\nhand blower\nhand-held computer\nhandkerchief\nhard disc\nharmonica\nharp\nharvester\nhatchet\nholster\nhome theater\nhoneycomb\nhook\nhoopskirt\nhorizontal bar\nhorse cart\nhourglass\niPod\niron\njack-o'-lantern\njean\njeep\njersey\njigsaw puzzle\njinrikisha\njoystick\nkimono\nknee pad\nknot\nlab coat\nladle\nlampshade\nlaptop\nlawn mower\nlens cap\nletter opener\nlibrary\nlifeboat\nlighter\nlimousine\nliner\nlipstick\nLoafer\nlotion\nloudspeaker\nloupe\nlumbermill\nmagnetic compass\nmailbag\nmailbox\nmaillot\nmaillot\nmanhole cover\nmaraca\nmarimba\nmask\nmatchstick\nmaypole\nmaze\nmeasuring cup\nmedicine chest\nmegalith\nmicrophone\nmicrowave\nmilitary uniform\nmilk can\nminibus\nminiskirt\nminivan\nmissile\nmitten\nmixing bowl\nmobile home\nModel T\nmodem\nmonastery\nmonitor\nmoped\nmortar\nmortarboard\nmosque\nmosquito net\nmotor scooter\nmountain bike\nmountain tent\nmouse\nmousetrap\nmoving van\nmuzzle\nnail\nneck brace\nnecklace\nnipple\nnotebook\nobelisk\noboe\nocarina\nodometer\noil filter\norgan\noscilloscope\noverskirt\noxcart\noxygen mask\npacket\npaddle\npaddlewheel\npadlock\npaintbrush\npajama\npalace\npanpipe\npaper towel\nparachute\nparallel bars\npark bench\nparking meter\npassenger car\npatio\npay-phone\npedestal\npencil box\npencil sharpener\nperfume\nPetri dish\nphotocopier\npick\npickelhaube\npicket fence\npickup\npier\npiggy bank\npill bottle\npillow\nping-pong ball\npinwheel\npirate\npitcher\nplane\nplanetarium\nplastic bag\nplate rack\nplow\nplunger\nPolaroid camera\npole\npolice van\nponcho\npool table\npop bottle\npot\npotter's wheel\npower drill\nprayer rug\nprinter\nprison\nprojectile\nprojector\npuck\npunching bag\npurse\nquill\nquilt\nracer\nracket\nradiator\nradio\nradio telescope\nrain barrel\nrecreational vehicle\nreel\nreflex camera\nrefrigerator\nremote control\nrestaurant\nrevolver\nrifle\nrocking chair\nrotisserie\nrubber eraser\nrugby ball\nrule\nrunning shoe\nsafe\nsafety pin\nsaltshaker\nsandal\nsarong\nsax\nscabbard\nscale\nschool bus\nschooner\nscoreboard\nscreen\nscrew\nscrewdriver\nseat belt\nsewing machine\nshield\nshoe shop\nshoji\nshopping basket\nshopping cart\nshovel\nshower cap\nshower curtain\nski\nski mask\nsleeping bag\nslide rule\nsliding door\nslot\nsnorkel\nsnowmobile\nsnowplow\nsoap dispenser\nsoccer ball\nsock\nsolar dish\nsombrero\nsoup bowl\nspace bar\nspace heater\nspace shuttle\nspatula\nspeedboat\nspider web\nspindle\nsports car\nspotlight\nstage\nsteam locomotive\nsteel arch bridge\nsteel drum\nstethoscope\nstole\nstone wall\nstopwatch\nstove\nstrainer\nstreetcar\nstretcher\nstudio couch\nstupa\nsubmarine\nsuit\nsundial\nsunglass\nsunglasses\nsunscreen\nsuspension bridge\nswab\nsweatshirt\nswimming trunks\nswing\nswitch\nsyringe\ntable lamp\ntank\ntape player\nteapot\nteddy\ntelevision\ntennis ball\nthatch\ntheater curtain\nthimble\nthresher\nthrone\ntile roof\ntoaster\ntobacco shop\ntoilet seat\ntorch\ntotem pole\ntow truck\ntoyshop\ntractor\ntrailer truck\ntray\ntrench coat\ntricycle\ntrimaran\ntripod\ntriumphal arch\ntrolleybus\ntrombone\ntub\nturnstile\ntypewriter keyboard\numbrella\nunicycle\nupright\nvacuum\nvase\nvault\nvelvet\nvending machine\nvestment\nviaduct\nviolin\nvolleyball\nwaffle iron\nwall clock\nwallet\nwardrobe\nwarplane\nwashbasin\nwasher\nwater bottle\nwater jug\nwater tower\nwhiskey jug\nwhistle\nwig\nwindow screen\nwindow shade\nWindsor tie\nwine bottle\nwing\nwok\nwooden spoon\nwool\nworm fence\nwreck\nyawl\nyurt\nweb site\ncomic book\ncrossword puzzle\nstreet sign\ntraffic light\nbook jacket\nmenu\nplate\nguacamole\nconsomme\nhot pot\ntrifle\nice cream\nice lolly\nFrench loaf\nbagel\npretzel\ncheeseburger\nhotdog\nmashed potato\nhead cabbage\nbroccoli\ncauliflower\nzucchini\nspaghetti squash\nacorn squash\nbutternut squash\ncucumber\nartichoke\nbell pepper\ncardoon\nmushroom\nGranny Smith\nstrawberry\norange\nlemon\nfig\npineapple\nbanana\njackfruit\ncustard apple\npomegranate\nhay\ncarbonara\nchocolate sauce\ndough\nmeat loaf\npizza\npotpie\nburrito\nred wine\nespresso\ncup\neggnog\nalp\nbubble\ncliff\ncoral reef\ngeyser\nlakeside\npromontory\nsandbar\nseashore\nvalley\nvolcano\nballplayer\ngroom\nscuba diver\nrapeseed\ndaisy\nyellow lady's slipper\ncorn\nacorn\nhip\nbuckeye\ncoral fungus\nagaric\ngyromitra\nstinkhorn\nearthstar\nhen-of-the-woods\nbolete\near\ntoilet tissue" }, { "path": "imageai/Detection/Custom/CUSTOMDETECTION.md", "content": "# ImageAI : Custom Object Detection\n\n### TABLE OF CONTENTS\n\n- :white_square_button: Custom Object Detection\n- :white_square_button: Object Detection, Extraction and Fine-tune\n- :white_square_button: Hiding/Showing Object Name and Probability\n- :white_square_button: Image Input & Output Types\n- :white_square_button: Documentation\n\n\nImageAI provides very convenient and powerful methods to perform object detection on images and extract each object from the image using your own **custom YOLOv3 or TinyYOLOv3 model** and the corresponding **.json** generated during the training. To test the custom object detection, you can download a sample custom model we have trained to detect the Hololens headset and its **.json** file via the links below:\n\n* [**yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt**](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt) _(Size = 236 mb)_\n* [**hololens-yolo_yolov3_detection_config.json**](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/hololens-yolo_yolov3_detection_config.json)\n\n\n Once you download the custom object detection model file, you should copy the model file to the your project folder where your **.py** files will be.\n Then create a python file and give it a name; an example is FirstCustomDetection.py. Then write the code below into the python file: \n\n### FirstCustomDetection.py\n
\n\n```python\nfrom imageai.Detection.Custom import CustomObjectDetection\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\")\ndetector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\")\ndetector.loadModel()\ndetections = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", output_image_path=\"holo2-detected.jpg\")\nfor detection in detections:\n print(detection[\"name\"], \" : \", detection[\"percentage_probability\"], \" : \", detection[\"box_points\"])\n\n```\n\nSample Result - Input:\n\n![Input](../../../data-images/holo2.jpg)\n\n Output: \n \n![Output](../../../data-images/holo2-detected.jpg)\n \n```\nhololens : 39.69653248786926 : [611, 74, 751, 154]\nhololens : 87.6643180847168 : [23, 46, 90, 79]\nhololens : 89.25175070762634 : [191, 66, 243, 95]\nhololens : 64.49641585350037 : [437, 81, 514, 133]\nhololens : 91.78624749183655 : [380, 113, 423, 138]\n\n```\n\n\nLet us make a breakdown of the object detection code that we used above.\n\n```python\nfrom imageai.Detection.Custom import CustomObjectDetection\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\n```\n In the 3 lines above , we import the **ImageAI custom object detection** class in the first line, created the class instance on the second line and set the model type to YOLOv3.\n \n```python\ndetector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\")\ndetector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\")\ndetector.loadModel()\n```\n\n In the 3 lines above, we specified the file path to our downloaded model file in the first line , specified the path to our **hololens-yolo_yolov3_detection_config.json** file in the second line and loaded the model on the third line.\n\n```python\ndetections = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", output_image_path=\"holo2-detected.jpg\")\nfor detection in detections:\n print(detection[\"name\"], \" : \", detection[\"percentage_probability\"], \" : \", detection[\"box_points\"])\n\n```\n\nIn the 3 lines above, we ran the `detectObjectsFromImage()` function and parse in the path to our test image, and the path to the new\n image which the function will save. Then the function returns an array of dictionaries with each dictionary corresponding\n to the number of objects detected in the image. Each dictionary has the properties `name` (name of the object),\n`percentage_probability` (percentage probability of the detection) and `box_points` (the x1,y1,x2 and y2 coordinates of the bounding box of the object). \n\n\n\n\n### Object Detection, Extraction and Fine-tune\n
\n\nIn the examples we used above, we ran the object detection on an image and it\nreturned the detected objects in an array as well as save a new image with rectangular markers drawn on each object. In our next examples, we will be able to extract each object from the input image and save it independently.\n \n \n\nIn the example code below which is very identical to the previous object detection code, we will save each object detected as a separate image.\n\n```python\nfrom imageai.Detection.Custom import CustomObjectDetection\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\")\ndetector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\") \ndetector.loadModel()\ndetections, extracted_objects_array = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", output_image_path=\"holo2-detected.jpg\", extract_detected_objects=True)\n\nfor detection, object_path in zip(detections, extracted_objects_array):\n print(object_path)\n print(detection[\"name\"], \" : \", detection[\"percentage_probability\"], \" : \", detection[\"box_points\"])\n print(\"---------------\")\n```\n\n\nSample Result: Output Images\n \n![](../../../data-images/holo2-detected-objects/hololens-1.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-2.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-3.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-4.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-5.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-6.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-7.jpg)\n\n\n\n\nLet us review the part of the code that perform the object detection and extract the images:\n\n```python\ndetections, extracted_objects_array = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", output_image_path=\"holo2-detected.jpg\", extract_detected_objects=True)\n\nfor detection, object_path in zip(detections, extracted_objects_array):\n print(object_path)\n print(detection[\"name\"], \" : \", detection[\"percentage_probability\"], \" : \", detection[\"box_points\"])\n print(\"---------------\")\n```\n\nIn the above above lines, we called the `detectObjectsFromImage()` , parse in the input image path, output image part, and an\nextra parameter `extract_detected_objects=True`. This parameter states that the function should extract each object detected from the image\nand save it has a seperate image. The parameter is false by default. Once set to `true`, the function will create a directory\n which is the `output image path + \"-objects\"`. Then it saves all the extracted images into this new directory with\n each image's name being the `detected object name + \"-\" + a number` which corresponds to the order at which the objects\n were detected.\n \nThis new parameter we set to extract and save detected objects as an image will make the function to return 2 values. The\n first is the array of dictionaries with each dictionary corresponding to a detected object. The second is an array of the paths\n to the saved images of each object detected and extracted, and they are arranged in order at which the objects are in the\n first array.\n\n \n\n### And one important feature you need to know!\n\nYou will recall that the percentage probability\n for each detected object is sent back by the `detectObjectsFromImage()` function. The function has a parameter\n `minimum_percentage_probability` , whose default value is `30` (value ranges between 0 - 100) , but it set to 30 in this example. That means the function will only return a detected\n object if it's percentage probability is **30 or above**. The value was kept at this number to ensure the integrity of the\n detection results. You fine-tune the object\n detection by setting `minimum_percentage_probability` equal to a smaller value to detect more number of objects or higher value to detect less number of objects.\n\n\n\n\n### Hiding/Showing Object Name and Probability\n
\n\n**ImageAI** provides options to hide the name of objects detected and/or the percentage probability from being shown on the saved/returned detected image. Using the `detectObjectsFromImage()` and `detectCustomObjectsFromImage()` functions, the parameters `'display_object_name'` and `'display_percentage_probability'` can be set to True of False individually. Take a look at the code below: \n```python\ndetections = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"holo2.jpg\"), output_image_path=os.path.join(execution_path , \"holo2_nodetails.jpg\"), minimum_percentage_probability=30, display_percentage_probability=False, display_object_name=False)\n```\n\nIn the above code, we specified that both the object name and percentage probability should not be shown. As you can see in the result below, both the names of the objects and their individual percentage probability is not shown in the detected image. \n\n**Result**\n\n![](../../../data-images/holo2-nodetails.jpg)\n\n\n### Image Input & Output Types\n
\n\n**ImageAI** custom object detection supports 2 input types of inputs which are **file path to image file**(default) and **numpy array of an image**\nas well as 2 types of output which are image **file**(default) and numpy **array **.\nThis means you can now perform object detection in production applications such as on a web server and system\n that returns file in any of the above stated formats.\n To perform object detection with numpy array input, you just need to state the input type\nin the `.detectObjectsFromImage()` function. See example below.\n\n```python\ndetections = detector.detectObjectsFromImage(input_type=\"array\", input_image=image_array , output_image_path=os.path.join(execution_path , \"holo2-detected.jpg\")) # For numpy array input type\n```\nTo perform object detection with numpy array output you just need to state the output type\nin the `.detectObjectsFromImage()` function. See example below.\n\n```python\ndetected_image_array, detections = detector.detectObjectsFromImage(output_type=\"array\", input_image=\"holo2.jpg\" ) # For numpy array output type\n```\n\n\n\n### Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions. Find links below: \n\n* Documentation - **English Version** [https://imageai.readthedocs.io](https://imageai.readthedocs.io)" }, { "path": "imageai/Detection/Custom/CUSTOMDETECTIONTRAINING.md", "content": "# ImageAI : Custom Detection Model Training \n\n## ---------------------------------------------------\n## Introducing Jarvis and TheiaEngine.\n\nWe the creators of ImageAI are glad to announce 2 new AI projects to provide state-of-the-art Generative AI, LLM and Image Understanding on your personal computer and servers. \n\n\n[![](../../../jarvis.png)](https://jarvis.genxr.co)\n\nInstall Jarvis on PC/Mac to setup limitless access to LLM powered AI Chats for your every day work, research and generative AI needs with 100% privacy and full offline capability.\n\n\nVisit [https://jarvis.genxr.co](https://jarvis.genxr.co/) to get started.\n\n\n[![](../../../theiaengine.png)](https://www.genxr.co/theia-engine)\n\n\n[TheiaEngine](https://www.genxr.co/theia-engine), the next-generation computer Vision AI API capable of all Generative and Understanding computer vision tasks in a single API call and available via REST API to all programming languages. Features include\n- **Detect 300+ objects** ( 220 more objects than ImageAI)\n- **Provide answers to any content or context questions** asked on an image\n - very useful to get information on any object, action or information without needing to train a new custom model for every tasks\n- **Generate scene description and summary**\n- **Convert 2D image to 3D pointcloud and triangular mesh**\n- **Semantic Scene mapping of objects, walls, floors, etc**\n- **Stateless Face recognition and emotion detection**\n- **Image generation and augmentation from prompt**\n- etc.\n\nVisit [https://www.genxr.co/theia-engine](https://www.genxr.co/theia-engine) to try the demo and join in the beta testing today.\n## ---------------------------------------------------\n\n**ImageAI** provides the most simple and powerful approach to training custom object detection models\nusing the YOLOv3 architeture, which\nwhich you can load into the `imageai.Detection.Custom.CustomObjectDetection` class. This allows\n you to train your own **YOLOv3** or **TinyYOLOv3** model on any set of images that corresponds to any type of objects of interest.\nThe training process generates a JSON file that maps the objects names in your image dataset and the detection anchors, as well as creates lots of models. In choosing the best model for your custom object detection task, an `evaluateModel()` function has been provided to compute the **mAP** of your saved models by allowing you to state your desired **IoU** and **Non-maximum Suppression** values. Then you can perform custom\nobject detection using the model and the JSON file generated. \n\n### TABLE OF CONTENTS\n- :white_square_button: Preparing your custom dataset\n- :white_square_button: Training on your custom Dataset\n- :white_square_button: Evaluating your saved detection models' mAP\n\n\n### Preparing your custom dataset\n
\n\nTo train a custom detection model, you need to prepare the images you want to use to train the model. \nYou will prepare the images as follows: \n\n1. Decide the type of object(s) you want to detect and collect about **200 (minimum recommendation)** or more picture of each of the object(s)\n2. Once you have collected the images, you need to annotate the object(s) in the images. **ImageAI** uses the **YOLO** for image annotation. You can generate this annotation for your images using the easy to use [**LabelImg**](https://github.com/tzutalin/labelImg) image annotation tool, available for Windows, Linux and MacOS systems. Open the link below to install the annotation tool. See: [https://github.com/tzutalin/labelImg](https://github.com/tzutalin/labelImg)\n3. When you are done annotating your images, **annotation .txt** files will be generated for each image in your dataset. The **annotation .txt** file describes each or **all** of the objects in the image. For example, if each image your image names are **image(1).jpg**, **image(2).jpg**, **image(3).jpg** till **image(z).jpg**; the corresponding annotation for each of the images will be **image(1).txt**, **image(2).txt**, **image(3).txt** till **image(z).txt**. \n4. Once you have the annotations for all your images, create a folder for your dataset (E.g headsets) and in this parent folder, create child folders **train** and **validation**\n5. In the train folder, create **images** and **annotations**\n sub-folders. Put about 70-80% of your dataset of each object's images in the **images** folder and put the corresponding annotations for these images in the **annotations** folder. \n6. In the validation folder, create **images** and **annotations** sub-folders. Put the rest of your dataset images in the **images** folder and put the corresponding annotations for these images in the **annotations** folder.\n7. Once you have done this, the structure of your image dataset folder should look like below: \n ```\n >> train >> images >> img_1.jpg (shows Object_1)\n >> images >> img_2.jpg (shows Object_2)\n >> images >> img_3.jpg (shows Object_1, Object_3 and Object_n)\n >> annotations >> img_1.txt (describes Object_1)\n >> annotations >> img_2.txt (describes Object_2)\n >> annotations >> img_3.txt (describes Object_1, Object_3 and Object_n)\n \n >> validation >> images >> img_151.jpg (shows Object_1, Object_3 and Object_n)\n >> images >> img_152.jpg (shows Object_2)\n >> images >> img_153.jpg (shows Object_1)\n >> annotations >> img_151.txt (describes Object_1, Object_3 and Object_n)\n >> annotations >> img_152.txt (describes Object_2)\n >> annotations >> img_153.txt (describes Object_1)\n ```\n8. You can train your custom detection model completely from scratch or use transfer learning (recommended for better accuracy) from a pre-trained YOLOv3 model. Also, we have provided a sample annotated Hololens and Headsets (Hololens and Oculus) dataset for you to train with. Download the pre-trained YOLOv3 model and the sample datasets in the link below. \n\nDownload dataset `hololens-yolo.zip` [here](https://github.com/OlafenwaMoses/ImageAI/releases/tag/test-resources-v3) and pre-trained model `yolov3.pt` [here](https://github.com/OlafenwaMoses/ImageAI/releases/tag/3.0.0-pretrained)\n\n\n### Training on your custom dataset\n
\n\nBefore you start training your custom detection model, kindly take note of the following: \n\n- The default **batch_size** is 4. If you are training with **Google Colab**, this will be fine. However, I will advice you use a more powerful GPU than the K80 offered by Colab as the higher your **batch_size (8, 16)**, the better the accuracy of your detection model. \n\nThen your training code goes as follows: \n```python\nfrom imageai.Detection.Custom import DetectionModelTrainer\n\ntrainer = DetectionModelTrainer()\ntrainer.setModelTypeAsYOLOv3()\ntrainer.setDataDirectory(data_directory=\"hololens-yolo\")\ntrainer.setTrainConfig(object_names_array=[\"hololens\"], batch_size=4, num_experiments=200, train_from_pretrained_model=\"yolov3.pt\")\n# In the above,when training for detecting multiple objects,\n#set object_names_array=[\"object1\", \"object2\", \"object3\",...\"objectz\"]\ntrainer.trainModel()\n```\n\n Yes! Just 6 lines of code and you can train object detection models on your custom dataset.\nNow lets take a look at how the code above works. \n\n```python\nfrom imageai.Detection.Custom import DetectionModelTrainer\n\ntrainer = DetectionModelTrainer()\ntrainer.setModelTypeAsYOLOv3()\ntrainer.setDataDirectory(data_directory=\"hololens-yolo\")\n```\n\nIn the first line, we import the **ImageAI** detection model training class, then we define the model trainer in the second line,\n we set the network type in the third line and set the path to the image dataset we want to train the network on.\n\n```python\ntrainer.setTrainConfig(object_names_array=[\"hololens\"], batch_size=4, num_experiments=200, train_from_pretrained_model=\"yolov3.pt\")\n```\n\n\nIn the line above, we configured our detection model trainer. The parameters we stated in the function as as below: \n\n- **num_objects** : this is an array containing the names of the objects in our dataset\n- **batch_size** : this is to state the batch size for the training\n- **num_experiments** : this is to state the number of times the network will train over all the training images,\n which is also called epochs \n- **train_from_pretrained_model(optional)** : this is to train using transfer learning from a pre-trained **YOLOv3** model\n\n```python\ntrainer.trainModel()\n```\n\n\nWhen you start the training, you should see something like this in the console: \n```\nGenerating anchor boxes for training images...\nthr=0.25: 1.0000 best possible recall, 6.93 anchors past thr\nn=9, img_size=416, metric_all=0.463/0.856-mean/best, past_thr=0.549-mean:\n====================\nPretrained YOLOv3 model loaded to initialize weights\n====================\nEpoch 1/100\n----------\nTrain:\n30it [00:14, 2.09it/s]\n box loss-> 0.09820, object loss-> 0.27985, class loss-> 0.00000\nValidation:\n15it [01:45, 7.05s/it]\n recall: 0.085714 precision: 0.000364 mAP@0.5: 0.000186, mAP@0.5-0.95: 0.000030\n\nEpoch 2/100\n----------\nTrain:\n30it [00:07, 4.25it/s]\n box loss-> 0.08691, object loss-> 0.07011, class loss-> 0.00000\nValidation:\n15it [01:37, 6.53s/it]\n recall: 0.214286 precision: 0.000854 mAP@0.5: 0.000516, mAP@0.5-0.95: 0.000111\n.\n.\n.\n.\n\n```\n\nLet us explain the details shown above: \n```\nGenerating anchor boxes for training images...\nthr=0.25: 1.0000 best possible recall, 6.93 anchors past thr\nn=9, img_size=416, metric_all=0.463/0.856-mean/best, past_thr=0.549-mean:\n====================\nPretrained YOLOv3 model loaded to initialize weights\n====================\n```\n\nThe above details signifies the following: \n- **ImageAI** autogenerates the best match detection **anchor boxes** for your image dataset. \n\n- A the pretrained **yolov3.pt** was loaded to initalize the weights used to train the model.\n\n```\nEpoch 1/100\n----------\nTrain:\n30it [00:14, 2.09it/s]\n box loss-> 0.09820, object loss-> 0.27985, class loss-> 0.00000\nValidation:\n15it [01:45, 7.05s/it]\n recall: 0.085714 precision: 0.000364 mAP@0.5: 0.000186, mAP@0.5-0.95: 0.000030\n\nEpoch 2/100\n----------\nTrain:\n30it [00:07, 4.25it/s]\n box loss-> 0.08691, object loss-> 0.07011, class loss-> 0.00000\nValidation:\n15it [01:37, 6.53s/it]\n recall: 0.214286 precision: 0.000854 mAP@0.5: 0.000516, mAP@0.5-0.95: 0.000111\n```\n\n- The above signifies the progress of the training. \n- For each experiment (Epoch), a number of metrics are computed. The important once fo chosing an accuate models is detailed below\n - The bounding box loss `box loss` is reported and expected to drop as the training progresses\n - The object localization loss `object loss` is reported and expected to drop as the training progresses\n - The class loss `class loss` is reported and expected to drop as the training progresses. If the class loss persists at 0.0000, it's because your dataset has a single class.\n - The `mAP50` and `mAP0.5-0.95` metrics are expected to increase. This signifies the models accuracy increases. There might be flunctuations in these metrics sometimes.\n- For each increase in the `mAP50` after an experiment, a model is saved in the **hololens-yolo/models** folder. The higher the mAP50, the better the model. \n\nOnce you are done training, you can visit the link below for performing object detection with your **custom detection model** and **detection_config.json** file.\n\n[Detection/Custom/CUSTOMDETECTION.md](./CUSTOMDETECTION.md)\n \n \n### >> Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions. Find links below: \n\n* Documentation - **English Version** [https://imageai.readthedocs.io](https://imageai.readthedocs.io)\n\n\n\n\n\n\n" }, { "path": "imageai/Detection/Custom/CUSTOMVIDEODETECTION.md", "content": "# ImageAI : Custom Video Object Detection, Tracking and Analysis\n\n### TABLE OF CONTENTS\n\n- :white_square_button: First Custom Video Object Detection\n- :white_square_button: Camera / Live Stream Video Detection\n- :white_square_button: Video Analysis\n- :white_square_button: Hiding/Showing Object Name and Probability\n- :white_square_button: Frame Detection Intervals\n- :white_square_button: Video Detection Timeout (NEW)\n- :white_square_button: Documentation\n\n\nImageAI provides convenient, flexible and powerful methods to perform object detection on videos using your own **custom YOLOv3 model** and the corresponding **.json** file generated during the training. This version of **ImageAI** provides commercial grade video objects detection features, which include but not limited to device/IP camera inputs, per frame, per second, per minute and entire video analysis for storing in databases and/or real-time visualizations and for future insights.\nTo test the custom video object detection,you can download a sample custom model we have trained to detect the Hololens headset and its **.json** file via the links below:\n\n* [**yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt**](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt) _(Size = 236 mb)_\n* [**hololens-yolo_yolov3_detection_config.json**](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/hololens-yolo_yolov3_detection_config.json)\n\n\nBecause video object detection is a compute intensive tasks, we advise you perform this experiment using a computer with a NVIDIA GPU and the GPU version of Tensorflow\n installed. Performing Video Object Detection CPU will be slower than using an NVIDIA GPU powered computer. You can use Google Colab for this\n experiment as it has an NVIDIA K80 GPU available for free.\n
\n Once you download the custom object detection model and JSON files, you should copy the model and the JSON files to the your project folder where your .py files will be.\n Then create a python file and give it a name; an example is FirstCustomVideoObjectDetection.py. Then write the code below into the python file:
\n\n\n### FirstCustomVideoObjectDetection.py\n
\n\n```python\nfrom imageai.Detection.Custom import CustomVideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\")\nvideo_detector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\")\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(input_file_path=\"holo1.mp4\",\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20,\n minimum_percentage_probability=40,\n log_progress=True)\n```\n\n[**Input Video**](../../../data-videos/holo1.mp4)\n[![Input Video](../../../data-images/holo-video.jpg)](../../../data-videos/holo1.mp4)\n[**Output Video**](https://www.youtube.com/watch?v=4o5GyAR4Mpw)\n[![Output Video](../../../data-images/holo-video-detected.jpg)](https://www.youtube.com/watch?v=4o5GyAR4Mpw)\n\n\n\nLet us make a breakdown of the object detection code that we used above.\n\n```python\nfrom imageai.Detection.Custom import CustomVideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n```\n\nIn the 3 lines above , we import the **ImageAI custom video object detection** class in the first line, import the **os** in the second line and obtained\n the path to folder where our python file runs.\n```python\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\")\nvideo_detector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\")\nvideo_detector.loadModel()\n```\nIn the 4 lines above, we created a new instance of the `CustomVideoObjectDetection` class in the first line, set the model type to YOLOv3 in the second line,\n set the model path to our custom YOLOv3 model file in the third line, specified the path to the model's corresponding **hololens-yolo_yolov3_detection_config.json** in the fourth line and load the model in the fifth line.\n\n```python\nvideo_detector.detectObjectsFromVideo(input_file_path=\"holo1.mp4\",\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20,\n minimum_percentage_probability=40,\n log_progress=True)\n```\n\nIn the code above, we ran the `detectObjectsFromVideo()` function and parse in the path to our video,the path to the new\n video (without the extension, it saves a .mp4 video by default) which the function will save, the number of frames per second (fps) that\n you we desire the output video to have and option to log the progress of the detection in the console. Then the function returns a the path to the saved video\n which contains boxes and percentage probabilities rendered on objects detected in the video.\n\n\n### Camera / Live Stream Video Detection\n
\n\n**ImageAI** now allows live-video detection with support for camera inputs. Using **OpenCV**'s **VideoCapture()** function, you can load live-video streams from a device camera, cameras connected by cable or IP cameras, and parse it into **ImageAI**'s **detectObjectsFromVideo()** function. All features that are supported for detecting objects in a video file is also available for detecting objects in a camera's live-video feed. Find below an example of detecting live-video feed from the device camera. \n\n```python\nfrom imageai.Detection.Custom import CustomVideoObjectDetection\nimport os\nimport cv2\n\nexecution_path = os.getcwd()\ncamera = cv2.VideoCapture(0)\n\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\")\nvideo_detector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\")\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20,\n minimum_percentage_probability=40,\n log_progress=True)\n```\n\nThe difference in the code above and the code for the detection of a video file is that we defined an **OpenCV VideoCapture** instance and loaded the default device camera into it. Then we parsed the camera we defined into the parameter **camera_input** which replaces the **input_file_path** that is used for video file. \n\n\n### Video Analysis\n
\n\n**ImageAI** now provide commercial-grade video analysis in the Custom Video Object Detection class, for both video file inputs and camera inputs. This feature allows developers to obtain deep insights into any video processed with **ImageAI**. This insights can be visualized in real-time, stored in a NoSQL database for future review or analysis.
\n\nFor video analysis, the **detectObjectsFromVideo()** now allows you to state your own defined functions which will be executed for every frame, seconds and/or minute of the video detected as well as a state a function that will be executed at the end of a video detection. Once this functions are stated, they will receive raw but comprehensive analytical data on the index of the frame/second/minute, objects detected (name, percentage_probability and box_points), number of instances of each unique object detected and average number of occurrence of each unique object detected over a second/minute and entire video.\n\nTo obtain the video analysis, all you need to do is specify a function, state the corresponding parameters it will be receiving and parse the function name into the **per_frame_function**, **per_second_function**, **per_minute_function** and **video_complete_function** parameters in the detection function. Find below examples of video analysis functions. \n\n```python\ndef forFrame(frame_number, output_array, output_count):\n print(\"FOR FRAME \" , frame_number)\n print(\"Output for each object : \", output_array)\n print(\"Output count for unique objects : \", output_count)\n print(\"------------END OF A FRAME --------------\")\n\ndef forSeconds(second_number, output_arrays, count_arrays, average_output_count):\n print(\"SECOND : \", second_number)\n print(\"Array for the outputs of each frame \", output_arrays)\n print(\"Array for output count for unique objects in each frame : \", count_arrays)\n print(\"Output average count for unique objects in the last second: \", average_output_count)\n print(\"------------END OF A SECOND --------------\")\n\ndef forMinute(minute_number, output_arrays, count_arrays, average_output_count):\n print(\"MINUTE : \", minute_number)\n print(\"Array for the outputs of each frame \", output_arrays)\n print(\"Array for output count for unique objects in each frame : \", count_arrays)\n print(\"Output average count for unique objects in the last minute: \", average_output_count)\n print(\"------------END OF A MINUTE --------------\")\n\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\")\nvideo_detector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\")\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20, per_second_function=forSeconds, per_frame_function = forFrame, per_minute_function= forMinute,\n minimum_percentage_probability=40,\n log_progress=True)\n```\n\n\n**ImageAI** also allows you to obtain complete analysis of the entire video processed. All you need is to define a function like the forSecond or forMinute function and set the **video_complete_function** parameter into your **.detectObjectsFromVideo()** function. The same values for the per_second-function and per_minute_function will be returned. The difference is that no index will be returned and the other 3 values will be returned, and the 3 values will cover all frames in the video. Below is a sample function: \n```python\ndef forFull(output_arrays, count_arrays, average_output_count):\n #Perform action on the 3 parameters returned into the function\n\n\nvideo_detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n video_complete_function=forFull,\n minimum_percentage_probability=40,\n log_progress=True)\n\n```\n\n**FINAL NOTE ON VIDEO ANALYSIS** : **ImageAI** allows you to obtain the detected video frame as a Numpy array at each frame, second and minute function. All you need to do is specify one more parameter in your function and set **return_detected_frame=True** in your **detectObjectsFromVideo()** function. Once this is set, the extra parameter you sepecified in your function will be the Numpy array of the detected frame. See a sample below:\n\n```python\ndef forFrame(frame_number, output_array, output_count, detected_frame):\n print(\"FOR FRAME \" , frame_number)\n print(\"Output for each object : \", output_array)\n print(\"Output count for unique objects : \", output_count)\n\tprint(\"Returned Objects is : \", type(detected_frame))\n print(\"------------END OF A FRAME --------------\")\n\n\nvideo_detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n per_frame_function=forFrame,\n minimum_percentage_probability=40,\n log_progress=True, return_detected_frame=True)\n```\n\n\n### Frame Detection Intervals\n
\n\nThe above video objects detection task are optimized for frame-real-time object detections that ensures that objects in every frame\nof the video is detected. **ImageAI** provides you the option to adjust the video frame detections which can speed up\nyour video detection process. When calling the `.detectObjectsFromVideo()`, you can\nspecify at which frame interval detections should be made. By setting the **frame_detection_interval** parameter to be\n equal to 5 or 20, that means the object detections in the video will be updated after 5 frames or 20 frames.\nIf your output video **frames_per_second** is set to 20, that means the object detections in the video will\n be updated once in every quarter of a second or every second. This is useful in case scenarios where the available\n compute is less powerful and speeds of moving objects are low. This ensures you can have objects detected as second-real-time\n, half-a-second-real-time or whichever way suits your needs. \n\n\n### Custom Video Detection Timeout\n
\n\n**ImageAI** now allows you to set a timeout in seconds for detection of objects in videos or camera live feed. To set a timeout for your video detection code, all you need to do is specify the `detection_timeout` parameter in the `detectObjectsFromVideo()` function to the number of desired seconds. In the example code below, we set `detection_timeout` to 120 seconds (2 minutes). \n\n\n```python\nfrom imageai.Detection.Custom import CustomVideoObjectDetection\nimport os\nimport cv2\n\nexecution_path = os.getcwd()\ncamera = cv2.VideoCapture(0)\n\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"yolov3_hololens-yolo_mAP-0.82726_epoch-73.pt\")\nvideo_detector.setJsonPath(\"hololens-yolo_yolov3_detection_config.json\")\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20, minimum_percentage_probability=40,\n detection_timeout=120)\n```\n\n\n### >> Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions. Find links below: \n\n* Documentation - **English Version** [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**\n\n" }, { "path": "imageai/Detection/Custom/__init__.py", "content": "import os\nimport time\nimport math\nimport json\nimport warnings\nfrom typing import List, Union, Tuple, Dict\nfrom collections import defaultdict\n\nimport numpy as np\nfrom PIL import Image\nimport cv2\nimport torch\nfrom torch.cuda import amp\nfrom torch.utils.data import DataLoader\nfrom torch.optim import SGD, lr_scheduler\nfrom tqdm import tqdm\n\nfrom .yolo.dataset import LoadImagesAndLabels\nfrom .yolo.custom_anchors import generate_anchors\nfrom .yolo.compute_loss import compute_loss\nfrom .yolo import validate\nfrom ...yolov3.tiny_yolov3 import YoloV3Tiny\nfrom ...yolov3.yolov3 import YoloV3\nfrom ...yolov3.utils import draw_bbox_and_label, get_predictions, prepare_image\n\nfrom ...backend_check.model_extension import extension_check\n\n\nclass DetectionModelTrainer:\n \"\"\"\n This is the Detection Model training class, which allows you to train object detection models\n on image datasets that are in YOLO format, using the YOLOv3.\n \"\"\"\n\n def __init__(self) -> None:\n self.__device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.__cuda = (self.__device != \"cpu\")\n self.__model_type = \"\"\n self.__model = None\n self.__optimizer = None\n self.__data_dir = \"\"\n self.__classes: List[str] = None\n self.__num_classes = None\n self.__anchors = None\n self.__dataset_name = None\n self.__mini_batch_size: int = None\n self.__scaler = amp.GradScaler(enabled=self.__cuda)\n self.__lr_lambda = None\n self.__custom_train_dataset = None\n self.__custom_val_dataset = None\n self.__train_loader = None\n self.__val_loader = None\n\n self.__model_path: str = None\n self.__epochs: int = None\n self.__output_models_dir: str = None\n self.__output_json_dir: str = None\n\n def __set_training_param(self, epochs : int, accumulate : int) -> None:\n # self.__lr_lambda = lambda x : ((1 - math.cos(x * math.pi / epochs)) / 2 ) * (0.1 - 1.0) + 1.0\n self.__lr_lambda = lambda x: (1 - x / (epochs - 1)) * (1.0 - 0.01) + 0.01\n self.__anchors = generate_anchors(\n self.__custom_train_dataset,\n n=9 if self.__model_type==\"yolov3\" else 6\n )\n self.__anchors = [round(i) for i in self.__anchors.reshape(-1).tolist()]\n if self.__model_type == \"yolov3\":\n self.__model = YoloV3(\n num_classes=self.__num_classes,\n anchors=self.__anchors,\n device=self.__device\n )\n elif self.__model_type == \"tiny-yolov3\":\n self.__model = YoloV3Tiny(\n num_classes=self.__num_classes,\n anchors=self.__anchors,\n device=self.__device\n )\n if self.__model_path:\n self.__load_model()\n\n w_d = (5e-4) * (self.__mini_batch_size * accumulate / 64) # scale weight decay\n g0, g1, g2 = [], [], [] # optimizer parameter groups\n for m in self.__model.modules():\n if hasattr(m, 'bias') and isinstance(m.bias, torch.nn.Parameter): # bias\n g2.append(m.bias)\n if isinstance(m, torch.nn.BatchNorm2d): # weight (no decay)\n g0.append(m.weight)\n elif hasattr(m, 'weight') and isinstance(m.weight, torch.nn.Parameter): # weight (with decay)\n g1.append(m.weight)\n\n self.__optimizer = SGD(\n g0,\n lr=1e-2,\n momentum=0.6,\n # weight_decay=w_d,\n nesterov=True\n )\n self.__optimizer.add_param_group({'params': g1, 'weight_decay': w_d}) # add g1 with weight_decay\n self.__optimizer.add_param_group({'params': g2}) # add g2 (biases)\n self.__lr_scheduler = lr_scheduler.LambdaLR(\n self.__optimizer,\n lr_lambda=self.__lr_lambda\n )\n del g0, g1, g2\n self.__model.to(self.__device)\n\n def __load_model(self) -> None:\n try:\n state_dict = torch.load(self.__model_path, map_location=self.__device)\n # check against cases where number of classes differs, causing the\n # channel of the convolutional layer just before the detection layer\n # to differ.\n new_state_dict = {k:v for k,v in state_dict.items() if k in self.__model.state_dict().keys() and v.shape==self.__model.state_dict()[k].shape}\n self.__model.load_state_dict(new_state_dict, strict=False)\n print(\"=\"*20)\n print(\"Pretrained YOLOv3 model loaded to initialize weights\")\n print(\"=\"*20)\n except Exception as e:\n print(\"=\"*20)\n print(\"pretrained weight loading failed. Defaulting to using random weight.\")\n print(\"=\"*20)\n\n def __load_data(self) -> None:\n self.__num_classes = len(self.__classes)\n self.__dataset_name = os.path.basename(os.path.dirname(self.__data_dir+os.path.sep))\n self.__custom_train_dataset = LoadImagesAndLabels(self.__data_dir, train=True)\n self.__custom_val_dataset = LoadImagesAndLabels(self.__data_dir, train=False)\n self.__train_loader = DataLoader(\n self.__custom_train_dataset, batch_size=self.__mini_batch_size,\n shuffle=True,\n collate_fn=self.__custom_train_dataset.collate_fn\n )\n self.__val_loader = DataLoader(\n self.__custom_val_dataset, batch_size=self.__mini_batch_size//2,\n shuffle=True, collate_fn=self.__custom_val_dataset.collate_fn\n )\n\n def setModelTypeAsYOLOv3(self) -> None:\n \"\"\"\n 'setModelTypeAsYOLOv3()' is used to set the model type to the YOLOv3 model.\n :return:\n \"\"\"\n self.__model_type = \"yolov3\"\n\n def setModelTypeAsTinyYOLOv3(self) -> None:\n \"\"\"\n 'setModelTypeAsTinyYOLOv3()' is used to set the model type to the TinyYOLOv3 model.\n :return:\n \"\"\"\n self.__model_type = \"tiny-yolov3\"\n\n def setDataDirectory(self, data_directory: str):\n \"\"\"\n 'setDataDirectory()' is required to set the path to which the data/dataset to be used for training is kept. The input dataset must be in the YOLO format. The directory can have any name, but it must have 'train' and 'validation'\n sub-directory. In the 'train' and 'validation' sub-directories, there must be 'images' and 'annotations'\n sub-directories respectively. The 'images' folder will contain the pictures for the dataset and the\n 'annotations' folder will contain the TXT files with details of the annotations for each image in the\n 'images folder'.\n N.B: Strictly take note that the filenames (without the extension) of the pictures in the 'images folder'\n must be the same as the filenames (except the extension) of their corresponding annotation TXT files in\n the 'annotations' folder.\n The structure of the 'train' and 'validation' folder must be as follows:\n >> train >> images >> img_1.jpg\n >> images >> img_2.jpg\n >> images >> img_3.jpg\n >> annotations >> img_1.txt\n >> annotations >> img_2.txt\n >> annotations >> img_3.txt\n >> validation >> images >> img_151.jpg\n >> images >> img_152.jpg\n >> images >> img_153.jpg\n >> annotations >> img_151.txt\n >> annotations >> img_152.txt\n >> annotations >> img_153.txt\n :param data_directory:\n :return:\n \"\"\"\n if os.path.isdir(data_directory):\n self.__data_dir = data_directory\n else:\n raise ValueError(\n \"The parameter passed should point to a valid directory\"\n )\n def setTrainConfig(self, object_names_array: List[str], batch_size: int=4, num_experiments=100, train_from_pretrained_model: str = None):\n \"\"\"\n 'setTrainConfig()' function allows you to set the properties for the training instances. It accepts the following values:\n - object_names_array , this is an array of the names of the different objects in your dataset, in the index order your dataset is annotated\n - batch_size (optional), this is the batch size for the training instance\n - num_experiments (optional), also known as epochs, it is the number of times the network will train on all the training dataset\n - train_from_pretrained_model (optional), this is used to perform transfer learning by specifying the path to a pre-trained YOLOv3 or TinyYOLOv3 model\n :param object_names_array:\n :param batch_size:\n :param num_experiments:\n :param train_from_pretrained_model:\n :return:\n \"\"\"\n self.__model_path = train_from_pretrained_model\n if self.__model_path:\n extension_check(self.__model_path)\n self.__classes = object_names_array\n self.__mini_batch_size = batch_size\n self.__epochs = num_experiments\n self.__output_models_dir = os.path.join(self.__data_dir, \"models\")\n self.__output_json_dir = os.path.join(self.__data_dir, \"json\")\n\n def trainModel(self) -> None:\n \"\"\"\n 'trainModel()' function starts the actual model training. Once the training starts, the training instance\n creates 3 sub-folders in your dataset folder which are:\n - json, where the JSON configuration file for using your trained model is stored\n - models, where your trained models are stored once they are generated after each improved experiments\n - cache , where temporary traing configuraton files are stored\n :return:\n \"\"\"\n\n self.__load_data()\n os.makedirs(self.__output_models_dir, exist_ok=True)\n os.makedirs(self.__output_json_dir, exist_ok=True)\n\n mp, mr, map50, map50_95, best_fitness = 0, 0, 0, 0, 0.0\n nbs = 64 # norminal batch size\n nb = len(self.__train_loader) # number of batches\n nw = max(3 * nb, 1000) # number of warmup iterations.\n last_opt_step = -1\n prev_save_name, recent_save_name = \"\", \"\"\n\n accumulate = max(round(nbs / self.__mini_batch_size), 1) # accumulate loss before optimizing.\n\n self.__set_training_param(self.__epochs, accumulate)\n\n with open(os.path.join(self.__output_json_dir, f\"{self.__dataset_name}_{self.__model_type}_detection_config.json\"), \"w\") as configWriter:\n json.dump(\n {\n \"labels\": self.__classes,\n \"anchors\": self.__anchors\n },\n configWriter\n )\n\n since = time.time()\n\n self.__lr_scheduler.last_epoch = -1\n\n for epoch in range(1, self.__epochs+1):\n self.__optimizer.zero_grad()\n mloss = torch.zeros(3, device=self.__device)\n print(f\"Epoch {epoch}/{self.__epochs}\", \"-\"*10, sep=\"\\n\")\n\n for phase in [\"train\", \"validation\"]:\n if phase==\"train\":\n self.__model.train()\n print(\"Train: \")\n for batch_i, (data, anns) in tqdm(enumerate(self.__train_loader)):\n batches_done = batch_i + nb * epoch\n\n data = data.to(self.__device)\n anns = anns.to(self.__device)\n\n # warmup\n if batches_done <= nw:\n xi = [0, nw] # x interp\n accumulate = max(1, np.interp(batches_done, xi, [1, nbs / self.__mini_batch_size]).round())\n for j, x in enumerate(self.__optimizer.param_groups):\n # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0\n x['lr'] = np.interp(batches_done, xi, [0.1 if j == 2 else 0.0, 0.01 * self.__lr_lambda(epoch)])\n if 'momentum' in x:\n x['momentum'] = np.interp(batches_done, xi, [0.8, 0.9])\n\n with amp.autocast(enabled=self.__cuda):\n _ = self.__model(data)\n loss_layers = self.__model.get_loss_layers()\n loss, loss_components = compute_loss(loss_layers, anns.detach(), self.__device)\n\n self.__scaler.scale(loss).backward()\n mloss = (mloss * batch_i + loss_components) / (batch_i + 1)\n\n # Optimize\n if batches_done - last_opt_step >= accumulate:\n self.__scaler.step(self.__optimizer) # optimizer.step\n self.__scaler.update()\n self.__optimizer.zero_grad()\n last_opt_step = batches_done\n\n print(f\" box loss-> {float(mloss[0]):.5f}, object loss-> {float(mloss[1]):.5f}, class loss-> {float(mloss[2]):.5f}\")\n\n self.__lr_scheduler.step()\n\n else:\n self.__model.eval()\n print(\"Validation:\")\n\n mp, mr, map50, map50_95 = validate.run(\n self.__model, self.__val_loader,\n self.__num_classes, device=self.__device\n )\n \n print(f\" recall: {mr:0.6f} precision: {mp:0.6f} mAP@0.5: {map50:0.6f}, mAP@0.5-0.95: {map50_95:0.6f}\" \"\\n\")\n\n if map50 > best_fitness:\n best_fitness = map50\n recent_save_name = self.__model_type+f\"_{self.__dataset_name}_mAP-{best_fitness:0.5f}_epoch-{epoch}.pt\"\n if prev_save_name:\n os.remove(os.path.join(self.__output_models_dir, prev_save_name))\n torch.save(\n self.__model.state_dict(),\n os.path.join(self.__output_models_dir, recent_save_name)\n )\n prev_save_name = recent_save_name\n\n if epoch == self.__epochs:\n torch.save(\n self.__model.state_dict(),\n os.path.join(self.__output_models_dir, self.__model_type+f\"_{self.__dataset_name}_last.pt\")\n )\n\n elapsed_time = time.time() - since\n print(f\"Training completed in {elapsed_time//60:.0f}m {elapsed_time % 60:.0f}s\")\n torch.cuda.empty_cache()\n\n\nclass CustomObjectDetection:\n \"\"\"\n This is the object detection class for using your custom trained models. \n It supports your custom trained YOLOv3 and TinyYOLOv3 model and allows \n to you to perform object detection in images.\n \"\"\"\n def __init__(self) -> None:\n self.__device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.__anchors: List[int] = None\n self.__classes: List[str] = None \n self.__model = None\n self.__model_loaded: bool = False\n self.__model_path: str = None\n self.__json_path: str = None\n self.__model_type: str = None\n self.__nms_score = 0.4\n self.__objectness_score = 0.4\n \n def setModelTypeAsYOLOv3(self) -> None:\n \"\"\"\n 'setModelTypeAsYOLOv3()' is used to set the model type to the YOLOv3 model.\n :return:\n \"\"\"\n self.__model_type = \"yolov3\"\n\n def setModelTypeAsTinyYOLOv3(self) -> None:\n \"\"\"\n 'setModelTypeAsTinyYOLOv3()' is used to set the model type to the TinyYOLOv3 model.\n :return:\n \"\"\"\n self.__model_type = \"tiny-yolov3\"\n \n def setModelPath(self, model_path: str):\n if os.path.isfile(model_path):\n extension_check(model_path)\n self.__model_path = model_path\n self.__model_loaded = False\n else:\n raise ValueError(\n \"invalid path, path not pointing to the weightfile.\"\n ) from None\n self.__model_path = model_path\n \n def setJsonPath(self, configuration_json: str):\n self.__json_path = configuration_json\n \n def __load_classes_and_anchors(self) -> List[str]:\n\n with open(self.__json_path) as f:\n json_config = json.load(f)\n self.__anchors = json_config[\"anchors\"]\n self.__classes = json_config[\"labels\"]\n\n def __load_image_yolo(self, input_image : Union[str, np.ndarray, Image.Image]) -> Tuple[List[str], List[np.ndarray], torch.Tensor, torch.Tensor]:\n \"\"\"\n Loads image/images from the given path. If the given path is a directory,\n this function only load the images in the directory (it does noot visit the\n subdirectories).\n \"\"\"\n allowed_exts = [\"jpg\", \"jpeg\", \"png\"]\n fnames = []\n original_dims = []\n inputs = []\n original_imgs = []\n if type(input_image) == str:\n if os.path.isfile(input_image):\n if input_image.rsplit('.')[-1].lower() in allowed_exts:\n img = cv2.imread(input_image)\n else:\n raise ValueError(f\"image path '{input_image}' is not found or a valid file\")\n elif type(input_image) == np.ndarray:\n img = input_image\n elif \"PIL\" in str(type(input_image)):\n img = np.asarray(input_image)\n else:\n raise ValueError(f\"Invalid image input format\")\n \n img_h, img_w, _ = img.shape\n\n original_imgs.append(np.array(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)).astype(np.uint8))\n original_dims.append((img_w, img_h))\n if type(input_image) == str:\n fnames.append(os.path.basename(input_image)) \n else:\n fnames.append(\"\") \n inputs.append(prepare_image(img, (416, 416)))\n\n if original_dims:\n return (\n fnames,\n original_imgs,\n torch.FloatTensor(original_dims).repeat(1,2).to(self.__device),\n torch.cat(inputs, 0).to(self.__device)\n )\n raise RuntimeError(\n f\"Error loading image.\"\n \"\\nEnsure the file is a valid image,\"\n \" allowed file extensions are .jpg, .jpeg, .png\"\n )\n \n def useCPU(self):\n \"\"\"\n Used to force classification to be done on CPU.\n By default, classification will occur on GPU compute if available else CPU compute.\n \"\"\"\n\n self.__device = \"cpu\"\n if self.__model_loaded:\n self.__model_loaded = False\n self.loadModel()\n \n def loadModel(self) -> None:\n \"\"\"\n Loads the pretrained weights in the specified model path.\n \"\"\"\n self.__load_classes_and_anchors()\n\n if self.__model_type == \"yolov3\":\n self.__model = YoloV3(\n anchors=self.__anchors,\n num_classes=len(self.__classes),\n device=self.__device\n )\n elif self.__model_type == \"tiny-yolov3\":\n self.__model = YoloV3Tiny(\n anchors=self.__anchors,\n num_classes=len(self.__classes),\n device=self.__device\n )\n else:\n raise ValueError(f\"Invalid model type. Call setModelTypeAsYOLOv3() or setModelTypeAsTinyYOLOv3() to set a model type before loading the model\")\n \n self.__model.to(self.__device)\n\n state_dict = torch.load(self.__model_path, map_location=self.__device)\n try:\n self.__model.load_state_dict(state_dict)\n self.__model_loaded = True\n self.__model.to(self.__device).eval()\n except Exception as e:\n raise RuntimeError(f\"Invalid weights!!! {e}\")\n\n\n def detectObjectsFromImage(self,\n input_image: Union[str, np.ndarray, Image.Image],\n output_image_path: str=None,\n output_type: str =\"file\",\n extract_detected_objects: bool=False, minimum_percentage_probability: int=40,\n display_percentage_probability: bool=True, display_object_name: bool=True,\n display_box: bool=True,\n custom_objects: List=None,\n nms_treshold: float= 0.4,\n objectness_treshold: float= 0.4,\n ) -> Union[List[List[Tuple[str, float, Dict[str, int]]]], np.ndarray, List[np.ndarray], List[str]]:\n \"\"\"\n Detects objects in an image using the unique classes provided\n by COCO.\n\n :param input_image: path to an image file, cv2 image or PIL image\n :param output_image_path: path to save input image with predictions rendered\n :param output_type: type of output for rendered image. Acceptable values are 'file' and 'array` ( a cv2 image )\n :param extract_detected_objects: extract each object based on the output type\n :param minimum_percentage_probability: the minimum confidence a detected object must have\n :param display_percentage_probability: to diplay/not display the confidence on rendered image \n :param display_object_name: to diplay/not display the object name on rendered image \n :param display_box: to diplay/not display the object bounding box on rendered image \n :param custom_objects: a dictionary of detectable objects set to boolean values\n \n :returns: A list of tuples containing the label of detected object and the\n confidence.\n \"\"\"\n \n self.__nms_score = nms_treshold\n self.__objectness_score = objectness_treshold\n \n self.__model.eval()\n if not self.__model_loaded:\n if self.__model_path:\n warnings.warn(\n \"Model path has changed but pretrained weights in the\"\n \" new path is yet to be loaded.\",\n ResourceWarning\n )\n else:\n raise RuntimeError(\n \"Model path isn't set, pretrained weights aren't used.\"\n )\n \n predictions = defaultdict(lambda : [])\n \n\n if self.__model_type == \"yolov3\" or self.__model_type == \"tiny-yolov3\":\n fnames, original_imgs, input_dims, imgs = self.__load_image_yolo(input_image)\n \n with torch.no_grad():\n output = self.__model(imgs)\n \n output = get_predictions(\n pred=output.to(self.__device), num_classes=len(self.__classes),\n nms_confidence_level=self.__nms_score, objectness_confidence= self.__objectness_score,\n device=self.__device\n )\n \n if output is None:\n if output_type == \"array\":\n if extract_detected_objects:\n return original_imgs[0], [], []\n else:\n return original_imgs[0], []\n else:\n if extract_detected_objects:\n return original_imgs[0], []\n else:\n return []\n \n # scale the output to match the dimension of the original image\n input_dims = torch.index_select(input_dims, 0, output[:, 0].long())\n scaling_factor = torch.min(416 / input_dims, 1)[0].view(-1, 1)\n output[:, [1,3]] -= (416 - (scaling_factor * input_dims[:, 0].view(-1,1))) / 2\n output[:, [2,4]] -= (416 - (scaling_factor * input_dims[:, 1].view(-1,1))) / 2\n output[:, 1:5] /= scaling_factor\n\n #clip bounding box for those that extended outside the detected image.\n for idx in range(output.shape[0]):\n output[idx, [1,3]] = torch.clamp(output[idx, [1,3]], 0.0, input_dims[idx, 0])\n output[idx, [2,4]] = torch.clamp(output[idx, [2,4]], 0.0, input_dims[idx, 1])\n\n for pred in output:\n pred_label = self.__classes[int(pred[-1])]\n if custom_objects:\n if pred_label.replace(\" \", \"_\") in custom_objects.keys():\n if not custom_objects[pred_label.replace(\" \", \"_\")]:\n continue\n else:\n continue\n predictions[int(pred[0])].append((\n pred_label,\n float(pred[-2]),\n {k:v for k,v in zip([\"x1\", \"y1\", \"x2\", \"y2\"], map(int, pred[1:5]))},\n ))\n \n # Render detection on copy of input image\n original_input_image = None\n output_image_array = None\n extracted_objects = []\n\n if self.__model_type == \"yolov3\" or self.__model_type == \"tiny-yolov3\":\n original_input_image = cv2.cvtColor(original_imgs[0], cv2.COLOR_RGB2BGR)\n if isinstance(output, torch.Tensor):\n for pred in output:\n percentage_conf = round(float(pred[-2]) * 100, 2)\n if percentage_conf < minimum_percentage_probability:\n continue\n\n displayed_label = \"\"\n if display_object_name:\n displayed_label = f\"{self.__classes[int(pred[-1].item())]} : \"\n if display_percentage_probability:\n displayed_label += f\" {percentage_conf}%\"\n\n\n original_imgs[int(pred[0].item())] = draw_bbox_and_label(pred[1:5].int() if display_box else None,\n displayed_label,\n original_imgs[int(pred[0].item())]\n )\n output_image_array = cv2.cvtColor(original_imgs[0], cv2.COLOR_RGB2BGR)\n\n # Format predictions for function reponse\n predictions_batch = list(predictions.values())\n predictions_list = predictions_batch[0] if len(predictions_batch) > 0 else []\n min_probability = minimum_percentage_probability / 100\n\n\n if output_type == \"file\":\n if output_image_path:\n cv2.imwrite(output_image_path, output_image_array)\n\n if extract_detected_objects:\n extraction_dir = \".\".join(output_image_path.split(\".\")[:-1]) + \"-extracted\"\n os.mkdir(extraction_dir)\n count = 0\n for obj_prediction in predictions_list: \n if obj_prediction[1] >= min_probability:\n count += 1\n extracted_path = os.path.join(\n extraction_dir, \n \".\".join(os.path.basename(output_image_path).split(\".\")[:-1]) + f\"-{count}.jpg\"\n )\n obj_bbox = obj_prediction[2]\n cv2.imwrite(extracted_path, original_input_image[obj_bbox[\"y1\"] : obj_bbox[\"y2\"], obj_bbox[\"x1\"] : obj_bbox[\"x2\"]])\n\n extracted_objects.append(extracted_path)\n\n elif output_type == \"array\":\n if extract_detected_objects:\n for obj_prediction in predictions_list: \n if obj_prediction[1] >= min_probability:\n obj_bbox = obj_prediction[2]\n\n extracted_objects.append(original_input_image[obj_bbox[\"y1\"] : obj_bbox[\"y2\"], obj_bbox[\"x1\"] : obj_bbox[\"x2\"]])\n else:\n raise ValueError(f\"Invalid output_type '{output_type}'. Supported values are 'file' and 'array' \")\n\n \n predictions_list = [\n {\n \"name\": prediction[0], \"percentage_probability\": round(prediction[1] * 100, 2),\n \"box_points\": [prediction[2][\"x1\"], prediction[2][\"y1\"], prediction[2][\"x2\"], prediction[2][\"y2\"]]\n } for prediction in predictions_list if prediction[1] >= min_probability\n ]\n\n\n if output_type == \"array\":\n if extract_detected_objects:\n return output_image_array, predictions_list, extracted_objects\n else:\n return output_image_array, predictions_list\n else:\n if extract_detected_objects:\n return predictions_list, extracted_objects\n else:\n return predictions_list\n\n\nclass CustomVideoObjectDetection:\n \"\"\"\n This is the custom objects detection class for videos and camera live stream inputs in the ImageAI library. It provides support for YOLOv3 and TinyYOLOv3 object detection networks. After instantiating this class, you can set it's properties and\n make object detections using it's pre-defined functions.\n The following functions are required to be called before object detection can be made\n * setModelPath()\n * At least of of the following and it must correspond to the model set in the setModelPath()\n [setModelTypeAsRetinaNet(), setModelTypeAsYOLOv3(), setModelTinyYOLOv3()]\n * loadModel() [This must be called once only before performing object detection]\n Once the above functions have been called, you can call the detectObjectsFromVideo() function\n or the detectCustomObjectsFromVideo() of the object detection instance object at anytime to\n obtain observable objects in any video or camera live stream.\n \"\"\"\n\n def __init__(self):\n self.__detector = CustomObjectDetection()\n\n def setModelTypeAsYOLOv3(self):\n self.__detector.setModelTypeAsYOLOv3()\n \n def setModelTypeAsTinyYOLOv3(self):\n self.__detector.setModelTypeAsTinyYOLOv3()\n\n def setModelPath(self, model_path: str):\n extension_check(model_path)\n self.__detector.setModelPath(model_path)\n \n def setJsonPath(self, configuration_json: str):\n self.__detector.setJsonPath(configuration_json)\n\n def loadModel(self):\n self.__detector.loadModel()\n \n def useCPU(self):\n self.__detector.useCPU()\n\n def detectObjectsFromVideo(self, input_file_path=\"\", camera_input=None, output_file_path=\"\", frames_per_second=20,\n frame_detection_interval=1, minimum_percentage_probability=40, log_progress=False,\n display_percentage_probability=True, display_object_name=True, display_box=True, save_detected_video=True,\n per_frame_function=None, per_second_function=None, per_minute_function=None,\n video_complete_function=None, return_detected_frame=False, detection_timeout = None):\n\n \"\"\"\n 'detectObjectsFromVideo()' function is used to detect objects observable in the given video path or a camera input:\n * input_file_path , which is the file path to the input video. It is required only if 'camera_input' is not set\n * camera_input , allows you to parse in camera input for live video detections\n * output_file_path , which is the path to the output video. It is required only if 'save_detected_video' is not set to False\n * frames_per_second , which is the number of frames to be used in the output video\n * frame_detection_interval (optional, 1 by default) , which is the intervals of frames that will be detected.\n * minimum_percentage_probability (optional, 50 by default) , option to set the minimum percentage probability for nominating a detected object for output.\n * log_progress (optional) , which states if the progress of the frame processed is to be logged to console\n * display_percentage_probability (optional), can be used to hide or show probability scores on the detected video frames\n * display_object_name (optional), can be used to show or hide object names on the detected video frames\n * save_save_detected_video (optional, True by default), can be set to or not to save the detected video\n * per_frame_function (optional), this parameter allows you to parse in a function you will want to execute after each frame of the video is detected. If this parameter is set to a function, after every video frame is detected, the function will be executed with the following values parsed into it:\n -- position number of the frame\n -- an array of dictinaries, with each dictionary corresponding to each object detected. Each dictionary contains 'name', 'percentage_probability' and 'box_points'\n -- a dictionary with with keys being the name of each unique objects and value are the number of instances of the object present\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed as the fourth value into the function\n * per_second_function (optional), this parameter allows you to parse in a function you will want to execute after each second of the video is detected. If this parameter is set to a function, after every second of a video is detected, the function will be executed with the following values parsed into it:\n -- position number of the second\n -- an array of dictionaries whose keys are position number of each frame present in the last second , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n -- an array of dictionaries, with each dictionary corresponding to each frame in the past second, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n -- a dictionary with its keys being the name of each unique object detected throughout the past second, and the key values are the average number of instances of the object found in all the frames contained in the past second\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed\n as the fifth value into the function\n * per_minute_function (optional), this parameter allows you to parse in a function you will want to execute after each minute of the video is detected. If this parameter is set to a function, after every minute of a video is detected, the function will be executed with the following values parsed into it:\n -- position number of the minute\n -- an array of dictionaries whose keys are position number of each frame present in the last minute , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n -- an array of dictionaries, with each dictionary corresponding to each frame in the past minute, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n -- a dictionary with its keys being the name of each unique object detected throughout the past minute, and the key values are the average number of instances of the object found in all the frames contained in the past minute\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed as the fifth value into the function\n * video_complete_function (optional), this parameter allows you to parse in a function you will want to execute after all of the video frames have been detected. If this parameter is set to a function, after all of frames of a video is detected, the function will be executed with the following values parsed into it:\n -- an array of dictionaries whose keys are position number of each frame present in the entire video , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n -- an array of dictionaries, with each dictionary corresponding to each frame in the entire video, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n -- a dictionary with its keys being the name of each unique object detected throughout the entire video, and the key values are the average number of instances of the object found in all the frames contained in the entire video\n * return_detected_frame (optionally, False by default), option to obtain the return the last detected video frame into the per_per_frame_function, per_per_second_function or per_per_minute_function\n * detection_timeout (optionally, None by default), option to state the number of seconds of a video that should be detected after which the detection function stop processing the video\n * thread_safe (optional, False by default), enforce the loaded detection model works across all threads if set to true, made possible by forcing all Tensorflow inference to run on the default graph.\n :param input_file_path:\n :param camera_input\n :param output_file_path:\n :param save_detected_video:\n :param frames_per_second:\n :param frame_detection_interval:\n :param minimum_percentage_probability:\n :param log_progress:\n :param display_percentage_probability:\n :param display_object_name:\n :param per_frame_function:\n :param per_second_function:\n :param per_minute_function:\n :param video_complete_function:\n :param return_detected_frame:\n :param detection_timeout:\n :param thread_safe:\n :return output_video_filepath:\n :return counting:\n :return output_objects_array:\n :return output_objects_count:\n :return detected_copy:\n :return this_second_output_object_array:\n :return this_second_counting_array:\n :return this_second_counting:\n :return this_minute_output_object_array:\n :return this_minute_counting_array:\n :return this_minute_counting:\n :return this_video_output_object_array:\n :return this_video_counting_array:\n :return this_video_counting:\n \"\"\"\n\n if (input_file_path == \"\" and camera_input == None):\n raise ValueError(\n \"You must set 'input_file_path' to a valid video file, or set 'camera_input' to a valid camera\")\n elif (save_detected_video == True and output_file_path == \"\"):\n raise ValueError(\n \"You must set 'output_video_filepath' to a valid video file name, in which the detected video will be saved. If you don't intend to save the detected video, set 'save_detected_video=False'\")\n\n else:\n\n output_frames_dict = {}\n output_frames_count_dict = {}\n\n input_video = cv2.VideoCapture(input_file_path)\n if (camera_input != None):\n input_video = camera_input\n\n output_video_filepath = output_file_path + '.mp4'\n\n frame_width = int(input_video.get(3))\n frame_height = int(input_video.get(4))\n output_video = cv2.VideoWriter(output_video_filepath, cv2.VideoWriter_fourcc(*\"MP4V\"),\n frames_per_second,\n (frame_width, frame_height))\n\n counting = 0\n\n detection_timeout_count = 0\n video_frames_count = 0\n\n while (input_video.isOpened()):\n ret, frame = input_video.read()\n\n if (ret == True):\n\n video_frames_count += 1\n if (detection_timeout != None):\n if ((video_frames_count % frames_per_second) == 0):\n detection_timeout_count += 1\n\n if (detection_timeout_count >= detection_timeout):\n break\n\n output_objects_array = []\n\n counting += 1\n\n if (log_progress == True):\n print(\"Processing Frame : \", str(counting))\n\n detected_copy = frame.copy()\n\n check_frame_interval = counting % frame_detection_interval\n\n if (counting == 1 or check_frame_interval == 0):\n try:\n detected_copy, output_objects_array = self.__detector.detectObjectsFromImage(\n input_image=frame, output_type=\"array\",\n minimum_percentage_probability=minimum_percentage_probability,\n display_percentage_probability=display_percentage_probability,\n display_object_name=display_object_name,\n display_box=display_box)\n \n except Exception as e:\n warnings.warn()\n \n if (save_detected_video == True):\n output_video.write(detected_copy)\n\n if detected_copy is not None and output_objects_array is not None:\n\n output_frames_dict[counting] = output_objects_array\n\n output_objects_count = {}\n for eachItem in output_objects_array:\n eachItemName = eachItem[\"name\"]\n try:\n output_objects_count[eachItemName] = output_objects_count[eachItemName] + 1\n except:\n output_objects_count[eachItemName] = 1\n\n output_frames_count_dict[counting] = output_objects_count\n\n if (counting == 1 or check_frame_interval == 0):\n if (per_frame_function != None):\n if (return_detected_frame == True):\n per_frame_function(counting, output_objects_array, output_objects_count,\n detected_copy)\n elif (return_detected_frame == False):\n per_frame_function(counting, output_objects_array, output_objects_count)\n\n if (per_second_function != None):\n if (counting != 1 and (counting % frames_per_second) == 0):\n\n this_second_output_object_array = []\n this_second_counting_array = []\n this_second_counting = {}\n\n for aa in range(counting):\n if (aa >= (counting - frames_per_second)):\n this_second_output_object_array.append(output_frames_dict[aa + 1])\n this_second_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_second_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_second_counting[eachItem] = this_second_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_second_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_second_counting:\n this_second_counting[eachCountingItem] = int(this_second_counting[eachCountingItem] / frames_per_second)\n\n if (return_detected_frame == True):\n per_second_function(int(counting / frames_per_second),\n this_second_output_object_array, this_second_counting_array,\n this_second_counting, detected_copy)\n\n elif (return_detected_frame == False):\n per_second_function(int(counting / frames_per_second),\n this_second_output_object_array, this_second_counting_array,\n this_second_counting)\n\n if (per_minute_function != None):\n\n if (counting != 1 and (counting % (frames_per_second * 60)) == 0):\n\n this_minute_output_object_array = []\n this_minute_counting_array = []\n this_minute_counting = {}\n\n for aa in range(counting):\n if (aa >= (counting - (frames_per_second * 60))):\n this_minute_output_object_array.append(output_frames_dict[aa + 1])\n this_minute_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_minute_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_minute_counting[eachItem] = this_minute_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_minute_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_minute_counting:\n this_minute_counting[eachCountingItem] = int(this_minute_counting[eachCountingItem] / (frames_per_second * 60))\n\n if (return_detected_frame == True):\n per_minute_function(int(counting / (frames_per_second * 60)),\n this_minute_output_object_array, this_minute_counting_array,\n this_minute_counting, detected_copy)\n\n elif (return_detected_frame == False):\n per_minute_function(int(counting / (frames_per_second * 60)),\n this_minute_output_object_array, this_minute_counting_array,\n this_minute_counting)\n else:\n break\n\n if (video_complete_function != None):\n\n this_video_output_object_array = []\n this_video_counting_array = []\n this_video_counting = {}\n\n for aa in range(counting):\n this_video_output_object_array.append(output_frames_dict[aa + 1])\n this_video_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_video_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_video_counting[eachItem] = this_video_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_video_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_video_counting:\n this_video_counting[eachCountingItem] = int(this_video_counting[eachCountingItem] / counting)\n\n video_complete_function(this_video_output_object_array, this_video_counting_array,\n this_video_counting)\n\n input_video.release()\n output_video.release()\n\n if (save_detected_video == True):\n return output_video_filepath\n\n " }, { "path": "imageai/Detection/Custom/yolo/__init__.py", "content": "" }, { "path": "imageai/Detection/Custom/yolo/compute_loss.py", "content": "import math\n\nimport torch\nimport torch.nn as nn\n\n# This new loss function is based on https://github.com/ultralytics/yolov3/blob/master/utils/loss.py\n\ndef bbox_iou(box1, box2, x1y1x2y2=True, GIoU=False, DIoU=False, CIoU=False, eps=1e-9):\n # Returns the IoU of box1 to box2. box1 is 4, box2 is nx4\n box2 = box2.T\n\n # Get the coordinates of bounding boxes\n if x1y1x2y2: # x1, y1, x2, y2 = box1\n b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]\n b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]\n else: # transform from xywh to xyxy\n b1_x1, b1_x2 = box1[0] - box1[2] / 2, box1[0] + box1[2] / 2\n b1_y1, b1_y2 = box1[1] - box1[3] / 2, box1[1] + box1[3] / 2\n b2_x1, b2_x2 = box2[0] - box2[2] / 2, box2[0] + box2[2] / 2\n b2_y1, b2_y2 = box2[1] - box2[3] / 2, box2[1] + box2[3] / 2\n\n # Intersection area\n inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \\\n (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0)\n\n # Union Area\n w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps\n w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps\n union = w1 * h1 + w2 * h2 - inter + eps\n\n iou = inter / union\n if GIoU or DIoU or CIoU:\n # convex (smallest enclosing box) width\n cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1)\n ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height\n if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1\n c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared\n rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 +\n (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center distance squared\n if DIoU:\n return iou - rho2 / c2 # DIoU\n elif CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47\n v = (4 / math.pi ** 2) * \\\n torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2)\n with torch.no_grad():\n alpha = v / ((1 + eps) - iou + v)\n return iou - (rho2 / c2 + v * alpha) # CIoU\n else: # GIoU https://arxiv.org/pdf/1902.09630.pdf\n c_area = cw * ch + eps # convex area\n return iou - (c_area - union) / c_area # GIoU\n else:\n return iou # IoU\n\n\ndef compute_loss(loss_layers, targets, device=\"cpu\"):\n nc = loss_layers[0].num_classes\n nl = len(loss_layers)\n # output at each layer\n predictions = [layer.pred for layer in loss_layers]\n\n # placeholders for the losses.\n lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)\n\n # Build yolo targets\n tcls, tbox, indices, anchors = build_targets(predictions, targets, loss_layers, device) # targets\n\n BCEcls = nn.BCEWithLogitsLoss(\n pos_weight=torch.tensor([1.0], device=device))\n BCEobj = nn.BCEWithLogitsLoss(\n pos_weight=torch.tensor([1.0], device=device))\n\n balance = [4.0, 1.0, 0.4]\n\n # Calculate losses for each yolo layer\n for layer_index, layer_predictions in enumerate(predictions):\n # Get image ids, anchors, grid index i and j for each target in the current yolo layer\n b, anchor, grid_j, grid_i = indices[layer_index]\n # Build empty object target tensor with the same shape as the object prediction\n tobj = torch.zeros_like(layer_predictions[..., 0], device=device) # target obj\n # Get the number of targets for this layer.\n # Each target is a label box with some scaling and the association of an anchor box.\n # Label boxes may be associated to 0 or multiple anchors. So they are multiple times or not at all in the targets.\n num_targets = b.shape[0]\n # Check if there are targets for this batch\n if num_targets:\n # Load the corresponding values from the predictions for each of the targets\n ps = layer_predictions[b, anchor, grid_j, grid_i]\n\n # Regression of the box\n # Apply sigmoid to xy offset predictions in each cell that has a target\n pxy = ps[:, :2].sigmoid() * 2 - 0.5\n # Apply exponent to wh predictions and multiply with the anchor box that matched best with the label for each cell that has a target\n pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[layer_index]\n # Build box out of xy and wh\n pbox = torch.cat((pxy, pwh), 1)\n # Calculate CIoU or GIoU for each target with the predicted box for its cell + anchor\n iou = bbox_iou(pbox.T, tbox[layer_index], x1y1x2y2=False, CIoU=True)\n # We want to minimize our loss so we and the best possible IoU is 1 so we take 1 - IoU and reduce it with a mean\n lbox += (1.0 - iou).mean() # iou loss\n\n # Classification of the objectness\n # Fill our empty object target tensor with the IoU we just calculated for each target at the targets position\n tobj[b, anchor, grid_j, grid_i] = iou.detach().clamp(0).type(tobj.dtype) # Use cells with iou > 0 as object targets\n\n # Classification of the class\n # Check if we need to do a classification (number of classes > 1)\n if nc > 1:\n # Hot one class encoding\n t = torch.full_like(ps[:, 5:], 0.0, device=device) # targets\n t[range(num_targets), tcls[layer_index]] = 1\n # Use the tensor to calculate the BCE loss\n lcls += BCEcls(ps[:, 5:], t) # BCE\n\n # Classification of the objectness the sequel\n # Calculate the BCE loss between the on the fly generated target and the network prediction\n obji = BCEobj(layer_predictions[..., 4], tobj) # obj loss\n lobj += obji * balance[layer_index]\n\n lbox *= 0.05\n lobj *= (1.0 * ((416 / 640) ** 2)) # scale to image size\n lcls *= (0.5 * (nc / 80)) # scale to classes\n\n # Merge losses\n loss = (lbox + lobj + lcls) * tobj.shape[0]\n\n return loss, (torch.cat((lbox, lobj, lcls))).detach()\n\n\ndef build_targets(p, targets, loss_layers, device=\"cpu\"):\n # Build targets for compute_loss(), input targets(image,class,x,y,w,h)\n na, nt = len(loss_layers[0].anchors), targets.shape[0] # number of anchors, targets\n tcls, tbox, indices, anch = [], [], [], []\n gain = torch.ones(7, device=device) # normalized to gridspace gain\n # Make a tensor that iterates 0-2 for 3 anchors and repeat that as many times as we have target boxes\n ai = torch.arange(na, device=device).float().view(na, 1).repeat(1, nt)\n # Copy target boxes anchor size times and append an anchor index to each copy the anchor index is also expressed by the new first dimension\n targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2)\n\n g = 0.5\n off = torch.tensor([\n [0, 0], [1, 0], [0, 1],\n [-1, 0], [0, -1]\n ], device=device).float() * g #offsets\n\n for i, yolo_layer in enumerate(loss_layers):\n # Scale anchors by the yolo grid cell size so that an anchor with the size of the cell would result in 1\n anchors = yolo_layer.anchors / yolo_layer.stride\n # Add the number of yolo cells in this layer the gain tensor\n # The gain tensor matches the collums of our targets (img id, class, x, y, w, h, anchor id)\n gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain\n # Scale targets by the number of yolo layer cells, they are now in the yolo cell coordinate system\n t = targets * gain\n # Check if we have targets\n if nt:\n # Calculate ration between anchor and target box for both width and height\n r = t[:, :, 4:6] / anchors[:, None]\n # Select the ratios that have the highest divergence in any axis and check if the ratio is less than 4\n j = torch.max(r, 1.0 / r).max(2)[0] < 4.0 # compare\n # Only use targets that have the correct ratios for their anchors\n # That means we only keep ones that have a matching anchor and we loose the anchor dimension\n # The anchor id is still saved in the 7th value of each target\n t = t[j]\n\n #offsets\n gxy = t[:, 2:4] #grid xy\n gxi = gain[[2,3]] - gxy\n j, k = ((gxy % 1 < g) & (gxy > 1)).T\n l, m = ((gxi % 1 < g) & (gxi > 1)).T\n j = torch.stack((torch.ones_like(j), j, k, l, m))\n t = t.repeat((5, 1, 1))[j]\n offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j]\n else:\n t = targets[0]\n offsets = 0\n\n # Extract image id in batch and class id\n b, c = t[:, :2].long().T\n # We isolate the target cell associations.\n # x, y, w, h are allready in the cell coordinate system meaning an x = 1.2 would be 1.2 times cellwidth\n gxy = t[:, 2:4] #grid xy\n gwh = t[:, 4:6] # grid wh\n # Cast to int to get an cell index e.g. 1.2 gets associated to cell 1\n gij = (gxy - offsets).long()\n # Isolate x and y index dimensions\n gi, gj = gij.T # grid xy indices\n\n # Convert anchor indexes to int\n a = t[:, 6].long()\n # Add target tensors for this yolo layer to the output lists\n # Add to index list and limit index range to prevent out of bounds\n indices.append((b, a, gj.clamp_(0, int(gain[3] - 1)), gi.clamp_(0, int(gain[2] - 1))))\n # Add to target box list and convert box coordinates from global grid coordinates to local offsets in the grid cell\n tbox.append(torch.cat((gxy - gij, gwh), 1)) # box\n # Add correct anchor for each target to the list\n anch.append(anchors[a])\n # Add class for each target to the list\n tcls.append(c)\n\n return tcls, tbox, indices, anch\n" }, { "path": "imageai/Detection/Custom/yolo/custom_anchors.py", "content": "import random\n\nimport torch\nimport numpy as np\nfrom scipy.cluster.vq import kmeans\n\n# This new anchor generator function is based on https://github.com/ultralytics/yolov3/blob/master/utils/autoanchor.py\n\ndef generate_anchors(dataset, n=9, img_size=416, thr=4.0, gen=1000, verbose=True):\n \"\"\" Creates kmeans-evolved anchors from training dataset\n\n Arguments:\n dataset: a loaded dataset i.e. subclass of torch.utils.data.Dataset\n n: number of anchors\n img_size: image size used for training\n thr: anchor-label wh ratio threshold used for training, default=4.0\n gen: generations to evolve anchors using genetic algorithm\n verbose: print all results\n\n Return:\n k: kmeans evolved anchors\n \"\"\"\n thr = 1 / thr\n\n def metric(k, wh): # compute metrics\n r = wh[:, None] / k[None]\n x = torch.min(r, 1 / r).min(2)[0] # ratio metric\n return x, x.max(1)[0] # x, best_x\n\n def anchor_fitness(k): # mutation fitness\n _, best = metric(torch.tensor(k, dtype=torch.float32), wh)\n return (best * (best > thr).float()).mean() # fitness\n\n def print_results(k, verbose=True):\n k = k[np.argsort(k.prod(1))] # sort small to large\n if verbose:\n x, best = metric(k, wh0)\n bpr, aat = (best > thr).float().mean(), (x > thr).float().mean() * n # best possible recall, anch > thr\n s = f'thr={thr:.2f}: {bpr:.4f} best possible recall, {aat:.2f} anchors past thr\\n' \\\n f'n={n}, img_size={img_size}, metric_all={x.mean():.3f}/{best.mean():.3f}-mean/best, ' \\\n f'past_thr={x[x > thr].mean():.3f}-mean: '\n print(s)\n return k\n\n # Get label wh\n shapes = img_size * dataset.shapes / dataset.shapes.max(1, keepdims=True)\n wh0 = np.concatenate([l[:, 3:5] * s for s, l in zip(shapes, dataset.labels)]) # wh\n\n # Filter\n i = (wh0 < 3.0).any(1).sum()\n if i and verbose:\n print(f'WARNING: Extremely small objects found. {i} of {len(wh0)} labels are < 3 pixels in size.')\n wh = wh0[(wh0 >= 2.0).any(1)] # filter > 2 pixels\n # wh = wh * (np.random.rand(wh.shape[0], 1) * 0.9 + 0.1) # multiply by random scale 0-1\n\n # Kmeans calculation\n s = wh.std(0) # sigmas for whitening\n k, dist = kmeans(wh / s, n, iter=30) # points, mean distance\n assert len(k) == n, f'ERROR: scipy.cluster.vq.kmeans requested {n} points but returned only {len(k)}'\n k *= s\n wh = torch.tensor(wh, dtype=torch.float32) # filtered\n wh0 = torch.tensor(wh0, dtype=torch.float32) # unfiltered\n k = print_results(k, verbose=False)\n\n # Evolve\n npr = np.random\n f, sh, mp, s = anchor_fitness(k), k.shape, 0.9, 0.1 # fitness, generations, mutation prob, sigma\n if verbose:\n print(\"Generating anchor boxes for training images...\")\n for _ in range(gen):\n v = np.ones(sh)\n while (v == 1).all(): # mutate until a change occurs (prevent duplicates)\n v = ((npr.random(sh) < mp) * random.random() * npr.randn(*sh) * s + 1).clip(0.3, 3.0)\n kg = (k.copy() * v).clip(min=2.0)\n fg = anchor_fitness(kg)\n if fg > f:\n f, k = fg, kg.copy()\n\n return print_results(k)\n" }, { "path": "imageai/Detection/Custom/yolo/dataset.py", "content": "import os\nimport warnings\nfrom typing import Tuple, List\n\nimport cv2 as cv\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\n\nfrom ....yolov3.utils import prepare_image\n\nclass LoadImagesAndLabels(Dataset):\n\n def __init__(self, path : str, net_dim=(416, 416), train=True):\n if not os.path.isdir(path):\n raise NotADirectoryError(\"path is not a valid directory!!!\")\n\n super().__init__()\n\n if train:\n path = os.path.join(path, \"train\")\n else:\n path = os.path.join(path, \"validation\")\n\n self.__net_width, self.__net_height = net_dim\n self.__images_paths = []\n self.shapes = []\n self.labels = []\n for img in os.listdir(os.path.join(path, \"images\")):\n p = os.path.join(path, \"images\", img)\n image = cv.imread(p)\n if isinstance(image, np.ndarray):\n l_p = self.__img_path2label_path(p)\n self.__images_paths.append(p)\n self.shapes.append((image.shape[1], image.shape[0]))\n self.labels.append(self.__load_raw_label(l_p))\n\n self.__nsamples = len(self.__images_paths)\n self.shapes = np.array(self.shapes)\n\n def __len__(self) -> int:\n return self.__nsamples\n\n def __img_path2label_path(self, path : str) -> str:\n im, lb = os.sep+\"images\"+os.sep, os.sep+\"annotations\"+os.sep\n return lb.join(path.rsplit(im, 1)).rsplit(\".\", 1)[0] + \".txt\"\n\n def __getitem__(self, idx) -> Tuple[torch.Tensor, torch.Tensor]:\n if idx >= self.__nsamples:\n raise IndexError(\"Index out of range.\")\n image_path = self.__images_paths[idx]\n label = self.labels[idx].copy()\n image, label = self.__load_data(image_path, label)\n return image, label\n\n def __xywhn2xyxy(self, nlabel : torch.Tensor, width : int, height : int) -> torch.Tensor:\n \"\"\"\n Transformed label from normalized center_x, center_y, width, height to\n x_1, y_1, x_2, y_2\n \"\"\"\n label = nlabel.clone()\n label[:, 1] = (nlabel[:, 1] - (nlabel[:, 3] / 2)) * width\n label[:, 2] = (nlabel[:, 2] - (nlabel[:, 4] / 2)) * height\n label[:, 3] = (nlabel[:, 1] + (nlabel[:, 3] / 2)) * width\n label[:, 4] = (nlabel[:, 2] + (nlabel[:, 4] / 2)) * height\n\n return label\n\n def __load_data(self, img_path : str, label : np.ndarray) -> Tuple[torch.Tensor, torch.Tensor]:\n img = cv.imread(img_path)\n img_h, img_w = img.shape[:2]\n img = prepare_image(img[:, :, :3], [self.__net_width, self.__net_height])\n lab = self.__process_label(label, img_w, img_h)\n return img.squeeze(), lab\n\n def __load_raw_label(self, label_path : str):\n if os.path.isfile(label_path):\n with warnings.catch_warnings():\n l = np.loadtxt(label_path).reshape(-1,5)\n assert (l >= 0).all(), \"bounding box values should be positive and in range 0 - 1\"\n assert (l[:, 1:] <= 1).all(), \"bounding box values should be in the range 0 - 1\"\n else:\n l = np.zeros((0,5), dtype=np.float32)\n return l\n\n def __process_label(self, label : np.ndarray, image_width : int, image_height : int) -> torch.Tensor:\n \"\"\"\n Process corresponding label and resize the ground truth bounding boxes\n to match the dimension of the resizes image.\n \"\"\"\n #max_box = 50\n scaling_factor = min(\n self.__net_width/image_width,\n self.__net_width/image_height\n )\n #bs = torch.zeros((max_box, 6))\n bs = torch.zeros((len(label), 6))\n if label.size > 0:\n nlabels = torch.from_numpy(label)\n labels = self.__xywhn2xyxy(nlabels, image_width, image_height)\n # scale bounding box to match new image size\n labels[:, [1,3]] = ((labels[:, [1,3]] * scaling_factor) +\\\n (self.__net_width - (image_width * scaling_factor))/2)\n labels[:, [2,4]] = ((labels[:, [2,4]] * scaling_factor) +\\\n (self.__net_width - (image_height * scaling_factor))/2)\n \n # convert x1, y1, x2, y2 to center_x, center_y, width, height\n label_copy = labels.clone()\n labels[:, 1] = (label_copy[:, 3] + label_copy[:, 1])/2\n labels[:, 2] = (label_copy[:, 4] + label_copy[:, 2])/2\n labels[:, 3] = (label_copy[:, 3] - label_copy[:, 1])\n labels[:, 4] = (label_copy[:, 4] - label_copy[:, 2])\n\n\n # scale labels by new image dimension\n labels[:, 1:5] /= self.__net_width\n bs[:, 1:] = labels[:, :]\n return bs\n\n def collate_fn(self, batch) -> Tuple[torch.Tensor, torch.Tensor]:\n batch = [data for data in batch if data is not None]\n imgs, bboxes = list(zip(*batch))\n\n imgs = torch.stack(imgs)\n\n for i, boxes in enumerate(bboxes):\n boxes[:, 0] = i\n bboxes = torch.cat(bboxes, 0)\n\n return imgs, bboxes\n\n" }, { "path": "imageai/Detection/Custom/yolo/metric.py", "content": "import math\nimport warnings\n\nimport numpy as np\nimport torch\n\n# This new metric functions is based on https://github.com/ultralytics/yolov3/blob/master/utils/metric.py\n\ndef ap_per_class(tp, conf, pred_cls, target_cls):\n \"\"\" Compute the average precision, given the recall and precision curves.\n Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.\n # Arguments\n tp: True positives (nparray, nx1 or nx10).\n conf: Objectness value from 0-1 (nparray).\n pred_cls: Predicted object classes (nparray).\n target_cls: True object classes (nparray).\n # Returns\n The average precision as computed in py-faster-rcnn.\n \"\"\"\n\n # Sort by objectness\n i = np.argsort(-conf)\n tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]\n\n # Find unique classes\n unique_classes = np.unique(target_cls)\n nc = unique_classes.shape[0] # number of classes, number of detections\n\n # Create Precision-Recall curve and compute AP for each class\n px = np.linspace(0, 1, 1000)\n ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000))\n for ci, c in enumerate(unique_classes):\n i = pred_cls == c\n n_l = (target_cls == c).sum() # number of labels\n n_p = i.sum() # number of predictions\n\n if n_p == 0 or n_l == 0:\n continue\n else:\n # Accumulate FPs and TPs\n fpc = (1 - tp[i]).cumsum(0)\n tpc = tp[i].cumsum(0)\n\n # Recall\n recall = tpc / (n_l + 1e-16) # recall curve\n r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases\n\n # Precision\n precision = tpc / (tpc + fpc) # precision curve\n p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score\n\n # AP from recall-precision curve\n for j in range(tp.shape[1]):\n ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j])\n\n # Compute F1 (harmonic mean of precision and recall)\n f1 = 2 * p * r / (p + r + 1e-16)\n i = f1.mean(0).argmax() # max F1 index\n\n return p[:, i], r[:, i], ap, f1[:, i], unique_classes.astype('int32')\n\n\ndef compute_ap(recall, precision):\n \"\"\" Compute the average precision, given the recall and precision curves\n # Arguments\n recall: The recall curve (list)\n precision: The precision curve (list)\n # Returns\n Average precision, precision curve, recall curve\n \"\"\"\n\n # Append sentinel values to beginning and end\n mrec = np.concatenate(([0.0], recall, [1.0]))\n mpre = np.concatenate(([1.0], precision, [0.0]))\n\n # Compute the precision envelope\n mpre = np.flip(np.maximum.accumulate(np.flip(mpre)))\n\n # Integrate area under curve\n x = np.linspace(0, 1, 101) # 101-point interp (COCO)\n ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate\n\n return ap, mpre, mrec\n\n" }, { "path": "imageai/Detection/Custom/yolo/validate.py", "content": "import os\n\nimport numpy as np\nimport torch\nfrom torchvision.ops import box_iou\n\nfrom ....yolov3.utils import get_predictions\nfrom .metric import ap_per_class\nfrom tqdm import tqdm\n\n# This new validation function is based on https://github.com/ultralytics/yolov3/blob/master/val.py\n\n\ndef xywh2xyxy(box_coord : torch.Tensor):\n \"\"\"\n Convert bounding box coordinates from center_x, center_y, width, height\n to x_1, y_1, x_2, x_3\n \"\"\"\n n = box_coord.clone()\n n[:, 0] = (box_coord[:, 0] - (box_coord[:, 2] / 2))\n n[:, 1] = (box_coord[:, 1] - (box_coord[:, 3] / 2))\n n[:, 2] = (box_coord[:, 0] + (box_coord[:, 2] / 2))\n n[:, 3] = (box_coord[:, 1] + (box_coord[:, 3] / 2))\n\n return n\n\ndef process_batch(detections, labels, iouv):\n \"\"\"\n Return correct predictions matrix. Both sets of boxes are in (x1, y1, x2, y2) format.\n Arguments:\n detections (Array[N, 6]), x1, y1, x2, y2, conf, class\n labels (Array[M, 5]), class, x1, y1, x2, y2\n Returns:\n correct (Array[N, 10]), for 10 IoU levels\n \"\"\"\n detections[:, [1,3]] = torch.clamp(detections[:, [1,3]], 0.0, 416)\n detections[:, [2,4]] = torch.clamp(detections[:, [2,4]], 0.0, 416)\n \n correct = torch.zeros(detections.shape[0], iouv.shape[0], dtype=torch.bool, device=iouv.device)\n iou = box_iou(labels[:, 1:], detections[:, 1:5])\n x = torch.where((iou >= iouv[0]) & (labels[:, 0:1] == detections[:, 7])) # IoU above threshold and classes match\n if x[0].shape[0]:\n matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() # [label, detection, iou]\n if x[0].shape[0] > 1:\n matches = matches[matches[:, 2].argsort()[::-1]]\n matches = matches[np.unique(matches[:, 1], return_index=True)[1]]\n matches = matches[np.unique(matches[:, 0], return_index=True)[1]]\n matches = torch.Tensor(matches).to(iouv.device)\n correct[matches[:, 1].long()] = matches[:, 2:3] >= iouv\n return correct\n\n@torch.no_grad()\ndef run(model, val_dataloader, num_class, net_dim=416, nms_thresh=0.6, objectness_thresh=0.001, device=\"cpu\"):\n model.eval()\n nc = int(num_class) # number of classes\n iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for mAP@0.5:0.95\n niou = iouv.numel()\n\n p, r, f1, mp, mr, map50, map = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0\n stats, ap, ap_class = [], [], []\n \n for batch_i, (im, targets) in tqdm(enumerate(val_dataloader)):\n im = im.to(device)\n targets = targets.to(device)\n nb = im.shape[0] # batch\n\n # Inference\n out = model(im) # inference\n\n # NMS\n targets[:, 2:] *= torch.Tensor([net_dim, net_dim, net_dim, net_dim]).to(device) # to pixels\n out = get_predictions(\n pred=out.to(device), num_classes=nc,\n objectness_confidence=objectness_thresh,\n nms_confidence_level=nms_thresh, device=device\n )\n\n # Metrics\n for si in range(nb):\n labels = targets[targets[:, 0] == si, 1:]\n pred = out[out[:, 0]==si, :] if isinstance(out, torch.Tensor) else torch.zeros((0,0), device=device)\n nl = len(labels)\n tcls = labels[:, 0].tolist() if nl else [] # target class\n\n if len(pred) == 0:\n if nl:\n stats.append((torch.zeros(0, niou, dtype=torch.bool, device=\"cpu\"), torch.Tensor(device=\"cpu\"), torch.Tensor(device=\"cpu\"), tcls))\n continue\n\n # Predictions\n if nc==1:\n pred[:, 7] = 0\n \n if pred.shape[0] > 300:\n pred = pred[:300, :] # sorted by confidence\n \n predn = pred.clone()\n\n # Evaluate\n if nl:\n tbox = xywh2xyxy(labels[:, 1:5]).to(device) # target boxes\n labelsn = torch.cat((labels[:, 0:1], tbox), 1).to(device) # native-space labels\n correct = process_batch(predn, labelsn, iouv)\n else:\n correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool)\n stats.append((correct.cpu(), pred[:, 5].cpu(), pred[:, 7].cpu(), tcls)) # (correct, conf, pcls, tcls)\n\n # Compute metrics\n stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy\n if len(stats) and stats[0].any():\n p, r, ap, f1, ap_class = ap_per_class(*stats)\n ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95\n mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()\n\n return mp, mr, map50, map\n\n" }, { "path": "imageai/Detection/README.md", "content": "# ImageAI : Object Detection\n\n## ---------------------------------------------------\n## Introducing Jarvis and TheiaEngine.\n\nWe the creators of ImageAI are glad to announce 2 new AI projects to provide state-of-the-art Generative AI, LLM and Image Understanding on your personal computer and servers. \n\n\n[![](../../jarvis.png)](https://jarvis.genxr.co)\n\nInstall Jarvis on PC/Mac to setup limitless access to LLM powered AI Chats for your every day work, research and generative AI needs with 100% privacy and full offline capability.\n\n\nVisit [https://jarvis.genxr.co](https://jarvis.genxr.co/) to get started.\n\n\n[![](../../theiaengine.png)](https://www.genxr.co/theia-engine)\n\n\n[TheiaEngine](https://www.genxr.co/theia-engine), the next-generation computer Vision AI API capable of all Generative and Understanding computer vision tasks in a single API call and available via REST API to all programming languages. Features include\n- **Detect 300+ objects** ( 220 more objects than ImageAI)\n- **Provide answers to any content or context questions** asked on an image\n - very useful to get information on any object, action or information without needing to train a new custom model for every tasks\n- **Generate scene description and summary**\n- **Convert 2D image to 3D pointcloud and triangular mesh**\n- **Semantic Scene mapping of objects, walls, floors, etc**\n- **Stateless Face recognition and emotion detection**\n- **Image generation and augmentation from prompt**\n- etc.\n\nVisit [https://www.genxr.co/theia-engine](https://www.genxr.co/theia-engine) to try the demo and join in the beta testing today.\n## ---------------------------------------------------\n\n### TABLE OF CONTENTS\n\n- :white_square_button: First Object Detection\n- :white_square_button: Object Detection, Extraction and Fine-tune\n- :white_square_button: Custom Object Detection\n- :white_square_button: Detection Speed\n- :white_square_button: Hiding/Showing Object Name and Probability\n- :white_square_button: Image Input & Output Types\n- :white_square_button: Documentation\n\n\nImageAI provides very convenient and powerful methods to perform object detection on images and extract each object from the image. The object detection class supports RetinaNet, YOLOv3 and TinyYOLOv3. To start performing object detection, you must download the RetinaNet, YOLOv3 or TinyYOLOv3 object detection model via the links below: \n* **[RetinaNet](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/retinanet_resnet50_fpn_coco-eeacb38b.pth)** _(Size = 130 mb, high performance and accuracy, with longer detection time)_\n* **[YOLOv3](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt)** _(Size = 237 mb, moderate performance and accuracy, with a moderate detection time)_\n* **[TinyYOLOv3](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/tiny-yolov3.pt)** _(Size = 34 mb, optimized for speed and moderate performance, with fast detection time)_\n\n\n Once you download the object detection model file, you should copy the model file to the your project folder where your .py files will be.\n Then create a python file and give it a name; an example is FirstObjectDetection.py. Then write the code below into the python file:\n\n### FirstObjectDetection.py\n
\n\n```python\nfrom imageai.Detection import ObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = ObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"yolov3.pt\"))\ndetector.loadModel()\ndetections = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"image2.jpg\"), output_image_path=os.path.join(execution_path , \"image2new.jpg\"), minimum_percentage_probability=30)\n\nfor eachObject in detections:\n print(eachObject[\"name\"] , \" : \", eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"--------------------------------\")\n```\n\nSample Result:\nInput Image\n![Input Image](../../data-images/image2.jpg)\nOutput Image\n![Output Image](../../data-images/yolo.jpg)\n\n```\nlaptop : 87.32235431671143 : (306, 238, 390, 284)\n--------------------------------\nlaptop : 96.86298966407776 : (121, 209, 258, 293)\n--------------------------------\nlaptop : 98.6301600933075 : (279, 321, 401, 425)\n--------------------------------\nlaptop : 99.78572130203247 : (451, 204, 579, 285)\n--------------------------------\nbed : 94.02391314506531 : (23, 205, 708, 553)\n--------------------------------\napple : 48.03136885166168 : (527, 343, 557, 364)\n--------------------------------\ncup : 34.09906327724457 : (462, 347, 496, 379)\n--------------------------------\ncup : 44.65090036392212 : (582, 342, 618, 386)\n--------------------------------\nperson : 57.70219564437866 : (27, 311, 341, 437)\n--------------------------------\nperson : 85.26121377944946 : (304, 173, 387, 253)\n--------------------------------\nperson : 96.33603692054749 : (415, 130, 538, 266)\n--------------------------------\nperson : 96.95255160331726 : (174, 108, 278, 269)\n--------------------------------\n```\n\nLet us make a breakdown of the object detection code that we used above.\n\n```python\nfrom imageai.Detection import ObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n```\n\n In the 3 lines above , we import the **ImageAI object detection** class in the first line, import the `os` in the second line and obtained the path to folder where our python file runs.\n \n```python\ndetector = ObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"yolov3.pt\"))\ndetector.loadModel()\n```\n\nIn the 4 lines above, we created a new instance of the `ObjectDetection` class in the first line, set the model type to YOLOv3 in the second line, set the model path to the YOLOv3 model file we downloaded and copied to the python file folder in the third line and load the model in the fourth line.\n\n```python\ndetections = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"image2.jpg\"), output_image_path=os.path.join(execution_path , \"image2new.jpg\"))\n\nfor eachObject in detections:\n print(eachObject[\"name\"] , \" : \", eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"--------------------------------\")\n```\n\nIn the 2 lines above, we ran the `detectObjectsFromImage()` function and parse in the path to our image, and the path to the new image which the function will save. Then the function returns an array of dictionaries with each dictionary corresponding to the number of objects detected in the image. Each dictionary has the properties `name` (name of the object), `percentage_probability` (percentage probability of the detection) and `box_points` (the x1,y1,x2 and y2 coordinates of the bounding box of the object).\n\nShould you want to use the RetinaNet which is appropriate for high-performance and high-accuracy demanding detection tasks, you will download the RetinaNet model file from the links above, copy it to your python file's folder, set the model type and model path in your python code as seen below:\n\n```python\ndetector = ObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath( os.path.join(execution_path , \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\ndetector.loadModel()\n```\n\nHowever, if you desire TinyYOLOv3 which is optimized for speed and embedded devices, you will download the TinyYOLOv3 model file from the links above, copy it to your python file's folder, set the model type and model path in your python code as seen below:\n\n```python\ndetector = ObjectDetection()\ndetector.setModelTypeAsTinyYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"tiny-yolov3.pt\"))\ndetector.loadModel()\n```\n\n## Object Detection, Extraction and Fine-tune\n
\n\nIn the examples we used above, we ran the object detection on an image and it returned the detected objects in an array as well as save a new image with rectangular markers drawn on each object. In our next examples, we will be able to extract each object from the input image\n and save it independently.\n\nIn the example code below which is very identical to the previous object detction code, we will save each object detected as a seperate image.\n\n```python\nfrom imageai.Detection import ObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = ObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"yolov3.pt\"))\ndetector.loadModel()\n\ndetections, objects_path = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"image3.jpg\"), output_image_path=os.path.join(execution_path , \"image3new.jpg\"), minimum_percentage_probability=30, extract_detected_objects=True)\n\nfor eachObject, eachObjectPath in zip(detections, objects_path):\n print(eachObject[\"name\"] , \" : \" , eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"Object's image saved in \" + eachObjectPath)\n print(\"--------------------------------\")\n```\n\n![Input Image](../../data-images/image3.jpg)\n![Output Images](../../data-images/image3new.jpg)\n\n![dog](../../data-images/image3new-objects/dog-1.jpg)\n![motorcycle](../../data-images/image3new-objects/motorcycle-3.jpg)\n![car](../../data-images/image3new-objects/car-4.jpg)\n![bicycle](../../data-images/image3new-objects/bicycle-5.jpg)\n![person](../../data-images/image3new-objects/person-6.jpg)\n![person](../../data-images/image3new-objects/person-7.jpg)\n![person](../../data-images/image3new-objects/person-8.jpg)\n![person](../../data-images/image3new-objects/person-9.jpg)\n![person](../../data-images/image3new-objects/person-10.jpg)\n\n\nLet us review the part of the code that perform the object detection and extract the images:\n\n```python\ndetections, objects_path = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"image3.jpg\"), output_image_path=os.path.join(execution_path , \"image3new.jpg\"), minimum_percentage_probability=30, extract_detected_objects=True)\n\nfor eachObject, eachObjectPath in zip(detections, objects_path):\n print(eachObject[\"name\"] , \" : \" , eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"Object's image saved in \" + eachObjectPath)\n print(\"--------------------------------\")\n```\n\nIn the above above lines, we called the `detectObjectsFromImage()` , parse in the input image path, output image path, and an extra parameter `extract_detected_objects=True`. This parameter states that the function should extract each object detected from the image and save it has a seperate image. The parameter is false by default. Once set to `true`, the function will create a directory which is the **output image path + \"-objects\"** . Then it saves all the extracted images into this new directory with each image's name being the **detected object name + \"-\" + a number** which corresponds to the order at which the objects were detected.\n\nThis new parameter we set to extract and save detected objects as an image will make the function to return 2 values. The first is the array of dictionaries with each dictionary corresponding to a detected object. The second is an array of the paths to the saved images of each object detected and extracted, and they are arranged in order at which the objects are in the first array.\n\n\n**And one important feature you need to know!** You will recall that the percentage probability\n for each detected object is sent back by the `detectObjectsFromImage()` function. The function has a parameter `minimum_percentage_probability`, whose default value is `50` (value ranges between 0 - 100) , but it set to 30 in this example. That means the function will only return a detected object if it's percentage probability is **30 or above**. The value was kept at this number to ensure the integrity of the detection results. You fine-tune the object detection by setting **minimum_percentage_probability** equal to a smaller value to detect more number of objects or higher value to detect less number of objects.\n\n\n## Custom Object Detection\n
\n\nThe object detection model (**RetinaNet**) supported by **ImageAI** can detect 80 different types of objects. They include:\n```\nperson, bicycle, car, motorcycle, airplane, bus, train, truck, boat, traffic light, fire hydrant, stop_sign,\nparking meter, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra,\ngiraffe, backpack, umbrella, handbag, tie, suitcase, frisbee, skis, snowboard,\nsports ball, kite, baseball bat, baseball glove, skateboard, surfboard, tennis racket,\nbottle, wine glass, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange,\nbroccoli, carrot, hot dog, pizza, donot, cake, chair, couch, potted plant, bed,\ndining table, toilet, tv, laptop, mouse, remote, keyboard, cell phone, microwave, oven,\ntoaster, sink, refrigerator, book, clock, vase, scissors, teddy bear, hair dryer, toothbrush.\n```\n\nInterestingly, **ImageAI** allow you to perform detection for one or more of the items above. That means you can\n customize the type of object(s) you want to be detected in the image. Let's take a look at the code below:\n\n```python\nfrom imageai.Detection import ObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = ObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"yolov3.pt\"))\ndetector.loadModel()\n\ncustom_objects = detector.CustomObjects(car=True, motorcycle=True)\ndetections = detector.detectCustomObjectsFromImage(custom_objects=custom_objects, input_image=os.path.join(execution_path , \"image3.jpg\"), output_image_path=os.path.join(execution_path , \"image3custom.jpg\"), minimum_percentage_probability=30)\n\nfor eachObject in detections:\n print(eachObject[\"name\"] , \" : \", eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"--------------------------------\")\n```\n\n![Result](../../data-images/image3custom.jpg)\n\n\nLet us take a look at the part of the code that made this possible.\n```python\ncustom_objects = detector.CustomObjects(car=True, motorcycle=True)\ndetections = detector.detectCustomObjectsFromImage(custom_objects=custom_objects, input_image=os.path.join(execution_path , \"image3.jpg\"), output_image_path=os.path.join(execution_path , \"image3custom.jpg\"), minimum_percentage_probability=30)\n```\n\nIn the above code, after loading the model (can be done before loading the model as well), we defined a new variable\n`custom_objects = detector.CustomObjects()`, in which we set its car and motorcycle properties equal to **True**.\nThis is to tell the model to detect only the object we set to True. Then we call the `detector.detectCustomObjectsFromImage()`\nwhich is the function that allows us to perform detection of custom objects. Then we will set the `custom_objects` value\n to the custom objects variable we defined.\n\n\n\n## Hiding/Showing Object Name and Probability\n
\n\n**ImageAI** provides options to hide the name of objects detected and/or the percentage probability from being shown on the saved/returned detected image. Using the `detectObjectsFromImage()` and `detectCustomObjectsFromImage()` functions, the parameters `display_object_name` and `display_percentage_probability` can be set to True of False individually. Take a look at the code below:\n\n```python\ndetections = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"image3.jpg\"), output_image_path=os.path.join(execution_path , \"image3new_nodetails.jpg\"), minimum_percentage_probability=30, display_percentage_probability=False, display_object_name=False)\n```\n\nIn the above code, we specified that both the object name and percentage probability should not be shown. As you can see in the result below, both the names of the objects and their individual percentage probability is not shown in the detected image.\n\n![Result](../../data-images/nodetails.jpg)\n\n\n## Image Input & Output Types\n
\n\n**ImageAI** supports 3 types of inputs which are **file path to image file**(default), **numpy array of image** and **image file stream**\nas well as 2 types of output which are image **file**(default) and numpy **array **.\nThis means you can now perform object detection in production applications such as on a web server and system\n that returns file in any of the above stated formats.\n\nTo perform object detection with numpy array or file stream input, you just need to state the input type\nin the `.detectObjectsFromImage()` function or the `.detectCustomObjectsFromImage()` function. See example below.\n\n```python\ndetections = detector.detectObjectsFromImage(input_type=\"array\", input_image=image_array , output_image_path=os.path.join(execution_path , \"image.jpg\")) # For numpy array input type\ndetections = detector.detectObjectsFromImage(input_type=\"stream\", input_image=image_stream , output_image_path=os.path.join(execution_path , \"test2new.jpg\")) # For file stream input type\n```\n\nTo perform object detection with numpy array output you just need to state the output type\nin the `.detectObjectsFromImage()` function or the `.detectCustomObjectsFromImage()` function. See example below.\n\n```python\ndetected_image_array, detections = detector.detectObjectsFromImage(output_type=\"array\", input_image=\"image.jpg\" ) # For numpy array output type\n```\n\n\n## Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions. Find links below:\n\n* Documentation - **English Version [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**" }, { "path": "imageai/Detection/VIDEO.md", "content": "# ImageAI : Video Object Detection, Tracking and Analysis\n\n## ---------------------------------------------------\n## Introducing Jarvis and TheiaEngine.\n\nWe the creators of ImageAI are glad to announce 2 new AI projects to provide state-of-the-art Generative AI, LLM and Image Understanding on your personal computer and servers. \n\n\n[![](../../jarvis.png)](https://jarvis.genxr.co)\n\nInstall Jarvis on PC/Mac to setup limitless access to LLM powered AI Chats for your every day work, research and generative AI needs with 100% privacy and full offline capability.\n\n\nVisit [https://jarvis.genxr.co](https://jarvis.genxr.co/) to get started.\n\n\n[![](../../theiaengine.png)](https://www.genxr.co/theia-engine)\n\n\n[TheiaEngine](https://www.genxr.co/theia-engine), the next-generation computer Vision AI API capable of all Generative and Understanding computer vision tasks in a single API call and available via REST API to all programming languages. Features include\n- **Detect 300+ objects** ( 220 more objects than ImageAI)\n- **Provide answers to any content or context questions** asked on an image\n - very useful to get information on any object, action or information without needing to train a new custom model for every tasks\n- **Generate scene description and summary**\n- **Convert 2D image to 3D pointcloud and triangular mesh**\n- **Semantic Scene mapping of objects, walls, floors, etc**\n- **Stateless Face recognition and emotion detection**\n- **Image generation and augmentation from prompt**\n- etc.\n\nVisit [https://www.genxr.co/theia-engine](https://www.genxr.co/theia-engine) to try the demo and join in the beta testing today.\n## ---------------------------------------------------\n\n## TABLE OF CONTENTS\n\n- :white_square_button: First Video Object Detection\n- :white_square_button: Custom Video Object Detection (Object Tracking)\n- :white_square_button: Camera / Live Stream Video Detection\n- :white_square_button: Video Analysis\n- :white_square_button: Detection Speed\n- :white_square_button: Hiding/Showing Object Name and Probability\n- :white_square_button: Frame Detection Intervals\n- :white_square_button: Video Detection Timeout (NEW)\n- :white_square_button: Documentation\n\nImageAI provides convenient, flexible and powerful methods to perform object detection on videos. The video object detection class provided only supports RetinaNet, YOLOv3 and TinyYOLOv3. This version of **ImageAI** provides commercial grade video objects detection features, which include but not limited to device/IP camera inputs, per frame, per second, per minute and entire video analysis for storing in databases and/or real-time visualizations and for future insights.\n\nTo start performing video object detection, you must download the RetinaNet, YOLOv3 or TinyYOLOv3 object detection model via the links below:\n\n* **[RetinaNet](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/retinanet_resnet50_fpn_coco-eeacb38b.pth)** _(Size = 130 mb, high performance and accuracy, with longer detection time)_\n* **[YOLOv3](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/yolov3.pt)** _(Size = 237 mb, moderate performance and accuracy, with a moderate detection time)_\n* **[TinyYOLOv3](https://github.com/OlafenwaMoses/ImageAI/releases/download/3.0.0-pretrained/tiny-yolov3.pt)** _(Size = 34 mb, optimized for speed and moderate performance, with fast detection time)_\n\nBecause video object detection is a compute intensive tasks, we advise you perform this experiment using a computer with a NVIDIA GPU and the GPU version of Tensorflow installed. Performing Video Object Detection CPU will be slower than using an NVIDIA GPU powered computer. You can use Google Colab for this experiment as it has an NVIDIA K80 GPU available for free.\n\n Once you download the object detection model file, you should copy the model file to the your project folder where your .py files will be.\n Then create a python file and give it a name; an example is `FirstVideoObjectDetection.py`. Then write the code below into the python file:\n\n\n### FirstVideoObjectDetection.py\n
\n\n```python\nfrom imageai.Detection import VideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath( os.path.join(execution_path , \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\ndetector.loadModel()\n\nvideo_path = detector.detectObjectsFromVideo(input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\")\n , frames_per_second=20, log_progress=True)\nprint(video_path)\n```\n\n\nInput Video (a 1min 24seconds video)\n\n[![](../../data-images/video--1.jpg)](https://github.com/OlafenwaMoses/ImageAI/blob/master/data-videos/traffic.mp4)\n\nOutput Video\n[![](../../data-images/video-2.jpg)](https://www.youtube.com/embed/qplVDqOmElI?rel=0)\n\nLet us make a breakdown of the object detection code that we used above.\n\n```python\nfrom imageai.Detection import VideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n```\n\n In the 3 lines above , we import the **ImageAI video object detection ** class in the first line, import the **os** in the second line and obtained\n the path to folder where our python file runs.\n\n```python\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath( os.path.join(execution_path , \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\ndetector.loadModel()\n```\n\nIn the 4 lines above, we created a new instance of the **VideoObjectDetection** class in the first line, set the model type to RetinaNet in the second line, set the model path to the RetinaNet model file we downloaded and copied to the python file folder in the third line and load the model in the fourth line.\n\n```python\nvideo_path = detector.detectObjectsFromVideo(input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\"),\n frames_per_second=20, log_progress=True)\nprint(video_path)\n```\n\nIn the 2 lines above, we ran the `detectObjectsFromVideo()` function and parse in the path to our video,the path to the new video (without the extension, it saves a .avi video by default) which the function will save, the number of frames per second (fps) that you we desire the output video to have and option to log the progress of the detection in the console. Then the function returns a the path to the saved video which contains boxes and percentage probabilities rendered on objects detected in the video.\n\n\n### Custom Video Object Detection\n
\n\nThe video object detection model (**RetinaNet**) supported by **ImageAI** can detect 80 different types of objects. They include: \n```\n person, bicycle, car, motorcycle, airplane, bus, train, truck, boat, traffic light, fire hydrant, stop_sign,\n parking meter, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra,\n giraffe, backpack, umbrella, handbag, tie, suitcase, frisbee, skis, snowboard,\n sports ball, kite, baseball bat, baseball glove, skateboard, surfboard, tennis racket,\n bottle, wine glass, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange,\n broccoli, carrot, hot dog, pizza, donot, cake, chair, couch, potted plant, bed,\n dining table, toilet, tv, laptop, mouse, remote, keyboard, cell phone, microwave,\n oven, toaster, sink, refrigerator, book, clock, vase, scissors, teddy bear, hair dryer,\n toothbrush.\n```\n\n\nInterestingly, **ImageAI** allow you to perform detection for one or more of the items above. That means you can customize the type of object(s) you want to be detected in the video. Let's take a look at the code below:\n\n```python\nfrom imageai.Detection import VideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath( os.path.join(execution_path , \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\ndetector.loadModel()\n\ncustom_objects = detector.CustomObjects(person=True, bicycle=True, motorcycle=True)\n\nvideo_path = detector.detectCustomObjectsFromVideo(\n custom_objects=custom_objects,\n input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_custom_detected\"),\n frames_per_second=20, log_progress=True)\nprint(video_path)\n```\n\nLet us take a look at the part of the code that made this possible.\n\n```python\ncustom_objects = detector.CustomObjects(person=True, bicycle=True, motorcycle=True)\n\nvideo_path = detector.detectCustomObjectsFromVideo(\n custom_objects=custom_objects, \n input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_custom_detected\"),\n frames_per_second=20, log_progress=True)\n```\n\nIn the above code, after loading the model (can be done before loading the model as well), we defined a new variable\n`custom_objects = detector.CustomObjects()`, in which we set its person, car and motorcycle properties equal to **True**.\nThis is to tell the model to detect only the object we set to True. Then we call the `detector.detectCustomObjectsFromVideo()`\nwhich is the function that allows us to perform detection of custom objects. Then we will set the `custom_objects` value\n to the custom objects variable we defined.\n\nOutput Video\n[![Output Video](../../data-images/video-3.jpg)](https://www.youtube.com/embed/YfAycAzkwPM?rel=0)\nC:\\Users\\User\\PycharmProjects\\ImageAITest\\traffic_custom_detected.avi\n\n\n### Camera / Live Stream Video Detection\n
\n\n**ImageAI** now allows live-video detection with support for camera inputs. Using **OpenCV**'s `VideoCapture()` function, you can load live-video streams from a device camera, cameras connected by cable or IP cameras, and parse it into **ImageAI**'s `detectObjectsFromVideo()` and `detectCustomObjectsFromVideo()` functions. All features that are supported for detecting objects in a video file is also available for detecting objects in a camera's live-video feed. Find below an example of detecting live-video feed from the device camera.\n\n```python\nfrom imageai.Detection import VideoObjectDetection\nimport os\nimport cv2\n\nexecution_path = os.getcwd()\n\n\ncamera = cv2.VideoCapture(0)\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath(os.path.join(execution_path , \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\ndetector.loadModel()\n\n\nvideo_path = detector.detectObjectsFromVideo(\n camera_input=camera,\n output_file_path=os.path.join(execution_path, \"camera_detected_video\"),\n frames_per_second=20, log_progress=True, minimum_percentage_probability=40)\n```\n\nThe difference in the code above and the code for the detection of a video file is that we defined an **OpenCV VideoCapture** instance and loaded the default device camera into it. Then we parsed the camera we defined into the parameter `camera_input` which replaces the `input_file_path` that is used for video file.\n\n### Video Analysis\n
\n\n**ImageAI** now provide commercial-grade video analysis in the Video Object Detection class, for both video file inputs and camera inputs. This feature allows developers to obtain deep insights into any video processed with **ImageAI**. This insights can be visualized in real-time, stored in a NoSQL database for future review or analysis.\n\nFor video analysis, the `detectObjectsFromVideo()` and `detectCustomObjectsFromVideo()` now allows you to state your own defined functions which will be executed for every frame, seconds and/or minute of the video detected as well as a state a function that will be executed at the end of a video detection. Once this functions are stated, they will receive raw but comprehensive analytical data on the index of the frame/second/minute, objects detected (name, percentage_probability and box_points), number of instances of each unique object detected and average number of occurrence of each unique object detected over a second/minute and entire video.\n\nTo obtain the video analysis, all you need to do is specify a function, state the corresponding parameters it will be receiving and parse the function name into the `per_frame_function`, `per_second_function`, `per_minute_function` and `video_complete_function` parameters in the detection function. Find below examples of video analysis functions.\n\n```python\ndef forFrame(frame_number, output_array, output_count):\n print(\"FOR FRAME \" , frame_number)\n print(\"Output for each object : \", output_array)\n print(\"Output count for unique objects : \", output_count)\n print(\"------------END OF A FRAME --------------\")\n\ndef forSeconds(second_number, output_arrays, count_arrays, average_output_count):\n print(\"SECOND : \", second_number)\n print(\"Array for the outputs of each frame \", output_arrays)\n print(\"Array for output count for unique objects in each frame : \", count_arrays)\n print(\"Output average count for unique objects in the last second: \", average_output_count)\n print(\"------------END OF A SECOND --------------\")\n\ndef forMinute(minute_number, output_arrays, count_arrays, average_output_count):\n print(\"MINUTE : \", minute_number)\n print(\"Array for the outputs of each frame \", output_arrays)\n print(\"Array for output count for unique objects in each frame : \", count_arrays)\n print(\"Output average count for unique objects in the last minute: \", average_output_count)\n print(\"------------END OF A MINUTE --------------\")\n\nvideo_detector = VideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(os.path.join(execution_path, \"yolov3.pt\"))\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(\n input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\"),\n frames_per_second=10,\n per_second_function=forSeconds,\n per_frame_function=forFrame,\n per_minute_function=forMinute,\n minimum_percentage_probability=30\n)\n```\n\nWhen the detection starts on a video feed, be it from a video file or camera input, the result will have the format as below:\n\n**Results for the Frame function**\n```\nFOR FRAME : 1\n \nOutput for each object : [{'box_points': (362, 295, 443, 355), 'name': 'boat', 'percentage_probability': 26.666194200515747}, {'box_points': (319, 245, 386, 296), 'name': 'boat', 'percentage_probability': 30.052968859672546}, {'box_points': (219, 308, 341, 358), 'name': 'boat', 'percentage_probability': 47.46982455253601}, {'box_points': (589, 198, 621, 241), 'name': 'bus', 'percentage_probability': 24.62330162525177}, {'box_points': (519, 181, 583, 263), 'name': 'bus', 'percentage_probability': 27.446213364601135}, {'box_points': (493, 197, 561, 272), 'name': 'bus', 'percentage_probability': 59.81815457344055}, {'box_points': (432, 187, 491, 240), 'name': 'bus', 'percentage_probability': 64.42965269088745}, {'box_points': (157, 225, 220, 255), 'name': 'car', 'percentage_probability': 21.150341629981995}, {'box_points': (324, 249, 377, 293), 'name': 'car', 'percentage_probability': 24.089913070201874}, {'box_points': (152, 275, 260, 327), 'name': 'car', 'percentage_probability': 30.341443419456482}, {'box_points': (433, 198, 485, 244), 'name': 'car', 'percentage_probability': 37.205660343170166}, {'box_points': (184, 226, 233, 260), 'name': 'car', 'percentage_probability': 38.52525353431702}, {'box_points': (3, 296, 134, 359), 'name': 'car', 'percentage_probability': 47.80363142490387}, {'box_points': (357, 302, 439, 359), 'name': 'car', 'percentage_probability': 47.94844686985016}, {'box_points': (481, 266, 546, 314), 'name': 'car', 'percentage_probability': 65.8585786819458}, {'box_points': (597, 269, 624, 318), 'name': 'person', 'percentage_probability': 27.125394344329834}]\n \nOutput count for unique objects : {'bus': 4, 'boat': 3, 'person': 1, 'car': 8}\n\n------------END OF A FRAME --------------\n```\n\nFor any function you parse into the **per_frame_function**, the function will be executed after every single video frame is processed and he following will be parsed into it:\n\n* **Frame Index:** This is the position number of the frame inside the video (e.g 1 for first frame and 20 for twentieth frame).\n* **Output Array:** This is an array of dictionaries. Each dictionary corresponds to each detected object in the image and it contains the \"name\", \"percentage_probabaility\" and \"box_points\"(x1,y1,x2,y2) values of the object.\n* **Output Count:** This is a dictionary that has the name of each unique object detected as its keys and the number of instances of the objects detected as the values.\n\n**Results for the Second function**\n```\nFOR SECOND : 1\n \n Array for the outputs of each frame [[{'box_points': (362, 295, 443, 355), 'name': 'boat', 'percentage_probability': 26.666194200515747}, {'box_points': (319, 245, 386, 296), 'name': 'boat', 'percentage_probability': 30.052968859672546}, {'box_points': (219, 308, 341, 358), 'name': 'boat', 'percentage_probability': 47.46982455253601}, {'box_points': (589, 198, 621, 241), 'name': 'bus', 'percentage_probability': 24.62330162525177}, {'box_points': (519, 181, 583, 263), 'name': 'bus', 'percentage_probability': 27.446213364601135}, {'box_points': (493, 197, 561, 272), 'name': 'bus', 'percentage_probability': 59.81815457344055}, {'box_points': (432, 187, 491, 240), 'name': 'bus', 'percentage_probability': 64.42965269088745}, {'box_points': (157, 225, 220, 255), 'name': 'car', 'percentage_probability': 21.150341629981995}, {'box_points': (324, 249, 377, 293), 'name': 'car', 'percentage_probability': 24.089913070201874}, {'box_points': (152, 275, 260, 327), 'name': 'car', 'percentage_probability': 30.341443419456482}, {'box_points': (433, 198, 485, 244), 'name': 'car', 'percentage_probability': 37.205660343170166}, {'box_points': (184, 226, 233, 260), 'name': 'car', 'percentage_probability': 38.52525353431702}, {'box_points': (3, 296, 134, 359), 'name': 'car', 'percentage_probability': 47.80363142490387}, {'box_points': (357, 302, 439, 359), 'name': 'car', 'percentage_probability': 47.94844686985016}, {'box_points': (481, 266, 546, 314), 'name': 'car', 'percentage_probability': 65.8585786819458}, {'box_points': (597, 269, 624, 318), 'name': 'person', 'percentage_probability': 27.125394344329834}],\n [{'box_points': (316, 240, 384, 302), 'name': 'boat', 'percentage_probability': 29.594269394874573}, {'box_points': (361, 295, 441, 354), 'name': 'boat', 'percentage_probability': 36.11513376235962}, {'box_points': (216, 305, 340, 357), 'name': 'boat', 'percentage_probability': 44.89373862743378}, {'box_points': (432, 198, 488, 244), 'name': 'truck', 'percentage_probability': 22.914741933345795}, {'box_points': (589, 199, 623, 240), 'name': 'bus', 'percentage_probability': 20.545457303524017}, {'box_points': (519, 182, 583, 263), 'name': 'bus', 'percentage_probability': 24.467085301876068}, {'box_points': (492, 197, 563, 271), 'name': 'bus', 'percentage_probability': 61.112016439437866}, {'box_points': (433, 188, 490, 241), 'name': 'bus', 'percentage_probability': 65.08989334106445}, {'box_points': (352, 303, 442, 357), 'name': 'car', 'percentage_probability': 20.025095343589783}, {'box_points': (136, 172, 188, 195), 'name': 'car', 'percentage_probability': 21.571354568004608}, {'box_points': (152, 276, 261, 326), 'name': 'car', 'percentage_probability': 33.07966589927673}, {'box_points': (181, 225, 230, 256), 'name': 'car', 'percentage_probability': 35.111838579177856}, {'box_points': (432, 198, 488, 244), 'name': 'car', 'percentage_probability': 36.25282347202301}, {'box_points': (3, 292, 130, 360), 'name': 'car', 'percentage_probability': 67.55480170249939}, {'box_points': (479, 265, 546, 314), 'name': 'car', 'percentage_probability': 71.47912979125977}, {'box_points': (597, 269, 625, 318), 'name': 'person', 'percentage_probability': 25.903674960136414}],................, \n[{'box_points': (133, 250, 187, 278), 'name': 'umbrella', 'percentage_probability': 21.518094837665558}, {'box_points': (154, 233, 218, 259), 'name': 'umbrella', 'percentage_probability': 23.687003552913666}, {'box_points': (348, 311, 425, 360), 'name': 'boat', 'percentage_probability': 21.015766263008118}, {'box_points': (11, 164, 137, 225), 'name': 'bus', 'percentage_probability': 32.20453858375549}, {'box_points': (424, 187, 485, 243), 'name': 'bus', 'percentage_probability': 38.043853640556335}, {'box_points': (496, 186, 570, 264), 'name': 'bus', 'percentage_probability': 63.83994221687317}, {'box_points': (588, 197, 622, 240), 'name': 'car', 'percentage_probability': 23.51653128862381}, {'box_points': (58, 268, 111, 303), 'name': 'car', 'percentage_probability': 24.538707733154297}, {'box_points': (2, 246, 72, 301), 'name': 'car', 'percentage_probability': 28.433072566986084}, {'box_points': (472, 273, 539, 323), 'name': 'car', 'percentage_probability': 87.17672824859619}, {'box_points': (597, 270, 626, 317), 'name': 'person', 'percentage_probability': 27.459821105003357}]\n ]\n \nArray for output count for unique objects in each frame : [{'bus': 4, 'boat': 3, 'person': 1, 'car': 8},\n {'truck': 1, 'bus': 4, 'boat': 3, 'person': 1, 'car': 7},\n {'bus': 5, 'boat': 2, 'person': 1, 'car': 5},\n {'bus': 5, 'boat': 1, 'person': 1, 'car': 9},\n {'truck': 1, 'bus': 2, 'car': 6, 'person': 1},\n {'truck': 2, 'bus': 4, 'boat': 2, 'person': 1, 'car': 7},\n {'truck': 1, 'bus': 3, 'car': 7, 'person': 1, 'umbrella': 1},\n {'bus': 4, 'car': 7, 'person': 1, 'umbrella': 2},\n {'bus': 3, 'car': 6, 'boat': 1, 'person': 1, 'umbrella': 3},\n {'bus': 3, 'car': 4, 'boat': 1, 'person': 1, 'umbrella': 2}]\n \nOutput average count for unique objects in the last second: {'truck': 0.5, 'bus': 3.7, 'umbrella': 0.8, 'boat': 1.3, 'person': 1.0, 'car': 6.6}\n\n------------END OF A SECOND --------------\n```\n\nIn the above result, the video was processed and saved in 10 frames per second (FPS). For any function you parse into the **per_second_function**, the function will be executed after every single second of the video that is processed and he following will be parsed into it:\n\n- **Second Index:** This is the position number of the second inside the video (e.g 1 for first second and 20 for twentieth second).\n- **Output Array:** This is an array of arrays, with each contained array and its position (array index + 1) corresponding to the equivalent frame in the last second of the video (In the above example, their are 10 arrays which corresponds to the 10 frames contained in one second). Each contained array contains dictionaries. Each dictionary corresponds to each detected object in the image and it contains the \"name\", \"percentage_probabaility\" and \"box_points\"(x1,y1,x2,y2) values of the object.\n- **Count arrays:** This is an array of dictionaries. Each dictionary and its position (array index + 1) corresponds to the equivalent frame in the last second of he video. Each dictionary has the name of each unique object detected as its keys and the number of instances of the objects detected as the values.\n- **Average Output Count:** This is a dictionary that has the name of each unique object detected in the last second as its keys and the average number of instances of the objects detected across the number of frames as the values.\n\n**Results for the Minute function**\nThe above set of **4 parameters** that are returned for every second of the video processed is the same parameters to that will be returned for every minute of the video processed. The difference is that the index returned corresponds to the minute index, the **output_arrays** is an array that contains the number of FPS * 60 number of arrays (in the code example above, 10 frames per second(fps) * 60 seconds = 600 frames = 600 arrays), and the **count_arrays** is an array that contains the number of FPS * 60 number of dictionaries (in the code example above, 10 frames per second(fps) * 60 seconds = 600 frames = 600 dictionaries) and the **average_output_count** is a dictionary that covers all the objects detected in all the frames contained in the last minute.\n\n**Results for the Video Complete Function**\n**ImageAI** allows you to obtain complete analysis of the entire video processed. All you need is to define a function like the forSecond or forMinute function and set the **video_complete_function** parameter into your `.detectObjectsFromVideo()` or `.detectCustomObjectsFromVideo()` function. The same values for the per_second-function and per_minute_function will be returned. The difference is that no index will be returned and the other 3 values will be returned, and the 3 values will cover all frames in the video. Below is a sample function: \n\n```python\ndef forFull(output_arrays, count_arrays, average_output_count):\n #Perform action on the 3 parameters returned into the function\n\nvideo_detector.detectObjectsFromVideo(\n input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\"),\n frames_per_second=10,\n video_complete_function=forFull,\n minimum_percentage_probability=30\n)\n```\n\n**FINAL NOTE ON VIDEO ANALYSIS** : **ImageAI** allows you to obtain the detected video frame as a Numpy array at each frame, second and minute function. All you need to do is specify one more parameter in your function and set `return_detected_frame=True` in your `detectObjectsFromVideo()` or `detectCustomObjectsFrom()` function. Once this is set, the extra parameter you sepecified in your function will be the Numpy array of the detected frame. See a sample below:\n\n```python\ndef forFrame(frame_number, output_array, output_count, detected_frame):\n print(\"FOR FRAME \" , frame_number)\n print(\"Output for each object : \", output_array)\n print(\"Output count for unique objects : \", output_count)\n\tprint(\"Returned Objects is : \", type(detected_frame))\n print(\"------------END OF A FRAME --------------\")\n\nvideo_detector.detectObjectsFromVideo(\n input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\"),\n frames_per_second=10,\n per_frame_function=forFrame,\n minimum_percentage_probability=30,\n return_detected_frame=True\n)\n```\n\n### Frame Detection Intervals\n
\n\nThe above video objects detection task are optimized for frame-real-time object detections that ensures that objects in every frame of the video is detected. **ImageAI** provides you the option to adjust the video frame detections which can speed up your video detection process. When calling the `.detectObjectsFromVideo()` or `.detectCustomObjectsFromVideo()`, you can specify at which frame interval detections should be made. By setting the **frame_detection_interval** parameter to be equal to 5 or 20, that means the object detections in the video will be updated after 5 frames or 20 frames.\nIf your output video **frames_per_second** is set to 20, that means the object detections in the video will be updated once in every quarter of a second or every second. This is useful in case scenarious where the available compute is less powerful and speeds of moving objects are low. This ensures you can have objects detected as second-real-time , half-a-second-real-time or whichever way suits your needs. We conducted video object detection on the same input video we have been using all this while by applying a **frame_detection_interval** value equal to 5.\n\n\n###Video Detection Timeout\n
\n\n**ImageAI** now allows you to set a timeout in seconds for detection of objects in videos or camera live feed.\nTo set a timeout for your video detection code, all you need to do is specify the `detection_timeout` parameter in the `detectObjectsFromVideo()` function to the number of desired seconds. In the example code below, we set `detection_timeout` to 120 seconds (2 minutes). \n\n```python\nfrom imageai.Detection import VideoObjectDetection\nimport os\nimport cv2\n\nexecution_path = os.getcwd()\ncamera = cv2.VideoCapture(0)\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath(os.path.join(execution_path , \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\ndetector.loadModel()\n\n\nvideo_path = detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"camera_detected_video\"),\n frames_per_second=20,\n log_progress=True,\n minimum_percentage_probability=40,\n detection_timeout=120)\n```\n\n\n### Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions. Find links below: \n\n- Documentation - **English Version [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**\n\n" }, { "path": "imageai/Detection/__init__.py", "content": "import os, warnings\nfrom tkinter import Image\nfrom collections import defaultdict\nfrom typing import List, Tuple, Dict, Union\nfrom PIL import Image\nimport torchvision\n\nimport numpy as np\nfrom enum import Enum\nimport torch\nimport cv2\nfrom typing import Union, List\n\nfrom ..yolov3.yolov3 import YoloV3\nfrom ..yolov3.tiny_yolov3 import YoloV3Tiny\nfrom ..yolov3.utils import draw_bbox_and_label, get_predictions, prepare_image\nfrom ..retinanet.utils import read_image, draw_bounding_boxes_and_labels, tensor_to_ndarray\nimport uuid\n\nfrom ..backend_check.model_extension import extension_check\n\nwarnings.filterwarnings(\"once\", category=ResourceWarning)\n\n\nclass ImageReadMode(Enum):\n \"\"\"\n Support for various modes while reading images.\n\n Use ``ImageReadMode.UNCHANGED`` for loading the image as-is,\n ``ImageReadMode.GRAY`` for converting to grayscale,\n ``ImageReadMode.GRAY_ALPHA`` for grayscale with transparency,\n ``ImageReadMode.RGB`` for RGB and ``ImageReadMode.RGB_ALPHA`` for\n RGB with transparency.\n \"\"\"\n\n UNCHANGED = 0\n GRAY = 1\n GRAY_ALPHA = 2\n RGB = 3\n RGB_ALPHA = 4\n\nclass ObjectDetection:\n \"\"\"\n This is the object detection class for images in the ImageAI library. It allows you to detect the 80 objects in the COCO dataset [ https://cocodataset.org/#home ] in any image. \n \n This class provides support for RetinaNet, YOLOv3 and TinyYOLOv3 object detection networks . After instantiating this class, you can set its properties and make object detections using pretrained models.\n\n The following functions are required to be called before object detection can be made\n\n * setModelPath: Used to specify the filepath to the pretrained model.\n\n * At least of of the following and it must correspond to the model set in the setModelPath()\n [setModelTypeAsRetinaNet(), setModelTypeAsYOLOv3(), setModelTypeAsTinyYOLOv3()]\n\n * loadModel: [This must be called once only before performing object detection]\n Once the above functions have been called, you can call the detectObjectsFromImage() function of\n the object detection instance object at anytime to obtain observable objects in any image.\n\n * detectObjectsFromImage: Used to perform object detection on an image\n \"\"\"\n def __init__(self) -> None:\n self.__device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.__nms_score: float = 0.4\n self.__objectness_score: float = 0.5\n self.__anchors: List[int] = None\n self.__anchors_yolov3: List[int] = [10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, 373, 326]\n self.__anchors_tiny_yolov3: List[int] = [10, 14, 23, 27, 37, 58, 81, 82, 135, 169, 344, 319]\n \n self.__classes = self.__load_classes(os.path.join(os.path.dirname(os.path.abspath(__file__)), \"coco_classes.txt\"))\n self.__model_type = \"\"\n self.__model = None\n self.__model_loaded = False\n self.__model_path = \"\"\n \n def __load_classes(self, path: str) -> List[str]:\n with open(path) as f:\n unique_classes = [c.strip() for c in f.readlines()]\n return unique_classes\n\n def __load_image_yolo(self, input_image : Union[str, np.ndarray, Image.Image]) -> Tuple[List[str], List[np.ndarray], torch.Tensor, torch.Tensor]:\n allowed_exts = [\"jpg\", \"jpeg\", \"png\"]\n fnames = []\n original_dims = []\n inputs = []\n original_imgs = []\n if type(input_image) == str:\n if os.path.isfile(input_image):\n if input_image.rsplit('.')[-1].lower() in allowed_exts:\n img = cv2.imread(input_image)\n else:\n raise ValueError(f\"image path '{input_image}' is not found or a valid file\")\n elif type(input_image) == np.ndarray:\n img = input_image\n elif \"PIL\" in str(type(input_image)):\n img = np.asarray(input_image)\n else:\n raise ValueError(f\"Invalid image input format\")\n \n img_h, img_w, _ = img.shape\n\n original_imgs.append(np.array(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)).astype(np.uint8))\n original_dims.append((img_w, img_h))\n if type(input_image) == str:\n fnames.append(os.path.basename(input_image)) \n else:\n fnames.append(\"\") \n inputs.append(prepare_image(img, (416, 416)))\n\n if original_dims:\n return (\n fnames,\n original_imgs,\n torch.FloatTensor(original_dims).repeat(1,2).to(self.__device),\n torch.cat(inputs, 0).to(self.__device)\n )\n raise RuntimeError(\n f\"Error loading image.\"\n \"\\nEnsure the file is a valid image,\"\n \" allowed file extensions are .jpg, .jpeg, .png\"\n )\n \n def __save_temp_img(self, input_image : Union[np.ndarray, Image.Image]) -> str:\n\n temp_path = os.path.join(\n os.path.dirname(os.path.abspath(__file__)),\n f\"{str(uuid.uuid4())}.jpg\" \n ) \n if type(input_image) == np.ndarray:\n cv2.imwrite(temp_path, input_image)\n elif \"PIL\" in str(type(input_image)):\n input_image.save(temp_path)\n else:\n raise ValueError(\n f\"Invalid image input. Supported formats are OpenCV/Numpy array, PIL image or image file path\"\n )\n\n return temp_path\n\n def __load_image_retinanet(self, input_image : str) -> Tuple[List[str], List[torch.Tensor], List[torch.Tensor]]:\n \"\"\"\n Loads image from the given path.\n \"\"\"\n allowed_file_extensions = [\"jpg\", \"jpeg\", \"png\"]\n images = []\n scaled_images = []\n fnames = []\n \n delete_file = False\n if type(input_image) is not str:\n input_image = self.__save_temp_img(input_image=input_image)\n delete_file = True\n\n\n if os.path.isfile(input_image):\n if input_image.rsplit('.')[-1].lower() in allowed_file_extensions:\n img = read_image(input_image, ImageReadMode.RGB)\n images.append(img)\n scaled_images.append(img.div(255.0).to(self.__device))\n fnames.append(os.path.basename(input_image))\n else:\n raise ValueError(f\"Input image with path {input_image} not a valid file\")\n\n if delete_file:\n os.remove(input_image)\n \n if images:\n return (fnames, images, scaled_images)\n raise RuntimeError(\n f\"Error loading image from input.\"\n \"\\nEnsure the folder contains images,\"\n \" allowed file extensions are .jpg, .jpeg, .png\"\n )\n \n def setModelTypeAsYOLOv3(self):\n \"\"\"\n 'setModelTypeAsYOLOv3()' is used to set the model type to the YOLOv3 model.\n :return:\n \"\"\"\n self.__anchors = self.__anchors_yolov3\n self.__model_type = \"yolov3\"\n \n def setModelTypeAsTinyYOLOv3(self):\n \"\"\"\n 'setModelTypeAsTinyYOLOv3()' is used to set the model type to the TinyYOLOv3 model.\n :return:\n \"\"\"\n self.__anchors = self.__anchors_tiny_yolov3\n self.__model_type = \"tiny-yolov3\"\n \n def setModelTypeAsRetinaNet(self):\n \"\"\"\n 'setModelTypeAsRetinaNet()' is used to set the model type to the RetinaNet model.\n :return:\n \"\"\"\n self.__anchors = self.__anchors_tiny_yolov3\n self.__model_type = \"retinanet\"\n\n def setModelPath(self, path: str) -> None:\n \"\"\"\n 'setModelPath()' function is required and is used to set the file path to the model adopted from the list of the\n available 3 model types. The model path must correspond to the model type.\n :param model_path:\n :return:\n \"\"\"\n if os.path.isfile(path):\n extension_check(path)\n self.__model_path = path\n self.__model_loaded = False\n else:\n raise ValueError(\n \"invalid path, path not pointing to a valid file.\"\n ) from None\n \n def useCPU(self):\n \"\"\"\n Used to force classification to be done on CPU.\n By default, classification will occur on GPU compute if available else CPU compute.\n \"\"\"\n\n self.__device = \"cpu\"\n if self.__model_loaded:\n self.__model_loaded = False\n self.loadModel()\n \n def loadModel(self) -> None:\n \"\"\"\n 'loadModel()' function is used to load the model weights into the model architecture from the file path defined\n in the setModelPath() function.\n :return:\n \"\"\"\n if not self.__model_loaded:\n if self.__model_type==\"yolov3\":\n self.__model = YoloV3(\n anchors=self.__anchors ,\n num_classes=len(self.__classes),\\\n device=self.__device\n )\n elif self.__model_type==\"tiny-yolov3\":\n self.__model = YoloV3Tiny(\n anchors=self.__anchors,\n num_classes=len(self.__classes),\n device=self.__device\n )\n elif self.__model_type==\"retinanet\":\n\n self.__classes = self.__load_classes(os.path.join(os.path.dirname(os.path.abspath(__file__)), \"coco91_classes.txt\"))\n\n self.__model = torchvision.models.detection.retinanet_resnet50_fpn(\n pretrained=False, num_classes=91,\n pretrained_backbone = False\n )\n else:\n raise ValueError(f\"Invalid model type. Call setModelTypeAsYOLOv3(), setModelTypeAsTinyYOLOv3() or setModelTypeAsRetinaNet to set a model type before loading the model\")\n\n state_dict = torch.load(self.__model_path, map_location=self.__device)\n try:\n self.__model.load_state_dict(state_dict)\n self.__model_loaded = True\n self.__model.to(self.__device).eval()\n except:\n raise RuntimeError(\"Invalid weights!!!\") from None\n \n def CustomObjects(self, **kwargs):\n\n \"\"\"\n The 'CustomObjects()' function allows you to handpick the type of objects ( from the COCO classes ) you want to detect\n from an image. The objects are pre-initiated in the function variables and predefined as 'False',\n which you can easily set to true for any number of objects available. This function\n returns a dictionary which must be parsed into the 'detectObjectsFromImage()'. Detecting\n custom objects only happens when you call the function 'detectObjectsFromImage()'\n\n Acceptable values are 'True' and 'False' for all object values present\n :param boolean_values:\n :return: custom_objects_dict\n \"\"\"\n\n if not self.__model_loaded:\n self.loadModel()\n all_objects_str = (obj_label.replace(\" \", \"_\") for obj_label in self.__classes)\n all_objects_dict = {}\n for object_str in all_objects_str:\n all_objects_dict[object_str] = False\n \n for karg in kwargs:\n if karg in all_objects_dict:\n all_objects_dict[karg] = kwargs[karg]\n else:\n raise ValueError(f\" object '{karg}' doesn't exist in the supported object classes\")\n\n return all_objects_dict\n\n \n\n def detectObjectsFromImage(self,\n input_image: Union[str, np.ndarray, Image.Image],\n output_image_path: str=None,\n output_type: str =\"file\",\n extract_detected_objects: bool=False, minimum_percentage_probability: int=50,\n display_percentage_probability: bool=True, display_object_name: bool=True,\n display_box: bool=True,\n custom_objects: List=None\n ) -> Union[List[List[Tuple[str, float, Dict[str, int]]]], np.ndarray, List[np.ndarray], List[str]]:\n \"\"\"\n Detects objects in an image using the unique classes provided\n by COCO.\n\n :param input_image: path to an image file, cv2 image or PIL image\n :param output_image_path: path to save input image with predictions rendered\n :param output_type: type of output for rendered image. Acceptable values are 'file' and 'array` ( a cv2 image )\n :param extract_detected_objects: extract each object based on the output type\n :param minimum_percentage_probability: the minimum confidence a detected object must have\n :param display_percentage_probability: to diplay/not display the confidence on rendered image \n :param display_object_name: to diplay/not display the object name on rendered image \n :param display_box: to diplay/not display the object bounding box on rendered image \n :param custom_objects: a dictionary of detectable objects set to boolean values\n \n :returns: A list of tuples containing the label of detected object and the\n confidence.\n \"\"\"\n \n \n self.__model.eval()\n if not self.__model_loaded:\n if self.__model_path:\n warnings.warn(\n \"Model path has changed but pretrained weights in the\"\n \" new path is yet to be loaded.\",\n ResourceWarning\n )\n else:\n raise RuntimeError(\n \"Model path isn't set, pretrained weights aren't used.\"\n )\n predictions = defaultdict(lambda : [])\n \n\n if self.__model_type == \"yolov3\" or self.__model_type == \"tiny-yolov3\":\n fnames, original_imgs, input_dims, imgs = self.__load_image_yolo(input_image)\n \n with torch.no_grad():\n output = self.__model(imgs)\n \n output = get_predictions(\n pred=output.to(self.__device), num_classes=len(self.__classes),\n nms_confidence_level=self.__nms_score, objectness_confidence= self.__objectness_score,\n device=self.__device\n )\n \n if output is None:\n if output_type == \"array\":\n if extract_detected_objects:\n return original_imgs[0], [], []\n else:\n return original_imgs[0], []\n else:\n if extract_detected_objects:\n return original_imgs[0], []\n else:\n return []\n \n # scale the output to match the dimension of the original image\n input_dims = torch.index_select(input_dims, 0, output[:, 0].long())\n scaling_factor = torch.min(416 / input_dims, 1)[0].view(-1, 1)\n output[:, [1,3]] -= (416 - (scaling_factor * input_dims[:, 0].view(-1,1))) / 2\n output[:, [2,4]] -= (416 - (scaling_factor * input_dims[:, 1].view(-1,1))) / 2\n output[:, 1:5] /= scaling_factor\n\n #clip bounding box for those that extended outside the detected image.\n for idx in range(output.shape[0]):\n output[idx, [1,3]] = torch.clamp(output[idx, [1,3]], 0.0, input_dims[idx, 0])\n output[idx, [2,4]] = torch.clamp(output[idx, [2,4]], 0.0, input_dims[idx, 1])\n\n for pred in output:\n pred_label = self.__classes[int(pred[-1])]\n if custom_objects:\n if pred_label.replace(\" \", \"_\") in custom_objects.keys():\n if not custom_objects[pred_label.replace(\" \", \"_\")]:\n continue\n else:\n continue\n predictions[int(pred[0])].append((\n pred_label,\n float(pred[-2]),\n {k:v for k,v in zip([\"x1\", \"y1\", \"x2\", \"y2\"], map(int, pred[1:5]))},\n ))\n elif self.__model_type == \"retinanet\":\n fnames, original_imgs, scaled_images = self.__load_image_retinanet(input_image)\n with torch.no_grad():\n output = self.__model(scaled_images)\n \n if output is None:\n if output_type == \"array\":\n if extract_detected_objects:\n return original_imgs[0], [], []\n else:\n return original_imgs[0], []\n else:\n if extract_detected_objects:\n return original_imgs[0], []\n else:\n return []\n\n for idx, pred in enumerate(output):\n for id in range(pred[\"labels\"].shape[0]):\n if pred[\"scores\"][id] >= self.__objectness_score:\n pred_label = self.__classes[pred[\"labels\"][id]]\n\n if custom_objects:\n if pred_label.replace(\" \", \"_\") in custom_objects.keys():\n if not custom_objects[pred_label.replace(\" \", \"_\")]:\n continue\n else:\n continue\n\n predictions[idx].append(\n (\n pred_label,\n pred[\"scores\"][id].item(),\n {k:v for k,v in zip([\"x1\", \"y1\", \"x2\", \"y2\"], map(int, pred[\"boxes\"][id]))}\n )\n )\n \n # Render detection on copy of input image\n original_input_image = None\n output_image_array = None\n extracted_objects = []\n\n if self.__model_type == \"yolov3\" or self.__model_type == \"tiny-yolov3\":\n original_input_image = cv2.cvtColor(original_imgs[0], cv2.COLOR_RGB2BGR)\n if isinstance(output, torch.Tensor):\n for pred in output:\n percentage_conf = round(float(pred[-2]) * 100, 2)\n if percentage_conf < minimum_percentage_probability:\n continue\n\n displayed_label = \"\"\n if display_object_name:\n displayed_label = f\"{self.__classes[int(pred[-1].item())]} : \"\n if display_percentage_probability:\n displayed_label += f\" {percentage_conf}%\"\n\n\n original_imgs[int(pred[0].item())] = draw_bbox_and_label(pred[1:5].int() if display_box else None,\n displayed_label,\n original_imgs[int(pred[0].item())]\n )\n \n output_image_array = cv2.cvtColor(original_imgs[0], cv2.COLOR_RGB2BGR)\n \n elif self.__model_type == \"retinanet\":\n original_input_image = tensor_to_ndarray(original_imgs[0].div(255.0))\n original_input_image = cv2.cvtColor(original_input_image, cv2.COLOR_RGB2BGR)\n for idx, pred in predictions.items():\n \n max_dim = max(list(original_imgs[idx].size()))\n\n for label, score, bbox in pred:\n percentage_conf = round(score * 100, 2)\n if percentage_conf < minimum_percentage_probability:\n continue\n \n displayed_label = \"\"\n if display_object_name:\n displayed_label = f\"{label} :\"\n if display_percentage_probability:\n displayed_label += f\" {percentage_conf}%\"\n\n original_imgs[idx] = draw_bounding_boxes_and_labels(\n image=original_imgs[idx],\n boxes=torch.Tensor([[bbox[\"x1\"], bbox[\"y1\"], bbox[\"x2\"], bbox[\"y2\"]]]),\n draw_boxes=display_box,\n labels=[displayed_label],\n label_color=(0, 0, 255),\n box_color=(0, 255, 0),\n width=1,\n fill=False,\n font_size=int(max_dim / 30)\n )\n \n output_image_array = tensor_to_ndarray(original_imgs[0].div(255.0))\n output_image_array = cv2.cvtColor(output_image_array, cv2.COLOR_RGB2BGR)\n \n\n # Format predictions for function reponse\n predictions_batch = list(predictions.values())\n predictions_list = predictions_batch[0] if len(predictions_batch) > 0 else []\n min_probability = minimum_percentage_probability / 100\n\n\n if output_type == \"file\":\n if output_image_path:\n cv2.imwrite(output_image_path, output_image_array)\n\n if extract_detected_objects:\n extraction_dir = \".\".join(output_image_path.split(\".\")[:-1]) + \"-extracted\"\n os.mkdir(extraction_dir)\n count = 0\n for obj_prediction in predictions_list: \n if obj_prediction[1] >= min_probability:\n count += 1\n extracted_path = os.path.join(\n extraction_dir, \n \".\".join(os.path.basename(output_image_path).split(\".\")[:-1]) + f\"-{count}.jpg\"\n )\n obj_bbox = obj_prediction[2]\n cv2.imwrite(extracted_path, original_input_image[obj_bbox[\"y1\"] : obj_bbox[\"y2\"], obj_bbox[\"x1\"] : obj_bbox[\"x2\"]])\n\n extracted_objects.append(extracted_path)\n\n elif output_type == \"array\":\n if extract_detected_objects:\n for obj_prediction in predictions_list: \n if obj_prediction[1] >= min_probability:\n obj_bbox = obj_prediction[2]\n\n extracted_objects.append(original_input_image[obj_bbox[\"y1\"] : obj_bbox[\"y2\"], obj_bbox[\"x1\"] : obj_bbox[\"x2\"]])\n else:\n raise ValueError(f\"Invalid output_type '{output_type}'. Supported values are 'file' and 'array' \")\n\n \n predictions_list = [\n {\n \"name\": prediction[0], \"percentage_probability\": round(prediction[1] * 100, 2),\n \"box_points\": [prediction[2][\"x1\"], prediction[2][\"y1\"], prediction[2][\"x2\"], prediction[2][\"y2\"]]\n } for prediction in predictions_list if prediction[1] >= min_probability\n ]\n\n\n if output_type == \"array\":\n if extract_detected_objects:\n return output_image_array, predictions_list, extracted_objects\n else:\n return output_image_array, predictions_list\n else:\n if extract_detected_objects:\n return predictions_list, extracted_objects\n else:\n return predictions_list\n\n\nclass VideoObjectDetection:\n \"\"\"\n This is the object detection class for videos and camera live stream inputs in the ImageAI library. It provides support for RetinaNet,\n YOLOv3 and TinyYOLOv3 object detection networks. After instantiating this class, you can set it's properties and\n make object detections using it's pre-defined functions.\n The following functions are required to be called before object detection can be made\n * setModelPath()\n * At least of of the following and it must correspond to the model set in the setModelPath()\n [setModelTypeAsRetinaNet(), setModelTypeAsYOLOv3(), setModelTinyYOLOv3()]\n * loadModel() [This must be called once only before performing object detection]\n Once the above functions have been called, you can call the detectObjectsFromVideo() function\n or the detectCustomObjectsFromVideo() of the object detection instance object at anytime to\n obtain observable objects in any video or camera live stream.\n \"\"\"\n\n def __init__(self):\n self.__detector = ObjectDetection()\n\n def setModelTypeAsYOLOv3(self):\n self.__detector.setModelTypeAsYOLOv3()\n \n def setModelTypeAsTinyYOLOv3(self):\n self.__detector.setModelTypeAsTinyYOLOv3()\n \n def setModelTypeAsRetinaNet(self):\n self.__detector.setModelTypeAsRetinaNet()\n\n def setModelPath(self, model_path: str):\n extension_check(model_path)\n self.__detector.setModelPath(model_path)\n\n def loadModel(self):\n self.__detector.loadModel()\n \n def useCPU(self):\n self.__detector.useCPU()\n \n def CustomObjects(self, **kwargs):\n return self.__detector.CustomObjects(**kwargs)\n\n def detectObjectsFromVideo(self, input_file_path=\"\", camera_input=None, output_file_path=\"\", frames_per_second=20,\n frame_detection_interval=1, minimum_percentage_probability=50, log_progress=False,\n display_percentage_probability=True, display_object_name=True, display_box=True, save_detected_video=True,\n per_frame_function=None, per_second_function=None, per_minute_function=None,\n video_complete_function=None, return_detected_frame=False, detection_timeout = None, custom_objects=None):\n\n \"\"\"\n 'detectObjectsFromVideo()' function is used to detect objects observable in the given video path or a camera input:\n * input_file_path , which is the file path to the input video. It is required only if 'camera_input' is not set\n * camera_input , allows you to parse in camera input for live video detections\n * output_file_path , which is the path to the output video. It is required only if 'save_detected_video' is not set to False\n * frames_per_second , which is the number of frames to be used in the output video\n * frame_detection_interval (optional, 1 by default) , which is the intervals of frames that will be detected.\n * minimum_percentage_probability (optional, 50 by default) , option to set the minimum percentage probability for nominating a detected object for output.\n * log_progress (optional) , which states if the progress of the frame processed is to be logged to console\n * display_percentage_probability (optional), can be used to hide or show probability scores on the detected video frames\n * display_object_name (optional), can be used to show or hide object names on the detected video frames\n * save_save_detected_video (optional, True by default), can be set to or not to save the detected video\n * per_frame_function (optional), this parameter allows you to parse in a function you will want to execute after each frame of the video is detected. If this parameter is set to a function, after every video frame is detected, the function will be executed with the following values parsed into it:\n -- position number of the frame\n -- an array of dictinaries, with each dictionary corresponding to each object detected. Each dictionary contains 'name', 'percentage_probability' and 'box_points'\n -- a dictionary with with keys being the name of each unique objects and value are the number of instances of the object present\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed as the fourth value into the function\n * per_second_function (optional), this parameter allows you to parse in a function you will want to execute after each second of the video is detected. If this parameter is set to a function, after every second of a video is detected, the function will be executed with the following values parsed into it:\n -- position number of the second\n -- an array of dictionaries whose keys are position number of each frame present in the last second , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n -- an array of dictionaries, with each dictionary corresponding to each frame in the past second, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n -- a dictionary with its keys being the name of each unique object detected throughout the past second, and the key values are the average number of instances of the object found in all the frames contained in the past second\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed\n as the fifth value into the function\n * per_minute_function (optional), this parameter allows you to parse in a function you will want to execute after each minute of the video is detected. If this parameter is set to a function, after every minute of a video is detected, the function will be executed with the following values parsed into it:\n -- position number of the minute\n -- an array of dictionaries whose keys are position number of each frame present in the last minute , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n -- an array of dictionaries, with each dictionary corresponding to each frame in the past minute, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n -- a dictionary with its keys being the name of each unique object detected throughout the past minute, and the key values are the average number of instances of the object found in all the frames contained in the past minute\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed as the fifth value into the function\n * video_complete_function (optional), this parameter allows you to parse in a function you will want to execute after all of the video frames have been detected. If this parameter is set to a function, after all of frames of a video is detected, the function will be executed with the following values parsed into it:\n -- an array of dictionaries whose keys are position number of each frame present in the entire video , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n -- an array of dictionaries, with each dictionary corresponding to each frame in the entire video, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n -- a dictionary with its keys being the name of each unique object detected throughout the entire video, and the key values are the average number of instances of the object found in all the frames contained in the entire video\n * return_detected_frame (optionally, False by default), option to obtain the return the last detected video frame into the per_per_frame_function, per_per_second_function or per_per_minute_function\n * detection_timeout (optionally, None by default), option to state the number of seconds of a video that should be detected after which the detection function stop processing the video\n * thread_safe (optional, False by default), enforce the loaded detection model works across all threads if set to true, made possible by forcing all Tensorflow inference to run on the default graph.\n :param input_file_path:\n :param camera_input\n :param output_file_path:\n :param save_detected_video:\n :param frames_per_second:\n :param frame_detection_interval:\n :param minimum_percentage_probability:\n :param log_progress:\n :param display_percentage_probability:\n :param display_object_name:\n :param per_frame_function:\n :param per_second_function:\n :param per_minute_function:\n :param video_complete_function:\n :param return_detected_frame:\n :param detection_timeout:\n :param thread_safe:\n :return output_video_filepath:\n :return counting:\n :return output_objects_array:\n :return output_objects_count:\n :return detected_copy:\n :return this_second_output_object_array:\n :return this_second_counting_array:\n :return this_second_counting:\n :return this_minute_output_object_array:\n :return this_minute_counting_array:\n :return this_minute_counting:\n :return this_video_output_object_array:\n :return this_video_counting_array:\n :return this_video_counting:\n \"\"\"\n\n if (input_file_path == \"\" and camera_input == None):\n raise ValueError(\n \"You must set 'input_file_path' to a valid video file, or set 'camera_input' to a valid camera\")\n elif (save_detected_video == True and output_file_path == \"\"):\n raise ValueError(\n \"You must set 'output_video_filepath' to a valid video file name, in which the detected video will be saved. If you don't intend to save the detected video, set 'save_detected_video=False'\")\n\n else:\n try:\n\n output_frames_dict = {}\n output_frames_count_dict = {}\n\n input_video = cv2.VideoCapture(input_file_path)\n if (camera_input != None):\n input_video = camera_input\n\n output_video_filepath = output_file_path + '.mp4'\n\n frame_width = int(input_video.get(3))\n frame_height = int(input_video.get(4))\n output_video = cv2.VideoWriter(output_video_filepath, cv2.VideoWriter_fourcc(*\"MP4V\"),\n frames_per_second,\n (frame_width, frame_height))\n\n counting = 0\n\n detection_timeout_count = 0\n video_frames_count = 0\n\n while (input_video.isOpened()):\n ret, frame = input_video.read()\n\n if (ret == True):\n\n video_frames_count += 1\n if (detection_timeout != None):\n if ((video_frames_count % frames_per_second) == 0):\n detection_timeout_count += 1\n\n if (detection_timeout_count >= detection_timeout):\n break\n\n output_objects_array = []\n\n counting += 1\n\n if (log_progress == True):\n print(\"Processing Frame : \", str(counting))\n\n detected_copy = frame.copy()\n\n check_frame_interval = counting % frame_detection_interval\n\n if (counting == 1 or check_frame_interval == 0):\n try:\n detected_copy, output_objects_array = self.__detector.detectObjectsFromImage(\n input_image=frame, output_type=\"array\",\n minimum_percentage_probability=minimum_percentage_probability,\n display_percentage_probability=display_percentage_probability,\n display_object_name=display_object_name,\n display_box=display_box,\n custom_objects=custom_objects)\n except:\n None\n\n output_frames_dict[counting] = output_objects_array\n\n output_objects_count = {}\n for eachItem in output_objects_array:\n eachItemName = eachItem[\"name\"]\n try:\n output_objects_count[eachItemName] = output_objects_count[eachItemName] + 1\n except:\n output_objects_count[eachItemName] = 1\n\n output_frames_count_dict[counting] = output_objects_count\n\n \n if (save_detected_video == True):\n output_video.write(detected_copy)\n\n if (counting == 1 or check_frame_interval == 0):\n if (per_frame_function != None):\n if (return_detected_frame == True):\n per_frame_function(counting, output_objects_array, output_objects_count,\n detected_copy)\n elif (return_detected_frame == False):\n per_frame_function(counting, output_objects_array, output_objects_count)\n\n if (per_second_function != None):\n if (counting != 1 and (counting % frames_per_second) == 0):\n\n this_second_output_object_array = []\n this_second_counting_array = []\n this_second_counting = {}\n\n for aa in range(counting):\n if (aa >= (counting - frames_per_second)):\n this_second_output_object_array.append(output_frames_dict[aa + 1])\n this_second_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_second_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_second_counting[eachItem] = this_second_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_second_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_second_counting:\n this_second_counting[eachCountingItem] = int(this_second_counting[eachCountingItem] / frames_per_second)\n\n if (return_detected_frame == True):\n per_second_function(int(counting / frames_per_second),\n this_second_output_object_array, this_second_counting_array,\n this_second_counting, detected_copy)\n\n elif (return_detected_frame == False):\n per_second_function(int(counting / frames_per_second),\n this_second_output_object_array, this_second_counting_array,\n this_second_counting)\n\n if (per_minute_function != None):\n\n if (counting != 1 and (counting % (frames_per_second * 60)) == 0):\n\n this_minute_output_object_array = []\n this_minute_counting_array = []\n this_minute_counting = {}\n\n for aa in range(counting):\n if (aa >= (counting - (frames_per_second * 60))):\n this_minute_output_object_array.append(output_frames_dict[aa + 1])\n this_minute_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_minute_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_minute_counting[eachItem] = this_minute_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_minute_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_minute_counting:\n this_minute_counting[eachCountingItem] = int(this_minute_counting[eachCountingItem] / (frames_per_second * 60))\n\n if (return_detected_frame == True):\n per_minute_function(int(counting / (frames_per_second * 60)),\n this_minute_output_object_array, this_minute_counting_array,\n this_minute_counting, detected_copy)\n\n elif (return_detected_frame == False):\n per_minute_function(int(counting / (frames_per_second * 60)),\n this_minute_output_object_array, this_minute_counting_array,\n this_minute_counting)\n\n\n else:\n break\n\n if (video_complete_function != None):\n\n this_video_output_object_array = []\n this_video_counting_array = []\n this_video_counting = {}\n\n for aa in range(counting):\n this_video_output_object_array.append(output_frames_dict[aa + 1])\n this_video_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_video_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_video_counting[eachItem] = this_video_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_video_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_video_counting:\n this_video_counting[eachCountingItem] = int(this_video_counting[eachCountingItem] / counting)\n\n video_complete_function(this_video_output_object_array, this_video_counting_array,\n this_video_counting)\n\n input_video.release()\n output_video.release()\n\n if (save_detected_video == True):\n return output_video_filepath\n\n except:\n raise ValueError(\n \"An error occured. It may be that your input video is invalid. Ensure you specified a proper string value for 'output_file_path' is 'save_detected_video' is not False. \"\n \"Also ensure your per_frame, per_second, per_minute or video_complete_analysis function is properly configured to receive the right parameters. \")" }, { "path": "imageai/Detection/coco91_classes.txt", "content": "unlabeled\nperson\nbicycle\ncar\nmotorcycle\nairplane\nbus\ntrain\ntruck\nboat\ntraffic light\nfire hydrant\nstreet sign\nstop sign\nparking meter\nbench\nbird\ncat\ndog\nhorse\nsheep\ncow\nelephant\nbear\nzebra\ngiraffe\nhat\nbackpack\numbrella\nshoe\neye glasses\nhandbag\ntie\nsuitcase\nfrisbee\nskis\nsnowboard\nsports ball\nkite\nbaseball bat\nbaseball glove\nskateboard\nsurfboard\ntennis racket\nbottle\nplate\nwine glass\ncup\nfork\nknife\nspoon\nbowl\nbanana\napple\nsandwich\norange\nbroccoli\ncarrot\nhot dog\npizza\ndonut\ncake\nchair\ncouch\npotted plant\nbed\nmirror\ndining table\nwindow\ndesk\ntoilet\ndoor\ntv\nlaptop\nmouse\nremote\nkeyboard\ncell phone\nmicrowave\noven\ntoaster\nsink\nrefrigerator\nblender\nbook\nclock\nvase\nscissors\nteddy bear\nhair drier\ntoothbrush\nhair brush" }, { "path": "imageai/Detection/coco_classes.txt", "content": "person\nbicycle\ncar\nmotorbike\naeroplane\nbus\ntrain\ntruck\nboat\ntraffic light\nfire hydrant\nstop sign\nparking meter\nbench\nbird\ncat\ndog\nhorse\nsheep\ncow\nelephant\nbear\nzebra\ngiraffe\nbackpack\numbrella\nhandbag\ntie\nsuitcase\nfrisbee\nskis\nsnowboard\nsports ball\nkite\nbaseball bat\nbaseball glove\nskateboard\nsurfboard\ntennis racket\nbottle\nwine glass\ncup\nfork\nknife\nspoon\nbowl\nbanana\napple\nsandwich\norange\nbroccoli\ncarrot\nhot dog\npizza\ndonut\ncake\nchair\nsofa\npottedplant\nbed\ndiningtable\ntoilet\ntvmonitor\nlaptop\nmouse\nremote\nkeyboard\ncell phone\nmicrowave\noven\ntoaster\nsink\nrefrigerator\nbook\nclock\nvase\nscissors\nteddy bear\nhair drier\ntoothbrush" }, { "path": "imageai/__init__.py", "content": "from .backend_check import backend_check" }, { "path": "imageai/backend_check/__init__.py", "content": "" }, { "path": "imageai/backend_check/backend_check.py", "content": "try:\n import torch\n import torchvision\nexcept:\n try:\n import tensorflow\n import keras\n\n raise RuntimeError(\"Dependency error!!! It appears you are trying to use ImageAI with a Tensorflow backend. ImageAI now uses PyTorch as backed as from version 3.0.2 . If you want to use the Tensorflow models or a customly trained '.h5' model, install ImageAI 2.1.6 or earlier. To use the latest Pytorch models, see the documentation in https://imageai.readthedocs.io/\")\n except:\n raise RuntimeError(\"Dependency error!!! PyTorch and TorchVision are not installed. Please see installation instructions in the documentation https://imageai.readthedocs.io/\")" }, { "path": "imageai/backend_check/model_extension.py", "content": "import os\n\ndef extension_check(file_path: str):\n if file_path.endswith(\".h5\"):\n raise RuntimeError(\"You are trying to use a Tensorflow model with ImageAI. ImageAI now uses PyTorch as backed as from version 3.0.2 . If you want to use the Tensorflow models or a customly trained '.h5' model, install ImageAI 2.1.6 or earlier. To use the latest Pytorch models, see the documentation in https://imageai.readthedocs.io/\")\n elif file_path.endswith(\".pt\") == False and file_path.endswith(\".pth\") == False:\n raise ValueError(f\"Invalid model file {os.path.basename(file_path)}. Please parse in a '.pt' and '.pth' model file.\")\n" }, { "path": "imageai/densenet121/__init__.py", "content": "import os, warnings\nfrom pathlib import Path\nfrom typing import List, Tuple\n\nimport torch, torchvision\nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom PIL import Image\n\nwarnings.filterwarnings(\"once\", category=ResourceWarning)\n\nclass DenseNet121Pretrained:\n \"\"\"\n An implementation that allows for easy classification of images\n using the state of the art MobileNet computer vision model.\n \"\"\"\n def __init__(self, label_path : str) -> None:\n self.__model = torchvision.models.densenet121(pretrained=False)\n self.__classes = self.__load_classes(label_path)\n self.__has_loaded_weights = False\n self.__device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.__model_path = \"\"\n \n def __load_classes(self, path : str) -> List[str]:\n with open(path) as f:\n unique_classes = [c.strip() for c in f.readlines()]\n return unique_classes\n\n def __load_image(self, image_path : str) -> Tuple[List[str], torch.Tensor]:\n \"\"\"\n Loads image/images from the given path. If image_path is a directory, this\n function only load the images in the directory (it does not visit the sub-\n directories). This function also convert the loaded image/images to the\n specification expected by the MobileNetV2 architecture.\n \"\"\"\n allowed_file_extensions = [\"jpg\", \"jpeg\", \"png\"]\n images = []\n fnames = []\n preprocess = transforms.Compose([\n transforms.Resize(256),\n transforms.CenterCrop(224),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n ])\n if os.path.isfile(image_path):\n img = Image.open(image_path).convert(\"RGB\")\n images.append(preprocess(img))\n fnames.append(os.path.basename(image_path))\n\n elif os.path.isdir(image_path):\n for file in os.listdir(image_path):\n if os.path.isfile(os.path.join(image_path, file)) and\\\n file.rsplit('.')[-1].lower() in allowed_file_extensions:\n img = Image.open(os.path.join(image_path, file)).convert(\"RGB\")\n images.append(preprocess(img))\n fnames.append(file)\n if images:\n return fnames, torch.stack(images)\n raise RuntimeError(\n f\"Error loading images from {os.path.abspath(image_path)}.\"\n \"\\nEnsure the folder contains images,\"\n \" allowed file extensions are .jpg, .jpeg, .png\"\n )\n\n # properties\n model_path = property(\n fget=lambda self : self.__model_path,\n fset=lambda self, path: self.set_model_path(path),\n doc=\"Path containing the pretrained weight.\"\n )\n\n def set_model_path(self, path : str) -> None:\n \"\"\"\n Sets the path to the pretrained weight.\n \"\"\"\n if os.path.isfile(path):\n self.__model_path = path\n self.__has_loaded_weights = False\n else:\n raise ValueError(\n \"parameter path should be a path to the pretrianed weight file.\"\n )\n\n def load_model(self) -> None:\n \"\"\"\n Loads the mobilenet vison weight into the model architecture.\n \"\"\"\n if not self.__has_loaded_weights:\n try:\n import re\n state_dict = torch.load(self.__model_path, map_location=self.__device)\n # '.'s are no longer allowed in module names, but previous densenet layers\n # as provided by the pytorch organization has names that uses '.'s.\n pattern = re.compile(\n r\"^(.*denselayer\\d+\\.(?:norm|relu|conv))\\.((?:[12])\\.\"\n \"(?:weight|bias|running_mean|running_var))$\"\n )\n for key in list(state_dict.keys()):\n res = pattern.match(key)\n if res:\n new_key = res.group(1) + res.group(2)\n state_dict[new_key] = state_dict[key]\n del state_dict[key]\n self.__model.load_state_dict(state_dict)\n self.__has_loaded_weights = True\n self.__model.eval()\n except Exception:\n print(\"Weight loading failed.\\nEnsure the model path is\"\n \" set and the weight file is in the specified model path.\")\n\n def classify(self, image_path : str, top_n : int = 5, verbose : bool = True) -> List[List[Tuple[str, str]]]:\n \"\"\"\n Classfies image/images according to the classes provided by imagenet.\n\n Parameters:\n -----------\n image_path: a path to a single image or a path to a directory containing\n images. If image_path is a path to a file, this functions\n classifies the image according to the categories provided\n by imagenet, else, if image_path is a path to a directory\n that contains images, this function classifies all images in\n the given directory (it doesn't visit the subdirectories).\n\n top_n: number of top predictions to return.\n verbose: if true, it prints the top_n predictions.\n \"\"\"\n if not self.__has_loaded_weights:\n warnings.warn(\"Pretrained weights aren't loaded\", ResourceWarning)\n\n fnames, images = self.__load_image(image_path)\n images = images.to(self.__device)\n \n with torch.no_grad():\n output = self.__model(images)\n probabilities = torch.softmax(output, dim=1)\n top5_prob, top5_catid = torch.topk(probabilities, 5)\n\n predictions = [\n [\n (self.__classes[top5_catid[i][j]], f\"{top5_prob[i][j].item()*100:.5f}%\")\n for j in range(top5_prob.shape[1])\n ]\n for i in range(top5_prob.shape[0])\n ]\n\n if verbose:\n for idx, pred in enumerate(predictions):\n print(\"-\"*50, f\"Top 5 predictions for {fnames[idx]}\", \"-\"*50, sep=\"\\n\")\n for label, score in pred:\n print(f\"\\t{label}:{score: >10}\")\n print(\"-\"*50, \"\\n\")\n return predictions\n\n" }, { "path": "imageai/inceptionv3/__init__.py", "content": "import os, warnings\nfrom pathlib import Path\nfrom typing import List, Tuple\n\nimport torch, torchvision\nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom PIL import Image\n\nwarnings.filterwarnings(\"once\", category=ResourceWarning)\n\nclass InceptionV3Pretrained:\n \"\"\"\n An implementation that allows for easy classification of images\n using the state of the art MobileNet computer vision model.\n \"\"\"\n def __init__(self, label_path : str) -> None:\n self.__model = torchvision.models.inception_v3(pretrained=False)\n self.__classes = self.__load_classes(label_path)\n self.__has_loaded_weights = False\n self.__device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.__model_path = \"\"\n \n def __load_classes(self, path : str) -> List[str]:\n with open(path) as f:\n unique_classes = [c.strip() for c in f.readlines()]\n return unique_classes\n\n def __load_image(self, image_path : str) -> Tuple[List[str], torch.Tensor]:\n \"\"\"\n Loads image/images from the given path. If image_path is a directory, this\n function only load the images in the directory (it does not visit the sub-\n directories). This function also convert the loaded image/images to the\n specification expected by the MobileNetV2 architecture.\n \"\"\"\n allowed_file_extensions = [\"jpg\", \"jpeg\", \"png\"]\n images = []\n fnames = []\n preprocess = transforms.Compose([\n transforms.Resize(299),\n transforms.CenterCrop(299),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n ])\n if os.path.isfile(image_path):\n img = Image.open(image_path).convert(\"RGB\")\n images.append(preprocess(img))\n fnames.append(os.path.basename(image_path))\n\n elif os.path.isdir(image_path):\n for file in os.listdir(image_path):\n if os.path.isfile(os.path.join(image_path, file)) and\\\n file.rsplit('.')[-1].lower() in allowed_file_extensions:\n img = Image.open(os.path.join(image_path, file)).convert(\"RGB\")\n images.append(preprocess(img))\n fnames.append(file)\n if images:\n return fnames, torch.stack(images)\n raise RuntimeError(\n f\"Error loading images from {os.path.abspath(image_path)}.\"\n \"\\nEnsure the folder contains images,\"\n \" allowed file extensions are .jpg, .jpeg, .png\"\n )\n\n # properties\n model_path = property(\n fget=lambda self : self.__model_path,\n fset=lambda self, path: self.set_model_path(path),\n doc=\"Path containing the pretrained weight.\"\n )\n\n def set_model_path(self, path : str) -> None:\n \"\"\"\n Sets the path to the pretrained weight.\n \"\"\"\n if os.path.isfile(path):\n self.__model_path = path\n self.__has_loaded_weights = False\n else:\n raise ValueError(\n \"parameter path should be a path to the pretrianed weight file.\"\n )\n\n def load_model(self) -> None:\n \"\"\"\n Loads the mobilenet vison weight into the model architecture.\n \"\"\"\n if not self.__has_loaded_weights:\n try:\n self.__model.load_state_dict(\n torch.load(self.__model_path, map_location=self.__device)\n )\n self.__has_loaded_weights = True\n self.__model.eval()\n except Exception:\n print(\"Weight loading failed.\\nEnsure the model path is\"\n \" set and the weight file is in the specified model path.\")\n\n def classify(self, image_path : str, top_n : int = 5, verbose : bool = True) -> List[List[Tuple[str, str]]]:\n \"\"\"\n Classfies image/images according to the classes provided by imagenet.\n\n Parameters:\n -----------\n image_path: a path to a single image or a path to a directory containing\n images. If image_path is a path to a file, this functions\n classifies the image according to the categories provided\n by imagenet, else, if image_path is a path to a directory\n that contains images, this function classifies all images in\n the given directory (it doesn't visit the subdirectories).\n\n top_n: number of top predictions to return.\n verbose: if true, it prints the top_n predictions.\n \"\"\"\n if not self.__has_loaded_weights:\n if self.__model_path:\n warnings.warn(\n \"Model path has changed but pretrained weights in the\"\n \" new path are yet to be loaded.\",\n ResourceWarning\n )\n else:\n warnings.warn(\n \"Model path isn't set, pretrained weights aren't used.\",\n ResourceWarning\n )\n\n fnames, images = self.__load_image(image_path)\n images = images.to(self.__device)\n print(images.shape)\n \n with torch.no_grad():\n output = self.__model(images)\n probabilities = torch.softmax(output, dim=1)\n top5_prob, top5_catid = torch.topk(probabilities, 5)\n\n with open(os.path.join(str(Path(__file__).resolve().parent.parent), \"imagenet_classes.txt\")) as f:\n categories = [c.strip() for c in f.readlines()]\n predictions = [\n [\n (categories[top5_catid[i][j]], f\"{top5_prob[i][j].item()*100:.5f}%\")\n for j in range(top5_prob.shape[1])\n ]\n for i in range(top5_prob.shape[0])\n ]\n\n if verbose:\n for idx, pred in enumerate(predictions):\n print(\"-\"*50, f\"Top 5 predictions for {fnames[idx]}\", \"-\"*50, sep=\"\\n\")\n for label, score in pred:\n print(f\"\\t{label}:{score: >10}\")\n print(\"-\"*50, \"\\n\")\n return predictions\n\n" }, { "path": "imageai/mobilenetv2/__init__.py", "content": "import os, warnings\nfrom pathlib import Path\nfrom typing import List, Tuple\n\nimport torch, torchvision\nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom PIL import Image\n\nwarnings.filterwarnings(\"once\", category=ResourceWarning)\n\nclass MobileNetV2Pretrained:\n \"\"\"\n An implementation that allows for easy classification of images\n using the state of the art MobileNet computer vision model.\n \"\"\"\n def __init__(self, label_path : str) -> None:\n self.__model = torchvision.models.mobilenet_v2(pretrained=False)\n self.__classes = self.__load_classes(label_path)\n self.__has_loaded_weights = False\n self.__device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.__model_path = \"\"\n \n def __load_classes(self, path : str) -> List[str]:\n with open(path) as f:\n unique_classes = [c.strip() for c in f.readlines()]\n return unique_classes\n\n def __load_image(self, image_path : str) -> Tuple[List[str], torch.Tensor]:\n \"\"\"\n Loads image/images from the given path. If image_path is a directory, this\n function only load the images in the directory (it does not visit the sub-\n directories). This function also convert the loaded image/images to the\n specification expected by the MobileNetV2 architecture.\n \"\"\"\n allowed_file_extensions = [\"jpg\", \"jpeg\", \"png\"]\n images = []\n fnames = []\n preprocess = transforms.Compose([\n transforms.Resize(256),\n transforms.CenterCrop(224),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n ])\n if os.path.isfile(image_path):\n img = Image.open(image_path).convert(\"RGB\")\n images.append(preprocess(img))\n fnames.append(os.path.basename(image_path))\n\n elif os.path.isdir(image_path):\n for file in os.listdir(image_path):\n if os.path.isfile(os.path.join(image_path, file)) and\\\n file.rsplit('.')[-1].lower() in allowed_file_extensions:\n img = Image.open(os.path.join(image_path, file)).convert(\"RGB\")\n images.append(preprocess(img))\n fnames.append(file)\n if images:\n return fnames, torch.stack(images)\n raise RuntimeError(\n f\"Error loading images from {os.path.abspath(image_path)}.\"\n \"\\nEnsure the folder contains images,\"\n \" allowed file extensions are .jpg, .jpeg, .png\"\n )\n\n # properties\n model_path = property(\n fget=lambda self : self.__model_path,\n fset=lambda self, path: self.set_model_path(path),\n doc=\"Path containing the pretrained weight.\"\n )\n\n def set_model_path(self, path : str) -> None:\n \"\"\"\n Sets the path to the pretrained weight.\n \"\"\"\n if os.path.isfile(path):\n self.__model_path = path\n self.__has_loaded_weight = False\n else:\n raise ValueError(\n \"parameter path should be a valid path to the pretrianed weight file.\"\n )\n\n def load_model(self) -> None:\n \"\"\"\n Loads the mobilenet vison weight into the model architecture.\n \"\"\"\n if not self.__has_loaded_weights:\n try:\n self.__model.load_state_dict(\n torch.load(self.__model_path, map_location=self.__device)\n )\n self.__has_loaded_weights = True\n self.__model.eval()\n except Exception:\n print(\"Weight loading failed.\\nEnsure the model path is\"\n \" set and the weight file is in the specified model path.\")\n\n def classify(self, image_path : str, top_n : int = 5, verbose : bool = True) -> List[List[Tuple[str, str]]]:\n \"\"\"\n Classfies image/images according to the classes provided by imagenet.\n\n Parameters:\n -----------\n image_path: a path to a single image or a path to a directory containing\n images. If image_path is a path to a file, this functions\n classifies the image according to the categories provided\n by imagenet, else, if image_path is a path to a directory\n that contains images, this function classifies all images in\n the given directory (it doesn't visit the subdirectories).\n\n top_n: number of top predictions to return.\n verbose: if true, it prints the top_n predictions.\n \"\"\"\n if not self.__has_loaded_weights:\n if self.__model_path:\n warnings.warn(\n \"Model path has changed but pretrained weights in the\"\n \" new path are yet to be loaded.\",\n ResourceWarning\n )\n else:\n warnings.warn(\n \"Model path isn't set, pretrained weights aren't used.\",\n ResourceWarning\n )\n\n fnames, images = self.__load_image(image_path)\n images = images.to(self.__device)\n \n with torch.no_grad():\n output = self.__model(images)\n probabilities = torch.softmax(output, dim=1)\n top5_prob, top5_catid = torch.topk(probabilities, 5)\n\n predictions = [\n [\n (self.__classes[top5_catid[i][j]], f\"{top5_prob[i][j].item()*100:.5f}%\")\n for j in range(top5_prob.shape[1])\n ]\n for i in range(top5_prob.shape[0])\n ]\n\n if verbose:\n for idx, pred in enumerate(predictions):\n print(\"-\"*50, f\"Top 5 predictions for {fnames[idx]}\", \"-\"*50, sep=\"\\n\")\n for label, score in pred:\n print(f\"\\t{label}:{score: >10}\")\n print(\"-\"*50, \"\\n\")\n return predictions\n" }, { "path": "imageai/resnet50/__init__.py", "content": "import os, warnings\nfrom typing import List, Tuple\n\nimport torch, torchvision\nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom PIL import Image\n\nwarnings.filterwarnings(\"once\", category=ResourceWarning)\n\nclass ResNet50Pretrained:\n \"\"\"\n An implementation that allows for easy classification of images\n using the state of the art MobileNet computer vision model.\n \"\"\"\n def __init__(self, label_path : str) -> None:\n self.__model = torchvision.models.resnet50(pretrained=False)\n self.__classes = self.__load_classes(label_path)\n self.__has_loaded_weights = False\n self.__device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.__model_path = \"\"\n \n def __load_classes(self, path : str) -> List[str]:\n with open(path) as f:\n unique_classes = [c.strip() for c in f.readlines()]\n return unique_classes\n\n def __load_image(self, image_path : str) -> Tuple[List[str], torch.Tensor]:\n \"\"\"\n Loads image/images from the given path. If image_path is a directory, this\n function only load the images in the directory (it does not visit the sub-\n directories). This function also convert the loaded image/images to the\n specification expected by the MobileNetV2 architecture.\n \"\"\"\n allowed_file_extensions = [\"jpg\", \"jpeg\", \"png\"]\n images = []\n fnames = []\n preprocess = transforms.Compose([\n transforms.Resize(256),\n transforms.CenterCrop(224),\n transforms.ToTensor(),\n transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n ])\n if os.path.isfile(image_path):\n img = Image.open(image_path).convert(\"RGB\")\n images.append(preprocess(img))\n fnames.append(os.path.basename(image_path))\n\n elif os.path.isdir(image_path):\n for file in os.listdir(image_path):\n if os.path.isfile(os.path.join(image_path, file)) and\\\n file.rsplit('.')[-1].lower() in allowed_file_extensions:\n img = Image.open(os.path.join(image_path, file)).convert(\"RGB\")\n images.append(preprocess(img))\n fnames.append(file)\n if images:\n return fnames, torch.stack(images)\n raise RuntimeError(\n f\"Error loading images from {os.path.abspath(image_path)}.\"\n \"\\nEnsure the folder contains images,\"\n \" allowed file extensions are .jpg, .jpeg, .png\"\n )\n\n # properties\n model_path = property(\n fget=lambda self : self.__model_path,\n fset=lambda self, path: self.set_model_path(path),\n doc=\"Path containing the pretrained weight.\"\n )\n\n def set_model_path(self, path : str) -> None:\n \"\"\"\n Sets the path to the pretrained weight.\n \"\"\"\n if os.path.isfile(path):\n self.__model_path = path\n self.__has_loaded_weights = False\n else:\n raise ValueError(\n \"parameter path should be a path to the pretrianed weight file.\"\n )\n\n def load_model(self) -> None:\n \"\"\"\n Loads the mobilenet vison weight into the model architecture.\n \"\"\"\n if not self.__has_loaded_weights:\n try:\n self.__model.load_state_dict(\n torch.load(self.__model_path, map_location=self.__device)\n )\n self.__has_loaded_weights = True\n self.__model.eval()\n except Exception:\n print(\"Weight loading failed.\\nEnsure the model path is\"\n \" set and the weight file is in the specified model path.\")\n\n def classify(self, image_path : str, top_n : int = 5, verbose : bool = True) -> List[List[Tuple[str, str]]]:\n \"\"\"\n Classfies image/images according to the classes provided by imagenet.\n\n Parameters:\n -----------\n image_path: a path to a single image or a path to a directory containing\n images. If image_path is a path to a file, this functions\n classifies the image according to the categories provided\n by imagenet, else, if image_path is a path to a directory\n that contains images, this function classifies all images in\n the given directory (it doesn't visit the subdirectories).\n\n top_n: number of top predictions to return.\n verbose: if true, it prints the top_n predictions.\n \"\"\"\n if not self.__has_loaded_weights:\n if self.__model_path:\n warnings.warn(\n \"Model path has changed but pretrained weights in the\"\n \" new path are yet to be loaded.\",\n ResourceWarning\n )\n else:\n warnings.warn(\n \"Model path isn't set, pretrained weights aren't used.\",\n ResourceWarning\n )\n\n fnames, images = self.__load_image(image_path)\n images = images.to(self.__device)\n \n with torch.no_grad():\n output = self.__model(images)\n probabilities = torch.softmax(output, dim=1)\n top5_prob, top5_catid = torch.topk(probabilities, 5)\n \n predictions = [\n [\n (self.__classes[top5_catid[i][j]], f\"{top5_prob[i][j].item()*100:.5f}%\")\n for j in range(top5_prob.shape[1])\n ]\n for i in range(top5_prob.shape[0])\n ]\n\n if verbose:\n for idx, pred in enumerate(predictions):\n print(\"-\"*50, f\"Top 5 predictions for {fnames[idx]}\", \"-\"*50, sep=\"\\n\")\n for label, score in pred:\n print(f\"\\t{label}:{score: >10}\")\n print(\"-\"*50, \"\\n\")\n return predictions\n" }, { "path": "imageai/retinanet/__init__.py", "content": "" }, { "path": "imageai/retinanet/utils.py", "content": "\nfrom torchvision.io import ImageReadMode\nimport torch\nfrom PIL import Image, ImageColor, ImageDraw, ImageFont\nfrom typing import List, Optional, Union, Tuple, BinaryIO\nimport numpy as np\nimport math\nimport warnings\nimport pathlib\n\ndef read_file(path: str) -> torch.Tensor:\n \"\"\"\n Reads and outputs the bytes contents of a file as a uint8 Tensor\n with one dimension.\n\n Args:\n path (str): the path to the file to be read\n\n Returns:\n data (Tensor)\n \"\"\"\n data = torch.ops.image.read_file(path)\n return data\n\ndef decode_image(input: torch.Tensor, mode: ImageReadMode = ImageReadMode.UNCHANGED) -> torch.Tensor:\n \"\"\"\n Detects whether an image is a JPEG or PNG and performs the appropriate\n operation to decode the image into a 3 dimensional RGB or grayscale Tensor.\n\n Optionally converts the image to the desired format.\n The values of the output tensor are uint8 in [0, 255].\n\n Args:\n input (Tensor): a one dimensional uint8 tensor containing the raw bytes of the\n PNG or JPEG image.\n mode (ImageReadMode): the read mode used for optionally converting the image.\n Default: ``ImageReadMode.UNCHANGED``.\n See ``ImageReadMode`` class for more information on various\n available modes.\n\n Returns:\n output (Tensor[image_channels, image_height, image_width])\n \"\"\"\n output = torch.ops.image.decode_image(input, mode.value)\n return output\n\ndef read_image(path: str, mode: ImageReadMode = ImageReadMode.UNCHANGED) -> torch.Tensor:\n \"\"\"\n Reads a JPEG or PNG image into a 3 dimensional RGB or grayscale Tensor.\n Optionally converts the image to the desired format.\n The values of the output tensor are uint8 in [0, 255].\n\n Args:\n path (str): path of the JPEG or PNG image.\n mode (ImageReadMode): the read mode used for optionally converting the image.\n Default: ``ImageReadMode.UNCHANGED``.\n See ``ImageReadMode`` class for more information on various\n available modes.\n\n Returns:\n output (Tensor[image_channels, image_height, image_width])\n \"\"\"\n \n data = read_file(path)\n return decode_image(data, mode)\n\ndef _generate_color_palette(num_objects: int):\n palette = torch.tensor([2 ** 25 - 1, 2 ** 15 - 1, 2 ** 21 - 1])\n return [tuple((i * palette) % 255) for i in range(num_objects)]\n\n@torch.no_grad()\ndef make_grid(\n tensor: Union[torch.Tensor, List[torch.Tensor]],\n nrow: int = 8,\n padding: int = 2,\n normalize: bool = False,\n value_range: Optional[Tuple[int, int]] = None,\n scale_each: bool = False,\n pad_value: float = 0.0,\n **kwargs,\n) -> torch.Tensor:\n \"\"\"\n Make a grid of images.\n\n Args:\n tensor (Tensor or list): 4D mini-batch Tensor of shape (B x C x H x W)\n or a list of images all of the same size.\n nrow (int, optional): Number of images displayed in each row of the grid.\n The final grid size is ``(B / nrow, nrow)``. Default: ``8``.\n padding (int, optional): amount of padding. Default: ``2``.\n normalize (bool, optional): If True, shift the image to the range (0, 1),\n by the min and max values specified by ``value_range``. Default: ``False``.\n value_range (tuple, optional): tuple (min, max) where min and max are numbers,\n then these numbers are used to normalize the image. By default, min and max\n are computed from the tensor.\n range (tuple. optional):\n .. warning::\n This parameter was deprecated in ``0.12`` and will be removed in ``0.14``. Please use ``value_range``\n instead.\n scale_each (bool, optional): If ``True``, scale each image in the batch of\n images separately rather than the (min, max) over all images. Default: ``False``.\n pad_value (float, optional): Value for the padded pixels. Default: ``0``.\n\n Returns:\n grid (Tensor): the tensor containing grid of images.\n \"\"\"\n if not (torch.is_tensor(tensor) or (isinstance(tensor, list) and all(torch.is_tensor(t) for t in tensor))):\n raise TypeError(f\"tensor or list of tensors expected, got {type(tensor)}\")\n\n if \"range\" in kwargs.keys():\n warnings.warn(\n \"The parameter 'range' is deprecated since 0.12 and will be removed in 0.14. \"\n \"Please use 'value_range' instead.\"\n )\n value_range = kwargs[\"range\"]\n\n # if list of tensors, convert to a 4D mini-batch Tensor\n if isinstance(tensor, list):\n tensor = torch.stack(tensor, dim=0)\n\n if tensor.dim() == 2: # single image H x W\n tensor = tensor.unsqueeze(0)\n if tensor.dim() == 3: # single image\n if tensor.size(0) == 1: # if single-channel, convert to 3-channel\n tensor = torch.cat((tensor, tensor, tensor), 0)\n tensor = tensor.unsqueeze(0)\n\n if tensor.dim() == 4 and tensor.size(1) == 1: # single-channel images\n tensor = torch.cat((tensor, tensor, tensor), 1)\n\n if normalize is True:\n tensor = tensor.clone() # avoid modifying tensor in-place\n if value_range is not None:\n assert isinstance(\n value_range, tuple\n ), \"value_range has to be a tuple (min, max) if specified. min and max are numbers\"\n\n def norm_ip(img, low, high):\n img.clamp_(min=low, max=high)\n img.sub_(low).div_(max(high - low, 1e-5))\n\n def norm_range(t, value_range):\n if value_range is not None:\n norm_ip(t, value_range[0], value_range[1])\n else:\n norm_ip(t, float(t.min()), float(t.max()))\n\n if scale_each is True:\n for t in tensor: # loop over mini-batch dimension\n norm_range(t, value_range)\n else:\n norm_range(tensor, value_range)\n\n assert isinstance(tensor, torch.Tensor)\n if tensor.size(0) == 1:\n return tensor.squeeze(0)\n\n # make the mini-batch of images into a grid\n nmaps = tensor.size(0)\n xmaps = min(nrow, nmaps)\n ymaps = int(math.ceil(float(nmaps) / xmaps))\n height, width = int(tensor.size(2) + padding), int(tensor.size(3) + padding)\n num_channels = tensor.size(1)\n grid = tensor.new_full((num_channels, height * ymaps + padding, width * xmaps + padding), pad_value)\n k = 0\n for y in range(ymaps):\n for x in range(xmaps):\n if k >= nmaps:\n break\n # Tensor.copy_() is a valid method but seems to be missing from the stubs\n # https://pytorch.org/docs/stable/tensors.html#torch.Tensor.copy_\n grid.narrow(1, y * height + padding, height - padding).narrow( # type: ignore[attr-defined]\n 2, x * width + padding, width - padding\n ).copy_(tensor[k])\n k = k + 1\n return grid\n\n\n@torch.no_grad()\ndef draw_bounding_boxes_and_labels(\n image: torch.Tensor,\n boxes: torch.Tensor,\n draw_boxes: bool,\n labels: Optional[List[str]] = None,\n label_color: Optional[Union[List[Union[str, Tuple[int, int, int]]], str, Tuple[int, int, int]]] = None,\n box_color: Optional[Union[List[Union[str, Tuple[int, int, int]]], str, Tuple[int, int, int]]] = None,\n fill: Optional[bool] = False,\n width: int = 1,\n font: Optional[str] = None,\n font_size: int = 10,\n) -> torch.Tensor:\n\n \"\"\"\n Draws bounding boxes on given image.\n The values of the input image should be uint8 between 0 and 255.\n If fill is True, Resulting Tensor should be saved as PNG image.\n\n Args:\n image (Tensor): Tensor of shape (C x H x W) and dtype uint8.\n boxes (Tensor): Tensor of size (N, 4) containing bounding boxes in (xmin, ymin, xmax, ymax) format. Note that\n the boxes are absolute coordinates with respect to the image. In other words: `0 <= xmin < xmax < W` and\n `0 <= ymin < ymax < H`.\n labels (List[str]): List containing the labels of bounding boxes.\n colors (color or list of colors, optional): List containing the colors\n of the boxes or single color for all boxes. The color can be represented as\n PIL strings e.g. \"red\" or \"#FF00FF\", or as RGB tuples e.g. ``(240, 10, 157)``.\n By default, random colors are generated for boxes.\n fill (bool): If `True` fills the bounding box with specified color.\n width (int): Width of bounding box.\n font (str): A filename containing a TrueType font. If the file is not found in this filename, the loader may\n also search in other directories, such as the `fonts/` directory on Windows or `/Library/Fonts/`,\n `/System/Library/Fonts/` and `~/Library/Fonts/` on macOS.\n font_size (int): The requested font size in points.\n\n Returns:\n img (Tensor[C, H, W]): Image Tensor of dtype uint8 with bounding boxes plotted.\n \"\"\"\n\n if not isinstance(image, torch.Tensor):\n raise TypeError(f\"Tensor expected, got {type(image)}\")\n elif image.dtype != torch.uint8:\n raise ValueError(f\"Tensor uint8 expected, got {image.dtype}\")\n elif image.dim() != 3:\n raise ValueError(\"Pass individual images, not batches\")\n elif image.size(0) not in {1, 3}:\n raise ValueError(\"Only grayscale and RGB images are supported\")\n\n num_boxes = boxes.shape[0]\n\n if labels is None:\n labels: Union[List[str], List[None]] = [None] * num_boxes # type: ignore[no-redef]\n elif len(labels) != num_boxes:\n raise ValueError(\n f\"Number of boxes ({num_boxes}) and labels ({len(labels)}) mismatch. Please specify labels for each box.\"\n )\n\n\n # Handle Grayscale images\n if image.size(0) == 1:\n image = torch.tile(image, (3, 1, 1))\n\n ndarr = image.permute(1, 2, 0).cpu().numpy()\n img_to_draw = Image.fromarray(ndarr)\n img_boxes = boxes.to(torch.int64).tolist()\n\n if fill:\n draw = ImageDraw.Draw(img_to_draw, \"RGBA\")\n else:\n draw = ImageDraw.Draw(img_to_draw)\n\n txt_font = ImageFont.load_default() if font is None else ImageFont.truetype(font=font, size=font_size)\n\n for bbox, label in zip(img_boxes, labels):\n if draw_boxes:\n if fill:\n fill_color = label_color + (100,)\n draw.rectangle(bbox, width=width, outline=label_color, fill=fill_color)\n else:\n draw.rectangle(bbox, width=width, outline=box_color)\n\n if label is not None:\n margin = width + 1\n draw.text((bbox[0] + margin, bbox[1] + margin), label, fill=label_color, font=txt_font)\n\n return torch.from_numpy(np.array(img_to_draw)).permute(2, 0, 1).to(dtype=torch.uint8)\n\n\n@torch.no_grad()\ndef tensor_to_ndarray(\n tensor: Union[torch.Tensor, List[torch.Tensor]],\n **kwargs,\n) -> None:\n \"\"\"\n Convert a Tensor into ndarray and return the array\n\n Args:\n tensor (Tensor or list): Image to be saved. If given a mini-batch tensor,\n saves the tensor as a grid of images by calling ``make_grid``.\n fp (string or file object): A filename or a file object\n format(Optional): If omitted, the format to use is determined from the filename extension.\n If a file object was used instead of a filename, this parameter should always be used.\n **kwargs: Other arguments are documented in ``make_grid``.\n \"\"\"\n\n grid = make_grid(tensor, **kwargs)\n # Add 0.5 after unnormalizing to [0, 255] to round to nearest integer\n ndarr = grid.mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to(\"cpu\", torch.uint8).numpy()\n \n return ndarr\n" }, { "path": "imageai/yolov3/__init__.py", "content": "" }, { "path": "imageai/yolov3/tiny_yolov3.py", "content": "from typing import Union, List, Tuple, Optional\n\nimport torch\nimport torch.nn as nn\nimport numpy as np\n\nfrom .yolov3 import DetectionLayer, ConvLayer\n\n\nclass YoloV3Tiny(nn.Module):\n\n def __init__(\n self,\n anchors : Union[List[int], Tuple[int,...]],\n num_classes : int=80,\n device : str=\"cpu\"\n ):\n super().__init__()\n\n # Network Layers\n self.conv1 = ConvLayer(3, 16)\n self.maxpool1 = nn.MaxPool2d(2, 2)\n self.conv2 = ConvLayer(16, 32)\n self.maxpool2 = nn.MaxPool2d(2, 2)\n self.conv3 = ConvLayer(32, 64)\n self.maxpool3 = nn.MaxPool2d(2, 2)\n self.conv4 = ConvLayer(64, 128)\n self.maxpool4 = nn.MaxPool2d(2, 2)\n self.conv5 = ConvLayer(128, 256)\n self.maxpool5 = nn.MaxPool2d(2, 2)\n self.conv6 = ConvLayer(256, 512)\n self.zeropad = nn.ZeroPad2d((0, 1, 0, 1))\n self.maxpool6 = nn.MaxPool2d(2, 1)\n self.conv7 = ConvLayer(512, 1024)\n self.conv8 = ConvLayer(1024, 256, 1, 1)\n self.conv9 = ConvLayer(256, 512)\n self.conv10 = ConvLayer(\n 512, (3 * (5+num_classes)), 1, 1,\n use_batch_norm=False,\n activation=\"linear\"\n )\n self.yolo1 = DetectionLayer(\n num_classes=num_classes, anchors=anchors,\n anchor_masks=(3, 4, 5), device=device, layer=1\n )\n # self.__route_layer(conv8)\n self.conv11 = ConvLayer(256, 128, 1, 1)\n self.upsample1 = nn.Upsample(\n scale_factor=2, mode=\"nearest\"\n #align_corners=True\n )\n # self.__route_layer(upsample1, conv5)\n self.conv12 = ConvLayer(384, 256)\n self.conv13 = ConvLayer(\n 256, (3 * (5 + num_classes)), 1, 1,\n use_batch_norm=False,\n activation=\"linear\"\n )\n self.yolo2 = DetectionLayer(\n num_classes=num_classes, anchors=anchors,\n anchor_masks=(0, 1, 2), device=device, layer=2\n )\n \n def get_loss_layers(self) -> List[torch.Tensor]:\n return [self.yolo1, self.yolo2]\n\n def __route_layer(self, y1 : torch.Tensor, y2 : Optional[torch.Tensor]=None) -> torch.Tensor:\n if isinstance(y2, torch.Tensor):\n return torch.cat([y1, y2], 1)\n return y1\n\n def forward(self, x : torch.Tensor) -> torch.Tensor:\n y = self.maxpool2(self.conv2(self.maxpool1(self.conv1(x))))\n y = self.maxpool4(self.conv4(self.maxpool3(self.conv3(y))))\n r1 = self.conv5(y) # route layer\n y = self.zeropad(self.conv6(self.maxpool5(r1)))\n y = self.conv7(self.maxpool6(y))\n r2 = self.conv8(y) # route layer\n y = self.conv10(self.conv9(r2))\n\n # first detection layer\n out = self.yolo1(y)\n y = self.conv11(self.__route_layer(r2))\n y = self.__route_layer(self.upsample1(y), r1)\n y = self.conv13(self.conv12(y))\n \n # second detection layer\n out = torch.cat([out, self.yolo2(y)], 1)\n\n return out\n" }, { "path": "imageai/yolov3/utils.py", "content": "import math\nfrom typing import Union, List, Tuple\n\nimport torch\nimport numpy as np\nimport cv2 as cv\nfrom torchvision.ops import batched_nms\n\n\ndef draw_bbox_and_label(x : torch.Tensor, label : str, img : np.ndarray) -> np.ndarray:\n \"\"\"\n Draws the predicted bounding boxes on the original image.\n \"\"\"\n x1,y1,x2,y2 = tuple(map(int, x))\n if x is not None:\n img = cv.rectangle(img, (x1,y1), (x2,y2), (0, 255, 0), 1)\n t_size = cv.getTextSize(label, cv.FONT_HERSHEY_PLAIN, 1, 1)[0]\n c2 = (x1 + t_size[0] + 3, y1 + t_size[1] + 4)\n img = cv.putText(img, label, (x1, y1+t_size[1]+4), cv.FONT_HERSHEY_PLAIN, 1, (0,0,255), 1)\n\n return img \n\ndef letterbox_image(\n image : np.ndarray,\n inp_dim : Tuple[int, int]) -> np.ndarray:\n \"\"\"\n Resizes images into the dimension expected by the network. This\n function fills extra spaces in the image with grayscale, if the\n image is smaller than the expected dimesion. This implementation\n keeps the aspect ration of the original image.\n \"\"\"\n img_w, img_h = image.shape[1], image.shape[0] # original image dimension\n net_w, net_h = inp_dim # the dimension expected by the network.\n\n # calculate the new dimension with same aspect ration as\n # the original image.\n scale_factor = min(net_w/img_w, net_h/img_h)\n new_w = int(round(img_w * scale_factor))\n new_h = int(round(img_h * scale_factor))\n\n resized_image = cv.resize(image, (new_w, new_h), interpolation=cv.INTER_CUBIC)\n canvas = np.full((net_w, net_h, 3), 128)\n canvas[(net_h - new_h)//2 : (net_h - new_h)//2 + new_h, (net_w - new_w)//2 : (net_w - new_w)//2 + new_w, :] = resized_image\n return canvas\n\ndef prepare_image(\n image : np.ndarray,\n inp_dim : Tuple[int, int]) -> torch.Tensor:\n \"\"\"\n Prepared the input to match the expectation of the network.\n \"\"\"\n img = letterbox_image(image, inp_dim)\n img = img[:, :, ::-1].transpose((2, 0, 1)).copy()\n img = torch.from_numpy(img).float().div(255.0).unsqueeze(0)\n return img\n\ndef bbox_iou(bbox1 : torch.Tensor, bbox2 : torch.Tensor, device=\"cpu\"):\n \"\"\"\n Returns the IoU value of overlapping boxes\n \"\"\"\n b1_x1, b1_y1, b1_x2, b1_y2 = bbox1[:, 0], bbox1[:, 1], bbox1[:, 2], bbox1[:, 3]\n b2_x1, b2_y1, b2_x2, b2_y2 = bbox2[:, 0], bbox2[:, 1], bbox2[:, 2], bbox2[:, 3]\n\n # intersections\n inter_rect_x1 = torch.max(b1_x1, b2_x1)\n inter_rect_y1 = torch.max(b1_y1, b2_y1)\n inter_rect_x2 = torch.min(b1_x2, b2_x2)\n inter_rect_y2 = torch.min(b1_y2, b2_y2)\n inter_area = torch.max(inter_rect_x2 - inter_rect_x1+1, torch.zeros(inter_rect_x2.shape, device=device)) * \\\n torch.max(inter_rect_y2 - inter_rect_y1+1, torch.zeros(inter_rect_y2.shape, device=device))\n\n b1_area = (b1_x2 - b1_x1 + 1) * (b1_y2 - b1_y1 + 1)\n b2_area = (b2_x2 - b2_x1 + 1) * (b2_y2 - b2_y1 + 1)\n \n return inter_area / (b1_area + b2_area - inter_area)\n\ndef transform_prediction(\n pred : torch.Tensor,\n inp_dim : int,\n anchors : Union[List[int], Tuple[int, ...], torch.Tensor],\n num_classes : int,\n device : str = \"cpu\"\n ) -> torch.Tensor:\n \"\"\"\n Transforms the predictions of the convolutional layers\n from\n batch_size x (3 * 5+num_classes) x grid_size x grid_size\n to\n batch_size x (grid_size * grid_size * anchors) x num_classes\n aids the concatenation of the prediction at the three detection layers\n and also for easy representation of the predicted bounding boxes.\n\n Also, transforms the bounding box predictions and the objectness score\n to match the discription specified in the paper:\n Bx = sigmoid(Tx) + Cx\n By = sigmoid(Ty) + Cy\n Bw = Pw(exp(Tw))\n Bh = Ph(exp(Th))\n\n Parameters:\n -----------\n pred: prediction of the convolutional layer\n inp_dim: the dimension of images expected by the yolo neural network\n anchors: a list of anchors\n num_classes: the numbers of unique classes as specified by COCO.\n\n Returns:\n --------\n the transformed input.\n \"\"\"\n batch_size = pred.shape[0]\n grid_size = pred.shape[2]\n stride = inp_dim // grid_size\n bbox_attrs = 5 + num_classes\n num_anchors = len(anchors)\n\n # transform input shape\n pred = pred.view(batch_size, bbox_attrs*num_anchors, grid_size*grid_size)\n pred = pred.transpose(1, 2).contiguous()\n pred = pred.view(batch_size, grid_size*grid_size*num_anchors, bbox_attrs)\n\n # since the dimensions of the anchors are in accordance with the original\n # dimension of the image, it's required to scale the dimension of the\n # anchors to match the dimension of the output of the convolutional\n # layer\n anchors = [(a[0] / stride, a[1] / stride) for a in anchors]\n\n # sigmoid the center_x, center_y and the objectness score\n pred[:, :, 0] = torch.sigmoid(pred[:, :, 0])\n pred[:, :, 1] = torch.sigmoid(pred[:, :, 1])\n pred[:, :, 4] = torch.sigmoid(pred[:, :, 4])\n\n # add the center offsets\n grid = torch.arange(grid_size, dtype=torch.float)\n grid = np.arange(grid_size)\n x_o, y_o = np.meshgrid(grid, grid)\n #x_offset, y_offset = torch.meshgrid(grid, grid)\n\n x_offset = torch.FloatTensor(x_o).view(-1, 1).to(device)\n y_offset = torch.FloatTensor(y_o).view(-1, 1).to(device)\n #x_offset = x_offset.transpose(0,1).reshape(-1,1).to(device)\n #y_offset = y_offset.transpose(0,1).reshape(-1,1).to(device)\n\n x_y_offset = torch.cat([x_offset, y_offset], dim=1).repeat(1, num_anchors).view(-1,2).unsqueeze(0)\n pred[:, :, :2] += x_y_offset\n \n # transform height and width\n anchors = torch.FloatTensor(anchors).to(device)\n anchors = anchors.repeat(grid_size*grid_size, 1).unsqueeze(0)\n pred[:, :, 2:4] = torch.exp(pred[:, :, 2:4])*anchors\n\n # apply sigmoid to class scores\n pred[:, :, 5:5+num_classes] = torch.sigmoid(pred[:, :, 5:5+num_classes])\n\n # resize bounding box prediction to the original image dimension\n pred[:, :, :4] *= stride\n\n return pred\n\ndef get_predictions(\n pred : torch.Tensor,\n num_classes : int,\n objectness_confidence : float = 0.5,\n nms_confidence_level : float = 0.4,\n device : str = \"cpu\") -> Union[torch.Tensor, int]:\n \"\"\"\n This function filters the bounding boxes predicted by the network by first\n discarding bounding boxes that has low objectness score, and then proceeds\n to filter overlapping bounding boxes using the non-maximum suppression\n algorithm.\n\n Parameters:\n -----------\n pred: a tensor (predicted output) of shape \n 'batch_size x num_bboxes x bbox_attrs'\n num_classes: the number of unique classes as provided by COCO.\n objectness_confidence_level: probability threshold for bounding boxes\n containing a valid object.\n nms_convidence_level: threshold for overlapping bounding boxes\n\n Returns:\n --------\n The prediction with reasonable bounding boxes.\n \"\"\"\n nB = pred.shape[0] # number of batches\n bbox_attr = pred.shape[2] # center_x, center_y, height, width, class_probabilites\n nBBOX = pred.shape[1] # number of bounding boxes\n conf_mask = (pred[:, :, 4] > objectness_confidence).float().unsqueeze(2)\n pred = pred * conf_mask\n\n # transform the predicted centers, height and width to top-left corner and\n # right bottom corner coordinates to aid the ease computation of the IoU\n bbox_corner = pred.new(pred.shape)\n bbox_corner[:, :, 0] = (pred[:, :, 0] - (pred[:, :, 2] / 2)) # top-left_x\n bbox_corner[:, :, 1] = (pred[:, :, 1] - (pred[:, :, 3] / 2)) # top-left_y\n bbox_corner[:, :, 2] = (pred[:, :, 0] + (pred[:, :, 2] / 2)) # bottom_right_x\n bbox_corner[:, :, 3] = (pred[:, :, 1] + (pred[:, :, 3] / 2)) # bottom_right_y\n pred[:, :, :4] = bbox_corner[:, :, :4]\n\n n_pred = pred.view(-1, bbox_attr)\n idxs = torch.arange(nB).reshape(-1,1).repeat(1, nBBOX).view(-1).to(device) # image indices\n\n max_conf, max_idx = torch.max(n_pred[:, 5:5+num_classes], 1) # maximum class score and the index\n max_conf = max_conf.float().unsqueeze(1).to(device)\n max_idx = max_idx.float().unsqueeze(1).to(device)\n n_pred = torch.cat([idxs.unsqueeze(1), n_pred[:, :5], max_conf, max_idx], 1) # batch_idx, x1, y1, x2, y2, objectness_score, class_score, class_idx\n\n valid_bbox_indices = batched_nms(n_pred[:, 1:5].clone(), n_pred[:, 5].clone(), n_pred[:, 7].clone(), nms_confidence_level)\n\n if len(valid_bbox_indices):\n return n_pred[valid_bbox_indices, :]\n return None\n" }, { "path": "imageai/yolov3/yolov3.py", "content": "from typing import Union, List, Tuple, Optional\n\nimport torch\nimport torch.nn as nn\nimport numpy as np\n\nfrom .utils import transform_prediction\n\n\ndef noop(x):\n return x\n\nclass DetectionLayer(nn.Module):\n\n def __init__(\n self,\n anchors : Union[List[int], Tuple[int, ...]],\n anchor_masks : Tuple[int, int, int],\n layer : int,\n num_classes : int=80,\n device : str=\"cpu\"\n ):\n super().__init__()\n self.height = 416\n self.width = 416\n self.num_classes = num_classes\n self.ignore_thresh = 0.7\n self.truth_thresh = 1\n self.rescore = 1\n self.device = device\n self.anchors = self.__get_anchors(anchors, anchor_masks)\n self.layer = layer\n self.layer_width = None\n self.layer_height = None\n self.layer_output = None\n self.pred = None\n self.stride = None\n self.grid = None\n self.anchor_grid = None\n\n def __get_anchors(\n self, anchors : Union[List[int], Tuple[int, ...]],\n anchor_masks : Tuple[int, int, int]\n ) -> torch.Tensor:\n a = [(anchors[i], anchors[i+1]) for i in range(0, len(anchors), 2)]\n return torch.tensor([a[i] for i in anchor_masks]).to(self.device)\n\n def forward(self, x : torch.Tensor):\n self.layer_height, self.layer_width = x.shape[2], x.shape[3]\n self.stride = self.height // self.layer_height\n if self.training:\n batch_size = x.shape[0]\n grid_size = x.shape[2]\n bbox_attrs = 5 + self.num_classes\n num_anchors = len(self.anchors)\n\n # transform input shape\n self.layer_output = x.detach()\n self.pred = x.view(batch_size, num_anchors, bbox_attrs, grid_size, grid_size).permute(0, 1, 3, 4, 2).contiguous()\n \n self.layer_output = self.layer_output.view(batch_size, bbox_attrs*num_anchors, grid_size*grid_size)\n self.layer_output = self.layer_output.transpose(1, 2).contiguous()\n self.layer_output = self.layer_output.view(batch_size, grid_size*grid_size*num_anchors, bbox_attrs)\n\n else:\n # transform the output of the network and scale it to match the\n # network dimension : 416x416\n self.layer_output = transform_prediction(\n x.data, self.width, self.anchors, self.num_classes,\n self.device\n )\n return self.layer_output\n\n\nclass ConvLayer(nn.Module):\n\n def __init__(self, in_f : int, out_f : int, kernel_size : int = 3,\n stride : int = 1, use_batch_norm : bool = True,\n activation : str =\"leaky\"):\n super().__init__()\n self.conv = nn.Conv2d(\n in_f, out_f, stride=stride, kernel_size=kernel_size,\n padding= kernel_size//2,\n bias=False if use_batch_norm else True\n )\n self.batch_norm = nn.BatchNorm2d(out_f) if use_batch_norm else noop\n self.leaky_relu = nn.LeakyReLU(0.1, inplace=True) if activation==\"leaky\" else noop\n\n def forward(self, x : torch.Tensor):\n return self.leaky_relu(self.batch_norm(self.conv(x)))\n\nclass YoloV3(nn.Module):\n\n def __init__(\n self,\n anchors : Union[List[int], Tuple[int, ...]],\n num_classes : int = 80,\n device : str =\"cpu\"):\n super().__init__()\n\n # Network Layers\n self.conv1 = ConvLayer(3, 32)\n self.conv2 = ConvLayer(32, 64, stride=2)\n self.conv3 = ConvLayer(64, 32, 1, 1)\n self.conv4 = ConvLayer(32, 64)\n # self.__shortcut_layer1(self.conv4, self.conv2)\n self.conv5 = ConvLayer(64, 128, stride=2)\n self.conv6 = ConvLayer(128, 64, 1, 1)\n self.conv7 = ConvLayer(64, 128, stride=1)\n # self.__shortcut_layer2(self.conv7, self.conv5)\n self.conv8 = ConvLayer(128, 64, 1, 1)\n self.conv9 = ConvLayer(64, 128, stride=1)\n # self.__shortcut_layer3(self.conv9, shortcut2)\n self.conv10 = ConvLayer(128, 256, stride=2)\n self.conv11 = ConvLayer(256, 128, 1, 1)\n self.conv12 = ConvLayer(128, 256)\n # self.__shortcut_layer4(self.con12, self.conv10)\n self.conv13 = ConvLayer(256, 128, 1, 1)\n self.conv14 = ConvLayer(128, 256)\n # self.__shortcut_layer5(self.conv14, shortcut4)\n self.conv15 = ConvLayer(256, 128, 1, 1)\n self.conv16 = ConvLayer(128, 256)\n # self.__shortcut_layer6(self.conv16, shortcut5)\n self.conv17 = ConvLayer(256, 128, 1, 1)\n self.conv18 = ConvLayer(128, 256)\n # self.__shortcut_layer7(self.conv18, shortcut6)\n self.conv19 = ConvLayer(256, 128, 1, 1)\n self.conv20 = ConvLayer(128, 256)\n # self.__shortcut_layer8(self.conv20, shortcut7)\n self.conv21 = ConvLayer(256, 128, 1, 1)\n self.conv22 = ConvLayer(128, 256)\n # self.__shortcut_layer9(self.conv22, shortcut8)\n self.conv23 = ConvLayer(256, 128, 1, 1)\n self.conv24 = ConvLayer(128, 256)\n # self.__shortcut_layer10(self.conv24, shortcut9)\n self.conv25 = ConvLayer(256, 128, 1, 1)\n self.conv26 = ConvLayer(128, 256)\n # self.__shortcut_layer11(self.conv26, shortcut10)\n self.conv27 = ConvLayer(256, 512, stride=2)\n self.conv28 = ConvLayer(512, 256, 1, 1)\n self.conv29 = ConvLayer(256, 512)\n # self.__shortcut_layer12(self.conv29, self.conv27)\n self.conv30 = ConvLayer(512, 256, 1, 1)\n self.conv31 = ConvLayer(256, 512)\n # self.__shortcut_layer13(self.conv31, shortcut12)\n self.conv32 = ConvLayer(512, 256, 1, 1)\n self.conv33 = ConvLayer(256, 512)\n # self.__shortcut_layer14(self.conv33, shortcut13)\n self.conv34 = ConvLayer(512, 256, 1, 1)\n self.conv35 = ConvLayer(256, 512)\n # self.__shortcut_layer15(self.conv35, shortcut14)\n self.conv36 = ConvLayer(512, 256, 1, 1)\n self.conv37 = ConvLayer(256, 512)\n # self.__shortcut_layer16(self.conv37, shortcut15)\n self.conv38 = ConvLayer(512, 256, 1, 1)\n self.conv39 = ConvLayer(256, 512)\n # self.__shortcut_layer17(self.conv39, shortcut16)\n self.conv40 = ConvLayer(512, 256, 1, 1)\n self.conv41 = ConvLayer(256, 512)\n # self.__shortcut_layer18(self.conv41, shortcut17)\n self.conv42 = ConvLayer(512, 256, 1, 1)\n self.conv43 = ConvLayer(256, 512)\n # self.__shortcut_layer19(self.conv43, shortcut18)\n self.conv44 = ConvLayer(512, 1024, stride=2)\n self.conv45 = ConvLayer(1024, 512, 1, 1)\n self.conv46 = ConvLayer(512, 1024)\n # self.__shortcut_layer20(self.conv46, self.conv44)\n self.conv47 = ConvLayer(1024, 512, 1, 1)\n self.conv48 = ConvLayer(512, 1024)\n # self.__shortcut_layer21(self.conv48, shortcut20)\n self.conv49 = ConvLayer(1024, 512, 1, 1)\n self.conv50 = ConvLayer(512, 1024)\n # self.__shortcut_layer22(self.conv50, shortcut21)\n self.conv51 = ConvLayer(1024, 512, 1, 1)\n self.conv52 = ConvLayer(512, 1024)\n # self.__shortcut_layer23(self.conv52, shortcut22)\n self.conv53 = ConvLayer(1024, 512, 1, 1)\n self.conv54 = ConvLayer(512, 1024)\n self.conv55 = ConvLayer(1024, 512, 1, 1)\n self.conv56 = ConvLayer(512, 1024)\n self.conv57 = ConvLayer(1024, 512, 1, 1)\n self.conv58 = ConvLayer(512, 1024)\n self.conv59 = ConvLayer(\n 1024, (3 * (5 + num_classes)), 1, 1, use_batch_norm=False,\n activation=\"linear\"\n )\n\n # yolo layer\n self.yolo1 = DetectionLayer(\n num_classes=num_classes, anchors=anchors,\n anchor_masks=(6, 7, 8), device=device, layer=1\n )\n\n # self.__route_layer(self.conv57)\n self.conv60 = ConvLayer(512, 256, 1, 1)\n self.upsample1 = nn.Upsample(\n scale_factor=2, mode=\"nearest\"\n #align_corners=True\n )\n # self.__route_layer(self.upsample1, shortcut19)\n self.conv61 = ConvLayer(768, 256, 1, 1)\n self.conv62 = ConvLayer(256, 512)\n self.conv63 = ConvLayer(512, 256, 1, 1)\n self.conv64 = ConvLayer(256, 512)\n self.conv65 = ConvLayer(512, 256, 1, 1)\n self.conv66 = ConvLayer(256, 512)\n self.conv67 = ConvLayer(\n 512, (3 * (5 + num_classes)), 1, 1, use_batch_norm=False,\n activation=\"linear\"\n )\n \n # yolo layer\n self.yolo2 = DetectionLayer(\n num_classes=num_classes, anchors=anchors,\n anchor_masks=(3, 4, 5), device=device, layer=2\n )\n \n # self.__route_layer(self.conv65)\n self.conv68 = ConvLayer(256, 128, 1, 1)\n self.upsample2 = nn.Upsample(\n scale_factor=2, mode=\"nearest\"\n #align_corners=True\n )\n # self.__route_layer(self.upsample2, shortcut11)\n\n self.conv69 = ConvLayer(384, 128, 1, 1)\n self.conv70 = ConvLayer(128, 256)\n self.conv71 = ConvLayer(256, 128, 1, 1)\n self.conv72 = ConvLayer(128, 256)\n self.conv73 = ConvLayer(256, 128, 1, 1)\n self.conv74 = ConvLayer(128, 256)\n self.conv75 = ConvLayer(\n 256, (3 * (5 + num_classes)), 1, 1, use_batch_norm=False,\n activation=\"linear\"\n )\n\n # yolo layer\n self.yolo3 = DetectionLayer(\n num_classes=num_classes, anchors=anchors,\n anchor_masks=(0, 1, 2), device=device, layer=3\n )\n \n def get_loss_layers(self) -> List[torch.Tensor]:\n return [self.yolo1, self.yolo2, self.yolo3]\n\n def __route_layer(self, y1 : torch.Tensor, y2 : Optional[torch.Tensor]=None):\n if isinstance(y2, torch.Tensor):\n return torch.cat([y1, y2], 1)\n return y1\n\n def __shortcut_layer(self,\n y1 : torch.Tensor, y2 : torch.Tensor,\n activation : str=\"linear\"\n ) -> torch.Tensor:\n actv = noop if activation==\"linear\" else nn.LeakyReLU(0.1)\n return actv(y1 + y2)\n\n def forward(self, x : torch.Tensor) -> torch.Tensor:\n y = self.conv2(self.conv1(x))\n # shortcut1\n y = self.conv5(self.__shortcut_layer(self.conv4(self.conv3(y)), y))\n y2 = self.conv7(self.conv6(y))\n # shortcut2\n y = self.__shortcut_layer(y2, y)\n y2 = self.conv9(self.conv8(y))\n # shortcut3\n y2 = self.conv10(self.__shortcut_layer(y2, y))\n y = self.conv12(self.conv11(y2))\n # shortcut4\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv14(self.conv13(y2))\n # shortcut5\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv16(self.conv15(self.__shortcut_layer(y2, y)))\n # shortcut6\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv18(self.conv17(y2))\n # shortcut7\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv20(self.conv19(y2))\n # shortcut8\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv22(self.conv21(y2))\n # shortcut9\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv24(self.conv23(y2))\n # shortcut10\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv26(self.conv25(y2))\n # shortcut11\n r1 = self.__shortcut_layer(y, y2) # route_layer\n y = self.conv27(r1)\n y2 = self.conv29(self.conv28(y))\n # shortcut12\n y = self.__shortcut_layer(y2, y)\n y2 = self.conv31(self.conv30(y))\n # shortcut13\n y = self.__shortcut_layer(y2, y)\n y2 = self.conv33(self.conv32(y))\n # shortcut14\n y = self.__shortcut_layer(y2, y)\n y2 = self.conv35(self.conv34(y))\n # shortcut15\n y = self.__shortcut_layer(y2, y)\n y2 = self.conv37(self.conv36(y))\n # shortcut16\n y = self.__shortcut_layer(y2, y)\n y2 = self.conv39(self.conv38(y))\n # shortcut17\n y = self.__shortcut_layer(y2, y)\n y2 = self.conv41(self.conv40(y))\n # shortcut18\n y = self.__shortcut_layer(y2, y)\n y2 = self.conv43(self.conv42(y))\n # shortcut19\n r2 = self.__shortcut_layer(y2, y) # route_layer\n y2 = self.conv44(r2)\n y = self.conv46(self.conv45(y2))\n # shortcut20\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv48(self.conv47(y2))\n # shortcut21\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv50(self.conv49(y2))\n # shortcut22\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv52(self.conv51(y2))\n # shortcut23\n y2 = self.__shortcut_layer(y, y2)\n y = self.conv54(self.conv53(y2))\n r3 = self.conv57(self.conv56(self.conv55(y))) # route_layer\n y = self.conv59(self.conv58(r3))\n\n # first detection layer\n out = self.yolo1(y)\n y = self.conv60(self.__route_layer(r3))\n y = self.conv62(self.conv61(self.__route_layer(self.upsample1(y), r2)))\n r4 = self.conv65(self.conv64(self.conv63(y))) # route_layer\n y = self.conv67(self.conv66(r4))\n\n # second detection layer\n out = torch.cat([out, self.yolo2(y)], dim=1)\n y = self.conv68(self.__route_layer(r4))\n y = self.conv70(self.conv69(self.__route_layer(self.upsample2(y), r1)))\n y = self.conv75(self.conv74(self.conv73(self.conv72(self.conv71(y)))))\n\n # third detection layer\n out = torch.cat([out, self.yolo3(y)], dim=1)\n\n return out\n" }, { "path": "imageai_tf_deprecated/Classification/CUSTOMCLASSIFICATION.md", "content": "# ImageAI : Custom Image Classification\nA **DeepQuest AI** project https://deepquestai.com

\n\n---\n\nImageAI provides 4 different algorithms and model types to perform custom image prediction using your custom models.\nYou will be able to use your model trained with **ImageAI** and the corresponding model_class JSON file to predict custom objects\nthat you have trained the model on.\n\n### TABLE OF CONTENTS\n\n- :white_square_button: Custom Model Prediction\n- :white_square_button: Custom Model Prediction with Full Model (NEW)\n- :white_square_button: Custom Prediction with multiple models (NEW)\n- :white_square_button: Convert custom model to Tensorflow's format (NEW)\n- :white_square_button: Convert custom model to DeepStack's format (NEW)\n\n\n### Custom Model Prediction\n
\n\nIn this example, we will be using the model trained for 20 experiments on **IdenProf**, a dataset of uniformed professionals and achieved 65.17% accuracy on the test dataset.\n(You can use your own trained model and generated JSON file. This 'class' is provided mainly for the purpose to use your own custom models.)\nDownload the ResNet model of the model and JSON files in links below:\n\n- [**ResNet50**](https://github.com/OlafenwaMoses/ImageAI/releases/download/essentials-v5/idenprof_resnet_ex-056_acc-0.993062.h5) _(Size = 90.4 mb)_\n- [**IdenProf model_class.json file**](https://github.com/OlafenwaMoses/ImageAI/releases/download/essentials-v5/idenprof.json)\n\nGreat!\nOnce you have downloaded this model file and the JSON file, start a new python project, and then copy the model file and the JSON file to your project folder where your python files (.py files) will be.\nDownload the image below, or take any image on your computer that include any of the following professionals(Chef, Doctor, Engineer, Farmer, Fireman, Judge, Mechanic, Pilot, Police and Waiter) and copy it to your python project's folder.\nThen create a python file and give it a name; an example is **FirstCustomPrediction.py**.\nThen write the code below into the python file:\n\n### FirstCustomPrediction.py\n\n```python\nfrom imageai.Classification.Custom import CustomImageClassification\nimport os\n\nexecution_path = os.getcwd()\n\nprediction = CustomImageClassification()\nprediction.setModelTypeAsResNet50()\nprediction.setModelPath(os.path.join(execution_path, \"idenprof_resnet_ex-056_acc-0.993062.h5\"))\nprediction.setJsonPath(os.path.join(execution_path, \"idenprof.json\"))\nprediction.loadModel(num_objects=10)\n\npredictions, probabilities = prediction.classifyImage(os.path.join(execution_path, \"4.jpg\"), result_count=5)\n\nfor eachPrediction, eachProbability in zip(predictions, probabilities):\n print(eachPrediction + \" : \" + eachProbability)\n```\n\n**Sample Result:**\n\n![Sample Result](../../data-images/4.jpg)\n```\nmechanic : 76.82620286941528\nchef : 10.106072574853897\nwaiter : 4.036874696612358\npolice : 2.6663416996598244\npilot : 2.239348366856575\n```\n\nThe code above works as follows:\n```python\nfrom imageai.Classification.Custom import CustomImageClassification\nimport os\n```\nThe code above imports the **ImageAI** library for custom image prediction and the python **os** class.\n\n```python\nexecution_path = os.getcwd()\n```\n\nThe above line obtains the path to the folder that contains your python file (in this example, your FirstCustomPrediction.py).\n\n```python\nprediction = CustomImageClassification()\nprediction.setModelTypeAsResNet50()\nprediction.setModelPath(os.path.join(execution_path, \"idenprof_resnet_ex-056_acc-0.993062.h5\"))\nprediction.setJsonPath(os.path.join(execution_path, \"idenprof.json\"))\nprediction.loadModel(num_objects=10)\n```\n\nIn the lines above, we created and instance of the `CustomImageClassification()`\n class in the first line, then we set the model type of the prediction object to ResNet by caling the `.setModelTypeAsResNet50()`\n in the second line, we set the model path of the prediction object to the path of the custom model file (`idenprof_resnet_ex-056_acc-0.993062.h5`) we copied to the python file folder\n in the third line, we set the path to the model_class.json of the model, we load the model and parse the number of objected that can be predicted in the model.\n\n```python\npredictions, probabilities = prediction.classifyImage(os.path.join(execution_path, \"4.jpg\"), result_count=5)\n```\n\nIn the above line, we defined 2 variables to be equal to the function called to predict an image, which is the `.classifyImage()` function, into which we parsed the path to our image and also state the number of prediction results we want to have (values from 1 to 10 in this case) parsing `result_count=5`. The `.classifyImage()` function will return 2 array objects with the first (**predictions**) being an array of predictions and the second (**percentage_probabilities**) being an array of the corresponding percentage probability for each prediction.\n\n```python\nfor eachPrediction, eachProbability in zip(predictions, probabilities):\n print(eachPrediction + \" : \" + eachProbability)\n```\n\nThe above line obtains each object in the **predictions** array, and also obtains the corresponding percentage probability from the **percentage_probabilities**, and finally prints the result of both to console.\n\n**CustomImageClassification** class also supports the multiple predictions, input types and prediction speeds that are contained\nin the **ImageClassification** class. Follow this [link](README.md) to see all the details.\n\n\n\n### Custom Prediction with multiple models\n
\n\n\nIn previous versions of **ImageAI**, running more than one custom model at once wasn't supported.\nNow you can run multiple custom models, as many as your computer memory can accommodate.\nSee the example code below for running multiple custom prediction models.\n\n```python\nfrom imageai.Classification.Custom import CustomImageClassification\nimport os\n\nexecution_path = os.getcwd()\n\npredictor = CustomImageClassification()\npredictor.setModelPath(model_path=os.path.join(execution_path, \"idenprof_resnet.h5\"))\npredictor.setJsonPath(model_json=os.path.join(execution_path, \"idenprof.json\"))\npredictor.setModelTypeAsResNet50()\npredictor.loadModel(num_objects=10)\n\npredictor2 = CustomImageClassification()\npredictor2.setModelPath(model_path=os.path.join(execution_path, \"idenprof_inception_0.719500.h5\"))\npredictor2.setJsonPath(model_json=os.path.join(execution_path, \"idenprof.json\"))\npredictor2.setModelTypeAsInceptionV3()\npredictor2.loadModel(num_objects=10)\n\nresults, probabilities = predictor.classifyImage(image_input=os.path.join(execution_path, \"9.jpg\"), result_count=5)\nprint(results)\nprint(probabilities)\n\n\nresults2, probabilities2 = predictor3.classifyImage(image_input=os.path.join(execution_path, \"9.jpg\"),\n result_count=5)\nprint(results2)\nprint(probabilities2)\nprint(\"-------------------------------\")\n```\n\n### Documentation\n\nWe have provided full documentation for all **ImageAI** classes and functions in 3 major languages. Find links below:**\n\n* Documentation - **English Version [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**\n* Documentation - **Chinese Version [https://imageai-cn.readthedocs.io](https://imageai-cn.readthedocs.io)**\n* Documentation - **French Version [https://imageai-fr.readthedocs.io](https://imageai-fr.readthedocs.io)**\n\n" }, { "path": "imageai_tf_deprecated/Classification/CUSTOMTRAINING.md", "content": "# ImageAI : Custom Prediction Model Training \n\n---\n\n**ImageAI** provides the most simple and powerful approach to training custom image prediction models\nusing state-of-the-art SqueezeNet, ResNet50, InceptionV3 and DenseNet\nwhich you can load into the `imageai.Classification.Custom.CustomImageClassification` class. This allows\n you to train your own model on any set of images that corresponds to any type of objects/persons.\nThe training process generates a JSON file that maps the objects types in your image dataset\nand creates lots of models. You will then pick the model with the highest accuracy and perform custom\nimage prediction using the model and the JSON file generated.\n\n### TABLE OF CONTENTS\n- :white_square_button: Custom Model Training Prediction \n- :white_square_button: Saving Full Custom Model \n- :white_square_button: Training on the IdenProf Dataset \n- :white_square_button: Continuous Model Training \n- :white_square_button: Transfer Learning (Training from a pre-trained model)\n\n\n### Custom Model Training\n
\n\nBecause model training is a compute intensive tasks, we strongly advise you perform this experiment using a computer with a NVIDIA GPU and the GPU version of Tensorflow installed. Performing model training on CPU will my take hours or days. With NVIDIA GPU powered computer system, this will take a few hours. You can use Google Colab for this experiment as it has an NVIDIA K80 GPU available.\n\nTo train a custom prediction model, you need to prepare the images you want to use to train the model.\nYou will prepare the images as follows:\n\n1. Create a dataset folder with the name you will like your dataset to be called (e.g pets) \n2. In the dataset folder, create a folder by the name **train** \n3. In the dataset folder, create a folder by the name **test** \n4. In the train folder, create a folder for each object you want to the model to predict and give the folder a name that corresponds to the respective object name (e.g dog, cat, squirrel, snake) \n5. In the test folder, create a folder for each object you want to the model to predict and give\n the folder a name that corresponds to the respective object name (e.g dog, cat, squirrel, snake) \n6. In each folder present in the train folder, put the images of each object in its respective folder. This images are the ones to be used to train the model To produce a model that can perform well in practical applications, I recommend you about 500 or more images per object. 1000 images per object is just great \n7. In each folder present in the test folder, put about 100 to 200 images of each object in its respective folder. These images are the ones to be used to test the model as it trains \n8. Once you have done this, the structure of your image dataset folder should look like below: \n ```\n pets//train//dog//dog-train-images\n pets//train//cat//cat-train-images\n pets//train//squirrel//squirrel-train-images\n pets//train//snake//snake-train-images \n pets//test//dog//dog-test-images\n pets//test//cat//cat-test-images\n pets//test//squirrel//squirrel-test-images\n pets//test//snake//snake-test-images\n ```\n9. Then your training code goes as follows: \n ```python\n from imageai.Classification.Custom import ClassificationModelTrainer\n model_trainer = ClassificationModelTrainer()\n model_trainer.setModelTypeAsResNet50()\n model_trainer.setDataDirectory(\"pets\")\n model_trainer.trainModel(num_objects=4, num_experiments=100, enhance_data=True, batch_size=32, show_network_summary=True)\n ```\n\n Yes! Just 5 lines of code and you can train any of the available 4 state-of-the-art Deep Learning algorithms on your custom dataset.\nNow lets take a look at how the code above works.\n\n```python\nfrom imageai.Classification.Custom import ClassificationModelTrainer\nmodel_trainer = ClassificationModelTrainer()\nmodel_trainer.setModelTypeAsResNet50()\nmodel_trainer.setDataDirectory(\"pets\")\n```\n\nIn the first line, we import the **ImageAI** model training class, then we define the model trainer in the second line,\n we set the network type in the third line and set the path to the image dataset we want to train the network on.\n\n```python\nmodel_trainer.trainModel(num_objects=4, num_experiments=100, enhance_data=True, batch_size=32, show_network_summary=True)\n```\n\nIn the code above, we start the training process. The parameters stated in the function are as below:\n- **num_objects** : this is to state the number of object types in the image dataset \n- **num_experiments** : this is to state the number of times the network will train over all the training images,\n which is also called epochs \n- **enhance_data (optional)** : This is used to state if we want the network to produce modified copies of the training\nimages for better performance. \n- **batch_size** : This is to state the number of images the network will process at ones. The images\n are processed in batches until they are exhausted per each experiment performed. \n- **show_network_summary** : This is to state if the network should show the structure of the training\n network in the console.\n \n\nWhen you start the training, you should see something like this in the console:\n```\nTotal params: 23,608,202\nTrainable params: 23,555,082\nNon-trainable params: 53,120\n____________________________________________________________________________________________________\nUsing Enhanced Data Generation\nFound 4000 images belonging to 4 classes.\nFound 800 images belonging to 4 classes.\nJSON Mapping for the model classes saved to C:\\Users\\User\\PycharmProjects\\ImageAITest\\pets\\json\\model_class.json\nNumber of experiments (Epochs) : 100\n```\n\nWhen the training progress progresses, you will see results as follows in the console: \n```\nEpoch 1/100\n 1/25 [>.............................] - ETA: 52s - loss: 2.3026 - acc: 0.2500\n 2/25 [=>............................] - ETA: 41s - loss: 2.3027 - acc: 0.1250\n 3/25 [==>...........................] - ETA: 37s - loss: 2.2961 - acc: 0.1667\n 4/25 [===>..........................] - ETA: 36s - loss: 2.2980 - acc: 0.1250\n 5/25 [=====>........................] - ETA: 33s - loss: 2.3178 - acc: 0.1000\n 6/25 [======>.......................] - ETA: 31s - loss: 2.3214 - acc: 0.0833\n 7/25 [=======>......................] - ETA: 30s - loss: 2.3202 - acc: 0.0714\n 8/25 [========>.....................] - ETA: 29s - loss: 2.3207 - acc: 0.0625\n 9/25 [=========>....................] - ETA: 27s - loss: 2.3191 - acc: 0.0556\n10/25 [===========>..................] - ETA: 25s - loss: 2.3167 - acc: 0.0750\n11/25 [============>.................] - ETA: 23s - loss: 2.3162 - acc: 0.0682\n12/25 [=============>................] - ETA: 21s - loss: 2.3143 - acc: 0.0833\n13/25 [==============>...............] - ETA: 20s - loss: 2.3135 - acc: 0.0769\n14/25 [===============>..............] - ETA: 18s - loss: 2.3132 - acc: 0.0714\n15/25 [=================>............] - ETA: 16s - loss: 2.3128 - acc: 0.0667\n16/25 [==================>...........] - ETA: 15s - loss: 2.3121 - acc: 0.0781\n17/25 [===================>..........] - ETA: 13s - loss: 2.3116 - acc: 0.0735\n18/25 [====================>.........] - ETA: 12s - loss: 2.3114 - acc: 0.0694\n19/25 [=====================>........] - ETA: 10s - loss: 2.3112 - acc: 0.0658\n20/25 [=======================>......] - ETA: 8s - loss: 2.3109 - acc: 0.0625\n21/25 [========================>.....] - ETA: 7s - loss: 2.3107 - acc: 0.0595\n22/25 [=========================>....] - ETA: 5s - loss: 2.3104 - acc: 0.0568\n23/25 [==========================>...] - ETA: 3s - loss: 2.3101 - acc: 0.0543\n24/25 [===========================>..] - ETA: 1s - loss: 2.3097 - acc: 0.0625Epoch 00000: saving model to C:\\Users\\Moses\\Documents\\Moses\\W7\\AI\\Custom Datasets\\IDENPROF\\idenprof-small-test\\idenprof\\models\\model_ex-000_acc-0.100000.h5\n\n25/25 [==============================] - 51s - loss: 2.3095 - acc: 0.0600 - val_loss: 2.3026 - val_acc: 0.1000\n```\n\nLet us explain the details shown above: \n1. The line **Epoch 1/100** means the network is training the first experiment of the targeted 100 \n2. The line `1/25 [>.............................] - ETA: 52s - loss: 2.3026 - acc: 0.2500` represents the number of batches that has been trained in the present experiment\n3. The line `Epoch 00000: saving model to C:\\Users\\User\\PycharmProjects\\ImageAITest\\pets\\models\\model_ex-000_acc-0.100000.h5` refers to the model saved after the present experiment. The **ex_000** represents the experiment at this stage while the **acc_0.100000** and **val_acc: 0.1000** represents the accuracy of the model on the test images after the present experiment (maximum value value of accuracy is 1.0). This result helps to know the best performed model you can use for custom image prediction. \n \n Once you are done training your custom model, you can use the \"CustomImagePrediction\" class to perform image prediction with your model. Simply follow the link below.\n[imageai/Classification/CUSTOMCLASSIFICATION.md](https://github.com/OlafenwaMoses/ImageAI/blob/master/imageai/Classification/CUSTOMCLASSIFICATION.md)\n\n\n### Training on the IdenProf data\n\nA sample from the IdenProf Dataset used to train a Model for predicting professionals.\n![](../../data-images/idenprof.jpg)\n\nBelow we provide a sample code to train on **IdenProf**, a dataset which contains images of 10 uniformed professionals. The code below will download the dataset and initiate the training:\n\n```python\nfrom io import open\nimport requests\nimport shutil\nfrom zipfile import ZipFile\nimport os\nfrom imageai.Classification.Custom import ClassificationModelTrainer\n\nexecution_path = os.getcwd()\n\nTRAIN_ZIP_ONE = os.path.join(execution_path, \"idenprof-train1.zip\")\nTRAIN_ZIP_TWO = os.path.join(execution_path, \"idenprof-train2.zip\")\nTEST_ZIP = os.path.join(execution_path, \"idenprof-test.zip\")\n\nDATASET_DIR = os.path.join(execution_path, \"idenprof\")\nDATASET_TRAIN_DIR = os.path.join(DATASET_DIR, \"train\")\nDATASET_TEST_DIR = os.path.join(DATASET_DIR, \"test\")\n\nif(os.path.exists(DATASET_DIR) == False):\n os.mkdir(DATASET_DIR)\nif(os.path.exists(DATASET_TRAIN_DIR) == False):\n os.mkdir(DATASET_TRAIN_DIR)\nif(os.path.exists(DATASET_TEST_DIR) == False):\n os.mkdir(DATASET_TEST_DIR)\n\nif(len(os.listdir(DATASET_TRAIN_DIR)) < 10):\n if(os.path.exists(TRAIN_ZIP_ONE) == False):\n print(\"Downloading idenprof-train1.zip\")\n data = requests.get(\"https://github.com/OlafenwaMoses/IdenProf/releases/download/v1.0/idenprof-train1.zip\", stream = True)\n with open(TRAIN_ZIP_ONE, \"wb\") as file:\n shutil.copyfileobj(data.raw, file)\n del data\n if (os.path.exists(TRAIN_ZIP_TWO) == False):\n print(\"Downloading idenprof-train2.zip\")\n data = requests.get(\"https://github.com/OlafenwaMoses/IdenProf/releases/download/v1.0/idenprof-train2.zip\", stream=True)\n with open(TRAIN_ZIP_TWO, \"wb\") as file:\n shutil.copyfileobj(data.raw, file)\n del data\n print(\"Extracting idenprof-train1.zip\")\n extract1 = ZipFile(TRAIN_ZIP_ONE)\n extract1.extractall(DATASET_TRAIN_DIR)\n extract1.close()\n print(\"Extracting idenprof-train2.zip\")\n extract2 = ZipFile(TRAIN_ZIP_TWO)\n extract2.extractall(DATASET_TRAIN_DIR)\n extract2.close()\n\nif(len(os.listdir(DATASET_TEST_DIR)) < 10):\n if (os.path.exists(TEST_ZIP) == False):\n print(\"Downloading idenprof-test.zip\")\n data = requests.get(\"https://github.com/OlafenwaMoses/IdenProf/releases/download/v1.0/idenprof-test.zip\", stream=True)\n with open(TEST_ZIP, \"wb\") as file:\n shutil.copyfileobj(data.raw, file)\n del data\n print(\"Extracting idenprof-test.zip\")\n extract = ZipFile(TEST_ZIP)\n extract.extractall(DATASET_TEST_DIR)\n extract.close()\n\n\nmodel_trainer = ClassificationModelTrainer()\nmodel_trainer.setModelTypeAsResNet50()\nmodel_trainer.setDataDirectory(DATASET_DIR)\nmodel_trainer.trainModel(num_objects=10, num_experiments=100, enhance_data=True, batch_size=32, show_network_summary=True)\n```\n\n### Continuous Model Training\n
\n\n**ImageAI** now allows you to continue training your custom model on your previously saved model.\nThis is useful in cases of incomplete training due compute time limits/large size of dataset or should you intend to further train your model.\nKindly note that **continuous training** is for using a previously saved model to train on the same dataset the model was trained on.\nAll you need to do is specify the `continue_from_model` parameter to the path of the previously saved model in your `trainModel()` function.\nSee an example code below.\n\n```python\nfrom imageai.Classification.Custom import ClassificationModelTrainer\nimport os\n\ntrainer = ClassificationModelTrainer()\ntrainer.setModelTypeAsDenseNet121()\ntrainer.setDataDirectory(\"idenprof\")\ntrainer.trainModel(num_objects=10, num_experiments=50, enhance_data=True, batch_size=8, show_network_summary=True, continue_from_model=\"idenprof_densenet-0.763500.h5\")\n```\n\n### Transfer Learning (Training from a pre-trained model)\n
\n\nFrom the feedbacks we have received over the past months, we discovered most custom models trained with **ImageAI** were based on datasets with few number of images as they fall short the minimum recommendation of 500 images per each class of objects, for a achieving a viable accuracy. \n\nTo ensure they can still train very accurate custom models using few number of images, **ImageAI** now allows you to train by leveraging **transfer learning** . This means you can take any pre-trained **ResNet50**, **Squeezenet**, **InceptionV3** and **DenseNet121** model trained on larger datasets and use it to kickstart your custom model training.\nAll you need to do is specify the `transfer_from_model` parameter to the path of the pre-trained model, `initial_num_objects` parameter which corresponds to the number of objects in the previous dataset the pre-trained model was trained on, all in your `trainModel()` function. See an example code below, showing how to perform transfer learning from a ResNet50 model trained on the ImageNet dataset.\n\n```python\nfrom imageai.Classification.Custom import ClassificationModelTrainer\nimport os\n\ntrainer = ClassificationModelTrainer()\ntrainer.setModelTypeAsResNet50()\ntrainer.setDataDirectory(\"idenprof\")\ntrainer.trainModel(num_objects=10, num_experiments=50, enhance_data=True, batch_size=32, show_network_summary=True,transfer_from_model=\"resnet50_imagenet_tf.2.0.h5\", initial_num_objects=1000)\n```\n\n\n### Contact Developer\n- **Moses Olafenwa**\n * _Email:_ guymodscientist@gmail.com\n * _Website:_ [https://moses.aicommons.science](https://moses.aicommons.science)\n * _Twitter:_ [@OlafenwaMoses](https://twitter.com/OlafenwaMoses)\n * _Medium:_ [@guymodscientist](https://medium.com/@guymodscientist)\n * _Facebook:_ [moses.olafenwa](https://facebook.com/moses.olafenwa)\n\n\n### Documentation\n\nWe have provided full documentation for all **ImageAI** classes and functions in 3 major languages. Find links below:\n\n* Documentation - **English Version [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**\n* Documentation - **Chinese Version [https://imageai-cn.readthedocs.io](https://imageai-cn.readthedocs.io)**\n* Documentation - **French Version [https://imageai-fr.readthedocs.io](https://imageai-fr.readthedocs.io)**\n" }, { "path": "imageai_tf_deprecated/Classification/Custom/__init__.py", "content": "import tensorflow as tf\nfrom PIL import Image\nimport time\nimport numpy as np\nimport os\nimport warnings\nfrom matplotlib.cbook import deprecated\nimport json\n\nclass ClassificationModelTrainer:\n \"\"\"\n This is the Classification Model training class, that allows you to define a deep learning network\n from the 4 available networks types supported by ImageAI which are MobileNetv2, ResNet50,\n InceptionV3 and DenseNet121.\n \"\"\"\n\n def __init__(self):\n self.__modelType = \"\"\n self.__use_pretrained_model = False\n self.__data_dir = \"\"\n self.__train_dir = \"\"\n self.__test_dir = \"\"\n self.__logs_dir = \"\"\n self.__num_epochs = 10\n self.__trained_model_dir = \"\"\n self.__model_class_dir = \"\"\n self.__initial_learning_rate = 1e-3\n self.__model_collection = []\n\n\n def setModelTypeAsSqueezeNet(self):\n raise ValueError(\"ImageAI no longer support SqueezeNet. You can use MobileNetV2 instead by downloading the MobileNetV2 model and call the function 'setModelTypeAsMobileNetV2'\")\n\n def setModelTypeAsMobileNetV2(self):\n \"\"\"\n 'setModelTypeAsMobileNetV2()' is used to set the model type to the MobileNetV2 model\n for the training instance object .\n :return:\n \"\"\"\n self.__modelType = \"mobilenetv2\"\n\n @deprecated(since=\"2.1.6\", message=\"'.setModelTypeAsResNet()' has been deprecated! Please use 'setModelTypeAsResNet50()' instead.\")\n def setModelTypeAsResNet(self):\n return self.setModelTypeAsResNet50()\n\n def setModelTypeAsResNet50(self):\n \"\"\"\n 'setModelTypeAsResNet()' is used to set the model type to the ResNet model\n for the training instance object .\n :return:\n \"\"\"\n self.__modelType = \"resnet50\"\n\n \n @deprecated(since=\"2.1.6\", message=\"'.setModelTypeAsDenseNet()' has been deprecated! Please use 'setModelTypeAsDenseNet121()' instead.\")\n def setModelTypeAsDenseNet(self):\n return self.setModelTypeAsDenseNet121()\n\n def setModelTypeAsDenseNet121(self):\n \"\"\"\n 'setModelTypeAsDenseNet()' is used to set the model type to the DenseNet model\n for the training instance object .\n :return:\n \"\"\"\n self.__modelType = \"densenet121\"\n\n def setModelTypeAsInceptionV3(self):\n \"\"\"\n 'setModelTypeAsInceptionV3()' is used to set the model type to the InceptionV3 model\n for the training instance object .\n :return:\n \"\"\"\n self.__modelType = \"inceptionv3\"\n\n def setDataDirectory(self, data_directory=\"\", train_subdirectory=\"train\", test_subdirectory=\"test\",\n models_subdirectory=\"models\", json_subdirectory=\"json\"):\n \"\"\"\n 'setDataDirectory()'\n\n - data_directory , is required to set the path to which the data/dataset to be used for\n training is kept. The directory can have any name, but it must have 'train' and 'test'\n sub-directory. In the 'train' and 'test' sub-directories, there must be sub-directories\n with each having it's name corresponds to the name/label of the object whose images are\n to be kept. The structure of the 'test' and 'train' folder must be as follows:\n\n >> train >> class1 >> class1_train_images\n >> class2 >> class2_train_images\n >> class3 >> class3_train_images\n >> class4 >> class4_train_images\n >> class5 >> class5_train_images\n\n >> test >> class1 >> class1_test_images\n >> class2 >> class2_test_images\n >> class3 >> class3_test_images\n >> class4 >> class4_test_images\n >> class5 >> class5_test_images\n\n - train_subdirectory (optional), subdirectory within 'data_directory' where the training set is. Defaults to 'train'.\n - test_subdirectory (optional), subdirectory within 'data_directory' where the testing set is. Defaults to 'test'.\n - models_subdirectory (optional), subdirectory within 'data_directory' where the output models will be saved. Defaults to 'models'.\n - json_subdirectory (optional), subdirectory within 'data_directory' where the model classes json file will be saved. Defaults to 'json'.\n\n :param data_directory:\n :param train_subdirectory:\n :param test_subdirectory:\n :param models_subdirectory:\n :param json_subdirectory:\n :return:\n \"\"\"\n\n self.__data_dir = data_directory\n\n self.__train_dir = os.path.join(self.__data_dir, train_subdirectory)\n self.__test_dir = os.path.join(self.__data_dir, test_subdirectory)\n self.__trained_model_dir = os.path.join(self.__data_dir, models_subdirectory)\n self.__model_class_dir = os.path.join(self.__data_dir, json_subdirectory)\n self.__logs_dir = os.path.join(self.__data_dir, \"logs\")\n\n def lr_schedule(self, epoch):\n\n # Learning Rate Schedule\n\n\n lr = self.__initial_learning_rate\n total_epochs = self.__num_epochs\n\n check_1 = int(total_epochs * 0.9)\n check_2 = int(total_epochs * 0.8)\n check_3 = int(total_epochs * 0.6)\n check_4 = int(total_epochs * 0.4)\n\n if epoch > check_1:\n lr *= 1e-4\n elif epoch > check_2:\n lr *= 1e-3\n elif epoch > check_3:\n lr *= 1e-2\n elif epoch > check_4:\n lr *= 1e-1\n\n\n return lr\n\n\n\n\n def trainModel(self, num_objects, num_experiments=200, enhance_data=False, batch_size = 32, initial_learning_rate=1e-3, show_network_summary=False, training_image_size = 224, continue_from_model=None, transfer_from_model=None, transfer_with_full_training=True, initial_num_objects = None, save_full_model = False):\n\n \"\"\"\n 'trainModel()' function starts the model actual training. It accepts the following values:\n - num_objects , which is the number of classes present in the dataset that is to be used for training\n - num_experiments , also known as epochs, it is the number of times the network will train on all the training dataset\n - enhance_data (optional) , this is used to modify the dataset and create more instance of the training set to enhance the training result\n - batch_size (optional) , due to memory constraints, the network trains on a batch at once, until all the training set is exhausted. The value is set to 32 by default, but can be increased or decreased depending on the meormory of the compute used for training. The batch_size is conventionally set to 16, 32, 64, 128.\n - initial_learning_rate(optional) , this value is used to adjust the weights generated in the network. You rae advised to keep this value as it is if you don't have deep understanding of this concept.\n - show_network_summary(optional) , this value is used to show the structure of the network should you desire to see it. It is set to False by default\n - training_image_size(optional) , this value is used to define the image size on which the model will be trained. The value is 224 by default and is kept at a minimum of 100.\n - continue_from_model (optional) , this is used to set the path to a model file trained on the same dataset. It is primarily for continuos training from a previously saved model.\n - transfer_from_model (optional) , this is used to set the path to a model file trained on another dataset. It is primarily used to perform tramsfer learning.\n - transfer_with_full_training (optional) , this is used to set the pre-trained model to be re-trained across all the layers or only at the top layers.\n - initial_num_objects (required if 'transfer_from_model' is set ), this is used to set the number of objects the model used for transfer learning is trained on. If 'transfer_from_model' is set, this must be set as well.\n - save_full_model ( optional ), this is used to save the trained models with their network types. Any model saved by this specification can be loaded without specifying the network type.\n\n\n :param num_objects:\n :param num_experiments:\n :param enhance_data:\n :param batch_size:\n :param initial_learning_rate:\n :param show_network_summary:\n :param training_image_size:\n :param continue_from_model:\n :param transfer_from_model:\n :param initial_num_objects:\n :param save_full_model:\n :return:\n \"\"\"\n self.__num_epochs = num_experiments\n self.__initial_learning_rate = initial_learning_rate\n lr_scheduler = tf.keras.callbacks.LearningRateScheduler(self.lr_schedule)\n\n\n if(training_image_size < 100):\n warnings.warn(\"The specified training_image_size {} is less than 100. Hence the training_image_size will default to 100.\".format(training_image_size))\n training_image_size = 100\n\n\n\n if (self.__modelType == \"mobilenetv2\"):\n if (continue_from_model != None):\n model = tf.keras.applications.MobileNetV2(input_shape=(training_image_size, training_image_size, 3), weights=continue_from_model, classes=num_objects,\n include_top=True)\n if (show_network_summary == True):\n print(\"Training using weights from a previouly model\")\n elif (transfer_from_model != None):\n base_model = tf.keras.applications.MobileNetV2(input_shape=(training_image_size, training_image_size, 3), weights= transfer_from_model,\n include_top=False, pooling=\"avg\")\n\n network = base_model.output\n network = tf.keras.layers.Dense(num_objects, activation='softmax',\n use_bias=True)(network)\n \n model = tf.keras.model.Models(inputs=base_model.input, outputs=network)\n\n if (show_network_summary == True):\n print(\"Training using weights from a pre-trained ImageNet model\")\n else:\n base_model = tf.keras.applications.MobileNetV2(input_shape=(training_image_size, training_image_size, 3), weights= None, classes=num_objects,\n include_top=False, pooling=\"avg\")\n \n network = base_model.output\n network = tf.keras.layers.Dense(num_objects, activation='softmax',\n use_bias=True)(network)\n \n model = tf.keras.models.Model(inputs=base_model.input, outputs=network)\n\n elif (self.__modelType == \"resnet50\"):\n if (continue_from_model != None):\n model = tf.keras.applications.ResNet50(input_shape=(training_image_size, training_image_size, 3), weights=continue_from_model, classes=num_objects,\n include_top=True)\n if (show_network_summary == True):\n print(\"Training using weights from a previouly model\")\n elif (transfer_from_model != None):\n base_model = tf.keras.applications.ResNet50(input_shape=(training_image_size, training_image_size, 3), weights= transfer_from_model,\n include_top=False, pooling=\"avg\")\n\n network = base_model.output\n network = tf.keras.layers.Dense(num_objects, activation='softmax',\n use_bias=True)(network)\n \n model = tf.keras.model.Models(inputs=base_model.input, outputs=network)\n\n if (show_network_summary == True):\n print(\"Training using weights from a pre-trained ImageNet model\")\n else:\n base_model = tf.keras.applications.ResNet50(input_shape=(training_image_size, training_image_size, 3), weights= None, classes=num_objects,\n include_top=False, pooling=\"avg\")\n\n network = base_model.output\n network = tf.keras.layers.Dense(num_objects, activation='softmax',\n use_bias=True)(network)\n \n model = tf.keras.models.Model(inputs=base_model.input, outputs=network)\n\n elif (self.__modelType == \"inceptionv3\"):\n\n if (continue_from_model != None):\n model = tf.keras.applications.InceptionV3(input_shape=(training_image_size, training_image_size, 3), weights=continue_from_model, classes=num_objects,\n include_top=True)\n if (show_network_summary == True):\n print(\"Training using weights from a previouly model\")\n elif (transfer_from_model != None):\n base_model = tf.keras.applications.InceptionV3(input_shape=(training_image_size, training_image_size, 3), weights= transfer_from_model,\n include_top=False, pooling=\"avg\")\n\n network = base_model.output\n network = tf.keras.layers.Dense(num_objects, activation='softmax',\n use_bias=True)(network)\n \n model = tf.keras.model.Models(inputs=base_model.input, outputs=network)\n\n if (show_network_summary == True):\n print(\"Training using weights from a pre-trained ImageNet model\")\n else:\n base_model = tf.keras.applications.InceptionV3(input_shape=(training_image_size, training_image_size, 3), weights= None, classes=num_objects,\n include_top=False, pooling=\"avg\")\n\n network = base_model.output\n network = tf.keras.layers.Dense(num_objects, activation='softmax',\n use_bias=True)(network)\n \n model = tf.keras.models.Model(inputs=base_model.input, outputs=network)\n\n base_model = tf.keras.applications.InceptionV3(input_shape=(training_image_size, training_image_size, 3), weights= None, classes=num_objects,\n include_top=False, pooling=\"avg\")\n\n elif (self.__modelType == \"densenet121\"):\n if (continue_from_model != None):\n model = tf.keras.applications.DenseNet121(input_shape=(training_image_size, training_image_size, 3), weights=continue_from_model, classes=num_objects,\n include_top=True)\n if (show_network_summary == True):\n print(\"Training using weights from a previouly model\")\n elif (transfer_from_model != None):\n base_model = tf.keras.applications.DenseNet121(input_shape=(training_image_size, training_image_size, 3), weights= transfer_from_model,\n include_top=False, pooling=\"avg\")\n\n network = base_model.output\n network = tf.keras.layers.Dense(num_objects, activation='softmax',\n use_bias=True)(network)\n \n model = tf.keras.model.Models(inputs=base_model.input, outputs=network)\n\n if (show_network_summary == True):\n print(\"Training using weights from a pre-trained ImageNet model\")\n else:\n base_model = tf.keras.applications.DenseNet121(input_shape=(training_image_size, training_image_size, 3), weights= None, classes=num_objects,\n include_top=False, pooling=\"avg\")\n\n network = base_model.output\n network = tf.keras.layers.Dense(num_objects, activation='softmax',\n use_bias=True)(network)\n \n model = tf.keras.models.Model(inputs=base_model.input, outputs=network)\n\n base_model = tf.keras.applications.DenseNet121(input_shape=(training_image_size, training_image_size, 3), weights= None, classes=num_objects,\n include_top=False, pooling=\"avg\")\n\n\n optimizer = tf.keras.optimizers.Adam(lr=self.__initial_learning_rate, decay=1e-4)\n model.compile(loss=\"categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n if (show_network_summary == True):\n model.summary()\n\n model_name = 'model_ex-{epoch:03d}_acc-{accuracy:03f}.h5'\n\n log_name = '{}_lr-{}_{}'.format(self.__modelType, initial_learning_rate, time.strftime(\"%Y-%m-%d-%H-%M-%S\"))\n\n if not os.path.isdir(self.__trained_model_dir):\n os.makedirs(self.__trained_model_dir)\n\n if not os.path.isdir(self.__model_class_dir):\n os.makedirs(self.__model_class_dir)\n\n if not os.path.isdir(self.__logs_dir):\n os.makedirs(self.__logs_dir)\n\n model_path = os.path.join(self.__trained_model_dir, model_name)\n\n\n logs_path = os.path.join(self.__logs_dir, log_name)\n if not os.path.isdir(logs_path):\n os.makedirs(logs_path)\n\n save_weights_condition = True\n\n if(save_full_model == True ):\n save_weights_condition = False\n\n\n checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=model_path,\n monitor='accuracy',\n verbose=1,\n save_weights_only=save_weights_condition,\n save_best_only=True,\n period=1)\n\n\n tensorboard = tf.keras.callbacks.TensorBoard(log_dir=logs_path, \n histogram_freq=0, \n write_graph=False, \n write_images=False)\n \n\n if (enhance_data == True):\n print(\"Using Enhanced Data Generation\")\n\n height_shift = 0\n width_shift = 0\n if (enhance_data == True):\n height_shift = 0.1\n width_shift = 0.1\n\n train_datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n rescale=1. / 255,\n horizontal_flip=enhance_data, height_shift_range=height_shift, width_shift_range=width_shift)\n\n test_datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n rescale=1. / 255)\n\n train_generator = train_datagen.flow_from_directory(self.__train_dir, target_size=(training_image_size, training_image_size),\n batch_size=batch_size,\n class_mode=\"categorical\")\n test_generator = test_datagen.flow_from_directory(self.__test_dir, target_size=(training_image_size, training_image_size),\n batch_size=batch_size,\n class_mode=\"categorical\")\n\n class_indices = train_generator.class_indices\n class_json = {}\n for eachClass in class_indices:\n class_json[str(class_indices[eachClass])] = eachClass\n\n with open(os.path.join(self.__model_class_dir, \"model_class.json\"), \"w+\") as json_file:\n json.dump(class_json, json_file, indent=4, separators=(\",\", \" : \"),\n ensure_ascii=True)\n json_file.close()\n print(\"JSON Mapping for the model classes saved to \", os.path.join(self.__model_class_dir, \"model_class.json\"))\n\n num_train = len(train_generator.filenames)\n num_test = len(test_generator.filenames)\n print(\"Number of experiments (Epochs) : \", self.__num_epochs)\n\n \n model.fit_generator(train_generator, steps_per_epoch=int(num_train / batch_size), epochs=self.__num_epochs,\n validation_data=test_generator,\n validation_steps=int(num_test / batch_size), callbacks=[checkpoint, lr_scheduler])\n\n\n\n\n\nclass CustomImageClassification:\n \"\"\"\n This is the image classification class for custom models trained with the 'ClassificationModelTrainer' class. It provides support for 4 different models which are:\n ResNet50, MobileNetV2, DenseNet121 and Inception V3. After instantiating this class, you can set it's properties and\n make image classification using it's pre-defined functions.\n\n The following functions are required to be called before a classification can be made\n * setModelPath() , path to your custom model\n * setJsonPath , , path to your custom model's corresponding JSON file\n * At least of of the following and it must correspond to the model set in the setModelPath()\n [setModelTypeAsMobileNetV2(), setModelTypeAsResNet50(), setModelTypeAsDenseNet121, setModelTypeAsInceptionV3]\n * loadModel() [This must be called once only before making a classification]\n\n Once the above functions have been called, you can call the classifyImage() function of the classification instance\n object at anytime to predict an image.\n \"\"\"\n def __init__(self):\n self.__modelType = \"\"\n self.modelPath = \"\"\n self.jsonPath = \"\"\n self.numObjects = 10\n self.__model_classes = dict()\n self.__modelLoaded = False\n self.__model_collection = []\n self.__input_image_size = 224\n \n def setModelPath(self, model_path):\n \"\"\"\n 'setModelPath()' function is required and is used to set the file path to the model adopted from the list of the\n available 4 model types. The model path must correspond to the model type set for the classification instance object.\n\n :param model_path:\n :return:\n \"\"\"\n self.modelPath = model_path\n\n def setJsonPath(self, model_json):\n \"\"\"\n 'setJsonPath()'\n\n :param model_path:\n :return:\n \"\"\"\n self.jsonPath = model_json\n\n def setModelTypeAsMobileNetV2(self):\n \"\"\"\n 'setModelTypeAsMobileNetV2()' is used to set the model type to the MobileNetV2 model\n for the classification instance object .\n :return:\n \"\"\"\n self.__modelType = \"mobilenetv2\"\n\n def setModelTypeAsResNet50(self):\n \"\"\"\n 'setModelTypeAsResNet50()' is used to set the model type to the ResNet50 model\n for the classification instance object .\n :return:\n \"\"\"\n self.__modelType = \"resnet50\"\n\n def setModelTypeAsDenseNet121(self):\n \"\"\"\n 'setModelTypeAsDenseNet121()' is used to set the model type to the DenseNet121 model\n for the classification instance object .\n :return:\n \"\"\"\n self.__modelType = \"densenet121\"\n\n def setModelTypeAsInceptionV3(self):\n \"\"\"\n 'setModelTypeAsInceptionV3()' is used to set the model type to the InceptionV3 model\n for the classification instance object .\n :return:\n \"\"\"\n self.__modelType = \"inceptionv3\"\n\n def loadModel(self, classification_speed=\"normal\", num_objects=10):\n \"\"\"\n 'loadModel()' function is used to load the model structure into the program from the file path defined\n in the setModelPath() function. This function receives an optional value which is \"classification_speed\".\n The value is used to reduce the time it takes to classify an image, down to about 50% of the normal time,\n with just slight changes or drop in classification accuracy, depending on the nature of the image.\n * classification_speed (optional); Acceptable values are \"normal\", \"fast\", \"faster\" and \"fastest\"\n\n :param classification_speed :\n :return:\n \"\"\"\n\n self.__model_classes = json.load(open(self.jsonPath))\n\n if(classification_speed==\"normal\"):\n self.__input_image_size = 224\n elif(classification_speed==\"fast\"):\n self.__input_image_size = 160\n elif(classification_speed==\"faster\"):\n self.__input_image_size = 120\n elif (classification_speed == \"fastest\"):\n self.__input_image_size = 100\n\n if (self.__modelLoaded == False):\n\n image_input = tf.keras.layers.Input(shape=(self.__input_image_size, self.__input_image_size, 3))\n\n if(self.__modelType == \"\" ):\n raise ValueError(\"You must set a valid model type before loading the model.\")\n\n elif(self.__modelType == \"mobilenetv2\"):\n model = tf.keras.applications.MobileNetV2(input_shape=(self.__input_image_size, self.__input_image_size, 3), weights=self.modelPath, classes = num_objects )\n self.__model_collection.append(model)\n self.__modelLoaded = True\n try:\n None\n except:\n raise ValueError(\"An error occured. Ensure your model file is a MobileNetV2 Model and is located in the path {}\".format(self.modelPath))\n\n elif(self.__modelType == \"resnet50\"):\n try:\n model = tf.keras.applications.ResNet50(input_shape=(self.__input_image_size, self.__input_image_size, 3), weights=None, classes = num_objects )\n model.load_weights(self.modelPath)\n self.__model_collection.append(model)\n self.__modelLoaded = True\n except:\n raise ValueError(\"An error occured. Ensure your model file is a ResNet50 Model and is located in the path {}\".format(self.modelPath))\n\n elif (self.__modelType == \"densenet121\"):\n try:\n model = tf.keras.applications.DenseNet121(input_shape=(self.__input_image_size, self.__input_image_size, 3), weights=self.modelPath, classes = num_objects)\n self.__model_collection.append(model)\n self.__modelLoaded = True\n except:\n raise ValueError(\"An error occured. Ensure your model file is a DenseNet121 Model and is located in the path {}\".format(self.modelPath))\n\n elif (self.__modelType == \"inceptionv3\"):\n try:\n model = tf.keras.applications.InceptionV3(input_shape=(self.__input_image_size, self.__input_image_size, 3), weights=self.modelPath, classes = num_objects )\n self.__model_collection.append(model)\n self.__modelLoaded = True\n except:\n raise ValueError(\"An error occured. Ensure your model file is in {}\".format(self.modelPath))\n def loadFullModel(self, classification_speed=\"normal\", num_objects=10):\n \"\"\"\n 'loadFullModel()' function is used to load the model structure into the program from the file path defined\n in the setModelPath() function. As opposed to the 'loadModel()' function, you don't need to specify the model type. This means you can load any Keras model trained with or without ImageAI and perform image prediction.\n - prediction_speed (optional), Acceptable values are \"normal\", \"fast\", \"faster\" and \"fastest\"\n - num_objects (required), the number of objects the model is trained to recognize\n\n :param prediction_speed:\n :param num_objects:\n :return:\n \"\"\"\n\n self.numObjects = num_objects\n self.__model_classes = json.load(open(self.jsonPath))\n\n if (classification_speed == \"normal\"):\n self.__input_image_size = 224\n elif (classification_speed == \"fast\"):\n self.__input_image_size = 160\n elif (classification_speed == \"faster\"):\n self.__input_image_size = 120\n elif (classification_speed == \"fastest\"):\n self.__input_image_size = 100\n\n if (self.__modelLoaded == False):\n \n model = tf.keras.models.load_model(filepath=self.modelPath)\n self.__model_collection.append(model)\n self.__modelLoaded = True\n self.__modelType = \"full\"\n\n def getModels(self):\n \"\"\"\n 'getModels()' provides access to the internal model collection. Helpful if models are used down the line with tools like lime.\n :return:\n \"\"\"\n return self.__model_collection\n\n\n def classifyImage(self, image_input, result_count=5, input_type=\"file\"):\n \"\"\"\n 'classifyImage()' function is used to classify a given image by receiving the following arguments:\n * input_type (optional) , the type of input to be parsed. Acceptable values are \"file\", \"array\" and \"stream\"\n * image_input , file path/numpy array/image file stream of the image.\n * result_count (optional) , the number of classifications to be sent which must be whole numbers between\n 1 and 1000. The default is 5.\n\n This function returns 2 arrays namely 'classification_results' and 'classification_probabilities'. The 'classification_results'\n contains possible objects classes arranged in descending of their percentage probabilities. The 'classification_probabilities'\n contains the percentage probability of each object class. The position of each object class in the 'classification_results'\n array corresponds with the positions of the percentage probability in the 'classification_probabilities' array.\n\n\n :param input_type:\n :param image_input:\n :param result_count:\n :return classification_results, classification_probabilities:\n \"\"\"\n classification_results = []\n classification_probabilities = []\n if (self.__modelLoaded == False):\n raise ValueError(\"You must call the loadModel() function before making classification.\")\n\n else:\n if (input_type == \"file\"):\n try:\n image_to_predict = tf.keras.preprocessing.image.load_img(image_input, target_size=(self.__input_image_size, self.__input_image_size))\n image_to_predict = tf.keras.preprocessing.image.img_to_array(image_to_predict, data_format=\"channels_last\")\n image_to_predict = np.expand_dims(image_to_predict, axis=0)\n except:\n raise ValueError(\"You have set a path to an invalid image file.\")\n elif (input_type == \"array\"):\n try:\n image_input = Image.fromarray(np.uint8(image_input))\n image_input = image_input.resize((self.__input_image_size, self.__input_image_size))\n image_input = np.expand_dims(image_input, axis=0)\n image_to_predict = image_input.copy()\n image_to_predict = np.asarray(image_to_predict, dtype=np.float64)\n except:\n raise ValueError(\"You have parsed in a wrong numpy array for the image\")\n elif (input_type == \"stream\"):\n try:\n image_input = Image.open(image_input)\n image_input = image_input.resize((self.__input_image_size, self.__input_image_size))\n image_input = np.expand_dims(image_input, axis=0)\n image_to_predict = image_input.copy()\n image_to_predict = np.asarray(image_to_predict, dtype=np.float64)\n \n except:\n raise ValueError(\"You have parsed in a wrong stream for the image\")\n\n if (self.__modelType == \"mobilenetv2\"):\n image_to_predict = tf.keras.applications.mobilenet_v2.preprocess_input(image_to_predict)\n elif (self.__modelType == \"full\"):\n image_to_predict = tf.keras.applications.mobilenet_v2.preprocess_input(image_to_predict)\n elif (self.__modelType == \"inceptionv3\"):\n image_to_predict = tf.keras.applications.inception_v3.preprocess_input(image_to_predict)\n elif (self.__modelType == \"densenet121\"):\n image_to_predict = tf.keras.applications.densenet.preprocess_input(image_to_predict)\n try:\n model = self.__model_collection[0]\n prediction = model.predict(image_to_predict, steps=1)\n\n predictiondata = []\n for pred in prediction:\n top_indices = pred.argsort()[-result_count:][::-1]\n for i in top_indices:\n each_result = []\n each_result.append(self.__model_classes[str(i)])\n each_result.append(pred[i])\n predictiondata.append(each_result)\n\n for result in predictiondata:\n classification_results.append(str(result[0]))\n classification_probabilities.append(result[1] * 100)\n \n except:\n raise ValueError(\"Error. Ensure your input image is valid\")\n\n return classification_results, classification_probabilities\n \n\n @deprecated(since=\"2.1.6\", message=\"'.predictImage()' has been deprecated! Please use 'classifyImage()' instead.\")\n def predictImage(self, image_input, result_count=5, input_type=\"file\"):\n\n return self.classifyImage(image_input, result_count, input_type)" }, { "path": "imageai_tf_deprecated/Classification/README.md", "content": "# ImageAI : Image Prediction\nA **DeepQuest AI** project [https://deepquestai.com](https://deepquestai.com)\n\n---\n\n### TABLE OF CONTENTS\n- :white_square_button: First Prediction\n- :white_square_button: Prediction Speed\n- :white_square_button: Image Input Types\n- :white_square_button: Prediction in MultiThreading\n- :white_square_button: Documentation\n\nImageAI provides 4 different algorithms and model types to perform image prediction.\nTo perform image prediction on any picture, take the following simple steps. The 4 algorithms provided for\n image prediction include **MobileNetV2**, **ResNet50**, **InceptionV3** and **DenseNet121**. Each of these\n algorithms have individual model files which you must use depending on the choice of your algorithm. To download the\n model file for your choice of algorithm, click on any of the links below:\n \n- **[MobileNetV2](https://github.com/OlafenwaMoses/ImageAI/releases/download/essentials-v5/mobilenet_v2.h5)** _(Size = 4.82 mb, fastest prediction time and moderate accuracy)_\n- **[ResNet50](https://github.com/OlafenwaMoses/ImageAI/releases/download/essentials-v5/resnet50_imagenet_tf.2.0.h5)** by Microsoft Research _(Size = 98 mb, fast prediction time and high accuracy)_\n - **[InceptionV3](https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/inception_v3_weights_tf_dim_ordering_tf_kernels.h5)** by Google Brain team _(Size = 91.6 mb, slow prediction time and higher accuracy)_\n - **[DenseNet121](https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/DenseNet-BC-121-32.h5)** by Facebook AI Research _(Size = 31.6 mb, slower prediction time and highest accuracy)_\n\n Great! Once you have downloaded this model file, start a new python project, and then copy the model file to your project\n folder where your python files (.py files) will be . Download the image below, or take any image on your computer\n and copy it to your python project's folder. Then create a python file and give it a name; an example is `FirstPrediction.py`.\n Then write the code below into the python file:\n \n### FirstPrediction.py\n
\n\n```python\nfrom imageai.Classification import ImageClassification\nimport os\n\nexecution_path = os.getcwd()\n\nprediction = ImageClassification()\nprediction.setModelTypeAsResNet50()\nprediction.setModelPath(os.path.join(execution_path, \"resnet50_imagenet_tf.2.0.h5\"))\nprediction.loadModel()\n\npredictions, probabilities = prediction.classifyImage(os.path.join(execution_path, \"1.jpg\"), result_count=5 )\nfor eachPrediction, eachProbability in zip(predictions, probabilities):\n print(eachPrediction , \" : \" , eachProbability)\n```\n\nSample Result:\n![](../../data-images/1.jpg)\n\n```\nconvertible : 52.459555864334106\nsports_car : 37.61284649372101\npickup : 3.1751200556755066\ncar_wheel : 1.817505806684494\nminivan : 1.7487050965428352\n```\n\nThe code above works as follows:\n```python\nfrom imageai.Classification import ImageClassification\nimport os\n```\nThe code above imports the `ImageAI` library and the python `os` class.\n```python\nexecution_path = os.getcwd()\n```\nThe above line obtains the path to the folder that contains your python file (in this example, your FirstPrediction.py).\n\n```python\nprediction = ImageClassification()\nprediction.setModelTypeAsResNet50()\nprediction.setModelPath(os.path.join(execution_path, \"resnet50_imagenet_tf.2.0.h5\"))\n```\nIn the lines above, we created and instance of the `ImagePrediction()` class in the first line, then we set the model type of the prediction object to ResNet by caling the `.setModelTypeAsResNet50()` in the second line and then we set the model path of the prediction object to the path of the model file (`resnet50_imagenet_tf.2.0.h5`) we copied to the python file folder in the third line.\n\n```python\npredictions, probabilities = prediction.classifyImage(os.path.join(execution_path, \"1.jpg\"), result_count=5 )\n```\n\nIn the above line, we defined 2 variables to be equal to the function called to predict an image, which is the `.classifyImage()` function, into which we parsed the path to our image and also state the number of prediction results we want to have (values from 1 to 1000) parsing `result_count=5`. The `.classifyImage()` function will return 2 array objects with the first (**predictions**) being an array of predictions and the second (**percentage_probabilities**) being an array of the corresponding percentage probability for each prediction.\n\n```python\nfor eachPrediction, eachProbability in zip(predictions, probabilities):\n print(eachPrediction, \" : \" , eachProbability)\n```\nThe above line obtains each object in the **predictions** array, and also obtains the corresponding percentage probability from the **percentage_probabilities**, and finally prints the result of both to console.\n\n\n### Prediction Speed\n
\n\n**ImageAI** now provides prediction speeds for all image prediction tasks. The prediction speeds allow you to reduce the time of prediction at a rate between 20% - 60%, and yet having just slight changes but accurate prediction results. The available prediction speeds are **\"normal\"**(default), **\"fast\"**, **\"faster\"** and **\"fastest\"**.\nAll you need to do is to state the speed mode you desire when loading the model as seen below.\n\n```python\nprediction.loadModel(prediction_speed=\"fast\")\n```\n\nTo observe the differences in the prediction speeds, look below for each speed applied to multiple prediction with time taken to predict and predictions given. The results below are obtained from predictions performed on a Windows 8 laptop with Intel Celeron N2820 CPU, with processor speed of 2.13GHz\n\n**Prediction Speed = \"normal\" , Prediction Time = 5.9 seconds**\n```\nconvertible : 52.459555864334106\nsports_car : 37.61284649372101\npickup : 3.1751200556755066\ncar_wheel : 1.817505806684494\nminivan : 1.7487050965428352\n-----------------------\ntoilet_tissue : 13.99008333683014\njeep : 6.842949986457825\ncar_wheel : 6.71963095664978\nseat_belt : 6.704962253570557\nminivan : 5.861184373497963\n-----------------------\nbustard : 52.03368067741394\nvulture : 20.936034619808197\ncrane : 10.620515048503876\nkite : 10.20539253950119\nwhite_stork : 1.6472270712256432\n-----------------------\n```\n\n**Prediction Speed = \"fast\" , Prediction Time = 3.4 seconds**\n```\nsports_car : 55.5136501789093\npickup : 19.860029220581055\nconvertible : 17.88402795791626\ntow_truck : 2.357563190162182\ncar_wheel : 1.8646160140633583\n-----------------------\ndrum : 12.241223454475403\ntoilet_tissue : 10.96322312951088\ncar_wheel : 10.776633024215698\ndial_telephone : 9.840480983257294\ntoilet_seat : 8.989936858415604\n-----------------------\nvulture : 52.81011462211609\nbustard : 45.628002285957336\nkite : 0.8065823465585709\ngoose : 0.3629807382822037\ncrane : 0.21266008261591196\n-----------------------\n```\n\n**Prediction Speed = \"faster\" , Prediction Time = 2.7 seconds**\n```\nsports_car : 79.90474104881287\ntow_truck : 9.751049429178238\nconvertible : 7.056044787168503\nracer : 1.8735893070697784\ncar_wheel : 0.7379394955933094\n-----------------------\noil_filter : 73.52778315544128\njeep : 11.926891654729843\nreflex_camera : 7.9965077340602875\nPolaroid_camera : 0.9798810817301273\nbarbell : 0.8661789819598198\n-----------------------\nvulture : 93.00530552864075\nbustard : 6.636220961809158\nkite : 0.15161558985710144\nbald_eagle : 0.10513027664273977\ncrane : 0.05982434959150851\n-----------------------\n```\n\n**Prediction Speed = \"fastest\" , Prediction Time = 2.2 seconds**\n```\ntow_truck : 62.5033438205719\nsports_car : 31.26143217086792\nracer : 2.2139860317111015\nfire_engine : 1.7813067883253098\nambulance : 0.8790366351604462\n-----------------------\nreflex_camera : 94.00787949562073\nracer : 2.345871739089489\njeep : 1.6016140580177307\noil_filter : 1.4121259562671185\nlens_cap : 0.1283118617720902\n-----------------------\nkite : 98.5377550125122\nvulture : 0.7469987496733665\nbustard : 0.36855682265013456\nbald_eagle : 0.2437378279864788\ngreat_grey_owl : 0.0699841941241175\n-----------------------\n```\n\n**PLEASE NOTE:** When adjusting speed modes, it is best to use models that have higher accuracies like the DenseNet or InceptionV3 models, or use it in case scenarios where the images predicted are iconic.\n\n\n### Image Input Types\n
\n\nPrevious version of **ImageAI** supported only file inputs and accepts file paths to an image for image prediction.\nNow, **ImageAI** supports 3 input types which are **file path to image file**(default), **numpy array of image** and **image file stream**.\nThis means you can now perform image prediction in production applications such as on a web server and system\n that returns file in any of the above stated formats.\n\nTo perform image prediction with numpy array or file stream input, you just need to state the input type\nin the `.classifyImage()` function. See example below.\n\n```python\npredictions, probabilities = prediction.classifyImage(image_array, result_count=5 , input_type=\"array\" ) # For numpy array input type\npredictions, probabilities = prediction.classifyImage(image_stream, result_count=5 , input_type=\"stream\" ) # For file stream input type\n```\n\n### Prediction in MultiThreading\n
\n\nWhen developing programs that run heavy task on the deafult thread like User Interfaces (UI),\n you should consider running your predictions in a new thread. When running image prediction using ImageAI in\n a new thread, you must take note the following:\n- You can create your prediction object, set its model type, set model path and json path\noutside the new thread.\n- The `.loadModel()` must be in the new thread and image prediction (`classifyImage()`) must take place in th new thread.\n\nTake a look of a sample code below on image prediction using multithreading:\n```python\nfrom imageai.Prediction import ImageClassification\nimport os\nimport threading\n\nexecution_path = os.getcwd()\n\nprediction = ImageClassification()\nprediction.setModelTypeAsResNet()\nprediction.setModelPath( os.path.join(execution_path, \"resnet50_imagenet_tf.2.0.h5\"))\n\npicturesfolder = os.environ[\"USERPROFILE\"] + \"\\\\Pictures\\\\\"\nallfiles = os.listdir(picturesfolder)\n\nclass PredictionThread(threading.Thread):\n def __init__(self):\n threading.Thread.__init__(self)\n def run(self):\n prediction.loadModel()\n for eachPicture in allfiles:\n if eachPicture.endswith(\".png\") or eachPicture.endswith(\".jpg\"):\n predictions, percentage_probabilities = prediction.predictImage(picturesfolder + eachPicture, result_count=1)\n for prediction, percentage_probability in zip(predictions, probabilities):\n print(prediction , \" : \" , percentage_probability)\n\npredictionThread = PredictionThread ()\npredictionThread.start()\n\n```\n\n\n### Documentation\n\nWe have provided full documentation for all **ImageAI** classes and functions in 3 major languages. Find links below:**\n\n* Documentation - **English Version [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**\n* Documentation - **Chinese Version [https://imageai-cn.readthedocs.io](https://imageai-cn.readthedocs.io)**\n* Documentation - **French Version [https://imageai-fr.readthedocs.io](https://imageai-fr.readthedocs.io)**\n\n" }, { "path": "imageai_tf_deprecated/Classification/__init__.py", "content": "import tensorflow as tf\nfrom PIL import Image\nimport numpy as np\nfrom matplotlib.cbook import deprecated\n\n\nclass ImageClassification:\n \"\"\"\n This is the image classification class in the ImageAI library. It provides support for 4 different models which are:\n ResNet, MobileNetV2, DenseNet and Inception V3. After instantiating this class, you can set it's properties and\n make image classification using it's pre-defined functions.\n\n The following functions are required to be called before a classification can be made\n * setModelPath()\n * At least of of the following and it must correspond to the model set in the setModelPath()\n [setModelTypeAsMobileNetv2(), setModelTypeAsResNet(), setModelTypeAsDenseNet, setModelTypeAsInceptionV3]\n * loadModel() [This must be called once only before making a classification]\n\n Once the above functions have been called, you can call the classifyImage() function of the classification instance\n object at anytime to classify an image.\n \"\"\"\n def __init__(self):\n self.__modelType = \"\"\n self.modelPath = \"\"\n self.__modelLoaded = False\n self.__model_collection = []\n self.__input_image_size = 224\n \n def setModelPath(self, model_path):\n \"\"\"\n 'setModelPath()' function is required and is used to set the file path to the model adopted from the list of the\n available 4 model types. The model path must correspond to the model type set for the classification instance object.\n\n :param model_path:\n :return:\n \"\"\"\n self.modelPath = model_path\n\n def setModelTypeAsSqueezeNet(self):\n raise ValueError(\"ImageAI no longer support SqueezeNet. You can use MobileNetV2 instead by downloading the MobileNetV2 model and call the function 'setModelTypeAsMobileNetV2'\")\n\n def setModelTypeAsMobileNetV2(self):\n \"\"\"\n 'setModelTypeAsMobileNetV2()' is used to set the model type to the MobileNetV2 model\n for the classification instance object .\n :return:\n \"\"\"\n self.__modelType = \"mobilenetv2\"\n\n @deprecated(since=\"2.1.6\", message=\"'.setModelTypeAsResNet()' has been deprecated! Please use 'setModelTypeAsResNet50()' instead.\")\n def setModelTypeAsResNet(self):\n return self.setModelTypeAsResNet50()\n\n def setModelTypeAsResNet50(self):\n \"\"\"\n 'setModelTypeAsResNet50()' is used to set the model type to the ResNet50 model\n for the classification instance object .\n :return:\n \"\"\"\n self.__modelType = \"resnet50\"\n\n @deprecated(since=\"2.1.6\", message=\"'.setModelTypeAsDenseNet()' has been deprecated! Please use 'setModelTypeAsDenseNet121()' instead.\")\n def setModelTypeAsDenseNet(self):\n return self.setModelTypeAsDenseNet121()\n\n def setModelTypeAsDenseNet121(self):\n \"\"\"\n 'setModelTypeAsDenseNet121()' is used to set the model type to the DenseNet121 model\n for the classification instance object .\n :return:\n \"\"\"\n self.__modelType = \"densenet121\"\n\n def setModelTypeAsInceptionV3(self):\n \"\"\"\n 'setModelTypeAsInceptionV3()' is used to set the model type to the InceptionV3 model\n for the classification instance object .\n :return:\n \"\"\"\n self.__modelType = \"inceptionv3\"\n\n def loadModel(self, classification_speed=\"normal\"):\n \"\"\"\n 'loadModel()' function is used to load the model structure into the program from the file path defined\n in the setModelPath() function. This function receives an optional value which is \"classification_speed\".\n The value is used to reduce the time it takes to classify an image, down to about 50% of the normal time,\n with just slight changes or drop in classification accuracy, depending on the nature of the image.\n * classification_speed (optional); Acceptable values are \"normal\", \"fast\", \"faster\" and \"fastest\"\n\n :param classification_speed :\n :return:\n \"\"\"\n\n if(classification_speed==\"normal\"):\n self.__input_image_size = 224\n elif(classification_speed==\"fast\"):\n self.__input_image_size = 160\n elif(classification_speed==\"faster\"):\n self.__input_image_size = 120\n elif (classification_speed == \"fastest\"):\n self.__input_image_size = 100\n\n if (self.__modelLoaded == False):\n\n if(self.__modelType == \"\" ):\n raise ValueError(\"You must set a valid model type before loading the model.\")\n\n elif(self.__modelType == \"mobilenetv2\"):\n model = tf.keras.applications.MobileNetV2(input_shape=(self.__input_image_size, self.__input_image_size, 3), weights=None, classes = 1000 )\n model.load_weights(self.modelPath)\n self.__model_collection.append(model)\n self.__modelLoaded = True\n try:\n None\n except:\n raise ValueError(\"An error occured. Ensure your model file is a MobileNetV2 Model and is located in the path {}\".format(self.modelPath))\n\n elif(self.__modelType == \"resnet50\"):\n try:\n model = tf.keras.applications.ResNet50(input_shape=(self.__input_image_size, self.__input_image_size, 3), weights=None, classes = 1000 )\n model.load_weights(self.modelPath)\n self.__model_collection.append(model)\n self.__modelLoaded = True\n except Exception as e:\n raise ValueError(\"An error occured. Ensure your model file is a ResNet50 Model and is located in the path {}\".format(self.modelPath))\n\n elif (self.__modelType == \"densenet121\"):\n try:\n model = tf.keras.applications.DenseNet121(input_shape=(self.__input_image_size, self.__input_image_size, 3), weights=None, classes = 1000 )\n model.load_weights(self.modelPath)\n self.__model_collection.append(model)\n self.__modelLoaded = True\n except:\n raise ValueError(\"An error occured. Ensure your model file is a DenseNet121 Model and is located in the path {}\".format(self.modelPath))\n\n elif (self.__modelType == \"inceptionv3\"):\n try:\n model = tf.keras.applications.InceptionV3(input_shape=(self.__input_image_size, self.__input_image_size, 3), weights=None, classes = 1000 )\n model.load_weights(self.modelPath)\n self.__model_collection.append(model)\n self.__modelLoaded = True\n except:\n raise ValueError(\"An error occured. Ensure your model file is in {}\".format(self.modelPath))\n\n\n def classifyImage(self, image_input, result_count=5, input_type=\"file\"):\n \"\"\"\n 'classifyImage()' function is used to classify a given image by receiving the following arguments:\n * input_type (optional) , the type of input to be parsed. Acceptable values are \"file\", \"array\" and \"stream\"\n * image_input , file path/numpy array/image file stream of the image.\n * result_count (optional) , the number of classifications to be sent which must be whole numbers between\n 1 and 1000. The default is 5.\n\n This function returns 2 arrays namely 'classification_results' and 'classification_probabilities'. The 'classification_results'\n contains possible objects classes arranged in descending of their percentage probabilities. The 'classification_probabilities'\n contains the percentage probability of each object class. The position of each object class in the 'classification_results'\n array corresponds with the positions of the percentage probability in the 'classification_probabilities' array.\n\n\n :param input_type:\n :param image_input:\n :param result_count:\n :return classification_results, classification_probabilities:\n \"\"\"\n classification_results = []\n classification_probabilities = []\n if (self.__modelLoaded == False):\n raise ValueError(\"You must call the loadModel() function before making classification.\")\n\n else:\n if (input_type == \"file\"):\n try:\n image_to_predict = tf.keras.preprocessing.image.load_img(image_input, target_size=(self.__input_image_size, self.__input_image_size))\n image_to_predict = tf.keras.preprocessing.image.img_to_array(image_to_predict, data_format=\"channels_last\")\n image_to_predict = np.expand_dims(image_to_predict, axis=0)\n except:\n raise ValueError(\"You have set a path to an invalid image file.\")\n elif (input_type == \"array\"):\n try:\n image_input = Image.fromarray(np.uint8(image_input))\n image_input = image_input.resize((self.__input_image_size, self.__input_image_size))\n image_input = np.expand_dims(image_input, axis=0)\n image_to_predict = image_input.copy()\n image_to_predict = np.asarray(image_to_predict, dtype=np.float64)\n except:\n raise ValueError(\"You have parsed in a wrong numpy array for the image\")\n elif (input_type == \"stream\"):\n try:\n image_input = Image.open(image_input)\n image_input = image_input.resize((self.__input_image_size, self.__input_image_size))\n image_input = np.expand_dims(image_input, axis=0)\n image_to_predict = image_input.copy()\n image_to_predict = np.asarray(image_to_predict, dtype=np.float64)\n \n except:\n raise ValueError(\"You have parsed in a wrong stream for the image\")\n \n if (self.__modelType == \"mobilenetv2\"):\n image_to_predict = tf.keras.applications.mobilenet_v2.preprocess_input(image_to_predict)\n elif (self.__modelType == \"densenet121\"):\n image_to_predict = tf.keras.applications.densenet.preprocess_input(image_to_predict)\n elif (self.__modelType == \"inceptionv3\"):\n image_to_predict = tf.keras.applications.inception_v3.preprocess_input(image_to_predict)\n\n try:\n model = self.__model_collection[0]\n prediction = model.predict(image_to_predict, steps=1)\n\n if (self.__modelType == \"mobilenetv2\"):\n predictiondata = tf.keras.applications.mobilenet_v2.decode_predictions(prediction, top=int(result_count))\n elif (self.__modelType == \"resnet50\"):\n predictiondata = tf.keras.applications.resnet50.decode_predictions(prediction, top=int(result_count))\n elif (self.__modelType == \"inceptionv3\"):\n predictiondata = tf.keras.applications.inception_v3.decode_predictions(prediction, top=int(result_count))\n elif (self.__modelType == \"densenet121\"):\n predictiondata = tf.keras.applications.densenet.decode_predictions(prediction, top=int(result_count))\n\n \n\n for results in predictiondata:\n for result in results:\n classification_results.append(str(result[1]))\n classification_probabilities.append(result[2] * 100)\n except:\n raise ValueError(\"An error occured! Try again.\")\n\n return classification_results, classification_probabilities\n \n\n @deprecated(since=\"2.1.6\", message=\"'.predictImage()' has been deprecated! Please use 'classifyImage()' instead.\")\n def predictImage(self, image_input, result_count=5, input_type=\"file\"):\n \n return self.classifyImage(image_input, result_count, input_type)" }, { "path": "imageai_tf_deprecated/Detection/Custom/CUSTOMDETECTION.md", "content": "# ImageAI : Custom Object Detection\n\nAn **DeepQuest AI** project [https://deepquestai.com](https://deepquestai.com)\n\n---\n\n\n### TABLE OF CONTENTS\n\n- :white_square_button: Custom Object Detection\n- :white_square_button: Object Detection, Extraction and Fine-tune\n- :white_square_button: Hiding/Showing Object Name and Probability\n- :white_square_button: Image Input & Output Types\n- :white_square_button: Documentation\n\n\nImageAI provides very convenient and powerful methods to perform object detection on images and extract each object from the image using your own **custom YOLOv3 model** and the corresponding **detection_config.json** generated during the training. To test the custom object detection, you can download a sample custom model we have trained to detect the Hololens headset and its **detection_config.json** file via the links below:\n\n* [**hololens-ex-60--loss-2.76.h5**](https://github.com/OlafenwaMoses/ImageAI/releases/download/essential-v4/hololens-ex-60--loss-2.76.h5) _(Size = 236 mb)_\n* [**detection_config.json**](https://github.com/OlafenwaMoses/ImageAI/releases/download/essential-v4/detection_config.json)\n\n\n Once you download the custom object detection model file, you should copy the model file to the your project folder where your **.py** files will be.\n Then create a python file and give it a name; an example is FirstCustomDetection.py. Then write the code below into the python file: \n\n### FirstCustomDetection.py\n
\n\n```python\nfrom imageai.Detection.Custom import CustomObjectDetection\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(\"hololens-ex-60--loss-2.76.h5\")\ndetector.setJsonPath(\"detection_config.json\")\ndetector.loadModel()\ndetections = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", output_image_path=\"holo2-detected.jpg\")\nfor detection in detections:\n print(detection[\"name\"], \" : \", detection[\"percentage_probability\"], \" : \", detection[\"box_points\"])\n\n```\n\nSample Result - Input:\n\n![Input](../../../data-images/holo2.jpg)\n\n Output: \n \n![Output](../../../data-images/holo2-detected.jpg)\n \n```\nhololens : 39.69653248786926 : [611, 74, 751, 154]\nhololens : 87.6643180847168 : [23, 46, 90, 79]\nhololens : 89.25175070762634 : [191, 66, 243, 95]\nhololens : 64.49641585350037 : [437, 81, 514, 133]\nhololens : 91.78624749183655 : [380, 113, 423, 138]\n\n```\n\n\nLet us make a breakdown of the object detection code that we used above.\n\n```python\nfrom imageai.Detection.Custom import CustomObjectDetection\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\n```\n In the 3 lines above , we import the **ImageAI custom object detection** class in the first line, created the class instance on the second line and set the model type to YOLOv3.\n \n```python\ndetector.setModelPath(\"hololens-ex-60--loss-2.76.h5\")\ndetector.setJsonPath(\"detection_config.json\")\ndetector.loadModel()\n```\n\n In the 3 lines above, we specified the file path to our downloaded model file in the first line , specified the path to our **detection_config.json** file in the second line and loaded the model on the third line.\n\n```python\ndetections = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", output_image_path=\"holo2-detected.jpg\")\nfor detection in detections:\n print(detection[\"name\"], \" : \", detection[\"percentage_probability\"], \" : \", detection[\"box_points\"])\n\n```\n\nIn the 3 lines above, we ran the `detectObjectsFromImage()` function and parse in the path to our test image, and the path to the new\n image which the function will save. Then the function returns an array of dictionaries with each dictionary corresponding\n to the number of objects detected in the image. Each dictionary has the properties `name` (name of the object),\n`percentage_probability` (percentage probability of the detection) and `box_points` (the x1,y1,x2 and y2 coordinates of the bounding box of the object). \n\n\n\n\n### Object Detection, Extraction and Fine-tune\n
\n\nIn the examples we used above, we ran the object detection on an image and it\nreturned the detected objects in an array as well as save a new image with rectangular markers drawn on each object. In our next examples, we will be able to extract each object from the input image and save it independently.\n \n \n\nIn the example code below which is very identical to the previous object detection code, we will save each object detected as a separate image.\n\n```python\nfrom imageai.Detection.Custom import CustomObjectDetection\n\ndetector = CustomObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath(\"hololens-ex-60--loss-2.76.h5\")\ndetector.setJsonPath(\"detection_config.json\") \ndetector.loadModel()\ndetections, extracted_objects_array = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", output_image_path=\"holo2-detected.jpg\", extract_detected_objects=True)\n\nfor detection, object_path in zip(detections, extracted_objects_array):\n print(object_path)\n print(detection[\"name\"], \" : \", detection[\"percentage_probability\"], \" : \", detection[\"box_points\"])\n print(\"---------------\")\n```\n\n\nSample Result: Output Images\n \n![](../../../data-images/holo2-detected-objects/hololens-1.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-2.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-3.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-4.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-5.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-6.jpg)\n![](../../../data-images/holo2-detected-objects/hololens-7.jpg)\n\n\n\n\nLet us review the part of the code that perform the object detection and extract the images:\n\n```python\ndetections, extracted_objects_array = detector.detectObjectsFromImage(input_image=\"holo2.jpg\", output_image_path=\"holo2-detected.jpg\", extract_detected_objects=True)\n\nfor detection, object_path in zip(detections, extracted_objects_array):\n print(object_path)\n print(detection[\"name\"], \" : \", detection[\"percentage_probability\"], \" : \", detection[\"box_points\"])\n print(\"---------------\")\n```\n\nIn the above above lines, we called the `detectObjectsFromImage()` , parse in the input image path, output image part, and an\nextra parameter `extract_detected_objects=True`. This parameter states that the function should extract each object detected from the image\nand save it has a seperate image. The parameter is false by default. Once set to `true`, the function will create a directory\n which is the `output image path + \"-objects\"`. Then it saves all the extracted images into this new directory with\n each image's name being the `detected object name + \"-\" + a number` which corresponds to the order at which the objects\n were detected.\n \nThis new parameter we set to extract and save detected objects as an image will make the function to return 2 values. The\n first is the array of dictionaries with each dictionary corresponding to a detected object. The second is an array of the paths\n to the saved images of each object detected and extracted, and they are arranged in order at which the objects are in the\n first array.\n\n \n\n### And one important feature you need to know!\n\nYou will recall that the percentage probability\n for each detected object is sent back by the `detectObjectsFromImage()` function. The function has a parameter\n `minimum_percentage_probability` , whose default value is `30` (value ranges between 0 - 100) , but it set to 30 in this example. That means the function will only return a detected\n object if it's percentage probability is **30 or above**. The value was kept at this number to ensure the integrity of the\n detection results. You fine-tune the object\n detection by setting `minimum_percentage_probability` equal to a smaller value to detect more number of objects or higher value to detect less number of objects.\n\n\n\n\n### Hiding/Showing Object Name and Probability\n
\n\n**ImageAI** provides options to hide the name of objects detected and/or the percentage probability from being shown on the saved/returned detected image. Using the `detectObjectsFromImage()` and `detectCustomObjectsFromImage()` functions, the parameters `'display_object_name'` and `'display_percentage_probability'` can be set to True of False individually. Take a look at the code below: \n```python\ndetections = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"holo2.jpg\"), output_image_path=os.path.join(execution_path , \"holo2_nodetails.jpg\"), minimum_percentage_probability=30, display_percentage_probability=False, display_object_name=False)\n```\n\nIn the above code, we specified that both the object name and percentage probability should not be shown. As you can see in the result below, both the names of the objects and their individual percentage probability is not shown in the detected image. \n\n**Result**\n\n![](../../../data-images/holo2-nodetails.jpg)\n\n\n### Image Input & Output Types\n
\n\n**ImageAI** custom object detection supports 2 input types of inputs which are **file path to image file**(default) and **numpy array of an image**\nas well as 2 types of output which are image **file**(default) and numpy **array **.\nThis means you can now perform object detection in production applications such as on a web server and system\n that returns file in any of the above stated formats.\n To perform object detection with numpy array input, you just need to state the input type\nin the `.detectObjectsFromImage()` function. See example below.\n\n```python\ndetections = detector.detectObjectsFromImage(input_type=\"array\", input_image=image_array , output_image_path=os.path.join(execution_path , \"holo2-detected.jpg\")) # For numpy array input type\n```\nTo perform object detection with numpy array output you just need to state the output type\nin the `.detectObjectsFromImage()` function. See example below.\n\n```python\ndetected_image_array, detections = detector.detectObjectsFromImage(output_type=\"array\", input_image=\"holo2.jpg\" ) # For numpy array output type\n```\n\n\n\n### Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions in 3 major languages. Find links below: \n\n* Documentation - **English Version** [https://imageai.readthedocs.io](https://imageai.readthedocs.io)** \n* Documentation - **Chinese Version** [https://imageai-cn.readthedocs.io](https://imageai-cn.readthedocs.io)**\n* Documentation - **French Version** [https://imageai-fr.readthedocs.io](https://imageai-fr.readthedocs.io)**\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/CUSTOMDETECTIONTRAINING.md", "content": "# ImageAI : Custom Detection Model Training \n\n---\n\n**ImageAI** provides the most simple and powerful approach to training custom object detection models\nusing the YOLOv3 architeture, which\nwhich you can load into the `imageai.Detection.Custom.CustomObjectDetection` class. This allows\n you to train your own model on any set of images that corresponds to any type of objects of interest.\nThe training process generates a JSON file that maps the objects names in your image dataset and the detection anchors, as well as creates lots of models. In choosing the best model for your custom object detection task, an `evaluateModel()` function has been provided to compute the **mAP** of your saved models by allowing you to state your desired **IoU** and **Non-maximum Suppression** values. Then you can perform custom\nobject detection using the model and the JSON file generated. \n\n### TABLE OF CONTENTS\n- :white_square_button: Preparing your custom dataset\n- :white_square_button: Training on your custom Dataset\n- :white_square_button: Evaluating your saved detection models' mAP\n\n\n### Preparing your custom dataset\n
\n\nTo train a custom detection model, you need to prepare the images you want to use to train the model. \nYou will prepare the images as follows: \n\n1. Decide the type of object(s) you want to detect and collect about **200 (minimum recommendation)** or more picture of each of the object(s)\n2. Once you have collected the images, you need to annotate the object(s) in the images. **ImageAI** uses the **Pascal VOC format** for image annotation. You can generate this annotation for your images using the easy to use [**LabelImg**](https://github.com/tzutalin/labelImg) image annotation tool, available for Windows, Linux and MacOS systems. Open the link below to install the annotation tool. See: [https://github.com/tzutalin/labelImg](https://github.com/tzutalin/labelImg)\n3. When you are done annotating your images, **annotation XML** files will be generated for each image in your dataset. The **annotation XML** file describes each or **all** of the objects in the image. For example, if each image your image names are **image(1).jpg**, **image(2).jpg**, **image(3).jpg** till **image(z).jpg**; the corresponding annotation for each of the images will be **image(1).xml**, **image(2).xml**, **image(3).xml** till **image(z).xml**. \n4. Once you have the annotations for all your images, create a folder for your dataset (E.g headsets) and in this parent folder, create child folders **train** and **validation**\n5. In the train folder, create **images** and **annotations**\n sub-folders. Put about 70-80% of your dataset of each object's images in the **images** folder and put the corresponding annotations for these images in the **annotations** folder. \n6. In the validation folder, create **images** and **annotations** sub-folders. Put the rest of your dataset images in the **images** folder and put the corresponding annotations for these images in the **annotations** folder.\n7. Once you have done this, the structure of your image dataset folder should look like below: \n ```\n >> train >> images >> img_1.jpg (shows Object_1)\n >> images >> img_2.jpg (shows Object_2)\n >> images >> img_3.jpg (shows Object_1, Object_3 and Object_n)\n >> annotations >> img_1.xml (describes Object_1)\n >> annotations >> img_2.xml (describes Object_2)\n >> annotations >> img_3.xml (describes Object_1, Object_3 and Object_n)\n \n >> validation >> images >> img_151.jpg (shows Object_1, Object_3 and Object_n)\n >> images >> img_152.jpg (shows Object_2)\n >> images >> img_153.jpg (shows Object_1)\n >> annotations >> img_151.xml (describes Object_1, Object_3 and Object_n)\n >> annotations >> img_152.xml (describes Object_2)\n >> annotations >> img_153.xml (describes Object_1)\n ```\n8. You can train your custom detection model completely from scratch or use transfer learning (recommended for better accuracy) from a pre-trained YOLOv3 model. Also, we have provided a sample annotated Hololens and Headsets (Hololens and Oculus) dataset for you to train with. Download the pre-trained YOLOv3 model and the sample datasets in the link below. \n\n[https://github.com/OlafenwaMoses/ImageAI/releases/tag/essential-v4](https://github.com/OlafenwaMoses/ImageAI/releases/tag/essential-v4)\n\n\n### Training on your custom dataset\n
\n\nBefore you start training your custom detection model, kindly take note of the following: \n\n- The default **batch_size** is 4. If you are training with **Google Colab**, this will be fine. However, I will advice you use a more powerful GPU than the K80 offered by Colab as the higher your **batch_size (8, 16)**, the better the accuracy of your detection model. \n- If you experience '_TfDeviceCaptureOp' object has no attribute '_set_device_from_string' error in Google Colab, it is due to a bug in **Tensorflow**. You can solve this by installing **Tensorflow GPU 1.13.1**. \n ```bash\n pip3 install tensorflow-gpu==1.13.1\n ```\n\nThen your training code goes as follows: \n```python\nfrom imageai.Detection.Custom import DetectionModelTrainer\n\ntrainer = DetectionModelTrainer()\ntrainer.setModelTypeAsYOLOv3()\ntrainer.setDataDirectory(data_directory=\"hololens\")\ntrainer.setTrainConfig(object_names_array=[\"hololens\"], batch_size=4, num_experiments=200, train_from_pretrained_model=\"pretrained-yolov3.h5\")\n# In the above,when training for detecting multiple objects,\n#set object_names_array=[\"object1\", \"object2\", \"object3\",...\"objectz\"]\ntrainer.trainModel()\n```\n\n Yes! Just 6 lines of code and you can train object detection models on your custom dataset.\nNow lets take a look at how the code above works. \n\n```python\nfrom imageai.Detection.Custom import DetectionModelTrainer\n\ntrainer = DetectionModelTrainer()\ntrainer.setModelTypeAsYOLOv3()\ntrainer.setDataDirectory(data_directory=\"hololens\")\n```\n\nIn the first line, we import the **ImageAI** detection model training class, then we define the model trainer in the second line,\n we set the network type in the third line and set the path to the image dataset we want to train the network on.\n\n```python\ntrainer.setTrainConfig(object_names_array=[\"hololens\"], batch_size=4, num_experiments=200, train_from_pretrained_model=\"pretrained-yolov3.h5\")\n```\n\n\nIn the line above, we configured our detection model trainer. The parameters we stated in the function as as below: \n\n- **num_objects** : this is an array containing the names of the objects in our dataset\n- **batch_size** : this is to state the batch size for the training\n- **num_experiments** : this is to state the number of times the network will train over all the training images,\n which is also called epochs \n- **train_from_pretrained_model(optional)** : this is to train using transfer learning from a pre-trained **YOLOv3** model\n\n```python\ntrainer.trainModel()\n```\n\n\nWhen you start the training, you should see something like this in the console: \n```\nUsing TensorFlow backend.\nGenerating anchor boxes for training images and annotation...\nAverage IOU for 9 anchors: 0.78\nAnchor Boxes generated.\nDetection configuration saved in hololens/json/detection_config.json\nTraining on: \t['hololens']\nTraining with Batch Size: 4\nNumber of Experiments: 200\n\nEpoch 1/200\n480/480 [==============================] - 395s 823ms/step - loss: 36.9000 - yolo_layer_1_loss: 3.2970 - yolo_layer_2_loss: 9.4923 - yolo_layer_3_loss: 24.1107 - val_loss: 15.6321 - val_yolo_layer_1_loss: 2.0275 - val_yolo_layer_2_loss: 6.4191 - val_yolo_layer_3_loss: 7.1856\nEpoch 2/200\n480/480 [==============================] - 293s 610ms/step - loss: 11.9330 - yolo_layer_1_loss: 1.3968 - yolo_layer_2_loss: 4.2894 - yolo_layer_3_loss: 6.2468 - val_loss: 7.9868 - val_yolo_layer_1_loss: 1.7054 - val_yolo_layer_2_loss: 2.9156 - val_yolo_layer_3_loss: 3.3657\nEpoch 3/200\n480/480 [==============================] - 293s 610ms/step - loss: 7.1228 - yolo_layer_1_loss: 1.0583 - yolo_layer_2_loss: 2.2863 - yolo_layer_3_loss: 3.7782 - val_loss: 6.4964 - val_yolo_layer_1_loss: 1.1391 - val_yolo_layer_2_loss: 2.2058 - val_yolo_layer_3_loss: 3.1514\nEpoch 4/200\n480/480 [==============================] - 297s 618ms/step - loss: 5.5802 - yolo_layer_1_loss: 0.9742 - yolo_layer_2_loss: 1.8916 - yolo_layer_3_loss: 2.7144 - val_loss: 6.4275 - val_yolo_layer_1_loss: 1.6153 - val_yolo_layer_2_loss: 2.1203 - val_yolo_layer_3_loss: 2.6919\nEpoch 5/200\n480/480 [==============================] - 295s 615ms/step - loss: 4.8717 - yolo_layer_1_loss: 0.7568 - yolo_layer_2_loss: 1.6641 - yolo_layer_3_loss: 2.4508 - val_loss: 6.3723 - val_yolo_layer_1_loss: 1.6434 - val_yolo_layer_2_loss: 2.1188 - val_yolo_layer_3_loss: 2.6101\nEpoch 6/200\n480/480 [==============================] - 300s 624ms/step - loss: 4.7989 - yolo_layer_1_loss: 0.8708 - yolo_layer_2_loss: 1.6683 - yolo_layer_3_loss: 2.2598 - val_loss: 5.8672 - val_yolo_layer_1_loss: 1.2349 - val_yolo_layer_2_loss: 2.0504 - val_yolo_layer_3_loss: 2.5820\nEpoch 7/200\n```\n\nLet us explain the details shown above: \n```\nUsing TensorFlow backend.\nGenerating anchor boxes for training images and annotation...\nAverage IOU for 9 anchors: 0.78\nAnchor Boxes generated.\nDetection configuration saved in hololens/json/detection_config.json\nTraining on: \t['hololens']\nTraining with Batch Size: 4\nNumber of Experiments: 200\n```\n\nThe above details signifies the following: \n- **ImageAI** autogenerates the best match detection **anchor boxes** for your image dataset. \n\n- The anchor boxes and the object names mapping are saved in \n**json/detection_config.json** path of in the image dataset folder. Please note that for every new training you start, a new **detection_config.json** file is generated and is only compatible with the model saved during that training.\n\n```\nEpoch 1/200\n480/480 [==============================] - 395s 823ms/step - loss: 36.9000 - yolo_layer_1_loss: 3.2970 - yolo_layer_2_loss: 9.4923 - yolo_layer_3_loss: 24.1107 - val_loss: 15.6321 - val_yolo_layer_1_loss: 2.0275 - val_yolo_layer_2_loss: 6.4191 - val_yolo_layer_3_loss: 7.1856\nEpoch 2/200\n480/480 [==============================] - 293s 610ms/step - loss: 11.9330 - yolo_layer_1_loss: 1.3968 - yolo_layer_2_loss: 4.2894 - yolo_layer_3_loss: 6.2468 - val_loss: 7.9868 - val_yolo_layer_1_loss: 1.7054 - val_yolo_layer_2_loss: 2.9156 - val_yolo_layer_3_loss: 3.3657\nEpoch 3/200\n480/480 [==============================] - 293s 610ms/step - loss: 7.1228 - yolo_layer_1_loss: 1.0583 - yolo_layer_2_loss: 2.2863 - yolo_layer_3_loss: 3.7782 - val_loss: 6.4964 - val_yolo_layer_1_loss: 1.1391 - val_yolo_layer_2_loss: 2.2058 - val_yolo_layer_3_loss: 3.1514\nEpoch 4/200\n480/480 [==============================] - 297s 618ms/step - loss: 5.5802 - yolo_layer_1_loss: 0.9742 - yolo_layer_2_loss: 1.8916 - yolo_layer_3_loss: 2.7144 - val_loss: 6.4275 - val_yolo_layer_1_loss: 1.6153 - val_yolo_layer_2_loss: 2.1203 - val_yolo_layer_3_loss: 2.6919\nEpoch 5/200\n480/480 [==============================] - 295s 615ms/step - loss: 4.8717 - yolo_layer_1_loss: 0.7568 - yolo_layer_2_loss: 1.6641 - yolo_layer_3_loss: 2.4508 - val_loss: 6.3723 - val_yolo_layer_1_loss: 1.6434 - val_yolo_layer_2_loss: 2.1188 - val_yolo_layer_3_loss: 2.6101\nEpoch 6/200\n480/480 [==============================] - 300s 624ms/step - loss: 4.7989 - yolo_layer_1_loss: 0.8708 - yolo_layer_2_loss: 1.6683 - yolo_layer_3_loss: 2.2598 - val_loss: 5.8672 - val_yolo_layer_1_loss: 1.2349 - val_yolo_layer_2_loss: 2.0504 - val_yolo_layer_3_loss: 2.5820\nEpoch 7/200\n```\n\n- The above signifies the progress of the training. \n- For each experiment (Epoch), the general total validation loss (E.g - loss: 4.7582) is reported. \n- For each drop in the loss after an experiment, a model is saved in the **hololens/models** folder. The lower the loss, the better the model. \n- **Tensorboard** report file for the training will be saved in the **hololens/logs** folder.\n\nOnce you are done training, you can visit the link below for performing object detection with your **custom detection model** and **detection_config.json** file.\n\n[Detection/Custom/CUSTOMDETECTION.md](./CUSTOMDETECTION.md)\n \n \n \n### Evaluating your saved detection models' mAP\n
\n\nAfter training on your custom dataset, you can evaluate the mAP of your saved models by specifying your desired IoU and Non-maximum suppression values. See details as below:\n\n- **Single Model Evaluation:** To evaluate a single model, simply use the example code below with the path to your dataset directory, the model file and the **detection_config.json** file saved during the training. In the example, we used an **object_threshold** of 0.3 ( percentage_score >= 30% ), **IoU** of 0.5 and **Non-maximum suppression** value of 0.5.\n ```python\n from imageai.Detection.Custom import DetectionModelTrainer\n \n trainer = DetectionModelTrainer()\n trainer.setModelTypeAsYOLOv3()\n trainer.setDataDirectory(data_directory=\"hololens\")\n metrics = trainer.evaluateModel(model_path=\"detection_model-ex-60--loss-2.76.h5\", json_path=\"detection_config.json\", iou_threshold=0.5, object_threshold=0.3, nms_threshold=0.5)\n ```\n Consider that `trainer.evaluateModel` method will show the metrics on standard output as shown below, \n but also returns a list of dicts containing all the information that is displayed. \n \n Sample Result:\n ```\n Model File: hololens_detection_model-ex-09--loss-4.01.h5 \n Using IoU : 0.5\n Using Object Threshold : 0.3\n Using Non-Maximum Suppression : 0.5\n hololens: 0.9613\n mAP: 0.9613\n ===============================\n ```\n Let's see how those metrics looks like:\n ```\n [{\n 'average_precision': {'hololens': 0.9613334437735249},\n 'map': 0.9613334437735249,\n 'model_file': 'hololens_detection_model-ex-09--loss-4.01.h5',\n 'using_iou': 0.5,\n 'using_non_maximum_suppression': 0.5,\n 'using_object_threshold': 0.3\n }]\n ```\n- **Multi Model Evaluation:** To evaluate all your saved models, simply parse in the path to the folder containing the models as the **model_path** as seen in the example below:\n ```python\n from imageai.Detection.Custom import DetectionModelTrainer\n \n trainer = DetectionModelTrainer()\n trainer.setModelTypeAsYOLOv3()\n trainer.setDataDirectory(data_directory=\"hololens\")\n metrics = trainer.evaluateModel(model_path=\"hololens/models\", json_path=\"hololens/json/detection_config.json\", iou_threshold=0.5, object_threshold=0.3, nms_threshold=0.5)\n ```\n Sample Result:\n ```\n Model File: hololens/models/detection_model-ex-07--loss-4.42.h5 \n Using IoU : 0.5\n Using Object Threshold : 0.3\n Using Non-Maximum Suppression : 0.5\n hololens: 0.9231\n mAP: 0.9231\n ===============================\n Model File: hololens/models/detection_model-ex-10--loss-3.95.h5 \n Using IoU : 0.5\n Using Object Threshold : 0.3\n Using Non-Maximum Suppression : 0.5\n hololens: 0.9725\n mAP: 0.9725\n ===============================\n Model File: hololens/models/detection_model-ex-05--loss-5.26.h5 \n Using IoU : 0.5\n Using Object Threshold : 0.3\n Using Non-Maximum Suppression : 0.5\n hololens: 0.9204\n mAP: 0.9204\n ===============================\n Model File: hololens/models/detection_model-ex-03--loss-6.44.h5 \n Using IoU : 0.5\n Using Object Threshold : 0.3\n Using Non-Maximum Suppression : 0.5\n hololens: 0.8120\n mAP: 0.8120\n ===============================\n Model File: hololens/models/detection_model-ex-18--loss-2.96.h5 \n Using IoU : 0.5\n Using Object Threshold : 0.3\n Using Non-Maximum Suppression : 0.5\n hololens: 0.9431\n mAP: 0.9431\n ===============================\n Model File: hololens/models/detection_model-ex-17--loss-3.10.h5 \n Using IoU : 0.5\n Using Object Threshold : 0.3\n Using Non-Maximum Suppression : 0.5\n hololens: 0.9404\n mAP: 0.9404\n ===============================\n Model File: hololens/models/detection_model-ex-08--loss-4.16.h5 \n Using IoU : 0.5\n Using Object Threshold : 0.3\n Using Non-Maximum Suppression : 0.5\n hololens: 0.9725\n mAP: 0.9725\n ===============================\n ```\n Let's see how those metrics looks like:\n ```\n [{\n 'average_precision': {'hololens': 0.9231334437735249},\n 'map': 0.9231334437735249,\n 'model_file': 'hololens/models/detection_model-ex-07--loss-4.42.h5',\n 'using_iou': 0.5,\n 'using_non_maximum_suppression': 0.5,\n 'using_object_threshold': 0.3\n },\n {\n 'average_precision': {'hololens': 0.9725334437735249},\n 'map': 0.97251334437735249,\n 'model_file': 'hololens/models/detection_model-ex-10--loss-3.95.h5',\n 'using_iou': 0.5,\n 'using_non_maximum_suppression': 0.5,\n 'using_object_threshold': 0.3\n },\n {\n 'average_precision': {'hololens': 0.92041334437735249},\n 'map': 0.92041334437735249,\n 'model_file': 'hololens/models/detection_model-ex-05--loss-5.26.h5',\n 'using_iou': 0.5,\n 'using_non_maximum_suppression': 0.5,\n 'using_object_threshold': 0.3\n },\n {\n 'average_precision': {'hololens': 0.81201334437735249},\n 'map': 0.81201334437735249,\n 'model_file': 'hololens/models/detection_model-ex-03--loss-6.44.h5',\n 'using_iou': 0.5,\n 'using_non_maximum_suppression': 0.5,\n 'using_object_threshold': 0.3\n },\n {\n 'average_precision': {'hololens': 0.94311334437735249},\n 'map': 0.94311334437735249,\n 'model_file': 'hololens/models/detection_model-ex-18--loss-2.96.h5',\n 'using_iou': 0.5,\n 'using_non_maximum_suppression': 0.5,\n 'using_object_threshold': 0.3\n },\n {\n 'average_precision': {'hololens': 0.94041334437735249},\n 'map': 0.94041334437735249,\n 'model_file': 'hololens/models/detection_model-ex-17--loss-3.10.h5',\n 'using_iou': 0.5,\n 'using_non_maximum_suppression': 0.5,\n 'using_object_threshold': 0.3\n },\n {\n 'average_precision': {'hololens': 0.97251334437735249},\n 'map': 0.97251334437735249,\n 'model_file': 'hololens/models/detection_model-ex-08--loss-4.16.h5',\n 'using_iou': 0.5,\n 'using_non_maximum_suppression': 0.5,\n 'using_object_threshold': 0.3\n }\n ]\n ```\n\n\n### >> Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions in 3 major languages. Find links below: \n\n* Documentation - **English Version** [https://imageai.readthedocs.io](https://imageai.readthedocs.io) \n* Documentation - **Chinese Version** [https://imageai-cn.readthedocs.io](https://imageai-cn.readthedocs.io)\n* Documentation - **French Version** [https://imageai-fr.readthedocs.io](https://imageai-fr.readthedocs.io)\n\n\n\n\n\n\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/CUSTOMVIDEODETECTION.md", "content": "# ImageAI : Custom Video Object Detection, Tracking and Analysis\n\nAn **DeepQuest AI** project [https://deepquestai.com](https://deepquestai.com)\n\n---\n\n### TABLE OF CONTENTS\n\n- :white_square_button: First Custom Video Object Detection\n- :white_square_button: Camera / Live Stream Video Detection\n- :white_square_button: Video Analysis\n- :white_square_button: Hiding/Showing Object Name and Probability\n- :white_square_button: Frame Detection Intervals\n- :white_square_button: Video Detection Timeout (NEW)\n- :white_square_button: Documentation\n\n\nImageAI provides convenient, flexible and powerful methods to perform object detection on videos using your own **custom YOLOv3 model** and the corresponding **detection_config.json** generated during the training. This version of **ImageAI** provides commercial grade video objects detection features, which include but not limited to device/IP camera inputs, per frame, per second, per minute and entire video analysis for storing in databases and/or real-time visualizations and for future insights.\nTo test the custom video object detection,you can download a sample custom model we have trained to detect the Hololens headset and its **detection_config.json** file via the links below:\n- [**hololens-ex-60--loss-2.76.h5**](https://github.com/OlafenwaMoses/ImageAI/releases/download/essential-v4/hololens-ex-60--loss-2.76.h5) _(Size = 236 mb)_\n- [**detection_config.json**](https://github.com/OlafenwaMoses/ImageAI/releases/download/essential-v4/detection_config.json)\n\n\nBecause video object detection is a compute intensive tasks, we advise you perform this experiment using a computer with a NVIDIA GPU and the GPU version of Tensorflow\n installed. Performing Video Object Detection CPU will be slower than using an NVIDIA GPU powered computer. You can use Google Colab for this\n experiment as it has an NVIDIA K80 GPU available for free.\n
\n Once you download the custom object detection model and JSON files, you should copy the model and the JSON files to the your project folder where your .py files will be.\n Then create a python file and give it a name; an example is FirstCustomVideoObjectDetection.py. Then write the code below into the python file:
\n\n\n### FirstCustomVideoObjectDetection.py\n
\n\n```python\nfrom imageai.Detection.Custom import CustomVideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"hololens-ex-60--loss-2.76.h5\")\nvideo_detector.setJsonPath(\"detection_config.json\")\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(input_file_path=\"holo1.mp4\",\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20,\n minimum_percentage_probability=40,\n log_progress=True)\n```\n\n[**Input Video**](../../../data-videos/holo1.mp4)\n[![Input Video](../../../data-images/holo-video.jpg)](../../../data-videos/holo1.mp4)\n[**Output Video**](https://www.youtube.com/watch?v=4o5GyAR4Mpw)\n[![Output Video](../../../data-images/holo-video-detected.jpg)](https://www.youtube.com/watch?v=4o5GyAR4Mpw)\n\n\n\nLet us make a breakdown of the object detection code that we used above.\n\n```python\nfrom imageai.Detection.Custom import CustomVideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n```\n\nIn the 3 lines above , we import the **ImageAI custom video object detection** class in the first line, import the **os** in the second line and obtained\n the path to folder where our python file runs.\n```python\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"hololens-ex-60--loss-2.76.h5\")\nvideo_detector.setJsonPath(\"detection_config.json\")\nvideo_detector.loadModel()\n```\nIn the 4 lines above, we created a new instance of the `CustomVideoObjectDetection` class in the first line, set the model type to YOLOv3 in the second line,\n set the model path to our custom YOLOv3 model file in the third line, specified the path to the model's corresponding **detection_config.json** in the fourth line and load the model in the fifth line.\n\n```python\nvideo_detector.detectObjectsFromVideo(input_file_path=\"holo1.mp4\",\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20,\n minimum_percentage_probability=40,\n log_progress=True)\n```\n\nIn the code above, we ran the `detectObjectsFromVideo()` function and parse in the path to our video,the path to the new\n video (without the extension, it saves a .avi video by default) which the function will save, the number of frames per second (fps) that\n you we desire the output video to have and option to log the progress of the detection in the console. Then the function returns a the path to the saved video\n which contains boxes and percentage probabilities rendered on objects detected in the video.\n\n\n### Camera / Live Stream Video Detection\n
\n\n**ImageAI** now allows live-video detection with support for camera inputs. Using **OpenCV**'s **VideoCapture()** function, you can load live-video streams from a device camera, cameras connected by cable or IP cameras, and parse it into **ImageAI**'s **detectObjectsFromVideo()** function. All features that are supported for detecting objects in a video file is also available for detecting objects in a camera's live-video feed. Find below an example of detecting live-video feed from the device camera. \n\n```python\nfrom imageai.Detection.Custom import CustomVideoObjectDetection\nimport os\nimport cv2\n\nexecution_path = os.getcwd()\ncamera = cv2.VideoCapture(0)\n\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"hololens-ex-60--loss-2.76.h5\")\nvideo_detector.setJsonPath(\"detection_config.json\")\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20,\n minimum_percentage_probability=40,\n log_progress=True)\n```\n\nThe difference in the code above and the code for the detection of a video file is that we defined an **OpenCV VideoCapture** instance and loaded the default device camera into it. Then we parsed the camera we defined into the parameter **camera_input** which replaces the **input_file_path** that is used for video file. \n\n\n### Video Analysis\n
\n\n**ImageAI** now provide commercial-grade video analysis in the Custom Video Object Detection class, for both video file inputs and camera inputs. This feature allows developers to obtain deep insights into any video processed with **ImageAI**. This insights can be visualized in real-time, stored in a NoSQL database for future review or analysis.
\n\nFor video analysis, the **detectObjectsFromVideo()** now allows you to state your own defined functions which will be executed for every frame, seconds and/or minute of the video detected as well as a state a function that will be executed at the end of a video detection. Once this functions are stated, they will receive raw but comprehensive analytical data on the index of the frame/second/minute, objects detected (name, percentage_probability and box_points), number of instances of each unique object detected and average number of occurrence of each unique object detected over a second/minute and entire video.\n\nTo obtain the video analysis, all you need to do is specify a function, state the corresponding parameters it will be receiving and parse the function name into the **per_frame_function**, **per_second_function**, **per_minute_function** and **video_complete_function** parameters in the detection function. Find below examples of video analysis functions. \n\n```python\ndef forFrame(frame_number, output_array, output_count):\n print(\"FOR FRAME \" , frame_number)\n print(\"Output for each object : \", output_array)\n print(\"Output count for unique objects : \", output_count)\n print(\"------------END OF A FRAME --------------\")\n\ndef forSeconds(second_number, output_arrays, count_arrays, average_output_count):\n print(\"SECOND : \", second_number)\n print(\"Array for the outputs of each frame \", output_arrays)\n print(\"Array for output count for unique objects in each frame : \", count_arrays)\n print(\"Output average count for unique objects in the last second: \", average_output_count)\n print(\"------------END OF A SECOND --------------\")\n\ndef forMinute(minute_number, output_arrays, count_arrays, average_output_count):\n print(\"MINUTE : \", minute_number)\n print(\"Array for the outputs of each frame \", output_arrays)\n print(\"Array for output count for unique objects in each frame : \", count_arrays)\n print(\"Output average count for unique objects in the last minute: \", average_output_count)\n print(\"------------END OF A MINUTE --------------\")\n\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"hololens-ex-60--loss-2.76.h5\")\nvideo_detector.setJsonPath(\"detection_config.json\")\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20, per_second_function=forSeconds, per_frame_function = forFrame, per_minute_function= forMinute,\n minimum_percentage_probability=40,\n log_progress=True)\n```\n\n\n**ImageAI** also allows you to obtain complete analysis of the entire video processed. All you need is to define a function like the forSecond or forMinute function and set the **video_complete_function** parameter into your **.detectObjectsFromVideo()** function. The same values for the per_second-function and per_minute_function will be returned. The difference is that no index will be returned and the other 3 values will be returned, and the 3 values will cover all frames in the video. Below is a sample function: \n```python\ndef forFull(output_arrays, count_arrays, average_output_count):\n #Perform action on the 3 parameters returned into the function\n\n\nvideo_detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n video_complete_function=forFull,\n minimum_percentage_probability=40,\n log_progress=True)\n\n```\n\n**FINAL NOTE ON VIDEO ANALYSIS** : **ImageAI** allows you to obtain the detected video frame as a Numpy array at each frame, second and minute function. All you need to do is specify one more parameter in your function and set **return_detected_frame=True** in your **detectObjectsFromVideo()** function. Once this is set, the extra parameter you sepecified in your function will be the Numpy array of the detected frame. See a sample below:\n\n```python\ndef forFrame(frame_number, output_array, output_count, detected_frame):\n print(\"FOR FRAME \" , frame_number)\n print(\"Output for each object : \", output_array)\n print(\"Output count for unique objects : \", output_count)\n\tprint(\"Returned Objects is : \", type(detected_frame))\n print(\"------------END OF A FRAME --------------\")\n\n\nvideo_detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n per_frame_function=forFrame,\n minimum_percentage_probability=40,\n log_progress=True, return_detected_frame=True)\n```\n\n\n### Frame Detection Intervals\n
\n\nThe above video objects detection task are optimized for frame-real-time object detections that ensures that objects in every frame\nof the video is detected. **ImageAI** provides you the option to adjust the video frame detections which can speed up\nyour video detection process. When calling the `.detectObjectsFromVideo()`, you can\nspecify at which frame interval detections should be made. By setting the **frame_detection_interval** parameter to be\n equal to 5 or 20, that means the object detections in the video will be updated after 5 frames or 20 frames.\nIf your output video **frames_per_second** is set to 20, that means the object detections in the video will\n be updated once in every quarter of a second or every second. This is useful in case scenarios where the available\n compute is less powerful and speeds of moving objects are low. This ensures you can have objects detected as second-real-time\n, half-a-second-real-time or whichever way suits your needs. \n\n\n### Custom Video Detection Timeout\n
\n\n**ImageAI** now allows you to set a timeout in seconds for detection of objects in videos or camera live feed. To set a timeout for your video detection code, all you need to do is specify the `detection_timeout` parameter in the `detectObjectsFromVideo()` function to the number of desired seconds. In the example code below, we set `detection_timeout` to 120 seconds (2 minutes). \n\n\n```python\nfrom imageai.Detection.Custom import CustomVideoObjectDetection\nimport os\nimport cv2\n\nexecution_path = os.getcwd()\ncamera = cv2.VideoCapture(0)\n\nvideo_detector = CustomVideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(\"hololens-ex-60--loss-2.76.h5\")\nvideo_detector.setJsonPath(\"detection_config.json\")\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"holo1-detected3\"),\n frames_per_second=20, minimum_percentage_probability=40,\n detection_timeout=120)\n```\n\n\n### >> Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions in 3 major languages. Find links below: \n\n* Documentation - **English Version** [https://imageai.readthedocs.io](https://imageai.readthedocs.io)** \n* Documentation - **Chinese Version** [https://imageai-cn.readthedocs.io](https://imageai-cn.readthedocs.io)**\n* Documentation - **French Version** [https://imageai-fr.readthedocs.io](https://imageai-fr.readthedocs.io)**\n\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/__init__.py", "content": "import os\nimport re\nimport numpy as np\nimport json\nfrom imageai.Detection.Custom.voc import parse_voc_annotation\nfrom imageai.Detection.YOLO.yolov3 import yolov3_main, yolov3_train, dummy_loss\nfrom imageai.Detection.Custom.generator import BatchGenerator\nfrom imageai.Detection.Custom.utils.utils import normalize, evaluate, makedirs\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.keras.optimizers import Adam\nfrom imageai.Detection.Custom.callbacks import CustomModelCheckpoint\nfrom imageai.Detection.Custom.utils.multi_gpu_model import multi_gpu_model\nfrom imageai.Detection.Custom.gen_anchors import generateAnchors\nimport tensorflow as tf\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras import Input\nfrom tensorflow.keras.callbacks import TensorBoard\nimport tensorflow.keras.backend as K\nimport cv2\n\ntf.config.run_functions_eagerly(True)\nos.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\"\n\n\nclass DetectionModelTrainer:\n\n \"\"\"\n This is the Detection Model training class, which allows you to train object detection models\n on image datasets that are in Pascal VOC annotation format, using the YOLOv3.\n \"\"\"\n\n def __init__(self):\n self.__model_type = \"\"\n self.__training_mode = True\n\n self.__model_min_input_size = 288\n self.__model_max_input_size = 448\n self.__model_anchors = []\n self.__inference_anchors = []\n self.__json_directory = \"\"\n self.__model_labels = []\n self.__num_objects = 0\n self.__pre_trained_model = \"\"\n\n self.__train_images_folder = \"\"\n self.__train_annotations_folder = \"\"\n self.__train_cache_file = \"\"\n self.__train_times = 8\n self.__train_batch_size = 4\n self.__train_learning_rate = 1e-4\n self.__train_epochs = 100\n self.__train_warmup_epochs = 3\n self.__train_ignore_treshold = 0.5\n self.__train_gpus = \"0\"\n self.__train_grid_scales = [1, 1, 1]\n self.__train_obj_scale = 5\n self.__train_noobj_scale = 1\n self.__train_xywh_scale = 1\n self.__train_class_scale = 1\n self.__model_directory = \"\"\n self.__train_weights_name = \"\"\n self.__train_debug = True\n self.__logs_directory = \"\"\n\n self.__validation_images_folder = \"\"\n self.__validation_annotations_folder = \"\"\n self.__validation_cache_file = \"\"\n self.__validation_times = 1\n\n def setModelTypeAsYOLOv3(self):\n \"\"\"\n 'setModelTypeAsYOLOv3()' is used to set the model type to the YOLOv3 model\n for the training instance object .\n :return:\n \"\"\"\n self.__model_type = \"yolov3\"\n\n def setDataDirectory(self, data_directory):\n\n \"\"\"\n\n 'setDataDirectory()' is required to set the path to which the data/dataset to be used for\n training is kept. The directory can have any name, but it must have 'train' and 'validation'\n sub-directory. In the 'train' and 'validation' sub-directories, there must be 'images' and 'annotations'\n sub-directories respectively. The 'images' folder will contain the pictures for the dataset and the\n 'annotations' folder will contain the XML files with details of the annotations for each image in the\n 'images folder'.\n\n N.B: Strictly take note that the filenames (without the extension) of the pictures in the 'images folder'\n must be the same as the filenames (without the extension) of their corresponding annotation XML files in\n the 'annotations' folder.\n\n The structure of the 'train' and 'validation' folder must be as follows:\n\n >> train >> images >> img_1.jpg\n >> images >> img_2.jpg\n >> images >> img_3.jpg\n >> annotations >> img_1.xml\n >> annotations >> img_2.xml\n >> annotations >> img_3.xml\n\n\n >> validation >> images >> img_151.jpg\n >> images >> img_152.jpg\n >> images >> img_153.jpg\n >> annotations >> img_151.xml\n >> annotations >> img_152.xml\n >> annotations >> img_153.xml\n\n :param data_directory:\n :return:\n \"\"\"\n\n self.__train_images_folder = os.path.join(data_directory, \"train\", \"images\")\n self.__train_annotations_folder = os.path.join(data_directory, \"train\", \"annotations\")\n self.__validation_images_folder = os.path.join(data_directory, \"validation\", \"images\")\n self.__validation_annotations_folder = os.path.join(data_directory, \"validation\", \"annotations\")\n\n os.makedirs(os.path.join(data_directory, \"cache\"), exist_ok=True)\n self.__train_cache_file = os.path.join(data_directory, \"cache\", \"detection_train_data.pkl\")\n self.__validation_cache_file = os.path.join(data_directory, \"cache\", \"detection_test_data.pkl\")\n\n os.makedirs(os.path.join(data_directory, \"models\"), exist_ok=True)\n\n os.makedirs(os.path.join(data_directory, \"json\"), exist_ok=True)\n\n os.makedirs(os.path.join(data_directory, \"logs\"), exist_ok=True)\n\n self.__model_directory = os.path.join(data_directory, \"models\")\n self.__train_weights_name = os.path.join(self.__model_directory, \"detection_model-\")\n self.__json_directory = os.path.join(data_directory, \"json\")\n self.__logs_directory = os.path.join(data_directory, \"logs\")\n\n def setGpuUsage(self, train_gpus):\n \"\"\"\n 'setGpuUsage' function allows you to set the GPUs to be used while training\n train_gpu can be:\n - an integer, indicating the number of GPUs to use\n - a list of integers, indicating the id of the GPUs to be used\n - a string, indicating the it og the id of the GPUs to be used, separated by commas\n :param train_gpus: gpus where to run\n :return:\n \"\"\"\n # train_gpus, could be a string separated by comma, or a list of int or the number of GPUs to be used\n if type(train_gpus) == str:\n train_gpus = train_gpus.split(',')\n if type(train_gpus) == int:\n train_gpus = range(train_gpus)\n # let it as a string separated by commas\n self.__train_gpus = ','.join([str(gpu) for gpu in train_gpus])\n\n def setTrainConfig(self, object_names_array, batch_size=4, num_experiments=100, train_from_pretrained_model=\"\"):\n\n \"\"\"\n\n 'setTrainConfig()' function allows you to set the properties for the training instances. It accepts the following values:\n\n - object_names_array , this is an array of the names of the different objects in your dataset\n - batch_size (optional), this is the batch size for the training instance\n - num_experiments (optional), also known as epochs, it is the number of times the network will train on all the training dataset\n - train_from_pretrained_model (optional), this is used to perform transfer learning by specifying the path to a pre-trained YOLOv3 model\n\n :param object_names_array:\n :param batch_size:\n :param num_experiments:\n :param train_from_pretrained_model:\n :return:\n \"\"\"\n\n # Remove cache files\n if os.path.isfile(self.__train_cache_file) == True:\n os.remove(self.__train_cache_file)\n\n if os.path.isfile(self.__validation_cache_file) == True:\n os.remove(self.__validation_cache_file)\n\n self.__model_anchors, self.__inference_anchors = generateAnchors(self.__train_annotations_folder,\n self.__train_images_folder,\n self.__train_cache_file, self.__model_labels)\n\n self.__model_labels = sorted(object_names_array)\n self.__num_objects = len(object_names_array)\n\n self.__train_batch_size = batch_size\n self.__train_epochs = num_experiments\n self.__pre_trained_model = train_from_pretrained_model\n\n json_data = dict()\n json_data[\"labels\"] = self.__model_labels\n json_data[\"anchors\"] = self.__inference_anchors\n\n with open(os.path.join(self.__json_directory, \"detection_config.json\"), \"w+\") as json_file:\n json.dump(json_data, json_file, indent=4, separators=(\",\", \" : \"),\n ensure_ascii=True)\n\n print(\"Detection configuration saved in \", os.path.join(self.__json_directory, \"detection_config.json\"))\n\n def trainModel(self):\n\n \"\"\"\n 'trainModel()' function starts the actual model training. Once the training starts, the training instance\n creates 3 sub-folders in your dataset folder which are:\n\n - json, where the JSON configuration file for using your trained model is stored\n - models, where your trained models are stored once they are generated after each improved experiments\n - cache , where temporary traing configuraton files are stored\n\n :return:\n \"\"\"\n\n train_ints, valid_ints, labels, max_box_per_image = self._create_training_instances(\n self.__train_annotations_folder,\n self.__train_images_folder,\n self.__train_cache_file,\n self.__validation_annotations_folder,\n self.__validation_images_folder,\n self.__validation_cache_file,\n self.__model_labels\n\n )\n if self.__training_mode:\n print('Training on: \\t' + str(labels) + '')\n print(\"Training with Batch Size: \", self.__train_batch_size)\n print(\"Number of Training Samples: \", len(train_ints))\n print(\"Number of Validation Samples: \", len(valid_ints))\n print(\"Number of Experiments: \", self.__train_epochs)\n\n ###############################\n # Create the generators\n ###############################\n train_generator = BatchGenerator(\n instances=train_ints,\n anchors=self.__model_anchors,\n labels=labels,\n downsample=32, # ratio between network input's size and network output's size, 32 for YOLOv3\n max_box_per_image=max_box_per_image,\n batch_size=self.__train_batch_size,\n min_net_size=self.__model_min_input_size,\n max_net_size=self.__model_max_input_size,\n shuffle=True,\n jitter=0.3,\n norm=normalize\n )\n\n valid_generator = BatchGenerator(\n instances=valid_ints,\n anchors=self.__model_anchors,\n labels=labels,\n downsample=32, # ratio between network input's size and network output's size, 32 for YOLOv3\n max_box_per_image=max_box_per_image,\n batch_size=self.__train_batch_size,\n min_net_size=self.__model_min_input_size,\n max_net_size=self.__model_max_input_size,\n shuffle=True,\n jitter=0.0,\n norm=normalize\n )\n\n ###############################\n # Create the model\n ###############################\n if os.path.exists(self.__pre_trained_model):\n self.__train_warmup_epochs = 0\n warmup_batches = self.__train_warmup_epochs * (self.__train_times * len(train_generator))\n\n os.environ['CUDA_VISIBLE_DEVICES'] = self.__train_gpus\n multi_gpu = [int(gpu) for gpu in self.__train_gpus.split(',')]\n\n \"\"\"train_model, infer_model = self._create_model(\n nb_class=len(labels),\n anchors=self.__model_anchors,\n max_box_per_image=max_box_per_image,\n max_grid=[self.__model_max_input_size, self.__model_max_input_size],\n batch_size=self.__train_batch_size,\n warmup_batches=warmup_batches,\n ignore_thresh=self.__train_ignore_treshold,\n multi_gpu=multi_gpu,\n lr=self.__train_learning_rate,\n grid_scales=self.__train_grid_scales,\n obj_scale=self.__train_obj_scale,\n noobj_scale=self.__train_noobj_scale,\n xywh_scale=self.__train_xywh_scale,\n class_scale=self.__train_class_scale,\n )\"\"\"\n\n train_model, infer_model = self._create_model(\n nb_class=len(labels),\n anchors=self.__model_anchors,\n max_box_per_image=max_box_per_image,\n max_grid=[self.__model_max_input_size, self.__model_max_input_size],\n batch_size=self.__train_batch_size,\n warmup_batches=warmup_batches,\n ignore_thresh=self.__train_ignore_treshold,\n multi_gpu=multi_gpu,\n lr=self.__train_learning_rate,\n grid_scales=self.__train_grid_scales,\n obj_scale=self.__train_obj_scale,\n noobj_scale=self.__train_noobj_scale,\n xywh_scale=self.__train_xywh_scale,\n class_scale=self.__train_class_scale,\n )\n\n ###############################\n # Kick off the training\n ###############################\n callbacks = self._create_callbacks(self.__train_weights_name, infer_model)\n\n train_model.fit_generator(\n generator=train_generator,\n steps_per_epoch=len(train_generator) * self.__train_times,\n validation_data=valid_generator,\n validation_steps=len(valid_generator) * self.__train_times,\n epochs=self.__train_epochs + self.__train_warmup_epochs,\n verbose=1,\n callbacks=callbacks,\n workers=4,\n max_queue_size=8\n )\n\n def evaluateModel(self, model_path, json_path, batch_size=4, iou_threshold=0.5, object_threshold=0.2, nms_threshold=0.45):\n \"\"\"\n\n 'evaluateModel()' is used to obtain the mAP metrics for your model(s). It accepts the following values:\n\n - model_path ( model file or folder), this value can be the part to your model file or the path to the folder containing all your saved model files\n - json_path , this is the path the the 'detection_config.json' file saved for the dataset during the training\n - iou_threshold , this value is used to set the desired 'IoU' to obtain the mAP metrics for your model(s)\n - object_threshold , this is used to set your desired minimum 'class score' to obtain the mAP metrics for your model(s)\n - nms_threshold , this is used to set your desired 'Non-maximum suppresion' to obtain the mAP metrics for your model(s)\n\n :param model_path:\n :param json_path:\n :param batch_size:\n :param iou_threshold:\n :param object_threshold:\n :param nms_threshold:\n :return: list of dictionaries, containing one dict per evaluated model.\n Each dict contains exactly the same metrics that are printed on standard output\n \"\"\"\n\n self.__training_mode = False\n\n with open(json_path, 'r') as json_file:\n detection_model_json = json.load(json_file)\n\n temp_anchor_array = []\n new_anchor_array = []\n\n temp_anchor_array.append(detection_model_json[\"anchors\"][2])\n temp_anchor_array.append(detection_model_json[\"anchors\"][1])\n temp_anchor_array.append(detection_model_json[\"anchors\"][0])\n\n for aa in temp_anchor_array:\n for aaa in aa:\n new_anchor_array.append(aaa)\n\n self.__model_anchors = new_anchor_array\n self.__model_labels = detection_model_json[\"labels\"]\n self.__num_objects = len(self.__model_labels)\n\n self.__train_batch_size = batch_size\n self.__train_epochs = 100\n\n print(\"Starting Model evaluation....\")\n\n _, valid_ints, labels, max_box_per_image = self._create_training_instances(\n self.__train_annotations_folder,\n self.__train_images_folder,\n self.__train_cache_file,\n self.__validation_annotations_folder,\n self.__validation_images_folder,\n self.__validation_cache_file,\n self.__model_labels\n\n )\n\n if len(valid_ints) == 0:\n print('Validation samples were not provided.')\n print('Please, check your validation samples are correctly provided:')\n print('\\tAnnotations: {}\\n\\tImages: {}'.format(self.__validation_annotations_folder,\n self.__validation_images_folder))\n\n valid_generator = BatchGenerator(\n instances=valid_ints,\n anchors=self.__model_anchors,\n labels=labels,\n downsample=32, # ratio between network input's size and network output's size, 32 for YOLOv3\n max_box_per_image=max_box_per_image,\n batch_size=self.__train_batch_size,\n min_net_size=self.__model_min_input_size,\n max_net_size=self.__model_max_input_size,\n shuffle=True,\n jitter=0.0,\n norm=normalize\n )\n\n results = list()\n\n if os.path.isfile(model_path):\n # model_files must be a list containing the complete path to the files,\n # if a file is given, then the list contains just this file\n model_files = [model_path]\n elif os.path.isdir(model_path):\n # model_files must be a list containing the complete path to the files,\n # if a folder is given, then the list contains the complete path to each file on that folder\n model_files = sorted([os.path.join(model_path, file_name) for file_name in os.listdir(model_path)])\n # sort the files to make sure we're always evaluating them on same order\n else:\n print('model_path must be the path to a .h5 file or a directory. Found {}'.format(model_path))\n return results\n\n for model_file in model_files:\n if str(model_file).endswith(\".h5\"):\n try:\n infer_model = load_model(model_file)\n\n ###############################\n # Run the evaluation\n ###############################\n # compute mAP for all the classes\n average_precisions = evaluate(infer_model, valid_generator, iou_threshold=iou_threshold,\n obj_thresh=object_threshold, nms_thresh=nms_threshold)\n\n result_dict = {\n 'model_file': model_file,\n 'using_iou': iou_threshold,\n 'using_object_threshold': object_threshold,\n 'using_non_maximum_suppression': nms_threshold,\n 'average_precision': dict(),\n 'evaluation_samples': len(valid_ints)\n }\n # print the score\n print(\"Model File: \", model_file, '\\n')\n print(\"Evaluation samples: \", len(valid_ints))\n print(\"Using IoU: \", iou_threshold)\n print(\"Using Object Threshold: \", object_threshold)\n print(\"Using Non-Maximum Suppression: \", nms_threshold)\n\n for label, average_precision in average_precisions.items():\n print(labels[label] + ': {:.4f}'.format(average_precision))\n result_dict['average_precision'][labels[label]] = average_precision\n\n print('mAP: {:.4f}'.format(sum(average_precisions.values()) / len(average_precisions)))\n result_dict['map'] = sum(average_precisions.values()) / len(average_precisions)\n print(\"===============================\")\n\n results.append(result_dict)\n except Exception as e:\n print('skipping the evaluation of {} because following exception occurred: {}'.format(model_file, e))\n continue\n else:\n print('skipping the evaluation of {} since it\\'s not a .h5 file'.format(model_file))\n\n return results\n\n def _create_training_instances(self,\n train_annot_folder,\n train_image_folder,\n train_cache,\n valid_annot_folder,\n valid_image_folder,\n valid_cache,\n labels,\n ):\n\n # parse annotations of the training set\n train_ints, train_labels = parse_voc_annotation(train_annot_folder, train_image_folder, train_cache, labels)\n\n # parse annotations of the validation set, if any, otherwise split the training set\n\n if os.path.exists(valid_annot_folder):\n valid_ints, valid_labels = parse_voc_annotation(valid_annot_folder, valid_image_folder, valid_cache, labels)\n print('Evaluating over {} samples taken from {}'.format(len(valid_ints),\n os.path.dirname(valid_annot_folder)))\n else:\n\n train_portion = 0.8 # use 80% to train and the remaining 20% to evaluate\n train_valid_split = int(round(train_portion * len(train_ints)))\n np.random.seed(0)\n np.random.shuffle(train_ints)\n\n valid_ints = train_ints[train_valid_split:]\n train_ints = train_ints[:train_valid_split]\n print('Evaluating over {} samples taken as {:5.2f}% of the training set '\n 'given at {}'.format(len(valid_ints),\n (1 - train_portion)*100,\n os.path.dirname(train_annot_folder)))\n\n print('Training over {} samples given at {}'.format(len(train_ints), os.path.dirname(train_annot_folder)))\n\n # compare the seen labels with the given labels in config.json\n if len(labels) > 0:\n overlap_labels = set(labels).intersection(set(train_labels.keys()))\n\n # return None, None, None if some given label is not in the dataset\n if len(overlap_labels) < len(labels):\n if self.__training_mode:\n print('Some labels have no annotations! Please revise the list of labels in your configuration.')\n return None, None, None, None\n else:\n if self.__training_mode:\n print('No labels are provided. Train on all seen labels.')\n print(train_labels)\n\n labels = train_labels.keys()\n\n max_box_per_image = max([len(inst['object']) for inst in (train_ints + valid_ints)])\n\n return train_ints, valid_ints, sorted(labels), max_box_per_image\n\n def _create_callbacks(self, saved_weights_name, model_to_save):\n\n checkpoint = CustomModelCheckpoint(\n model_to_save=model_to_save,\n filepath=saved_weights_name + 'ex-{epoch:03d}--loss-{loss:08.3f}.h5',\n monitor='loss',\n verbose=0,\n save_best_only=True,\n mode='min',\n period=1\n )\n reduce_on_plateau = ReduceLROnPlateau(\n monitor='loss',\n factor=0.1,\n patience=2,\n verbose=0,\n mode='min',\n epsilon=0.01,\n cooldown=0,\n min_lr=0\n )\n tensor_board = TensorBoard(\n log_dir=self.__logs_directory\n )\n return [checkpoint, reduce_on_plateau, tensor_board]\n\n def _create_model(\n self,\n nb_class,\n anchors,\n max_box_per_image,\n max_grid, batch_size,\n warmup_batches,\n ignore_thresh,\n multi_gpu,\n lr,\n grid_scales,\n obj_scale,\n noobj_scale,\n xywh_scale,\n class_scale\n ):\n if len(multi_gpu) > 1:\n with tf.device('/cpu:0'):\n template_model, infer_model = yolov3_train(\n num_classes=nb_class,\n anchors=anchors,\n max_box_per_image=max_box_per_image,\n max_grid=max_grid,\n batch_size=batch_size // len(multi_gpu),\n warmup_batches=warmup_batches,\n ignore_thresh=ignore_thresh,\n grid_scales=grid_scales,\n obj_scale=obj_scale,\n noobj_scale=noobj_scale,\n xywh_scale=xywh_scale,\n class_scale=class_scale\n )\n else:\n template_model, infer_model = yolov3_train(\n num_classes=nb_class,\n anchors=anchors,\n max_box_per_image=max_box_per_image,\n max_grid=max_grid,\n batch_size=batch_size,\n warmup_batches=warmup_batches,\n ignore_thresh=ignore_thresh,\n grid_scales=grid_scales,\n obj_scale=obj_scale,\n noobj_scale=noobj_scale,\n xywh_scale=xywh_scale,\n class_scale=class_scale\n )\n\n # load the pretrained weight if exists, otherwise load the backend weight only\n\n if len(self.__pre_trained_model) > 3:\n if self.__training_mode:\n print(\"Training with transfer learning from pretrained Model\")\n template_model.load_weights(self.__pre_trained_model, by_name=True)\n else:\n if self.__training_mode:\n print(\"Pre-trained Model not provided. Transfer learning not in use.\")\n print(\"Training will start with 3 warmup experiments\")\n\n if len(multi_gpu) > 1:\n train_model = multi_gpu_model(template_model, gpus=multi_gpu)\n else:\n train_model = template_model\n\n optimizer = Adam(lr=lr, clipnorm=0.001)\n train_model.compile(loss=dummy_loss, optimizer=optimizer)\n\n return train_model, infer_model\n\n\nclass CustomObjectDetection:\n\n \"\"\"\n This is the object detection class for using your custom trained models. It supports your custom trained YOLOv3 model and allows to you to perform object detection in images.\n \"\"\"\n\n def __init__(self):\n self.__model_type = \"\"\n self.__model_path = \"\"\n self.__model_labels = []\n self.__model_anchors = []\n self.__detection_config_json_path = \"\"\n self.__input_size = 416\n self.__object_threshold = 0.4\n self.__nms_threshold = 0.4\n self.__model = None\n self.__detection_utils = CustomDetectionUtils(labels=[])\n\n def setModelTypeAsYOLOv3(self):\n \"\"\"\n 'setModelTypeAsYOLOv3' is used to set your custom detection model as YOLOv3\n :return:\n \"\"\"\n self.__model_type = \"yolov3\"\n\n def setModelPath(self, detection_model_path):\n \"\"\"\n 'setModelPath' is used to specify the filepath to your custom detection model\n :param detection_model_path: path to the .h5 model file.\n Usually is one of those under /models/detection_model-ex-ddd--loss-dddd.ddd.h5\n :return: None\n \"\"\"\n self.__model_path = detection_model_path\n\n def setJsonPath(self, configuration_json):\n \"\"\"\n 'setJsonPath' is used to set the filepath to the configuration JSON file for your custom detection model\n :param configuration_json: path to the .json file. Usually it is /json/detection_config.json\n :return: None\n \"\"\"\n self.__detection_config_json_path = configuration_json\n\n def loadModel(self):\n\n \"\"\"\n 'loadModel' is used to load the model into the CustomObjectDetection class\n :return: None\n \"\"\"\n\n if self.__model_type == \"yolov3\":\n detection_model_json = json.load(open(self.__detection_config_json_path))\n\n self.__model_labels = detection_model_json[\"labels\"]\n self.__model_anchors = detection_model_json[\"anchors\"]\n\n self.__detection_utils = CustomDetectionUtils(labels=self.__model_labels)\n\n self.__model = yolov3_main(Input(shape=(None, None, 3)), 3, len(self.__model_labels))\n\n self.__model.load_weights(self.__model_path)\n\n def detectObjectsFromImage(self, input_image=\"\", output_image_path=\"\", input_type=\"file\", output_type=\"file\",\n extract_detected_objects=False, minimum_percentage_probability=50, nms_treshold=0.4,\n display_percentage_probability=True, display_object_name=True, thread_safe=False):\n\n \"\"\"\n\n 'detectObjectsFromImage()' function is used to detect objects observable in the given image:\n * input_image , which can be a filepath or image numpy array in BGR\n * output_image_path (only if output_type = file) , file path to the output image that will contain the detection boxes and label, if output_type=\"file\"\n * input_type (optional) , filepath/numpy array of the image. Acceptable values are \"file\" and \"array\"\n * output_type (optional) , file path/numpy array/image file stream of the image. Acceptable values are \"file\" and \"array\"\n * extract_detected_objects (optional) , option to save each object detected individually as an image and return an array of the objects' image path.\n * minimum_percentage_probability (optional, 30 by default) , option to set the minimum percentage probability for nominating a detected object for output.\n * nms_threshold (optional, o.45 by default) , option to set the Non-maximum suppression for the detection\n * display_percentage_probability (optional, True by default), option to show or hide the percentage probability of each object in the saved/returned detected image\n * display_display_object_name (optional, True by default), option to show or hide the name of each object in the saved/returned detected image\n * thread_safe (optional, False by default), enforce the loaded detection model works across all threads if set to true, made possible by forcing all Keras inference to run on the default graph\n\n\n The values returned by this function depends on the parameters parsed. The possible values returnable\n are stated as below\n - If extract_detected_objects = False or at its default value and output_type = 'file' or\n at its default value, you must parse in the 'output_image_path' as a string to the path you want\n the detected image to be saved. Then the function will return:\n 1. an array of dictionaries, with each dictionary corresponding to the objects\n detected in the image. Each dictionary contains the following property:\n * name (string)\n * percentage_probability (float)\n * box_points (list of x1,y1,x2 and y2 coordinates)\n\n - If extract_detected_objects = False or at its default value and output_type = 'array' ,\n Then the function will return:\n\n 1. a numpy array of the detected image\n 2. an array of dictionaries, with each dictionary corresponding to the objects\n detected in the image. Each dictionary contains the following property:\n * name (string)\n * percentage_probability (float)\n * box_points (list of x1,y1,x2 and y2 coordinates)\n\n - If extract_detected_objects = True and output_type = 'file' or\n at its default value, you must parse in the 'output_image_path' as a string to the path you want\n the detected image to be saved. Then the function will return:\n 1. an array of dictionaries, with each dictionary corresponding to the objects\n detected in the image. Each dictionary contains the following property:\n * name (string)\n * percentage_probability (float)\n * box_points (list of x1,y1,x2 and y2 coordinates)\n 2. an array of string paths to the image of each object extracted from the image\n\n - If extract_detected_objects = True and output_type = 'array', the the function will return:\n 1. a numpy array of the detected image\n 2. an array of dictionaries, with each dictionary corresponding to the objects\n detected in the image. Each dictionary contains the following property:\n * name (string)\n * percentage_probability (float)\n * box_points (list of x1,y1,x2 and y2 coordinates)\n 3. an array of numpy arrays of each object detected in the image\n\n :param input_image:\n :param output_image_path:\n :param input_type:\n :param output_type:\n :param extract_detected_objects:\n :param minimum_percentage_probability:\n :param nms_treshold:\n :param display_percentage_probability:\n :param display_object_name:\n :param thread_safe:\n :return image_frame:\n :return output_objects_array:\n :return detected_objects_image_array:\n \"\"\"\n\n if self.__model is None:\n raise ValueError(\"You must call the loadModel() function before making object detection.\")\n else:\n if output_type == \"file\":\n # from the image file, lets keep the directory and the filename, but remove its format\n # if output_image_path is path/to/the/output/image.png\n # then output_image_folder is path/to/the/output/image\n # let's check if it is in the appropriated format soon to fail early\n output_image_folder, n_subs = re.subn(r'\\.(?:jpe?g|png|tif|webp|PPM|PGM)$', '', output_image_path, flags=re.I)\n if n_subs == 0:\n # if no substitution was done, the given output_image_path is not in a supported format,\n # raise an error\n raise ValueError(\"output_image_path must be the path where to write the image. \"\n \"Therefore it must end as one the following: \"\n \"'.jpg', '.png', '.tif', '.webp', '.PPM', '.PGM'. {} found\".format(output_image_path))\n elif extract_detected_objects:\n # Results must be written as files and need to extract detected objects as images,\n # let's create a folder to store the object's images\n objects_dir = output_image_folder + \"-objects\"\n\n os.makedirs(objects_dir, exist_ok=True)\n\n self.__object_threshold = minimum_percentage_probability / 100\n self.__nms_threshold = nms_treshold\n\n output_objects_array = []\n detected_objects_image_array = []\n\n if input_type == \"file\":\n image = cv2.imread(input_image)\n elif input_type == \"array\":\n image = input_image\n else:\n raise ValueError(\"input_type must be 'file' or 'array'. {} found\".format(input_type))\n\n image_frame = image.copy()\n\n height, width, channels = image.shape\n\n image = cv2.resize(image, (self.__input_size, self.__input_size))\n\n image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n image = image.astype(\"float32\") / 255.\n\n # expand the image to batch\n image = np.expand_dims(image, 0)\n\n if self.__model_type == \"yolov3\":\n if thread_safe == True:\n with K.get_session().graph.as_default():\n yolo_results = self.__model.predict(image)\n else:\n yolo_results = self.__model.predict(image)\n\n boxes = list()\n\n for idx, result in enumerate(yolo_results):\n box_set = self.__detection_utils.decode_netout(result[0], self.__model_anchors[idx],\n self.__object_threshold, self.__input_size,\n self.__input_size)\n boxes += box_set\n\n self.__detection_utils.correct_yolo_boxes(boxes, height, width, self.__input_size, self.__input_size)\n\n self.__detection_utils.do_nms(boxes, self.__nms_threshold)\n\n all_boxes, all_labels, all_scores = self.__detection_utils.get_boxes(boxes, self.__model_labels,\n self.__object_threshold)\n\n for object_box, object_label, object_score in zip(all_boxes, all_labels, all_scores):\n each_object_details = dict()\n each_object_details[\"name\"] = object_label\n each_object_details[\"percentage_probability\"] = object_score\n\n if object_box.xmin < 0:\n object_box.xmin = 0\n if object_box.ymin < 0:\n object_box.ymin = 0\n\n each_object_details[\"box_points\"] = [object_box.xmin, object_box.ymin, object_box.xmax, object_box.ymax]\n output_objects_array.append(each_object_details)\n\n drawn_image = self.__detection_utils.draw_boxes_and_caption(image_frame.copy(), all_boxes, all_labels,\n all_scores, show_names=display_object_name,\n show_percentage=display_percentage_probability)\n\n if extract_detected_objects:\n\n for cnt, each_object in enumerate(output_objects_array):\n\n splitted_image = image_frame[each_object[\"box_points\"][1]:each_object[\"box_points\"][3],\n each_object[\"box_points\"][0]:each_object[\"box_points\"][2]]\n if output_type == \"file\":\n splitted_image_path = os.path.join(objects_dir, \"{}-{:05d}.jpg\".format(each_object[\"name\"],\n cnt))\n\n cv2.imwrite(splitted_image_path, splitted_image)\n detected_objects_image_array.append(splitted_image_path)\n elif output_type == \"array\":\n detected_objects_image_array.append(splitted_image.copy())\n\n if output_type == \"file\":\n # we already validated that the output_image_path is a supported by OpenCV one\n cv2.imwrite(output_image_path, drawn_image)\n\n if extract_detected_objects:\n if output_type == \"file\":\n return output_objects_array, detected_objects_image_array\n elif output_type == \"array\":\n return drawn_image, output_objects_array, detected_objects_image_array\n\n else:\n if output_type == \"file\":\n return output_objects_array\n elif output_type == \"array\":\n return drawn_image, output_objects_array\n\n\nclass CustomVideoObjectDetection:\n\n\n \"\"\"\n\n This is the object detection class for videos and camera live stream inputs using your custom trained detection models. It provides support for your custom YOLOv3 models.\n\n \"\"\"\n\n def __init__(self):\n self.__model_type = \"\"\n self.__model_path = \"\"\n self.__model_labels = []\n self.__model_anchors = []\n self.__detection_config_json_path = \"\"\n self.__model_loaded = False\n self.__input_size = 416\n self.__object_threshold = 0.4\n self.__nms_threshold = 0.4\n self.__detector = []\n self.__detection_utils = CustomDetectionUtils(labels=[])\n\n def setModelTypeAsYOLOv3(self):\n\n \"\"\"\n 'setModelTypeAsYOLOv3' is used to set your custom detection model as YOLOv3\n :return:\n \"\"\"\n\n self.__model_type = \"yolov3\"\n\n\n def setModelPath(self, detection_model_path):\n \"\"\"\n 'setModelPath' is used to specify the filepath to your custom detection model\n\n :param detection_model_path:\n :return:\n \"\"\"\n self.__model_path = detection_model_path\n\n\n def setJsonPath(self, configuration_json):\n \"\"\"\n 'setJsonPath' is used to set the filepath to the configuration JSON file for your custom detection model\n\n :param configuration_json:\n :return:\n \"\"\"\n self.__detection_config_json_path = configuration_json\n\n def loadModel(self):\n \"\"\"\n 'loadModel' is used to load the model into the CustomVideoObjectDetection class\n\n :return:\n \"\"\"\n\n if (self.__model_loaded == False):\n if(self.__model_type == \"yolov3\"):\n detector = CustomObjectDetection()\n detector.setModelTypeAsYOLOv3()\n detector.setModelPath(self.__model_path)\n detector.setJsonPath(self.__detection_config_json_path)\n detector.loadModel()\n\n self.__detector = detector\n self.__model_loaded = True\n\n\n def detectObjectsFromVideo(self, input_file_path=\"\", camera_input=None, output_file_path=\"\", frames_per_second=20,\n frame_detection_interval=1, minimum_percentage_probability=50, log_progress=False,\n display_percentage_probability=True, display_object_name=True, save_detected_video=True,\n per_frame_function=None, per_second_function=None, per_minute_function=None,\n video_complete_function=None, return_detected_frame=False, detection_timeout = None):\n\n\n\n\n \"\"\"\n\n 'detectObjectsFromVideo()' function is used to detect objects observable in the given video path or a camera input:\n * input_file_path , which is the file path to the input video. It is required only if 'camera_input' is not set\n * camera_input , allows you to parse in camera input for live video detections\n * output_file_path , which is the path to the output video. It is required only if 'save_detected_video' is not set to False\n * frames_per_second , which is the number of frames to be used in the output video\n * frame_detection_interval (optional, 1 by default) , which is the intervals of frames that will be detected.\n * minimum_percentage_probability (optional, 50 by default) , option to set the minimum percentage probability for nominating a detected object for output.\n * log_progress (optional) , which states if the progress of the frame processed is to be logged to console\n * display_percentage_probability (optional), can be used to hide or show probability scores on the detected video frames\n * display_object_name (optional), can be used to show or hide object names on the detected video frames\n * save_save_detected_video (optional, True by default), can be set to or not to save the detected video\n * per_frame_function (optional), this parameter allows you to parse in a function you will want to execute after each frame of the video is detected. If this parameter is set to a function, after every video frame is detected, the function will be executed with the following values parsed into it:\n -- position number of the frame\n -- an array of dictinaries, with each dictinary corresponding to each object detected. Each dictionary contains 'name', 'percentage_probability' and 'box_points'\n -- a dictionary with with keys being the name of each unique objects and value are the number of instances of the object present\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed as the fourth value into the function\n\n * per_second_function (optional), this parameter allows you to parse in a function you will want to execute after each second of the video is detected. If this parameter is set to a function, after every second of a video is detected, the function will be executed with the following values parsed into it:\n -- position number of the second\n -- an array of dictionaries whose keys are position number of each frame present in the last second , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n -- an array of dictionaries, with each dictionary corresponding to each frame in the past second, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n -- a dictionary with its keys being the name of each unique object detected throughout the past second, and the key values are the average number of instances of the object found in all the frames contained in the past second\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed\n as the fifth value into the function\n\n * per_minute_function (optional), this parameter allows you to parse in a function you will want to execute after each minute of the video is detected. If this parameter is set to a function, after every minute of a video is detected, the function will be executed with the following values parsed into it:\n -- position number of the minute\n -- an array of dictionaries whose keys are position number of each frame present in the last minute , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n\n -- an array of dictionaries, with each dictionary corresponding to each frame in the past minute, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n\n -- a dictionary with its keys being the name of each unique object detected throughout the past minute, and the key values are the average number of instances of the object found in all the frames contained in the past minute\n\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed as the fifth value into the function\n\n * video_complete_function (optional), this parameter allows you to parse in a function you will want to execute after all of the video frames have been detected. If this parameter is set to a function, after all of frames of a video is detected, the function will be executed with the following values parsed into it:\n -- an array of dictionaries whose keys are position number of each frame present in the entire video , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n -- an array of dictionaries, with each dictionary corresponding to each frame in the entire video, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n -- a dictionary with its keys being the name of each unique object detected throughout the entire video, and the key values are the average number of instances of the object found in all the frames contained in the entire video\n\n * return_detected_frame (optionally, False by default), option to obtain the return the last detected video frame into the per_per_frame_function, per_per_second_function or per_per_minute_function\n\n * detection_timeout (optionally, None by default), option to state the number of seconds of a video that should be detected after which the detection function stop processing the video\n\n :param input_file_path:\n :param camera_input:\n :param output_file_path:\n :param frames_per_second:\n :param frame_detection_interval:\n :param minimum_percentage_probability:\n :param log_progress:\n :param display_percentage_probability:\n :param display_object_name:\n :param save_detected_video:\n :param per_frame_function:\n :param per_second_function:\n :param per_minute_function:\n :param video_complete_function:\n :param return_detected_frame:\n :param detection_timeout:\n :return output_video_filepath:\n :return counting:\n :return output_objects_array:\n :return output_objects_count:\n :return detected_copy:\n :return this_second_output_object_array:\n :return this_second_counting_array:\n :return this_second_counting:\n :return this_minute_output_object_array:\n :return this_minute_counting_array:\n :return this_minute_counting:\n :return this_video_output_object_array:\n :return this_video_counting_array:\n :return this_video_counting:\n \"\"\"\n\n output_frames_dict = {}\n output_frames_count_dict = {}\n\n input_video = cv2.VideoCapture(input_file_path)\n if (camera_input != None):\n input_video = camera_input\n\n output_video_filepath = output_file_path + '.avi'\n\n frame_width = int(input_video.get(3))\n frame_height = int(input_video.get(4))\n output_video = cv2.VideoWriter(output_video_filepath, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),\n frames_per_second,\n (frame_width, frame_height))\n\n counting = 0\n predicted_numbers = None\n scores = None\n detections = None\n\n\n detection_timeout_count = 0\n video_frames_count = 0\n\n\n if(self.__model_type == \"yolov3\"):\n\n\n\n while (input_video.isOpened()):\n ret, frame = input_video.read()\n\n if (ret == True):\n\n detected_frame = frame.copy()\n\n video_frames_count += 1\n if (detection_timeout != None):\n if ((video_frames_count % frames_per_second) == 0):\n detection_timeout_count += 1\n\n if (detection_timeout_count >= detection_timeout):\n break\n\n output_objects_array = []\n\n counting += 1\n\n if (log_progress == True):\n print(\"Processing Frame : \", str(counting))\n\n\n\n check_frame_interval = counting % frame_detection_interval\n\n if (counting == 1 or check_frame_interval == 0):\n try:\n detected_frame, output_objects_array = self.__detector.detectObjectsFromImage(\n input_image=frame, input_type=\"array\", output_type=\"array\",\n minimum_percentage_probability=minimum_percentage_probability,\n display_percentage_probability=display_percentage_probability,\n display_object_name=display_object_name)\n except:\n None\n\n\n output_frames_dict[counting] = output_objects_array\n\n output_objects_count = {}\n for eachItem in output_objects_array:\n eachItemName = eachItem[\"name\"]\n try:\n output_objects_count[eachItemName] = output_objects_count[eachItemName] + 1\n except:\n output_objects_count[eachItemName] = 1\n\n output_frames_count_dict[counting] = output_objects_count\n\n\n if (save_detected_video == True):\n output_video.write(detected_frame)\n\n if (counting == 1 or check_frame_interval == 0):\n if (per_frame_function != None):\n if (return_detected_frame == True):\n per_frame_function(counting, output_objects_array, output_objects_count,\n detected_frame)\n elif (return_detected_frame == False):\n per_frame_function(counting, output_objects_array, output_objects_count)\n\n if (per_second_function != None):\n if (counting != 1 and (counting % frames_per_second) == 0):\n\n this_second_output_object_array = []\n this_second_counting_array = []\n this_second_counting = {}\n\n for aa in range(counting):\n if (aa >= (counting - frames_per_second)):\n this_second_output_object_array.append(output_frames_dict[aa + 1])\n this_second_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_second_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_second_counting[eachItem] = this_second_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_second_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_second_counting:\n this_second_counting[eachCountingItem] = int(this_second_counting[eachCountingItem] / frames_per_second)\n\n if (return_detected_frame == True):\n per_second_function(int(counting / frames_per_second),\n this_second_output_object_array, this_second_counting_array,\n this_second_counting, detected_frame)\n\n elif (return_detected_frame == False):\n per_second_function(int(counting / frames_per_second),\n this_second_output_object_array, this_second_counting_array,\n this_second_counting)\n\n if (per_minute_function != None):\n\n if (counting != 1 and (counting % (frames_per_second * 60)) == 0):\n\n this_minute_output_object_array = []\n this_minute_counting_array = []\n this_minute_counting = {}\n\n for aa in range(counting):\n if (aa >= (counting - (frames_per_second * 60))):\n this_minute_output_object_array.append(output_frames_dict[aa + 1])\n this_minute_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_minute_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_minute_counting[eachItem] = this_minute_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_minute_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_minute_counting:\n this_minute_counting[eachCountingItem] = int(this_minute_counting[eachCountingItem] / (frames_per_second * 60))\n\n if (return_detected_frame == True):\n per_minute_function(int(counting / (frames_per_second * 60)),\n this_minute_output_object_array, this_minute_counting_array,\n this_minute_counting, detected_frame)\n\n elif (return_detected_frame == False):\n per_minute_function(int(counting / (frames_per_second * 60)),\n this_minute_output_object_array, this_minute_counting_array,\n this_minute_counting)\n\n\n else:\n break\n\n if (video_complete_function != None):\n\n this_video_output_object_array = []\n this_video_counting_array = []\n this_video_counting = {}\n\n for aa in range(counting):\n this_video_output_object_array.append(output_frames_dict[aa + 1])\n this_video_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_video_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_video_counting[eachItem] = this_video_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_video_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_video_counting:\n this_video_counting[eachCountingItem] = this_video_counting[\n eachCountingItem] / counting\n\n video_complete_function(this_video_output_object_array, this_video_counting_array,\n this_video_counting)\n\n input_video.release()\n output_video.release()\n\n if (save_detected_video == True):\n return output_video_filepath\n\n\nclass BoundBox:\n def __init__(self, xmin, ymin, xmax, ymax, objness=None, classes=None):\n self.xmin = xmin\n self.ymin = ymin\n self.xmax = xmax\n self.ymax = ymax\n self.objness = objness\n self.classes = classes\n self.label = -1\n self.score = -1\n\n def get_label(self):\n if self.label == -1:\n self.label = np.argmax(self.classes)\n\n return self.label\n\n def get_score(self):\n if self.score == -1:\n self.score = self.classes[self.get_label()]\n\n return self.score\n\n\nclass CustomDetectionUtils:\n def __init__(self, labels):\n self.__labels = labels\n self.__colors = []\n\n for i in range(len(labels)):\n color_space_values = np.random.randint(50, 255, size=(3,))\n red, green, blue = color_space_values\n red, green, blue = int(red), int(green), int(blue)\n self.__colors.append([red, green, blue])\n\n @staticmethod\n def _sigmoid(x):\n return 1. / (1. + np.exp(-x))\n\n def decode_netout(self, netout, anchors, obj_thresh, net_h, net_w):\n grid_h, grid_w = netout.shape[:2]\n nb_box = 3\n netout = netout.reshape((grid_h, grid_w, nb_box, -1))\n nb_class = netout.shape[-1] - 5\n boxes = []\n netout[..., :2] = self._sigmoid(netout[..., :2])\n netout[..., 4:] = self._sigmoid(netout[..., 4:])\n netout[..., 5:] = netout[..., 4][..., np.newaxis] * netout[..., 5:]\n netout[..., 5:] *= netout[..., 5:] > obj_thresh\n\n for row in range(grid_h):\n for col in range(grid_w):\n for b in range(nb_box):\n # 4th element is objectness score\n objectness = netout[row, col, b, 4]\n\n if objectness <= obj_thresh:\n continue\n\n # first 4 elements are x, y, w, and h\n x, y, w, h = netout[row, col, b, :4]\n x = (col + x) / grid_w # center position, unit: image width\n y = (row + y) / grid_h # center position, unit: image height\n w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width\n h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height\n # last elements are class probabilities\n classes = netout[row, col, b, 5:]\n box = BoundBox(x - w / 2, y - h / 2, x + w / 2, y + h / 2, objectness, classes)\n boxes.append(box)\n\n return boxes\n\n @staticmethod\n def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w):\n new_w, new_h = net_w, net_h\n for i in range(len(boxes)):\n x_offset, x_scale = (net_w - new_w) / 2. / net_w, float(new_w) / net_w\n y_offset, y_scale = (net_h - new_h) / 2. / net_h, float(new_h) / net_h\n boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w)\n boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w)\n boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h)\n boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h)\n\n def _interval_overlap(self, interval_a, interval_b):\n x1, x2 = interval_a\n x3, x4 = interval_b\n if x3 < x1:\n if x4 < x1:\n return 0\n else:\n return min(x2, x4) - x1\n else:\n if x2 < x3:\n return 0\n else:\n return min(x2, x4) - x3\n\n def bbox_iou(self, box1, box2):\n intersect_w = self._interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])\n intersect_h = self._interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax])\n intersect = intersect_w * intersect_h\n w1, h1 = box1.xmax - box1.xmin, box1.ymax - box1.ymin\n w2, h2 = box2.xmax - box2.xmin, box2.ymax - box2.ymin\n union = w1 * h1 + w2 * h2 - intersect\n\n try:\n result = float(intersect) / float(union)\n return result\n except:\n return 0.0\n\n def do_nms(self, boxes, nms_thresh):\n if len(boxes) > 0:\n nb_class = len(boxes[0].classes)\n else:\n return\n\n for c in range(nb_class):\n sorted_indices = np.argsort([-box.classes[c] for box in boxes])\n\n for i in range(len(sorted_indices)):\n index_i = sorted_indices[i]\n\n if boxes[index_i].classes[c] == 0: continue\n\n for j in range(i + 1, len(sorted_indices)):\n index_j = sorted_indices[j]\n\n if self.bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh:\n boxes[index_j].classes[c] = 0\n\n def get_boxes(self, boxes, labels, thresh):\n v_boxes, v_labels, v_scores = list(), list(), list()\n # enumerate all boxes\n for box in boxes:\n # enumerate all possible labels\n for i in range(len(labels)):\n # check if the threshold for this label is high enough\n if box.classes[i] > thresh:\n v_boxes.append(box)\n v_labels.append(labels[i])\n v_scores.append(box.classes[i] * 100)\n # don't break, many labels may trigger for one box\n return v_boxes, v_labels, v_scores\n\n def label_color(self, label):\n \"\"\" Return a color from a set of predefined colors. Contains 80 colors in total.\n\n Args\n label: The label to get the color for.\n\n Returns\n A list of three values representing a RGB color.\n\n If no color is defined for a certain label, the color green is returned and a warning is printed.\n \"\"\"\n if label < len(self.__colors):\n return self.__colors[label]\n else:\n return 0, 255, 0\n\n def draw_boxes_and_caption(self, image_frame, v_boxes, v_labels, v_scores, show_names=False, show_percentage=False):\n\n for i in range(len(v_boxes)):\n box = v_boxes[i]\n y1, x1, y2, x2 = box.ymin, box.xmin, box.ymax, box.xmax\n width, height = x2 - x1, y2 - y1\n class_color = self.label_color(self.__labels.index(v_labels[i]))\n\n image_frame = cv2.rectangle(image_frame, (x1, y1), (x2, y2), class_color, 2)\n\n label = \"\"\n if show_names and show_percentage:\n label = \"%s : %.3f\" % (v_labels[i], v_scores[i])\n elif show_names:\n label = \"%s\" % (v_labels[i])\n elif show_percentage:\n label = \"%.3f\" % (v_scores[i])\n\n if show_names or show_percentage:\n b = np.array([x1, y1, x2, y2]).astype(int)\n cv2.putText(image_frame, label, (b[0], b[1] - 10), cv2.FONT_HERSHEY_PLAIN, 1, (200, 0, 0), 3)\n cv2.putText(image_frame, label, (b[0], b[1] - 10), cv2.FONT_HERSHEY_PLAIN, 1, (255, 255, 255), 2)\n\n return image_frame\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/callbacks.py", "content": "from tensorflow.keras.callbacks import TensorBoard, ModelCheckpoint\nimport tensorflow as tf\nimport numpy as np\nimport warnings\n\nclass CustomTensorBoard(TensorBoard):\n \"\"\" to log the loss after each batch\n \"\"\" \n def __init__(self, log_every=1, **kwargs):\n super(CustomTensorBoard, self).__init__(**kwargs)\n self.log_every = log_every\n self.counter = 0\n \n def on_batch_end(self, batch, logs=None):\n self.counter+=1\n if self.counter%self.log_every==0:\n for name, value in logs.items():\n if name in ['batch', 'size']:\n continue\n summary = tf.Summary()\n summary_value = summary.value.add()\n summary_value.simple_value = value.item()\n summary_value.tag = name\n self.writer.add_summary(summary, self.counter)\n self.writer.flush()\n \n super(CustomTensorBoard, self).on_batch_end(batch, logs)\n\nclass CustomModelCheckpoint(ModelCheckpoint):\n \"\"\" to save the template model, not the multi-GPU model\n \"\"\"\n def __init__(self, model_to_save, **kwargs):\n super(CustomModelCheckpoint, self).__init__(**kwargs)\n self.model_to_save = model_to_save\n\n def on_epoch_end(self, epoch, logs=None):\n logs = logs or {}\n self.epochs_since_last_save += 1\n if self.epochs_since_last_save >= self.period:\n self.epochs_since_last_save = 0\n filepath = self.filepath.format(epoch=epoch + 1, **logs)\n if self.save_best_only:\n current = logs.get(self.monitor)\n if current is None:\n warnings.warn('Can save best model only with %s available, '\n 'skipping.' % (self.monitor), RuntimeWarning)\n else:\n if self.monitor_op(current, self.best):\n if self.verbose > 0:\n print('\\nEpoch %05d: %s improved from %0.5f to %0.5f,'\n ' saving model to %s'\n % (epoch + 1, self.monitor, self.best,\n current, filepath))\n self.best = current\n if self.save_weights_only:\n self.model_to_save.save_weights(filepath, overwrite=True)\n else:\n self.model_to_save.save(filepath, overwrite=True)\n else:\n if self.verbose > 0:\n print('\\nEpoch %05d: %s did not improve from %0.5f' %\n (epoch + 1, self.monitor, self.best))\n else:\n if self.verbose > 0:\n print('\\nEpoch %05d: saving model to %s' % (epoch + 1, filepath))\n if self.save_weights_only:\n self.model_to_save.save_weights(filepath, overwrite=True)\n else:\n self.model_to_save.save(filepath, overwrite=True)\n\n super(CustomModelCheckpoint, self).on_batch_end(epoch, logs)" }, { "path": "imageai_tf_deprecated/Detection/Custom/evaluate.py", "content": "#! /usr/bin/env python\n\nimport argparse\nimport os\nimport json\nfrom imageai.Detection.Custom.voc import parse_voc_annotation\nfrom imageai.Detection.Custom.generator import BatchGenerator\nfrom imageai.Detection.Custom.utils.utils import normalize, evaluate\nfrom keras.models import load_model\n\n\ndef _main_(args):\n config_path = args.conf\n\n with open(config_path) as config_buffer: \n config = json.loads(config_buffer.read())\n\n ###############################\n # Create the validation generator\n ############################### \n valid_ints, labels = parse_voc_annotation(\n config['valid']['valid_annot_folder'], \n config['valid']['valid_image_folder'], \n config['valid']['cache_name'],\n config['model']['labels']\n )\n\n labels = labels.keys() if len(config['model']['labels']) == 0 else config['model']['labels']\n labels = sorted(labels)\n \n valid_generator = BatchGenerator(\n instances = valid_ints, \n anchors = config['model']['anchors'], \n labels = labels, \n downsample = 32, # ratio between network input's size and network output's size, 32 for YOLOv3\n max_box_per_image = 0,\n batch_size = config['train']['batch_size'],\n min_net_size = config['model']['min_input_size'],\n max_net_size = config['model']['max_input_size'], \n shuffle = True, \n jitter = 0.0, \n norm = normalize\n )\n\n ###############################\n # Load the model and do evaluation\n ###############################\n os.environ['CUDA_VISIBLE_DEVICES'] = config['train']['gpus']\n\n infer_model = load_model(config['train']['saved_weights_name'])\n\n # compute mAP for all the classes\n average_precisions = evaluate(infer_model, valid_generator)\n\n # print the score\n for label, average_precision in average_precisions.items():\n print(labels[label] + ': {:.4f}'.format(average_precision))\n print('mAP: {:.4f}'.format(sum(average_precisions.values()) / len(average_precisions))) \n\n\nif __name__ == '__main__':\n argparser = argparse.ArgumentParser(description='Evaluate YOLO_v3 model on any dataset')\n argparser.add_argument('-c', '--conf', help='path to configuration file') \n \n args = argparser.parse_args()\n _main_(args)\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/gen_anchors.py", "content": "import random\nimport numpy as np\n\nfrom imageai.Detection.Custom.voc import parse_voc_annotation\n\n\ndef IOU(ann, centroids):\n w, h = ann\n similarities = []\n\n for centroid in centroids:\n c_w, c_h = centroid\n\n if c_w >= w and c_h >= h:\n similarity = w*h/(c_w*c_h)\n elif c_w >= w and c_h <= h:\n similarity = w*c_h/(w*h + (c_w-w)*c_h)\n elif c_w <= w and c_h >= h:\n similarity = c_w*h/(w*h + c_w*(c_h-h))\n else: #means both w,h are bigger than c_w and c_h respectively\n similarity = (c_w*c_h)/(w*h)\n similarities.append(similarity) # will become (k,) shape\n\n return np.array(similarities)\n\n\ndef avg_IOU(anns, centroids):\n n,d = anns.shape\n sum = 0.\n\n for i in range(anns.shape[0]):\n sum+= max(IOU(anns[i], centroids))\n\n return sum/n\n\n\ndef run_kmeans(ann_dims, anchor_num):\n ann_num = ann_dims.shape[0]\n iterations = 0\n prev_assignments = np.ones(ann_num)*(-1)\n iteration = 0\n old_distances = np.zeros((ann_num, anchor_num))\n\n indices = [random.randrange(ann_dims.shape[0]) for i in range(anchor_num)]\n centroids = ann_dims[indices]\n anchor_dim = ann_dims.shape[1]\n\n while True:\n distances = []\n iteration += 1\n for i in range(ann_num):\n d = 1 - IOU(ann_dims[i], centroids)\n distances.append(d)\n distances = np.array(distances) # distances.shape = (ann_num, anchor_num)\n\n #assign samples to centroids\n assignments = np.argmin(distances,axis=1)\n\n if (assignments == prev_assignments).all() :\n return centroids\n\n #calculate new centroids\n centroid_sums=np.zeros((anchor_num, anchor_dim), np.float)\n for i in range(ann_num):\n centroid_sums[assignments[i]]+=ann_dims[i]\n for j in range(anchor_num):\n centroids[j] = centroid_sums[j]/(np.sum(assignments==j) + 1e-6)\n\n prev_assignments = assignments.copy()\n old_distances = distances.copy()\n\n\ndef generateAnchors(train_annotation_folder, train_image_folder, train_cache_file, model_labels):\n\n print(\"Generating anchor boxes for training images and annotation...\")\n num_anchors = 9\n\n train_imgs, train_labels = parse_voc_annotation(\n train_annotation_folder,\n train_image_folder,\n train_cache_file,\n model_labels\n )\n\n # run k_mean to find the anchors\n annotation_dims = []\n for image in train_imgs:\n\n for obj in image['object']:\n relative_w = (float(obj['xmax']) - float(obj['xmin']))/image['width']\n relative_h = (float(obj[\"ymax\"]) - float(obj['ymin']))/image['height']\n annotation_dims.append(tuple(map(float, (relative_w,relative_h))))\n\n annotation_dims = np.array(annotation_dims)\n centroids = run_kmeans(annotation_dims, num_anchors)\n\n # write anchors to file\n print('Average IOU for', num_anchors, 'anchors:', '%0.2f' % avg_IOU(annotation_dims, centroids))\n\n anchors = centroids.copy()\n\n widths = anchors[:, 0]\n sorted_indices = np.argsort(widths)\n\n anchor_array = []\n reverse_anchor_array = []\n out_string = \"\"\n r = \"anchors: [\"\n for i in sorted_indices:\n anchor_array.append(int(anchors[i, 0] * 416))\n anchor_array.append(int(anchors[i, 1] * 416))\n\n out_string += str(int(anchors[i, 0] * 416)) + ',' + str(int(anchors[i, 1] * 416)) + ', '\n\n reverse_anchor_array.append(anchor_array[12:18])\n reverse_anchor_array.append(anchor_array[6:12])\n reverse_anchor_array.append(anchor_array[0:6])\n\n print(\"Anchor Boxes generated.\")\n return anchor_array, reverse_anchor_array\n\n\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/generator.py", "content": "import cv2\nimport copy\nimport numpy as np\nfrom tensorflow.keras.utils import Sequence\nfrom imageai.Detection.Custom.utils.bbox import BoundBox, bbox_iou\nfrom imageai.Detection.Custom.utils.image import apply_random_scale_and_crop, random_distort_image, random_flip, correct_bounding_boxes\n\nclass BatchGenerator(Sequence):\n def __init__(self, \n instances, \n anchors, \n labels, \n downsample=32, # ratio between network input's size and network output's size, 32 for YOLOv3\n max_box_per_image=30,\n batch_size=1,\n min_net_size=320,\n max_net_size=608, \n shuffle=True, \n jitter=True, \n norm=None\n ):\n self.instances = instances\n self.batch_size = batch_size\n self.labels = labels\n self.downsample = downsample\n self.max_box_per_image = max_box_per_image\n self.min_net_size = (min_net_size//self.downsample)*self.downsample\n self.max_net_size = (max_net_size//self.downsample)*self.downsample\n self.shuffle = shuffle\n self.jitter = jitter\n self.norm = norm\n self.anchors = [BoundBox(0, 0, anchors[2*i], anchors[2*i+1]) for i in range(len(anchors)//2)]\n self.net_h = 416 \n self.net_w = 416\n\n if shuffle: np.random.shuffle(self.instances)\n \n def __len__(self):\n return int(np.ceil(float(len(self.instances))/self.batch_size)) \n\n def __getitem__(self, idx):\n # get image input size, change every 10 batches\n net_h, net_w = self._get_net_size(idx)\n base_grid_h, base_grid_w = net_h//self.downsample, net_w//self.downsample\n\n # determine the first and the last indices of the batch\n l_bound = idx * self.batch_size\n r_bound = (idx+1) * self.batch_size\n\n if r_bound > len(self.instances):\n r_bound = len(self.instances)\n l_bound = r_bound - self.batch_size\n\n x_batch = np.zeros((r_bound - l_bound, net_h, net_w, 3)) # input images\n t_batch = np.zeros((r_bound - l_bound, 1, 1, 1, self.max_box_per_image, 4)) # list of groundtruth boxes\n\n # initialize the inputs and the outputs\n yolo_1 = np.zeros((r_bound - l_bound, 1*base_grid_h, 1*base_grid_w, len(self.anchors)//3, 4+1+len(self.labels))) # desired network output 1\n yolo_2 = np.zeros((r_bound - l_bound, 2*base_grid_h, 2*base_grid_w, len(self.anchors)//3, 4+1+len(self.labels))) # desired network output 2\n yolo_3 = np.zeros((r_bound - l_bound, 4*base_grid_h, 4*base_grid_w, len(self.anchors)//3, 4+1+len(self.labels))) # desired network output 3\n yolos = [yolo_3, yolo_2, yolo_1]\n\n dummy_yolo_1 = np.zeros((r_bound - l_bound, 1))\n dummy_yolo_2 = np.zeros_like(dummy_yolo_1)\n dummy_yolo_3 = np.zeros_like(dummy_yolo_1)\n \n instance_count = 0\n true_box_index = 0\n\n # do the logic to fill in the inputs and the output\n for train_instance in self.instances[l_bound:r_bound]:\n # augment input image and fix object's position and size\n img, all_objs = self._aug_image(train_instance, net_h, net_w)\n \n for obj in all_objs:\n # find the best anchor box for this object\n max_anchor = None \n max_index = -1\n max_iou = -1\n\n shifted_box = BoundBox(0, \n 0,\n obj['xmax']-obj['xmin'], \n obj['ymax']-obj['ymin']) \n \n for i in range(len(self.anchors)):\n anchor = self.anchors[i]\n iou = bbox_iou(shifted_box, anchor)\n\n if max_iou < iou:\n max_anchor = anchor\n max_index = i\n max_iou = iou \n \n # determine the yolo to be responsible for this bounding box\n yolo = yolos[max_index//3]\n grid_h, grid_w = yolo.shape[1:3]\n \n # determine the position of the bounding box on the grid\n center_x = .5*(obj['xmin'] + obj['xmax'])\n center_x = center_x / float(net_w) * grid_w # sigma(t_x) + c_x\n center_y = .5*(obj['ymin'] + obj['ymax'])\n center_y = center_y / float(net_h) * grid_h # sigma(t_y) + c_y\n \n # determine the sizes of the bounding box\n w = np.log((obj['xmax'] - obj['xmin']) / float(max_anchor.xmax)) # t_w\n h = np.log((obj['ymax'] - obj['ymin']) / float(max_anchor.ymax)) # t_h\n\n box = [center_x, center_y, w, h]\n\n # determine the index of the label\n obj_indx = self.labels.index(obj['name']) \n\n # determine the location of the cell responsible for this object\n grid_x = int(np.floor(center_x))\n grid_y = int(np.floor(center_y))\n\n # assign ground truth x, y, w, h, confidence and class probs to y_batch\n yolo[instance_count, grid_y, grid_x, max_index%3] = 0\n yolo[instance_count, grid_y, grid_x, max_index%3, 0:4] = box\n yolo[instance_count, grid_y, grid_x, max_index%3, 4 ] = 1.\n yolo[instance_count, grid_y, grid_x, max_index%3, 5+obj_indx] = 1\n\n # assign the true box to t_batch\n true_box = [center_x, center_y, obj['xmax'] - obj['xmin'], obj['ymax'] - obj['ymin']]\n t_batch[instance_count, 0, 0, 0, true_box_index] = true_box\n\n true_box_index += 1\n true_box_index = true_box_index % self.max_box_per_image \n\n # assign input image to x_batch\n if self.norm != None: \n x_batch[instance_count] = self.norm(img)\n else:\n # plot image and bounding boxes for sanity check\n for obj in all_objs:\n cv2.rectangle(img, (obj['xmin'],obj['ymin']), (obj['xmax'],obj['ymax']), (255,0,0), 3)\n cv2.putText(img, obj['name'], \n (obj['xmin']+2, obj['ymin']+12), \n 0, 1.2e-3 * img.shape[0], \n (0,255,0), 2)\n \n x_batch[instance_count] = img\n\n # increase instance counter in the current batch\n instance_count += 1 \n \n return [x_batch, t_batch, yolo_1, yolo_2, yolo_3], [dummy_yolo_1, dummy_yolo_2, dummy_yolo_3]\n\n def _get_net_size(self, idx):\n if idx % 10 == 0:\n net_size = self.downsample*np.random.randint(self.min_net_size/self.downsample, \\\n self.max_net_size/self.downsample+1)\n\n self.net_h, self.net_w = net_size, net_size\n return self.net_h, self.net_w\n \n def _aug_image(self, instance, net_h, net_w):\n image_name = instance['filename']\n image = cv2.imread(image_name) # BGR image\n \n if image is None:\n print('Cannot find ', image_name)\n\n image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # RGB image\n \n image_h, image_w, _ = image.shape\n \n # determine the amount of scaling and cropping\n dw = self.jitter * image_w\n dh = self.jitter * image_h\n\n new_ar = (image_w + np.random.uniform(-dw, dw)) / (image_h + np.random.uniform(-dh, dh))\n scale = np.random.uniform(0.25, 2)\n\n if new_ar < 1:\n new_h = int(scale * net_h)\n new_w = int(net_h * new_ar)\n else:\n new_w = int(scale * net_w)\n new_h = int(net_w / new_ar)\n \n dx = int(np.random.uniform(0, net_w - new_w))\n dy = int(np.random.uniform(0, net_h - new_h))\n \n # apply scaling and cropping\n im_sized = apply_random_scale_and_crop(image, new_w, new_h, net_w, net_h, dx, dy)\n \n # randomly distort hsv space\n im_sized = random_distort_image(im_sized)\n \n # randomly flip\n flip = np.random.randint(2)\n im_sized = random_flip(im_sized, flip)\n \n # correct the size and pos of bounding boxes\n all_objs = correct_bounding_boxes(instance['object'], new_w, new_h, net_w, net_h, dx, dy, flip, image_w, image_h)\n \n return im_sized, all_objs \n\n def on_epoch_end(self):\n if self.shuffle:\n np.random.shuffle(self.instances)\n \n def num_classes(self):\n return len(self.labels)\n\n def size(self):\n return len(self.instances) \n\n def get_anchors(self):\n anchors = []\n\n for anchor in self.anchors:\n anchors += [anchor.xmax, anchor.ymax]\n\n return anchors\n\n def load_annotation(self, i):\n annots = []\n\n for obj in self.instances[i]['object']:\n annot = [obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], self.labels.index(obj['name'])]\n annots += [annot]\n\n if len(annots) == 0:\n annots = [[]]\n\n return np.array(annots)\n\n def load_image(self, i):\n return cv2.imread(self.instances[i]['filename']) # BGR image\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/utils/__init__.py", "content": "" }, { "path": "imageai_tf_deprecated/Detection/Custom/utils/bbox.py", "content": "import numpy as np\nimport os\nimport cv2\nfrom .colors import get_color\n\nclass BoundBox:\n def __init__(self, xmin, ymin, xmax, ymax, c = None, classes = None):\n self.xmin = xmin\n self.ymin = ymin\n self.xmax = xmax\n self.ymax = ymax\n \n self.c = c\n self.classes = classes\n\n self.label = -1\n self.score = -1\n\n def get_label(self):\n if self.label == -1:\n self.label = np.argmax(self.classes)\n \n return self.label\n \n def get_score(self):\n if self.score == -1:\n self.score = self.classes[self.get_label()]\n \n return self.score \n\ndef _interval_overlap(interval_a, interval_b):\n x1, x2 = interval_a\n x3, x4 = interval_b\n\n if x3 < x1:\n if x4 < x1:\n return 0\n else:\n return min(x2,x4) - x1\n else:\n if x2 < x3:\n return 0\n else:\n return min(x2,x4) - x3 \n\ndef bbox_iou(box1, box2):\n intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])\n intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax]) \n \n intersect = intersect_w * intersect_h\n\n w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin\n w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin\n \n union = w1*h1 + w2*h2 - intersect\n\n if(union <= 0):\n union = 1\n\n return float(intersect) / float(union)\n\ndef draw_boxes(image, boxes, labels, obj_thresh, quiet=True):\n for box in boxes:\n label_str = ''\n label = -1\n \n for i in range(len(labels)):\n if box.classes[i] > obj_thresh:\n if label_str != '': label_str += ', '\n label_str += (labels[i] + ' ' + str(round(box.get_score()*100, 2)) + '%')\n label = i\n if not quiet: print(label_str)\n \n if label >= 0:\n text_size = cv2.getTextSize(label_str, cv2.FONT_HERSHEY_SIMPLEX, 1.1e-3 * image.shape[0], 5)\n width, height = text_size[0][0], text_size[0][1]\n region = np.array([[box.xmin-3, box.ymin], \n [box.xmin-3, box.ymin-height-26], \n [box.xmin+width+13, box.ymin-height-26], \n [box.xmin+width+13, box.ymin]], dtype='int32') \n\n cv2.rectangle(img=image, pt1=(box.xmin,box.ymin), pt2=(box.xmax,box.ymax), color=get_color(label), thickness=5)\n cv2.fillPoly(img=image, pts=[region], color=get_color(label))\n cv2.putText(img=image, \n text=label_str, \n org=(box.xmin+13, box.ymin - 13), \n fontFace=cv2.FONT_HERSHEY_SIMPLEX, \n fontScale=1e-3 * image.shape[0], \n color=(0,0,0), \n thickness=2)\n \n return image " }, { "path": "imageai_tf_deprecated/Detection/Custom/utils/colors.py", "content": "def get_color(label):\n \"\"\" Return a color from a set of predefined colors. Contains 80 colors in total.\n code originally from https://github.com/fizyr/keras-retinanet/\n Args\n label: The label to get the color for.\n Returns\n A list of three values representing a RGB color.\n \"\"\"\n if label < len(colors):\n return colors[label]\n else:\n print('Label {} has no color, returning default.'.format(label))\n return (0, 255, 0)\n\ncolors = [\n [31 , 0 , 255] ,\n [0 , 159 , 255] ,\n [255 , 95 , 0] ,\n [255 , 19 , 0] ,\n [255 , 0 , 0] ,\n [255 , 38 , 0] ,\n [0 , 255 , 25] ,\n [255 , 0 , 133] ,\n [255 , 172 , 0] ,\n [108 , 0 , 255] ,\n [0 , 82 , 255] ,\n [0 , 255 , 6] ,\n [255 , 0 , 152] ,\n [223 , 0 , 255] ,\n [12 , 0 , 255] ,\n [0 , 255 , 178] ,\n [108 , 255 , 0] ,\n [184 , 0 , 255] ,\n [255 , 0 , 76] ,\n [146 , 255 , 0] ,\n [51 , 0 , 255] ,\n [0 , 197 , 255] ,\n [255 , 248 , 0] ,\n [255 , 0 , 19] ,\n [255 , 0 , 38] ,\n [89 , 255 , 0] ,\n [127 , 255 , 0] ,\n [255 , 153 , 0] ,\n [0 , 255 , 255] ,\n [0 , 255 , 216] ,\n [0 , 255 , 121] ,\n [255 , 0 , 248] ,\n [70 , 0 , 255] ,\n [0 , 255 , 159] ,\n [0 , 216 , 255] ,\n [0 , 6 , 255] ,\n [0 , 63 , 255] ,\n [31 , 255 , 0] ,\n [255 , 57 , 0] ,\n [255 , 0 , 210] ,\n [0 , 255 , 102] ,\n [242 , 255 , 0] ,\n [255 , 191 , 0] ,\n [0 , 255 , 63] ,\n [255 , 0 , 95] ,\n [146 , 0 , 255] ,\n [184 , 255 , 0] ,\n [255 , 114 , 0] ,\n [0 , 255 , 235] ,\n [255 , 229 , 0] ,\n [0 , 178 , 255] ,\n [255 , 0 , 114] ,\n [255 , 0 , 57] ,\n [0 , 140 , 255] ,\n [0 , 121 , 255] ,\n [12 , 255 , 0] ,\n [255 , 210 , 0] ,\n [0 , 255 , 44] ,\n [165 , 255 , 0] ,\n [0 , 25 , 255] ,\n [0 , 255 , 140] ,\n [0 , 101 , 255] ,\n [0 , 255 , 82] ,\n [223 , 255 , 0] ,\n [242 , 0 , 255] ,\n [89 , 0 , 255] ,\n [165 , 0 , 255] ,\n [70 , 255 , 0] ,\n [255 , 0 , 172] ,\n [255 , 76 , 0] ,\n [203 , 255 , 0] ,\n [204 , 0 , 255] ,\n [255 , 0 , 229] ,\n [255 , 133 , 0] ,\n [127 , 0 , 255] ,\n [0 , 235 , 255] ,\n [0 , 255 , 197] ,\n [255 , 0 , 191] ,\n [0 , 44 , 255] ,\n [50 , 255 , 0]\n]\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/utils/image.py", "content": "import cv2\nimport numpy as np\nimport copy\n\n\ndef _rand_scale(scale):\n scale = np.random.uniform(1, scale)\n return scale if np.random.randint(2) == 0 else 1./scale\n\n\ndef _constrain(min_v, max_v, value):\n\n if value < min_v:\n return min_v\n\n if value > max_v:\n return max_v\n\n return value \n\n\ndef random_flip(image, flip):\n if flip == 1:\n return cv2.flip(image, 1)\n return image\n\n\ndef correct_bounding_boxes(boxes, new_w, new_h, net_w, net_h, dx, dy, flip, image_w, image_h):\n boxes = copy.deepcopy(boxes)\n\n # randomize boxes' order\n np.random.shuffle(boxes)\n\n # correct sizes and positions\n sx, sy = float(new_w)/image_w, float(new_h)/image_h\n zero_boxes = []\n\n for i in range(len(boxes)):\n boxes[i]['xmin'] = int(_constrain(0, net_w, boxes[i]['xmin']*sx + dx))\n boxes[i]['xmax'] = int(_constrain(0, net_w, boxes[i]['xmax']*sx + dx))\n boxes[i]['ymin'] = int(_constrain(0, net_h, boxes[i]['ymin']*sy + dy))\n boxes[i]['ymax'] = int(_constrain(0, net_h, boxes[i]['ymax']*sy + dy))\n\n if boxes[i]['xmax'] <= boxes[i]['xmin'] or boxes[i]['ymax'] <= boxes[i]['ymin']:\n zero_boxes += [i]\n continue\n\n if flip == 1:\n swap = boxes[i]['xmin']\n boxes[i]['xmin'] = net_w - boxes[i]['xmax']\n boxes[i]['xmax'] = net_w - swap\n\n boxes = [boxes[i] for i in range(len(boxes)) if i not in zero_boxes]\n\n return boxes\n\n\ndef random_distort_image(image, hue=18, saturation=1.5, exposure=1.5):\n # determine scale factors\n dhue = np.random.uniform(-hue, hue)\n dsat = _rand_scale(saturation)\n dexp = _rand_scale(exposure)\n\n # convert RGB space to HSV space\n image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV).astype('float')\n \n # change satuation and exposure\n image[:, :, 1] *= dsat\n image[:, :, 2] *= dexp\n \n # change hue\n image[:, :, 0] += dhue\n image[:, :, 0] -= (image[:, :, 0] > 180) * 180\n image[:, :, 0] += (image[:, :, 0] < 0) * 180\n \n # convert back to RGB from HSV\n return cv2.cvtColor(image.astype('uint8'), cv2.COLOR_HSV2RGB)\n\n\ndef apply_random_scale_and_crop(image, new_w, new_h, net_w, net_h, dx, dy):\n\n im_sized = cv2.resize(image, (new_w, new_h))\n \n if dx > 0: \n im_sized = np.pad(im_sized, ((0, 0), (dx, 0), (0, 0)), mode='constant', constant_values=127)\n else:\n im_sized = im_sized[:, -dx:, :]\n if (new_w + dx) < net_w:\n im_sized = np.pad(im_sized, ((0, 0), (0, net_w - (new_w+dx)), (0, 0)), mode='constant', constant_values=127)\n \n if dy > 0: \n im_sized = np.pad(im_sized, ((dy, 0), (0, 0), (0, 0)), mode='constant', constant_values=127)\n else:\n im_sized = im_sized[-dy:, :, :]\n \n if (new_h + dy) < net_h:\n im_sized = np.pad(im_sized, ((0, net_h - (new_h+dy)), (0, 0), (0, 0)), mode='constant', constant_values=127)\n \n return im_sized[:net_h, :net_w, :]\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/utils/multi_gpu_model.py", "content": "from keras.layers import Lambda, concatenate\nfrom keras.models import Model\nimport tensorflow as tf\n\n\ndef multi_gpu_model(model, gpus):\n if isinstance(gpus, (list, tuple)):\n num_gpus = len(gpus)\n target_gpu_ids = gpus\n else:\n num_gpus = gpus\n target_gpu_ids = range(num_gpus)\n\n def get_slice(data, i, parts):\n shape = tf.shape(data)\n batch_size = shape[:1]\n input_shape = shape[1:]\n step = batch_size // parts\n if i == num_gpus - 1:\n size = batch_size - step * i\n else:\n size = step\n size = tf.concat([size, input_shape], axis=0)\n stride = tf.concat([step, input_shape * 0], axis=0)\n start = stride * i\n return tf.slice(data, start, size)\n\n all_outputs = []\n for i in range(len(model.outputs)):\n all_outputs.append([])\n\n # Place a copy of the model on each GPU,\n # each getting a slice of the inputs.\n for i, gpu_id in enumerate(target_gpu_ids):\n with tf.device('/gpu:%d' % gpu_id):\n with tf.name_scope('replica_%d' % gpu_id):\n inputs = []\n # Retrieve a slice of the input.\n for x in model.inputs:\n input_shape = tuple(x.get_shape().as_list())[1:]\n slice_i = Lambda(get_slice, output_shape=input_shape,\n arguments={'i': i, 'parts': num_gpus})(x)\n inputs.append(slice_i)\n\n # Apply model on slice\n # (creating a model replica on the target device).\n outputs = model(inputs)\n if not isinstance(outputs, list):\n outputs = [outputs]\n\n # Save the outputs for merging back together later.\n for o in range(len(outputs)):\n all_outputs[o].append(outputs[o])\n\n # Merge outputs on CPU.\n with tf.device('/cpu:0'):\n merged = []\n for name, outputs in zip(model.output_names, all_outputs):\n merged.append(concatenate(outputs, axis=0, name=name))\n return Model(model.inputs, merged)\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/utils/utils.py", "content": "import cv2\nimport numpy as np\nimport os\nfrom .bbox import BoundBox, bbox_iou\nfrom scipy.special import expit\n\n\ndef _sigmoid(x):\n return expit(x)\n\n\ndef makedirs(path):\n try:\n os.makedirs(path)\n except OSError:\n if not os.path.isdir(path):\n raise\n\n\ndef evaluate(model,\n generator, \n iou_threshold,\n obj_thresh,\n nms_thresh,\n net_h=416,\n net_w=416,\n save_path=None):\n \"\"\" Evaluate a given dataset using a given model.\n code originally from https://github.com/fizyr/keras-retinanet\n\n # Arguments\n model : The model to evaluate.\n generator : The generator that represents the dataset to evaluate.\n iou_threshold : The threshold used to consider when a detection is positive or negative.\n obj_thresh : The threshold used to distinguish between object and non-object\n nms_thresh : The threshold used to determine whether two detections are duplicates\n net_h : The height of the input image to the model, higher value results in better accuracy\n net_w : The width of the input image to the model\n save_path : The path to save images with visualized detections to.\n # Returns\n A dict mapping class names to mAP scores.\n \"\"\" \n # gather all detections and annotations\n all_detections = [[None for i in range(generator.num_classes())] for j in range(generator.size())]\n all_annotations = [[None for i in range(generator.num_classes())] for j in range(generator.size())]\n\n for i in range(generator.size()):\n raw_image = [generator.load_image(i)]\n\n # make the boxes and the labels\n pred_boxes = get_yolo_boxes(model, raw_image, net_h, net_w, generator.get_anchors(), obj_thresh, nms_thresh)[0]\n\n score = np.array([box.get_score() for box in pred_boxes])\n pred_labels = np.array([box.label for box in pred_boxes]) \n \n if len(pred_boxes) > 0:\n pred_boxes = np.array([[box.xmin, box.ymin, box.xmax, box.ymax, box.get_score()] for box in pred_boxes]) \n else:\n pred_boxes = np.array([[]]) \n \n # sort the boxes and the labels according to scores\n score_sort = np.argsort(-score)\n pred_labels = pred_labels[score_sort]\n pred_boxes = pred_boxes[score_sort]\n \n # copy detections to all_detections\n for label in range(generator.num_classes()):\n all_detections[i][label] = pred_boxes[pred_labels == label, :]\n\n annotations = generator.load_annotation(i)\n \n # copy detections to all_annotations\n for label in range(generator.num_classes()):\n all_annotations[i][label] = annotations[annotations[:, 4] == label, :4].copy()\n\n # compute mAP by comparing all detections and all annotations\n average_precisions = {}\n \n for label in range(generator.num_classes()):\n false_positives = np.zeros((0,))\n true_positives = np.zeros((0,))\n scores = np.zeros((0,))\n num_annotations = 0.0\n\n for i in range(generator.size()):\n detections = all_detections[i][label]\n annotations = all_annotations[i][label]\n num_annotations += annotations.shape[0]\n detected_annotations = []\n\n for d in detections:\n scores = np.append(scores, d[4])\n\n if annotations.shape[0] == 0:\n false_positives = np.append(false_positives, 1)\n true_positives = np.append(true_positives, 0)\n continue\n\n overlaps = compute_overlap(np.expand_dims(d, axis=0), annotations)\n assigned_annotation = np.argmax(overlaps, axis=1)\n max_overlap = overlaps[0, assigned_annotation]\n\n if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:\n false_positives = np.append(false_positives, 0)\n true_positives = np.append(true_positives, 1)\n detected_annotations.append(assigned_annotation)\n else:\n false_positives = np.append(false_positives, 1)\n true_positives = np.append(true_positives, 0)\n\n # no annotations -> AP for this class is 0 (is this correct?)\n if num_annotations == 0:\n average_precisions[label] = 0\n continue\n\n # sort by score\n indices = np.argsort(-scores)\n false_positives = false_positives[indices]\n true_positives = true_positives[indices]\n\n # compute false positives and true positives\n false_positives = np.cumsum(false_positives)\n true_positives = np.cumsum(true_positives)\n\n # compute recall and precision\n recall = true_positives / num_annotations\n precision = true_positives / np.maximum(true_positives + false_positives, np.finfo(np.float64).eps)\n\n # compute average precision\n average_precision = compute_ap(recall, precision) \n average_precisions[label] = average_precision\n\n return average_precisions \n\n\ndef correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w):\n if (float(net_w)/image_w) < (float(net_h)/image_h):\n new_w = net_w\n new_h = (image_h*net_w)/image_w\n else:\n new_h = net_w\n new_w = (image_w*net_h)/image_h\n \n for i in range(len(boxes)):\n x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w\n y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h\n \n boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w)\n boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w)\n boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h)\n boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h)\n \n\ndef do_nms(boxes, nms_thresh):\n if len(boxes) > 0:\n nb_class = len(boxes[0].classes)\n else:\n return\n \n for c in range(nb_class):\n sorted_indices = np.argsort([-box.classes[c] for box in boxes])\n\n for i in range(len(sorted_indices)):\n index_i = sorted_indices[i]\n\n if boxes[index_i].classes[c] == 0: continue\n\n for j in range(i+1, len(sorted_indices)):\n index_j = sorted_indices[j]\n\n if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh:\n boxes[index_j].classes[c] = 0\n\n\ndef decode_netout(netout, anchors, obj_thresh, net_h, net_w):\n grid_h, grid_w = netout.shape[:2]\n nb_box = 3\n netout = netout.reshape((grid_h, grid_w, nb_box, -1))\n nb_class = netout.shape[-1] - 5\n\n boxes = []\n\n netout[..., :2] = _sigmoid(netout[..., :2])\n netout[..., 4] = _sigmoid(netout[..., 4])\n netout[..., 5:] = netout[..., 4][..., np.newaxis] * _softmax(netout[..., 5:])\n netout[..., 5:] *= netout[..., 5:] > obj_thresh\n\n for i in range(grid_h*grid_w):\n row = i // grid_w\n col = i % grid_w\n \n for b in range(nb_box):\n # 4th element is objectness score\n objectness = netout[row, col, b, 4]\n \n if objectness <= obj_thresh:\n continue\n \n # first 4 elements are x, y, w, and h\n x, y, w, h = netout[row, col, b, :4]\n\n x = (col + x) / grid_w # center position, unit: image width\n y = (row + y) / grid_h # center position, unit: image height\n w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width\n h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height\n \n # last elements are class probabilities\n classes = netout[row, col, b, 5:]\n \n box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes)\n\n boxes.append(box)\n\n return boxes\n\n\ndef preprocess_input(image, net_h, net_w):\n new_h, new_w, _ = image.shape\n\n # determine the new size of the image\n if (float(net_w)/new_w) < (float(net_h)/new_h):\n new_h = (new_h * net_w)//new_w\n new_w = net_w\n else:\n new_w = (new_w * net_h)//new_h\n new_h = net_h\n\n # resize the image to the new size\n resized = cv2.resize(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)/255., (new_w, new_h))\n\n # embed the image into the standard letter box\n new_image = np.ones((net_h, net_w, 3)) * 0.5\n new_image[(net_h-new_h)//2:(net_h+new_h)//2, (net_w-new_w)//2:(net_w+new_w)//2, :] = resized\n new_image = np.expand_dims(new_image, 0)\n\n return new_image\n\n\ndef normalize(image):\n return image/255.\n\n\ndef get_yolo_boxes(model, images, net_h, net_w, anchors, obj_thresh, nms_thresh):\n image_h, image_w, _ = images[0].shape\n nb_images = len(images)\n batch_input = np.zeros((nb_images, net_h, net_w, 3))\n\n # preprocess the input\n for i in range(nb_images):\n batch_input[i] = preprocess_input(images[i], net_h, net_w) \n\n # run the prediction\n batch_output = model.predict_on_batch(batch_input)\n batch_boxes = [None]*nb_images\n\n for i in range(nb_images):\n yolos = [batch_output[0][i], batch_output[1][i], batch_output[2][i]]\n boxes = []\n\n # decode the output of the network\n for j in range(len(yolos)):\n yolo_anchors = anchors[(2-j)*6:(3-j)*6] # config['model']['anchors']\n boxes += decode_netout(yolos[j], yolo_anchors, obj_thresh, net_h, net_w)\n\n # correct the sizes of the bounding boxes\n correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w)\n\n # suppress non-maximal boxes\n do_nms(boxes, nms_thresh) \n \n batch_boxes[i] = boxes\n\n return batch_boxes \n\n\ndef compute_overlap(a, b):\n \"\"\"\n Code originally from https://github.com/rbgirshick/py-faster-rcnn.\n Parameters\n ----------\n a: (N, 4) ndarray of float\n b: (K, 4) ndarray of float\n Returns\n -------\n overlaps: (N, K) ndarray of overlap between boxes and query_boxes\n \"\"\"\n area = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1])\n\n iw = np.minimum(np.expand_dims(a[:, 2], axis=1), b[:, 2]) - np.maximum(np.expand_dims(a[:, 0], 1), b[:, 0])\n ih = np.minimum(np.expand_dims(a[:, 3], axis=1), b[:, 3]) - np.maximum(np.expand_dims(a[:, 1], 1), b[:, 1])\n\n iw = np.maximum(iw, 0)\n ih = np.maximum(ih, 0)\n\n ua = np.expand_dims((a[:, 2] - a[:, 0]) * (a[:, 3] - a[:, 1]), axis=1) + area - iw * ih\n\n ua = np.maximum(ua, np.finfo(float).eps)\n\n intersection = iw * ih\n\n return intersection / ua \n\n\ndef compute_ap(recall, precision):\n \"\"\" Compute the average precision, given the recall and precision curves.\n Code originally from https://github.com/rbgirshick/py-faster-rcnn.\n\n # Arguments\n recall: The recall curve (list).\n precision: The precision curve (list).\n # Returns\n The average precision as computed in py-faster-rcnn.\n \"\"\"\n # correct AP calculation\n # first append sentinel values at the end\n mrec = np.concatenate(([0.], recall, [1.]))\n mpre = np.concatenate(([0.], precision, [0.]))\n\n # compute the precision envelope\n for i in range(mpre.size - 1, 0, -1):\n mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])\n\n # to calculate area under PR curve, look for points\n # where X axis (recall) changes value\n i = np.where(mrec[1:] != mrec[:-1])[0]\n\n # and sum (\\Delta recall) * prec\n ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])\n return ap \n\n\ndef _softmax(x, axis=-1):\n x = x - np.amax(x, axis, keepdims=True)\n e_x = np.exp(x)\n \n return e_x / e_x.sum(axis, keepdims=True)\n" }, { "path": "imageai_tf_deprecated/Detection/Custom/voc.py", "content": "import os\nimport xml.etree.ElementTree as ET\nimport pickle\n\n\ndef parse_voc_annotation(ann_dir, img_dir, cache_name, labels=[]):\n if os.path.exists(cache_name):\n with open(cache_name, 'rb') as handle:\n cache = pickle.load(handle)\n all_insts, seen_labels = cache['all_insts'], cache['seen_labels']\n else:\n all_insts = list()\n seen_labels = dict()\n \n for ann in sorted(os.listdir(ann_dir)):\n img = {'object': list()}\n\n try:\n tree = ET.parse(os.path.join(ann_dir, ann))\n except Exception as e:\n print(e)\n print('Ignore this bad annotation: ' + os.path.join(ann_dir, ann))\n continue\n \n for elem in tree.iter():\n if 'filename' in elem.tag:\n img['filename'] = os.path.join(img_dir, elem.text)\n if 'width' in elem.tag:\n img['width'] = int(elem.text)\n if 'height' in elem.tag:\n img['height'] = int(elem.text)\n if 'object' in elem.tag or 'part' in elem.tag:\n obj = {}\n \n for attr in list(elem):\n if 'name' in attr.tag:\n obj['name'] = attr.text\n\n if obj['name'] in seen_labels:\n seen_labels[obj['name']] += 1\n else:\n seen_labels[obj['name']] = 1\n \n if len(labels) > 0 and obj['name'] not in labels:\n break\n else:\n img['object'] += [obj]\n \n if 'bndbox' in attr.tag:\n for dim in list(attr):\n if 'xmin' in dim.tag:\n obj['xmin'] = int(round(float(dim.text)))\n if 'ymin' in dim.tag:\n obj['ymin'] = int(round(float(dim.text)))\n if 'xmax' in dim.tag:\n obj['xmax'] = int(round(float(dim.text)))\n if 'ymax' in dim.tag:\n obj['ymax'] = int(round(float(dim.text)))\n\n if len(img['object']) > 0:\n all_insts += [img]\n\n cache = {'all_insts': all_insts, 'seen_labels': seen_labels}\n with open(cache_name, 'wb') as handle:\n pickle.dump(cache, handle, protocol=pickle.HIGHEST_PROTOCOL) \n \n return all_insts, seen_labels\n" }, { "path": "imageai_tf_deprecated/Detection/README.md", "content": "# ImageAI : Object Detection\n\nA **DeepQuest AI** project [https://deepquestai.com](https://deepquestai.com)\n\n### TABLE OF CONTENTS\n\n- :white_square_button: First Object Detection\n- :white_square_button: Object Detection, Extraction and Fine-tune\n- :white_square_button: Custom Object Detection\n- :white_square_button: Detection Speed\n- :white_square_button: Hiding/Showing Object Name and Probability\n- :white_square_button: Image Input & Output Types\n- :white_square_button: Documentation\n\n\nImageAI provides very convenient and powerful methods to perform object detection on images and extract each object from the image. The object detection class supports RetinaNet, YOLOv3 and TinyYOLOv3. To start performing object detection, you must download the RetinaNet, YOLOv3 or TinyYOLOv3 object detection model via the links below: \n* **[RetinaNet](https://github.com/OlafenwaMoses/ImageAI/releases/download/essentials-v5/resnet50_coco_best_v2.1.0.h5)** _(Size = 145 mb, high performance and accuracy, with longer detection time)_\n* **[YOLOv3](https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/yolo.h5)** _(Size = 237 mb, moderate performance and accuracy, with a moderate detection time)_\n* **[TinyYOLOv3](https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/yolo-tiny.h5)** _(Size = 34 mb, optimized for speed and moderate performance, with fast detection time)_\n\n\n Once you download the object detection model file, you should copy the model file to the your project folder where your .py files will be.\n Then create a python file and give it a name; an example is FirstObjectDetection.py. Then write the code below into the python file:\n\n### FirstObjectDetection.py\n
\n\n```python\nfrom imageai.Detection import ObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = ObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"yolo.h5\"))\ndetector.loadModel()\ndetections = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"image2.jpg\"), output_image_path=os.path.join(execution_path , \"image2new.jpg\"), minimum_percentage_probability=30)\n\nfor eachObject in detections:\n print(eachObject[\"name\"] , \" : \", eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"--------------------------------\")\n```\n\nSample Result:\nInput Image\n![Input Image](../../data-images/image2.jpg)\nOutput Image\n![Output Image](../../data-images/yolo.jpg)\n\n```\nlaptop : 87.32235431671143 : (306, 238, 390, 284)\n--------------------------------\nlaptop : 96.86298966407776 : (121, 209, 258, 293)\n--------------------------------\nlaptop : 98.6301600933075 : (279, 321, 401, 425)\n--------------------------------\nlaptop : 99.78572130203247 : (451, 204, 579, 285)\n--------------------------------\nbed : 94.02391314506531 : (23, 205, 708, 553)\n--------------------------------\napple : 48.03136885166168 : (527, 343, 557, 364)\n--------------------------------\ncup : 34.09906327724457 : (462, 347, 496, 379)\n--------------------------------\ncup : 44.65090036392212 : (582, 342, 618, 386)\n--------------------------------\nperson : 57.70219564437866 : (27, 311, 341, 437)\n--------------------------------\nperson : 85.26121377944946 : (304, 173, 387, 253)\n--------------------------------\nperson : 96.33603692054749 : (415, 130, 538, 266)\n--------------------------------\nperson : 96.95255160331726 : (174, 108, 278, 269)\n--------------------------------\n```\n\nLet us make a breakdown of the object detection code that we used above.\n\n```python\nfrom imageai.Detection import ObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n```\n\n In the 3 lines above , we import the **ImageAI object detection** class in the first line, import the `os` in the second line and obtained the path to folder where our python file runs.\n \n```python\ndetector = ObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"yolo.h5\"))\ndetector.loadModel()\n```\n\nIn the 4 lines above, we created a new instance of the `ObjectDetection` class in the first line, set the model type to YOLOv3 in the second line, set the model path to the YOLOv3 model file we downloaded and copied to the python file folder in the third line and load the model in the fourth line.\n\n```python\ndetections = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"image2.jpg\"), output_image_path=os.path.join(execution_path , \"image2new.jpg\"))\n\nfor eachObject in detections:\n print(eachObject[\"name\"] , \" : \", eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"--------------------------------\")\n```\n\nIn the 2 lines above, we ran the `detectObjectsFromImage()` function and parse in the path to our image, and the path to the new image which the function will save. Then the function returns an array of dictionaries with each dictionary corresponding to the number of objects detected in the image. Each dictionary has the properties `name` (name of the object), `percentage_probability` (percentage probability of the detection) and `box_points` (the x1,y1,x2 and y2 coordinates of the bounding box of the object).\n\nShould you want to use the RetinaNet which is appropriate for high-performance and high-accuracy demanding detection tasks, you will download the RetinaNet model file from the links above, copy it to your python file's folder, set the model type and model path in your python code as seen below:\n\n```python\ndetector = ObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath( os.path.join(execution_path , \"resnet50_coco_best_v2.0.1.h5\"))\ndetector.loadModel()\n```\n\nHowever, if you desire TinyYOLOv3 which is optimized for speed and embedded devices, you will download the TinyYOLOv3 model file from the links above, copy it to your python file's folder, set the model type and model path in your python code as seen below:\n\n```python\ndetector = ObjectDetection()\ndetector.setModelTypeAsTinyYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"yolo-tiny.h5\"))\ndetector.loadModel()\n```\n\n## Object Detection, Extraction and Fine-tune\n
\n\nIn the examples we used above, we ran the object detection on an image and it returned the detected objects in an array as well as save a new image with rectangular markers drawn on each object. In our next examples, we will be able to extract each object from the input image\n and save it independently.\n\nIn the example code below which is very identical to the previous object detction code, we will save each object detected as a seperate image.\n\n```python\nfrom imageai.Detection import ObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = ObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"yolo.h5\"))\ndetector.loadModel()\n\ndetections, objects_path = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"image3.jpg\"), output_image_path=os.path.join(execution_path , \"image3new.jpg\"), minimum_percentage_probability=30, extract_detected_objects=True)\n\nfor eachObject, eachObjectPath in zip(detections, objects_path):\n print(eachObject[\"name\"] , \" : \" , eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"Object's image saved in \" + eachObjectPath)\n print(\"--------------------------------\")\n```\n\n![Input Image](../../data-images/image3.jpg)\n![Output Images](../../data-images/image3new.jpg)\n\n![dog](../../data-images/image3new-objects/dog-1.jpg)\n![motorcycle](../../data-images/image3new-objects/motorcycle-3.jpg)\n![car](../../data-images/image3new-objects/car-4.jpg)\n![bicycle](../../data-images/image3new-objects/bicycle-5.jpg)\n![person](../../data-images/image3new-objects/person-6.jpg)\n![person](../../data-images/image3new-objects/person-7.jpg)\n![person](../../data-images/image3new-objects/person-8.jpg)\n![person](../../data-images/image3new-objects/person-9.jpg)\n![person](../../data-images/image3new-objects/person-10.jpg)\n\n\nLet us review the part of the code that perform the object detection and extract the images:\n\n```python\ndetections, objects_path = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"image3.jpg\"), output_image_path=os.path.join(execution_path , \"image3new.jpg\"), minimum_percentage_probability=30, extract_detected_objects=True)\n\nfor eachObject, eachObjectPath in zip(detections, objects_path):\n print(eachObject[\"name\"] , \" : \" , eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"Object's image saved in \" + eachObjectPath)\n print(\"--------------------------------\")\n```\n\nIn the above above lines, we called the `detectObjectsFromImage()` , parse in the input image path, output image path, and an extra parameter `extract_detected_objects=True`. This parameter states that the function should extract each object detected from the image and save it has a seperate image. The parameter is false by default. Once set to `true`, the function will create a directory which is the **output image path + \"-objects\"** . Then it saves all the extracted images into this new directory with each image's name being the **detected object name + \"-\" + a number** which corresponds to the order at which the objects were detected.\n\nThis new parameter we set to extract and save detected objects as an image will make the function to return 2 values. The first is the array of dictionaries with each dictionary corresponding to a detected object. The second is an array of the paths to the saved images of each object detected and extracted, and they are arranged in order at which the objects are in the first array.\n\n\n**And one important feature you need to know!** You will recall that the percentage probability\n for each detected object is sent back by the `detectObjectsFromImage()` function. The function has a parameter `minimum_percentage_probability`, whose default value is `50` (value ranges between 0 - 100) , but it set to 30 in this example. That means the function will only return a detected object if it's percentage probability is **30 or above**. The value was kept at this number to ensure the integrity of the detection results. You fine-tune the object detection by setting **minimum_percentage_probability** equal to a smaller value to detect more number of objects or higher value to detect less number of objects.\n\n\n## Custom Object Detection\n
\n\nThe object detection model (**RetinaNet**) supported by **ImageAI** can detect 80 different types of objects. They include:\n```\nperson, bicycle, car, motorcycle, airplane, bus, train, truck, boat, traffic light, fire hydrant, stop_sign,\nparking meter, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra,\ngiraffe, backpack, umbrella, handbag, tie, suitcase, frisbee, skis, snowboard,\nsports ball, kite, baseball bat, baseball glove, skateboard, surfboard, tennis racket,\nbottle, wine glass, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange,\nbroccoli, carrot, hot dog, pizza, donot, cake, chair, couch, potted plant, bed,\ndining table, toilet, tv, laptop, mouse, remote, keyboard, cell phone, microwave, oven,\ntoaster, sink, refrigerator, book, clock, vase, scissors, teddy bear, hair dryer, toothbrush.\n```\n\nInterestingly, **ImageAI** allow you to perform detection for one or more of the items above. That means you can\n customize the type of object(s) you want to be detected in the image. Let's take a look at the code below:\n\n```python\nfrom imageai.Detection import ObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = ObjectDetection()\ndetector.setModelTypeAsYOLOv3()\ndetector.setModelPath( os.path.join(execution_path , \"yolo.h5\"))\ndetector.loadModel()\n\ncustom_objects = detector.CustomObjects(car=True, motorcycle=True)\ndetections = detector.detectCustomObjectsFromImage(custom_objects=custom_objects, input_image=os.path.join(execution_path , \"image3.jpg\"), output_image_path=os.path.join(execution_path , \"image3custom.jpg\"), minimum_percentage_probability=30)\n\nfor eachObject in detections:\n print(eachObject[\"name\"] , \" : \", eachObject[\"percentage_probability\"], \" : \", eachObject[\"box_points\"] )\n print(\"--------------------------------\")\n```\n\n![Result](../../data-images/image3custom.jpg)\n\n\nLet us take a look at the part of the code that made this possible.\n```python\ncustom_objects = detector.CustomObjects(car=True, motorcycle=True)\ndetections = detector.detectCustomObjectsFromImage(custom_objects=custom_objects, input_image=os.path.join(execution_path , \"image3.jpg\"), output_image_path=os.path.join(execution_path , \"image3custom.jpg\"), minimum_percentage_probability=30)\n```\n\nIn the above code, after loading the model (can be done before loading the model as well), we defined a new variable\n`custom_objects = detector.CustomObjects()`, in which we set its car and motorcycle properties equal to **True**.\nThis is to tell the model to detect only the object we set to True. Then we call the `detector.detectCustomObjectsFromImage()`\nwhich is the function that allows us to perform detection of custom objects. Then we will set the `custom_objects` value\n to the custom objects variable we defined.\n\n\n## Detection Speed\n
\n\n**ImageAI** now provides detection speeds for all object detection tasks. The detection speeds allow you to reduce\n the time of detection at a rate between 20% - 80%, and yet having just slight changes but accurate detection\nresults. Coupled with lowering the `minimum_percentage_probability` parameter, detections can match the normal\nspeed and yet reduce detection time drastically. The available detection speeds are **\"normal\"**(default), **\"fast\"**, **\"faster\"** , **\"fastest\"** and **\"flash\"**.\nAll you need to do is to state the speed mode you desire when loading the model as seen below.\n\n```python\ndetector.loadModel(detection_speed=\"fast\")\n```\n\n\n## Hiding/Showing Object Name and Probability\n
\n\n**ImageAI** provides options to hide the name of objects detected and/or the percentage probability from being shown on the saved/returned detected image. Using the `detectObjectsFromImage()` and `detectCustomObjectsFromImage()` functions, the parameters `display_object_name` and `display_percentage_probability` can be set to True of False individually. Take a look at the code below:\n\n```python\ndetections = detector.detectObjectsFromImage(input_image=os.path.join(execution_path , \"image3.jpg\"), output_image_path=os.path.join(execution_path , \"image3new_nodetails.jpg\"), minimum_percentage_probability=30, display_percentage_probability=False, display_object_name=False)\n```\n\nIn the above code, we specified that both the object name and percentage probability should not be shown. As you can see in the result below, both the names of the objects and their individual percentage probability is not shown in the detected image.\n\n![Result](../../data-images/nodetails.jpg)\n\n\n## Image Input & Output Types\n
\n\n**ImageAI** supports 3 types of inputs which are **file path to image file**(default), **numpy array of image** and **image file stream**\nas well as 2 types of output which are image **file**(default) and numpy **array **.\nThis means you can now perform object detection in production applications such as on a web server and system\n that returns file in any of the above stated formats.\n\nTo perform object detection with numpy array or file stream input, you just need to state the input type\nin the `.detectObjectsFromImage()` function or the `.detectCustomObjectsFromImage()` function. See example below.\n\n```python\ndetections = detector.detectObjectsFromImage(input_type=\"array\", input_image=image_array , output_image_path=os.path.join(execution_path , \"image.jpg\")) # For numpy array input type\ndetections = detector.detectObjectsFromImage(input_type=\"stream\", input_image=image_stream , output_image_path=os.path.join(execution_path , \"test2new.jpg\")) # For file stream input type\n```\n\nTo perform object detection with numpy array output you just need to state the output type\nin the `.detectObjectsFromImage()` function or the `.detectCustomObjectsFromImage()` function. See example below.\n\n```python\ndetected_image_array, detections = detector.detectObjectsFromImage(output_type=\"array\", input_image=\"image.jpg\" ) # For numpy array output type\n```\n\n\n## Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions in 3 major languages. Find links below:\n\n* Documentation - **English Version [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**\n* Documentation - **Chinese Version [https://imageai-cn.readthedocs.io](https://imageai-cn.readthedocs.io)**\n* Documentation - **French Version [https://imageai-fr.readthedocs.io](https://imageai-fr.readthedocs.io)**\n" }, { "path": "imageai_tf_deprecated/Detection/VIDEO.md", "content": "# ImageAI : Video Object Detection, Tracking and Analysis\n\nA **DeepQuest AI** project [https://deepquestai.com](https://deepquestai.com)\n\n---\n\n## TABLE OF CONTENTS\n\n- :white_square_button: First Video Object Detection\n- :white_square_button: Custom Video Object Detection (Object Tracking)\n- :white_square_button: Camera / Live Stream Video Detection\n- :white_square_button: Video Analysis\n- :white_square_button: Detection Speed\n- :white_square_button: Hiding/Showing Object Name and Probability\n- :white_square_button: Frame Detection Intervals\n- :white_square_button: Video Detection Timeout (NEW)\n- :white_square_button: Documentation\n\nImageAI provides convenient, flexible and powerful methods to perform object detection on videos. The video object detection class provided only supports RetinaNet, YOLOv3 and TinyYOLOv3. This version of **ImageAI** provides commercial grade video objects detection features, which include but not limited to device/IP camera inputs, per frame, per second, per minute and entire video analysis for storing in databases and/or real-time visualizations and for future insights.\n\nTo start performing video object detection, you must download the RetinaNet, YOLOv3 or TinyYOLOv3 object detection model via the links below:\n\n- **[RetinaNet](https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/resnet50_coco_best_v2.0.1.h5)** _(Size = 145 mb, high performance and accuracy, with longer detection time)_\n- **[YOLOv3](https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/yolo.h5)** _(Size = 237 mb, moderate performance and accuracy, with a moderate detection time)_\n- **[TinyYOLOv3](https://github.com/OlafenwaMoses/ImageAI/releases/download/1.0/yolo-tiny.h5)** _(Size = 34 mb, optimized for speed and moderate performance, with fast detection time)_\n\nBecause video object detection is a compute intensive tasks, we advise you perform this experiment using a computer with a NVIDIA GPU and the GPU version of Tensorflow installed. Performing Video Object Detection CPU will be slower than using an NVIDIA GPU powered computer. You can use Google Colab for this experiment as it has an NVIDIA K80 GPU available for free.\n\n Once you download the object detection model file, you should copy the model file to the your project folder where your .py files will be.\n Then create a python file and give it a name; an example is `FirstVideoObjectDetection.py`. Then write the code below into the python file:\n\n\n### FirstVideoObjectDetection.py\n
\n\n```python\nfrom imageai.Detection import VideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath( os.path.join(execution_path , \"resnet50_coco_best_v2.0.1.h5\"))\ndetector.loadModel()\n\nvideo_path = detector.detectObjectsFromVideo(input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\")\n , frames_per_second=20, log_progress=True)\nprint(video_path)\n```\n\n\nInput Video (a 1min 24seconds video)\n\n[![](../../data-images/video--1.jpg)](https://github.com/OlafenwaMoses/ImageAI/blob/master/data-videos/traffic.mp4)\n\nOutput Video\n[![](../../data-images/video-2.jpg)](https://www.youtube.com/embed/qplVDqOmElI?rel=0)\n\nLet us make a breakdown of the object detection code that we used above.\n\n```python\nfrom imageai.Detection import VideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n```\n\n In the 3 lines above , we import the **ImageAI video object detection ** class in the first line, import the **os** in the second line and obtained\n the path to folder where our python file runs.\n\n```python\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath( os.path.join(execution_path , \"resnet50_coco_best_v2.0.1.h5\"))\ndetector.loadModel()\n```\n\nIn the 4 lines above, we created a new instance of the **VideoObjectDetection** class in the first line, set the model type to RetinaNet in the second line, set the model path to the RetinaNet model file we downloaded and copied to the python file folder in the third line and load the model in the fourth line.\n\n```python\nvideo_path = detector.detectObjectsFromVideo(input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\"),\n frames_per_second=20, log_progress=True)\nprint(video_path)\n```\n\nIn the 2 lines above, we ran the `detectObjectsFromVideo()` function and parse in the path to our video,the path to the new video (without the extension, it saves a .avi video by default) which the function will save, the number of frames per second (fps) that you we desire the output video to have and option to log the progress of the detection in the console. Then the function returns a the path to the saved video which contains boxes and percentage probabilities rendered on objects detected in the video.\n\n\n### Custom Video Object Detection\n
\n\nThe video object detection model (**RetinaNet**) supported by **ImageAI** can detect 80 different types of objects. They include: \n```\n person, bicycle, car, motorcycle, airplane, bus, train, truck, boat, traffic light, fire hydrant, stop_sign,\n parking meter, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra,\n giraffe, backpack, umbrella, handbag, tie, suitcase, frisbee, skis, snowboard,\n sports ball, kite, baseball bat, baseball glove, skateboard, surfboard, tennis racket,\n bottle, wine glass, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange,\n broccoli, carrot, hot dog, pizza, donot, cake, chair, couch, potted plant, bed,\n dining table, toilet, tv, laptop, mouse, remote, keyboard, cell phone, microwave,\n oven, toaster, sink, refrigerator, book, clock, vase, scissors, teddy bear, hair dryer,\n toothbrush.\n```\n\n\nInterestingly, **ImageAI** allow you to perform detection for one or more of the items above. That means you can customize the type of object(s) you want to be detected in the video. Let's take a look at the code below:\n\n```python\nfrom imageai.Detection import VideoObjectDetection\nimport os\n\nexecution_path = os.getcwd()\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath( os.path.join(execution_path , \"resnet50_coco_best_v2.0.1.h5\"))\ndetector.loadModel()\n\ncustom_objects = detector.CustomObjects(person=True, bicycle=True, motorcycle=True)\n\nvideo_path = detector.detectCustomObjectsFromVideo(\n custom_objects=custom_objects,\n input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_custom_detected\"),\n frames_per_second=20, log_progress=True)\nprint(video_path)\n```\n\nLet us take a look at the part of the code that made this possible.\n\n```python\ncustom_objects = detector.CustomObjects(person=True, bicycle=True, motorcycle=True)\n\nvideo_path = detector.detectCustomObjectsFromVideo(\n custom_objects=custom_objects, \n input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_custom_detected\"),\n frames_per_second=20, log_progress=True)\n```\n\nIn the above code, after loading the model (can be done before loading the model as well), we defined a new variable\n`custom_objects = detector.CustomObjects()`, in which we set its person, car and motorcycle properties equal to **True**.\nThis is to tell the model to detect only the object we set to True. Then we call the `detector.detectCustomObjectsFromVideo()`\nwhich is the function that allows us to perform detection of custom objects. Then we will set the `custom_objects` value\n to the custom objects variable we defined.\n\nOutput Video\n[![Output Video](../../data-images/video-3.jpg)](https://www.youtube.com/embed/YfAycAzkwPM?rel=0)\nC:\\Users\\User\\PycharmProjects\\ImageAITest\\traffic_custom_detected.avi\n\n\n### Camera / Live Stream Video Detection\n
\n\n**ImageAI** now allows live-video detection with support for camera inputs. Using **OpenCV**'s `VideoCapture()` function, you can load live-video streams from a device camera, cameras connected by cable or IP cameras, and parse it into **ImageAI**'s `detectObjectsFromVideo()` and `detectCustomObjectsFromVideo()` functions. All features that are supported for detecting objects in a video file is also available for detecting objects in a camera's live-video feed. Find below an example of detecting live-video feed from the device camera.\n\n```python\nfrom imageai.Detection import VideoObjectDetection\nimport os\nimport cv2\n\nexecution_path = os.getcwd()\n\n\ncamera = cv2.VideoCapture(0)\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath(os.path.join(execution_path , \"resnet50_coco_best_v2.0.1.h5\"))\ndetector.loadModel()\n\n\nvideo_path = detector.detectObjectsFromVideo(\n camera_input=camera,\n output_file_path=os.path.join(execution_path, \"camera_detected_video\"),\n frames_per_second=20, log_progress=True, minimum_percentage_probability=40)\n```\n\nThe difference in the code above and the code for the detection of a video file is that we defined an **OpenCV VideoCapture** instance and loaded the default device camera into it. Then we parsed the camera we defined into the parameter `camera_input` which replaces the `input_file_path` that is used for video file.\n\n### Video Analysis\n
\n\n**ImageAI** now provide commercial-grade video analysis in the Video Object Detection class, for both video file inputs and camera inputs. This feature allows developers to obtain deep insights into any video processed with **ImageAI**. This insights can be visualized in real-time, stored in a NoSQL database for future review or analysis.\n\nFor video analysis, the `detectObjectsFromVideo()` and `detectCustomObjectsFromVideo()` now allows you to state your own defined functions which will be executed for every frame, seconds and/or minute of the video detected as well as a state a function that will be executed at the end of a video detection. Once this functions are stated, they will receive raw but comprehensive analytical data on the index of the frame/second/minute, objects detected (name, percentage_probability and box_points), number of instances of each unique object detected and average number of occurrence of each unique object detected over a second/minute and entire video.\n\nTo obtain the video analysis, all you need to do is specify a function, state the corresponding parameters it will be receiving and parse the function name into the `per_frame_function`, `per_second_function`, `per_minute_function` and `video_complete_function` parameters in the detection function. Find below examples of video analysis functions.\n\n```python\ndef forFrame(frame_number, output_array, output_count):\n print(\"FOR FRAME \" , frame_number)\n print(\"Output for each object : \", output_array)\n print(\"Output count for unique objects : \", output_count)\n print(\"------------END OF A FRAME --------------\")\n\ndef forSeconds(second_number, output_arrays, count_arrays, average_output_count):\n print(\"SECOND : \", second_number)\n print(\"Array for the outputs of each frame \", output_arrays)\n print(\"Array for output count for unique objects in each frame : \", count_arrays)\n print(\"Output average count for unique objects in the last second: \", average_output_count)\n print(\"------------END OF A SECOND --------------\")\n\ndef forMinute(minute_number, output_arrays, count_arrays, average_output_count):\n print(\"MINUTE : \", minute_number)\n print(\"Array for the outputs of each frame \", output_arrays)\n print(\"Array for output count for unique objects in each frame : \", count_arrays)\n print(\"Output average count for unique objects in the last minute: \", average_output_count)\n print(\"------------END OF A MINUTE --------------\")\n\nvideo_detector = VideoObjectDetection()\nvideo_detector.setModelTypeAsYOLOv3()\nvideo_detector.setModelPath(os.path.join(execution_path, \"yolo.h5\"))\nvideo_detector.loadModel()\n\nvideo_detector.detectObjectsFromVideo(\n input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\"),\n frames_per_second=10,\n per_second_function=forSeconds,\n per_frame_function=forFrame,\n per_minute_function=forMinute,\n minimum_percentage_probability=30\n)\n```\n\nWhen the detection starts on a video feed, be it from a video file or camera input, the result will have the format as below:\n\n**Results for the Frame function**\n```\nFOR FRAME : 1\n \nOutput for each object : [{'box_points': (362, 295, 443, 355), 'name': 'boat', 'percentage_probability': 26.666194200515747}, {'box_points': (319, 245, 386, 296), 'name': 'boat', 'percentage_probability': 30.052968859672546}, {'box_points': (219, 308, 341, 358), 'name': 'boat', 'percentage_probability': 47.46982455253601}, {'box_points': (589, 198, 621, 241), 'name': 'bus', 'percentage_probability': 24.62330162525177}, {'box_points': (519, 181, 583, 263), 'name': 'bus', 'percentage_probability': 27.446213364601135}, {'box_points': (493, 197, 561, 272), 'name': 'bus', 'percentage_probability': 59.81815457344055}, {'box_points': (432, 187, 491, 240), 'name': 'bus', 'percentage_probability': 64.42965269088745}, {'box_points': (157, 225, 220, 255), 'name': 'car', 'percentage_probability': 21.150341629981995}, {'box_points': (324, 249, 377, 293), 'name': 'car', 'percentage_probability': 24.089913070201874}, {'box_points': (152, 275, 260, 327), 'name': 'car', 'percentage_probability': 30.341443419456482}, {'box_points': (433, 198, 485, 244), 'name': 'car', 'percentage_probability': 37.205660343170166}, {'box_points': (184, 226, 233, 260), 'name': 'car', 'percentage_probability': 38.52525353431702}, {'box_points': (3, 296, 134, 359), 'name': 'car', 'percentage_probability': 47.80363142490387}, {'box_points': (357, 302, 439, 359), 'name': 'car', 'percentage_probability': 47.94844686985016}, {'box_points': (481, 266, 546, 314), 'name': 'car', 'percentage_probability': 65.8585786819458}, {'box_points': (597, 269, 624, 318), 'name': 'person', 'percentage_probability': 27.125394344329834}]\n \nOutput count for unique objects : {'bus': 4, 'boat': 3, 'person': 1, 'car': 8}\n\n------------END OF A FRAME --------------\n```\n\nFor any function you parse into the **per_frame_function**, the function will be executed after every single video frame is processed and he following will be parsed into it:\n\n* **Frame Index:** This is the position number of the frame inside the video (e.g 1 for first frame and 20 for twentieth frame).\n* **Output Array:** This is an array of dictionaries. Each dictionary corresponds to each detected object in the image and it contains the \"name\", \"percentage_probabaility\" and \"box_points\"(x1,y1,x2,y2) values of the object.\n* **Output Count:** This is a dictionary that has the name of each unique object detected as its keys and the number of instances of the objects detected as the values.\n\n**Results for the Second function**\n```\nFOR SECOND : 1\n \n Array for the outputs of each frame [[{'box_points': (362, 295, 443, 355), 'name': 'boat', 'percentage_probability': 26.666194200515747}, {'box_points': (319, 245, 386, 296), 'name': 'boat', 'percentage_probability': 30.052968859672546}, {'box_points': (219, 308, 341, 358), 'name': 'boat', 'percentage_probability': 47.46982455253601}, {'box_points': (589, 198, 621, 241), 'name': 'bus', 'percentage_probability': 24.62330162525177}, {'box_points': (519, 181, 583, 263), 'name': 'bus', 'percentage_probability': 27.446213364601135}, {'box_points': (493, 197, 561, 272), 'name': 'bus', 'percentage_probability': 59.81815457344055}, {'box_points': (432, 187, 491, 240), 'name': 'bus', 'percentage_probability': 64.42965269088745}, {'box_points': (157, 225, 220, 255), 'name': 'car', 'percentage_probability': 21.150341629981995}, {'box_points': (324, 249, 377, 293), 'name': 'car', 'percentage_probability': 24.089913070201874}, {'box_points': (152, 275, 260, 327), 'name': 'car', 'percentage_probability': 30.341443419456482}, {'box_points': (433, 198, 485, 244), 'name': 'car', 'percentage_probability': 37.205660343170166}, {'box_points': (184, 226, 233, 260), 'name': 'car', 'percentage_probability': 38.52525353431702}, {'box_points': (3, 296, 134, 359), 'name': 'car', 'percentage_probability': 47.80363142490387}, {'box_points': (357, 302, 439, 359), 'name': 'car', 'percentage_probability': 47.94844686985016}, {'box_points': (481, 266, 546, 314), 'name': 'car', 'percentage_probability': 65.8585786819458}, {'box_points': (597, 269, 624, 318), 'name': 'person', 'percentage_probability': 27.125394344329834}],\n [{'box_points': (316, 240, 384, 302), 'name': 'boat', 'percentage_probability': 29.594269394874573}, {'box_points': (361, 295, 441, 354), 'name': 'boat', 'percentage_probability': 36.11513376235962}, {'box_points': (216, 305, 340, 357), 'name': 'boat', 'percentage_probability': 44.89373862743378}, {'box_points': (432, 198, 488, 244), 'name': 'truck', 'percentage_probability': 22.914741933345795}, {'box_points': (589, 199, 623, 240), 'name': 'bus', 'percentage_probability': 20.545457303524017}, {'box_points': (519, 182, 583, 263), 'name': 'bus', 'percentage_probability': 24.467085301876068}, {'box_points': (492, 197, 563, 271), 'name': 'bus', 'percentage_probability': 61.112016439437866}, {'box_points': (433, 188, 490, 241), 'name': 'bus', 'percentage_probability': 65.08989334106445}, {'box_points': (352, 303, 442, 357), 'name': 'car', 'percentage_probability': 20.025095343589783}, {'box_points': (136, 172, 188, 195), 'name': 'car', 'percentage_probability': 21.571354568004608}, {'box_points': (152, 276, 261, 326), 'name': 'car', 'percentage_probability': 33.07966589927673}, {'box_points': (181, 225, 230, 256), 'name': 'car', 'percentage_probability': 35.111838579177856}, {'box_points': (432, 198, 488, 244), 'name': 'car', 'percentage_probability': 36.25282347202301}, {'box_points': (3, 292, 130, 360), 'name': 'car', 'percentage_probability': 67.55480170249939}, {'box_points': (479, 265, 546, 314), 'name': 'car', 'percentage_probability': 71.47912979125977}, {'box_points': (597, 269, 625, 318), 'name': 'person', 'percentage_probability': 25.903674960136414}],................, \n[{'box_points': (133, 250, 187, 278), 'name': 'umbrella', 'percentage_probability': 21.518094837665558}, {'box_points': (154, 233, 218, 259), 'name': 'umbrella', 'percentage_probability': 23.687003552913666}, {'box_points': (348, 311, 425, 360), 'name': 'boat', 'percentage_probability': 21.015766263008118}, {'box_points': (11, 164, 137, 225), 'name': 'bus', 'percentage_probability': 32.20453858375549}, {'box_points': (424, 187, 485, 243), 'name': 'bus', 'percentage_probability': 38.043853640556335}, {'box_points': (496, 186, 570, 264), 'name': 'bus', 'percentage_probability': 63.83994221687317}, {'box_points': (588, 197, 622, 240), 'name': 'car', 'percentage_probability': 23.51653128862381}, {'box_points': (58, 268, 111, 303), 'name': 'car', 'percentage_probability': 24.538707733154297}, {'box_points': (2, 246, 72, 301), 'name': 'car', 'percentage_probability': 28.433072566986084}, {'box_points': (472, 273, 539, 323), 'name': 'car', 'percentage_probability': 87.17672824859619}, {'box_points': (597, 270, 626, 317), 'name': 'person', 'percentage_probability': 27.459821105003357}]\n ]\n \nArray for output count for unique objects in each frame : [{'bus': 4, 'boat': 3, 'person': 1, 'car': 8},\n {'truck': 1, 'bus': 4, 'boat': 3, 'person': 1, 'car': 7},\n {'bus': 5, 'boat': 2, 'person': 1, 'car': 5},\n {'bus': 5, 'boat': 1, 'person': 1, 'car': 9},\n {'truck': 1, 'bus': 2, 'car': 6, 'person': 1},\n {'truck': 2, 'bus': 4, 'boat': 2, 'person': 1, 'car': 7},\n {'truck': 1, 'bus': 3, 'car': 7, 'person': 1, 'umbrella': 1},\n {'bus': 4, 'car': 7, 'person': 1, 'umbrella': 2},\n {'bus': 3, 'car': 6, 'boat': 1, 'person': 1, 'umbrella': 3},\n {'bus': 3, 'car': 4, 'boat': 1, 'person': 1, 'umbrella': 2}]\n \nOutput average count for unique objects in the last second: {'truck': 0.5, 'bus': 3.7, 'umbrella': 0.8, 'boat': 1.3, 'person': 1.0, 'car': 6.6}\n\n------------END OF A SECOND --------------\n```\n\nIn the above result, the video was processed and saved in 10 frames per second (FPS). For any function you parse into the **per_second_function**, the function will be executed after every single second of the video that is processed and he following will be parsed into it:\n\n- **Second Index:** This is the position number of the second inside the video (e.g 1 for first second and 20 for twentieth second).\n- **Output Array:** This is an array of arrays, with each contained array and its position (array index + 1) corresponding to the equivalent frame in the last second of the video (In the above example, their are 10 arrays which corresponds to the 10 frames contained in one second). Each contained array contains dictionaries. Each dictionary corresponds to each detected object in the image and it contains the \"name\", \"percentage_probabaility\" and \"box_points\"(x1,y1,x2,y2) values of the object.\n- **Count arrays:** This is an array of dictionaries. Each dictionary and its position (array index + 1) corresponds to the equivalent frame in the last second of he video. Each dictionary has the name of each unique object detected as its keys and the number of instances of the objects detected as the values.\n- **Average Output Count:** This is a dictionary that has the name of each unique object detected in the last second as its keys and the average number of instances of the objects detected across the number of frames as the values.\n\n**Results for the Minute function**\nThe above set of **4 parameters** that are returned for every second of the video processed is the same parameters to that will be returned for every minute of the video processed. The difference is that the index returned corresponds to the minute index, the **output_arrays** is an array that contains the number of FPS * 60 number of arrays (in the code example above, 10 frames per second(fps) * 60 seconds = 600 frames = 600 arrays), and the **count_arrays** is an array that contains the number of FPS * 60 number of dictionaries (in the code example above, 10 frames per second(fps) * 60 seconds = 600 frames = 600 dictionaries) and the **average_output_count** is a dictionary that covers all the objects detected in all the frames contained in the last minute.\n\n**Results for the Video Complete Function**\n**ImageAI** allows you to obtain complete analysis of the entire video processed. All you need is to define a function like the forSecond or forMinute function and set the **video_complete_function** parameter into your `.detectObjectsFromVideo()` or `.detectCustomObjectsFromVideo()` function. The same values for the per_second-function and per_minute_function will be returned. The difference is that no index will be returned and the other 3 values will be returned, and the 3 values will cover all frames in the video. Below is a sample function: \n\n```python\ndef forFull(output_arrays, count_arrays, average_output_count):\n #Perform action on the 3 parameters returned into the function\n\nvideo_detector.detectObjectsFromVideo(\n input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\"),\n frames_per_second=10,\n video_complete_function=forFull,\n minimum_percentage_probability=30\n)\n```\n\n**FINAL NOTE ON VIDEO ANALYSIS** : **ImageAI** allows you to obtain the detected video frame as a Numpy array at each frame, second and minute function. All you need to do is specify one more parameter in your function and set `return_detected_frame=True` in your `detectObjectsFromVideo()` or `detectCustomObjectsFrom()` function. Once this is set, the extra parameter you sepecified in your function will be the Numpy array of the detected frame. See a sample below:\n\n```python\ndef forFrame(frame_number, output_array, output_count, detected_frame):\n print(\"FOR FRAME \" , frame_number)\n print(\"Output for each object : \", output_array)\n print(\"Output count for unique objects : \", output_count)\n\tprint(\"Returned Objects is : \", type(detected_frame))\n print(\"------------END OF A FRAME --------------\")\n\nvideo_detector.detectObjectsFromVideo(\n input_file_path=os.path.join(execution_path, \"traffic.mp4\"),\n output_file_path=os.path.join(execution_path, \"traffic_detected\"),\n frames_per_second=10,\n per_frame_function=forFrame,\n minimum_percentage_probability=30,\n return_detected_frame=True\n)\n```\n\n### Video Detection Speed\n
\n\n**ImageAI** now provides detection speeds for all video object detection tasks. The detection speeds allow you to reduce\n the time of detection at a rate between 20% - 80%, and yet having just slight changes but accurate detection\nresults. Coupled with lowering the **minimum_percentage_probability** parameter, detections can closely match the normal\nspeed and yet reduce detection time drastically. The available detection speeds are **\"normal\"**(default), **\"fast\"**, **\"faster\"** , **\"fastest\"** and **\"flash\"**.\nAll you need to do is to state the speed mode you desire when loading the model as seen below.\n\n```python\ndetector.loadModel(detection_speed=\"fast\")\n```\n\nTo observe the differences in the detection speeds, look below for each speed applied to object detection with\n coupled with the adjustment of the minimum_percentage_probability , time taken to detect and detections given.\nThe results below are obtained from detections performed on a NVIDIA K80 GPU. Links are provided below to download\n the videos for each detection speed applied.\n\nVideo Length = 1min 24seconds, Detection Speed = \"normal\" , Minimum Percentage Probability = 50 (default), Detection Time = 29min 3seconds\n[![](../../data-images/video-4.jpg)](https://www.youtube.com/embed/qplVDqOmElI?rel=0)\n\n\n**Video Length = 1min 24seconds, Detection Speed = \"fast\" , Minimum Percentage Probability = 40, Detection Time = 11min 6seconds**\n\n\n**Video Length = 1min 24seconds, Detection Speed = \"faster\" , Minimum Percentage Probability = 30, Detection Time = 7min 47seconds**\n\n\n**Video Length = 1min 24seconds, Detection Speed = \"fastest\" , Minimum Percentage Probability = 20, Detection Time = 6min 20seconds**\n\n\n**Video Length = 1min 24seconds, Detection Speed = \"flash\" , Minimum Percentage Probability = 10, Detection Time = 3min 55seconds**\n\n\nIf you use more powerful NVIDIA GPUs, you will definitely have faster detection time than stated above.\n\n### Frame Detection Intervals\n
\n\nThe above video objects detection task are optimized for frame-real-time object detections that ensures that objects in every frame of the video is detected. **ImageAI** provides you the option to adjust the video frame detections which can speed up your video detection process. When calling the `.detectObjectsFromVideo()` or `.detectCustomObjectsFromVideo()`, you can specify at which frame interval detections should be made. By setting the **frame_detection_interval** parameter to be equal to 5 or 20, that means the object detections in the video will be updated after 5 frames or 20 frames.\nIf your output video **frames_per_second** is set to 20, that means the object detections in the video will be updated once in every quarter of a second or every second. This is useful in case scenarious where the available compute is less powerful and speeds of moving objects are low. This ensures you can have objects detected as second-real-time , half-a-second-real-time or whichever way suits your needs. We conducted video object detection on the same input video we have been using all this while by applying a **frame_detection_interval** value equal to 5.\nThe results below are obtained from detections performed on a NVIDIA K80 GPU.\nSee the results and link to download the videos below:\n\n\n**Video Length = 1min 24seconds, Detection Speed = \"normal\" , Minimum Percentage Probability = 50 (default), Frame Detection Interval = 5, Detection Time = 15min 49seconds**\n\n\n\n\n**Video Length = 1min 24seconds, Detection Speed = \"fast\" , Minimum Percentage Probability = 40, Frame Detection Interval = 5, Detection Time = 5min 6seconds**\n\n\n\n\n**Video Length = 1min 24seconds, Detection Speed = \"faster\" , Minimum Percentage Probability = 30, Frame Detection Interval = 5, Detection Time = 3min 18seconds**\n\n\n\n\n**Video Length = 1min 24seconds, Detection Speed = \"fastest\" , Minimum Percentage Probability = 20 , Frame Detection Interval = 5, Detection Time = 2min 18seconds**\n[![](../../data-images/video-3.jpg)](https://www.youtube.com/embed/S-jgBTQgbd4?rel=0)\n\n\n**Video Length = 1min 24seconds, Detection Speed = \"flash\" , Minimum Percentage Probability = 10, Frame Detection Interval = 5, Detection Time = 1min 27seconds**\n\n[Download detected video at speed \"flash\" and interval=5](https://drive.google.com/open?id=1aN2nnVoFjhUWpcz2Und3dsCT9OKrakM0)\n\n\n###Video Detection Timeout\n
\n\n**ImageAI** now allows you to set a timeout in seconds for detection of objects in videos or camera live feed.\nTo set a timeout for your video detection code, all you need to do is specify the `detection_timeout` parameter in the `detectObjectsFromVideo()` function to the number of desired seconds. In the example code below, we set `detection_timeout` to 120 seconds (2 minutes). \n\n```python\nfrom imageai.Detection import VideoObjectDetection\nimport os\nimport cv2\n\nexecution_path = os.getcwd()\ncamera = cv2.VideoCapture(0)\n\ndetector = VideoObjectDetection()\ndetector.setModelTypeAsRetinaNet()\ndetector.setModelPath(os.path.join(execution_path , \"resnet50_coco_best_v2.0.1.h5\"))\ndetector.loadModel()\n\n\nvideo_path = detector.detectObjectsFromVideo(camera_input=camera,\n output_file_path=os.path.join(execution_path, \"camera_detected_video\"),\n frames_per_second=20,\n log_progress=True,\n minimum_percentage_probability=40,\n detection_timeout=120)\n```\n\n\n### Documentation\n
\n\nWe have provided full documentation for all **ImageAI** classes and functions in 3 major languages. Find links below: \n\n- Documentation - **English Version [https://imageai.readthedocs.io](https://imageai.readthedocs.io)**\n- Documentation - **Chinese Version [https://imageai-cn.readthedocs.io](https://imageai-cn.readthedocs.io)**\n- Documentation - **French Version [https://imageai-fr.readthedocs.io](https://imageai-fr.readthedocs.io)**\n\n" }, { "path": "imageai_tf_deprecated/Detection/YOLO/__init__.py", "content": "" }, { "path": "imageai_tf_deprecated/Detection/YOLO/utils.py", "content": "import tensorflow as tf\nfrom keras import backend as K\nimport numpy as np\nfrom PIL import Image\nimport cv2\n\n\ndef yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False):\n\n num_anchors = len(anchors)\n\n anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])\n\n grid_shape = K.shape(feats)[1:3]\n grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),\n [1, grid_shape[1], 1, 1])\n grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),\n [grid_shape[0], 1, 1, 1])\n grid = K.concatenate([grid_x, grid_y])\n grid = K.cast(grid, K.dtype(feats))\n\n feats = K.reshape(\n feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])\n\n\n box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(grid_shape[::-1], K.dtype(feats))\n box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats))\n box_confidence = K.sigmoid(feats[..., 4:5])\n box_class_probs = K.sigmoid(feats[..., 5:])\n\n if calc_loss == True:\n return grid, feats, box_xy, box_wh\n return box_xy, box_wh, box_confidence, box_class_probs\n\n\ndef yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape):\n\n box_yx = box_xy[..., ::-1]\n box_hw = box_wh[..., ::-1]\n input_shape = K.cast(input_shape, K.dtype(box_yx))\n image_shape = K.cast(image_shape, K.dtype(box_yx))\n new_shape = K.round(image_shape * K.min(input_shape/image_shape))\n offset = (input_shape-new_shape)/2./input_shape\n scale = input_shape/new_shape\n box_yx = (box_yx - offset) * scale\n box_hw *= scale\n\n box_mins = box_yx - (box_hw / 2.)\n box_maxes = box_yx + (box_hw / 2.)\n boxes = K.concatenate([\n box_mins[..., 0:1],\n box_mins[..., 1:2],\n box_maxes[..., 0:1],\n box_maxes[..., 1:2]\n ])\n\n\n boxes *= K.concatenate([image_shape, image_shape])\n return boxes\n\n\ndef yolo_boxes_and_scores(feats, anchors, num_classes, input_shape, image_shape):\n\n box_xy, box_wh, box_confidence, box_class_probs = yolo_head(feats,\n anchors, num_classes, input_shape)\n boxes = yolo_correct_boxes(box_xy, box_wh, input_shape, image_shape)\n boxes = K.reshape(boxes, [-1, 4])\n box_scores = box_confidence * box_class_probs\n box_scores = K.reshape(box_scores, [-1, num_classes])\n return boxes, box_scores\n\n\ndef yolo_eval(yolo_outputs,\n anchors,\n num_classes,\n image_shape,\n max_boxes=20,\n score_threshold=.6,\n iou_threshold=.5):\n\n num_layers = len(yolo_outputs)\n anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [1,2,3]]\n input_shape = K.shape(yolo_outputs[0])[1:3] * 32\n boxes = []\n box_scores = []\n for l in range(num_layers):\n _boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],\n anchors[anchor_mask[l]], num_classes, input_shape, image_shape)\n boxes.append(_boxes)\n box_scores.append(_box_scores)\n boxes = K.concatenate(boxes, axis=0)\n box_scores = K.concatenate(box_scores, axis=0)\n\n mask = box_scores >= score_threshold\n max_boxes_tensor = K.constant(max_boxes, dtype='int32')\n boxes_ = []\n scores_ = []\n classes_ = []\n for c in range(num_classes):\n class_boxes = tf.boolean_mask(boxes, mask[:, c])\n class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])\n nms_index = tf.image.non_max_suppression(\n class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold)\n class_boxes = K.gather(class_boxes, nms_index)\n class_box_scores = K.gather(class_box_scores, nms_index)\n classes = K.ones_like(class_box_scores, 'int32') * c\n boxes_.append(class_boxes)\n scores_.append(class_box_scores)\n classes_.append(classes)\n boxes_ = K.concatenate(boxes_, axis=0)\n scores_ = K.concatenate(scores_, axis=0)\n classes_ = K.concatenate(classes_, axis=0)\n\n return boxes_, scores_, classes_\n\n\n\ndef letterbox_image(image, size):\n iw, ih = image.size\n w, h = size\n scale = min(w/iw, h/ih)\n nw = int(iw*scale)\n nh = int(ih*scale)\n\n image = image.resize((nw,nh), Image.BICUBIC)\n new_image = Image.new('RGB', size, (128,128,128))\n new_image.paste(image, ((w-nw)//2, (h-nh)//2))\n return new_image\n\n\n\n\ndef correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w):\n if (float(net_w)/image_w) < (float(net_h)/image_h):\n new_w = net_w\n new_h = (image_h*net_w)/image_w\n else:\n new_h = net_w\n new_w = (image_w*net_h)/image_h\n \n for i in range(len(boxes)):\n x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w\n y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h\n \n boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w)\n boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w)\n boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h)\n boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h)\n \n\n\nclass BoundBox:\n def __init__(self, xmin, ymin, xmax, ymax, objness = None, classes = None):\n self.xmin = xmin\n self.ymin = ymin\n self.xmax = xmax\n self.ymax = ymax\n \n self.objness = objness\n self.classes = classes\n\n self.label = -1\n self.score = -1\n\n def get_label(self):\n if self.label == -1:\n self.label = np.argmax(self.classes)\n \n return self.label\n \n def get_score(self):\n if self.score == -1:\n self.score = self.classes[self.get_label()]\n \n return self.score\n\n\ndef _interval_overlap(interval_a, interval_b):\n x1, x2 = interval_a\n x3, x4 = interval_b\n\n if x3 < x1:\n if x4 < x1:\n return 0\n else:\n return min(x2,x4) - x1\n else:\n if x2 < x3:\n return 0\n else:\n return min(x2,x4) - x3 \n\ndef _sigmoid(x):\n return 1. / (1. + np.exp(-x))\n\ndef bbox_iou(box1, box2):\n intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])\n intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax])\n \n intersect = intersect_w * intersect_h\n\n w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin\n w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin\n \n union = w1*h1 + w2*h2 - intersect\n \n return float(intersect) / union\n\n\ndef do_nms(boxes, nms_thresh):\n if len(boxes) > 0:\n nb_class = len(boxes[0].classes)\n else:\n return\n \n for c in range(nb_class):\n sorted_indices = np.argsort([-box.classes[c] for box in boxes])\n\n for i in range(len(sorted_indices)):\n index_i = sorted_indices[i]\n\n if boxes[index_i].classes[c] == 0: continue\n\n for j in range(i+1, len(sorted_indices)):\n index_j = sorted_indices[j]\n\n if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh:\n boxes[index_j].classes[c] = 0\n\ndef decode_netout(netout, anchors, obj_thresh, nms_thresh, net_h, net_w):\n grid_h, grid_w = netout.shape[:2]\n nb_box = 3\n netout = netout.reshape((grid_h, grid_w, nb_box, -1))\n nb_class = netout.shape[-1] - 5\n\n boxes = []\n\n netout[..., :2] = _sigmoid(netout[..., :2])\n netout[..., 4:] = _sigmoid(netout[..., 4:])\n netout[..., 5:] = netout[..., 4][..., np.newaxis] * netout[..., 5:]\n netout[..., 5:] *= netout[..., 5:] > obj_thresh\n\n for i in range(grid_h*grid_w):\n row = i / grid_w\n col = i % grid_w\n \n for b in range(nb_box):\n # 4th element is objectness score\n objectness = netout[int(row)][int(col)][b][4]\n #objectness = netout[..., :4]\n \n if(objectness.all() <= obj_thresh): continue\n \n # first 4 elements are x, y, w, and h\n x, y, w, h = netout[int(row)][int(col)][b][:4]\n\n x = (col + x) / grid_w # center position, unit: image width\n y = (row + y) / grid_h # center position, unit: image height\n w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width\n h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height \n \n # last elements are class probabilities\n classes = netout[int(row)][col][b][5:]\n \n box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes)\n #box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, None, classes)\n\n boxes.append(box)\n\n return boxes\n\ndef preprocess_input(image, input_shape):\n net_h, net_w = input_shape\n new_h, new_w, _ = image.shape\n\n # determine the new size of the image\n if (float(net_w)/new_w) < (float(net_h)/new_h):\n new_h = int((new_h * net_w)/new_w)\n new_w = net_w\n else:\n new_w = int((new_w * net_h)/new_h)\n new_h = net_h\n\n # resize the image to the new size\n resized = cv2.resize(image[:,:,::-1]/255., (int(new_w), int(new_h)))\n\n # embed the image into the standard letter box\n new_image = np.ones((net_h, net_w, 3)) * 0.5\n new_image[int((net_h-new_h)//2):int((net_h+new_h)//2), int((net_w-new_w)//2):int((net_w+new_w)//2), :] = resized\n new_image = np.expand_dims(new_image, 0)\n\n return new_image\n\ndef retrieve_yolo_detections(yolo_result, anchors, min_probability, nms_thresh, image_input_size, image_size, labels_dict ):\n\n boxes = []\n\n for i in range(len(yolo_result)):\n # decode the output of the network\n boxes += decode_netout(yolo_result[i][0], \n anchors[i], \n min_probability, \n nms_thresh, \n image_input_size[0], \n image_input_size[1])\n\n # correct the sizes of the bounding boxes\n correct_yolo_boxes(boxes, image_size[1], image_size[0], image_input_size[0], image_input_size[1])\n\n # suppress non-maximal boxes\n do_nms(boxes, nms_thresh)\n\n detections = list()\n for box in boxes:\n label = -1\n \n for i in range(len(labels_dict.keys())):\n if box.classes[i] > min_probability:\n label = labels_dict[i]\n \n\n percentage_probability = box.classes[i] * 100\n xmin = box.xmin\n ymin = box.ymin\n xmax = box.xmax\n ymax = box.ymax\n \n if xmin < 0:\n xmin = 0\n \n if ymin < 0:\n ymin = 0\n\n detection = dict()\n detection[\"name\"] = label\n detection[\"percentage_probability\"] = percentage_probability\n detection[\"box_points\"] = [ xmin, ymin, xmax, ymax]\n\n detections.append(detection)\n \n return detections\n\n\ndef draw_boxes(image, box_points, draw_box, label, percentage_probability, color):\n \n xmin, ymin, xmax, ymax = box_points\n\n if draw_box is True:\n cv2.rectangle(image, (xmin,ymin), (xmax,ymax), color, 2)\n\n if label is not None:\n if percentage_probability is None:\n label = \"{}\".format(label)\n else:\n label = \"{} {:.2f}%\".format(label, percentage_probability)\n elif percentage_probability is not None:\n label = \"{:.2f}\".format(percentage_probability)\n \n if label is not None or percentage_probability is not None:\n cv2.putText(image, label, (xmin, ymin - 13), cv2.FONT_HERSHEY_SIMPLEX, 1e-3 * image.shape[0], (255, 0, 0), 2)\n cv2.putText(image, label, (xmin, ymin - 13), cv2.FONT_HERSHEY_SIMPLEX, 1e-3 * image.shape[0], (255, 255, 255), 1)\n \n return image " }, { "path": "imageai_tf_deprecated/Detection/YOLO/yolov3.py", "content": "from tensorflow.keras.layers import Conv2D, MaxPool2D, Add, ZeroPadding2D, UpSampling2D, Concatenate, LeakyReLU, Lambda\nfrom tensorflow.keras.layers import LeakyReLU\nfrom tensorflow.keras.layers import BatchNormalization\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import Input\nfrom tensorflow.keras.layers import add, concatenate\nfrom tensorflow.keras.layers import Layer\nimport tensorflow as tf\n\n\n\n\nclass YoloLayer(Layer):\n def __init__(self, anchors, max_grid, batch_size, warmup_batches, ignore_thresh, \n grid_scale, obj_scale, noobj_scale, xywh_scale, class_scale, \n **kwargs):\n # make the model settings persistent\n self.ignore_thresh = ignore_thresh\n self.warmup_batches = warmup_batches\n self.anchors = tf.constant(anchors, dtype='float', shape=[1,1,1,3,2])\n self.grid_scale = grid_scale\n self.obj_scale = obj_scale\n self.noobj_scale = noobj_scale\n self.xywh_scale = xywh_scale\n self.class_scale = class_scale \n\n # make a persistent mesh grid\n max_grid_h, max_grid_w = max_grid\n\n cell_x = tf.cast(tf.reshape(tf.tile(tf.range(max_grid_w), [max_grid_h]), (1, max_grid_h, max_grid_w, 1, 1)), dtype=tf.float32)\n cell_y = tf.transpose(cell_x, (0,2,1,3,4))\n self.cell_grid = tf.tile(tf.concat([cell_x,cell_y],-1), [batch_size, 1, 1, 3, 1])\n\n super(YoloLayer, self).__init__(**kwargs)\n\n def build(self, input_shape):\n super(YoloLayer, self).build(input_shape) # Be sure to call this somewhere!\n\n def call(self, x):\n input_image, y_pred, y_true, true_boxes = x\n\n # adjust the shape of the y_predict [batch, grid_h, grid_w, 3, 4+1+nb_class]\n y_pred = tf.reshape(y_pred, tf.concat([tf.shape(y_pred)[:3], tf.constant([3, -1])], axis=0))\n \n # initialize the masks\n object_mask = tf.expand_dims(y_true[..., 4], 4)\n\n # the variable to keep track of number of batches processed\n batch_seen = tf.Variable(0.) \n\n # compute grid factor and net factor\n grid_h = tf.shape(y_true)[1]\n grid_w = tf.shape(y_true)[2]\n grid_factor = tf.reshape(tf.cast([grid_w, grid_h], tf.float32), [1,1,1,1,2])\n\n net_h = tf.shape(input_image)[1]\n net_w = tf.shape(input_image)[2] \n net_factor = tf.reshape(tf.cast([net_w, net_h], tf.float32), [1,1,1,1,2])\n \n \"\"\"\n Adjust prediction\n \"\"\"\n pred_box_xy = (self.cell_grid[:,:grid_h,:grid_w,:,:] + tf.sigmoid(y_pred[..., :2])) # sigma(t_xy) + c_xy\n pred_box_wh = y_pred[..., 2:4] # t_wh\n pred_box_conf = tf.expand_dims(tf.sigmoid(y_pred[..., 4]), 4) # adjust confidence\n pred_box_class = y_pred[..., 5:] # adjust class probabilities \n\n \"\"\"\n Adjust ground truth\n \"\"\"\n true_box_xy = y_true[..., 0:2] # (sigma(t_xy) + c_xy)\n true_box_wh = y_true[..., 2:4] # t_wh\n true_box_conf = tf.expand_dims(y_true[..., 4], 4)\n true_box_class = tf.argmax(y_true[..., 5:], -1) \n\n \"\"\"\n Compare each predicted box to all true boxes\n \"\"\" \n # initially, drag all objectness of all boxes to 0\n conf_delta = pred_box_conf - 0 \n\n # then, ignore the boxes which have good overlap with some true box\n true_xy = true_boxes[..., 0:2] / grid_factor\n true_wh = true_boxes[..., 2:4] / net_factor\n \n true_wh_half = true_wh / 2.\n true_mins = true_xy - true_wh_half\n true_maxes = true_xy + true_wh_half\n \n pred_xy = tf.expand_dims(pred_box_xy / grid_factor, 4)\n pred_wh = tf.expand_dims(tf.exp(pred_box_wh) * self.anchors / net_factor, 4)\n \n pred_wh_half = pred_wh / 2.\n pred_mins = pred_xy - pred_wh_half\n pred_maxes = pred_xy + pred_wh_half \n\n intersect_mins = tf.maximum(pred_mins, true_mins)\n intersect_maxes = tf.minimum(pred_maxes, true_maxes)\n\n intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)\n intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]\n \n true_areas = true_wh[..., 0] * true_wh[..., 1]\n pred_areas = pred_wh[..., 0] * pred_wh[..., 1]\n\n union_areas = pred_areas + true_areas - intersect_areas\n iou_scores = tf.truediv(intersect_areas, union_areas)\n\n best_ious = tf.reduce_max(iou_scores, axis=4) \n conf_delta *= tf.expand_dims(tf.cast((best_ious < self.ignore_thresh), dtype=tf.float32), 4)\n\n \"\"\"\n Compute some online statistics\n \"\"\" \n true_xy = true_box_xy / grid_factor\n true_wh = tf.exp(true_box_wh) * self.anchors / net_factor\n\n true_wh_half = true_wh / 2.\n true_mins = true_xy - true_wh_half\n true_maxes = true_xy + true_wh_half\n\n pred_xy = pred_box_xy / grid_factor\n pred_wh = tf.exp(pred_box_wh) * self.anchors / net_factor \n \n pred_wh_half = pred_wh / 2.\n pred_mins = pred_xy - pred_wh_half\n pred_maxes = pred_xy + pred_wh_half \n\n intersect_mins = tf.maximum(pred_mins, true_mins)\n intersect_maxes = tf.minimum(pred_maxes, true_maxes)\n intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)\n intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]\n \n true_areas = true_wh[..., 0] * true_wh[..., 1]\n pred_areas = pred_wh[..., 0] * pred_wh[..., 1]\n\n union_areas = pred_areas + true_areas - intersect_areas\n iou_scores = tf.truediv(intersect_areas, union_areas)\n iou_scores = object_mask * tf.expand_dims(iou_scores, 4)\n \n count = tf.reduce_sum(object_mask)\n count_noobj = tf.reduce_sum(1 - object_mask)\n detect_mask = tf.cast((pred_box_conf*object_mask >= 0.5), dtype=tf.float32)\n class_mask = tf.expand_dims(tf.cast(tf.equal(tf.argmax(pred_box_class, -1), true_box_class), dtype=tf.float32), 4)\n recall50 = tf.reduce_sum(tf.cast((iou_scores >= 0.5), dtype=tf.float32) * detect_mask * class_mask) / (count + 1e-3)\n recall75 = tf.reduce_sum(tf.cast((iou_scores >= 0.75), dtype=tf.float32) * detect_mask * class_mask) / (count + 1e-3) \n avg_iou = tf.reduce_sum(iou_scores) / (count + 1e-3)\n avg_obj = tf.reduce_sum(pred_box_conf * object_mask) / (count + 1e-3)\n avg_noobj = tf.reduce_sum(pred_box_conf * (1-object_mask)) / (count_noobj + 1e-3)\n avg_cat = tf.reduce_sum(object_mask * class_mask) / (count + 1e-3) \n\n \"\"\"\n Warm-up training\n \"\"\"\n batch_seen = tf.compat.v1.assign_add(batch_seen, 1.)\n \n true_box_xy, true_box_wh, xywh_mask = tf.cond(tf.less(batch_seen, self.warmup_batches+1), \n lambda: [true_box_xy + (0.5 + self.cell_grid[:,:grid_h,:grid_w,:,:]) * (1-object_mask), \n true_box_wh + tf.zeros_like(true_box_wh) * (1-object_mask), \n tf.ones_like(object_mask)],\n lambda: [true_box_xy, \n true_box_wh,\n object_mask])\n\n \"\"\"\n Compare each true box to all anchor boxes\n \"\"\" \n wh_scale = tf.exp(true_box_wh) * self.anchors / net_factor\n wh_scale = tf.expand_dims(2 - wh_scale[..., 0] * wh_scale[..., 1], axis=4) # the smaller the box, the bigger the scale\n\n xy_delta = xywh_mask * (pred_box_xy-true_box_xy) * wh_scale * self.xywh_scale\n wh_delta = xywh_mask * (pred_box_wh-true_box_wh) * wh_scale * self.xywh_scale\n conf_delta = object_mask * (pred_box_conf-true_box_conf) * self.obj_scale + (1-object_mask) * conf_delta * self.noobj_scale\n class_delta = object_mask * \\\n tf.expand_dims(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class), 4) * \\\n self.class_scale\n\n loss_xy = tf.reduce_sum(tf.square(xy_delta), list(range(1,5)))\n loss_wh = tf.reduce_sum(tf.square(wh_delta), list(range(1,5)))\n loss_conf = tf.reduce_sum(tf.square(conf_delta), list(range(1,5)))\n loss_class = tf.reduce_sum(class_delta, list(range(1,5)))\n\n loss = loss_xy + loss_wh + loss_conf + loss_class\n\n\n return loss*self.grid_scale\n\n def compute_output_shape(self, input_shape):\n return [(None, 1)]\n\n\n\ndef dummy_loss(y_true, y_pred):\n return tf.sqrt(tf.reduce_sum(y_pred))\n\ndef NetworkConv2D_BN_Leaky(input, channels, kernel_size, kernel_regularizer = l2(5e-4), strides=(1,1), padding=\"same\", use_bias=False):\n\n network = Conv2D( filters=channels, kernel_size=kernel_size, strides=strides, padding=padding, kernel_regularizer=kernel_regularizer, use_bias=use_bias)(input)\n network = BatchNormalization()(network)\n network = LeakyReLU(alpha=0.1)(network)\n return network\n\ndef residual_block(input, channels, num_blocks):\n network = ZeroPadding2D(((1,0), (1,0)))(input)\n network = NetworkConv2D_BN_Leaky(input=network,channels=channels, kernel_size=(3,3), strides=(2,2), padding=\"valid\")\n\n for blocks in range(num_blocks):\n network_1 = NetworkConv2D_BN_Leaky(input=network, channels= channels // 2, kernel_size=(1,1))\n network_1 = NetworkConv2D_BN_Leaky(input=network_1,channels= channels, kernel_size=(3,3))\n\n network = Add()([network, network_1])\n return network\n\ndef darknet(input):\n network = NetworkConv2D_BN_Leaky(input=input, channels=32, kernel_size=(3,3))\n network = residual_block(input=network, channels=64, num_blocks=1)\n network = residual_block(input=network, channels=128, num_blocks=2)\n network = residual_block(input=network, channels=256, num_blocks=8)\n network = residual_block(input=network, channels=512, num_blocks=8)\n network = residual_block(input=network, channels=1024, num_blocks=4)\n\n\n return network\n\ndef last_layers(input, channels_in, channels_out, layer_name=\"\"):\n\n\n\n network = NetworkConv2D_BN_Leaky( input=input, channels=channels_in, kernel_size=(1,1))\n network = NetworkConv2D_BN_Leaky(input=network, channels= (channels_in * 2) , kernel_size=(3, 3))\n network = NetworkConv2D_BN_Leaky(input=network, channels=channels_in, kernel_size=(1, 1))\n network = NetworkConv2D_BN_Leaky(input=network, channels=(channels_in * 2), kernel_size=(3, 3))\n network = NetworkConv2D_BN_Leaky(input=network, channels=channels_in, kernel_size=(1, 1))\n\n network_1 = NetworkConv2D_BN_Leaky(input=network, channels=(channels_in * 2), kernel_size=(3, 3))\n network_1 = Conv2D(filters=channels_out, kernel_size=(1,1), name=layer_name)(network_1)\n\n return network, network_1\n\ndef yolov3_base(input, num_anchors, num_classes):\n \n darknet_network = Model(input, darknet(input))\n\n network, network_1 = last_layers(darknet_network.output, 512, num_anchors * (num_classes + 5), layer_name=\"last1\")\n\n network = NetworkConv2D_BN_Leaky( input=network, channels=256, kernel_size=(1,1))\n network = UpSampling2D(2)(network)\n network = Concatenate()([network, darknet_network.layers[152].output])\n\n network, network_2 = last_layers(network, 256, num_anchors * (num_classes + 5), layer_name=\"last2\")\n\n network = NetworkConv2D_BN_Leaky(input=network, channels=128, kernel_size=(1, 1))\n network = UpSampling2D(2)(network)\n network = Concatenate()([network, darknet_network.layers[92].output])\n\n network, network_3 = last_layers(network, 128, num_anchors * (num_classes + 5), layer_name=\"last3\")\n\n return input, network_1, network_2, network_3\n\ndef yolov3_main(input, num_anchors, num_classes):\n\n input, network_1, network_2, network_3 = yolov3_base(input, num_anchors, num_classes)\n\n return Model(input, [network_1, network_2, network_3])\n\n\ndef yolov3_train(num_classes,\n anchors,\n max_box_per_image, \n max_grid, \n batch_size, \n warmup_batches,\n ignore_thresh,\n grid_scales,\n obj_scale,\n noobj_scale,\n xywh_scale,\n class_scale):\n\n input_image = Input(shape=(None, None, 3)) # net_h, net_w, 3\n true_boxes = Input(shape=(1, 1, 1, max_box_per_image, 4))\n true_yolo_1 = Input(shape=(None, None, len(anchors)//6, 4+1+num_classes)) # grid_h, grid_w, nb_anchor, 5+nb_class\n true_yolo_2 = Input(shape=(None, None, len(anchors)//6, 4+1+num_classes)) # grid_h, grid_w, nb_anchor, 5+nb_class\n true_yolo_3 = Input(shape=(None, None, len(anchors)//6, 4+1+num_classes)) # grid_h, grid_w, nb_anchor, 5+nb_class\n \n \n \n _ , network_1, network_2, network_3 = yolov3_base(input_image, len(anchors)//6, num_classes)\n \n loss_yolo_1 = YoloLayer(anchors[12:], \n [1*num for num in max_grid], \n batch_size, \n warmup_batches, \n ignore_thresh, \n grid_scales[0],\n obj_scale,\n noobj_scale,\n xywh_scale,\n class_scale)([input_image, network_1, true_yolo_1, true_boxes])\n\n loss_yolo_2 = YoloLayer(anchors[6:12], \n [2*num for num in max_grid], \n batch_size, \n warmup_batches, \n ignore_thresh, \n grid_scales[1],\n obj_scale,\n noobj_scale,\n xywh_scale,\n class_scale)([input_image, network_2, true_yolo_2, true_boxes])\n\n loss_yolo_3 = YoloLayer(anchors[:6], \n [4*num for num in max_grid], \n batch_size, \n warmup_batches, \n ignore_thresh, \n grid_scales[2],\n obj_scale,\n noobj_scale,\n xywh_scale,\n class_scale)([input_image, network_3, true_yolo_3, true_boxes]) \n\n train_model = Model([input_image, true_boxes, true_yolo_1, true_yolo_2, true_yolo_3], [loss_yolo_1, loss_yolo_2, loss_yolo_3])\n infer_model = Model(input_image, [network_1, network_2, network_3])\n\n return [train_model, infer_model]\n\n\ndef tiny_yolov3_main(input, num_anchors, num_classes):\n\n network_1 = NetworkConv2D_BN_Leaky(input=input, channels=16, kernel_size=(3,3) )\n network_1 = MaxPool2D(pool_size=(2,2), strides=(2,2), padding=\"same\")(network_1)\n network_1 = NetworkConv2D_BN_Leaky(input=network_1, channels=32, kernel_size=(3, 3))\n network_1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\")(network_1)\n network_1 = NetworkConv2D_BN_Leaky(input=network_1, channels=64, kernel_size=(3, 3))\n network_1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\")(network_1)\n network_1 = NetworkConv2D_BN_Leaky(input=network_1, channels=128, kernel_size=(3, 3))\n network_1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\")(network_1)\n network_1 = NetworkConv2D_BN_Leaky(input=network_1, channels=256, kernel_size=(3, 3))\n\n network_2 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\")(network_1)\n network_2 = NetworkConv2D_BN_Leaky(input=network_2, channels=512, kernel_size=(3, 3))\n network_2 = MaxPool2D(pool_size=(2, 2), strides=(1, 1), padding=\"same\")(network_2)\n network_2 = NetworkConv2D_BN_Leaky(input=network_2, channels=1024, kernel_size=(3, 3))\n network_2 = NetworkConv2D_BN_Leaky(input=network_2, channels=256, kernel_size=(1, 1))\n\n network_3 = NetworkConv2D_BN_Leaky(input=network_2, channels=512, kernel_size=(3, 3))\n network_3 = Conv2D(num_anchors * (num_classes + 5), kernel_size=(1,1))(network_3)\n\n network_2 = NetworkConv2D_BN_Leaky(input=network_2, channels=128, kernel_size=(1, 1))\n network_2 = UpSampling2D(2)(network_2)\n\n network_4 = Concatenate()([network_2, network_1])\n network_4 = NetworkConv2D_BN_Leaky(input=network_4, channels=256, kernel_size=(3, 3))\n network_4 = Conv2D(num_anchors * (num_classes + 5), kernel_size=(1,1))(network_4)\n\n return Model(input, [network_3, network_4])\n\ndef dummy_loss(y_true, y_pred):\n return tf.sqrt(tf.reduce_sum(y_pred))" }, { "path": "imageai_tf_deprecated/Detection/__init__.py", "content": "import cv2\nfrom imageai.Detection.keras_retinanet import models as retinanet_models\nfrom imageai.Detection.keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image\nfrom imageai.Detection.keras_retinanet.utils.visualization import draw_box, draw_caption\nimport matplotlib.pyplot as plt\nimport matplotlib.image as pltimage\nimport numpy as np\nimport tensorflow as tf\nimport os\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.layers import Input\nfrom PIL import Image\nimport colorsys\nimport warnings\n\nfrom imageai.Detection.YOLO.yolov3 import tiny_yolov3_main, yolov3_main\nfrom imageai.Detection.YOLO.utils import letterbox_image, yolo_eval, preprocess_input, retrieve_yolo_detections, draw_boxes\n\n\n\nclass ObjectDetection:\n \"\"\"\n This is the object detection class for images in the ImageAI library. It provides support for RetinaNet\n , YOLOv3 and TinyYOLOv3 object detection networks . After instantiating this class, you can set it's properties and\n make object detections using it's pre-defined functions.\n\n The following functions are required to be called before object detection can be made\n * setModelPath()\n * At least of of the following and it must correspond to the model set in the setModelPath()\n [setModelTypeAsRetinaNet(), setModelTypeAsYOLOv3(), setModelTypeAsTinyYOLOv3()]\n * loadModel() [This must be called once only before performing object detection]\n\n Once the above functions have been called, you can call the detectObjectsFromImage() function of\n the object detection instance object at anytime to obtain observable objects in any image.\n \"\"\"\n\n def __init__(self):\n self.__modelType = \"\"\n self.modelPath = \"\"\n self.__modelPathAdded = False\n self.__modelLoaded = False\n self.__model_collection = []\n\n # Instance variables for RetinaNet Model\n self.__input_image_min = 1333\n self.__input_image_max = 800\n\n self.numbers_to_names = {0: 'person', 1: 'bicycle', 2: 'car', 3: 'motorcycle', 4: 'airplane', 5: 'bus',\n 6: 'train',\n 7: 'truck', 8: 'boat', 9: 'traffic light', 10: 'fire hydrant', 11: 'stop sign',\n 12: 'parking meter',\n 13: 'bench', 14: 'bird', 15: 'cat', 16: 'dog', 17: 'horse', 18: 'sheep', 19: 'cow',\n 20: 'elephant',\n 21: 'bear', 22: 'zebra', 23: 'giraffe', 24: 'backpack', 25: 'umbrella', 26: 'handbag',\n 27: 'tie',\n 28: 'suitcase', 29: 'frisbee', 30: 'skis', 31: 'snowboard', 32: 'sports ball',\n 33: 'kite',\n 34: 'baseball bat', 35: 'baseball glove', 36: 'skateboard', 37: 'surfboard',\n 38: 'tennis racket',\n 39: 'bottle', 40: 'wine glass', 41: 'cup', 42: 'fork', 43: 'knife', 44: 'spoon',\n 45: 'bowl',\n 46: 'banana', 47: 'apple', 48: 'sandwich', 49: 'orange', 50: 'broccoli', 51: 'carrot',\n 52: 'hot dog',\n 53: 'pizza', 54: 'donut', 55: 'cake', 56: 'chair', 57: 'couch', 58: 'potted plant',\n 59: 'bed',\n 60: 'dining table', 61: 'toilet', 62: 'tv', 63: 'laptop', 64: 'mouse', 65: 'remote',\n 66: 'keyboard',\n 67: 'cell phone', 68: 'microwave', 69: 'oven', 70: 'toaster', 71: 'sink',\n 72: 'refrigerator',\n 73: 'book', 74: 'clock', 75: 'vase', 76: 'scissors', 77: 'teddy bear',\n 78: 'hair dryer',\n 79: 'toothbrush'}\n\n # Unique instance variables for YOLOv3 and TinyYOLOv3 model\n self.__yolo_iou = 0.45\n self.__yolo_score = 0.1\n self.__nms_thresh = 0.45\n self.__yolo_anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]]\n self.__yolo_model_image_size = (416, 416)\n self.__yolo_boxes, self.__yolo_scores, self.__yolo_classes = \"\", \"\", \"\"\n self.__tiny_yolo_anchors = [[81, 82, 135, 169, 344, 319], [10, 14, 23, 27, 37, 58]]\n self.__box_color = (112, 19, 24)\n \n\n def setModelTypeAsRetinaNet(self):\n \"\"\"\n 'setModelTypeAsRetinaNet()' is used to set the model type to the RetinaNet model\n for the video object detection instance instance object .\n :return:\n \"\"\"\n self.__modelType = \"retinanet\"\n\n def setModelTypeAsYOLOv3(self):\n \"\"\"\n 'setModelTypeAsYOLOv3()' is used to set the model type to the YOLOv3 model\n for the video object detection instance instance object .\n :return:\n \"\"\"\n\n self.__modelType = \"yolov3\"\n\n def setModelTypeAsTinyYOLOv3(self):\n \"\"\"\n 'setModelTypeAsTinyYOLOv3()' is used to set the model type to the TinyYOLOv3 model\n for the video object detection instance instance object .\n :return:\n \"\"\"\n\n self.__modelType = \"tinyyolov3\"\n\n def setModelPath(self, model_path):\n \"\"\"\n 'setModelPath()' function is required and is used to set the file path to a RetinaNet\n object detection model trained on the COCO dataset.\n :param model_path:\n :return:\n \"\"\"\n\n if (self.__modelPathAdded == False):\n self.modelPath = model_path\n self.__modelPathAdded = True\n\n def loadModel(self, detection_speed=\"normal\"):\n \"\"\"\n 'loadModel()' function is required and is used to load the model structure into the program from the file path defined\n in the setModelPath() function. This function receives an optional value which is \"detection_speed\".\n The value is used to reduce the time it takes to detect objects in an image, down to about a 10% of the normal time, with\n with just slight reduction in the number of objects detected.\n\n\n * prediction_speed (optional); Acceptable values are \"normal\", \"fast\", \"faster\", \"fastest\" and \"flash\"\n\n :param detection_speed:\n :return:\n \"\"\"\n\n if (self.__modelType == \"retinanet\"):\n if (detection_speed == \"normal\"):\n self.__input_image_min = 800\n self.__input_image_max = 1333\n elif (detection_speed == \"fast\"):\n self.__input_image_min = 400\n self.__input_image_max = 700\n elif (detection_speed == \"faster\"):\n self.__input_image_min = 300\n self.__input_image_max = 500\n elif (detection_speed == \"fastest\"):\n self.__input_image_min = 200\n self.__input_image_max = 350\n elif (detection_speed == \"flash\"):\n self.__input_image_min = 100\n self.__input_image_max = 250\n elif (self.__modelType == \"yolov3\"):\n if (detection_speed == \"normal\"):\n self.__yolo_model_image_size = (416, 416)\n elif (detection_speed == \"fast\"):\n self.__yolo_model_image_size = (320, 320)\n elif (detection_speed == \"faster\"):\n self.__yolo_model_image_size = (208, 208)\n elif (detection_speed == \"fastest\"):\n self.__yolo_model_image_size = (128, 128)\n elif (detection_speed == \"flash\"):\n self.__yolo_model_image_size = (96, 96)\n\n elif (self.__modelType == \"tinyyolov3\"):\n if (detection_speed == \"normal\"):\n self.__yolo_model_image_size = (832, 832)\n elif (detection_speed == \"fast\"):\n self.__yolo_model_image_size = (576, 576)\n elif (detection_speed == \"faster\"):\n self.__yolo_model_image_size = (416, 416)\n elif (detection_speed == \"fastest\"):\n self.__yolo_model_image_size = (320, 320)\n elif (detection_speed == \"flash\"):\n self.__yolo_model_image_size = (272, 272)\n\n if (self.__modelLoaded == False):\n if (self.__modelType == \"\"):\n raise ValueError(\"You must set a valid model type before loading the model.\")\n elif (self.__modelType == \"retinanet\"):\n model = retinanet_models.load_model(self.modelPath, backbone_name='resnet50')\n self.__model_collection.append(model)\n self.__modelLoaded = True\n elif (self.__modelType == \"yolov3\" or self.__modelType == \"tinyyolov3\"):\n\n input_image = Input(shape=(None, None, 3))\n\n if self.__modelType == \"yolov3\":\n model = yolov3_main(input_image, len(self.__yolo_anchors),\n len(self.numbers_to_names.keys()))\n else:\n model = tiny_yolov3_main(input_image, 3,\n len(self.numbers_to_names.keys()))\n\n model.load_weights(self.modelPath)\n \n self.__model_collection.append(model)\n self.__modelLoaded = True\n\n def detectObjectsFromImage(self, input_image=\"\", output_image_path=\"\", input_type=\"file\", output_type=\"file\",\n extract_detected_objects=False, minimum_percentage_probability=50,\n display_percentage_probability=True, display_object_name=True,\n display_box=True, thread_safe=False, custom_objects=None):\n \"\"\"\n 'detectObjectsFromImage()' function is used to detect objects observable in the given image path:\n * input_image , which can be a filepath, image numpy array or image file stream\n * output_image_path (only if output_type = file) , file path to the output image that will contain the detection boxes and label, if output_type=\"file\"\n * input_type (optional) , file path/numpy array/image file stream of the image. Acceptable values are \"file\", \"array\" and \"stream\"\n * output_type (optional) , file path/numpy array/image file stream of the image. Acceptable values are \"file\" and \"array\"\n * extract_detected_objects (optional) , option to save each object detected individually as an image and return an array of the objects' image path.\n * minimum_percentage_probability (optional, 50 by default) , option to set the minimum percentage probability for nominating a detected object for output.\n * display_percentage_probability (optional, True by default), option to show or hide the percentage probability of each object in the saved/returned detected image\n * display_display_object_name (optional, True by default), option to show or hide the name of each object in the saved/returned detected image\n * thread_safe (optional, False by default), enforce the loaded detection model works across all threads if set to true, made possible by forcing all Tensorflow inference to run on the default graph.\n\n\n The values returned by this function depends on the parameters parsed. The possible values returnable\n are stated as below\n - If extract_detected_objects = False or at its default value and output_type = 'file' or\n at its default value, you must parse in the 'output_image_path' as a string to the path you want\n the detected image to be saved. Then the function will return:\n 1. an array of dictionaries, with each dictionary corresponding to the objects\n detected in the image. Each dictionary contains the following property:\n * name (string)\n * percentage_probability (float)\n * box_points (list of x1,y1,x2 and y2 coordinates)\n\n - If extract_detected_objects = False or at its default value and output_type = 'array' ,\n Then the function will return:\n\n 1. a numpy array of the detected image\n 2. an array of dictionaries, with each dictionary corresponding to the objects\n detected in the image. Each dictionary contains the following property:\n * name (string)\n * percentage_probability (float)\n * box_points (list of x1,y1,x2 and y2 coordinates)\n\n - If extract_detected_objects = True and output_type = 'file' or\n at its default value, you must parse in the 'output_image_path' as a string to the path you want\n the detected image to be saved. Then the function will return:\n 1. an array of dictionaries, with each dictionary corresponding to the objects\n detected in the image. Each dictionary contains the following property:\n * name (string)\n * percentage_probability (float)\n * box_points (list of x1,y1,x2 and y2 coordinates)\n 2. an array of string paths to the image of each object extracted from the image\n\n - If extract_detected_objects = True and output_type = 'array', the the function will return:\n 1. a numpy array of the detected image\n 2. an array of dictionaries, with each dictionary corresponding to the objects\n detected in the image. Each dictionary contains the following property:\n * name (string)\n * percentage_probability (float)\n * box_points (list of x1,y1,x2 and y2 coordinates)\n 3. an array of numpy arrays of each object detected in the image\n\n\n :param input_image:\n :param output_image_path:\n :param input_type:\n :param output_type:\n :param extract_detected_objects:\n :param minimum_percentage_probability:\n :param display_percentage_probability:\n :param display_object_name:\n :param thread_safe:\n :return image_frame:\n :return output_objects_array:\n :return detected_objects_image_array:\n \"\"\"\n\n if (self.__modelLoaded == False):\n raise ValueError(\"You must call the loadModel() function before making object detection.\")\n elif (self.__modelLoaded == True):\n try:\n\n model_detections = list()\n detections = list()\n image_copy = None\n\n detected_objects_image_array = []\n min_probability = minimum_percentage_probability / 100\n\n if (input_type == \"file\"):\n input_image = cv2.imread(input_image)\n elif (input_type == \"array\"):\n input_image = np.array(input_image)\n\n detected_copy = input_image\n image_copy = input_image\n\n if (self.__modelType == \"yolov3\" or self.__modelType == \"tinyyolov3\"):\n\n image_h, image_w, _ = detected_copy.shape\n detected_copy = preprocess_input(detected_copy, self.__yolo_model_image_size)\n\n model = self.__model_collection[0]\n yolo_result = model.predict(detected_copy)\n\n model_detections = retrieve_yolo_detections(yolo_result,\n self.__yolo_anchors,\n min_probability,\n self.__nms_thresh,\n self.__yolo_model_image_size,\n (image_w, image_h),\n self.numbers_to_names)\n \n elif (self.__modelType == \"retinanet\"):\n detected_copy = preprocess_image(detected_copy)\n detected_copy, scale = resize_image(detected_copy)\n\n model = self.__model_collection[0]\n boxes, scores, labels = model.predict_on_batch(np.expand_dims(detected_copy, axis=0))\n\n \n boxes /= scale\n\n for box, score, label in zip(boxes[0], scores[0], labels[0]):\n # scores are sorted so we can break\n if score < min_probability:\n break\n\n detection_dict = dict()\n detection_dict[\"name\"] = self.numbers_to_names[label]\n detection_dict[\"percentage_probability\"] = score * 100\n detection_dict[\"box_points\"] = box.astype(int).tolist()\n model_detections.append(detection_dict)\n\n counting = 0\n objects_dir = output_image_path + \"-objects\"\n\n for detection in model_detections:\n counting += 1\n label = detection[\"name\"]\n percentage_probability = detection[\"percentage_probability\"]\n box_points = detection[\"box_points\"]\n\n if (custom_objects is not None):\n if (custom_objects[label] != \"valid\"):\n continue\n \n detections.append(detection)\n\n if display_object_name == False:\n label = None\n\n if display_percentage_probability == False:\n percentage_probability = None\n\n \n image_copy = draw_boxes(image_copy, \n box_points,\n display_box,\n label, \n percentage_probability, \n self.__box_color)\n \n \n\n if (extract_detected_objects == True):\n splitted_copy = image_copy.copy()[box_points[1]:box_points[3],\n box_points[0]:box_points[2]]\n if (output_type == \"file\"):\n if (os.path.exists(objects_dir) == False):\n os.mkdir(objects_dir)\n splitted_image_path = os.path.join(objects_dir,\n detection[\"name\"] + \"-\" + str(\n counting) + \".jpg\")\n cv2.imwrite(splitted_image_path, splitted_copy)\n detected_objects_image_array.append(splitted_image_path)\n elif (output_type == \"array\"):\n detected_objects_image_array.append(splitted_copy)\n\n \n if (output_type == \"file\"):\n cv2.imwrite(output_image_path, image_copy)\n\n if (extract_detected_objects == True):\n if (output_type == \"file\"):\n return detections, detected_objects_image_array\n elif (output_type == \"array\"):\n return image_copy, detections, detected_objects_image_array\n\n else:\n if (output_type == \"file\"):\n return detections\n elif (output_type == \"array\"):\n return image_copy, detections\n\n except:\n raise ValueError(\n \"Ensure you specified correct input image, input type, output type and/or output image path \")\n\n def CustomObjects(self, person=False, bicycle=False, car=False, motorcycle=False, airplane=False,\n bus=False, train=False, truck=False, boat=False, traffic_light=False, fire_hydrant=False,\n stop_sign=False,\n parking_meter=False, bench=False, bird=False, cat=False, dog=False, horse=False, sheep=False,\n cow=False, elephant=False, bear=False, zebra=False,\n giraffe=False, backpack=False, umbrella=False, handbag=False, tie=False, suitcase=False,\n frisbee=False, skis=False, snowboard=False,\n sports_ball=False, kite=False, baseball_bat=False, baseball_glove=False, skateboard=False,\n surfboard=False, tennis_racket=False,\n bottle=False, wine_glass=False, cup=False, fork=False, knife=False, spoon=False, bowl=False,\n banana=False, apple=False, sandwich=False, orange=False,\n broccoli=False, carrot=False, hot_dog=False, pizza=False, donut=False, cake=False, chair=False,\n couch=False, potted_plant=False, bed=False,\n dining_table=False, toilet=False, tv=False, laptop=False, mouse=False, remote=False,\n keyboard=False, cell_phone=False, microwave=False,\n oven=False, toaster=False, sink=False, refrigerator=False, book=False, clock=False, vase=False,\n scissors=False, teddy_bear=False, hair_dryer=False,\n toothbrush=False):\n\n \"\"\"\n The 'CustomObjects()' function allows you to handpick the type of objects you want to detect\n from an image. The objects are pre-initiated in the function variables and predefined as 'False',\n which you can easily set to true for any number of objects available. This function\n returns a dictionary which must be parsed into the 'detectCustomObjectsFromImage()'. Detecting\n custom objects only happens when you call the function 'detectCustomObjectsFromImage()'\n\n\n * true_values_of_objects (array); Acceptable values are 'True' and False for all object values present\n\n :param boolean_values:\n :return: custom_objects_dict\n \"\"\"\n\n custom_objects_dict = {}\n input_values = [person, bicycle, car, motorcycle, airplane,\n bus, train, truck, boat, traffic_light, fire_hydrant, stop_sign,\n parking_meter, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra,\n giraffe, backpack, umbrella, handbag, tie, suitcase, frisbee, skis, snowboard,\n sports_ball, kite, baseball_bat, baseball_glove, skateboard, surfboard, tennis_racket,\n bottle, wine_glass, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange,\n broccoli, carrot, hot_dog, pizza, donut, cake, chair, couch, potted_plant, bed,\n dining_table, toilet, tv, laptop, mouse, remote, keyboard, cell_phone, microwave,\n oven, toaster, sink, refrigerator, book, clock, vase, scissors, teddy_bear, hair_dryer,\n toothbrush]\n actual_labels = [\"person\", \"bicycle\", \"car\", \"motorcycle\", \"airplane\",\n \"bus\", \"train\", \"truck\", \"boat\", \"traffic light\", \"fire hydrant\", \"stop sign\",\n \"parking meter\", \"bench\", \"bird\", \"cat\", \"dog\", \"horse\", \"sheep\", \"cow\", \"elephant\", \"bear\",\n \"zebra\",\n \"giraffe\", \"backpack\", \"umbrella\", \"handbag\", \"tie\", \"suitcase\", \"frisbee\", \"skis\",\n \"snowboard\",\n \"sports ball\", \"kite\", \"baseball bat\", \"baseball glove\", \"skateboard\", \"surfboard\",\n \"tennis racket\",\n \"bottle\", \"wine glass\", \"cup\", \"fork\", \"knife\", \"spoon\", \"bowl\", \"banana\", \"apple\", \"sandwich\",\n \"orange\",\n \"broccoli\", \"carrot\", \"hot dog\", \"pizza\", \"donut\", \"cake\", \"chair\", \"couch\", \"potted plant\",\n \"bed\",\n \"dining table\", \"toilet\", \"tv\", \"laptop\", \"mouse\", \"remote\", \"keyboard\", \"cell phone\",\n \"microwave\",\n \"oven\", \"toaster\", \"sink\", \"refrigerator\", \"book\", \"clock\", \"vase\", \"scissors\", \"teddy bear\",\n \"hair dryer\",\n \"toothbrush\"]\n\n for input_value, actual_label in zip(input_values, actual_labels):\n if (input_value == True):\n custom_objects_dict[actual_label] = \"valid\"\n else:\n custom_objects_dict[actual_label] = \"invalid\"\n\n return custom_objects_dict\n\n def detectCustomObjectsFromImage(self, input_image=\"\", output_image_path=\"\", input_type=\"file\", output_type=\"file\",\n extract_detected_objects=False, minimum_percentage_probability=50,\n display_percentage_probability=True, display_object_name=True,\n display_box=True, thread_safe=False, custom_objects=None):\n \n warnings.warn(\"'detectCustomObjectsFromImage()' function has been deprecated and will be removed in future versions of ImageAI. \\n Kindly use 'detectObjectsFromImage()' \",\n DeprecationWarning, stacklevel=2)\n \n return self.detectObjectsFromImage(input_image=input_image,\n output_image_path=output_image_path, \n input_type=input_type, \n output_type=output_type,\n extract_detected_objects=extract_detected_objects, \n minimum_percentage_probability=minimum_percentage_probability,\n display_percentage_probability=display_percentage_probability, \n display_object_name=display_object_name,\n display_box=display_box, \n thread_safe=thread_safe, \n custom_objects=custom_objects)\n \n\nclass VideoObjectDetection:\n \"\"\"\n This is the object detection class for videos and camera live stream inputs in the ImageAI library. It provides support for RetinaNet,\n YOLOv3 and TinyYOLOv3 object detection networks. After instantiating this class, you can set it's properties and\n make object detections using it's pre-defined functions.\n\n The following functions are required to be called before object detection can be made\n * setModelPath()\n * At least of of the following and it must correspond to the model set in the setModelPath()\n [setModelTypeAsRetinaNet(), setModelTypeAsYOLOv3(), setModelTinyYOLOv3()]\n * loadModel() [This must be called once only before performing object detection]\n\n Once the above functions have been called, you can call the detectObjectsFromVideo() function\n or the detectCustomObjectsFromVideo() of the object detection instance object at anytime to\n obtain observable objects in any video or camera live stream.\n \"\"\"\n\n def __init__(self):\n self.__modelType = \"\"\n self.modelPath = \"\"\n self.__modelPathAdded = False\n self.__modelLoaded = False\n self.__detector = None\n self.__input_image_min = 1333\n self.__input_image_max = 800\n self.__detection_storage = None\n\n\n self.numbers_to_names = {0: 'person', 1: 'bicycle', 2: 'car', 3: 'motorcycle', 4: 'airplane', 5: 'bus',\n 6: 'train',\n 7: 'truck', 8: 'boat', 9: 'traffic light', 10: 'fire hydrant', 11: 'stop sign',\n 12: 'parking meter',\n 13: 'bench', 14: 'bird', 15: 'cat', 16: 'dog', 17: 'horse', 18: 'sheep', 19: 'cow',\n 20: 'elephant',\n 21: 'bear', 22: 'zebra', 23: 'giraffe', 24: 'backpack', 25: 'umbrella', 26: 'handbag',\n 27: 'tie',\n 28: 'suitcase', 29: 'frisbee', 30: 'skis', 31: 'snowboard', 32: 'sports ball',\n 33: 'kite',\n 34: 'baseball bat', 35: 'baseball glove', 36: 'skateboard', 37: 'surfboard',\n 38: 'tennis racket',\n 39: 'bottle', 40: 'wine glass', 41: 'cup', 42: 'fork', 43: 'knife', 44: 'spoon',\n 45: 'bowl',\n 46: 'banana', 47: 'apple', 48: 'sandwich', 49: 'orange', 50: 'broccoli', 51: 'carrot',\n 52: 'hot dog',\n 53: 'pizza', 54: 'donut', 55: 'cake', 56: 'chair', 57: 'couch', 58: 'potted plant',\n 59: 'bed',\n 60: 'dining table', 61: 'toilet', 62: 'tv', 63: 'laptop', 64: 'mouse', 65: 'remote',\n 66: 'keyboard',\n 67: 'cell phone', 68: 'microwave', 69: 'oven', 70: 'toaster', 71: 'sink',\n 72: 'refrigerator',\n 73: 'book', 74: 'clock', 75: 'vase', 76: 'scissors', 77: 'teddy bear',\n 78: 'hair dryer',\n 79: 'toothbrush'}\n\n def setModelTypeAsRetinaNet(self):\n \"\"\"\n 'setModelTypeAsRetinaNet()' is used to set the model type to the RetinaNet model\n for the video object detection instance instance object .\n :return:\n \"\"\"\n self.__modelType = \"retinanet\"\n\n def setModelTypeAsYOLOv3(self):\n \"\"\"\n 'setModelTypeAsYOLOv3()' is used to set the model type to the YOLOv3 model\n for the video object detection instance instance object .\n :return:\n \"\"\"\n self.__modelType = \"yolov3\"\n\n def setModelTypeAsTinyYOLOv3(self):\n \"\"\"\n 'setModelTypeAsTinyYOLOv3()' is used to set the model type to the TinyYOLOv3 model\n for the video object detection instance instance object .\n :return:\n \"\"\"\n self.__modelType = \"tinyyolov3\"\n\n def setModelPath(self, model_path):\n \"\"\"\n 'setModelPath()' function is required and is used to set the file path to a RetinaNet,\n YOLOv3 or TinyYOLOv3 object detection model trained on the COCO dataset.\n :param model_path:\n :return:\n \"\"\"\n\n if (self.__modelPathAdded == False):\n self.modelPath = model_path\n self.__modelPathAdded = True\n\n def loadModel(self, detection_speed=\"normal\"):\n \"\"\"\n 'loadModel()' function is required and is used to load the model structure into the program from the file path defined\n in the setModelPath() function. This function receives an optional value which is \"detection_speed\".\n The value is used to reduce the time it takes to detect objects in an image, down to about a 10% of the normal time, with\n with just slight reduction in the number of objects detected.\n\n\n * prediction_speed (optional); Acceptable values are \"normal\", \"fast\", \"faster\", \"fastest\" and \"flash\"\n\n :param detection_speed:\n :return:\n \"\"\"\n\n if (self.__modelLoaded == False):\n\n frame_detector = ObjectDetection()\n\n if (self.__modelType == \"retinanet\"):\n frame_detector.setModelTypeAsRetinaNet()\n elif (self.__modelType == \"yolov3\"):\n frame_detector.setModelTypeAsYOLOv3()\n elif (self.__modelType == \"tinyyolov3\"):\n frame_detector.setModelTypeAsTinyYOLOv3()\n frame_detector.setModelPath(self.modelPath)\n frame_detector.loadModel(detection_speed)\n self.__detector = frame_detector\n self.__modelLoaded = True\n\n\n def detectObjectsFromVideo(self, input_file_path=\"\", camera_input=None, output_file_path=\"\", frames_per_second=20,\n frame_detection_interval=1, minimum_percentage_probability=50, log_progress=False,\n display_percentage_probability=True, display_object_name=True, display_box=True, save_detected_video=True,\n per_frame_function=None, per_second_function=None, per_minute_function=None,\n video_complete_function=None, return_detected_frame=False, detection_timeout = None, \n thread_safe=False, custom_objects=None):\n\n \"\"\"\n 'detectObjectsFromVideo()' function is used to detect objects observable in the given video path or a camera input:\n * input_file_path , which is the file path to the input video. It is required only if 'camera_input' is not set\n * camera_input , allows you to parse in camera input for live video detections\n * output_file_path , which is the path to the output video. It is required only if 'save_detected_video' is not set to False\n * frames_per_second , which is the number of frames to be used in the output video\n * frame_detection_interval (optional, 1 by default) , which is the intervals of frames that will be detected.\n * minimum_percentage_probability (optional, 50 by default) , option to set the minimum percentage probability for nominating a detected object for output.\n * log_progress (optional) , which states if the progress of the frame processed is to be logged to console\n * display_percentage_probability (optional), can be used to hide or show probability scores on the detected video frames\n * display_object_name (optional), can be used to show or hide object names on the detected video frames\n * save_save_detected_video (optional, True by default), can be set to or not to save the detected video\n * per_frame_function (optional), this parameter allows you to parse in a function you will want to execute after each frame of the video is detected. If this parameter is set to a function, after every video frame is detected, the function will be executed with the following values parsed into it:\n -- position number of the frame\n -- an array of dictinaries, with each dictionary corresponding to each object detected. Each dictionary contains 'name', 'percentage_probability' and 'box_points'\n -- a dictionary with with keys being the name of each unique objects and value are the number of instances of the object present\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed as the fourth value into the function\n\n * per_second_function (optional), this parameter allows you to parse in a function you will want to execute after each second of the video is detected. If this parameter is set to a function, after every second of a video is detected, the function will be executed with the following values parsed into it:\n -- position number of the second\n -- an array of dictionaries whose keys are position number of each frame present in the last second , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n -- an array of dictionaries, with each dictionary corresponding to each frame in the past second, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n -- a dictionary with its keys being the name of each unique object detected throughout the past second, and the key values are the average number of instances of the object found in all the frames contained in the past second\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed\n as the fifth value into the function\n\n * per_minute_function (optional), this parameter allows you to parse in a function you will want to execute after each minute of the video is detected. If this parameter is set to a function, after every minute of a video is detected, the function will be executed with the following values parsed into it:\n -- position number of the minute\n -- an array of dictionaries whose keys are position number of each frame present in the last minute , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n\n -- an array of dictionaries, with each dictionary corresponding to each frame in the past minute, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n\n -- a dictionary with its keys being the name of each unique object detected throughout the past minute, and the key values are the average number of instances of the object found in all the frames contained in the past minute\n\n -- If return_detected_frame is set to True, the numpy array of the detected frame will be parsed as the fifth value into the function\n\n * video_complete_function (optional), this parameter allows you to parse in a function you will want to execute after all of the video frames have been detected. If this parameter is set to a function, after all of frames of a video is detected, the function will be executed with the following values parsed into it:\n -- an array of dictionaries whose keys are position number of each frame present in the entire video , and the value for each key is the array for each frame that contains the dictionaries for each object detected in the frame\n -- an array of dictionaries, with each dictionary corresponding to each frame in the entire video, and the keys of each dictionary are the name of the number of unique objects detected in each frame, and the key values are the number of instances of the objects found in the frame\n -- a dictionary with its keys being the name of each unique object detected throughout the entire video, and the key values are the average number of instances of the object found in all the frames contained in the entire video\n\n * return_detected_frame (optionally, False by default), option to obtain the return the last detected video frame into the per_per_frame_function, per_per_second_function or per_per_minute_function\n\n * detection_timeout (optionally, None by default), option to state the number of seconds of a video that should be detected after which the detection function stop processing the video\n * thread_safe (optional, False by default), enforce the loaded detection model works across all threads if set to true, made possible by forcing all Tensorflow inference to run on the default graph.\n\n\n :param input_file_path:\n :param camera_input\n :param output_file_path:\n :param save_detected_video:\n :param frames_per_second:\n :param frame_detection_interval:\n :param minimum_percentage_probability:\n :param log_progress:\n :param display_percentage_probability:\n :param display_object_name:\n :param per_frame_function:\n :param per_second_function:\n :param per_minute_function:\n :param video_complete_function:\n :param return_detected_frame:\n :param detection_timeout:\n :param thread_safe:\n :return output_video_filepath:\n :return counting:\n :return output_objects_array:\n :return output_objects_count:\n :return detected_copy:\n :return this_second_output_object_array:\n :return this_second_counting_array:\n :return this_second_counting:\n :return this_minute_output_object_array:\n :return this_minute_counting_array:\n :return this_minute_counting:\n :return this_video_output_object_array:\n :return this_video_counting_array:\n :return this_video_counting:\n \"\"\"\n\n if (input_file_path == \"\" and camera_input == None):\n raise ValueError(\n \"You must set 'input_file_path' to a valid video file, or set 'camera_input' to a valid camera\")\n elif (save_detected_video == True and output_file_path == \"\"):\n raise ValueError(\n \"You must set 'output_video_filepath' to a valid video file name, in which the detected video will be saved. If you don't intend to save the detected video, set 'save_detected_video=False'\")\n\n else:\n try:\n\n output_frames_dict = {}\n output_frames_count_dict = {}\n\n input_video = cv2.VideoCapture(input_file_path)\n if (camera_input != None):\n input_video = camera_input\n\n output_video_filepath = output_file_path + '.avi'\n\n frame_width = int(input_video.get(3))\n frame_height = int(input_video.get(4))\n output_video = cv2.VideoWriter(output_video_filepath, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),\n frames_per_second,\n (frame_width, frame_height))\n\n counting = 0\n\n detection_timeout_count = 0\n video_frames_count = 0\n\n while (input_video.isOpened()):\n ret, frame = input_video.read()\n\n if (ret == True):\n\n video_frames_count += 1\n if (detection_timeout != None):\n if ((video_frames_count % frames_per_second) == 0):\n detection_timeout_count += 1\n\n if (detection_timeout_count >= detection_timeout):\n break\n\n output_objects_array = []\n\n counting += 1\n\n if (log_progress == True):\n print(\"Processing Frame : \", str(counting))\n\n detected_copy = frame.copy()\n\n check_frame_interval = counting % frame_detection_interval\n\n if (counting == 1 or check_frame_interval == 0):\n try:\n detected_copy, output_objects_array = self.__detector.detectObjectsFromImage(\n input_image=frame, input_type=\"array\", output_type=\"array\",\n minimum_percentage_probability=minimum_percentage_probability,\n display_percentage_probability=display_percentage_probability,\n display_object_name=display_object_name,\n display_box=display_box,\n custom_objects=custom_objects)\n except:\n None\n\n output_frames_dict[counting] = output_objects_array\n\n output_objects_count = {}\n for eachItem in output_objects_array:\n eachItemName = eachItem[\"name\"]\n try:\n output_objects_count[eachItemName] = output_objects_count[eachItemName] + 1\n except:\n output_objects_count[eachItemName] = 1\n\n output_frames_count_dict[counting] = output_objects_count\n\n \n if (save_detected_video == True):\n output_video.write(detected_copy)\n\n if (counting == 1 or check_frame_interval == 0):\n if (per_frame_function != None):\n if (return_detected_frame == True):\n per_frame_function(counting, output_objects_array, output_objects_count,\n detected_copy)\n elif (return_detected_frame == False):\n per_frame_function(counting, output_objects_array, output_objects_count)\n\n if (per_second_function != None):\n if (counting != 1 and (counting % frames_per_second) == 0):\n\n this_second_output_object_array = []\n this_second_counting_array = []\n this_second_counting = {}\n\n for aa in range(counting):\n if (aa >= (counting - frames_per_second)):\n this_second_output_object_array.append(output_frames_dict[aa + 1])\n this_second_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_second_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_second_counting[eachItem] = this_second_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_second_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_second_counting:\n this_second_counting[eachCountingItem] = int(this_second_counting[eachCountingItem] / frames_per_second)\n\n if (return_detected_frame == True):\n per_second_function(int(counting / frames_per_second),\n this_second_output_object_array, this_second_counting_array,\n this_second_counting, detected_copy)\n\n elif (return_detected_frame == False):\n per_second_function(int(counting / frames_per_second),\n this_second_output_object_array, this_second_counting_array,\n this_second_counting)\n\n if (per_minute_function != None):\n\n if (counting != 1 and (counting % (frames_per_second * 60)) == 0):\n\n this_minute_output_object_array = []\n this_minute_counting_array = []\n this_minute_counting = {}\n\n for aa in range(counting):\n if (aa >= (counting - (frames_per_second * 60))):\n this_minute_output_object_array.append(output_frames_dict[aa + 1])\n this_minute_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_minute_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_minute_counting[eachItem] = this_minute_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_minute_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_minute_counting:\n this_minute_counting[eachCountingItem] = int(this_minute_counting[eachCountingItem] / (frames_per_second * 60))\n\n if (return_detected_frame == True):\n per_minute_function(int(counting / (frames_per_second * 60)),\n this_minute_output_object_array, this_minute_counting_array,\n this_minute_counting, detected_copy)\n\n elif (return_detected_frame == False):\n per_minute_function(int(counting / (frames_per_second * 60)),\n this_minute_output_object_array, this_minute_counting_array,\n this_minute_counting)\n\n\n else:\n break\n\n if (video_complete_function != None):\n\n this_video_output_object_array = []\n this_video_counting_array = []\n this_video_counting = {}\n\n for aa in range(counting):\n this_video_output_object_array.append(output_frames_dict[aa + 1])\n this_video_counting_array.append(output_frames_count_dict[aa + 1])\n\n for eachCountingDict in this_video_counting_array:\n for eachItem in eachCountingDict:\n try:\n this_video_counting[eachItem] = this_video_counting[eachItem] + \\\n eachCountingDict[eachItem]\n except:\n this_video_counting[eachItem] = eachCountingDict[eachItem]\n\n for eachCountingItem in this_video_counting:\n this_video_counting[eachCountingItem] = int(this_video_counting[eachCountingItem] / counting)\n\n video_complete_function(this_video_output_object_array, this_video_counting_array,\n this_video_counting)\n\n input_video.release()\n output_video.release()\n\n if (save_detected_video == True):\n return output_video_filepath\n\n except:\n raise ValueError(\n \"An error occured. It may be that your input video is invalid. Ensure you specified a proper string value for 'output_file_path' is 'save_detected_video' is not False. \"\n \"Also ensure your per_frame, per_second, per_minute or video_complete_analysis function is properly configured to receive the right parameters. \")\n\n def CustomObjects(self, person=False, bicycle=False, car=False, motorcycle=False, airplane=False,\n bus=False, train=False, truck=False, boat=False, traffic_light=False, fire_hydrant=False,\n stop_sign=False,\n parking_meter=False, bench=False, bird=False, cat=False, dog=False, horse=False, sheep=False,\n cow=False, elephant=False, bear=False, zebra=False,\n giraffe=False, backpack=False, umbrella=False, handbag=False, tie=False, suitcase=False,\n frisbee=False, skis=False, snowboard=False,\n sports_ball=False, kite=False, baseball_bat=False, baseball_glove=False, skateboard=False,\n surfboard=False, tennis_racket=False,\n bottle=False, wine_glass=False, cup=False, fork=False, knife=False, spoon=False, bowl=False,\n banana=False, apple=False, sandwich=False, orange=False,\n broccoli=False, carrot=False, hot_dog=False, pizza=False, donut=False, cake=False, chair=False,\n couch=False, potted_plant=False, bed=False,\n dining_table=False, toilet=False, tv=False, laptop=False, mouse=False, remote=False,\n keyboard=False, cell_phone=False, microwave=False,\n oven=False, toaster=False, sink=False, refrigerator=False, book=False, clock=False, vase=False,\n scissors=False, teddy_bear=False, hair_dryer=False,\n toothbrush=False):\n\n \"\"\"\n The 'CustomObjects()' function allows you to handpick the type of objects you want to detect\n from a video. The objects are pre-initiated in the function variables and predefined as 'False',\n which you can easily set to true for any number of objects available. This function\n returns a dictionary which must be parsed into the 'detectCustomObjectsFromVideo()'. Detecting\n custom objects only happens when you call the function 'detectCustomObjectsFromVideo()'\n\n\n * true_values_of_objects (array); Acceptable values are 'True' and False for all object values present\n\n :param boolean_values:\n :return: custom_objects_dict\n \"\"\"\n\n custom_objects_dict = {}\n input_values = [person, bicycle, car, motorcycle, airplane,\n bus, train, truck, boat, traffic_light, fire_hydrant, stop_sign,\n parking_meter, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra,\n giraffe, backpack, umbrella, handbag, tie, suitcase, frisbee, skis, snowboard,\n sports_ball, kite, baseball_bat, baseball_glove, skateboard, surfboard, tennis_racket,\n bottle, wine_glass, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange,\n broccoli, carrot, hot_dog, pizza, donut, cake, chair, couch, potted_plant, bed,\n dining_table, toilet, tv, laptop, mouse, remote, keyboard, cell_phone, microwave,\n oven, toaster, sink, refrigerator, book, clock, vase, scissors, teddy_bear, hair_dryer,\n toothbrush]\n actual_labels = [\"person\", \"bicycle\", \"car\", \"motorcycle\", \"airplane\",\n \"bus\", \"train\", \"truck\", \"boat\", \"traffic light\", \"fire hydrant\", \"stop sign\",\n \"parking meter\", \"bench\", \"bird\", \"cat\", \"dog\", \"horse\", \"sheep\", \"cow\", \"elephant\", \"bear\",\n \"zebra\",\n \"giraffe\", \"backpack\", \"umbrella\", \"handbag\", \"tie\", \"suitcase\", \"frisbee\", \"skis\",\n \"snowboard\",\n \"sports ball\", \"kite\", \"baseball bat\", \"baseball glove\", \"skateboard\", \"surfboard\",\n \"tennis racket\",\n \"bottle\", \"wine glass\", \"cup\", \"fork\", \"knife\", \"spoon\", \"bowl\", \"banana\", \"apple\", \"sandwich\",\n \"orange\",\n \"broccoli\", \"carrot\", \"hot dog\", \"pizza\", \"donut\", \"cake\", \"chair\", \"couch\", \"potted plant\",\n \"bed\",\n \"dining table\", \"toilet\", \"tv\", \"laptop\", \"mouse\", \"remote\", \"keyboard\", \"cell phone\",\n \"microwave\",\n \"oven\", \"toaster\", \"sink\", \"refrigerator\", \"book\", \"clock\", \"vase\", \"scissors\", \"teddy bear\",\n \"hair dryer\",\n \"toothbrush\"]\n\n for input_value, actual_label in zip(input_values, actual_labels):\n if (input_value == True):\n custom_objects_dict[actual_label] = \"valid\"\n else:\n custom_objects_dict[actual_label] = \"invalid\"\n\n return custom_objects_dict\n\n def detectCustomObjectsFromVideo(self, input_file_path=\"\", camera_input=None, output_file_path=\"\", frames_per_second=20,\n frame_detection_interval=1, minimum_percentage_probability=50, log_progress=False,\n display_percentage_probability=True, display_object_name=True, display_box=True, save_detected_video=True,\n per_frame_function=None, per_second_function=None, per_minute_function=None,\n video_complete_function=None, return_detected_frame=False, detection_timeout = None, \n thread_safe=False, custom_objects=None):\n\n\n return self.detectObjectsFromVideo(input_file_path=input_file_path,\n camera_input=camera_input,\n output_file_path=output_file_path, \n frames_per_second=frames_per_second,\n frame_detection_interval=frame_detection_interval, \n minimum_percentage_probability=minimum_percentage_probability, \n log_progress=log_progress,\n display_percentage_probability=display_percentage_probability, \n display_object_name=display_object_name, \n display_box=display_box, \n save_detected_video=save_detected_video,\n per_frame_function=per_frame_function, \n per_second_function=per_second_function, \n per_minute_function=per_minute_function,\n video_complete_function=video_complete_function, \n return_detected_frame=return_detected_frame, \n detection_timeout = detection_timeout, \n thread_safe=thread_safe, \n custom_objects=custom_objects)" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/__init__.py", "content": "" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/backend/__init__.py", "content": "from .backend import * # noqa: F401,F403\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/backend/backend.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport tensorflow\nfrom tensorflow import keras\n\n\ndef bbox_transform_inv(boxes, deltas, mean=None, std=None):\n \"\"\" Applies deltas (usually regression results) to boxes (usually anchors).\n\n Before applying the deltas to the boxes, the normalization that was previously applied (in the generator) has to be removed.\n The mean and std are the mean and std as applied in the generator. They are unnormalized in this function and then applied to the boxes.\n\n Args\n boxes : np.array of shape (B, N, 4), where B is the batch size, N the number of boxes and 4 values for (x1, y1, x2, y2).\n deltas: np.array of same shape as boxes. These deltas (d_x1, d_y1, d_x2, d_y2) are a factor of the width/height.\n mean : The mean value used when computing deltas (defaults to [0, 0, 0, 0]).\n std : The standard deviation used when computing deltas (defaults to [0.2, 0.2, 0.2, 0.2]).\n\n Returns\n A np.array of the same shape as boxes, but with deltas applied to each box.\n The mean and std are used during training to normalize the regression values (networks love normalization).\n \"\"\"\n if mean is None:\n mean = [0, 0, 0, 0]\n if std is None:\n std = [0.2, 0.2, 0.2, 0.2]\n\n width = boxes[:, :, 2] - boxes[:, :, 0]\n height = boxes[:, :, 3] - boxes[:, :, 1]\n\n x1 = boxes[:, :, 0] + (deltas[:, :, 0] * std[0] + mean[0]) * width\n y1 = boxes[:, :, 1] + (deltas[:, :, 1] * std[1] + mean[1]) * height\n x2 = boxes[:, :, 2] + (deltas[:, :, 2] * std[2] + mean[2]) * width\n y2 = boxes[:, :, 3] + (deltas[:, :, 3] * std[3] + mean[3]) * height\n\n pred_boxes = keras.backend.stack([x1, y1, x2, y2], axis=2)\n\n return pred_boxes\n\n\ndef shift(shape, stride, anchors):\n \"\"\" Produce shifted anchors based on shape of the map and stride size.\n\n Args\n shape : Shape to shift the anchors over.\n stride : Stride to shift the anchors with over the shape.\n anchors: The anchors to apply at each location.\n \"\"\"\n shift_x = (keras.backend.arange(0, shape[1], dtype=keras.backend.floatx()) + keras.backend.constant(0.5, dtype=keras.backend.floatx())) * stride\n shift_y = (keras.backend.arange(0, shape[0], dtype=keras.backend.floatx()) + keras.backend.constant(0.5, dtype=keras.backend.floatx())) * stride\n\n shift_x, shift_y = tensorflow.meshgrid(shift_x, shift_y)\n shift_x = keras.backend.reshape(shift_x, [-1])\n shift_y = keras.backend.reshape(shift_y, [-1])\n\n shifts = keras.backend.stack([\n shift_x,\n shift_y,\n shift_x,\n shift_y\n ], axis=0)\n\n shifts = keras.backend.transpose(shifts)\n number_of_anchors = keras.backend.shape(anchors)[0]\n\n k = keras.backend.shape(shifts)[0] # number of base points = feat_h * feat_w\n\n shifted_anchors = keras.backend.reshape(anchors, [1, number_of_anchors, 4]) + keras.backend.cast(keras.backend.reshape(shifts, [k, 1, 4]), keras.backend.floatx())\n shifted_anchors = keras.backend.reshape(shifted_anchors, [k * number_of_anchors, 4])\n\n return shifted_anchors\n\n\ndef map_fn(*args, **kwargs):\n \"\"\" See https://www.tensorflow.org/api_docs/python/tf/map_fn .\n \"\"\"\n\n if \"shapes\" in kwargs:\n shapes = kwargs.pop(\"shapes\")\n dtype = kwargs.pop(\"dtype\")\n sig = [tensorflow.TensorSpec(shapes[i], dtype=t) for i, t in\n enumerate(dtype)]\n\n # Try to use the new feature fn_output_signature in TF 2.3, use fallback if this is not available\n try:\n return tensorflow.map_fn(*args, **kwargs, fn_output_signature=sig)\n except TypeError:\n kwargs[\"dtype\"] = dtype\n\n return tensorflow.map_fn(*args, **kwargs)\n\n\ndef resize_images(images, size, method='bilinear', align_corners=False):\n \"\"\" See https://www.tensorflow.org/versions/r1.14/api_docs/python/tf/image/resize_images .\n\n Args\n method: The method used for interpolation. One of ('bilinear', 'nearest', 'bicubic', 'area').\n \"\"\"\n methods = {\n 'bilinear': tensorflow.image.ResizeMethod.BILINEAR,\n 'nearest' : tensorflow.image.ResizeMethod.NEAREST_NEIGHBOR,\n 'bicubic' : tensorflow.image.ResizeMethod.BICUBIC,\n 'area' : tensorflow.image.ResizeMethod.AREA,\n }\n return tensorflow.compat.v1.image.resize_images(images, size, methods[method], align_corners)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/bin/__init__.py", "content": "" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/bin/convert_model.py", "content": "#!/usr/bin/env python\n\n\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport argparse\nimport os\nimport sys\n\n# Allow relative imports when being executed as script.\nif __name__ == \"__main__\" and __package__ is None:\n sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))\n import keras_retinanet.bin # noqa: F401\n __package__ = \"keras_retinanet.bin\"\n\n# Change these to absolute imports if you copy this script outside the keras_retinanet package.\nfrom .. import models\nfrom ..utils.config import read_config_file, parse_anchor_parameters, parse_pyramid_levels\nfrom ..utils.gpu import setup_gpu\nfrom ..utils.tf_version import check_tf_version\n\n\ndef parse_args(args):\n parser = argparse.ArgumentParser(description='Script for converting a training model to an inference model.')\n\n parser.add_argument('model_in', help='The model to convert.')\n parser.add_argument('model_out', help='Path to save the converted model to.')\n parser.add_argument('--backbone', help='The backbone of the model to convert.', default='resnet50')\n parser.add_argument('--no-nms', help='Disables non maximum suppression.', dest='nms', action='store_false')\n parser.add_argument('--no-class-specific-filter', help='Disables class specific filtering.', dest='class_specific_filter', action='store_false')\n parser.add_argument('--config', help='Path to a configuration parameters .ini file.')\n parser.add_argument('--nms-threshold', help='Value for non maximum suppression threshold.', type=float, default=0.5)\n parser.add_argument('--score-threshold', help='Threshold for prefiltering boxes.', type=float, default=0.05)\n parser.add_argument('--max-detections', help='Maximum number of detections to keep.', type=int, default=300)\n parser.add_argument('--parallel-iterations', help='Number of batch items to process in parallel.', type=int, default=32)\n\n return parser.parse_args(args)\n\n\ndef main(args=None):\n # parse arguments\n if args is None:\n args = sys.argv[1:]\n args = parse_args(args)\n\n # make sure tensorflow is the minimum required version\n check_tf_version()\n\n # set modified tf session to avoid using the GPUs\n setup_gpu('cpu')\n\n # optionally load config parameters\n anchor_parameters = None\n pyramid_levels = None\n if args.config:\n args.config = read_config_file(args.config)\n if 'anchor_parameters' in args.config:\n anchor_parameters = parse_anchor_parameters(args.config)\n\n if 'pyramid_levels' in args.config:\n pyramid_levels = parse_pyramid_levels(args.config)\n\n # load the model\n model = models.load_model(args.model_in, backbone_name=args.backbone)\n\n # check if this is indeed a training model\n models.check_training_model(model)\n\n # convert the model\n model = models.convert_model(\n model,\n nms=args.nms,\n class_specific_filter=args.class_specific_filter,\n anchor_params=anchor_parameters,\n pyramid_levels=pyramid_levels,\n nms_threshold=args.nms_threshold,\n score_threshold=args.score_threshold,\n max_detections=args.max_detections,\n parallel_iterations=args.parallel_iterations\n )\n\n # save model\n model.save(args.model_out)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/bin/debug.py", "content": "#!/usr/bin/env python\n\n\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport argparse\nimport os\nimport sys\nimport cv2\n\n# Set keycodes for changing images\n# 81, 83 are left and right arrows on linux in Ascii code (probably not needed)\n# 65361, 65363 are left and right arrows in linux\n# 2424832, 2555904 are left and right arrows on Windows\n# 110, 109 are 'n' and 'm' on mac, windows, linux\n# (unfortunately arrow keys not picked up on mac)\nleftkeys = (81, 110, 65361, 2424832)\nrightkeys = (83, 109, 65363, 2555904)\n\n# Allow relative imports when being executed as script.\nif __name__ == \"__main__\" and __package__ is None:\n sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))\n import keras_retinanet.bin # noqa: F401\n __package__ = \"keras_retinanet.bin\"\n\n# Change these to absolute imports if you copy this script outside the keras_retinanet package.\nfrom ..preprocessing.pascal_voc import PascalVocGenerator\nfrom ..preprocessing.csv_generator import CSVGenerator\nfrom ..preprocessing.kitti import KittiGenerator\nfrom ..preprocessing.open_images import OpenImagesGenerator\nfrom ..utils.anchors import anchors_for_shape, compute_gt_annotations\nfrom ..utils.config import read_config_file, parse_anchor_parameters, parse_pyramid_levels\nfrom ..utils.image import random_visual_effect_generator\nfrom ..utils.tf_version import check_tf_version\nfrom ..utils.transform import random_transform_generator\nfrom ..utils.visualization import draw_annotations, draw_boxes, draw_caption\n\n\ndef create_generator(args):\n \"\"\" Create the data generators.\n\n Args:\n args: parseargs arguments object.\n \"\"\"\n common_args = {\n 'config' : args.config,\n 'image_min_side' : args.image_min_side,\n 'image_max_side' : args.image_max_side,\n 'group_method' : args.group_method\n }\n\n # create random transform generator for augmenting training data\n transform_generator = random_transform_generator(\n min_rotation=-0.1,\n max_rotation=0.1,\n min_translation=(-0.1, -0.1),\n max_translation=(0.1, 0.1),\n min_shear=-0.1,\n max_shear=0.1,\n min_scaling=(0.9, 0.9),\n max_scaling=(1.1, 1.1),\n flip_x_chance=0.5,\n flip_y_chance=0.5,\n )\n\n visual_effect_generator = random_visual_effect_generator(\n contrast_range=(0.9, 1.1),\n brightness_range=(-.1, .1),\n hue_range=(-0.05, 0.05),\n saturation_range=(0.95, 1.05)\n )\n\n if args.dataset_type == 'coco':\n # import here to prevent unnecessary dependency on cocoapi\n from ..preprocessing.coco import CocoGenerator\n\n generator = CocoGenerator(\n args.coco_path,\n args.coco_set,\n transform_generator=transform_generator,\n visual_effect_generator=visual_effect_generator,\n **common_args\n )\n elif args.dataset_type == 'pascal':\n generator = PascalVocGenerator(\n args.pascal_path,\n args.pascal_set,\n image_extension=args.image_extension,\n transform_generator=transform_generator,\n visual_effect_generator=visual_effect_generator,\n **common_args\n )\n elif args.dataset_type == 'csv':\n generator = CSVGenerator(\n args.annotations,\n args.classes,\n transform_generator=transform_generator,\n visual_effect_generator=visual_effect_generator,\n **common_args\n )\n elif args.dataset_type == 'oid':\n generator = OpenImagesGenerator(\n args.main_dir,\n subset=args.subset,\n version=args.version,\n labels_filter=args.labels_filter,\n parent_label=args.parent_label,\n annotation_cache_dir=args.annotation_cache_dir,\n transform_generator=transform_generator,\n visual_effect_generator=visual_effect_generator,\n **common_args\n )\n elif args.dataset_type == 'kitti':\n generator = KittiGenerator(\n args.kitti_path,\n subset=args.subset,\n transform_generator=transform_generator,\n visual_effect_generator=visual_effect_generator,\n **common_args\n )\n else:\n raise ValueError('Invalid data type received: {}'.format(args.dataset_type))\n\n return generator\n\n\ndef parse_args(args):\n \"\"\" Parse the arguments.\n \"\"\"\n parser = argparse.ArgumentParser(description='Debug script for a RetinaNet network.')\n subparsers = parser.add_subparsers(help='Arguments for specific dataset types.', dest='dataset_type')\n subparsers.required = True\n\n coco_parser = subparsers.add_parser('coco')\n coco_parser.add_argument('coco_path', help='Path to dataset directory (ie. /tmp/COCO).')\n coco_parser.add_argument('--coco-set', help='Name of the set to show (defaults to val2017).', default='val2017')\n\n pascal_parser = subparsers.add_parser('pascal')\n pascal_parser.add_argument('pascal_path', help='Path to dataset directory (ie. /tmp/VOCdevkit).')\n pascal_parser.add_argument('--pascal-set', help='Name of the set to show (defaults to test).', default='test')\n pascal_parser.add_argument('--image-extension', help='Declares the dataset images\\' extension.', default='.jpg')\n\n kitti_parser = subparsers.add_parser('kitti')\n kitti_parser.add_argument('kitti_path', help='Path to dataset directory (ie. /tmp/kitti).')\n kitti_parser.add_argument('subset', help='Argument for loading a subset from train/val.')\n\n def csv_list(string):\n return string.split(',')\n\n oid_parser = subparsers.add_parser('oid')\n oid_parser.add_argument('main_dir', help='Path to dataset directory.')\n oid_parser.add_argument('subset', help='Argument for loading a subset from train/validation/test.')\n oid_parser.add_argument('--version', help='The current dataset version is v4.', default='v4')\n oid_parser.add_argument('--labels-filter', help='A list of labels to filter.', type=csv_list, default=None)\n oid_parser.add_argument('--annotation-cache-dir', help='Path to store annotation cache.', default='.')\n oid_parser.add_argument('--parent-label', help='Use the hierarchy children of this label.', default=None)\n\n csv_parser = subparsers.add_parser('csv')\n csv_parser.add_argument('annotations', help='Path to CSV file containing annotations for evaluation.')\n csv_parser.add_argument('classes', help='Path to a CSV file containing class label mapping.')\n\n parser.add_argument('--no-resize', help='Disable image resizing.', dest='resize', action='store_false')\n parser.add_argument('--anchors', help='Show positive anchors on the image.', action='store_true')\n parser.add_argument('--display-name', help='Display image name on the bottom left corner.', action='store_true')\n parser.add_argument('--show-annotations', help='Show annotations on the image. Green annotations have anchors, red annotations don\\'t and therefore don\\'t contribute to training.', action='store_true')\n parser.add_argument('--random-transform', help='Randomly transform image and annotations.', action='store_true')\n parser.add_argument('--image-min-side', help='Rescale the image so the smallest side is min_side.', type=int, default=800)\n parser.add_argument('--image-max-side', help='Rescale the image if the largest side is larger than max_side.', type=int, default=1333)\n parser.add_argument('--config', help='Path to a configuration parameters .ini file.')\n parser.add_argument('--no-gui', help='Do not open a GUI window. Save images to an output directory instead.', action='store_true')\n parser.add_argument('--output-dir', help='The output directory to save images to if --no-gui is specified.', default='.')\n parser.add_argument('--flatten-output', help='Flatten the folder structure of saved output images into a single folder.', action='store_true')\n parser.add_argument('--group-method', help='Determines how images are grouped together', type=str, default='ratio', choices=['none', 'random', 'ratio'])\n\n return parser.parse_args(args)\n\n\ndef run(generator, args, anchor_params, pyramid_levels):\n \"\"\" Main loop.\n\n Args\n generator: The generator to debug.\n args: parseargs args object.\n \"\"\"\n # display images, one at a time\n i = 0\n while True:\n # load the data\n image = generator.load_image(i)\n annotations = generator.load_annotations(i)\n if len(annotations['labels']) > 0 :\n # apply random transformations\n if args.random_transform:\n image, annotations = generator.random_transform_group_entry(image, annotations)\n image, annotations = generator.random_visual_effect_group_entry(image, annotations)\n\n # resize the image and annotations\n if args.resize:\n image, image_scale = generator.resize_image(image)\n annotations['bboxes'] *= image_scale\n\n anchors = anchors_for_shape(image.shape, anchor_params=anchor_params, pyramid_levels=pyramid_levels)\n positive_indices, _, max_indices = compute_gt_annotations(anchors, annotations['bboxes'])\n\n # draw anchors on the image\n if args.anchors:\n draw_boxes(image, anchors[positive_indices], (255, 255, 0), thickness=1)\n\n # draw annotations on the image\n if args.show_annotations:\n # draw annotations in red\n draw_annotations(image, annotations, color=(0, 0, 255), label_to_name=generator.label_to_name)\n\n # draw regressed anchors in green to override most red annotations\n # result is that annotations without anchors are red, with anchors are green\n draw_boxes(image, annotations['bboxes'][max_indices[positive_indices], :], (0, 255, 0))\n\n # display name on the image\n if args.display_name:\n draw_caption(image, [0, image.shape[0]], os.path.basename(generator.image_path(i)))\n\n # write to file and advance if no-gui selected\n if args.no_gui:\n output_path = make_output_path(args.output_dir, generator.image_path(i), flatten=args.flatten_output)\n os.makedirs(os.path.dirname(output_path), exist_ok=True)\n cv2.imwrite(output_path, image)\n i += 1\n if i == generator.size(): # have written all images\n break\n else:\n continue\n\n # if we are using the GUI, then show an image\n cv2.imshow('Image', image)\n key = cv2.waitKeyEx()\n\n # press right for next image and left for previous (linux or windows, doesn't work for macOS)\n # if you run macOS, press \"n\" or \"m\" (will also work on linux and windows)\n\n if key in rightkeys:\n i = (i + 1) % generator.size()\n if key in leftkeys:\n i -= 1\n if i < 0:\n i = generator.size() - 1\n\n # press q or Esc to quit\n if (key == ord('q')) or (key == 27):\n return False\n\n return True\n\n\ndef make_output_path(output_dir, image_path, flatten = False):\n \"\"\" Compute the output path for a debug image. \"\"\"\n\n # If the output hierarchy is flattened to a single folder, throw away all leading folders.\n if flatten:\n path = os.path.basename(image_path)\n\n # Otherwise, make sure absolute paths are taken relative to the filesystem root.\n else:\n # Make sure to drop drive letters on Windows, otherwise relpath wil fail.\n _, path = os.path.splitdrive(image_path)\n if os.path.isabs(path):\n path = os.path.relpath(path, '/')\n\n # In all cases, append \"_debug\" to the filename, before the extension.\n base, extension = os.path.splitext(path)\n path = base + \"_debug\" + extension\n\n # Finally, join the whole thing to the output directory.\n return os.path.join(output_dir, path)\n\n\ndef main(args=None):\n # parse arguments\n if args is None:\n args = sys.argv[1:]\n args = parse_args(args)\n\n # make sure tensorflow is the minimum required version\n check_tf_version()\n\n # create the generator\n generator = create_generator(args)\n\n # optionally load config parameters\n if args.config:\n args.config = read_config_file(args.config)\n\n # optionally load anchor parameters\n anchor_params = None\n if args.config and 'anchor_parameters' in args.config:\n anchor_params = parse_anchor_parameters(args.config)\n\n pyramid_levels = None\n if args.config and 'pyramid_levels' in args.config:\n pyramid_levels = parse_pyramid_levels(args.config)\n # create the display window if necessary\n if not args.no_gui:\n cv2.namedWindow('Image', cv2.WINDOW_NORMAL)\n\n run(generator, args, anchor_params=anchor_params, pyramid_levels=pyramid_levels)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/bin/evaluate.py", "content": "#!/usr/bin/env python\n\n\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n http://www.apache.org/licenses/LICENSE-2.0\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport argparse\nimport os\nimport sys\n\n# Allow relative imports when being executed as script.\nif __name__ == \"__main__\" and __package__ is None:\n sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))\n import keras_retinanet.bin # noqa: F401\n __package__ = \"keras_retinanet.bin\"\n\n# Change these to absolute imports if you copy this script outside the keras_retinanet package.\nfrom .. import models\nfrom ..preprocessing.csv_generator import CSVGenerator\nfrom ..preprocessing.pascal_voc import PascalVocGenerator\nfrom ..utils.anchors import make_shapes_callback\nfrom ..utils.config import read_config_file, parse_anchor_parameters, parse_pyramid_levels\nfrom ..utils.eval import evaluate\nfrom ..utils.gpu import setup_gpu\nfrom ..utils.tf_version import check_tf_version\n\n\ndef create_generator(args, preprocess_image):\n \"\"\" Create generators for evaluation.\n \"\"\"\n common_args = {\n 'config' : args.config,\n 'image_min_side' : args.image_min_side,\n 'image_max_side' : args.image_max_side,\n 'no_resize' : args.no_resize,\n 'preprocess_image' : preprocess_image,\n 'group_method' : args.group_method\n }\n\n if args.dataset_type == 'coco':\n # import here to prevent unnecessary dependency on cocoapi\n from ..preprocessing.coco import CocoGenerator\n\n validation_generator = CocoGenerator(\n args.coco_path,\n 'val2017',\n shuffle_groups=False,\n **common_args\n )\n elif args.dataset_type == 'pascal':\n validation_generator = PascalVocGenerator(\n args.pascal_path,\n 'test',\n image_extension=args.image_extension,\n shuffle_groups=False,\n **common_args\n )\n elif args.dataset_type == 'csv':\n validation_generator = CSVGenerator(\n args.annotations,\n args.classes,\n shuffle_groups=False,\n **common_args\n )\n else:\n raise ValueError('Invalid data type received: {}'.format(args.dataset_type))\n\n return validation_generator\n\n\ndef parse_args(args):\n \"\"\" Parse the arguments.\n \"\"\"\n parser = argparse.ArgumentParser(description='Evaluation script for a RetinaNet network.')\n subparsers = parser.add_subparsers(help='Arguments for specific dataset types.', dest='dataset_type')\n subparsers.required = True\n\n coco_parser = subparsers.add_parser('coco')\n coco_parser.add_argument('coco_path', help='Path to dataset directory (ie. /tmp/COCO).')\n\n pascal_parser = subparsers.add_parser('pascal')\n pascal_parser.add_argument('pascal_path', help='Path to dataset directory (ie. /tmp/VOCdevkit).')\n pascal_parser.add_argument('--image-extension', help='Declares the dataset images\\' extension.', default='.jpg')\n\n csv_parser = subparsers.add_parser('csv')\n csv_parser.add_argument('annotations', help='Path to CSV file containing annotations for evaluation.')\n csv_parser.add_argument('classes', help='Path to a CSV file containing class label mapping.')\n\n parser.add_argument('model', help='Path to RetinaNet model.')\n parser.add_argument('--convert-model', help='Convert the model to an inference model (ie. the input is a training model).', action='store_true')\n parser.add_argument('--backbone', help='The backbone of the model.', default='resnet50')\n parser.add_argument('--gpu', help='Id of the GPU to use (as reported by nvidia-smi).')\n parser.add_argument('--score-threshold', help='Threshold on score to filter detections with (defaults to 0.05).', default=0.05, type=float)\n parser.add_argument('--iou-threshold', help='IoU Threshold to count for a positive detection (defaults to 0.5).', default=0.5, type=float)\n parser.add_argument('--max-detections', help='Max Detections per image (defaults to 100).', default=100, type=int)\n parser.add_argument('--save-path', help='Path for saving images with detections (doesn\\'t work for COCO).')\n parser.add_argument('--image-min-side', help='Rescale the image so the smallest side is min_side.', type=int, default=800)\n parser.add_argument('--image-max-side', help='Rescale the image if the largest side is larger than max_side.', type=int, default=1333)\n parser.add_argument('--no-resize', help='Don''t rescale the image.', action='store_true')\n parser.add_argument('--config', help='Path to a configuration parameters .ini file (only used with --convert-model).')\n parser.add_argument('--group-method', help='Determines how images are grouped together', type=str, default='ratio', choices=['none', 'random', 'ratio'])\n\n return parser.parse_args(args)\n\n\ndef main(args=None):\n # parse arguments\n if args is None:\n args = sys.argv[1:]\n args = parse_args(args)\n\n # make sure tensorflow is the minimum required version\n check_tf_version()\n\n # optionally choose specific GPU\n if args.gpu:\n setup_gpu(args.gpu)\n\n # make save path if it doesn't exist\n if args.save_path is not None and not os.path.exists(args.save_path):\n os.makedirs(args.save_path)\n\n # optionally load config parameters\n if args.config:\n args.config = read_config_file(args.config)\n\n # create the generator\n backbone = models.backbone(args.backbone)\n generator = create_generator(args, backbone.preprocess_image)\n\n # optionally load anchor parameters\n anchor_params = None\n pyramid_levels = None\n if args.config and 'anchor_parameters' in args.config:\n anchor_params = parse_anchor_parameters(args.config)\n if args.config and 'pyramid_levels' in args.config:\n pyramid_levels = parse_pyramid_levels(args.config)\n\n # load the model\n print('Loading model, this may take a second...')\n model = models.load_model(args.model, backbone_name=args.backbone)\n generator.compute_shapes = make_shapes_callback(model)\n\n # optionally convert the model\n if args.convert_model:\n model = models.convert_model(model, anchor_params=anchor_params, pyramid_levels=pyramid_levels)\n\n # print model summary\n # print(model.summary())\n\n # start evaluation\n if args.dataset_type == 'coco':\n from ..utils.coco_eval import evaluate_coco\n evaluate_coco(generator, model, args.score_threshold)\n else:\n average_precisions, inference_time = evaluate(\n generator,\n model,\n iou_threshold=args.iou_threshold,\n score_threshold=args.score_threshold,\n max_detections=args.max_detections,\n save_path=args.save_path\n )\n\n # print evaluation\n total_instances = []\n precisions = []\n for label, (average_precision, num_annotations) in average_precisions.items():\n print('{:.0f} instances of class'.format(num_annotations),\n generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision))\n total_instances.append(num_annotations)\n precisions.append(average_precision)\n\n if sum(total_instances) == 0:\n print('No test instances found.')\n return\n\n print('Inference time for {:.0f} images: {:.4f}'.format(generator.size(), inference_time))\n\n print('mAP using the weighted average of precisions among classes: {:.4f}'.format(sum([a * b for a, b in zip(total_instances, precisions)]) / sum(total_instances)))\n print('mAP: {:.4f}'.format(sum(precisions) / sum(x > 0 for x in total_instances)))\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/bin/train.py", "content": "#!/usr/bin/env python\n\n\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport argparse\nimport os\nimport sys\nimport warnings\n\nfrom tensorflow import keras\nimport tensorflow as tf\n\n# Allow relative imports when being executed as script.\nif __name__ == \"__main__\" and __package__ is None:\n sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))\n import keras_retinanet.bin # noqa: F401\n __package__ = \"keras_retinanet.bin\"\n\n# Change these to absolute imports if you copy this script outside the keras_retinanet package.\nfrom .. import layers # noqa: F401\nfrom .. import losses\nfrom .. import models\nfrom ..callbacks import RedirectModel\nfrom ..callbacks.eval import Evaluate\nfrom ..models.retinanet import retinanet_bbox\nfrom ..preprocessing.csv_generator import CSVGenerator\nfrom ..preprocessing.kitti import KittiGenerator\nfrom ..preprocessing.open_images import OpenImagesGenerator\nfrom ..preprocessing.pascal_voc import PascalVocGenerator\nfrom ..utils.anchors import make_shapes_callback\nfrom ..utils.config import read_config_file, parse_anchor_parameters, parse_pyramid_levels\nfrom ..utils.gpu import setup_gpu\nfrom ..utils.image import random_visual_effect_generator\nfrom ..utils.model import freeze as freeze_model\nfrom ..utils.tf_version import check_tf_version\nfrom ..utils.transform import random_transform_generator\n\n\ndef makedirs(path):\n # Intended behavior: try to create the directory,\n # pass if the directory exists already, fails otherwise.\n # Meant for Python 2.7/3.n compatibility.\n try:\n os.makedirs(path)\n except OSError:\n if not os.path.isdir(path):\n raise\n\n\ndef model_with_weights(model, weights, skip_mismatch):\n \"\"\" Load weights for model.\n\n Args\n model : The model to load weights for.\n weights : The weights to load.\n skip_mismatch : If True, skips layers whose shape of weights doesn't match with the model.\n \"\"\"\n if weights is not None:\n model.load_weights(weights, by_name=True, skip_mismatch=skip_mismatch)\n return model\n\n\ndef create_models(backbone_retinanet, num_classes, weights, multi_gpu=0,\n freeze_backbone=False, lr=1e-5, optimizer_clipnorm=0.001, config=None):\n \"\"\" Creates three models (model, training_model, prediction_model).\n\n Args\n backbone_retinanet : A function to call to create a retinanet model with a given backbone.\n num_classes : The number of classes to train.\n weights : The weights to load into the model.\n multi_gpu : The number of GPUs to use for training.\n freeze_backbone : If True, disables learning for the backbone.\n config : Config parameters, None indicates the default configuration.\n\n Returns\n model : The base model. This is also the model that is saved in snapshots.\n training_model : The training model. If multi_gpu=0, this is identical to model.\n prediction_model : The model wrapped with utility functions to perform object detection (applies regression values and performs NMS).\n \"\"\"\n\n modifier = freeze_model if freeze_backbone else None\n\n # load anchor parameters, or pass None (so that defaults will be used)\n anchor_params = None\n num_anchors = None\n pyramid_levels = None\n if config and 'anchor_parameters' in config:\n anchor_params = parse_anchor_parameters(config)\n num_anchors = anchor_params.num_anchors()\n if config and 'pyramid_levels' in config:\n pyramid_levels = parse_pyramid_levels(config)\n\n # Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.\n # optionally wrap in a parallel model\n if multi_gpu > 1:\n from keras.utils import multi_gpu_model\n with tf.device('/cpu:0'):\n model = model_with_weights(backbone_retinanet(num_classes, num_anchors=num_anchors, modifier=modifier, pyramid_levels=pyramid_levels), weights=weights, skip_mismatch=True)\n training_model = multi_gpu_model(model, gpus=multi_gpu)\n else:\n model = model_with_weights(backbone_retinanet(num_classes, num_anchors=num_anchors, modifier=modifier, pyramid_levels=pyramid_levels), weights=weights, skip_mismatch=True)\n training_model = model\n\n # make prediction model\n prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params, pyramid_levels=pyramid_levels)\n\n # compile model\n training_model.compile(\n loss={\n 'regression' : losses.smooth_l1(),\n 'classification': losses.focal()\n },\n optimizer=keras.optimizers.Adam(lr=lr, clipnorm=optimizer_clipnorm)\n )\n\n return model, training_model, prediction_model\n\n\ndef create_callbacks(model, training_model, prediction_model, validation_generator, args):\n \"\"\" Creates the callbacks to use during training.\n\n Args\n model: The base model.\n training_model: The model that is used for training.\n prediction_model: The model that should be used for validation.\n validation_generator: The generator for creating validation data.\n args: parseargs args object.\n\n Returns:\n A list of callbacks used for training.\n \"\"\"\n callbacks = []\n\n tensorboard_callback = None\n\n if args.tensorboard_dir:\n makedirs(args.tensorboard_dir)\n update_freq = args.tensorboard_freq\n if update_freq not in ['epoch', 'batch']:\n update_freq = int(update_freq)\n tensorboard_callback = keras.callbacks.TensorBoard(\n log_dir = args.tensorboard_dir,\n histogram_freq = 0,\n batch_size = args.batch_size,\n write_graph = True,\n write_grads = False,\n write_images = False,\n update_freq = update_freq,\n embeddings_freq = 0,\n embeddings_layer_names = None,\n embeddings_metadata = None\n )\n\n if args.evaluation and validation_generator:\n if args.dataset_type == 'coco':\n from ..callbacks.coco import CocoEval\n\n # use prediction model for evaluation\n evaluation = CocoEval(validation_generator, tensorboard=tensorboard_callback)\n else:\n evaluation = Evaluate(validation_generator, tensorboard=tensorboard_callback, weighted_average=args.weighted_average)\n evaluation = RedirectModel(evaluation, prediction_model)\n callbacks.append(evaluation)\n\n # save the model\n if args.snapshots:\n # ensure directory created first; otherwise h5py will error after epoch.\n makedirs(args.snapshot_path)\n checkpoint = keras.callbacks.ModelCheckpoint(\n os.path.join(\n args.snapshot_path,\n '{backbone}_{dataset_type}_{{epoch:02d}}.h5'.format(backbone=args.backbone, dataset_type=args.dataset_type)\n ),\n verbose=1,\n # save_best_only=True,\n # monitor=\"mAP\",\n # mode='max'\n )\n checkpoint = RedirectModel(checkpoint, model)\n callbacks.append(checkpoint)\n\n callbacks.append(keras.callbacks.ReduceLROnPlateau(\n monitor = 'loss',\n factor = args.reduce_lr_factor,\n patience = args.reduce_lr_patience,\n verbose = 1,\n mode = 'auto',\n min_delta = 0.0001,\n cooldown = 0,\n min_lr = 0\n ))\n\n if args.evaluation and validation_generator:\n callbacks.append(keras.callbacks.EarlyStopping(\n monitor = 'mAP',\n patience = 5,\n mode = 'max',\n min_delta = 0.01\n ))\n\n if args.tensorboard_dir:\n callbacks.append(tensorboard_callback)\n\n return callbacks\n\n\ndef create_generators(args, preprocess_image):\n \"\"\" Create generators for training and validation.\n\n Args\n args : parseargs object containing configuration for generators.\n preprocess_image : Function that preprocesses an image for the network.\n \"\"\"\n common_args = {\n 'batch_size' : args.batch_size,\n 'config' : args.config,\n 'image_min_side' : args.image_min_side,\n 'image_max_side' : args.image_max_side,\n 'no_resize' : args.no_resize,\n 'preprocess_image' : preprocess_image,\n 'group_method' : args.group_method\n }\n\n # create random transform generator for augmenting training data\n if args.random_transform:\n transform_generator = random_transform_generator(\n min_rotation=-0.1,\n max_rotation=0.1,\n min_translation=(-0.1, -0.1),\n max_translation=(0.1, 0.1),\n min_shear=-0.1,\n max_shear=0.1,\n min_scaling=(0.9, 0.9),\n max_scaling=(1.1, 1.1),\n flip_x_chance=0.5,\n flip_y_chance=0.5,\n )\n visual_effect_generator = random_visual_effect_generator(\n contrast_range=(0.9, 1.1),\n brightness_range=(-.1, .1),\n hue_range=(-0.05, 0.05),\n saturation_range=(0.95, 1.05)\n )\n else:\n transform_generator = random_transform_generator(flip_x_chance=0.5)\n visual_effect_generator = None\n\n if args.dataset_type == 'coco':\n # import here to prevent unnecessary dependency on cocoapi\n from ..preprocessing.coco import CocoGenerator\n\n train_generator = CocoGenerator(\n args.coco_path,\n 'train2017',\n transform_generator=transform_generator,\n visual_effect_generator=visual_effect_generator,\n **common_args\n )\n\n validation_generator = CocoGenerator(\n args.coco_path,\n 'val2017',\n shuffle_groups=False,\n **common_args\n )\n elif args.dataset_type == 'pascal':\n train_generator = PascalVocGenerator(\n args.pascal_path,\n 'train',\n image_extension=args.image_extension,\n transform_generator=transform_generator,\n visual_effect_generator=visual_effect_generator,\n **common_args\n )\n\n validation_generator = PascalVocGenerator(\n args.pascal_path,\n 'val',\n image_extension=args.image_extension,\n shuffle_groups=False,\n **common_args\n )\n elif args.dataset_type == 'csv':\n train_generator = CSVGenerator(\n args.annotations,\n args.classes,\n transform_generator=transform_generator,\n visual_effect_generator=visual_effect_generator,\n **common_args\n )\n\n if args.val_annotations:\n validation_generator = CSVGenerator(\n args.val_annotations,\n args.classes,\n shuffle_groups=False,\n **common_args\n )\n else:\n validation_generator = None\n elif args.dataset_type == 'oid':\n train_generator = OpenImagesGenerator(\n args.main_dir,\n subset='train',\n version=args.version,\n labels_filter=args.labels_filter,\n annotation_cache_dir=args.annotation_cache_dir,\n parent_label=args.parent_label,\n transform_generator=transform_generator,\n visual_effect_generator=visual_effect_generator,\n **common_args\n )\n\n validation_generator = OpenImagesGenerator(\n args.main_dir,\n subset='validation',\n version=args.version,\n labels_filter=args.labels_filter,\n annotation_cache_dir=args.annotation_cache_dir,\n parent_label=args.parent_label,\n shuffle_groups=False,\n **common_args\n )\n elif args.dataset_type == 'kitti':\n train_generator = KittiGenerator(\n args.kitti_path,\n subset='train',\n transform_generator=transform_generator,\n visual_effect_generator=visual_effect_generator,\n **common_args\n )\n\n validation_generator = KittiGenerator(\n args.kitti_path,\n subset='val',\n shuffle_groups=False,\n **common_args\n )\n else:\n raise ValueError('Invalid data type received: {}'.format(args.dataset_type))\n\n return train_generator, validation_generator\n\n\ndef check_args(parsed_args):\n \"\"\" Function to check for inherent contradictions within parsed arguments.\n For example, batch_size < num_gpus\n Intended to raise errors prior to backend initialisation.\n\n Args\n parsed_args: parser.parse_args()\n\n Returns\n parsed_args\n \"\"\"\n\n if parsed_args.multi_gpu > 1 and parsed_args.batch_size < parsed_args.multi_gpu:\n raise ValueError(\n \"Batch size ({}) must be equal to or higher than the number of GPUs ({})\".format(parsed_args.batch_size,\n parsed_args.multi_gpu))\n\n if parsed_args.multi_gpu > 1 and parsed_args.snapshot:\n raise ValueError(\n \"Multi GPU training ({}) and resuming from snapshots ({}) is not supported.\".format(parsed_args.multi_gpu,\n parsed_args.snapshot))\n\n if parsed_args.multi_gpu > 1 and not parsed_args.multi_gpu_force:\n raise ValueError(\"Multi-GPU support is experimental, use at own risk! Run with --multi-gpu-force if you wish to continue.\")\n\n if 'resnet' not in parsed_args.backbone:\n warnings.warn('Using experimental backbone {}. Only resnet50 has been properly tested.'.format(parsed_args.backbone))\n\n return parsed_args\n\n\ndef parse_args(args):\n \"\"\" Parse the arguments.\n \"\"\"\n parser = argparse.ArgumentParser(description='Simple training script for training a RetinaNet network.')\n subparsers = parser.add_subparsers(help='Arguments for specific dataset types.', dest='dataset_type')\n subparsers.required = True\n\n coco_parser = subparsers.add_parser('coco')\n coco_parser.add_argument('coco_path', help='Path to dataset directory (ie. /tmp/COCO).')\n\n pascal_parser = subparsers.add_parser('pascal')\n pascal_parser.add_argument('pascal_path', help='Path to dataset directory (ie. /tmp/VOCdevkit).')\n pascal_parser.add_argument('--image-extension', help='Declares the dataset images\\' extension.', default='.jpg')\n\n kitti_parser = subparsers.add_parser('kitti')\n kitti_parser.add_argument('kitti_path', help='Path to dataset directory (ie. /tmp/kitti).')\n\n def csv_list(string):\n return string.split(',')\n\n oid_parser = subparsers.add_parser('oid')\n oid_parser.add_argument('main_dir', help='Path to dataset directory.')\n oid_parser.add_argument('--version', help='The current dataset version is v4.', default='v4')\n oid_parser.add_argument('--labels-filter', help='A list of labels to filter.', type=csv_list, default=None)\n oid_parser.add_argument('--annotation-cache-dir', help='Path to store annotation cache.', default='.')\n oid_parser.add_argument('--parent-label', help='Use the hierarchy children of this label.', default=None)\n\n csv_parser = subparsers.add_parser('csv')\n csv_parser.add_argument('annotations', help='Path to CSV file containing annotations for training.')\n csv_parser.add_argument('classes', help='Path to a CSV file containing class label mapping.')\n csv_parser.add_argument('--val-annotations', help='Path to CSV file containing annotations for validation (optional).')\n\n group = parser.add_mutually_exclusive_group()\n group.add_argument('--snapshot', help='Resume training from a snapshot.')\n group.add_argument('--imagenet-weights', help='Initialize the model with pretrained imagenet weights. This is the default behaviour.', action='store_const', const=True, default=True)\n group.add_argument('--weights', help='Initialize the model with weights from a file.')\n group.add_argument('--no-weights', help='Don\\'t initialize the model with any weights.', dest='imagenet_weights', action='store_const', const=False)\n parser.add_argument('--backbone', help='Backbone model used by retinanet.', default='resnet50', type=str)\n parser.add_argument('--batch-size', help='Size of the batches.', default=1, type=int)\n parser.add_argument('--gpu', help='Id of the GPU to use (as reported by nvidia-smi).')\n parser.add_argument('--multi-gpu', help='Number of GPUs to use for parallel processing.', type=int, default=0)\n parser.add_argument('--multi-gpu-force', help='Extra flag needed to enable (experimental) multi-gpu support.', action='store_true')\n parser.add_argument('--initial-epoch', help='Epoch from which to begin the train, useful if resuming from snapshot.', type=int, default=0)\n parser.add_argument('--epochs', help='Number of epochs to train.', type=int, default=50)\n parser.add_argument('--steps', help='Number of steps per epoch.', type=int, default=10000)\n parser.add_argument('--lr', help='Learning rate.', type=float, default=1e-5)\n parser.add_argument('--optimizer-clipnorm', help='Clipnorm parameter for optimizer.', type=float, default=0.001)\n parser.add_argument('--snapshot-path', help='Path to store snapshots of models during training (defaults to \\'./snapshots\\')', default='./snapshots')\n parser.add_argument('--tensorboard-dir', help='Log directory for Tensorboard output', default='') # default='./logs') => https://github.com/tensorflow/tensorflow/pull/34870\n parser.add_argument('--tensorboard-freq', help='Update frequency for Tensorboard output. Values \\'epoch\\', \\'batch\\' or int', default='epoch')\n parser.add_argument('--no-snapshots', help='Disable saving snapshots.', dest='snapshots', action='store_false')\n parser.add_argument('--no-evaluation', help='Disable per epoch evaluation.', dest='evaluation', action='store_false')\n parser.add_argument('--freeze-backbone', help='Freeze training of backbone layers.', action='store_true')\n parser.add_argument('--random-transform', help='Randomly transform image and annotations.', action='store_true')\n parser.add_argument('--image-min-side', help='Rescale the image so the smallest side is min_side.', type=int, default=800)\n parser.add_argument('--image-max-side', help='Rescale the image if the largest side is larger than max_side.', type=int, default=1333)\n parser.add_argument('--no-resize', help='Don''t rescale the image.', action='store_true')\n parser.add_argument('--config', help='Path to a configuration parameters .ini file.')\n parser.add_argument('--weighted-average', help='Compute the mAP using the weighted average of precisions among classes.', action='store_true')\n parser.add_argument('--compute-val-loss', help='Compute validation loss during training', dest='compute_val_loss', action='store_true')\n parser.add_argument('--reduce-lr-patience', help='Reduce learning rate after validation loss decreases over reduce_lr_patience epochs', type=int, default=2)\n parser.add_argument('--reduce-lr-factor', help='When learning rate is reduced due to reduce_lr_patience, multiply by reduce_lr_factor', type=float, default=0.1)\n parser.add_argument('--group-method', help='Determines how images are grouped together', type=str, default='ratio', choices=['none', 'random', 'ratio'])\n\n # Fit generator arguments\n parser.add_argument('--multiprocessing', help='Use multiprocessing in fit_generator.', action='store_true')\n parser.add_argument('--workers', help='Number of generator workers.', type=int, default=1)\n parser.add_argument('--max-queue-size', help='Queue length for multiprocessing workers in fit_generator.', type=int, default=10)\n\n return check_args(parser.parse_args(args))\n\n\ndef main(args=None):\n # parse arguments\n if args is None:\n args = sys.argv[1:]\n args = parse_args(args)\n\n # create object that stores backbone information\n backbone = models.backbone(args.backbone)\n\n # make sure tensorflow is the minimum required version\n check_tf_version()\n\n # optionally choose specific GPU\n if args.gpu is not None:\n setup_gpu(args.gpu)\n\n # optionally load config parameters\n if args.config:\n args.config = read_config_file(args.config)\n\n # create the generators\n train_generator, validation_generator = create_generators(args, backbone.preprocess_image)\n\n # create the model\n if args.snapshot is not None:\n print('Loading model, this may take a second...')\n model = models.load_model(args.snapshot, backbone_name=args.backbone)\n training_model = model\n anchor_params = None\n pyramid_levels = None\n if args.config and 'anchor_parameters' in args.config:\n anchor_params = parse_anchor_parameters(args.config)\n if args.config and 'pyramid_levels' in args.config:\n pyramid_levels = parse_pyramid_levels(args.config)\n\n prediction_model = retinanet_bbox(model=model, anchor_params=anchor_params, pyramid_levels=pyramid_levels)\n else:\n weights = args.weights\n # default to imagenet if nothing else is specified\n if weights is None and args.imagenet_weights:\n weights = backbone.download_imagenet()\n\n print('Creating model, this may take a second...')\n model, training_model, prediction_model = create_models(\n backbone_retinanet=backbone.retinanet,\n num_classes=train_generator.num_classes(),\n weights=weights,\n multi_gpu=args.multi_gpu,\n freeze_backbone=args.freeze_backbone,\n lr=args.lr,\n optimizer_clipnorm=args.optimizer_clipnorm,\n config=args.config\n )\n\n # print model summary\n print(model.summary())\n\n # this lets the generator compute backbone layer shapes using the actual backbone model\n if 'vgg' in args.backbone or 'densenet' in args.backbone:\n train_generator.compute_shapes = make_shapes_callback(model)\n if validation_generator:\n validation_generator.compute_shapes = train_generator.compute_shapes\n\n # create the callbacks\n callbacks = create_callbacks(\n model,\n training_model,\n prediction_model,\n validation_generator,\n args,\n )\n\n if not args.compute_val_loss:\n validation_generator = None\n\n # start training\n return training_model.fit_generator(\n generator=train_generator,\n steps_per_epoch=args.steps,\n epochs=args.epochs,\n verbose=1,\n callbacks=callbacks,\n workers=args.workers,\n use_multiprocessing=args.multiprocessing,\n max_queue_size=args.max_queue_size,\n validation_data=validation_generator,\n initial_epoch=args.initial_epoch\n )\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/callbacks/__init__.py", "content": "from .common import * # noqa: F401,F403\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/callbacks/coco.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom tensorflow import keras\nfrom ..utils.coco_eval import evaluate_coco\n\n\nclass CocoEval(keras.callbacks.Callback):\n \"\"\" Performs COCO evaluation on each epoch.\n \"\"\"\n def __init__(self, generator, tensorboard=None, threshold=0.05):\n \"\"\" CocoEval callback intializer.\n\n Args\n generator : The generator used for creating validation data.\n tensorboard : If given, the results will be written to tensorboard.\n threshold : The score threshold to use.\n \"\"\"\n self.generator = generator\n self.threshold = threshold\n self.tensorboard = tensorboard\n\n super(CocoEval, self).__init__()\n\n def on_epoch_end(self, epoch, logs=None):\n logs = logs or {}\n\n coco_tag = ['AP @[ IoU=0.50:0.95 | area= all | maxDets=100 ]',\n 'AP @[ IoU=0.50 | area= all | maxDets=100 ]',\n 'AP @[ IoU=0.75 | area= all | maxDets=100 ]',\n 'AP @[ IoU=0.50:0.95 | area= small | maxDets=100 ]',\n 'AP @[ IoU=0.50:0.95 | area=medium | maxDets=100 ]',\n 'AP @[ IoU=0.50:0.95 | area= large | maxDets=100 ]',\n 'AR @[ IoU=0.50:0.95 | area= all | maxDets= 1 ]',\n 'AR @[ IoU=0.50:0.95 | area= all | maxDets= 10 ]',\n 'AR @[ IoU=0.50:0.95 | area= all | maxDets=100 ]',\n 'AR @[ IoU=0.50:0.95 | area= small | maxDets=100 ]',\n 'AR @[ IoU=0.50:0.95 | area=medium | maxDets=100 ]',\n 'AR @[ IoU=0.50:0.95 | area= large | maxDets=100 ]']\n coco_eval_stats = evaluate_coco(self.generator, self.model, self.threshold)\n\n if coco_eval_stats is not None:\n for index, result in enumerate(coco_eval_stats):\n logs[coco_tag[index]] = result\n\n if self.tensorboard:\n import tensorflow as tf\n writer = tf.summary.create_file_writer(self.tensorboard.log_dir)\n with writer.as_default():\n for index, result in enumerate(coco_eval_stats):\n tf.summary.scalar('{}. {}'.format(index + 1, coco_tag[index]), result, step=epoch)\n writer.flush()\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/callbacks/common.py", "content": "from tensorflow import keras\n\n\nclass RedirectModel(keras.callbacks.Callback):\n \"\"\"Callback which wraps another callback, but executed on a different model.\n\n ```python\n model = keras.models.load_model('model.h5')\n model_checkpoint = ModelCheckpoint(filepath='snapshot.h5')\n parallel_model = multi_gpu_model(model, gpus=2)\n parallel_model.fit(X_train, Y_train, callbacks=[RedirectModel(model_checkpoint, model)])\n ```\n\n Args\n callback : callback to wrap.\n model : model to use when executing callbacks.\n \"\"\"\n\n def __init__(self,\n callback,\n model):\n super(RedirectModel, self).__init__()\n\n self.callback = callback\n self.redirect_model = model\n\n def on_epoch_begin(self, epoch, logs=None):\n self.callback.on_epoch_begin(epoch, logs=logs)\n\n def on_epoch_end(self, epoch, logs=None):\n self.callback.on_epoch_end(epoch, logs=logs)\n\n def on_batch_begin(self, batch, logs=None):\n self.callback.on_batch_begin(batch, logs=logs)\n\n def on_batch_end(self, batch, logs=None):\n self.callback.on_batch_end(batch, logs=logs)\n\n def on_train_begin(self, logs=None):\n # overwrite the model with our custom model\n self.callback.set_model(self.redirect_model)\n\n self.callback.on_train_begin(logs=logs)\n\n def on_train_end(self, logs=None):\n self.callback.on_train_end(logs=logs)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/callbacks/eval.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom tensorflow import keras\nfrom ..utils.eval import evaluate\n\n\nclass Evaluate(keras.callbacks.Callback):\n \"\"\" Evaluation callback for arbitrary datasets.\n \"\"\"\n\n def __init__(\n self,\n generator,\n iou_threshold=0.5,\n score_threshold=0.05,\n max_detections=100,\n save_path=None,\n tensorboard=None,\n weighted_average=False,\n verbose=1\n ):\n \"\"\" Evaluate a given dataset using a given model at the end of every epoch during training.\n\n # Arguments\n generator : The generator that represents the dataset to evaluate.\n iou_threshold : The threshold used to consider when a detection is positive or negative.\n score_threshold : The score confidence threshold to use for detections.\n max_detections : The maximum number of detections to use per image.\n save_path : The path to save images with visualized detections to.\n tensorboard : Instance of keras.callbacks.TensorBoard used to log the mAP value.\n weighted_average : Compute the mAP using the weighted average of precisions among classes.\n verbose : Set the verbosity level, by default this is set to 1.\n \"\"\"\n self.generator = generator\n self.iou_threshold = iou_threshold\n self.score_threshold = score_threshold\n self.max_detections = max_detections\n self.save_path = save_path\n self.tensorboard = tensorboard\n self.weighted_average = weighted_average\n self.verbose = verbose\n\n super(Evaluate, self).__init__()\n\n def on_epoch_end(self, epoch, logs=None):\n logs = logs or {}\n\n # run evaluation\n average_precisions, _ = evaluate(\n self.generator,\n self.model,\n iou_threshold=self.iou_threshold,\n score_threshold=self.score_threshold,\n max_detections=self.max_detections,\n save_path=self.save_path\n )\n\n # compute per class average precision\n total_instances = []\n precisions = []\n for label, (average_precision, num_annotations) in average_precisions.items():\n if self.verbose == 1:\n print('{:.0f} instances of class'.format(num_annotations),\n self.generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision))\n total_instances.append(num_annotations)\n precisions.append(average_precision)\n if self.weighted_average:\n self.mean_ap = sum([a * b for a, b in zip(total_instances, precisions)]) / sum(total_instances)\n else:\n self.mean_ap = sum(precisions) / sum(x > 0 for x in total_instances)\n\n if self.tensorboard:\n import tensorflow as tf\n writer = tf.summary.create_file_writer(self.tensorboard.log_dir)\n with writer.as_default():\n tf.summary.scalar(\"mAP\", self.mean_ap, step=epoch)\n if self.verbose == 1:\n for label, (average_precision, num_annotations) in average_precisions.items():\n tf.summary.scalar(\"AP_\" + self.generator.label_to_name(label), average_precision, step=epoch)\n writer.flush()\n\n logs['mAP'] = self.mean_ap\n\n if self.verbose == 1:\n print('mAP: {:.4f}'.format(self.mean_ap))\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/initializers.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom tensorflow import keras\n\nimport math\n\n\nclass PriorProbability(keras.initializers.Initializer):\n \"\"\" Apply a prior probability to the weights.\n \"\"\"\n\n def __init__(self, probability=0.01):\n self.probability = probability\n\n def get_config(self):\n return {\n 'probability': self.probability\n }\n\n def __call__(self, shape, dtype=None):\n # set bias to -log((1 - p)/p) for foreground\n result = keras.backend.ones(shape, dtype=dtype) * -math.log((1 - self.probability) / self.probability)\n\n return result\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/layers/__init__.py", "content": "from ._misc import RegressBoxes, UpsampleLike, Anchors, ClipBoxes # noqa: F401\nfrom .filter_detections import FilterDetections # noqa: F401\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/layers/_misc.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport tensorflow\nfrom tensorflow import keras\nfrom .. import backend\nfrom ..utils import anchors as utils_anchors\n\nimport numpy as np\n\n\nclass Anchors(keras.layers.Layer):\n \"\"\" Keras layer for generating achors for a given shape.\n \"\"\"\n\n def __init__(self, size, stride, ratios=None, scales=None, *args, **kwargs):\n \"\"\" Initializer for an Anchors layer.\n\n Args\n size: The base size of the anchors to generate.\n stride: The stride of the anchors to generate.\n ratios: The ratios of the anchors to generate (defaults to AnchorParameters.default.ratios).\n scales: The scales of the anchors to generate (defaults to AnchorParameters.default.scales).\n \"\"\"\n self.size = size\n self.stride = stride\n self.ratios = ratios\n self.scales = scales\n\n if ratios is None:\n self.ratios = utils_anchors.AnchorParameters.default.ratios\n elif isinstance(ratios, list):\n self.ratios = np.array(ratios)\n if scales is None:\n self.scales = utils_anchors.AnchorParameters.default.scales\n elif isinstance(scales, list):\n self.scales = np.array(scales)\n\n self.num_anchors = len(self.ratios) * len(self.scales)\n self.anchors = utils_anchors.generate_anchors(\n base_size=self.size,\n ratios=self.ratios,\n scales=self.scales,\n ).astype(np.float32)\n\n super(Anchors, self).__init__(*args, **kwargs)\n\n def call(self, inputs, **kwargs):\n features = inputs\n features_shape = keras.backend.shape(features)\n\n # generate proposals from bbox deltas and shifted anchors\n if keras.backend.image_data_format() == 'channels_first':\n anchors = backend.shift(features_shape[2:4], self.stride, self.anchors)\n else:\n anchors = backend.shift(features_shape[1:3], self.stride, self.anchors)\n anchors = keras.backend.tile(keras.backend.expand_dims(anchors, axis=0), (features_shape[0], 1, 1))\n\n return anchors\n\n def compute_output_shape(self, input_shape):\n if None not in input_shape[1:]:\n if keras.backend.image_data_format() == 'channels_first':\n total = np.prod(input_shape[2:4]) * self.num_anchors\n else:\n total = np.prod(input_shape[1:3]) * self.num_anchors\n\n return (input_shape[0], total, 4)\n else:\n return (input_shape[0], None, 4)\n\n def get_config(self):\n config = super(Anchors, self).get_config()\n config.update({\n 'size' : self.size,\n 'stride' : self.stride,\n 'ratios' : self.ratios.tolist(),\n 'scales' : self.scales.tolist(),\n })\n\n return config\n\n\nclass UpsampleLike(keras.layers.Layer):\n \"\"\" Keras layer for upsampling a Tensor to be the same shape as another Tensor.\n \"\"\"\n\n def call(self, inputs, **kwargs):\n source, target = inputs\n target_shape = keras.backend.shape(target)\n if keras.backend.image_data_format() == 'channels_first':\n source = tensorflow.transpose(source, (0, 2, 3, 1))\n output = backend.resize_images(source, (target_shape[2], target_shape[3]), method='nearest')\n output = tensorflow.transpose(output, (0, 3, 1, 2))\n return output\n else:\n return backend.resize_images(source, (target_shape[1], target_shape[2]), method='nearest')\n\n def compute_output_shape(self, input_shape):\n if keras.backend.image_data_format() == 'channels_first':\n return (input_shape[0][0], input_shape[0][1]) + input_shape[1][2:4]\n else:\n return (input_shape[0][0],) + input_shape[1][1:3] + (input_shape[0][-1],)\n\n\nclass RegressBoxes(keras.layers.Layer):\n \"\"\" Keras layer for applying regression values to boxes.\n \"\"\"\n\n def __init__(self, mean=None, std=None, *args, **kwargs):\n \"\"\" Initializer for the RegressBoxes layer.\n\n Args\n mean: The mean value of the regression values which was used for normalization.\n std: The standard value of the regression values which was used for normalization.\n \"\"\"\n if mean is None:\n mean = np.array([0, 0, 0, 0])\n if std is None:\n std = np.array([0.2, 0.2, 0.2, 0.2])\n\n if isinstance(mean, (list, tuple)):\n mean = np.array(mean)\n elif not isinstance(mean, np.ndarray):\n raise ValueError('Expected mean to be a np.ndarray, list or tuple. Received: {}'.format(type(mean)))\n\n if isinstance(std, (list, tuple)):\n std = np.array(std)\n elif not isinstance(std, np.ndarray):\n raise ValueError('Expected std to be a np.ndarray, list or tuple. Received: {}'.format(type(std)))\n\n self.mean = mean\n self.std = std\n super(RegressBoxes, self).__init__(*args, **kwargs)\n\n def call(self, inputs, **kwargs):\n anchors, regression = inputs\n return backend.bbox_transform_inv(anchors, regression, mean=self.mean, std=self.std)\n\n def compute_output_shape(self, input_shape):\n return input_shape[0]\n\n def get_config(self):\n config = super(RegressBoxes, self).get_config()\n config.update({\n 'mean': self.mean.tolist(),\n 'std' : self.std.tolist(),\n })\n\n return config\n\n\nclass ClipBoxes(keras.layers.Layer):\n \"\"\" Keras layer to clip box values to lie inside a given shape.\n \"\"\"\n def call(self, inputs, **kwargs):\n image, boxes = inputs\n shape = keras.backend.cast(keras.backend.shape(image), keras.backend.floatx())\n if keras.backend.image_data_format() == 'channels_first':\n _, _, height, width = tensorflow.unstack(shape, axis=0)\n else:\n _, height, width, _ = tensorflow.unstack(shape, axis=0)\n\n x1, y1, x2, y2 = tensorflow.unstack(boxes, axis=-1)\n x1 = tensorflow.clip_by_value(x1, 0, width - 1)\n y1 = tensorflow.clip_by_value(y1, 0, height - 1)\n x2 = tensorflow.clip_by_value(x2, 0, width - 1)\n y2 = tensorflow.clip_by_value(y2, 0, height - 1)\n\n return keras.backend.stack([x1, y1, x2, y2], axis=2)\n\n def compute_output_shape(self, input_shape):\n return input_shape[1]\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/layers/filter_detections.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport tensorflow\nfrom tensorflow import keras\nfrom .. import backend\n\n\ndef filter_detections(\n boxes,\n classification,\n other = [],\n class_specific_filter = True,\n nms = True,\n score_threshold = 0.05,\n max_detections = 300,\n nms_threshold = 0.5\n):\n \"\"\" Filter detections using the boxes and classification values.\n\n Args\n boxes : Tensor of shape (num_boxes, 4) containing the boxes in (x1, y1, x2, y2) format.\n classification : Tensor of shape (num_boxes, num_classes) containing the classification scores.\n other : List of tensors of shape (num_boxes, ...) to filter along with the boxes and classification scores.\n class_specific_filter : Whether to perform filtering per class, or take the best scoring class and filter those.\n nms : Flag to enable/disable non maximum suppression.\n score_threshold : Threshold used to prefilter the boxes with.\n max_detections : Maximum number of detections to keep.\n nms_threshold : Threshold for the IoU value to determine when a box should be suppressed.\n\n Returns\n A list of [boxes, scores, labels, other[0], other[1], ...].\n boxes is shaped (max_detections, 4) and contains the (x1, y1, x2, y2) of the non-suppressed boxes.\n scores is shaped (max_detections,) and contains the scores of the predicted class.\n labels is shaped (max_detections,) and contains the predicted label.\n other[i] is shaped (max_detections, ...) and contains the filtered other[i] data.\n In case there are less than max_detections detections, the tensors are padded with -1's.\n \"\"\"\n def _filter_detections(scores, labels):\n # threshold based on score\n indices = tensorflow.where(keras.backend.greater(scores, score_threshold))\n\n if nms:\n filtered_boxes = tensorflow.gather_nd(boxes, indices)\n filtered_scores = keras.backend.gather(scores, indices)[:, 0]\n\n # perform NMS\n nms_indices = tensorflow.image.non_max_suppression(filtered_boxes, filtered_scores, max_output_size=max_detections, iou_threshold=nms_threshold)\n\n # filter indices based on NMS\n indices = keras.backend.gather(indices, nms_indices)\n\n # add indices to list of all indices\n labels = tensorflow.gather_nd(labels, indices)\n indices = keras.backend.stack([indices[:, 0], labels], axis=1)\n\n return indices\n\n if class_specific_filter:\n all_indices = []\n # perform per class filtering\n for c in range(int(classification.shape[1])):\n scores = classification[:, c]\n labels = c * tensorflow.ones((keras.backend.shape(scores)[0],), dtype='int64')\n all_indices.append(_filter_detections(scores, labels))\n\n # concatenate indices to single tensor\n indices = keras.backend.concatenate(all_indices, axis=0)\n else:\n scores = keras.backend.max(classification, axis = 1)\n labels = keras.backend.argmax(classification, axis = 1)\n indices = _filter_detections(scores, labels)\n\n # select top k\n scores = tensorflow.gather_nd(classification, indices)\n labels = indices[:, 1]\n scores, top_indices = tensorflow.nn.top_k(scores, k=keras.backend.minimum(max_detections, keras.backend.shape(scores)[0]))\n\n # filter input using the final set of indices\n indices = keras.backend.gather(indices[:, 0], top_indices)\n boxes = keras.backend.gather(boxes, indices)\n labels = keras.backend.gather(labels, top_indices)\n other_ = [keras.backend.gather(o, indices) for o in other]\n\n # zero pad the outputs\n pad_size = keras.backend.maximum(0, max_detections - keras.backend.shape(scores)[0])\n boxes = tensorflow.pad(boxes, [[0, pad_size], [0, 0]], constant_values=-1)\n scores = tensorflow.pad(scores, [[0, pad_size]], constant_values=-1)\n labels = tensorflow.pad(labels, [[0, pad_size]], constant_values=-1)\n labels = keras.backend.cast(labels, 'int32')\n other_ = [tensorflow.pad(o, [[0, pad_size]] + [[0, 0] for _ in range(1, len(o.shape))], constant_values=-1) for o in other_]\n\n # set shapes, since we know what they are\n boxes.set_shape([max_detections, 4])\n scores.set_shape([max_detections])\n labels.set_shape([max_detections])\n for o, s in zip(other_, [list(keras.backend.int_shape(o)) for o in other]):\n o.set_shape([max_detections] + s[1:])\n\n return [boxes, scores, labels] + other_\n\n\nclass FilterDetections(keras.layers.Layer):\n \"\"\" Keras layer for filtering detections using score threshold and NMS.\n \"\"\"\n\n def __init__(\n self,\n nms = True,\n class_specific_filter = True,\n nms_threshold = 0.5,\n score_threshold = 0.05,\n max_detections = 300,\n parallel_iterations = 32,\n **kwargs\n ):\n \"\"\" Filters detections using score threshold, NMS and selecting the top-k detections.\n\n Args\n nms : Flag to enable/disable NMS.\n class_specific_filter : Whether to perform filtering per class, or take the best scoring class and filter those.\n nms_threshold : Threshold for the IoU value to determine when a box should be suppressed.\n score_threshold : Threshold used to prefilter the boxes with.\n max_detections : Maximum number of detections to keep.\n parallel_iterations : Number of batch items to process in parallel.\n \"\"\"\n self.nms = nms\n self.class_specific_filter = class_specific_filter\n self.nms_threshold = nms_threshold\n self.score_threshold = score_threshold\n self.max_detections = max_detections\n self.parallel_iterations = parallel_iterations\n super(FilterDetections, self).__init__(**kwargs)\n\n def call(self, inputs, **kwargs):\n \"\"\" Constructs the NMS graph.\n\n Args\n inputs : List of [boxes, classification, other[0], other[1], ...] tensors.\n \"\"\"\n boxes = inputs[0]\n classification = inputs[1]\n other = inputs[2:]\n\n # wrap nms with our parameters\n def _filter_detections(args):\n boxes = args[0]\n classification = args[1]\n other = args[2]\n\n return filter_detections(\n boxes,\n classification,\n other,\n nms = self.nms,\n class_specific_filter = self.class_specific_filter,\n score_threshold = self.score_threshold,\n max_detections = self.max_detections,\n nms_threshold = self.nms_threshold,\n )\n\n # call filter_detections on each batch\n dtypes = [keras.backend.floatx(), keras.backend.floatx(), 'int32'] + [o.dtype for o in other]\n shapes = [(self.max_detections, 4), (self.max_detections,), (self.max_detections,)]\n shapes.extend([(self.max_detections,) + o.shape[2:] for o in other])\n outputs = backend.map_fn(\n _filter_detections,\n elems=[boxes, classification, other],\n dtype=dtypes,\n shapes=shapes,\n parallel_iterations=self.parallel_iterations,\n )\n\n return outputs\n\n def compute_output_shape(self, input_shape):\n \"\"\" Computes the output shapes given the input shapes.\n\n Args\n input_shape : List of input shapes [boxes, classification, other[0], other[1], ...].\n\n Returns\n List of tuples representing the output shapes:\n [filtered_boxes.shape, filtered_scores.shape, filtered_labels.shape, filtered_other[0].shape, filtered_other[1].shape, ...]\n \"\"\"\n return [\n (input_shape[0][0], self.max_detections, 4),\n (input_shape[1][0], self.max_detections),\n (input_shape[1][0], self.max_detections),\n ] + [\n tuple([input_shape[i][0], self.max_detections] + list(input_shape[i][2:])) for i in range(2, len(input_shape))\n ]\n\n def compute_mask(self, inputs, mask=None):\n \"\"\" This is required in Keras when there is more than 1 output.\n \"\"\"\n return (len(inputs) + 1) * [None]\n\n def get_config(self):\n \"\"\" Gets the configuration of this layer.\n\n Returns\n Dictionary containing the parameters of this layer.\n \"\"\"\n config = super(FilterDetections, self).get_config()\n config.update({\n 'nms' : self.nms,\n 'class_specific_filter' : self.class_specific_filter,\n 'nms_threshold' : self.nms_threshold,\n 'score_threshold' : self.score_threshold,\n 'max_detections' : self.max_detections,\n 'parallel_iterations' : self.parallel_iterations,\n })\n\n return config\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/losses.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport tensorflow\nfrom tensorflow import keras\n\n\ndef focal(alpha=0.25, gamma=2.0, cutoff=0.5):\n \"\"\" Create a functor for computing the focal loss.\n\n Args\n alpha: Scale the focal weight with alpha.\n gamma: Take the power of the focal weight with gamma.\n cutoff: Positive prediction cutoff for soft targets\n\n Returns\n A functor that computes the focal loss using the alpha and gamma.\n \"\"\"\n def _focal(y_true, y_pred):\n \"\"\" Compute the focal loss given the target tensor and the predicted tensor.\n\n As defined in https://arxiv.org/abs/1708.02002\n\n Args\n y_true: Tensor of target data from the generator with shape (B, N, num_classes).\n y_pred: Tensor of predicted data from the network with shape (B, N, num_classes).\n\n Returns\n The focal loss of y_pred w.r.t. y_true.\n \"\"\"\n labels = y_true[:, :, :-1]\n anchor_state = y_true[:, :, -1] # -1 for ignore, 0 for background, 1 for object\n classification = y_pred\n\n # filter out \"ignore\" anchors\n indices = tensorflow.where(keras.backend.not_equal(anchor_state, -1))\n labels = tensorflow.gather_nd(labels, indices)\n classification = tensorflow.gather_nd(classification, indices)\n\n # compute the focal loss\n alpha_factor = keras.backend.ones_like(labels) * alpha\n alpha_factor = tensorflow.where(keras.backend.greater(labels, cutoff), alpha_factor, 1 - alpha_factor)\n focal_weight = tensorflow.where(keras.backend.greater(labels, cutoff), 1 - classification, classification)\n focal_weight = alpha_factor * focal_weight ** gamma\n\n cls_loss = focal_weight * keras.backend.binary_crossentropy(labels, classification)\n\n # compute the normalizer: the number of positive anchors\n normalizer = tensorflow.where(keras.backend.equal(anchor_state, 1))\n normalizer = keras.backend.cast(keras.backend.shape(normalizer)[0], keras.backend.floatx())\n normalizer = keras.backend.maximum(keras.backend.cast_to_floatx(1.0), normalizer)\n\n return keras.backend.sum(cls_loss) / normalizer\n\n return _focal\n\n\ndef smooth_l1(sigma=3.0):\n \"\"\" Create a smooth L1 loss functor.\n\n Args\n sigma: This argument defines the point where the loss changes from L2 to L1.\n\n Returns\n A functor for computing the smooth L1 loss given target data and predicted data.\n \"\"\"\n sigma_squared = sigma ** 2\n\n def _smooth_l1(y_true, y_pred):\n \"\"\" Compute the smooth L1 loss of y_pred w.r.t. y_true.\n\n Args\n y_true: Tensor from the generator of shape (B, N, 5). The last value for each box is the state of the anchor (ignore, negative, positive).\n y_pred: Tensor from the network of shape (B, N, 4).\n\n Returns\n The smooth L1 loss of y_pred w.r.t. y_true.\n \"\"\"\n # separate target and state\n regression = y_pred\n regression_target = y_true[:, :, :-1]\n anchor_state = y_true[:, :, -1]\n\n # filter out \"ignore\" anchors\n indices = tensorflow.where(keras.backend.equal(anchor_state, 1))\n regression = tensorflow.gather_nd(regression, indices)\n regression_target = tensorflow.gather_nd(regression_target, indices)\n\n # compute smooth L1 loss\n # f(x) = 0.5 * (sigma * x)^2 if |x| < 1 / sigma / sigma\n # |x| - 0.5 / sigma / sigma otherwise\n regression_diff = regression - regression_target\n regression_diff = keras.backend.abs(regression_diff)\n regression_loss = tensorflow.where(\n keras.backend.less(regression_diff, 1.0 / sigma_squared),\n 0.5 * sigma_squared * keras.backend.pow(regression_diff, 2),\n regression_diff - 0.5 / sigma_squared\n )\n\n # compute the normalizer: the number of positive anchors\n normalizer = keras.backend.maximum(1, keras.backend.shape(indices)[0])\n normalizer = keras.backend.cast(normalizer, dtype=keras.backend.floatx())\n return keras.backend.sum(regression_loss) / normalizer\n\n return _smooth_l1\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/models/__init__.py", "content": "from __future__ import print_function\nimport sys\n\n\nclass Backbone(object):\n \"\"\" This class stores additional information on backbones.\n \"\"\"\n def __init__(self, backbone):\n # a dictionary mapping custom layer names to the correct classes\n from .. import layers\n from .. import losses\n from .. import initializers\n self.custom_objects = {\n 'UpsampleLike' : layers.UpsampleLike,\n 'PriorProbability' : initializers.PriorProbability,\n 'RegressBoxes' : layers.RegressBoxes,\n 'FilterDetections' : layers.FilterDetections,\n 'Anchors' : layers.Anchors,\n 'ClipBoxes' : layers.ClipBoxes,\n '_smooth_l1' : losses.smooth_l1(),\n '_focal' : losses.focal(),\n }\n\n self.backbone = backbone\n self.validate()\n\n def retinanet(self, *args, **kwargs):\n \"\"\" Returns a retinanet model using the correct backbone.\n \"\"\"\n raise NotImplementedError('retinanet method not implemented.')\n\n def download_imagenet(self):\n \"\"\" Downloads ImageNet weights and returns path to weights file.\n \"\"\"\n raise NotImplementedError('download_imagenet method not implemented.')\n\n def validate(self):\n \"\"\" Checks whether the backbone string is correct.\n \"\"\"\n raise NotImplementedError('validate method not implemented.')\n\n def preprocess_image(self, inputs):\n \"\"\" Takes as input an image and prepares it for being passed through the network.\n Having this function in Backbone allows other backbones to define a specific preprocessing step.\n \"\"\"\n raise NotImplementedError('preprocess_image method not implemented.')\n\n\ndef backbone(backbone_name):\n \"\"\" Returns a backbone object for the given backbone.\n \"\"\"\n if 'densenet' in backbone_name:\n from .densenet import DenseNetBackbone as b\n elif 'seresnext' in backbone_name or 'seresnet' in backbone_name or 'senet' in backbone_name:\n from .senet import SeBackbone as b\n elif 'resnet' in backbone_name:\n from .resnet import ResNetBackbone as b\n elif 'mobilenet' in backbone_name:\n from .mobilenet import MobileNetBackbone as b\n elif 'vgg' in backbone_name:\n from .vgg import VGGBackbone as b\n elif 'EfficientNet' in backbone_name:\n from .effnet import EfficientNetBackbone as b\n else:\n raise NotImplementedError('Backbone class for \\'{}\\' not implemented.'.format(backbone))\n\n return b(backbone_name)\n\n\ndef load_model(filepath, backbone_name='resnet50'):\n \"\"\" Loads a retinanet model using the correct custom objects.\n\n Args\n filepath: one of the following:\n - string, path to the saved model, or\n - h5py.File object from which to load the model\n backbone_name : Backbone with which the model was trained.\n\n Returns\n A keras.models.Model object.\n\n Raises\n ImportError: if h5py is not available.\n ValueError: In case of an invalid savefile.\n \"\"\"\n from tensorflow import keras\n return keras.models.load_model(filepath, custom_objects=backbone(backbone_name).custom_objects)\n\n\ndef convert_model(model, nms=True, class_specific_filter=True, anchor_params=None, **kwargs):\n \"\"\" Converts a training model to an inference model.\n\n Args\n model : A retinanet training model.\n nms : Boolean, whether to add NMS filtering to the converted model.\n class_specific_filter : Whether to use class specific filtering or filter for the best scoring class only.\n anchor_params : Anchor parameters object. If omitted, default values are used.\n **kwargs : Inference and minimal retinanet model settings.\n\n Returns\n A keras.models.Model object.\n\n Raises\n ImportError: if h5py is not available.\n ValueError: In case of an invalid savefile.\n \"\"\"\n from .retinanet import retinanet_bbox\n return retinanet_bbox(model=model, nms=nms, class_specific_filter=class_specific_filter, anchor_params=anchor_params, **kwargs)\n\n\ndef assert_training_model(model):\n \"\"\" Assert that the model is a training model.\n \"\"\"\n assert(all(output in model.output_names for output in ['regression', 'classification'])), \\\n \"Input is not a training model (no 'regression' and 'classification' outputs were found, outputs are: {}).\".format(model.output_names)\n\n\ndef check_training_model(model):\n \"\"\" Check that model is a training model and exit otherwise.\n \"\"\"\n try:\n assert_training_model(model)\n except AssertionError as e:\n print(e, file=sys.stderr)\n sys.exit(1)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/models/densenet.py", "content": "\"\"\"\nCopyright 2018 vidosits (https://github.com/vidosits/)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom tensorflow import keras\n\nfrom . import retinanet\nfrom . import Backbone\nfrom ..utils.image import preprocess_image\n\n\nallowed_backbones = {\n 'densenet121': ([6, 12, 24, 16], keras.applications.densenet.DenseNet121),\n 'densenet169': ([6, 12, 32, 32], keras.applications.densenet.DenseNet169),\n 'densenet201': ([6, 12, 48, 32], keras.applications.densenet.DenseNet201),\n}\n\n\nclass DenseNetBackbone(Backbone):\n \"\"\" Describes backbone information and provides utility functions.\n \"\"\"\n\n def retinanet(self, *args, **kwargs):\n \"\"\" Returns a retinanet model using the correct backbone.\n \"\"\"\n return densenet_retinanet(*args, backbone=self.backbone, **kwargs)\n\n def download_imagenet(self):\n \"\"\" Download pre-trained weights for the specified backbone name.\n This name is in the format {backbone}_weights_tf_dim_ordering_tf_kernels_notop\n where backbone is the densenet + number of layers (e.g. densenet121).\n For more info check the explanation from the keras densenet script itself:\n https://github.com/keras-team/keras/blob/master/keras/applications/densenet.py\n \"\"\"\n origin = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.8/'\n file_name = '{}_weights_tf_dim_ordering_tf_kernels_notop.h5'\n\n # load weights\n if keras.backend.image_data_format() == 'channels_first':\n raise ValueError('Weights for \"channels_first\" format are not available.')\n\n weights_url = origin + file_name.format(self.backbone)\n return keras.utils.get_file(file_name.format(self.backbone), weights_url, cache_subdir='models')\n\n def validate(self):\n \"\"\" Checks whether the backbone string is correct.\n \"\"\"\n backbone = self.backbone.split('_')[0]\n\n if backbone not in allowed_backbones:\n raise ValueError('Backbone (\\'{}\\') not in allowed backbones ({}).'.format(backbone, allowed_backbones.keys()))\n\n def preprocess_image(self, inputs):\n \"\"\" Takes as input an image and prepares it for being passed through the network.\n \"\"\"\n return preprocess_image(inputs, mode='tf')\n\n\ndef densenet_retinanet(num_classes, backbone='densenet121', inputs=None, modifier=None, **kwargs):\n \"\"\" Constructs a retinanet model using a densenet backbone.\n\n Args\n num_classes: Number of classes to predict.\n backbone: Which backbone to use (one of ('densenet121', 'densenet169', 'densenet201')).\n inputs: The inputs to the network (defaults to a Tensor of shape (None, None, 3)).\n modifier: A function handler which can modify the backbone before using it in retinanet (this can be used to freeze backbone layers for example).\n\n Returns\n RetinaNet model with a DenseNet backbone.\n \"\"\"\n # choose default input\n if inputs is None:\n inputs = keras.layers.Input((None, None, 3))\n\n blocks, creator = allowed_backbones[backbone]\n model = creator(input_tensor=inputs, include_top=False, pooling=None, weights=None)\n\n # get last conv layer from the end of each dense block\n layer_outputs = [model.get_layer(name='conv{}_block{}_concat'.format(idx + 2, block_num)).output for idx, block_num in enumerate(blocks)]\n\n # create the densenet backbone\n # layer_outputs contains 4 layers\n model = keras.models.Model(inputs=inputs, outputs=layer_outputs, name=model.name)\n\n # invoke modifier if given\n if modifier:\n model = modifier(model)\n\n # create the full model\n backbone_layers = {\n 'C2': model.outputs[0],\n 'C3': model.outputs[1],\n 'C4': model.outputs[2],\n 'C5': model.outputs[3]\n }\n\n model = retinanet.retinanet(inputs=inputs, num_classes=num_classes, backbone_layers=backbone_layers, **kwargs)\n\n return model\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/models/effnet.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom tensorflow import keras\n\nfrom . import retinanet\nfrom . import Backbone\nimport efficientnet.keras as efn\n\n\nclass EfficientNetBackbone(Backbone):\n \"\"\" Describes backbone information and provides utility functions.\n \"\"\"\n\n def __init__(self, backbone):\n super(EfficientNetBackbone, self).__init__(backbone)\n self.preprocess_image_func = None\n\n def retinanet(self, *args, **kwargs):\n \"\"\" Returns a retinanet model using the correct backbone.\n \"\"\"\n return effnet_retinanet(*args, backbone=self.backbone, **kwargs)\n\n def download_imagenet(self):\n \"\"\" Downloads ImageNet weights and returns path to weights file.\n \"\"\"\n from efficientnet.weights import IMAGENET_WEIGHTS_PATH\n from efficientnet.weights import IMAGENET_WEIGHTS_HASHES\n\n model_name = 'efficientnet-b' + self.backbone[-1]\n file_name = model_name + '_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'\n file_hash = IMAGENET_WEIGHTS_HASHES[model_name][1]\n weights_path = keras.utils.get_file(file_name, IMAGENET_WEIGHTS_PATH + file_name, cache_subdir='models', file_hash=file_hash)\n return weights_path\n\n def validate(self):\n \"\"\" Checks whether the backbone string is correct.\n \"\"\"\n allowed_backbones = ['EfficientNetB0', 'EfficientNetB1', 'EfficientNetB2', 'EfficientNetB3', 'EfficientNetB4',\n 'EfficientNetB5', 'EfficientNetB6', 'EfficientNetB7']\n backbone = self.backbone.split('_')[0]\n\n if backbone not in allowed_backbones:\n raise ValueError('Backbone (\\'{}\\') not in allowed backbones ({}).'.format(backbone, allowed_backbones))\n\n def preprocess_image(self, inputs):\n \"\"\" Takes as input an image and prepares it for being passed through the network.\n \"\"\"\n return efn.preprocess_input(inputs)\n\n\ndef effnet_retinanet(num_classes, backbone='EfficientNetB0', inputs=None, modifier=None, **kwargs):\n \"\"\" Constructs a retinanet model using a resnet backbone.\n\n Args\n num_classes: Number of classes to predict.\n backbone: Which backbone to use (one of ('resnet50', 'resnet101', 'resnet152')).\n inputs: The inputs to the network (defaults to a Tensor of shape (None, None, 3)).\n modifier: A function handler which can modify the backbone before using it in retinanet (this can be used to freeze backbone layers for example).\n\n Returns\n RetinaNet model with a ResNet backbone.\n \"\"\"\n # choose default input\n if inputs is None:\n if keras.backend.image_data_format() == 'channels_first':\n inputs = keras.layers.Input(shape=(3, None, None))\n else:\n # inputs = keras.layers.Input(shape=(224, 224, 3))\n inputs = keras.layers.Input(shape=(None, None, 3))\n\n # get last conv layer from the end of each block [28x28, 14x14, 7x7]\n if backbone == 'EfficientNetB0':\n model = efn.EfficientNetB0(input_tensor=inputs, include_top=False, weights=None)\n elif backbone == 'EfficientNetB1':\n model = efn.EfficientNetB1(input_tensor=inputs, include_top=False, weights=None)\n elif backbone == 'EfficientNetB2':\n model = efn.EfficientNetB2(input_tensor=inputs, include_top=False, weights=None)\n elif backbone == 'EfficientNetB3':\n model = efn.EfficientNetB3(input_tensor=inputs, include_top=False, weights=None)\n elif backbone == 'EfficientNetB4':\n model = efn.EfficientNetB4(input_tensor=inputs, include_top=False, weights=None)\n elif backbone == 'EfficientNetB5':\n model = efn.EfficientNetB5(input_tensor=inputs, include_top=False, weights=None)\n elif backbone == 'EfficientNetB6':\n model = efn.EfficientNetB6(input_tensor=inputs, include_top=False, weights=None)\n elif backbone == 'EfficientNetB7':\n model = efn.EfficientNetB7(input_tensor=inputs, include_top=False, weights=None)\n else:\n raise ValueError('Backbone (\\'{}\\') is invalid.'.format(backbone))\n\n layer_outputs = ['block4a_expand_activation', 'block6a_expand_activation', 'top_activation']\n\n layer_outputs = [\n model.get_layer(name=layer_outputs[0]).output, # 28x28\n model.get_layer(name=layer_outputs[1]).output, # 14x14\n model.get_layer(name=layer_outputs[2]).output, # 7x7\n ]\n # create the densenet backbone\n model = keras.models.Model(inputs=inputs, outputs=layer_outputs, name=model.name)\n\n # invoke modifier if given\n if modifier:\n model = modifier(model)\n\n # C2 not provided\n backbone_layers = {\n 'C3': model.outputs[0],\n 'C4': model.outputs[1],\n 'C5': model.outputs[2]\n }\n\n # create the full model\n return retinanet.retinanet(inputs=inputs, num_classes=num_classes, backbone_layers=backbone_layers, **kwargs)\n\n\ndef EfficientNetB0_retinanet(num_classes, inputs=None, **kwargs):\n return effnet_retinanet(num_classes=num_classes, backbone='EfficientNetB0', inputs=inputs, **kwargs)\n\n\ndef EfficientNetB1_retinanet(num_classes, inputs=None, **kwargs):\n return effnet_retinanet(num_classes=num_classes, backbone='EfficientNetB1', inputs=inputs, **kwargs)\n\n\ndef EfficientNetB2_retinanet(num_classes, inputs=None, **kwargs):\n return effnet_retinanet(num_classes=num_classes, backbone='EfficientNetB2', inputs=inputs, **kwargs)\n\n\ndef EfficientNetB3_retinanet(num_classes, inputs=None, **kwargs):\n return effnet_retinanet(num_classes=num_classes, backbone='EfficientNetB3', inputs=inputs, **kwargs)\n\n\ndef EfficientNetB4_retinanet(num_classes, inputs=None, **kwargs):\n return effnet_retinanet(num_classes=num_classes, backbone='EfficientNetB4', inputs=inputs, **kwargs)\n\n\ndef EfficientNetB5_retinanet(num_classes, inputs=None, **kwargs):\n return effnet_retinanet(num_classes=num_classes, backbone='EfficientNetB5', inputs=inputs, **kwargs)\n\n\ndef EfficientNetB6_retinanet(num_classes, inputs=None, **kwargs):\n return effnet_retinanet(num_classes=num_classes, backbone='EfficientNetB6', inputs=inputs, **kwargs)\n\n\ndef EfficientNetB7_retinanet(num_classes, inputs=None, **kwargs):\n return effnet_retinanet(num_classes=num_classes, backbone='EfficientNetB7', inputs=inputs, **kwargs)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/models/mobilenet.py", "content": "\"\"\"\nCopyright 2017-2018 lvaleriu (https://github.com/lvaleriu/)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom tensorflow import keras\nfrom ..utils.image import preprocess_image\n\nfrom . import retinanet\nfrom . import Backbone\n\n\nclass MobileNetBackbone(Backbone):\n \"\"\" Describes backbone information and provides utility functions.\n \"\"\"\n\n allowed_backbones = ['mobilenet128', 'mobilenet160', 'mobilenet192', 'mobilenet224']\n\n def retinanet(self, *args, **kwargs):\n \"\"\" Returns a retinanet model using the correct backbone.\n \"\"\"\n return mobilenet_retinanet(*args, backbone=self.backbone, **kwargs)\n\n def download_imagenet(self):\n \"\"\" Download pre-trained weights for the specified backbone name.\n This name is in the format mobilenet{rows}_{alpha} where rows is the\n imagenet shape dimension and 'alpha' controls the width of the network.\n For more info check the explanation from the keras mobilenet script itself.\n \"\"\"\n\n alpha = float(self.backbone.split('_')[1])\n rows = int(self.backbone.split('_')[0].replace('mobilenet', ''))\n\n # load weights\n if keras.backend.image_data_format() == 'channels_first':\n raise ValueError('Weights for \"channels_last\" format '\n 'are not available.')\n if alpha == 1.0:\n alpha_text = '1_0'\n elif alpha == 0.75:\n alpha_text = '7_5'\n elif alpha == 0.50:\n alpha_text = '5_0'\n else:\n alpha_text = '2_5'\n\n model_name = 'mobilenet_{}_{}_tf_no_top.h5'.format(alpha_text, rows)\n weights_url = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.6/' + model_name\n weights_path = keras.utils.get_file(model_name, weights_url, cache_subdir='models')\n\n return weights_path\n\n def validate(self):\n \"\"\" Checks whether the backbone string is correct.\n \"\"\"\n backbone = self.backbone.split('_')[0]\n\n if backbone not in MobileNetBackbone.allowed_backbones:\n raise ValueError('Backbone (\\'{}\\') not in allowed backbones ({}).'.format(backbone, MobileNetBackbone.allowed_backbones))\n\n def preprocess_image(self, inputs):\n \"\"\" Takes as input an image and prepares it for being passed through the network.\n \"\"\"\n return preprocess_image(inputs, mode='tf')\n\n\ndef mobilenet_retinanet(num_classes, backbone='mobilenet224_1.0', inputs=None, modifier=None, **kwargs):\n \"\"\" Constructs a retinanet model using a mobilenet backbone.\n\n Args\n num_classes: Number of classes to predict.\n backbone: Which backbone to use (one of ('mobilenet128', 'mobilenet160', 'mobilenet192', 'mobilenet224')).\n inputs: The inputs to the network (defaults to a Tensor of shape (None, None, 3)).\n modifier: A function handler which can modify the backbone before using it in retinanet (this can be used to freeze backbone layers for example).\n\n Returns\n RetinaNet model with a MobileNet backbone.\n \"\"\"\n alpha = float(backbone.split('_')[1])\n\n # choose default input\n if inputs is None:\n inputs = keras.layers.Input((None, None, 3))\n\n backbone = keras.applications.mobilenet.MobileNet(input_tensor=inputs, alpha=alpha, include_top=False, pooling=None, weights=None)\n\n # create the full model\n layer_names = ['conv_pw_5_relu', 'conv_pw_11_relu', 'conv_pw_13_relu']\n layer_outputs = [backbone.get_layer(name).output for name in layer_names]\n backbone = keras.models.Model(inputs=inputs, outputs=layer_outputs, name=backbone.name)\n\n # invoke modifier if given\n if modifier:\n backbone = modifier(backbone)\n\n # C2 not provided\n backbone_layers = {\n 'C3': backbone.outputs[0],\n 'C4': backbone.outputs[1],\n 'C5': backbone.outputs[2]\n }\n\n return retinanet.retinanet(inputs=inputs, num_classes=num_classes, backbone_layers=backbone_layers, **kwargs)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/models/resnet.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom tensorflow import keras\nimport keras_resnet\nimport keras_resnet.models\n\nfrom . import retinanet\nfrom . import Backbone\nfrom ..utils.image import preprocess_image\n\n\nclass ResNetBackbone(Backbone):\n \"\"\" Describes backbone information and provides utility functions.\n \"\"\"\n\n def __init__(self, backbone):\n super(ResNetBackbone, self).__init__(backbone)\n self.custom_objects.update(keras_resnet.custom_objects)\n\n def retinanet(self, *args, **kwargs):\n \"\"\" Returns a retinanet model using the correct backbone.\n \"\"\"\n return resnet_retinanet(*args, backbone=self.backbone, **kwargs)\n\n def download_imagenet(self):\n \"\"\" Downloads ImageNet weights and returns path to weights file.\n \"\"\"\n resnet_filename = 'ResNet-{}-model.keras.h5'\n resnet_resource = 'https://github.com/fizyr/keras-models/releases/download/v0.0.1/{}'.format(resnet_filename)\n depth = int(self.backbone.replace('resnet', ''))\n\n filename = resnet_filename.format(depth)\n resource = resnet_resource.format(depth)\n if depth == 50:\n checksum = '3e9f4e4f77bbe2c9bec13b53ee1c2319'\n elif depth == 101:\n checksum = '05dc86924389e5b401a9ea0348a3213c'\n elif depth == 152:\n checksum = '6ee11ef2b135592f8031058820bb9e71'\n\n return keras.utils.get_file(\n filename,\n resource,\n cache_subdir='models',\n md5_hash=checksum\n )\n\n def validate(self):\n \"\"\" Checks whether the backbone string is correct.\n \"\"\"\n allowed_backbones = ['resnet50', 'resnet101', 'resnet152']\n backbone = self.backbone.split('_')[0]\n\n if backbone not in allowed_backbones:\n raise ValueError('Backbone (\\'{}\\') not in allowed backbones ({}).'.format(backbone, allowed_backbones))\n\n def preprocess_image(self, inputs):\n \"\"\" Takes as input an image and prepares it for being passed through the network.\n \"\"\"\n return preprocess_image(inputs, mode='caffe')\n\n\ndef resnet_retinanet(num_classes, backbone='resnet50', inputs=None, modifier=None, **kwargs):\n \"\"\" Constructs a retinanet model using a resnet backbone.\n\n Args\n num_classes: Number of classes to predict.\n backbone: Which backbone to use (one of ('resnet50', 'resnet101', 'resnet152')).\n inputs: The inputs to the network (defaults to a Tensor of shape (None, None, 3)).\n modifier: A function handler which can modify the backbone before using it in retinanet (this can be used to freeze backbone layers for example).\n\n Returns\n RetinaNet model with a ResNet backbone.\n \"\"\"\n # choose default input\n if inputs is None:\n if keras.backend.image_data_format() == 'channels_first':\n inputs = keras.layers.Input(shape=(3, None, None))\n else:\n inputs = keras.layers.Input(shape=(None, None, 3))\n\n # create the resnet backbone\n if backbone == 'resnet50':\n resnet = keras_resnet.models.ResNet50(inputs, include_top=False, freeze_bn=True)\n elif backbone == 'resnet101':\n resnet = keras_resnet.models.ResNet101(inputs, include_top=False, freeze_bn=True)\n elif backbone == 'resnet152':\n resnet = keras_resnet.models.ResNet152(inputs, include_top=False, freeze_bn=True)\n else:\n raise ValueError('Backbone (\\'{}\\') is invalid.'.format(backbone))\n\n # invoke modifier if given\n if modifier:\n resnet = modifier(resnet)\n\n # create the full model\n # resnet.outputs contains 4 layers\n backbone_layers = {\n 'C2': resnet.outputs[0],\n 'C3': resnet.outputs[1],\n 'C4': resnet.outputs[2],\n 'C5': resnet.outputs[3]\n }\n\n return retinanet.retinanet(inputs=inputs, num_classes=num_classes, backbone_layers=backbone_layers, **kwargs)\n\n\ndef resnet50_retinanet(num_classes, inputs=None, **kwargs):\n return resnet_retinanet(num_classes=num_classes, backbone='resnet50', inputs=inputs, **kwargs)\n\n\ndef resnet101_retinanet(num_classes, inputs=None, **kwargs):\n return resnet_retinanet(num_classes=num_classes, backbone='resnet101', inputs=inputs, **kwargs)\n\n\ndef resnet152_retinanet(num_classes, inputs=None, **kwargs):\n return resnet_retinanet(num_classes=num_classes, backbone='resnet152', inputs=inputs, **kwargs)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/models/retinanet.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom tensorflow import keras\nfrom .. import initializers\nfrom .. import layers\nfrom ..utils.anchors import AnchorParameters\nfrom . import assert_training_model\n\n\ndef default_classification_model(\n num_classes,\n num_anchors,\n pyramid_feature_size=256,\n prior_probability=0.01,\n classification_feature_size=256,\n name='classification_submodel'\n):\n \"\"\" Creates the default classification submodel.\n\n Args\n num_classes : Number of classes to predict a score for at each feature level.\n num_anchors : Number of anchors to predict classification scores for at each feature level.\n pyramid_feature_size : The number of filters to expect from the feature pyramid levels.\n classification_feature_size : The number of filters to use in the layers in the classification submodel.\n name : The name of the submodel.\n\n Returns\n A keras.models.Model that predicts classes for each anchor.\n \"\"\"\n options = {\n 'kernel_size' : 3,\n 'strides' : 1,\n 'padding' : 'same',\n }\n\n if keras.backend.image_data_format() == 'channels_first':\n inputs = keras.layers.Input(shape=(pyramid_feature_size, None, None))\n else:\n inputs = keras.layers.Input(shape=(None, None, pyramid_feature_size))\n outputs = inputs\n for i in range(4):\n outputs = keras.layers.Conv2D(\n filters=classification_feature_size,\n activation='relu',\n name='pyramid_classification_{}'.format(i),\n kernel_initializer=keras.initializers.RandomNormal(mean=0.0, stddev=0.01, seed=None),\n bias_initializer='zeros',\n **options\n )(outputs)\n\n outputs = keras.layers.Conv2D(\n filters=num_classes * num_anchors,\n kernel_initializer=keras.initializers.RandomNormal(mean=0.0, stddev=0.01, seed=None),\n bias_initializer=initializers.PriorProbability(probability=prior_probability),\n name='pyramid_classification',\n **options\n )(outputs)\n\n # reshape output and apply sigmoid\n if keras.backend.image_data_format() == 'channels_first':\n outputs = keras.layers.Permute((2, 3, 1), name='pyramid_classification_permute')(outputs)\n outputs = keras.layers.Reshape((-1, num_classes), name='pyramid_classification_reshape')(outputs)\n outputs = keras.layers.Activation('sigmoid', name='pyramid_classification_sigmoid')(outputs)\n\n return keras.models.Model(inputs=inputs, outputs=outputs, name=name)\n\n\ndef default_regression_model(num_values, num_anchors, pyramid_feature_size=256, regression_feature_size=256, name='regression_submodel'):\n \"\"\" Creates the default regression submodel.\n\n Args\n num_values : Number of values to regress.\n num_anchors : Number of anchors to regress for each feature level.\n pyramid_feature_size : The number of filters to expect from the feature pyramid levels.\n regression_feature_size : The number of filters to use in the layers in the regression submodel.\n name : The name of the submodel.\n\n Returns\n A keras.models.Model that predicts regression values for each anchor.\n \"\"\"\n # All new conv layers except the final one in the\n # RetinaNet (classification) subnets are initialized\n # with bias b = 0 and a Gaussian weight fill with stddev = 0.01.\n options = {\n 'kernel_size' : 3,\n 'strides' : 1,\n 'padding' : 'same',\n 'kernel_initializer' : keras.initializers.RandomNormal(mean=0.0, stddev=0.01, seed=None),\n 'bias_initializer' : 'zeros'\n }\n\n if keras.backend.image_data_format() == 'channels_first':\n inputs = keras.layers.Input(shape=(pyramid_feature_size, None, None))\n else:\n inputs = keras.layers.Input(shape=(None, None, pyramid_feature_size))\n outputs = inputs\n for i in range(4):\n outputs = keras.layers.Conv2D(\n filters=regression_feature_size,\n activation='relu',\n name='pyramid_regression_{}'.format(i),\n **options\n )(outputs)\n\n outputs = keras.layers.Conv2D(num_anchors * num_values, name='pyramid_regression', **options)(outputs)\n if keras.backend.image_data_format() == 'channels_first':\n outputs = keras.layers.Permute((2, 3, 1), name='pyramid_regression_permute')(outputs)\n outputs = keras.layers.Reshape((-1, num_values), name='pyramid_regression_reshape')(outputs)\n\n return keras.models.Model(inputs=inputs, outputs=outputs, name=name)\n\n\ndef __create_pyramid_features(backbone_layers, pyramid_levels, feature_size=256):\n \"\"\" Creates the FPN layers on top of the backbone features.\n\n Args\n backbone_layers: a dictionary containing feature stages C3, C4, C5 from the backbone. Also contains C2 if provided.\n pyramid_levels: Pyramid levels in use.\n feature_size : The feature size to use for the resulting feature levels.\n\n Returns\n output_layers : A dict of feature levels. P3, P4, P5, P6 are always included. P2, P6, P7 included if in use.\n \"\"\"\n\n output_layers = {}\n\n # upsample C5 to get P5 from the FPN paper\n P5 = keras.layers.Conv2D(feature_size, kernel_size=1, strides=1, padding='same', name='C5_reduced')(backbone_layers['C5'])\n P5_upsampled = layers.UpsampleLike(name='P5_upsampled')([P5, backbone_layers['C4']])\n P5 = keras.layers.Conv2D(feature_size, kernel_size=3, strides=1, padding='same', name='P5')(P5)\n output_layers[\"P5\"] = P5\n\n # add P5 elementwise to C4\n P4 = keras.layers.Conv2D(feature_size, kernel_size=1, strides=1, padding='same', name='C4_reduced')(backbone_layers['C4'])\n P4 = keras.layers.Add(name='P4_merged')([P5_upsampled, P4])\n P4_upsampled = layers.UpsampleLike(name='P4_upsampled')([P4, backbone_layers['C3']])\n P4 = keras.layers.Conv2D(feature_size, kernel_size=3, strides=1, padding='same', name='P4')(P4)\n output_layers[\"P4\"] = P4\n\n # add P4 elementwise to C3\n P3 = keras.layers.Conv2D(feature_size, kernel_size=1, strides=1, padding='same', name='C3_reduced')(backbone_layers['C3'])\n P3 = keras.layers.Add(name='P3_merged')([P4_upsampled, P3])\n if 'C2' in backbone_layers and 2 in pyramid_levels:\n P3_upsampled = layers.UpsampleLike(name='P3_upsampled')([P3, backbone_layers['C2']])\n P3 = keras.layers.Conv2D(feature_size, kernel_size=3, strides=1, padding='same', name='P3')(P3)\n output_layers[\"P3\"] = P3\n\n if 'C2' in backbone_layers and 2 in pyramid_levels:\n P2 = keras.layers.Conv2D(feature_size, kernel_size=1, strides=1, padding='same', name='C2_reduced')(backbone_layers['C2'])\n P2 = keras.layers.Add(name='P2_merged')([P3_upsampled, P2])\n P2 = keras.layers.Conv2D(feature_size, kernel_size=3, strides=1, padding='same', name='P2')(P2)\n output_layers[\"P2\"] = P2\n\n # \"P6 is obtained via a 3x3 stride-2 conv on C5\"\n if 6 in pyramid_levels:\n P6 = keras.layers.Conv2D(feature_size, kernel_size=3, strides=2, padding='same', name='P6')(backbone_layers['C5'])\n output_layers[\"P6\"] = P6\n\n # \"P7 is computed by applying ReLU followed by a 3x3 stride-2 conv on P6\"\n if 7 in pyramid_levels:\n if 6 not in pyramid_levels:\n raise ValueError(\"P6 is required to use P7\")\n P7 = keras.layers.Activation('relu', name='C6_relu')(P6)\n P7 = keras.layers.Conv2D(feature_size, kernel_size=3, strides=2, padding='same', name='P7')(P7)\n output_layers[\"P7\"] = P7\n\n return output_layers\n\n\ndef default_submodels(num_classes, num_anchors):\n \"\"\" Create a list of default submodels used for object detection.\n\n The default submodels contains a regression submodel and a classification submodel.\n\n Args\n num_classes : Number of classes to use.\n num_anchors : Number of base anchors.\n\n Returns\n A list of tuple, where the first element is the name of the submodel and the second element is the submodel itself.\n \"\"\"\n return [\n ('regression', default_regression_model(4, num_anchors)),\n ('classification', default_classification_model(num_classes, num_anchors))\n ]\n\n\ndef __build_model_pyramid(name, model, features):\n \"\"\" Applies a single submodel to each FPN level.\n\n Args\n name : Name of the submodel.\n model : The submodel to evaluate.\n features : The FPN features.\n\n Returns\n A tensor containing the response from the submodel on the FPN features.\n \"\"\"\n return keras.layers.Concatenate(axis=1, name=name)([model(f) for f in features])\n\n\ndef __build_pyramid(models, features):\n \"\"\" Applies all submodels to each FPN level.\n\n Args\n models : List of submodels to run on each pyramid level (by default only regression, classifcation).\n features : The FPN features.\n\n Returns\n A list of tensors, one for each submodel.\n \"\"\"\n return [__build_model_pyramid(n, m, features) for n, m in models]\n\n\ndef __build_anchors(anchor_parameters, features):\n \"\"\" Builds anchors for the shape of the features from FPN.\n\n Args\n anchor_parameters : Parameteres that determine how anchors are generated.\n features : The FPN features.\n\n Returns\n A tensor containing the anchors for the FPN features.\n\n The shape is:\n ```\n (batch_size, num_anchors, 4)\n ```\n \"\"\"\n anchors = [\n layers.Anchors(\n size=anchor_parameters.sizes[i],\n stride=anchor_parameters.strides[i],\n ratios=anchor_parameters.ratios,\n scales=anchor_parameters.scales,\n name='anchors_{}'.format(i)\n )(f) for i, f in enumerate(features)\n ]\n\n return keras.layers.Concatenate(axis=1, name='anchors')(anchors)\n\n\ndef retinanet(\n inputs,\n backbone_layers,\n num_classes,\n num_anchors = None,\n create_pyramid_features = __create_pyramid_features,\n pyramid_levels = None,\n submodels = None,\n name = 'retinanet'\n):\n \"\"\" Construct a RetinaNet model on top of a backbone.\n\n This model is the minimum model necessary for training (with the unfortunate exception of anchors as output).\n\n Args\n inputs : keras.layers.Input (or list of) for the input to the model.\n num_classes : Number of classes to classify.\n num_anchors : Number of base anchors.\n create_pyramid_features : Functor for creating pyramid features given the features C3, C4, C5, and possibly C2 from the backbone.\n pyramid_levels : pyramid levels to use.\n submodels : Submodels to run on each feature map (default is regression and classification submodels).\n name : Name of the model.\n\n Returns\n A keras.models.Model which takes an image as input and outputs generated anchors and the result from each submodel on every pyramid level.\n\n The order of the outputs is as defined in submodels:\n ```\n [\n regression, classification, other[0], other[1], ...\n ]\n ```\n \"\"\"\n\n if num_anchors is None:\n num_anchors = AnchorParameters.default.num_anchors()\n\n if submodels is None:\n submodels = default_submodels(num_classes, num_anchors)\n\n if pyramid_levels is None:\n pyramid_levels = [3, 4, 5, 6, 7]\n\n if 2 in pyramid_levels and 'C2' not in backbone_layers:\n raise ValueError(\"C2 not provided by backbone model. Cannot create P2 layers.\")\n\n if 3 not in pyramid_levels or 4 not in pyramid_levels or 5 not in pyramid_levels:\n raise ValueError(\"pyramid levels 3, 4, and 5 required for functionality\")\n\n # compute pyramid features as per https://arxiv.org/abs/1708.02002\n features = create_pyramid_features(backbone_layers, pyramid_levels)\n feature_list = [features['P{}'.format(p)] for p in pyramid_levels]\n\n # for all pyramid levels, run available submodels\n pyramids = __build_pyramid(submodels, feature_list)\n\n return keras.models.Model(inputs=inputs, outputs=pyramids, name=name)\n\n\ndef retinanet_bbox(\n model = None,\n nms = True,\n class_specific_filter = True,\n name = 'retinanet-bbox',\n anchor_params = None,\n pyramid_levels = None,\n nms_threshold = 0.5,\n score_threshold = 0.05,\n max_detections = 300,\n parallel_iterations = 32,\n **kwargs\n):\n \"\"\" Construct a RetinaNet model on top of a backbone and adds convenience functions to output boxes directly.\n\n This model uses the minimum retinanet model and appends a few layers to compute boxes within the graph.\n These layers include applying the regression values to the anchors and performing NMS.\n\n Args\n model : RetinaNet model to append bbox layers to. If None, it will create a RetinaNet model using **kwargs.\n nms : Whether to use non-maximum suppression for the filtering step.\n class_specific_filter : Whether to use class specific filtering or filter for the best scoring class only.\n name : Name of the model.\n anchor_params : Struct containing anchor parameters. If None, default values are used.\n pyramid_levels : pyramid levels to use.\n nms_threshold : Threshold for the IoU value to determine when a box should be suppressed.\n score_threshold : Threshold used to prefilter the boxes with.\n max_detections : Maximum number of detections to keep.\n parallel_iterations : Number of batch items to process in parallel.\n **kwargs : Additional kwargs to pass to the minimal retinanet model.\n\n Returns\n A keras.models.Model which takes an image as input and outputs the detections on the image.\n\n The order is defined as follows:\n ```\n [\n boxes, scores, labels, other[0], other[1], ...\n ]\n ```\n \"\"\"\n\n # if no anchor parameters are passed, use default values\n if anchor_params is None:\n anchor_params = AnchorParameters.default\n\n # create RetinaNet model\n if model is None:\n model = retinanet(num_anchors=anchor_params.num_anchors(), **kwargs)\n else:\n assert_training_model(model)\n\n if pyramid_levels is None:\n pyramid_levels = [3, 4, 5, 6, 7]\n\n assert len(pyramid_levels) == len(anchor_params.sizes), \\\n \"number of pyramid levels {} should match number of anchor parameter sizes {}\".format(len(pyramid_levels),\n len(anchor_params.sizes))\n\n pyramid_layer_names = ['P{}'.format(p) for p in pyramid_levels]\n # compute the anchors\n features = [model.get_layer(p_name).output for p_name in pyramid_layer_names]\n anchors = __build_anchors(anchor_params, features)\n\n # we expect the anchors, regression and classification values as first output\n regression = model.outputs[0]\n classification = model.outputs[1]\n\n # \"other\" can be any additional output from custom submodels, by default this will be []\n other = model.outputs[2:]\n\n # apply predicted regression to anchors\n boxes = layers.RegressBoxes(name='boxes')([anchors, regression])\n boxes = layers.ClipBoxes(name='clipped_boxes')([model.inputs[0], boxes])\n\n # filter detections (apply NMS / score threshold / select top-k)\n detections = layers.FilterDetections(\n nms = nms,\n class_specific_filter = class_specific_filter,\n name = 'filtered_detections',\n nms_threshold = nms_threshold,\n score_threshold = score_threshold,\n max_detections = max_detections,\n parallel_iterations = parallel_iterations\n )([boxes, classification] + other)\n\n # construct the model\n return keras.models.Model(inputs=model.inputs, outputs=detections, name=name)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/models/senet.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom tensorflow import keras\n\nfrom . import retinanet\nfrom . import Backbone\nfrom classification_models.keras import Classifiers\n\n\nclass SeBackbone(Backbone):\n \"\"\" Describes backbone information and provides utility functions.\n \"\"\"\n\n def __init__(self, backbone):\n super(SeBackbone, self).__init__(backbone)\n _, self.preprocess_image_func = Classifiers.get(self.backbone)\n\n def retinanet(self, *args, **kwargs):\n \"\"\" Returns a retinanet model using the correct backbone.\n \"\"\"\n return senet_retinanet(*args, backbone=self.backbone, **kwargs)\n\n def download_imagenet(self):\n \"\"\" Downloads ImageNet weights and returns path to weights file.\n \"\"\"\n from classification_models.weights import WEIGHTS_COLLECTION\n\n weights_path = None\n for el in WEIGHTS_COLLECTION:\n if el['model'] == self.backbone and not el['include_top']:\n weights_path = keras.utils.get_file(el['name'], el['url'], cache_subdir='models', file_hash=el['md5'])\n\n if weights_path is None:\n raise ValueError('Unable to find imagenet weights for backbone {}!'.format(self.backbone))\n\n return weights_path\n\n def validate(self):\n \"\"\" Checks whether the backbone string is correct.\n \"\"\"\n allowed_backbones = ['seresnet18', 'seresnet34', 'seresnet50', 'seresnet101', 'seresnet152',\n 'seresnext50', 'seresnext101', 'senet154']\n backbone = self.backbone.split('_')[0]\n\n if backbone not in allowed_backbones:\n raise ValueError('Backbone (\\'{}\\') not in allowed backbones ({}).'.format(backbone, allowed_backbones))\n\n def preprocess_image(self, inputs):\n \"\"\" Takes as input an image and prepares it for being passed through the network.\n \"\"\"\n return self.preprocess_image_func(inputs)\n\n\ndef senet_retinanet(num_classes, backbone='seresnext50', inputs=None, modifier=None, **kwargs):\n \"\"\" Constructs a retinanet model using a resnet backbone.\n\n Args\n num_classes: Number of classes to predict.\n backbone: Which backbone to use (one of ('resnet50', 'resnet101', 'resnet152')).\n inputs: The inputs to the network (defaults to a Tensor of shape (None, None, 3)).\n modifier: A function handler which can modify the backbone before using it in retinanet (this can be used to freeze backbone layers for example).\n\n Returns\n RetinaNet model with a ResNet backbone.\n \"\"\"\n # choose default input\n if inputs is None:\n if keras.backend.image_data_format() == 'channels_first':\n inputs = keras.layers.Input(shape=(3, None, None))\n else:\n # inputs = keras.layers.Input(shape=(224, 224, 3))\n inputs = keras.layers.Input(shape=(None, None, 3))\n\n classifier, _ = Classifiers.get(backbone)\n model = classifier(input_tensor=inputs, include_top=False, weights=None)\n\n # get last conv layer from the end of each block [28x28, 14x14, 7x7]\n if backbone == 'seresnet18' or backbone == 'seresnet34':\n layer_outputs = ['stage3_unit1_relu1', 'stage4_unit1_relu1', 'relu1']\n elif backbone == 'seresnet50':\n layer_outputs = ['activation_36', 'activation_66', 'activation_81']\n elif backbone == 'seresnet101':\n layer_outputs = ['activation_36', 'activation_151', 'activation_166']\n elif backbone == 'seresnet152':\n layer_outputs = ['activation_56', 'activation_236', 'activation_251']\n elif backbone == 'seresnext50':\n layer_outputs = ['activation_37', 'activation_67', 'activation_81']\n elif backbone == 'seresnext101':\n layer_outputs = ['activation_37', 'activation_152', 'activation_166']\n elif backbone == 'senet154':\n layer_outputs = ['activation_59', 'activation_239', 'activation_253']\n else:\n raise ValueError('Backbone (\\'{}\\') is invalid.'.format(backbone))\n\n layer_outputs = [\n model.get_layer(name=layer_outputs[0]).output, # 28x28\n model.get_layer(name=layer_outputs[1]).output, # 14x14\n model.get_layer(name=layer_outputs[2]).output, # 7x7\n ]\n # create the densenet backbone\n model = keras.models.Model(inputs=inputs, outputs=layer_outputs, name=model.name)\n\n # invoke modifier if given\n if modifier:\n model = modifier(model)\n\n # C2 not provided\n backbone_layers = {\n 'C3': model.outputs[0],\n 'C4': model.outputs[1],\n 'C5': model.outputs[2]\n }\n\n # create the full model\n return retinanet.retinanet(inputs=inputs, num_classes=num_classes, backbone_layers=backbone_layers, **kwargs)\n\n\ndef seresnet18_retinanet(num_classes, inputs=None, **kwargs):\n return senet_retinanet(num_classes=num_classes, backbone='seresnet18', inputs=inputs, **kwargs)\n\n\ndef seresnet34_retinanet(num_classes, inputs=None, **kwargs):\n return senet_retinanet(num_classes=num_classes, backbone='seresnet34', inputs=inputs, **kwargs)\n\n\ndef seresnet50_retinanet(num_classes, inputs=None, **kwargs):\n return senet_retinanet(num_classes=num_classes, backbone='seresnet50', inputs=inputs, **kwargs)\n\n\ndef seresnet101_retinanet(num_classes, inputs=None, **kwargs):\n return senet_retinanet(num_classes=num_classes, backbone='seresnet101', inputs=inputs, **kwargs)\n\n\ndef seresnet152_retinanet(num_classes, inputs=None, **kwargs):\n return senet_retinanet(num_classes=num_classes, backbone='seresnet152', inputs=inputs, **kwargs)\n\n\ndef seresnext50_retinanet(num_classes, inputs=None, **kwargs):\n return senet_retinanet(num_classes=num_classes, backbone='seresnext50', inputs=inputs, **kwargs)\n\n\ndef seresnext101_retinanet(num_classes, inputs=None, **kwargs):\n return senet_retinanet(num_classes=num_classes, backbone='seresnext101', inputs=inputs, **kwargs)\n\n\ndef senet154_retinanet(num_classes, inputs=None, **kwargs):\n return senet_retinanet(num_classes=num_classes, backbone='senet154', inputs=inputs, **kwargs)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/models/vgg.py", "content": "\"\"\"\nCopyright 2017-2018 cgratie (https://github.com/cgratie/)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\n\nfrom tensorflow import keras\n\nfrom . import retinanet\nfrom . import Backbone\nfrom ..utils.image import preprocess_image\n\n\nclass VGGBackbone(Backbone):\n \"\"\" Describes backbone information and provides utility functions.\n \"\"\"\n\n def retinanet(self, *args, **kwargs):\n \"\"\" Returns a retinanet model using the correct backbone.\n \"\"\"\n return vgg_retinanet(*args, backbone=self.backbone, **kwargs)\n\n def download_imagenet(self):\n \"\"\" Downloads ImageNet weights and returns path to weights file.\n Weights can be downloaded at https://github.com/fizyr/keras-models/releases .\n \"\"\"\n if self.backbone == 'vgg16':\n resource = keras.applications.vgg16.vgg16.WEIGHTS_PATH_NO_TOP\n checksum = '6d6bbae143d832006294945121d1f1fc'\n elif self.backbone == 'vgg19':\n resource = keras.applications.vgg19.vgg19.WEIGHTS_PATH_NO_TOP\n checksum = '253f8cb515780f3b799900260a226db6'\n else:\n raise ValueError(\"Backbone '{}' not recognized.\".format(self.backbone))\n\n return keras.utils.get_file(\n '{}_weights_tf_dim_ordering_tf_kernels_notop.h5'.format(self.backbone),\n resource,\n cache_subdir='models',\n file_hash=checksum\n )\n\n def validate(self):\n \"\"\" Checks whether the backbone string is correct.\n \"\"\"\n allowed_backbones = ['vgg16', 'vgg19']\n\n if self.backbone not in allowed_backbones:\n raise ValueError('Backbone (\\'{}\\') not in allowed backbones ({}).'.format(self.backbone, allowed_backbones))\n\n def preprocess_image(self, inputs):\n \"\"\" Takes as input an image and prepares it for being passed through the network.\n \"\"\"\n return preprocess_image(inputs, mode='caffe')\n\n\ndef vgg_retinanet(num_classes, backbone='vgg16', inputs=None, modifier=None, **kwargs):\n \"\"\" Constructs a retinanet model using a vgg backbone.\n\n Args\n num_classes: Number of classes to predict.\n backbone: Which backbone to use (one of ('vgg16', 'vgg19')).\n inputs: The inputs to the network (defaults to a Tensor of shape (None, None, 3)).\n modifier: A function handler which can modify the backbone before using it in retinanet (this can be used to freeze backbone layers for example).\n\n Returns\n RetinaNet model with a VGG backbone.\n \"\"\"\n # choose default input\n if inputs is None:\n inputs = keras.layers.Input(shape=(None, None, 3))\n\n # create the vgg backbone\n if backbone == 'vgg16':\n vgg = keras.applications.VGG16(input_tensor=inputs, include_top=False, weights=None)\n elif backbone == 'vgg19':\n vgg = keras.applications.VGG19(input_tensor=inputs, include_top=False, weights=None)\n else:\n raise ValueError(\"Backbone '{}' not recognized.\".format(backbone))\n\n if modifier:\n vgg = modifier(vgg)\n\n # create the full model\n layer_names = [\"block3_pool\", \"block4_pool\", \"block5_pool\"]\n layer_outputs = [vgg.get_layer(name).output for name in layer_names]\n\n # C2 not provided\n backbone_layers = {\n 'C3': layer_outputs[0],\n 'C4': layer_outputs[1],\n 'C5': layer_outputs[2]\n }\n\n return retinanet.retinanet(inputs=inputs, num_classes=num_classes, backbone_layers=backbone_layers, **kwargs)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/preprocessing/__init__.py", "content": "" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/preprocessing/coco.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom ..preprocessing.generator import Generator\nfrom ..utils.image import read_image_bgr\n\nimport os\nimport numpy as np\n\nfrom pycocotools.coco import COCO\n\n\nclass CocoGenerator(Generator):\n \"\"\" Generate data from the COCO dataset.\n\n See https://github.com/cocodataset/cocoapi/tree/master/PythonAPI for more information.\n \"\"\"\n\n def __init__(self, data_dir, set_name, **kwargs):\n \"\"\" Initialize a COCO data generator.\n\n Args\n data_dir: Path to where the COCO dataset is stored.\n set_name: Name of the set to parse.\n \"\"\"\n self.data_dir = data_dir\n self.set_name = set_name\n self.coco = COCO(os.path.join(data_dir, 'annotations', 'instances_' + set_name + '.json'))\n self.image_ids = self.coco.getImgIds()\n\n self.load_classes()\n\n super(CocoGenerator, self).__init__(**kwargs)\n\n def load_classes(self):\n \"\"\" Loads the class to label mapping (and inverse) for COCO.\n \"\"\"\n # load class names (name -> label)\n categories = self.coco.loadCats(self.coco.getCatIds())\n categories.sort(key=lambda x: x['id'])\n\n self.classes = {}\n self.coco_labels = {}\n self.coco_labels_inverse = {}\n for c in categories:\n self.coco_labels[len(self.classes)] = c['id']\n self.coco_labels_inverse[c['id']] = len(self.classes)\n self.classes[c['name']] = len(self.classes)\n\n # also load the reverse (label -> name)\n self.labels = {}\n for key, value in self.classes.items():\n self.labels[value] = key\n\n def size(self):\n \"\"\" Size of the COCO dataset.\n \"\"\"\n return len(self.image_ids)\n\n def num_classes(self):\n \"\"\" Number of classes in the dataset. For COCO this is 80.\n \"\"\"\n return len(self.classes)\n\n def has_label(self, label):\n \"\"\" Return True if label is a known label.\n \"\"\"\n return label in self.labels\n\n def has_name(self, name):\n \"\"\" Returns True if name is a known class.\n \"\"\"\n return name in self.classes\n\n def name_to_label(self, name):\n \"\"\" Map name to label.\n \"\"\"\n return self.classes[name]\n\n def label_to_name(self, label):\n \"\"\" Map label to name.\n \"\"\"\n return self.labels[label]\n\n def coco_label_to_label(self, coco_label):\n \"\"\" Map COCO label to the label as used in the network.\n COCO has some gaps in the order of labels. The highest label is 90, but there are 80 classes.\n \"\"\"\n return self.coco_labels_inverse[coco_label]\n\n def coco_label_to_name(self, coco_label):\n \"\"\" Map COCO label to name.\n \"\"\"\n return self.label_to_name(self.coco_label_to_label(coco_label))\n\n def label_to_coco_label(self, label):\n \"\"\" Map label as used by the network to labels as used by COCO.\n \"\"\"\n return self.coco_labels[label]\n\n def image_path(self, image_index):\n \"\"\" Returns the image path for image_index.\n \"\"\"\n image_info = self.coco.loadImgs(self.image_ids[image_index])[0]\n path = os.path.join(self.data_dir, 'images', self.set_name, image_info['file_name'])\n return path\n\n def image_aspect_ratio(self, image_index):\n \"\"\" Compute the aspect ratio for an image with image_index.\n \"\"\"\n image = self.coco.loadImgs(self.image_ids[image_index])[0]\n return float(image['width']) / float(image['height'])\n\n def load_image(self, image_index):\n \"\"\" Load an image at the image_index.\n \"\"\"\n path = self.image_path(image_index)\n return read_image_bgr(path)\n\n def load_annotations(self, image_index):\n \"\"\" Load annotations for an image_index.\n \"\"\"\n # get ground truth annotations\n annotations_ids = self.coco.getAnnIds(imgIds=self.image_ids[image_index], iscrowd=False)\n annotations = {'labels': np.empty((0,)), 'bboxes': np.empty((0, 4))}\n\n # some images appear to miss annotations (like image with id 257034)\n if len(annotations_ids) == 0:\n return annotations\n\n # parse annotations\n coco_annotations = self.coco.loadAnns(annotations_ids)\n for idx, a in enumerate(coco_annotations):\n # some annotations have basically no width / height, skip them\n if a['bbox'][2] < 1 or a['bbox'][3] < 1:\n continue\n\n annotations['labels'] = np.concatenate([annotations['labels'], [self.coco_label_to_label(a['category_id'])]], axis=0)\n annotations['bboxes'] = np.concatenate([annotations['bboxes'], [[\n a['bbox'][0],\n a['bbox'][1],\n a['bbox'][0] + a['bbox'][2],\n a['bbox'][1] + a['bbox'][3],\n ]]], axis=0)\n\n return annotations\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/preprocessing/csv_generator.py", "content": "\"\"\"\nCopyright 2017-2018 yhenon (https://github.com/yhenon/)\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom .generator import Generator\nfrom ..utils.image import read_image_bgr\n\nimport numpy as np\nfrom PIL import Image\nfrom six import raise_from\n\nimport csv\nimport sys\nimport os.path\nfrom collections import OrderedDict\n\n\ndef _parse(value, function, fmt):\n \"\"\"\n Parse a string into a value, and format a nice ValueError if it fails.\n\n Returns `function(value)`.\n Any `ValueError` raised is catched and a new `ValueError` is raised\n with message `fmt.format(e)`, where `e` is the caught `ValueError`.\n \"\"\"\n try:\n return function(value)\n except ValueError as e:\n raise_from(ValueError(fmt.format(e)), None)\n\n\ndef _read_classes(csv_reader):\n \"\"\" Parse the classes file given by csv_reader.\n \"\"\"\n result = OrderedDict()\n for line, row in enumerate(csv_reader):\n line += 1\n\n try:\n class_name, class_id = row\n except ValueError:\n raise_from(ValueError('line {}: format should be \\'class_name,class_id\\''.format(line)), None)\n class_id = _parse(class_id, int, 'line {}: malformed class ID: {{}}'.format(line))\n\n if class_name in result:\n raise ValueError('line {}: duplicate class name: \\'{}\\''.format(line, class_name))\n result[class_name] = class_id\n return result\n\n\ndef _read_annotations(csv_reader, classes):\n \"\"\" Read annotations from the csv_reader.\n \"\"\"\n result = OrderedDict()\n for line, row in enumerate(csv_reader):\n line += 1\n\n try:\n img_file, x1, y1, x2, y2, class_name = row[:6]\n except ValueError:\n raise_from(ValueError('line {}: format should be \\'img_file,x1,y1,x2,y2,class_name\\' or \\'img_file,,,,,\\''.format(line)), None)\n\n if img_file not in result:\n result[img_file] = []\n\n # If a row contains only an image path, it's an image without annotations.\n if (x1, y1, x2, y2, class_name) == ('', '', '', '', ''):\n continue\n\n x1 = _parse(x1, int, 'line {}: malformed x1: {{}}'.format(line))\n y1 = _parse(y1, int, 'line {}: malformed y1: {{}}'.format(line))\n x2 = _parse(x2, int, 'line {}: malformed x2: {{}}'.format(line))\n y2 = _parse(y2, int, 'line {}: malformed y2: {{}}'.format(line))\n\n # Check that the bounding box is valid.\n if x2 <= x1:\n raise ValueError('line {}: x2 ({}) must be higher than x1 ({})'.format(line, x2, x1))\n if y2 <= y1:\n raise ValueError('line {}: y2 ({}) must be higher than y1 ({})'.format(line, y2, y1))\n\n # check if the current class name is correctly present\n if class_name not in classes:\n raise ValueError('line {}: unknown class name: \\'{}\\' (classes: {})'.format(line, class_name, classes))\n\n result[img_file].append({'x1': x1, 'x2': x2, 'y1': y1, 'y2': y2, 'class': class_name})\n return result\n\n\ndef _open_for_csv(path):\n \"\"\" Open a file with flags suitable for csv.reader.\n\n This is different for python2 it means with mode 'rb',\n for python3 this means 'r' with \"universal newlines\".\n \"\"\"\n if sys.version_info[0] < 3:\n return open(path, 'rb')\n else:\n return open(path, 'r', newline='')\n\n\nclass CSVGenerator(Generator):\n \"\"\" Generate data for a custom CSV dataset.\n\n See https://github.com/fizyr/keras-retinanet#csv-datasets for more information.\n \"\"\"\n\n def __init__(\n self,\n csv_data_file,\n csv_class_file,\n base_dir=None,\n **kwargs\n ):\n \"\"\" Initialize a CSV data generator.\n\n Args\n csv_data_file: Path to the CSV annotations file.\n csv_class_file: Path to the CSV classes file.\n base_dir: Directory w.r.t. where the files are to be searched (defaults to the directory containing the csv_data_file).\n \"\"\"\n self.image_names = []\n self.image_data = {}\n self.base_dir = base_dir\n\n # Take base_dir from annotations file if not explicitly specified.\n if self.base_dir is None:\n self.base_dir = os.path.dirname(csv_data_file)\n\n # parse the provided class file\n try:\n with _open_for_csv(csv_class_file) as file:\n self.classes = _read_classes(csv.reader(file, delimiter=','))\n except ValueError as e:\n raise_from(ValueError('invalid CSV class file: {}: {}'.format(csv_class_file, e)), None)\n\n self.labels = {}\n for key, value in self.classes.items():\n self.labels[value] = key\n\n # csv with img_path, x1, y1, x2, y2, class_name\n try:\n with _open_for_csv(csv_data_file) as file:\n self.image_data = _read_annotations(csv.reader(file, delimiter=','), self.classes)\n except ValueError as e:\n raise_from(ValueError('invalid CSV annotations file: {}: {}'.format(csv_data_file, e)), None)\n self.image_names = list(self.image_data.keys())\n\n super(CSVGenerator, self).__init__(**kwargs)\n\n def size(self):\n \"\"\" Size of the dataset.\n \"\"\"\n return len(self.image_names)\n\n def num_classes(self):\n \"\"\" Number of classes in the dataset.\n \"\"\"\n return max(self.classes.values()) + 1\n\n def has_label(self, label):\n \"\"\" Return True if label is a known label.\n \"\"\"\n return label in self.labels\n\n def has_name(self, name):\n \"\"\" Returns True if name is a known class.\n \"\"\"\n return name in self.classes\n\n def name_to_label(self, name):\n \"\"\" Map name to label.\n \"\"\"\n return self.classes[name]\n\n def label_to_name(self, label):\n \"\"\" Map label to name.\n \"\"\"\n return self.labels[label]\n\n def image_path(self, image_index):\n \"\"\" Returns the image path for image_index.\n \"\"\"\n return os.path.join(self.base_dir, self.image_names[image_index])\n\n def image_aspect_ratio(self, image_index):\n \"\"\" Compute the aspect ratio for an image with image_index.\n \"\"\"\n # PIL is fast for metadata\n image = Image.open(self.image_path(image_index))\n return float(image.width) / float(image.height)\n\n def load_image(self, image_index):\n \"\"\" Load an image at the image_index.\n \"\"\"\n return read_image_bgr(self.image_path(image_index))\n\n def load_annotations(self, image_index):\n \"\"\" Load annotations for an image_index.\n \"\"\"\n path = self.image_names[image_index]\n annotations = {'labels': np.empty((0,)), 'bboxes': np.empty((0, 4))}\n\n for idx, annot in enumerate(self.image_data[path]):\n annotations['labels'] = np.concatenate((annotations['labels'], [self.name_to_label(annot['class'])]))\n annotations['bboxes'] = np.concatenate((annotations['bboxes'], [[\n float(annot['x1']),\n float(annot['y1']),\n float(annot['x2']),\n float(annot['y2']),\n ]]))\n\n return annotations\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/preprocessing/generator.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport numpy as np\nimport random\nimport warnings\n\nfrom tensorflow import keras\n\nfrom ..utils.anchors import (\n anchor_targets_bbox,\n anchors_for_shape,\n guess_shapes\n)\nfrom ..utils.config import parse_anchor_parameters, parse_pyramid_levels\nfrom ..utils.image import (\n TransformParameters,\n adjust_transform_for_image,\n apply_transform,\n preprocess_image,\n resize_image,\n)\nfrom ..utils.transform import transform_aabb\n\n\nclass Generator(keras.utils.Sequence):\n \"\"\" Abstract generator class.\n \"\"\"\n\n def __init__(\n self,\n transform_generator = None,\n visual_effect_generator=None,\n batch_size=1,\n group_method='ratio', # one of 'none', 'random', 'ratio'\n shuffle_groups=True,\n image_min_side=800,\n image_max_side=1333,\n no_resize=False,\n transform_parameters=None,\n compute_anchor_targets=anchor_targets_bbox,\n compute_shapes=guess_shapes,\n preprocess_image=preprocess_image,\n config=None\n ):\n \"\"\" Initialize Generator object.\n\n Args\n transform_generator : A generator used to randomly transform images and annotations.\n batch_size : The size of the batches to generate.\n group_method : Determines how images are grouped together (defaults to 'ratio', one of ('none', 'random', 'ratio')).\n shuffle_groups : If True, shuffles the groups each epoch.\n image_min_side : After resizing the minimum side of an image is equal to image_min_side.\n image_max_side : If after resizing the maximum side is larger than image_max_side, scales down further so that the max side is equal to image_max_side.\n no_resize : If True, no image/annotation resizing is performed.\n transform_parameters : The transform parameters used for data augmentation.\n compute_anchor_targets : Function handler for computing the targets of anchors for an image and its annotations.\n compute_shapes : Function handler for computing the shapes of the pyramid for a given input.\n preprocess_image : Function handler for preprocessing an image (scaling / normalizing) for passing through a network.\n \"\"\"\n self.transform_generator = transform_generator\n self.visual_effect_generator = visual_effect_generator\n self.batch_size = int(batch_size)\n self.group_method = group_method\n self.shuffle_groups = shuffle_groups\n self.image_min_side = image_min_side\n self.image_max_side = image_max_side\n self.no_resize = no_resize\n self.transform_parameters = transform_parameters or TransformParameters()\n self.compute_anchor_targets = compute_anchor_targets\n self.compute_shapes = compute_shapes\n self.preprocess_image = preprocess_image\n self.config = config\n\n # Define groups\n self.group_images()\n\n # Shuffle when initializing\n if self.shuffle_groups:\n self.on_epoch_end()\n\n def on_epoch_end(self):\n if self.shuffle_groups:\n random.shuffle(self.groups)\n\n def size(self):\n \"\"\" Size of the dataset.\n \"\"\"\n raise NotImplementedError('size method not implemented')\n\n def num_classes(self):\n \"\"\" Number of classes in the dataset.\n \"\"\"\n raise NotImplementedError('num_classes method not implemented')\n\n def has_label(self, label):\n \"\"\" Returns True if label is a known label.\n \"\"\"\n raise NotImplementedError('has_label method not implemented')\n\n def has_name(self, name):\n \"\"\" Returns True if name is a known class.\n \"\"\"\n raise NotImplementedError('has_name method not implemented')\n\n def name_to_label(self, name):\n \"\"\" Map name to label.\n \"\"\"\n raise NotImplementedError('name_to_label method not implemented')\n\n def label_to_name(self, label):\n \"\"\" Map label to name.\n \"\"\"\n raise NotImplementedError('label_to_name method not implemented')\n\n def image_aspect_ratio(self, image_index):\n \"\"\" Compute the aspect ratio for an image with image_index.\n \"\"\"\n raise NotImplementedError('image_aspect_ratio method not implemented')\n\n def image_path(self, image_index):\n \"\"\" Get the path to an image.\n \"\"\"\n raise NotImplementedError('image_path method not implemented')\n\n def load_image(self, image_index):\n \"\"\" Load an image at the image_index.\n \"\"\"\n raise NotImplementedError('load_image method not implemented')\n\n def load_annotations(self, image_index):\n \"\"\" Load annotations for an image_index.\n \"\"\"\n raise NotImplementedError('load_annotations method not implemented')\n\n def load_annotations_group(self, group):\n \"\"\" Load annotations for all images in group.\n \"\"\"\n annotations_group = [self.load_annotations(image_index) for image_index in group]\n for annotations in annotations_group:\n assert(isinstance(annotations, dict)), '\\'load_annotations\\' should return a list of dictionaries, received: {}'.format(type(annotations))\n assert('labels' in annotations), '\\'load_annotations\\' should return a list of dictionaries that contain \\'labels\\' and \\'bboxes\\'.'\n assert('bboxes' in annotations), '\\'load_annotations\\' should return a list of dictionaries that contain \\'labels\\' and \\'bboxes\\'.'\n\n return annotations_group\n\n def filter_annotations(self, image_group, annotations_group, group):\n \"\"\" Filter annotations by removing those that are outside of the image bounds or whose width/height < 0.\n \"\"\"\n # test all annotations\n for index, (image, annotations) in enumerate(zip(image_group, annotations_group)):\n # test x2 < x1 | y2 < y1 | x1 < 0 | y1 < 0 | x2 <= 0 | y2 <= 0 | x2 >= image.shape[1] | y2 >= image.shape[0]\n invalid_indices = np.where(\n (annotations['bboxes'][:, 2] <= annotations['bboxes'][:, 0]) |\n (annotations['bboxes'][:, 3] <= annotations['bboxes'][:, 1]) |\n (annotations['bboxes'][:, 0] < 0) |\n (annotations['bboxes'][:, 1] < 0) |\n (annotations['bboxes'][:, 2] > image.shape[1]) |\n (annotations['bboxes'][:, 3] > image.shape[0])\n )[0]\n\n # delete invalid indices\n if len(invalid_indices):\n warnings.warn('Image {} with id {} (shape {}) contains the following invalid boxes: {}.'.format(\n self.image_path(group[index]),\n group[index],\n image.shape,\n annotations['bboxes'][invalid_indices, :]\n ))\n for k in annotations_group[index].keys():\n annotations_group[index][k] = np.delete(annotations[k], invalid_indices, axis=0)\n return image_group, annotations_group\n\n def load_image_group(self, group):\n \"\"\" Load images for all images in a group.\n \"\"\"\n return [self.load_image(image_index) for image_index in group]\n\n def random_visual_effect_group_entry(self, image, annotations):\n \"\"\" Randomly transforms image and annotation.\n \"\"\"\n visual_effect = next(self.visual_effect_generator)\n # apply visual effect\n image = visual_effect(image)\n return image, annotations\n\n def random_visual_effect_group(self, image_group, annotations_group):\n \"\"\" Randomly apply visual effect on each image.\n \"\"\"\n assert(len(image_group) == len(annotations_group))\n\n if self.visual_effect_generator is None:\n # do nothing\n return image_group, annotations_group\n\n for index in range(len(image_group)):\n # apply effect on a single group entry\n image_group[index], annotations_group[index] = self.random_visual_effect_group_entry(\n image_group[index], annotations_group[index]\n )\n\n return image_group, annotations_group\n\n def random_transform_group_entry(self, image, annotations, transform=None):\n \"\"\" Randomly transforms image and annotation.\n \"\"\"\n # randomly transform both image and annotations\n if transform is not None or self.transform_generator:\n if transform is None:\n transform = adjust_transform_for_image(next(self.transform_generator), image, self.transform_parameters.relative_translation)\n\n # apply transformation to image\n image = apply_transform(transform, image, self.transform_parameters)\n\n # Transform the bounding boxes in the annotations.\n annotations['bboxes'] = annotations['bboxes'].copy()\n for index in range(annotations['bboxes'].shape[0]):\n annotations['bboxes'][index, :] = transform_aabb(transform, annotations['bboxes'][index, :])\n\n return image, annotations\n\n def random_transform_group(self, image_group, annotations_group):\n \"\"\" Randomly transforms each image and its annotations.\n \"\"\"\n\n assert(len(image_group) == len(annotations_group))\n\n for index in range(len(image_group)):\n # transform a single group entry\n image_group[index], annotations_group[index] = self.random_transform_group_entry(image_group[index], annotations_group[index])\n\n return image_group, annotations_group\n\n def resize_image(self, image):\n \"\"\" Resize an image using image_min_side and image_max_side.\n \"\"\"\n if self.no_resize:\n return image, 1\n else:\n return resize_image(image, min_side=self.image_min_side, max_side=self.image_max_side)\n\n def preprocess_group_entry(self, image, annotations):\n \"\"\" Preprocess image and its annotations.\n \"\"\"\n # resize image\n image, image_scale = self.resize_image(image)\n\n # preprocess the image\n image = self.preprocess_image(image)\n\n # apply resizing to annotations too\n annotations['bboxes'] *= image_scale\n\n # convert to the wanted keras floatx\n image = keras.backend.cast_to_floatx(image)\n\n return image, annotations\n\n def preprocess_group(self, image_group, annotations_group):\n \"\"\" Preprocess each image and its annotations in its group.\n \"\"\"\n assert(len(image_group) == len(annotations_group))\n\n for index in range(len(image_group)):\n # preprocess a single group entry\n image_group[index], annotations_group[index] = self.preprocess_group_entry(image_group[index], annotations_group[index])\n\n return image_group, annotations_group\n\n def group_images(self):\n \"\"\" Order the images according to self.order and makes groups of self.batch_size.\n \"\"\"\n # determine the order of the images\n order = list(range(self.size()))\n if self.group_method == 'random':\n random.shuffle(order)\n elif self.group_method == 'ratio':\n order.sort(key=lambda x: self.image_aspect_ratio(x))\n\n # divide into groups, one group = one batch\n self.groups = [[order[x % len(order)] for x in range(i, i + self.batch_size)] for i in range(0, len(order), self.batch_size)]\n\n def compute_inputs(self, image_group):\n \"\"\" Compute inputs for the network using an image_group.\n \"\"\"\n # get the max image shape\n max_shape = tuple(max(image.shape[x] for image in image_group) for x in range(3))\n\n # construct an image batch object\n image_batch = np.zeros((self.batch_size,) + max_shape, dtype=keras.backend.floatx())\n\n # copy all images to the upper left part of the image batch object\n for image_index, image in enumerate(image_group):\n image_batch[image_index, :image.shape[0], :image.shape[1], :image.shape[2]] = image\n\n if keras.backend.image_data_format() == 'channels_first':\n image_batch = image_batch.transpose((0, 3, 1, 2))\n\n return image_batch\n\n def generate_anchors(self, image_shape):\n anchor_params = None\n pyramid_levels = None\n if self.config and 'anchor_parameters' in self.config:\n anchor_params = parse_anchor_parameters(self.config)\n if self.config and 'pyramid_levels' in self.config:\n pyramid_levels = parse_pyramid_levels(self.config)\n\n return anchors_for_shape(image_shape, anchor_params=anchor_params, pyramid_levels=pyramid_levels, shapes_callback=self.compute_shapes)\n\n def compute_targets(self, image_group, annotations_group):\n \"\"\" Compute target outputs for the network using images and their annotations.\n \"\"\"\n # get the max image shape\n max_shape = tuple(max(image.shape[x] for image in image_group) for x in range(3))\n anchors = self.generate_anchors(max_shape)\n\n batches = self.compute_anchor_targets(\n anchors,\n image_group,\n annotations_group,\n self.num_classes()\n )\n\n return list(batches)\n\n def compute_input_output(self, group):\n \"\"\" Compute inputs and target outputs for the network.\n \"\"\"\n # load images and annotations\n image_group = self.load_image_group(group)\n annotations_group = self.load_annotations_group(group)\n\n # check validity of annotations\n image_group, annotations_group = self.filter_annotations(image_group, annotations_group, group)\n\n # randomly apply visual effect\n image_group, annotations_group = self.random_visual_effect_group(image_group, annotations_group)\n\n # randomly transform data\n image_group, annotations_group = self.random_transform_group(image_group, annotations_group)\n\n # perform preprocessing steps\n image_group, annotations_group = self.preprocess_group(image_group, annotations_group)\n\n # compute network inputs\n inputs = self.compute_inputs(image_group)\n\n # compute network targets\n targets = self.compute_targets(image_group, annotations_group)\n\n return inputs, targets\n\n def __len__(self):\n \"\"\"\n Number of batches for generator.\n \"\"\"\n\n return len(self.groups)\n\n def __getitem__(self, index):\n \"\"\"\n Keras sequence method for generating batches.\n \"\"\"\n group = self.groups[index]\n inputs, targets = self.compute_input_output(group)\n\n return inputs, targets\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/preprocessing/kitti.py", "content": "\"\"\"\nCopyright 2017-2018 lvaleriu (https://github.com/lvaleriu/)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport csv\nimport os.path\n\nimport numpy as np\nfrom PIL import Image\n\nfrom .generator import Generator\nfrom ..utils.image import read_image_bgr\n\nkitti_classes = {\n 'Car': 0,\n 'Van': 1,\n 'Truck': 2,\n 'Pedestrian': 3,\n 'Person_sitting': 4,\n 'Cyclist': 5,\n 'Tram': 6,\n 'Misc': 7,\n 'DontCare': 7\n}\n\n\nclass KittiGenerator(Generator):\n \"\"\" Generate data for a KITTI dataset.\n\n See http://www.cvlibs.net/datasets/kitti/ for more information.\n \"\"\"\n\n def __init__(\n self,\n base_dir,\n subset='train',\n **kwargs\n ):\n \"\"\" Initialize a KITTI data generator.\n\n Args\n base_dir: Directory w.r.t. where the files are to be searched (defaults to the directory containing the csv_data_file).\n subset: The subset to generate data for (defaults to 'train').\n \"\"\"\n self.base_dir = base_dir\n\n label_dir = os.path.join(self.base_dir, subset, 'labels')\n image_dir = os.path.join(self.base_dir, subset, 'images')\n\n \"\"\"\n 1 type Describes the type of object: 'Car', 'Van', 'Truck',\n 'Pedestrian', 'Person_sitting', 'Cyclist', 'Tram',\n 'Misc' or 'DontCare'\n 1 truncated Float from 0 (non-truncated) to 1 (truncated), where\n truncated refers to the object leaving image boundaries\n 1 occluded Integer (0,1,2,3) indicating occlusion state:\n 0 = fully visible, 1 = partly occluded\n 2 = largely occluded, 3 = unknown\n 1 alpha Observation angle of object, ranging [-pi..pi]\n 4 bbox 2D bounding box of object in the image (0-based index):\n contains left, top, right, bottom pixel coordinates\n 3 dimensions 3D object dimensions: height, width, length (in meters)\n 3 location 3D object location x,y,z in camera coordinates (in meters)\n 1 rotation_y Rotation ry around Y-axis in camera coordinates [-pi..pi]\n \"\"\"\n\n self.labels = {}\n self.classes = kitti_classes\n for name, label in self.classes.items():\n self.labels[label] = name\n\n self.image_data = dict()\n self.images = []\n for i, fn in enumerate(os.listdir(label_dir)):\n label_fp = os.path.join(label_dir, fn)\n image_fp = os.path.join(image_dir, fn.replace('.txt', '.png'))\n\n self.images.append(image_fp)\n\n fieldnames = ['type', 'truncated', 'occluded', 'alpha', 'left', 'top', 'right', 'bottom', 'dh', 'dw', 'dl',\n 'lx', 'ly', 'lz', 'ry']\n with open(label_fp, 'r') as csv_file:\n reader = csv.DictReader(csv_file, delimiter=' ', fieldnames=fieldnames)\n boxes = []\n for line, row in enumerate(reader):\n label = row['type']\n cls_id = kitti_classes[label]\n\n annotation = {'cls_id': cls_id, 'x1': row['left'], 'x2': row['right'], 'y2': row['bottom'], 'y1': row['top']}\n boxes.append(annotation)\n\n self.image_data[i] = boxes\n\n super(KittiGenerator, self).__init__(**kwargs)\n\n def size(self):\n \"\"\" Size of the dataset.\n \"\"\"\n return len(self.images)\n\n def num_classes(self):\n \"\"\" Number of classes in the dataset.\n \"\"\"\n return max(self.classes.values()) + 1\n\n def has_label(self, label):\n \"\"\" Return True if label is a known label.\n \"\"\"\n return label in self.labels\n\n def has_name(self, name):\n \"\"\" Returns True if name is a known class.\n \"\"\"\n return name in self.classes\n\n def name_to_label(self, name):\n \"\"\" Map name to label.\n \"\"\"\n raise NotImplementedError()\n\n def label_to_name(self, label):\n \"\"\" Map label to name.\n \"\"\"\n return self.labels[label]\n\n def image_aspect_ratio(self, image_index):\n \"\"\" Compute the aspect ratio for an image with image_index.\n \"\"\"\n # PIL is fast for metadata\n image = Image.open(self.images[image_index])\n return float(image.width) / float(image.height)\n\n def image_path(self, image_index):\n \"\"\" Get the path to an image.\n \"\"\"\n return self.images[image_index]\n\n def load_image(self, image_index):\n \"\"\" Load an image at the image_index.\n \"\"\"\n return read_image_bgr(self.image_path(image_index))\n\n def load_annotations(self, image_index):\n \"\"\" Load annotations for an image_index.\n \"\"\"\n image_data = self.image_data[image_index]\n annotations = {'labels': np.empty((len(image_data),)), 'bboxes': np.empty((len(image_data), 4))}\n\n for idx, ann in enumerate(image_data):\n annotations['bboxes'][idx, 0] = float(ann['x1'])\n annotations['bboxes'][idx, 1] = float(ann['y1'])\n annotations['bboxes'][idx, 2] = float(ann['x2'])\n annotations['bboxes'][idx, 3] = float(ann['y2'])\n annotations['labels'][idx] = int(ann['cls_id'])\n\n return annotations\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/preprocessing/open_images.py", "content": "\"\"\"\nCopyright 2017-2018 lvaleriu (https://github.com/lvaleriu/)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport csv\nimport json\nimport os\nimport warnings\n\nimport numpy as np\nfrom PIL import Image\n\nfrom .generator import Generator\nfrom ..utils.image import read_image_bgr\n\n\ndef load_hierarchy(metadata_dir, version='v4'):\n hierarchy = None\n if version == 'challenge2018':\n hierarchy = 'bbox_labels_500_hierarchy.json'\n elif version == 'v4':\n hierarchy = 'bbox_labels_600_hierarchy.json'\n elif version == 'v3':\n hierarchy = 'bbox_labels_600_hierarchy.json'\n\n hierarchy_json = os.path.join(metadata_dir, hierarchy)\n with open(hierarchy_json) as f:\n hierarchy_data = json.loads(f.read())\n\n return hierarchy_data\n\n\ndef load_hierarchy_children(hierarchy):\n res = [hierarchy['LabelName']]\n\n if 'Subcategory' in hierarchy:\n for subcategory in hierarchy['Subcategory']:\n children = load_hierarchy_children(subcategory)\n\n for c in children:\n res.append(c)\n\n return res\n\n\ndef find_hierarchy_parent(hierarchy, parent_cls):\n if hierarchy['LabelName'] == parent_cls:\n return hierarchy\n elif 'Subcategory' in hierarchy:\n for child in hierarchy['Subcategory']:\n res = find_hierarchy_parent(child, parent_cls)\n if res is not None:\n return res\n\n return None\n\n\ndef get_labels(metadata_dir, version='v4'):\n if version == 'v4' or version == 'challenge2018':\n csv_file = 'class-descriptions-boxable.csv' if version == 'v4' else 'challenge-2018-class-descriptions-500.csv'\n\n boxable_classes_descriptions = os.path.join(metadata_dir, csv_file)\n id_to_labels = {}\n cls_index = {}\n\n i = 0\n with open(boxable_classes_descriptions) as f:\n for row in csv.reader(f):\n # make sure the csv row is not empty (usually the last one)\n if len(row):\n label = row[0]\n description = row[1].replace(\"\\\"\", \"\").replace(\"'\", \"\").replace('`', '')\n\n id_to_labels[i] = description\n cls_index[label] = i\n\n i += 1\n else:\n trainable_classes_path = os.path.join(metadata_dir, 'classes-bbox-trainable.txt')\n description_path = os.path.join(metadata_dir, 'class-descriptions.csv')\n\n description_table = {}\n with open(description_path) as f:\n for row in csv.reader(f):\n # make sure the csv row is not empty (usually the last one)\n if len(row):\n description_table[row[0]] = row[1].replace(\"\\\"\", \"\").replace(\"'\", \"\").replace('`', '')\n\n with open(trainable_classes_path, 'rb') as f:\n trainable_classes = f.read().split('\\n')\n\n id_to_labels = dict([(i, description_table[c]) for i, c in enumerate(trainable_classes)])\n cls_index = dict([(c, i) for i, c in enumerate(trainable_classes)])\n\n return id_to_labels, cls_index\n\n\ndef generate_images_annotations_json(main_dir, metadata_dir, subset, cls_index, version='v4'):\n validation_image_ids = {}\n\n if version == 'v4':\n annotations_path = os.path.join(metadata_dir, subset, '{}-annotations-bbox.csv'.format(subset))\n elif version == 'challenge2018':\n validation_image_ids_path = os.path.join(metadata_dir, 'challenge-2018-image-ids-valset-od.csv')\n\n with open(validation_image_ids_path, 'r') as csv_file:\n reader = csv.DictReader(csv_file, fieldnames=['ImageID'])\n next(reader)\n for line, row in enumerate(reader):\n image_id = row['ImageID']\n validation_image_ids[image_id] = True\n\n annotations_path = os.path.join(metadata_dir, 'challenge-2018-train-annotations-bbox.csv')\n else:\n annotations_path = os.path.join(metadata_dir, subset, 'annotations-human-bbox.csv')\n\n fieldnames = ['ImageID', 'Source', 'LabelName', 'Confidence',\n 'XMin', 'XMax', 'YMin', 'YMax',\n 'IsOccluded', 'IsTruncated', 'IsGroupOf', 'IsDepiction', 'IsInside']\n\n id_annotations = dict()\n with open(annotations_path, 'r') as csv_file:\n reader = csv.DictReader(csv_file, fieldnames=fieldnames)\n next(reader)\n\n images_sizes = {}\n for line, row in enumerate(reader):\n frame = row['ImageID']\n\n if version == 'challenge2018':\n if subset == 'train':\n if frame in validation_image_ids:\n continue\n elif subset == 'validation':\n if frame not in validation_image_ids:\n continue\n else:\n raise NotImplementedError('This generator handles only the train and validation subsets')\n\n class_name = row['LabelName']\n\n if class_name not in cls_index:\n continue\n\n cls_id = cls_index[class_name]\n\n if version == 'challenge2018':\n # We recommend participants to use the provided subset of the training set as a validation set.\n # This is preferable over using the V4 val/test sets, as the training set is more densely annotated.\n img_path = os.path.join(main_dir, 'images', 'train', frame + '.jpg')\n else:\n img_path = os.path.join(main_dir, 'images', subset, frame + '.jpg')\n\n if frame in images_sizes:\n width, height = images_sizes[frame]\n else:\n try:\n with Image.open(img_path) as img:\n width, height = img.width, img.height\n images_sizes[frame] = (width, height)\n except Exception as ex:\n if version == 'challenge2018':\n raise ex\n continue\n\n x1 = float(row['XMin'])\n x2 = float(row['XMax'])\n y1 = float(row['YMin'])\n y2 = float(row['YMax'])\n\n x1_int = int(round(x1 * width))\n x2_int = int(round(x2 * width))\n y1_int = int(round(y1 * height))\n y2_int = int(round(y2 * height))\n\n # Check that the bounding box is valid.\n if x2 <= x1:\n raise ValueError('line {}: x2 ({}) must be higher than x1 ({})'.format(line, x2, x1))\n if y2 <= y1:\n raise ValueError('line {}: y2 ({}) must be higher than y1 ({})'.format(line, y2, y1))\n\n if y2_int == y1_int:\n warnings.warn('filtering line {}: rounding y2 ({}) and y1 ({}) makes them equal'.format(line, y2, y1))\n continue\n\n if x2_int == x1_int:\n warnings.warn('filtering line {}: rounding x2 ({}) and x1 ({}) makes them equal'.format(line, x2, x1))\n continue\n\n img_id = row['ImageID']\n annotation = {'cls_id': cls_id, 'x1': x1, 'x2': x2, 'y1': y1, 'y2': y2}\n\n if img_id in id_annotations:\n annotations = id_annotations[img_id]\n annotations['boxes'].append(annotation)\n else:\n id_annotations[img_id] = {'w': width, 'h': height, 'boxes': [annotation]}\n return id_annotations\n\n\nclass OpenImagesGenerator(Generator):\n def __init__(\n self, main_dir, subset, version='v4',\n labels_filter=None, annotation_cache_dir='.',\n parent_label=None,\n **kwargs\n ):\n if version == 'challenge2018':\n metadata = 'challenge2018'\n elif version == 'v4':\n metadata = '2018_04'\n elif version == 'v3':\n metadata = '2017_11'\n else:\n raise NotImplementedError('There is currently no implementation for versions older than v3')\n\n if version == 'challenge2018':\n self.base_dir = os.path.join(main_dir, 'images', 'train')\n else:\n self.base_dir = os.path.join(main_dir, 'images', subset)\n\n metadata_dir = os.path.join(main_dir, metadata)\n annotation_cache_json = os.path.join(annotation_cache_dir, subset + '.json')\n\n self.hierarchy = load_hierarchy(metadata_dir, version=version)\n id_to_labels, cls_index = get_labels(metadata_dir, version=version)\n\n if os.path.exists(annotation_cache_json):\n with open(annotation_cache_json, 'r') as f:\n self.annotations = json.loads(f.read())\n else:\n self.annotations = generate_images_annotations_json(main_dir, metadata_dir, subset, cls_index, version=version)\n json.dump(self.annotations, open(annotation_cache_json, \"w\"))\n\n if labels_filter is not None or parent_label is not None:\n self.id_to_labels, self.annotations = self.__filter_data(id_to_labels, cls_index, labels_filter, parent_label)\n else:\n self.id_to_labels = id_to_labels\n\n self.id_to_image_id = dict([(i, k) for i, k in enumerate(self.annotations)])\n\n super(OpenImagesGenerator, self).__init__(**kwargs)\n\n def __filter_data(self, id_to_labels, cls_index, labels_filter=None, parent_label=None):\n \"\"\"\n If you want to work with a subset of the labels just set a list with trainable labels\n :param labels_filter: Ex: labels_filter = ['Helmet', 'Hat', 'Analog television']\n :param parent_label: If parent_label is set this will bring you the parent label\n but also its children in the semantic hierarchy as defined in OID, ex: Animal\n hierarchical tree\n :return:\n \"\"\"\n\n children_id_to_labels = {}\n\n if parent_label is None:\n # there is/are no other sublabel(s) other than the labels itself\n\n for label in labels_filter:\n for i, lb in id_to_labels.items():\n if lb == label:\n children_id_to_labels[i] = label\n break\n else:\n parent_cls = None\n for i, lb in id_to_labels.items():\n if lb == parent_label:\n parent_id = i\n for c, index in cls_index.items():\n if index == parent_id:\n parent_cls = c\n break\n\n if parent_cls is None:\n raise Exception('Couldnt find label {}'.format(parent_label))\n\n parent_tree = find_hierarchy_parent(self.hierarchy, parent_cls)\n\n if parent_tree is None:\n raise Exception('Couldnt find parent {} in the semantic hierarchical tree'.format(parent_label))\n\n children = load_hierarchy_children(parent_tree)\n\n for cls in children:\n index = cls_index[cls]\n label = id_to_labels[index]\n children_id_to_labels[index] = label\n\n id_map = dict([(ind, i) for i, ind in enumerate(children_id_to_labels.keys())])\n\n filtered_annotations = {}\n for k in self.annotations:\n img_ann = self.annotations[k]\n\n filtered_boxes = []\n for ann in img_ann['boxes']:\n cls_id = ann['cls_id']\n if cls_id in children_id_to_labels:\n ann['cls_id'] = id_map[cls_id]\n filtered_boxes.append(ann)\n\n if len(filtered_boxes) > 0:\n filtered_annotations[k] = {'w': img_ann['w'], 'h': img_ann['h'], 'boxes': filtered_boxes}\n\n children_id_to_labels = dict([(id_map[i], l) for (i, l) in children_id_to_labels.items()])\n\n return children_id_to_labels, filtered_annotations\n\n def size(self):\n return len(self.annotations)\n\n def num_classes(self):\n return len(self.id_to_labels)\n\n def has_label(self, label):\n \"\"\" Return True if label is a known label.\n \"\"\"\n return label in self.id_to_labels\n\n def has_name(self, name):\n \"\"\" Returns True if name is a known class.\n \"\"\"\n raise NotImplementedError()\n\n def name_to_label(self, name):\n raise NotImplementedError()\n\n def label_to_name(self, label):\n return self.id_to_labels[label]\n\n def image_aspect_ratio(self, image_index):\n img_annotations = self.annotations[self.id_to_image_id[image_index]]\n height, width = img_annotations['h'], img_annotations['w']\n return float(width) / float(height)\n\n def image_path(self, image_index):\n path = os.path.join(self.base_dir, self.id_to_image_id[image_index] + '.jpg')\n return path\n\n def load_image(self, image_index):\n return read_image_bgr(self.image_path(image_index))\n\n def load_annotations(self, image_index):\n image_annotations = self.annotations[self.id_to_image_id[image_index]]\n\n labels = image_annotations['boxes']\n height, width = image_annotations['h'], image_annotations['w']\n\n annotations = {'labels': np.empty((len(labels),)), 'bboxes': np.empty((len(labels), 4))}\n for idx, ann in enumerate(labels):\n cls_id = ann['cls_id']\n x1 = ann['x1'] * width\n x2 = ann['x2'] * width\n y1 = ann['y1'] * height\n y2 = ann['y2'] * height\n\n annotations['bboxes'][idx, 0] = x1\n annotations['bboxes'][idx, 1] = y1\n annotations['bboxes'][idx, 2] = x2\n annotations['bboxes'][idx, 3] = y2\n annotations['labels'][idx] = cls_id\n\n return annotations\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/preprocessing/pascal_voc.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom ..preprocessing.generator import Generator\nfrom ..utils.image import read_image_bgr\n\nimport os\nimport numpy as np\nfrom six import raise_from\nfrom PIL import Image\n\ntry:\n import xml.etree.cElementTree as ET\nexcept ImportError:\n import xml.etree.ElementTree as ET\n\nvoc_classes = {\n 'aeroplane' : 0,\n 'bicycle' : 1,\n 'bird' : 2,\n 'boat' : 3,\n 'bottle' : 4,\n 'bus' : 5,\n 'car' : 6,\n 'cat' : 7,\n 'chair' : 8,\n 'cow' : 9,\n 'diningtable' : 10,\n 'dog' : 11,\n 'horse' : 12,\n 'motorbike' : 13,\n 'person' : 14,\n 'pottedplant' : 15,\n 'sheep' : 16,\n 'sofa' : 17,\n 'train' : 18,\n 'tvmonitor' : 19\n}\n\n\ndef _findNode(parent, name, debug_name=None, parse=None):\n if debug_name is None:\n debug_name = name\n\n result = parent.find(name)\n if result is None:\n raise ValueError('missing element \\'{}\\''.format(debug_name))\n if parse is not None:\n try:\n return parse(result.text)\n except ValueError as e:\n raise_from(ValueError('illegal value for \\'{}\\': {}'.format(debug_name, e)), None)\n return result\n\n\nclass PascalVocGenerator(Generator):\n \"\"\" Generate data for a Pascal VOC dataset.\n\n See http://host.robots.ox.ac.uk/pascal/VOC/ for more information.\n \"\"\"\n\n def __init__(\n self,\n data_dir,\n set_name,\n classes=voc_classes,\n image_extension='.jpg',\n skip_truncated=False,\n skip_difficult=False,\n **kwargs\n ):\n \"\"\" Initialize a Pascal VOC data generator.\n\n Args\n base_dir: Directory w.r.t. where the files are to be searched (defaults to the directory containing the csv_data_file).\n csv_class_file: Path to the CSV classes file.\n \"\"\"\n self.data_dir = data_dir\n self.set_name = set_name\n self.classes = classes\n self.image_names = [line.strip().split(None, 1)[0] for line in open(os.path.join(data_dir, 'ImageSets', 'Main', set_name + '.txt')).readlines()]\n self.image_extension = image_extension\n self.skip_truncated = skip_truncated\n self.skip_difficult = skip_difficult\n\n self.labels = {}\n for key, value in self.classes.items():\n self.labels[value] = key\n\n super(PascalVocGenerator, self).__init__(**kwargs)\n\n def size(self):\n \"\"\" Size of the dataset.\n \"\"\"\n return len(self.image_names)\n\n def num_classes(self):\n \"\"\" Number of classes in the dataset.\n \"\"\"\n return len(self.classes)\n\n def has_label(self, label):\n \"\"\" Return True if label is a known label.\n \"\"\"\n return label in self.labels\n\n def has_name(self, name):\n \"\"\" Returns True if name is a known class.\n \"\"\"\n return name in self.classes\n\n def name_to_label(self, name):\n \"\"\" Map name to label.\n \"\"\"\n return self.classes[name]\n\n def label_to_name(self, label):\n \"\"\" Map label to name.\n \"\"\"\n return self.labels[label]\n\n def image_aspect_ratio(self, image_index):\n \"\"\" Compute the aspect ratio for an image with image_index.\n \"\"\"\n path = os.path.join(self.data_dir, 'JPEGImages', self.image_names[image_index] + self.image_extension)\n image = Image.open(path)\n return float(image.width) / float(image.height)\n\n def image_path(self, image_index):\n \"\"\" Get the path to an image.\n \"\"\"\n return os.path.join(self.data_dir, 'JPEGImages', self.image_names[image_index] + self.image_extension)\n\n def load_image(self, image_index):\n \"\"\" Load an image at the image_index.\n \"\"\"\n return read_image_bgr(self.image_path(image_index))\n\n def __parse_annotation(self, element):\n \"\"\" Parse an annotation given an XML element.\n \"\"\"\n truncated = _findNode(element, 'truncated', parse=int)\n difficult = _findNode(element, 'difficult', parse=int)\n\n class_name = _findNode(element, 'name').text\n if class_name not in self.classes:\n raise ValueError('class name \\'{}\\' not found in classes: {}'.format(class_name, list(self.classes.keys())))\n\n box = np.zeros((4,))\n label = self.name_to_label(class_name)\n\n bndbox = _findNode(element, 'bndbox')\n box[0] = _findNode(bndbox, 'xmin', 'bndbox.xmin', parse=float) - 1\n box[1] = _findNode(bndbox, 'ymin', 'bndbox.ymin', parse=float) - 1\n box[2] = _findNode(bndbox, 'xmax', 'bndbox.xmax', parse=float) - 1\n box[3] = _findNode(bndbox, 'ymax', 'bndbox.ymax', parse=float) - 1\n\n return truncated, difficult, box, label\n\n def __parse_annotations(self, xml_root):\n \"\"\" Parse all annotations under the xml_root.\n \"\"\"\n annotations = {'labels': np.empty((len(xml_root.findall('object')),)), 'bboxes': np.empty((len(xml_root.findall('object')), 4))}\n for i, element in enumerate(xml_root.iter('object')):\n try:\n truncated, difficult, box, label = self.__parse_annotation(element)\n except ValueError as e:\n raise_from(ValueError('could not parse object #{}: {}'.format(i, e)), None)\n\n if truncated and self.skip_truncated:\n continue\n if difficult and self.skip_difficult:\n continue\n\n annotations['bboxes'][i, :] = box\n annotations['labels'][i] = label\n\n return annotations\n\n def load_annotations(self, image_index):\n \"\"\" Load annotations for an image_index.\n \"\"\"\n filename = self.image_names[image_index] + '.xml'\n try:\n tree = ET.parse(os.path.join(self.data_dir, 'Annotations', filename))\n return self.__parse_annotations(tree.getroot())\n except ET.ParseError as e:\n raise_from(ValueError('invalid annotations file: {}: {}'.format(filename, e)), None)\n except ValueError as e:\n raise_from(ValueError('invalid annotations file: {}: {}'.format(filename, e)), None)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/__init__.py", "content": "" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/anchors.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport numpy as np\nfrom tensorflow import keras\n\n#from ..utils.compute_overlap import compute_overlap\n\n\nclass AnchorParameters:\n \"\"\" The parameteres that define how anchors are generated.\n\n Args\n sizes : List of sizes to use. Each size corresponds to one feature level.\n strides : List of strides to use. Each stride correspond to one feature level.\n ratios : List of ratios to use per location in a feature map.\n scales : List of scales to use per location in a feature map.\n \"\"\"\n def __init__(self, sizes, strides, ratios, scales):\n self.sizes = sizes\n self.strides = strides\n self.ratios = ratios\n self.scales = scales\n\n def num_anchors(self):\n return len(self.ratios) * len(self.scales)\n\n\n\"\"\"\nThe default anchor parameters.\n\"\"\"\nAnchorParameters.default = AnchorParameters(\n sizes = [32, 64, 128, 256, 512],\n strides = [8, 16, 32, 64, 128],\n ratios = np.array([0.5, 1, 2], keras.backend.floatx()),\n scales = np.array([2 ** 0, 2 ** (1.0 / 3.0), 2 ** (2.0 / 3.0)], keras.backend.floatx()),\n)\n\n\ndef anchor_targets_bbox(\n anchors,\n image_group,\n annotations_group,\n num_classes,\n negative_overlap=0.4,\n positive_overlap=0.5\n):\n \"\"\" Generate anchor targets for bbox detection.\n\n Args\n anchors: np.array of annotations of shape (N, 4) for (x1, y1, x2, y2).\n image_group: List of BGR images.\n annotations_group: List of annotation dictionaries with each annotation containing 'labels' and 'bboxes' of an image.\n num_classes: Number of classes to predict.\n mask_shape: If the image is padded with zeros, mask_shape can be used to mark the relevant part of the image.\n negative_overlap: IoU overlap for negative anchors (all anchors with overlap < negative_overlap are negative).\n positive_overlap: IoU overlap or positive anchors (all anchors with overlap > positive_overlap are positive).\n\n Returns\n labels_batch: batch that contains labels & anchor states (np.array of shape (batch_size, N, num_classes + 1),\n where N is the number of anchors for an image and the last column defines the anchor state (-1 for ignore, 0 for bg, 1 for fg).\n regression_batch: batch that contains bounding-box regression targets for an image & anchor states (np.array of shape (batch_size, N, 4 + 1),\n where N is the number of anchors for an image, the first 4 columns define regression targets for (x1, y1, x2, y2) and the\n last column defines anchor states (-1 for ignore, 0 for bg, 1 for fg).\n \"\"\"\n\n assert(len(image_group) == len(annotations_group)), \"The length of the images and annotations need to be equal.\"\n assert(len(annotations_group) > 0), \"No data received to compute anchor targets for.\"\n for annotations in annotations_group:\n assert('bboxes' in annotations), \"Annotations should contain bboxes.\"\n assert('labels' in annotations), \"Annotations should contain labels.\"\n\n batch_size = len(image_group)\n\n regression_batch = np.zeros((batch_size, anchors.shape[0], 4 + 1), dtype=keras.backend.floatx())\n labels_batch = np.zeros((batch_size, anchors.shape[0], num_classes + 1), dtype=keras.backend.floatx())\n\n # compute labels and regression targets\n for index, (image, annotations) in enumerate(zip(image_group, annotations_group)):\n if annotations['bboxes'].shape[0]:\n # obtain indices of gt annotations with the greatest overlap\n positive_indices, ignore_indices, argmax_overlaps_inds = compute_gt_annotations(anchors, annotations['bboxes'], negative_overlap, positive_overlap)\n\n labels_batch[index, ignore_indices, -1] = -1\n labels_batch[index, positive_indices, -1] = 1\n\n regression_batch[index, ignore_indices, -1] = -1\n regression_batch[index, positive_indices, -1] = 1\n\n # compute target class labels\n labels_batch[index, positive_indices, annotations['labels'][argmax_overlaps_inds[positive_indices]].astype(int)] = 1\n\n regression_batch[index, :, :-1] = bbox_transform(anchors, annotations['bboxes'][argmax_overlaps_inds, :])\n\n # ignore annotations outside of image\n if image.shape:\n anchors_centers = np.vstack([(anchors[:, 0] + anchors[:, 2]) / 2, (anchors[:, 1] + anchors[:, 3]) / 2]).T\n indices = np.logical_or(anchors_centers[:, 0] >= image.shape[1], anchors_centers[:, 1] >= image.shape[0])\n\n labels_batch[index, indices, -1] = -1\n regression_batch[index, indices, -1] = -1\n\n return regression_batch, labels_batch\n\n\ndef layer_shapes(image_shape, model):\n \"\"\"Compute layer shapes given input image shape and the model.\n\n Args\n image_shape: The shape of the image.\n model: The model to use for computing how the image shape is transformed in the pyramid.\n\n Returns\n A dictionary mapping layer names to image shapes.\n \"\"\"\n shape = {\n model.layers[0].name: (None,) + image_shape,\n }\n\n for layer in model.layers[1:]:\n nodes = layer._inbound_nodes\n for node in nodes:\n if isinstance(node.inbound_layers, keras.layers.Layer):\n inputs = [shape[node.inbound_layers.name]]\n else:\n inputs = [shape[lr.name] for lr in node.inbound_layers]\n if not inputs:\n continue\n shape[layer.name] = layer.compute_output_shape(inputs[0] if len(inputs) == 1 else inputs)\n\n return shape\n\n\ndef make_shapes_callback(model):\n \"\"\" Make a function for getting the shape of the pyramid levels.\n \"\"\"\n def get_shapes(image_shape, pyramid_levels):\n shape = layer_shapes(image_shape, model)\n image_shapes = [shape[\"P{}\".format(level)][1:3] for level in pyramid_levels]\n return image_shapes\n\n return get_shapes\n\n\ndef guess_shapes(image_shape, pyramid_levels):\n \"\"\"Guess shapes based on pyramid levels.\n\n Args\n image_shape: The shape of the image.\n pyramid_levels: A list of what pyramid levels are used.\n\n Returns\n A list of image shapes at each pyramid level.\n \"\"\"\n image_shape = np.array(image_shape[:2])\n image_shapes = [(image_shape + 2 ** x - 1) // (2 ** x) for x in pyramid_levels]\n return image_shapes\n\n\ndef anchors_for_shape(\n image_shape,\n pyramid_levels=None,\n anchor_params=None,\n shapes_callback=None,\n):\n \"\"\" Generators anchors for a given shape.\n\n Args\n image_shape: The shape of the image.\n pyramid_levels: List of ints representing which pyramids to use (defaults to [3, 4, 5, 6, 7]).\n anchor_params: Struct containing anchor parameters. If None, default values are used.\n shapes_callback: Function to call for getting the shape of the image at different pyramid levels.\n\n Returns\n np.array of shape (N, 4) containing the (x1, y1, x2, y2) coordinates for the anchors.\n \"\"\"\n\n if pyramid_levels is None:\n pyramid_levels = [3, 4, 5, 6, 7]\n\n if anchor_params is None:\n anchor_params = AnchorParameters.default\n\n if shapes_callback is None:\n shapes_callback = guess_shapes\n image_shapes = shapes_callback(image_shape, pyramid_levels)\n\n # compute anchors over all pyramid levels\n all_anchors = np.zeros((0, 4))\n for idx, p in enumerate(pyramid_levels):\n anchors = generate_anchors(\n base_size=anchor_params.sizes[idx],\n ratios=anchor_params.ratios,\n scales=anchor_params.scales\n )\n shifted_anchors = shift(image_shapes[idx], anchor_params.strides[idx], anchors)\n all_anchors = np.append(all_anchors, shifted_anchors, axis=0)\n\n return all_anchors\n\n\ndef shift(shape, stride, anchors):\n \"\"\" Produce shifted anchors based on shape of the map and stride size.\n\n Args\n shape : Shape to shift the anchors over.\n stride : Stride to shift the anchors with over the shape.\n anchors: The anchors to apply at each location.\n \"\"\"\n\n # create a grid starting from half stride from the top left corner\n shift_x = (np.arange(0, shape[1]) + 0.5) * stride\n shift_y = (np.arange(0, shape[0]) + 0.5) * stride\n\n shift_x, shift_y = np.meshgrid(shift_x, shift_y)\n\n shifts = np.vstack((\n shift_x.ravel(), shift_y.ravel(),\n shift_x.ravel(), shift_y.ravel()\n )).transpose()\n\n # add A anchors (1, A, 4) to\n # cell K shifts (K, 1, 4) to get\n # shift anchors (K, A, 4)\n # reshape to (K*A, 4) shifted anchors\n A = anchors.shape[0]\n K = shifts.shape[0]\n all_anchors = (anchors.reshape((1, A, 4)) + shifts.reshape((1, K, 4)).transpose((1, 0, 2)))\n all_anchors = all_anchors.reshape((K * A, 4))\n\n return all_anchors\n\n\ndef generate_anchors(base_size=16, ratios=None, scales=None):\n \"\"\"\n Generate anchor (reference) windows by enumerating aspect ratios X\n scales w.r.t. a reference window.\n \"\"\"\n\n if ratios is None:\n ratios = AnchorParameters.default.ratios\n\n if scales is None:\n scales = AnchorParameters.default.scales\n\n num_anchors = len(ratios) * len(scales)\n\n # initialize output anchors\n anchors = np.zeros((num_anchors, 4))\n\n # scale base_size\n anchors[:, 2:] = base_size * np.tile(scales, (2, len(ratios))).T\n\n # compute areas of anchors\n areas = anchors[:, 2] * anchors[:, 3]\n\n # correct for ratios\n anchors[:, 2] = np.sqrt(areas / np.repeat(ratios, len(scales)))\n anchors[:, 3] = anchors[:, 2] * np.repeat(ratios, len(scales))\n\n # transform from (x_ctr, y_ctr, w, h) -> (x1, y1, x2, y2)\n anchors[:, 0::2] -= np.tile(anchors[:, 2] * 0.5, (2, 1)).T\n anchors[:, 1::2] -= np.tile(anchors[:, 3] * 0.5, (2, 1)).T\n\n return anchors\n\n\ndef bbox_transform(anchors, gt_boxes, mean=None, std=None):\n \"\"\"Compute bounding-box regression targets for an image.\"\"\"\n\n # The Mean and std are calculated from COCO dataset.\n # Bounding box normalization was firstly introduced in the Fast R-CNN paper.\n # See https://github.com/fizyr/keras-retinanet/issues/1273#issuecomment-585828825 for more details\n if mean is None:\n mean = np.array([0, 0, 0, 0])\n if std is None:\n std = np.array([0.2, 0.2, 0.2, 0.2])\n\n if isinstance(mean, (list, tuple)):\n mean = np.array(mean)\n elif not isinstance(mean, np.ndarray):\n raise ValueError('Expected mean to be a np.ndarray, list or tuple. Received: {}'.format(type(mean)))\n\n if isinstance(std, (list, tuple)):\n std = np.array(std)\n elif not isinstance(std, np.ndarray):\n raise ValueError('Expected std to be a np.ndarray, list or tuple. Received: {}'.format(type(std)))\n\n anchor_widths = anchors[:, 2] - anchors[:, 0]\n anchor_heights = anchors[:, 3] - anchors[:, 1]\n\n # According to the information provided by a keras-retinanet author, they got marginally better results using\n # the following way of bounding box parametrization.\n # See https://github.com/fizyr/keras-retinanet/issues/1273#issuecomment-585828825 for more details\n targets_dx1 = (gt_boxes[:, 0] - anchors[:, 0]) / anchor_widths\n targets_dy1 = (gt_boxes[:, 1] - anchors[:, 1]) / anchor_heights\n targets_dx2 = (gt_boxes[:, 2] - anchors[:, 2]) / anchor_widths\n targets_dy2 = (gt_boxes[:, 3] - anchors[:, 3]) / anchor_heights\n\n targets = np.stack((targets_dx1, targets_dy1, targets_dx2, targets_dy2))\n targets = targets.T\n\n targets = (targets - mean) / std\n\n return targets\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/coco_eval.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom pycocotools.cocoeval import COCOeval\n\nfrom tensorflow import keras\nimport numpy as np\nimport json\n\nimport progressbar\nassert(callable(progressbar.progressbar)), \"Using wrong progressbar module, install 'progressbar2' instead.\"\n\n\ndef evaluate_coco(generator, model, threshold=0.05):\n \"\"\" Use the pycocotools to evaluate a COCO model on a dataset.\n\n Args\n generator : The generator for generating the evaluation data.\n model : The model to evaluate.\n threshold : The score threshold to use.\n \"\"\"\n # start collecting results\n results = []\n image_ids = []\n for index in progressbar.progressbar(range(generator.size()), prefix='COCO evaluation: '):\n image = generator.load_image(index)\n image = generator.preprocess_image(image)\n image, scale = generator.resize_image(image)\n\n if keras.backend.image_data_format() == 'channels_first':\n image = image.transpose((2, 0, 1))\n\n # run network\n boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))\n\n # correct boxes for image scale\n boxes /= scale\n\n # change to (x, y, w, h) (MS COCO standard)\n boxes[:, :, 2] -= boxes[:, :, 0]\n boxes[:, :, 3] -= boxes[:, :, 1]\n\n # compute predicted labels and scores\n for box, score, label in zip(boxes[0], scores[0], labels[0]):\n # scores are sorted, so we can break\n if score < threshold:\n break\n\n # append detection for each positively labeled class\n image_result = {\n 'image_id' : generator.image_ids[index],\n 'category_id' : generator.label_to_coco_label(label),\n 'score' : float(score),\n 'bbox' : box.tolist(),\n }\n\n # append detection to results\n results.append(image_result)\n\n # append image to list of processed images\n image_ids.append(generator.image_ids[index])\n\n if not len(results):\n return\n\n # write output\n json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4)\n json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4)\n\n # load results in COCO evaluation tool\n coco_true = generator.coco\n coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name))\n\n # run COCO evaluation\n coco_eval = COCOeval(coco_true, coco_pred, 'bbox')\n coco_eval.params.imgIds = image_ids\n coco_eval.evaluate()\n coco_eval.accumulate()\n coco_eval.summarize()\n return coco_eval.stats\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/colors.py", "content": "import warnings\n\n\ndef label_color(label):\n \"\"\" Return a color from a set of predefined colors. Contains 80 colors in total.\n\n Args\n label: The label to get the color for.\n\n Returns\n A list of three values representing a RGB color.\n\n If no color is defined for a certain label, the color green is returned and a warning is printed.\n \"\"\"\n if label < len(colors):\n return colors[label]\n else:\n warnings.warn('Label {} has no color, returning default.'.format(label))\n return (0, 255, 0)\n\n\n\"\"\"\nGenerated using:\n\n```\ncolors = [list((matplotlib.colors.hsv_to_rgb([x, 1.0, 1.0]) * 255).astype(int)) for x in np.arange(0, 1, 1.0 / 80)]\nshuffle(colors)\npprint(colors)\n```\n\"\"\"\ncolors = [\n [31 , 0 , 255] ,\n [0 , 159 , 255] ,\n [255 , 95 , 0] ,\n [255 , 19 , 0] ,\n [255 , 0 , 0] ,\n [255 , 38 , 0] ,\n [0 , 255 , 25] ,\n [255 , 0 , 133] ,\n [255 , 172 , 0] ,\n [108 , 0 , 255] ,\n [0 , 82 , 255] ,\n [0 , 255 , 6] ,\n [255 , 0 , 152] ,\n [223 , 0 , 255] ,\n [12 , 0 , 255] ,\n [0 , 255 , 178] ,\n [108 , 255 , 0] ,\n [184 , 0 , 255] ,\n [255 , 0 , 76] ,\n [146 , 255 , 0] ,\n [51 , 0 , 255] ,\n [0 , 197 , 255] ,\n [255 , 248 , 0] ,\n [255 , 0 , 19] ,\n [255 , 0 , 38] ,\n [89 , 255 , 0] ,\n [127 , 255 , 0] ,\n [255 , 153 , 0] ,\n [0 , 255 , 255] ,\n [0 , 255 , 216] ,\n [0 , 255 , 121] ,\n [255 , 0 , 248] ,\n [70 , 0 , 255] ,\n [0 , 255 , 159] ,\n [0 , 216 , 255] ,\n [0 , 6 , 255] ,\n [0 , 63 , 255] ,\n [31 , 255 , 0] ,\n [255 , 57 , 0] ,\n [255 , 0 , 210] ,\n [0 , 255 , 102] ,\n [242 , 255 , 0] ,\n [255 , 191 , 0] ,\n [0 , 255 , 63] ,\n [255 , 0 , 95] ,\n [146 , 0 , 255] ,\n [184 , 255 , 0] ,\n [255 , 114 , 0] ,\n [0 , 255 , 235] ,\n [255 , 229 , 0] ,\n [0 , 178 , 255] ,\n [255 , 0 , 114] ,\n [255 , 0 , 57] ,\n [0 , 140 , 255] ,\n [0 , 121 , 255] ,\n [12 , 255 , 0] ,\n [255 , 210 , 0] ,\n [0 , 255 , 44] ,\n [165 , 255 , 0] ,\n [0 , 25 , 255] ,\n [0 , 255 , 140] ,\n [0 , 101 , 255] ,\n [0 , 255 , 82] ,\n [223 , 255 , 0] ,\n [242 , 0 , 255] ,\n [89 , 0 , 255] ,\n [165 , 0 , 255] ,\n [70 , 255 , 0] ,\n [255 , 0 , 172] ,\n [255 , 76 , 0] ,\n [203 , 255 , 0] ,\n [204 , 0 , 255] ,\n [255 , 0 , 229] ,\n [255 , 133 , 0] ,\n [127 , 0 , 255] ,\n [0 , 235 , 255] ,\n [0 , 255 , 197] ,\n [255 , 0 , 191] ,\n [0 , 44 , 255] ,\n [50 , 255 , 0]\n]\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/compute_overlap.pyx", "content": "# --------------------------------------------------------\n# Fast R-CNN\n# Copyright (c) 2015 Microsoft\n# Licensed under The MIT License [see LICENSE for details]\n# Written by Sergey Karayev\n# --------------------------------------------------------\n\ncimport cython\nimport numpy as np\ncimport numpy as np\n\n\ndef compute_overlap(\n np.ndarray[double, ndim=2] boxes,\n np.ndarray[double, ndim=2] query_boxes\n):\n \"\"\"\n Args\n a: (N, 4) ndarray of float\n b: (K, 4) ndarray of float\n\n Returns\n overlaps: (N, K) ndarray of overlap between boxes and query_boxes\n \"\"\"\n cdef unsigned int N = boxes.shape[0]\n cdef unsigned int K = query_boxes.shape[0]\n cdef np.ndarray[double, ndim=2] overlaps = np.zeros((N, K), dtype=np.float64)\n cdef double iw, ih, box_area\n cdef double ua\n cdef unsigned int k, n\n for k in range(K):\n box_area = (\n (query_boxes[k, 2] - query_boxes[k, 0]) *\n (query_boxes[k, 3] - query_boxes[k, 1])\n )\n for n in range(N):\n iw = (\n min(boxes[n, 2], query_boxes[k, 2]) -\n max(boxes[n, 0], query_boxes[k, 0]) \n )\n if iw > 0:\n ih = (\n min(boxes[n, 3], query_boxes[k, 3]) -\n max(boxes[n, 1], query_boxes[k, 1]) \n )\n if ih > 0:\n ua = np.float64(\n (boxes[n, 2] - boxes[n, 0]) *\n (boxes[n, 3] - boxes[n, 1]) +\n box_area - iw * ih\n )\n overlaps[n, k] = iw * ih / ua\n return overlaps\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/config.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport configparser\nimport numpy as np\nfrom tensorflow import keras\nfrom ..utils.anchors import AnchorParameters\n\n\ndef read_config_file(config_path):\n config = configparser.ConfigParser()\n\n with open(config_path, 'r') as file:\n config.read_file(file)\n\n assert 'anchor_parameters' in config, \\\n \"Malformed config file. Verify that it contains the anchor_parameters section.\"\n\n config_keys = set(config['anchor_parameters'])\n default_keys = set(AnchorParameters.default.__dict__.keys())\n\n assert config_keys <= default_keys, \\\n \"Malformed config file. These keys are not valid: {}\".format(config_keys - default_keys)\n\n if 'pyramid_levels' in config:\n assert('levels' in config['pyramid_levels']), \"pyramid levels specified by levels key\"\n\n return config\n\n\ndef parse_anchor_parameters(config):\n ratios = np.array(list(map(float, config['anchor_parameters']['ratios'].split(' '))), keras.backend.floatx())\n scales = np.array(list(map(float, config['anchor_parameters']['scales'].split(' '))), keras.backend.floatx())\n sizes = list(map(int, config['anchor_parameters']['sizes'].split(' ')))\n strides = list(map(int, config['anchor_parameters']['strides'].split(' ')))\n assert (len(sizes) == len(strides)), \"sizes and strides should have an equal number of values\"\n\n return AnchorParameters(sizes, strides, ratios, scales)\n\n\ndef parse_pyramid_levels(config):\n levels = list(map(int, config['pyramid_levels']['levels'].split(' ')))\n\n return levels\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/eval.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom .anchors import compute_overlap\nfrom .visualization import draw_detections, draw_annotations\n\nfrom tensorflow import keras\nimport numpy as np\nimport os\nimport time\n\nimport cv2\nimport progressbar\nassert(callable(progressbar.progressbar)), \"Using wrong progressbar module, install 'progressbar2' instead.\"\n\n\ndef _compute_ap(recall, precision):\n \"\"\" Compute the average precision, given the recall and precision curves.\n\n Code originally from https://github.com/rbgirshick/py-faster-rcnn.\n\n # Arguments\n recall: The recall curve (list).\n precision: The precision curve (list).\n # Returns\n The average precision as computed in py-faster-rcnn.\n \"\"\"\n # correct AP calculation\n # first append sentinel values at the end\n mrec = np.concatenate(([0.], recall, [1.]))\n mpre = np.concatenate(([0.], precision, [0.]))\n\n # compute the precision envelope\n for i in range(mpre.size - 1, 0, -1):\n mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])\n\n # to calculate area under PR curve, look for points\n # where X axis (recall) changes value\n i = np.where(mrec[1:] != mrec[:-1])[0]\n\n # and sum (\\Delta recall) * prec\n ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])\n return ap\n\n\ndef _get_detections(generator, model, score_threshold=0.05, max_detections=100, save_path=None):\n \"\"\" Get the detections from the model using the generator.\n\n The result is a list of lists such that the size is:\n all_detections[num_images][num_classes] = detections[num_detections, 4 + num_classes]\n\n # Arguments\n generator : The generator used to run images through the model.\n model : The model to run on the images.\n score_threshold : The score confidence threshold to use.\n max_detections : The maximum number of detections to use per image.\n save_path : The path to save the images with visualized detections to.\n # Returns\n A list of lists containing the detections for each image in the generator.\n \"\"\"\n all_detections = [[None for i in range(generator.num_classes()) if generator.has_label(i)] for j in range(generator.size())]\n all_inferences = [None for i in range(generator.size())]\n\n for i in progressbar.progressbar(range(generator.size()), prefix='Running network: '):\n raw_image = generator.load_image(i)\n image, scale = generator.resize_image(raw_image.copy())\n image = generator.preprocess_image(image)\n\n if keras.backend.image_data_format() == 'channels_first':\n image = image.transpose((2, 0, 1))\n\n # run network\n start = time.time()\n boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))[:3]\n inference_time = time.time() - start\n\n # correct boxes for image scale\n boxes /= scale\n\n # select indices which have a score above the threshold\n indices = np.where(scores[0, :] > score_threshold)[0]\n\n # select those scores\n scores = scores[0][indices]\n\n # find the order with which to sort the scores\n scores_sort = np.argsort(-scores)[:max_detections]\n\n # select detections\n image_boxes = boxes[0, indices[scores_sort], :]\n image_scores = scores[scores_sort]\n image_labels = labels[0, indices[scores_sort]]\n image_detections = np.concatenate([image_boxes, np.expand_dims(image_scores, axis=1), np.expand_dims(image_labels, axis=1)], axis=1)\n\n if save_path is not None:\n draw_annotations(raw_image, generator.load_annotations(i), label_to_name=generator.label_to_name)\n draw_detections(raw_image, image_boxes, image_scores, image_labels, label_to_name=generator.label_to_name, score_threshold=score_threshold)\n\n cv2.imwrite(os.path.join(save_path, '{}.png'.format(i)), raw_image)\n\n # copy detections to all_detections\n for label in range(generator.num_classes()):\n if not generator.has_label(label):\n continue\n\n all_detections[i][label] = image_detections[image_detections[:, -1] == label, :-1]\n\n all_inferences[i] = inference_time\n\n return all_detections, all_inferences\n\n\ndef _get_annotations(generator):\n \"\"\" Get the ground truth annotations from the generator.\n\n The result is a list of lists such that the size is:\n all_detections[num_images][num_classes] = annotations[num_detections, 5]\n\n # Arguments\n generator : The generator used to retrieve ground truth annotations.\n # Returns\n A list of lists containing the annotations for each image in the generator.\n \"\"\"\n all_annotations = [[None for i in range(generator.num_classes())] for j in range(generator.size())]\n\n for i in progressbar.progressbar(range(generator.size()), prefix='Parsing annotations: '):\n # load the annotations\n annotations = generator.load_annotations(i)\n\n # copy detections to all_annotations\n for label in range(generator.num_classes()):\n if not generator.has_label(label):\n continue\n\n all_annotations[i][label] = annotations['bboxes'][annotations['labels'] == label, :].copy()\n\n return all_annotations\n\n\ndef evaluate(\n generator,\n model,\n iou_threshold=0.5,\n score_threshold=0.05,\n max_detections=100,\n save_path=None\n):\n \"\"\" Evaluate a given dataset using a given model.\n\n # Arguments\n generator : The generator that represents the dataset to evaluate.\n model : The model to evaluate.\n iou_threshold : The threshold used to consider when a detection is positive or negative.\n score_threshold : The score confidence threshold to use for detections.\n max_detections : The maximum number of detections to use per image.\n save_path : The path to save images with visualized detections to.\n # Returns\n A dict mapping class names to mAP scores.\n \"\"\"\n # gather all detections and annotations\n all_detections, all_inferences = _get_detections(generator, model, score_threshold=score_threshold, max_detections=max_detections, save_path=save_path)\n all_annotations = _get_annotations(generator)\n average_precisions = {}\n\n # all_detections = pickle.load(open('all_detections.pkl', 'rb'))\n # all_annotations = pickle.load(open('all_annotations.pkl', 'rb'))\n # pickle.dump(all_detections, open('all_detections.pkl', 'wb'))\n # pickle.dump(all_annotations, open('all_annotations.pkl', 'wb'))\n\n # process detections and annotations\n for label in range(generator.num_classes()):\n if not generator.has_label(label):\n continue\n\n false_positives = np.zeros((0,))\n true_positives = np.zeros((0,))\n scores = np.zeros((0,))\n num_annotations = 0.0\n\n for i in range(generator.size()):\n detections = all_detections[i][label]\n annotations = all_annotations[i][label]\n num_annotations += annotations.shape[0]\n detected_annotations = []\n\n for d in detections:\n scores = np.append(scores, d[4])\n\n if annotations.shape[0] == 0:\n false_positives = np.append(false_positives, 1)\n true_positives = np.append(true_positives, 0)\n continue\n\n overlaps = compute_overlap(np.expand_dims(d, axis=0), annotations)\n assigned_annotation = np.argmax(overlaps, axis=1)\n max_overlap = overlaps[0, assigned_annotation]\n\n if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:\n false_positives = np.append(false_positives, 0)\n true_positives = np.append(true_positives, 1)\n detected_annotations.append(assigned_annotation)\n else:\n false_positives = np.append(false_positives, 1)\n true_positives = np.append(true_positives, 0)\n\n # no annotations -> AP for this class is 0 (is this correct?)\n if num_annotations == 0:\n average_precisions[label] = 0, 0\n continue\n\n # sort by score\n indices = np.argsort(-scores)\n false_positives = false_positives[indices]\n true_positives = true_positives[indices]\n\n # compute false positives and true positives\n false_positives = np.cumsum(false_positives)\n true_positives = np.cumsum(true_positives)\n\n # compute recall and precision\n recall = true_positives / num_annotations\n precision = true_positives / np.maximum(true_positives + false_positives, np.finfo(np.float64).eps)\n\n # compute average precision\n average_precision = _compute_ap(recall, precision)\n average_precisions[label] = average_precision, num_annotations\n\n # inference time\n inference_time = np.sum(all_inferences) / generator.size()\n\n return average_precisions, inference_time\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/gpu.py", "content": "\"\"\"\nCopyright 2017-2019 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport tensorflow as tf\n\n\ndef setup_gpu(gpu_id):\n try:\n visible_gpu_indices = [int(id) for id in gpu_id.split(',')]\n available_gpus = tf.config.list_physical_devices('GPU')\n visible_gpus = [gpu for idx, gpu in enumerate(available_gpus) if idx in visible_gpu_indices]\n\n if visible_gpus:\n try:\n # Currently, memory growth needs to be the same across GPUs.\n for gpu in available_gpus:\n tf.config.experimental.set_memory_growth(gpu, True)\n\n # Use only the selcted gpu.\n tf.config.set_visible_devices(visible_gpus, 'GPU')\n except RuntimeError as e:\n # Visible devices must be set before GPUs have been initialized.\n print(e)\n\n logical_gpus = tf.config.list_logical_devices('GPU')\n print(len(available_gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n else:\n tf.config.set_visible_devices([], 'GPU')\n except ValueError:\n tf.config.set_visible_devices([], 'GPU')\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/image.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom __future__ import division\nimport numpy as np\nimport cv2\nfrom PIL import Image\n\nfrom .transform import change_transform_origin\n\n\ndef read_image_bgr(path):\n \"\"\" Read an image in BGR format.\n\n Args\n path: Path to the image.\n \"\"\"\n # We deliberately don't use cv2.imread here, since it gives no feedback on errors while reading the image.\n image = np.ascontiguousarray(Image.open(path).convert('RGB'))\n return image[:, :, ::-1]\n\n\ndef preprocess_image(x, mode='caffe'):\n \"\"\" Preprocess an image by subtracting the ImageNet mean.\n\n Args\n x: np.array of shape (None, None, 3) or (3, None, None).\n mode: One of \"caffe\" or \"tf\".\n - caffe: will zero-center each color channel with\n respect to the ImageNet dataset, without scaling.\n - tf: will scale pixels between -1 and 1, sample-wise.\n\n Returns\n The input with the ImageNet mean subtracted.\n \"\"\"\n # mostly identical to \"https://github.com/keras-team/keras-applications/blob/master/keras_applications/imagenet_utils.py\"\n # except for converting RGB -> BGR since we assume BGR already\n\n # covert always to float32 to keep compatibility with opencv\n x = x.astype(np.float32)\n\n if mode == 'tf':\n x /= 127.5\n x -= 1.\n elif mode == 'caffe':\n x -= [103.939, 116.779, 123.68]\n\n return x\n\n\ndef adjust_transform_for_image(transform, image, relative_translation):\n \"\"\" Adjust a transformation for a specific image.\n\n The translation of the matrix will be scaled with the size of the image.\n The linear part of the transformation will adjusted so that the origin of the transformation will be at the center of the image.\n \"\"\"\n height, width, channels = image.shape\n\n result = transform\n\n # Scale the translation with the image size if specified.\n if relative_translation:\n result[0:2, 2] *= [width, height]\n\n # Move the origin of transformation.\n result = change_transform_origin(transform, (0.5 * width, 0.5 * height))\n\n return result\n\n\nclass TransformParameters:\n \"\"\" Struct holding parameters determining how to apply a transformation to an image.\n\n Args\n fill_mode: One of: 'constant', 'nearest', 'reflect', 'wrap'\n interpolation: One of: 'nearest', 'linear', 'cubic', 'area', 'lanczos4'\n cval: Fill value to use with fill_mode='constant'\n relative_translation: If true (the default), interpret translation as a factor of the image size.\n If false, interpret it as absolute pixels.\n \"\"\"\n def __init__(\n self,\n fill_mode = 'nearest',\n interpolation = 'linear',\n cval = 0,\n relative_translation = True,\n ):\n self.fill_mode = fill_mode\n self.cval = cval\n self.interpolation = interpolation\n self.relative_translation = relative_translation\n\n def cvBorderMode(self):\n if self.fill_mode == 'constant':\n return cv2.BORDER_CONSTANT\n if self.fill_mode == 'nearest':\n return cv2.BORDER_REPLICATE\n if self.fill_mode == 'reflect':\n return cv2.BORDER_REFLECT_101\n if self.fill_mode == 'wrap':\n return cv2.BORDER_WRAP\n\n def cvInterpolation(self):\n if self.interpolation == 'nearest':\n return cv2.INTER_NEAREST\n if self.interpolation == 'linear':\n return cv2.INTER_LINEAR\n if self.interpolation == 'cubic':\n return cv2.INTER_CUBIC\n if self.interpolation == 'area':\n return cv2.INTER_AREA\n if self.interpolation == 'lanczos4':\n return cv2.INTER_LANCZOS4\n\n\ndef apply_transform(matrix, image, params):\n \"\"\"\n Apply a transformation to an image.\n\n The origin of transformation is at the top left corner of the image.\n\n The matrix is interpreted such that a point (x, y) on the original image is moved to transform * (x, y) in the generated image.\n Mathematically speaking, that means that the matrix is a transformation from the transformed image space to the original image space.\n\n Args\n matrix: A homogeneous 3 by 3 matrix holding representing the transformation to apply.\n image: The image to transform.\n params: The transform parameters (see TransformParameters)\n \"\"\"\n output = cv2.warpAffine(\n image,\n matrix[:2, :],\n dsize = (image.shape[1], image.shape[0]),\n flags = params.cvInterpolation(),\n borderMode = params.cvBorderMode(),\n borderValue = params.cval,\n )\n return output\n\n\ndef compute_resize_scale(image_shape, min_side=800, max_side=1333):\n \"\"\" Compute an image scale such that the image size is constrained to min_side and max_side.\n\n Args\n min_side: The image's min side will be equal to min_side after resizing.\n max_side: If after resizing the image's max side is above max_side, resize until the max side is equal to max_side.\n\n Returns\n A resizing scale.\n \"\"\"\n (rows, cols, _) = image_shape\n\n smallest_side = min(rows, cols)\n\n # rescale the image so the smallest side is min_side\n scale = min_side / smallest_side\n\n # check if the largest side is now greater than max_side, which can happen\n # when images have a large aspect ratio\n largest_side = max(rows, cols)\n if largest_side * scale > max_side:\n scale = max_side / largest_side\n\n return scale\n\n\ndef resize_image(img, min_side=800, max_side=1333):\n \"\"\" Resize an image such that the size is constrained to min_side and max_side.\n\n Args\n min_side: The image's min side will be equal to min_side after resizing.\n max_side: If after resizing the image's max side is above max_side, resize until the max side is equal to max_side.\n\n Returns\n A resized image.\n \"\"\"\n # compute scale to resize the image\n scale = compute_resize_scale(img.shape, min_side=min_side, max_side=max_side)\n\n # resize the image with the computed scale\n img = cv2.resize(img, None, fx=scale, fy=scale)\n\n return img, scale\n\n\ndef _uniform(val_range):\n \"\"\" Uniformly sample from the given range.\n\n Args\n val_range: A pair of lower and upper bound.\n \"\"\"\n return np.random.uniform(val_range[0], val_range[1])\n\n\ndef _check_range(val_range, min_val=None, max_val=None):\n \"\"\" Check whether the range is a valid range.\n\n Args\n val_range: A pair of lower and upper bound.\n min_val: Minimal value for the lower bound.\n max_val: Maximal value for the upper bound.\n \"\"\"\n if val_range[0] > val_range[1]:\n raise ValueError('interval lower bound > upper bound')\n if min_val is not None and val_range[0] < min_val:\n raise ValueError('invalid interval lower bound')\n if max_val is not None and val_range[1] > max_val:\n raise ValueError('invalid interval upper bound')\n\n\ndef _clip(image):\n \"\"\"\n Clip and convert an image to np.uint8.\n\n Args\n image: Image to clip.\n \"\"\"\n return np.clip(image, 0, 255).astype(np.uint8)\n\n\nclass VisualEffect:\n \"\"\" Struct holding parameters and applying image color transformation.\n\n Args\n contrast_factor: A factor for adjusting contrast. Should be between 0 and 3.\n brightness_delta: Brightness offset between -1 and 1 added to the pixel values.\n hue_delta: Hue offset between -1 and 1 added to the hue channel.\n saturation_factor: A factor multiplying the saturation values of each pixel.\n \"\"\"\n\n def __init__(\n self,\n contrast_factor,\n brightness_delta,\n hue_delta,\n saturation_factor,\n ):\n self.contrast_factor = contrast_factor\n self.brightness_delta = brightness_delta\n self.hue_delta = hue_delta\n self.saturation_factor = saturation_factor\n\n def __call__(self, image):\n \"\"\" Apply a visual effect on the image.\n\n Args\n image: Image to adjust\n \"\"\"\n\n if self.contrast_factor:\n image = adjust_contrast(image, self.contrast_factor)\n if self.brightness_delta:\n image = adjust_brightness(image, self.brightness_delta)\n\n if self.hue_delta or self.saturation_factor:\n\n image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)\n\n if self.hue_delta:\n image = adjust_hue(image, self.hue_delta)\n if self.saturation_factor:\n image = adjust_saturation(image, self.saturation_factor)\n\n image = cv2.cvtColor(image, cv2.COLOR_HSV2BGR)\n\n return image\n\n\ndef random_visual_effect_generator(\n contrast_range=(0.9, 1.1),\n brightness_range=(-.1, .1),\n hue_range=(-0.05, 0.05),\n saturation_range=(0.95, 1.05)\n):\n \"\"\" Generate visual effect parameters uniformly sampled from the given intervals.\n\n Args\n contrast_factor: A factor interval for adjusting contrast. Should be between 0 and 3.\n brightness_delta: An interval between -1 and 1 for the amount added to the pixels.\n hue_delta: An interval between -1 and 1 for the amount added to the hue channel.\n The values are rotated if they exceed 180.\n saturation_factor: An interval for the factor multiplying the saturation values of each\n pixel.\n \"\"\"\n _check_range(contrast_range, 0)\n _check_range(brightness_range, -1, 1)\n _check_range(hue_range, -1, 1)\n _check_range(saturation_range, 0)\n\n def _generate():\n while True:\n yield VisualEffect(\n contrast_factor=_uniform(contrast_range),\n brightness_delta=_uniform(brightness_range),\n hue_delta=_uniform(hue_range),\n saturation_factor=_uniform(saturation_range),\n )\n\n return _generate()\n\n\ndef adjust_contrast(image, factor):\n \"\"\" Adjust contrast of an image.\n\n Args\n image: Image to adjust.\n factor: A factor for adjusting contrast.\n \"\"\"\n mean = image.mean(axis=0).mean(axis=0)\n return _clip((image - mean) * factor + mean)\n\n\ndef adjust_brightness(image, delta):\n \"\"\" Adjust brightness of an image\n\n Args\n image: Image to adjust.\n delta: Brightness offset between -1 and 1 added to the pixel values.\n \"\"\"\n return _clip(image + delta * 255)\n\n\ndef adjust_hue(image, delta):\n \"\"\" Adjust hue of an image.\n\n Args\n image: Image to adjust.\n delta: An interval between -1 and 1 for the amount added to the hue channel.\n The values are rotated if they exceed 180.\n \"\"\"\n image[..., 0] = np.mod(image[..., 0] + delta * 180, 180)\n return image\n\n\ndef adjust_saturation(image, factor):\n \"\"\" Adjust saturation of an image.\n\n Args\n image: Image to adjust.\n factor: An interval for the factor multiplying the saturation values of each pixel.\n \"\"\"\n image[..., 1] = np.clip(image[..., 1] * factor, 0 , 255)\n return image\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/model.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\n\ndef freeze(model):\n \"\"\" Set all layers in a model to non-trainable.\n\n The weights for these layers will not be updated during training.\n\n This function modifies the given model in-place,\n but it also returns the modified model to allow easy chaining with other functions.\n \"\"\"\n for layer in model.layers:\n layer.trainable = False\n return model\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/tf_version.py", "content": "\"\"\"\nCopyright 2017-2019 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nfrom __future__ import print_function\n\nimport tensorflow as tf\nimport sys\n\nMINIMUM_TF_VERSION = 2, 3, 0\nBLACKLISTED_TF_VERSIONS = []\n\n\ndef tf_version():\n \"\"\" Get the Tensorflow version.\n Returns\n tuple of (major, minor, patch).\n \"\"\"\n return tuple(map(int, tf.version.VERSION.split('-')[0].split('.')))\n\n\ndef tf_version_ok(minimum_tf_version=MINIMUM_TF_VERSION, blacklisted=BLACKLISTED_TF_VERSIONS):\n \"\"\" Check if the current Tensorflow version is higher than the minimum version.\n \"\"\"\n return tf_version() >= minimum_tf_version and tf_version() not in blacklisted\n\n\ndef assert_tf_version(minimum_tf_version=MINIMUM_TF_VERSION, blacklisted=BLACKLISTED_TF_VERSIONS):\n \"\"\" Assert that the Tensorflow version is up to date.\n \"\"\"\n detected = tf.version.VERSION\n required = '.'.join(map(str, minimum_tf_version))\n assert(tf_version_ok(minimum_tf_version, blacklisted)), 'You are using tensorflow version {}. The minimum required version is {} (blacklisted: {}).'.format(detected, required, blacklisted)\n\n\ndef check_tf_version():\n \"\"\" Check that the Tensorflow version is up to date. If it isn't, print an error message and exit the script.\n \"\"\"\n try:\n assert_tf_version()\n except AssertionError as e:\n print(e, file=sys.stderr)\n sys.exit(1)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/transform.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport numpy as np\n\nDEFAULT_PRNG = np.random\n\n\ndef colvec(*args):\n \"\"\" Create a numpy array representing a column vector. \"\"\"\n return np.array([args]).T\n\n\ndef transform_aabb(transform, aabb):\n \"\"\" Apply a transformation to an axis aligned bounding box.\n\n The result is a new AABB in the same coordinate system as the original AABB.\n The new AABB contains all corner points of the original AABB after applying the given transformation.\n\n Args\n transform: The transformation to apply.\n x1: The minimum x value of the AABB.\n y1: The minimum y value of the AABB.\n x2: The maximum x value of the AABB.\n y2: The maximum y value of the AABB.\n Returns\n The new AABB as tuple (x1, y1, x2, y2)\n \"\"\"\n x1, y1, x2, y2 = aabb\n # Transform all 4 corners of the AABB.\n points = transform.dot([\n [x1, x2, x1, x2],\n [y1, y2, y2, y1],\n [1, 1, 1, 1 ],\n ])\n\n # Extract the min and max corners again.\n min_corner = points.min(axis=1)\n max_corner = points.max(axis=1)\n\n return [min_corner[0], min_corner[1], max_corner[0], max_corner[1]]\n\n\ndef _random_vector(min, max, prng=DEFAULT_PRNG):\n \"\"\" Construct a random vector between min and max.\n Args\n min: the minimum value for each component\n max: the maximum value for each component\n \"\"\"\n min = np.array(min)\n max = np.array(max)\n assert min.shape == max.shape\n assert len(min.shape) == 1\n return prng.uniform(min, max)\n\n\ndef rotation(angle):\n \"\"\" Construct a homogeneous 2D rotation matrix.\n Args\n angle: the angle in radians\n Returns\n the rotation matrix as 3 by 3 numpy array\n \"\"\"\n return np.array([\n [np.cos(angle), -np.sin(angle), 0],\n [np.sin(angle), np.cos(angle), 0],\n [0, 0, 1]\n ])\n\n\ndef random_rotation(min, max, prng=DEFAULT_PRNG):\n \"\"\" Construct a random rotation between -max and max.\n Args\n min: a scalar for the minimum absolute angle in radians\n max: a scalar for the maximum absolute angle in radians\n prng: the pseudo-random number generator to use.\n Returns\n a homogeneous 3 by 3 rotation matrix\n \"\"\"\n return rotation(prng.uniform(min, max))\n\n\ndef translation(translation):\n \"\"\" Construct a homogeneous 2D translation matrix.\n # Arguments\n translation: the translation 2D vector\n # Returns\n the translation matrix as 3 by 3 numpy array\n \"\"\"\n return np.array([\n [1, 0, translation[0]],\n [0, 1, translation[1]],\n [0, 0, 1]\n ])\n\n\ndef random_translation(min, max, prng=DEFAULT_PRNG):\n \"\"\" Construct a random 2D translation between min and max.\n Args\n min: a 2D vector with the minimum translation for each dimension\n max: a 2D vector with the maximum translation for each dimension\n prng: the pseudo-random number generator to use.\n Returns\n a homogeneous 3 by 3 translation matrix\n \"\"\"\n return translation(_random_vector(min, max, prng))\n\n\ndef shear(angle):\n \"\"\" Construct a homogeneous 2D shear matrix.\n Args\n angle: the shear angle in radians\n Returns\n the shear matrix as 3 by 3 numpy array\n \"\"\"\n return np.array([\n [1, -np.sin(angle), 0],\n [0, np.cos(angle), 0],\n [0, 0, 1]\n ])\n\n\ndef random_shear(min, max, prng=DEFAULT_PRNG):\n \"\"\" Construct a random 2D shear matrix with shear angle between -max and max.\n Args\n min: the minimum shear angle in radians.\n max: the maximum shear angle in radians.\n prng: the pseudo-random number generator to use.\n Returns\n a homogeneous 3 by 3 shear matrix\n \"\"\"\n return shear(prng.uniform(min, max))\n\n\ndef scaling(factor):\n \"\"\" Construct a homogeneous 2D scaling matrix.\n Args\n factor: a 2D vector for X and Y scaling\n Returns\n the zoom matrix as 3 by 3 numpy array\n \"\"\"\n return np.array([\n [factor[0], 0, 0],\n [0, factor[1], 0],\n [0, 0, 1]\n ])\n\n\ndef random_scaling(min, max, prng=DEFAULT_PRNG):\n \"\"\" Construct a random 2D scale matrix between -max and max.\n Args\n min: a 2D vector containing the minimum scaling factor for X and Y.\n min: a 2D vector containing The maximum scaling factor for X and Y.\n prng: the pseudo-random number generator to use.\n Returns\n a homogeneous 3 by 3 scaling matrix\n \"\"\"\n return scaling(_random_vector(min, max, prng))\n\n\ndef random_flip(flip_x_chance, flip_y_chance, prng=DEFAULT_PRNG):\n \"\"\" Construct a transformation randomly containing X/Y flips (or not).\n Args\n flip_x_chance: The chance that the result will contain a flip along the X axis.\n flip_y_chance: The chance that the result will contain a flip along the Y axis.\n prng: The pseudo-random number generator to use.\n Returns\n a homogeneous 3 by 3 transformation matrix\n \"\"\"\n flip_x = prng.uniform(0, 1) < flip_x_chance\n flip_y = prng.uniform(0, 1) < flip_y_chance\n # 1 - 2 * bool gives 1 for False and -1 for True.\n return scaling((1 - 2 * flip_x, 1 - 2 * flip_y))\n\n\ndef change_transform_origin(transform, center):\n \"\"\" Create a new transform representing the same transformation,\n only with the origin of the linear part changed.\n Args\n transform: the transformation matrix\n center: the new origin of the transformation\n Returns\n translate(center) * transform * translate(-center)\n \"\"\"\n center = np.array(center)\n return np.linalg.multi_dot([translation(center), transform, translation(-center)])\n\n\ndef random_transform(\n min_rotation=0,\n max_rotation=0,\n min_translation=(0, 0),\n max_translation=(0, 0),\n min_shear=0,\n max_shear=0,\n min_scaling=(1, 1),\n max_scaling=(1, 1),\n flip_x_chance=0,\n flip_y_chance=0,\n prng=DEFAULT_PRNG\n):\n \"\"\" Create a random transformation.\n\n The transformation consists of the following operations in this order (from left to right):\n * rotation\n * translation\n * shear\n * scaling\n * flip x (if applied)\n * flip y (if applied)\n\n Note that by default, the data generators in `keras_retinanet.preprocessing.generators` interpret the translation\n as factor of the image size. So an X translation of 0.1 would translate the image by 10% of it's width.\n Set `relative_translation` to `False` in the `TransformParameters` of a data generator to have it interpret\n the translation directly as pixel distances instead.\n\n Args\n min_rotation: The minimum rotation in radians for the transform as scalar.\n max_rotation: The maximum rotation in radians for the transform as scalar.\n min_translation: The minimum translation for the transform as 2D column vector.\n max_translation: The maximum translation for the transform as 2D column vector.\n min_shear: The minimum shear angle for the transform in radians.\n max_shear: The maximum shear angle for the transform in radians.\n min_scaling: The minimum scaling for the transform as 2D column vector.\n max_scaling: The maximum scaling for the transform as 2D column vector.\n flip_x_chance: The chance (0 to 1) that a transform will contain a flip along X direction.\n flip_y_chance: The chance (0 to 1) that a transform will contain a flip along Y direction.\n prng: The pseudo-random number generator to use.\n \"\"\"\n return np.linalg.multi_dot([\n random_rotation(min_rotation, max_rotation, prng),\n random_translation(min_translation, max_translation, prng),\n random_shear(min_shear, max_shear, prng),\n random_scaling(min_scaling, max_scaling, prng),\n random_flip(flip_x_chance, flip_y_chance, prng)\n ])\n\n\ndef random_transform_generator(prng=None, **kwargs):\n \"\"\" Create a random transform generator.\n\n Uses a dedicated, newly created, properly seeded PRNG by default instead of the global DEFAULT_PRNG.\n\n The transformation consists of the following operations in this order (from left to right):\n * rotation\n * translation\n * shear\n * scaling\n * flip x (if applied)\n * flip y (if applied)\n\n Note that by default, the data generators in `keras_retinanet.preprocessing.generators` interpret the translation\n as factor of the image size. So an X translation of 0.1 would translate the image by 10% of it's width.\n Set `relative_translation` to `False` in the `TransformParameters` of a data generator to have it interpret\n the translation directly as pixel distances instead.\n\n Args\n min_rotation: The minimum rotation in radians for the transform as scalar.\n max_rotation: The maximum rotation in radians for the transform as scalar.\n min_translation: The minimum translation for the transform as 2D column vector.\n max_translation: The maximum translation for the transform as 2D column vector.\n min_shear: The minimum shear angle for the transform in radians.\n max_shear: The maximum shear angle for the transform in radians.\n min_scaling: The minimum scaling for the transform as 2D column vector.\n max_scaling: The maximum scaling for the transform as 2D column vector.\n flip_x_chance: The chance (0 to 1) that a transform will contain a flip along X direction.\n flip_y_chance: The chance (0 to 1) that a transform will contain a flip along Y direction.\n prng: The pseudo-random number generator to use.\n \"\"\"\n\n if prng is None:\n # RandomState automatically seeds using the best available method.\n prng = np.random.RandomState()\n\n while True:\n yield random_transform(prng=prng, **kwargs)\n" }, { "path": "imageai_tf_deprecated/Detection/keras_retinanet/utils/visualization.py", "content": "\"\"\"\nCopyright 2017-2018 Fizyr (https://fizyr.com)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\"\"\"\n\nimport cv2\nimport numpy as np\n\nfrom .colors import label_color\n\n\ndef draw_box(image, box, color, thickness=2):\n \"\"\" Draws a box on an image with a given color.\n\n # Arguments\n image : The image to draw on.\n box : A list of 4 elements (x1, y1, x2, y2).\n color : The color of the box.\n thickness : The thickness of the lines to draw a box with.\n \"\"\"\n b = np.array(box).astype(int)\n cv2.rectangle(image, (b[0], b[1]), (b[2], b[3]), color, thickness, cv2.LINE_AA)\n\n\ndef draw_caption(image, box, caption):\n \"\"\" Draws a caption above the box in an image.\n\n # Arguments\n image : The image to draw on.\n box : A list of 4 elements (x1, y1, x2, y2).\n caption : String containing the text to draw.\n \"\"\"\n b = np.array(box).astype(int)\n cv2.putText(image, caption, (b[0], b[1] - 10), cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 0), 2)\n cv2.putText(image, caption, (b[0], b[1] - 10), cv2.FONT_HERSHEY_PLAIN, 1, (255, 255, 255), 1)\n\n\ndef draw_boxes(image, boxes, color, thickness=2):\n \"\"\" Draws boxes on an image with a given color.\n\n # Arguments\n image : The image to draw on.\n boxes : A [N, 4] matrix (x1, y1, x2, y2).\n color : The color of the boxes.\n thickness : The thickness of the lines to draw boxes with.\n \"\"\"\n for b in boxes:\n draw_box(image, b, color, thickness=thickness)\n\n\ndef draw_detections(image, boxes, scores, labels, color=None, label_to_name=None, score_threshold=0.5):\n \"\"\" Draws detections in an image.\n\n # Arguments\n image : The image to draw on.\n boxes : A [N, 4] matrix (x1, y1, x2, y2).\n scores : A list of N classification scores.\n labels : A list of N labels.\n color : The color of the boxes. By default the color from keras_retinanet.utils.colors.label_color will be used.\n label_to_name : (optional) Functor for mapping a label to a name.\n score_threshold : Threshold used for determining what detections to draw.\n \"\"\"\n selection = np.where(scores > score_threshold)[0]\n\n for i in selection:\n c = color if color is not None else label_color(labels[i])\n draw_box(image, boxes[i, :], color=c)\n\n # draw labels\n caption = (label_to_name(labels[i]) if label_to_name else labels[i]) + ': {0:.2f}'.format(scores[i])\n draw_caption(image, boxes[i, :], caption)\n\n\ndef draw_annotations(image, annotations, color=(0, 255, 0), label_to_name=None):\n \"\"\" Draws annotations in an image.\n\n # Arguments\n image : The image to draw on.\n annotations : A [N, 5] matrix (x1, y1, x2, y2, label) or dictionary containing bboxes (shaped [N, 4]) and labels (shaped [N]).\n color : The color of the boxes. By default the color from keras_retinanet.utils.colors.label_color will be used.\n label_to_name : (optional) Functor for mapping a label to a name.\n \"\"\"\n if isinstance(annotations, np.ndarray):\n annotations = {'bboxes': annotations[:, :4], 'labels': annotations[:, 4]}\n\n assert('bboxes' in annotations)\n assert('labels' in annotations)\n assert(annotations['bboxes'].shape[0] == annotations['labels'].shape[0])\n\n for i in range(annotations['bboxes'].shape[0]):\n label = annotations['labels'][i]\n c = color if color is not None else label_color(label)\n caption = '{}'.format(label_to_name(label) if label_to_name else label)\n draw_caption(image, annotations['bboxes'][i], caption)\n draw_box(image, annotations['bboxes'][i], color=c)\n" }, { "path": "imageai_tf_deprecated/Prediction/Custom/__init__.py", "content": "from ...Classification.Custom import ClassificationModelTrainer, CustomImageClassification\n\n\n\n\n\nclass ModelTraining(ClassificationModelTrainer):\n \"\"\"\n Deprecated! \n Replaced with 'imageai.Classification.Custom.ClassificationModelTrainer'\n \"\"\"\n def __call__(self):\n None\n\nclass CustomImagePrediction(CustomImageClassification):\n \"\"\"\n Deprecated! \n Replaced with 'imageai.Classification.Custom.CustomImageClassification'\n \"\"\"\n\n def __call__(self):\n None" }, { "path": "imageai_tf_deprecated/Prediction/Custom/custom_utils.py", "content": "import json\n\nCLASS_INDEX = None\n\n\n\ndef preprocess_input(x):\n \"\"\"Preprocesses a tensor encoding a batch of images.\n\n # Arguments\n x: input Numpy tensor, 4D.\n data_format: data format of the image tensor.\n\n # Returns\n Preprocessed tensor.\n \"\"\"\n\n\n # 'RGB'->'BGR'\n x *= (1./255)\n\n return x\n\n\ndef decode_predictions(preds, top=5, model_json=\"\"):\n\n\n global CLASS_INDEX\n\n if CLASS_INDEX is None:\n CLASS_INDEX = json.load(open(model_json))\n results = []\n for pred in preds:\n top_indices = pred.argsort()[-top:][::-1]\n for i in top_indices:\n each_result = []\n each_result.append(CLASS_INDEX[str(i)])\n each_result.append(pred[i])\n results.append(each_result)\n\n return results" }, { "path": "imageai_tf_deprecated/Prediction/__init__.py", "content": "from ..Classification import ImageClassification\nfrom matplotlib.cbook import deprecated\n\n\nclass ImagePrediction(ImageClassification):\n \"\"\"\n Deprecated! \n Replaced with 'imageai.Classification.ImageClassification'\n \"\"\"\n\n def __call__(self):\n None" }, { "path": "imageai_tf_deprecated/Prediction/imagenet_utils.py", "content": "CLASS_INDEX = None\n\n\n\ndef preprocess_input(x):\n \"\"\"Preprocesses a tensor encoding a batch of images.\n\n # Arguments\n x: input Numpy tensor, 4D.\n data_format: data format of the image tensor.\n\n # Returns\n Preprocessed tensor.\n \"\"\"\n\n\n # 'RGB'->'BGR'\n x = x[..., ::-1]\n # Zero-center by mean pixel\n x[..., 0] -= 103.939\n x[..., 1] -= 116.779\n x[..., 2] -= 123.68\n\n return x\n\n\ndef decode_predictions(preds, top=5):\n \"\"\"Decodes the prediction of an ImageNet model.\n\n # Arguments\n preds: Numpy tensor encoding a batch of predictions.\n top: integer, how many top-guesses to return.\n\n # Returns\n A list of lists of top class prediction tuples\n `(class_name, class_description, score)`.\n One list of tuples per sample in batch input.\n\n # Raises\n ValueError: in case of invalid shape of the `pred` array\n (must be 2D).\n \"\"\"\n global CLASS_INDEX\n if len(preds.shape) != 2 or preds.shape[1] != 1000:\n raise ValueError('`decode_predictions` expects '\n 'a batch of predictions '\n '(i.e. a 2D array of shape (samples, 1000)). '\n 'Found array with shape: ' + str(preds.shape))\n if CLASS_INDEX is None:\n CLASS_INDEX = {\"0\": [\"n01440764\", \"tench\"], \"1\": [\"n01443537\", \"goldfish\"], \"2\": [\"n01484850\", \"great_white_shark\"], \"3\": [\"n01491361\", \"tiger_shark\"], \"4\": [\"n01494475\", \"hammerhead\"], \"5\": [\"n01496331\", \"electric_ray\"], \"6\": [\"n01498041\", \"stingray\"], \"7\": [\"n01514668\", \"cock\"], \"8\": [\"n01514859\", \"hen\"], \"9\": [\"n01518878\", \"ostrich\"], \"10\": [\"n01530575\", \"brambling\"], \"11\": [\"n01531178\", \"goldfinch\"], \"12\": [\"n01532829\", \"house_finch\"], \"13\": [\"n01534433\", \"junco\"], \"14\": [\"n01537544\", \"indigo_bunting\"], \"15\": [\"n01558993\", \"robin\"], \"16\": [\"n01560419\", \"bulbul\"], \"17\": [\"n01580077\", \"jay\"], \"18\": [\"n01582220\", \"magpie\"], \"19\": [\"n01592084\", \"chickadee\"], \"20\": [\"n01601694\", \"water_ouzel\"], \"21\": [\"n01608432\", \"kite\"], \"22\": [\"n01614925\", \"bald_eagle\"], \"23\": [\"n01616318\", \"vulture\"], \"24\": [\"n01622779\", \"great_grey_owl\"], \"25\": [\"n01629819\", \"European_fire_salamander\"], \"26\": [\"n01630670\", \"common_newt\"], \"27\": [\"n01631663\", \"eft\"], \"28\": [\"n01632458\", \"spotted_salamander\"], \"29\": [\"n01632777\", \"axolotl\"], \"30\": [\"n01641577\", \"bullfrog\"], \"31\": [\"n01644373\", \"tree_frog\"], \"32\": [\"n01644900\", \"tailed_frog\"], \"33\": [\"n01664065\", \"loggerhead\"], \"34\": [\"n01665541\", \"leatherback_turtle\"], \"35\": [\"n01667114\", \"mud_turtle\"], \"36\": [\"n01667778\", \"terrapin\"], \"37\": [\"n01669191\", \"box_turtle\"], \"38\": [\"n01675722\", \"banded_gecko\"], \"39\": [\"n01677366\", \"common_iguana\"], \"40\": [\"n01682714\", \"American_chameleon\"], \"41\": [\"n01685808\", \"whiptail\"], \"42\": [\"n01687978\", \"agama\"], \"43\": [\"n01688243\", \"frilled_lizard\"], \"44\": [\"n01689811\", \"alligator_lizard\"], \"45\": [\"n01692333\", \"Gila_monster\"], \"46\": [\"n01693334\", \"green_lizard\"], \"47\": [\"n01694178\", \"African_chameleon\"], \"48\": [\"n01695060\", \"Komodo_dragon\"], \"49\": [\"n01697457\", \"African_crocodile\"], \"50\": [\"n01698640\", \"American_alligator\"], \"51\": [\"n01704323\", \"triceratops\"], \"52\": [\"n01728572\", \"thunder_snake\"], \"53\": [\"n01728920\", \"ringneck_snake\"], \"54\": [\"n01729322\", \"hognose_snake\"], \"55\": [\"n01729977\", \"green_snake\"], \"56\": [\"n01734418\", \"king_snake\"], \"57\": [\"n01735189\", \"garter_snake\"], \"58\": [\"n01737021\", \"water_snake\"], \"59\": [\"n01739381\", \"vine_snake\"], \"60\": [\"n01740131\", \"night_snake\"], \"61\": [\"n01742172\", \"boa_constrictor\"], \"62\": [\"n01744401\", \"rock_python\"], \"63\": [\"n01748264\", \"Indian_cobra\"], \"64\": [\"n01749939\", \"green_mamba\"], \"65\": [\"n01751748\", \"sea_snake\"], \"66\": [\"n01753488\", \"horned_viper\"], \"67\": [\"n01755581\", \"diamondback\"], \"68\": [\"n01756291\", \"sidewinder\"], \"69\": [\"n01768244\", \"trilobite\"], \"70\": [\"n01770081\", \"harvestman\"], \"71\": [\"n01770393\", \"scorpion\"], \"72\": [\"n01773157\", \"black_and_gold_garden_spider\"], \"73\": [\"n01773549\", \"barn_spider\"], \"74\": [\"n01773797\", \"garden_spider\"], \"75\": [\"n01774384\", \"black_widow\"], \"76\": [\"n01774750\", \"tarantula\"], \"77\": [\"n01775062\", \"wolf_spider\"], \"78\": [\"n01776313\", \"tick\"], \"79\": [\"n01784675\", \"centipede\"], \"80\": [\"n01795545\", \"black_grouse\"], \"81\": [\"n01796340\", \"ptarmigan\"], \"82\": [\"n01797886\", \"ruffed_grouse\"], \"83\": [\"n01798484\", \"prairie_chicken\"], \"84\": [\"n01806143\", \"peacock\"], \"85\": [\"n01806567\", \"quail\"], \"86\": [\"n01807496\", \"partridge\"], \"87\": [\"n01817953\", \"African_grey\"], \"88\": [\"n01818515\", \"macaw\"], \"89\": [\"n01819313\", \"sulphur-crested_cockatoo\"], \"90\": [\"n01820546\", \"lorikeet\"], \"91\": [\"n01824575\", \"coucal\"], \"92\": [\"n01828970\", \"bee_eater\"], \"93\": [\"n01829413\", \"hornbill\"], \"94\": [\"n01833805\", \"hummingbird\"], \"95\": [\"n01843065\", \"jacamar\"], \"96\": [\"n01843383\", \"toucan\"], \"97\": [\"n01847000\", \"drake\"], \"98\": [\"n01855032\", \"red-breasted_merganser\"], \"99\": [\"n01855672\", \"goose\"], \"100\": [\"n01860187\", \"black_swan\"], \"101\": [\"n01871265\", \"tusker\"], \"102\": [\"n01872401\", \"echidna\"], \"103\": [\"n01873310\", \"platypus\"], \"104\": [\"n01877812\", \"wallaby\"], \"105\": [\"n01882714\", \"koala\"], \"106\": [\"n01883070\", \"wombat\"], \"107\": [\"n01910747\", \"jellyfish\"], \"108\": [\"n01914609\", \"sea_anemone\"], \"109\": [\"n01917289\", \"brain_coral\"], \"110\": [\"n01924916\", \"flatworm\"], \"111\": [\"n01930112\", \"nematode\"], \"112\": [\"n01943899\", \"conch\"], \"113\": [\"n01944390\", \"snail\"], \"114\": [\"n01945685\", \"slug\"], \"115\": [\"n01950731\", \"sea_slug\"], \"116\": [\"n01955084\", \"chiton\"], \"117\": [\"n01968897\", \"chambered_nautilus\"], \"118\": [\"n01978287\", \"Dungeness_crab\"], \"119\": [\"n01978455\", \"rock_crab\"], \"120\": [\"n01980166\", \"fiddler_crab\"], \"121\": [\"n01981276\", \"king_crab\"], \"122\": [\"n01983481\", \"American_lobster\"], \"123\": [\"n01984695\", \"spiny_lobster\"], \"124\": [\"n01985128\", \"crayfish\"], \"125\": [\"n01986214\", \"hermit_crab\"], \"126\": [\"n01990800\", \"isopod\"], \"127\": [\"n02002556\", \"white_stork\"], \"128\": [\"n02002724\", \"black_stork\"], \"129\": [\"n02006656\", \"spoonbill\"], \"130\": [\"n02007558\", \"flamingo\"], \"131\": [\"n02009229\", \"little_blue_heron\"], \"132\": [\"n02009912\", \"American_egret\"], \"133\": [\"n02011460\", \"bittern\"], \"134\": [\"n02012849\", \"crane\"], \"135\": [\"n02013706\", \"limpkin\"], \"136\": [\"n02017213\", \"European_gallinule\"], \"137\": [\"n02018207\", \"American_coot\"], \"138\": [\"n02018795\", \"bustard\"], \"139\": [\"n02025239\", \"ruddy_turnstone\"], \"140\": [\"n02027492\", \"red-backed_sandpiper\"], \"141\": [\"n02028035\", \"redshank\"], \"142\": [\"n02033041\", \"dowitcher\"], \"143\": [\"n02037110\", \"oystercatcher\"], \"144\": [\"n02051845\", \"pelican\"], \"145\": [\"n02056570\", \"king_penguin\"], \"146\": [\"n02058221\", \"albatross\"], \"147\": [\"n02066245\", \"grey_whale\"], \"148\": [\"n02071294\", \"killer_whale\"], \"149\": [\"n02074367\", \"dugong\"], \"150\": [\"n02077923\", \"sea_lion\"], \"151\": [\"n02085620\", \"Chihuahua\"], \"152\": [\"n02085782\", \"Japanese_spaniel\"], \"153\": [\"n02085936\", \"Maltese_dog\"], \"154\": [\"n02086079\", \"Pekinese\"], \"155\": [\"n02086240\", \"Shih-Tzu\"], \"156\": [\"n02086646\", \"Blenheim_spaniel\"], \"157\": [\"n02086910\", \"papillon\"], \"158\": [\"n02087046\", \"toy_terrier\"], \"159\": [\"n02087394\", \"Rhodesian_ridgeback\"], \"160\": [\"n02088094\", \"Afghan_hound\"], \"161\": [\"n02088238\", \"basset\"], \"162\": [\"n02088364\", \"beagle\"], \"163\": [\"n02088466\", \"bloodhound\"], \"164\": [\"n02088632\", \"bluetick\"], \"165\": [\"n02089078\", \"black-and-tan_coonhound\"], \"166\": [\"n02089867\", \"Walker_hound\"], \"167\": [\"n02089973\", \"English_foxhound\"], \"168\": [\"n02090379\", \"redbone\"], \"169\": [\"n02090622\", \"borzoi\"], \"170\": [\"n02090721\", \"Irish_wolfhound\"], \"171\": [\"n02091032\", \"Italian_greyhound\"], \"172\": [\"n02091134\", \"whippet\"], \"173\": [\"n02091244\", \"Ibizan_hound\"], \"174\": [\"n02091467\", \"Norwegian_elkhound\"], \"175\": [\"n02091635\", \"otterhound\"], \"176\": [\"n02091831\", \"Saluki\"], \"177\": [\"n02092002\", \"Scottish_deerhound\"], \"178\": [\"n02092339\", \"Weimaraner\"], \"179\": [\"n02093256\", \"Staffordshire_bullterrier\"], \"180\": [\"n02093428\", \"American_Staffordshire_terrier\"], \"181\": [\"n02093647\", \"Bedlington_terrier\"], \"182\": [\"n02093754\", \"Border_terrier\"], \"183\": [\"n02093859\", \"Kerry_blue_terrier\"], \"184\": [\"n02093991\", \"Irish_terrier\"], \"185\": [\"n02094114\", \"Norfolk_terrier\"], \"186\": [\"n02094258\", \"Norwich_terrier\"], \"187\": [\"n02094433\", \"Yorkshire_terrier\"], \"188\": [\"n02095314\", \"wire-haired_fox_terrier\"], \"189\": [\"n02095570\", \"Lakeland_terrier\"], \"190\": [\"n02095889\", \"Sealyham_terrier\"], \"191\": [\"n02096051\", \"Airedale\"], \"192\": [\"n02096177\", \"cairn\"], \"193\": [\"n02096294\", \"Australian_terrier\"], \"194\": [\"n02096437\", \"Dandie_Dinmont\"], \"195\": [\"n02096585\", \"Boston_bull\"], \"196\": [\"n02097047\", \"miniature_schnauzer\"], \"197\": [\"n02097130\", \"giant_schnauzer\"], \"198\": [\"n02097209\", \"standard_schnauzer\"], \"199\": [\"n02097298\", \"Scotch_terrier\"], \"200\": [\"n02097474\", \"Tibetan_terrier\"], \"201\": [\"n02097658\", \"silky_terrier\"], \"202\": [\"n02098105\", \"soft-coated_wheaten_terrier\"], \"203\": [\"n02098286\", \"West_Highland_white_terrier\"], \"204\": [\"n02098413\", \"Lhasa\"], \"205\": [\"n02099267\", \"flat-coated_retriever\"], \"206\": [\"n02099429\", \"curly-coated_retriever\"], \"207\": [\"n02099601\", \"golden_retriever\"], \"208\": [\"n02099712\", \"Labrador_retriever\"], \"209\": [\"n02099849\", \"Chesapeake_Bay_retriever\"], \"210\": [\"n02100236\", \"German_short-haired_pointer\"], \"211\": [\"n02100583\", \"vizsla\"], \"212\": [\"n02100735\", \"English_setter\"], \"213\": [\"n02100877\", \"Irish_setter\"], \"214\": [\"n02101006\", \"Gordon_setter\"], \"215\": [\"n02101388\", \"Brittany_spaniel\"], \"216\": [\"n02101556\", \"clumber\"], \"217\": [\"n02102040\", \"English_springer\"], \"218\": [\"n02102177\", \"Welsh_springer_spaniel\"], \"219\": [\"n02102318\", \"cocker_spaniel\"], \"220\": [\"n02102480\", \"Sussex_spaniel\"], \"221\": [\"n02102973\", \"Irish_water_spaniel\"], \"222\": [\"n02104029\", \"kuvasz\"], \"223\": [\"n02104365\", \"schipperke\"], \"224\": [\"n02105056\", \"groenendael\"], \"225\": [\"n02105162\", \"malinois\"], \"226\": [\"n02105251\", \"briard\"], \"227\": [\"n02105412\", \"kelpie\"], \"228\": [\"n02105505\", \"komondor\"], \"229\": [\"n02105641\", \"Old_English_sheepdog\"], \"230\": [\"n02105855\", \"Shetland_sheepdog\"], \"231\": [\"n02106030\", \"collie\"], \"232\": [\"n02106166\", \"Border_collie\"], \"233\": [\"n02106382\", \"Bouvier_des_Flandres\"], \"234\": [\"n02106550\", \"Rottweiler\"], \"235\": [\"n02106662\", \"German_shepherd\"], \"236\": [\"n02107142\", \"Doberman\"], \"237\": [\"n02107312\", \"miniature_pinscher\"], \"238\": [\"n02107574\", \"Greater_Swiss_Mountain_dog\"], \"239\": [\"n02107683\", \"Bernese_mountain_dog\"], \"240\": [\"n02107908\", \"Appenzeller\"], \"241\": [\"n02108000\", \"EntleBucher\"], \"242\": [\"n02108089\", \"boxer\"], \"243\": [\"n02108422\", \"bull_mastiff\"], \"244\": [\"n02108551\", \"Tibetan_mastiff\"], \"245\": [\"n02108915\", \"French_bulldog\"], \"246\": [\"n02109047\", \"Great_Dane\"], \"247\": [\"n02109525\", \"Saint_Bernard\"], \"248\": [\"n02109961\", \"Eskimo_dog\"], \"249\": [\"n02110063\", \"malamute\"], \"250\": [\"n02110185\", \"Siberian_husky\"], \"251\": [\"n02110341\", \"dalmatian\"], \"252\": [\"n02110627\", \"affenpinscher\"], \"253\": [\"n02110806\", \"basenji\"], \"254\": [\"n02110958\", \"pug\"], \"255\": [\"n02111129\", \"Leonberg\"], \"256\": [\"n02111277\", \"Newfoundland\"], \"257\": [\"n02111500\", \"Great_Pyrenees\"], \"258\": [\"n02111889\", \"Samoyed\"], \"259\": [\"n02112018\", \"Pomeranian\"], \"260\": [\"n02112137\", \"chow\"], \"261\": [\"n02112350\", \"keeshond\"], \"262\": [\"n02112706\", \"Brabancon_griffon\"], \"263\": [\"n02113023\", \"Pembroke\"], \"264\": [\"n02113186\", \"Cardigan\"], \"265\": [\"n02113624\", \"toy_poodle\"], \"266\": [\"n02113712\", \"miniature_poodle\"], \"267\": [\"n02113799\", \"standard_poodle\"], \"268\": [\"n02113978\", \"Mexican_hairless\"], \"269\": [\"n02114367\", \"timber_wolf\"], \"270\": [\"n02114548\", \"white_wolf\"], \"271\": [\"n02114712\", \"red_wolf\"], \"272\": [\"n02114855\", \"coyote\"], \"273\": [\"n02115641\", \"dingo\"], \"274\": [\"n02115913\", \"dhole\"], \"275\": [\"n02116738\", \"African_hunting_dog\"], \"276\": [\"n02117135\", \"hyena\"], \"277\": [\"n02119022\", \"red_fox\"], \"278\": [\"n02119789\", \"kit_fox\"], \"279\": [\"n02120079\", \"Arctic_fox\"], \"280\": [\"n02120505\", \"grey_fox\"], \"281\": [\"n02123045\", \"tabby\"], \"282\": [\"n02123159\", \"tiger_cat\"], \"283\": [\"n02123394\", \"Persian_cat\"], \"284\": [\"n02123597\", \"Siamese_cat\"], \"285\": [\"n02124075\", \"Egyptian_cat\"], \"286\": [\"n02125311\", \"cougar\"], \"287\": [\"n02127052\", \"lynx\"], \"288\": [\"n02128385\", \"leopard\"], \"289\": [\"n02128757\", \"snow_leopard\"], \"290\": [\"n02128925\", \"jaguar\"], \"291\": [\"n02129165\", \"lion\"], \"292\": [\"n02129604\", \"tiger\"], \"293\": [\"n02130308\", \"cheetah\"], \"294\": [\"n02132136\", \"brown_bear\"], \"295\": [\"n02133161\", \"American_black_bear\"], \"296\": [\"n02134084\", \"ice_bear\"], \"297\": [\"n02134418\", \"sloth_bear\"], \"298\": [\"n02137549\", \"mongoose\"], \"299\": [\"n02138441\", \"meerkat\"], \"300\": [\"n02165105\", \"tiger_beetle\"], \"301\": [\"n02165456\", \"ladybug\"], \"302\": [\"n02167151\", \"ground_beetle\"], \"303\": [\"n02168699\", \"long-horned_beetle\"], \"304\": [\"n02169497\", \"leaf_beetle\"], \"305\": [\"n02172182\", \"dung_beetle\"], \"306\": [\"n02174001\", \"rhinoceros_beetle\"], \"307\": [\"n02177972\", \"weevil\"], \"308\": [\"n02190166\", \"fly\"], \"309\": [\"n02206856\", \"bee\"], \"310\": [\"n02219486\", \"ant\"], \"311\": [\"n02226429\", \"grasshopper\"], \"312\": [\"n02229544\", \"cricket\"], \"313\": [\"n02231487\", \"walking_stick\"], \"314\": [\"n02233338\", \"cockroach\"], \"315\": [\"n02236044\", \"mantis\"], \"316\": [\"n02256656\", \"cicada\"], \"317\": [\"n02259212\", \"leafhopper\"], \"318\": [\"n02264363\", \"lacewing\"], \"319\": [\"n02268443\", \"dragonfly\"], \"320\": [\"n02268853\", \"damselfly\"], \"321\": [\"n02276258\", \"admiral\"], \"322\": [\"n02277742\", \"ringlet\"], \"323\": [\"n02279972\", \"monarch\"], \"324\": [\"n02280649\", \"cabbage_butterfly\"], \"325\": [\"n02281406\", \"sulphur_butterfly\"], \"326\": [\"n02281787\", \"lycaenid\"], \"327\": [\"n02317335\", \"starfish\"], \"328\": [\"n02319095\", \"sea_urchin\"], \"329\": [\"n02321529\", \"sea_cucumber\"], \"330\": [\"n02325366\", \"wood_rabbit\"], \"331\": [\"n02326432\", \"hare\"], \"332\": [\"n02328150\", \"Angora\"], \"333\": [\"n02342885\", \"hamster\"], \"334\": [\"n02346627\", \"porcupine\"], \"335\": [\"n02356798\", \"fox_squirrel\"], \"336\": [\"n02361337\", \"marmot\"], \"337\": [\"n02363005\", \"beaver\"], \"338\": [\"n02364673\", \"guinea_pig\"], \"339\": [\"n02389026\", \"sorrel\"], \"340\": [\"n02391049\", \"zebra\"], \"341\": [\"n02395406\", \"hog\"], \"342\": [\"n02396427\", \"wild_boar\"], \"343\": [\"n02397096\", \"warthog\"], \"344\": [\"n02398521\", \"hippopotamus\"], \"345\": [\"n02403003\", \"ox\"], \"346\": [\"n02408429\", \"water_buffalo\"], \"347\": [\"n02410509\", \"bison\"], \"348\": [\"n02412080\", \"ram\"], \"349\": [\"n02415577\", \"bighorn\"], \"350\": [\"n02417914\", \"ibex\"], \"351\": [\"n02422106\", \"hartebeest\"], \"352\": [\"n02422699\", \"impala\"], \"353\": [\"n02423022\", \"gazelle\"], \"354\": [\"n02437312\", \"Arabian_camel\"], \"355\": [\"n02437616\", \"llama\"], \"356\": [\"n02441942\", \"weasel\"], \"357\": [\"n02442845\", \"mink\"], \"358\": [\"n02443114\", \"polecat\"], \"359\": [\"n02443484\", \"black-footed_ferret\"], \"360\": [\"n02444819\", \"otter\"], \"361\": [\"n02445715\", \"skunk\"], \"362\": [\"n02447366\", \"badger\"], \"363\": [\"n02454379\", \"armadillo\"], \"364\": [\"n02457408\", \"three-toed_sloth\"], \"365\": [\"n02480495\", \"orangutan\"], \"366\": [\"n02480855\", \"gorilla\"], \"367\": [\"n02481823\", \"chimpanzee\"], \"368\": [\"n02483362\", \"gibbon\"], \"369\": [\"n02483708\", \"siamang\"], \"370\": [\"n02484975\", \"guenon\"], \"371\": [\"n02486261\", \"patas\"], \"372\": [\"n02486410\", \"baboon\"], \"373\": [\"n02487347\", \"macaque\"], \"374\": [\"n02488291\", \"langur\"], \"375\": [\"n02488702\", \"colobus\"], \"376\": [\"n02489166\", \"proboscis_monkey\"], \"377\": [\"n02490219\", \"marmoset\"], \"378\": [\"n02492035\", \"capuchin\"], \"379\": [\"n02492660\", \"howler_monkey\"], \"380\": [\"n02493509\", \"titi\"], \"381\": [\"n02493793\", \"spider_monkey\"], \"382\": [\"n02494079\", \"squirrel_monkey\"], \"383\": [\"n02497673\", \"Madagascar_cat\"], \"384\": [\"n02500267\", \"indri\"], \"385\": [\"n02504013\", \"Indian_elephant\"], \"386\": [\"n02504458\", \"African_elephant\"], \"387\": [\"n02509815\", \"lesser_panda\"], \"388\": [\"n02510455\", \"giant_panda\"], \"389\": [\"n02514041\", \"barracouta\"], \"390\": [\"n02526121\", \"eel\"], \"391\": [\"n02536864\", \"coho\"], \"392\": [\"n02606052\", \"rock_beauty\"], \"393\": [\"n02607072\", \"anemone_fish\"], \"394\": [\"n02640242\", \"sturgeon\"], \"395\": [\"n02641379\", \"gar\"], \"396\": [\"n02643566\", \"lionfish\"], \"397\": [\"n02655020\", \"puffer\"], \"398\": [\"n02666196\", \"abacus\"], \"399\": [\"n02667093\", \"abaya\"], \"400\": [\"n02669723\", \"academic_gown\"], \"401\": [\"n02672831\", \"accordion\"], \"402\": [\"n02676566\", \"acoustic_guitar\"], \"403\": [\"n02687172\", \"aircraft_carrier\"], \"404\": [\"n02690373\", \"airliner\"], \"405\": [\"n02692877\", \"airship\"], \"406\": [\"n02699494\", \"altar\"], \"407\": [\"n02701002\", \"ambulance\"], \"408\": [\"n02704792\", \"amphibian\"], \"409\": [\"n02708093\", \"analog_clock\"], \"410\": [\"n02727426\", \"apiary\"], \"411\": [\"n02730930\", \"apron\"], \"412\": [\"n02747177\", \"ashcan\"], \"413\": [\"n02749479\", \"assault_rifle\"], \"414\": [\"n02769748\", \"backpack\"], \"415\": [\"n02776631\", \"bakery\"], \"416\": [\"n02777292\", \"balance_beam\"], \"417\": [\"n02782093\", \"balloon\"], \"418\": [\"n02783161\", \"ballpoint\"], \"419\": [\"n02786058\", \"Band_Aid\"], \"420\": [\"n02787622\", \"banjo\"], \"421\": [\"n02788148\", \"bannister\"], \"422\": [\"n02790996\", \"barbell\"], \"423\": [\"n02791124\", \"barber_chair\"], \"424\": [\"n02791270\", \"barbershop\"], \"425\": [\"n02793495\", \"barn\"], \"426\": [\"n02794156\", \"barometer\"], \"427\": [\"n02795169\", \"barrel\"], \"428\": [\"n02797295\", \"barrow\"], \"429\": [\"n02799071\", \"baseball\"], \"430\": [\"n02802426\", \"basketball\"], \"431\": [\"n02804414\", \"bassinet\"], \"432\": [\"n02804610\", \"bassoon\"], \"433\": [\"n02807133\", \"bathing_cap\"], \"434\": [\"n02808304\", \"bath_towel\"], \"435\": [\"n02808440\", \"bathtub\"], \"436\": [\"n02814533\", \"beach_wagon\"], \"437\": [\"n02814860\", \"beacon\"], \"438\": [\"n02815834\", \"beaker\"], \"439\": [\"n02817516\", \"bearskin\"], \"440\": [\"n02823428\", \"beer_bottle\"], \"441\": [\"n02823750\", \"beer_glass\"], \"442\": [\"n02825657\", \"bell_cote\"], \"443\": [\"n02834397\", \"bib\"], \"444\": [\"n02835271\", \"bicycle-built-for-two\"], \"445\": [\"n02837789\", \"bikini\"], \"446\": [\"n02840245\", \"binder\"], \"447\": [\"n02841315\", \"binoculars\"], \"448\": [\"n02843684\", \"birdhouse\"], \"449\": [\"n02859443\", \"boathouse\"], \"450\": [\"n02860847\", \"bobsled\"], \"451\": [\"n02865351\", \"bolo_tie\"], \"452\": [\"n02869837\", \"bonnet\"], \"453\": [\"n02870880\", \"bookcase\"], \"454\": [\"n02871525\", \"bookshop\"], \"455\": [\"n02877765\", \"bottlecap\"], \"456\": [\"n02879718\", \"bow\"], \"457\": [\"n02883205\", \"bow_tie\"], \"458\": [\"n02892201\", \"brass\"], \"459\": [\"n02892767\", \"brassiere\"], \"460\": [\"n02894605\", \"breakwater\"], \"461\": [\"n02895154\", \"breastplate\"], \"462\": [\"n02906734\", \"broom\"], \"463\": [\"n02909870\", \"bucket\"], \"464\": [\"n02910353\", \"buckle\"], \"465\": [\"n02916936\", \"bulletproof_vest\"], \"466\": [\"n02917067\", \"bullet_train\"], \"467\": [\"n02927161\", \"butcher_shop\"], \"468\": [\"n02930766\", \"cab\"], \"469\": [\"n02939185\", \"caldron\"], \"470\": [\"n02948072\", \"candle\"], \"471\": [\"n02950826\", \"cannon\"], \"472\": [\"n02951358\", \"canoe\"], \"473\": [\"n02951585\", \"can_opener\"], \"474\": [\"n02963159\", \"cardigan\"], \"475\": [\"n02965783\", \"car_mirror\"], \"476\": [\"n02966193\", \"carousel\"], \"477\": [\"n02966687\", \"carpenter's_kit\"], \"478\": [\"n02971356\", \"carton\"], \"479\": [\"n02974003\", \"car_wheel\"], \"480\": [\"n02977058\", \"cash_machine\"], \"481\": [\"n02978881\", \"cassette\"], \"482\": [\"n02979186\", \"cassette_player\"], \"483\": [\"n02980441\", \"castle\"], \"484\": [\"n02981792\", \"catamaran\"], \"485\": [\"n02988304\", \"CD_player\"], \"486\": [\"n02992211\", \"cello\"], \"487\": [\"n02992529\", \"cellular_telephone\"], \"488\": [\"n02999410\", \"chain\"], \"489\": [\"n03000134\", \"chainlink_fence\"], \"490\": [\"n03000247\", \"chain_mail\"], \"491\": [\"n03000684\", \"chain_saw\"], \"492\": [\"n03014705\", \"chest\"], \"493\": [\"n03016953\", \"chiffonier\"], \"494\": [\"n03017168\", \"chime\"], \"495\": [\"n03018349\", \"china_cabinet\"], \"496\": [\"n03026506\", \"Christmas_stocking\"], \"497\": [\"n03028079\", \"church\"], \"498\": [\"n03032252\", \"cinema\"], \"499\": [\"n03041632\", \"cleaver\"], \"500\": [\"n03042490\", \"cliff_dwelling\"], \"501\": [\"n03045698\", \"cloak\"], \"502\": [\"n03047690\", \"clog\"], \"503\": [\"n03062245\", \"cocktail_shaker\"], \"504\": [\"n03063599\", \"coffee_mug\"], \"505\": [\"n03063689\", \"coffeepot\"], \"506\": [\"n03065424\", \"coil\"], \"507\": [\"n03075370\", \"combination_lock\"], \"508\": [\"n03085013\", \"computer_keyboard\"], \"509\": [\"n03089624\", \"confectionery\"], \"510\": [\"n03095699\", \"container_ship\"], \"511\": [\"n03100240\", \"convertible\"], \"512\": [\"n03109150\", \"corkscrew\"], \"513\": [\"n03110669\", \"cornet\"], \"514\": [\"n03124043\", \"cowboy_boot\"], \"515\": [\"n03124170\", \"cowboy_hat\"], \"516\": [\"n03125729\", \"cradle\"], \"517\": [\"n03126707\", \"crane\"], \"518\": [\"n03127747\", \"crash_helmet\"], \"519\": [\"n03127925\", \"crate\"], \"520\": [\"n03131574\", \"crib\"], \"521\": [\"n03133878\", \"Crock_Pot\"], \"522\": [\"n03134739\", \"croquet_ball\"], \"523\": [\"n03141823\", \"crutch\"], \"524\": [\"n03146219\", \"cuirass\"], \"525\": [\"n03160309\", \"dam\"], \"526\": [\"n03179701\", \"desk\"], \"527\": [\"n03180011\", \"desktop_computer\"], \"528\": [\"n03187595\", \"dial_telephone\"], \"529\": [\"n03188531\", \"diaper\"], \"530\": [\"n03196217\", \"digital_clock\"], \"531\": [\"n03197337\", \"digital_watch\"], \"532\": [\"n03201208\", \"dining_table\"], \"533\": [\"n03207743\", \"dishrag\"], \"534\": [\"n03207941\", \"dishwasher\"], \"535\": [\"n03208938\", \"disk_brake\"], \"536\": [\"n03216828\", \"dock\"], \"537\": [\"n03218198\", \"dogsled\"], \"538\": [\"n03220513\", \"dome\"], \"539\": [\"n03223299\", \"doormat\"], \"540\": [\"n03240683\", \"drilling_platform\"], \"541\": [\"n03249569\", \"drum\"], \"542\": [\"n03250847\", \"drumstick\"], \"543\": [\"n03255030\", \"dumbbell\"], \"544\": [\"n03259280\", \"Dutch_oven\"], \"545\": [\"n03271574\", \"electric_fan\"], \"546\": [\"n03272010\", \"electric_guitar\"], \"547\": [\"n03272562\", \"electric_locomotive\"], \"548\": [\"n03290653\", \"entertainment_center\"], \"549\": [\"n03291819\", \"envelope\"], \"550\": [\"n03297495\", \"espresso_maker\"], \"551\": [\"n03314780\", \"face_powder\"], \"552\": [\"n03325584\", \"feather_boa\"], \"553\": [\"n03337140\", \"file\"], \"554\": [\"n03344393\", \"fireboat\"], \"555\": [\"n03345487\", \"fire_engine\"], \"556\": [\"n03347037\", \"fire_screen\"], \"557\": [\"n03355925\", \"flagpole\"], \"558\": [\"n03372029\", \"flute\"], \"559\": [\"n03376595\", \"folding_chair\"], \"560\": [\"n03379051\", \"football_helmet\"], \"561\": [\"n03384352\", \"forklift\"], \"562\": [\"n03388043\", \"fountain\"], \"563\": [\"n03388183\", \"fountain_pen\"], \"564\": [\"n03388549\", \"four-poster\"], \"565\": [\"n03393912\", \"freight_car\"], \"566\": [\"n03394916\", \"French_horn\"], \"567\": [\"n03400231\", \"frying_pan\"], \"568\": [\"n03404251\", \"fur_coat\"], \"569\": [\"n03417042\", \"garbage_truck\"], \"570\": [\"n03424325\", \"gasmask\"], \"571\": [\"n03425413\", \"gas_pump\"], \"572\": [\"n03443371\", \"goblet\"], \"573\": [\"n03444034\", \"go-kart\"], \"574\": [\"n03445777\", \"golf_ball\"], \"575\": [\"n03445924\", \"golfcart\"], \"576\": [\"n03447447\", \"gondola\"], \"577\": [\"n03447721\", \"gong\"], \"578\": [\"n03450230\", \"gown\"], \"579\": [\"n03452741\", \"grand_piano\"], \"580\": [\"n03457902\", \"greenhouse\"], \"581\": [\"n03459775\", \"grille\"], \"582\": [\"n03461385\", \"grocery_store\"], \"583\": [\"n03467068\", \"guillotine\"], \"584\": [\"n03476684\", \"hair_slide\"], \"585\": [\"n03476991\", \"hair_spray\"], \"586\": [\"n03478589\", \"half_track\"], \"587\": [\"n03481172\", \"hammer\"], \"588\": [\"n03482405\", \"hamper\"], \"589\": [\"n03483316\", \"hand_blower\"], \"590\": [\"n03485407\", \"hand-held_computer\"], \"591\": [\"n03485794\", \"handkerchief\"], \"592\": [\"n03492542\", \"hard_disc\"], \"593\": [\"n03494278\", \"harmonica\"], \"594\": [\"n03495258\", \"harp\"], \"595\": [\"n03496892\", \"harvester\"], \"596\": [\"n03498962\", \"hatchet\"], \"597\": [\"n03527444\", \"holster\"], \"598\": [\"n03529860\", \"home_theater\"], \"599\": [\"n03530642\", \"honeycomb\"], \"600\": [\"n03532672\", \"hook\"], \"601\": [\"n03534580\", \"hoopskirt\"], \"602\": [\"n03535780\", \"horizontal_bar\"], \"603\": [\"n03538406\", \"horse_cart\"], \"604\": [\"n03544143\", \"hourglass\"], \"605\": [\"n03584254\", \"iPod\"], \"606\": [\"n03584829\", \"iron\"], \"607\": [\"n03590841\", \"jack-o'-lantern\"], \"608\": [\"n03594734\", \"jean\"], \"609\": [\"n03594945\", \"jeep\"], \"610\": [\"n03595614\", \"jersey\"], \"611\": [\"n03598930\", \"jigsaw_puzzle\"], \"612\": [\"n03599486\", \"jinrikisha\"], \"613\": [\"n03602883\", \"joystick\"], \"614\": [\"n03617480\", \"kimono\"], \"615\": [\"n03623198\", \"knee_pad\"], \"616\": [\"n03627232\", \"knot\"], \"617\": [\"n03630383\", \"lab_coat\"], \"618\": [\"n03633091\", \"ladle\"], \"619\": [\"n03637318\", \"lampshade\"], \"620\": [\"n03642806\", \"laptop\"], \"621\": [\"n03649909\", \"lawn_mower\"], \"622\": [\"n03657121\", \"lens_cap\"], \"623\": [\"n03658185\", \"letter_opener\"], \"624\": [\"n03661043\", \"library\"], \"625\": [\"n03662601\", \"lifeboat\"], \"626\": [\"n03666591\", \"lighter\"], \"627\": [\"n03670208\", \"limousine\"], \"628\": [\"n03673027\", \"liner\"], \"629\": [\"n03676483\", \"lipstick\"], \"630\": [\"n03680355\", \"Loafer\"], \"631\": [\"n03690938\", \"lotion\"], \"632\": [\"n03691459\", \"loudspeaker\"], \"633\": [\"n03692522\", \"loupe\"], \"634\": [\"n03697007\", \"lumbermill\"], \"635\": [\"n03706229\", \"magnetic_compass\"], \"636\": [\"n03709823\", \"mailbag\"], \"637\": [\"n03710193\", \"mailbox\"], \"638\": [\"n03710637\", \"maillot\"], \"639\": [\"n03710721\", \"maillot\"], \"640\": [\"n03717622\", \"manhole_cover\"], \"641\": [\"n03720891\", \"maraca\"], \"642\": [\"n03721384\", \"marimba\"], \"643\": [\"n03724870\", \"mask\"], \"644\": [\"n03729826\", \"matchstick\"], \"645\": [\"n03733131\", \"maypole\"], \"646\": [\"n03733281\", \"maze\"], \"647\": [\"n03733805\", \"measuring_cup\"], \"648\": [\"n03742115\", \"medicine_chest\"], \"649\": [\"n03743016\", \"megalith\"], \"650\": [\"n03759954\", \"microphone\"], \"651\": [\"n03761084\", \"microwave\"], \"652\": [\"n03763968\", \"military_uniform\"], \"653\": [\"n03764736\", \"milk_can\"], \"654\": [\"n03769881\", \"minibus\"], \"655\": [\"n03770439\", \"miniskirt\"], \"656\": [\"n03770679\", \"minivan\"], \"657\": [\"n03773504\", \"missile\"], \"658\": [\"n03775071\", \"mitten\"], \"659\": [\"n03775546\", \"mixing_bowl\"], \"660\": [\"n03776460\", \"mobile_home\"], \"661\": [\"n03777568\", \"Model_T\"], \"662\": [\"n03777754\", \"modem\"], \"663\": [\"n03781244\", \"monastery\"], \"664\": [\"n03782006\", \"monitor\"], \"665\": [\"n03785016\", \"moped\"], \"666\": [\"n03786901\", \"mortar\"], \"667\": [\"n03787032\", \"mortarboard\"], \"668\": [\"n03788195\", \"mosque\"], \"669\": [\"n03788365\", \"mosquito_net\"], \"670\": [\"n03791053\", \"motor_scooter\"], \"671\": [\"n03792782\", \"mountain_bike\"], \"672\": [\"n03792972\", \"mountain_tent\"], \"673\": [\"n03793489\", \"mouse\"], \"674\": [\"n03794056\", \"mousetrap\"], \"675\": [\"n03796401\", \"moving_van\"], \"676\": [\"n03803284\", \"muzzle\"], \"677\": [\"n03804744\", \"nail\"], \"678\": [\"n03814639\", \"neck_brace\"], \"679\": [\"n03814906\", \"necklace\"], \"680\": [\"n03825788\", \"nipple\"], \"681\": [\"n03832673\", \"notebook\"], \"682\": [\"n03837869\", \"obelisk\"], \"683\": [\"n03838899\", \"oboe\"], \"684\": [\"n03840681\", \"ocarina\"], \"685\": [\"n03841143\", \"odometer\"], \"686\": [\"n03843555\", \"oil_filter\"], \"687\": [\"n03854065\", \"organ\"], \"688\": [\"n03857828\", \"oscilloscope\"], \"689\": [\"n03866082\", \"overskirt\"], \"690\": [\"n03868242\", \"oxcart\"], \"691\": [\"n03868863\", \"oxygen_mask\"], \"692\": [\"n03871628\", \"packet\"], \"693\": [\"n03873416\", \"paddle\"], \"694\": [\"n03874293\", \"paddlewheel\"], \"695\": [\"n03874599\", \"padlock\"], \"696\": [\"n03876231\", \"paintbrush\"], \"697\": [\"n03877472\", \"pajama\"], \"698\": [\"n03877845\", \"palace\"], \"699\": [\"n03884397\", \"panpipe\"], \"700\": [\"n03887697\", \"paper_towel\"], \"701\": [\"n03888257\", \"parachute\"], \"702\": [\"n03888605\", \"parallel_bars\"], \"703\": [\"n03891251\", \"park_bench\"], \"704\": [\"n03891332\", \"parking_meter\"], \"705\": [\"n03895866\", \"passenger_car\"], \"706\": [\"n03899768\", \"patio\"], \"707\": [\"n03902125\", \"pay-phone\"], \"708\": [\"n03903868\", \"pedestal\"], \"709\": [\"n03908618\", \"pencil_box\"], \"710\": [\"n03908714\", \"pencil_sharpener\"], \"711\": [\"n03916031\", \"perfume\"], \"712\": [\"n03920288\", \"Petri_dish\"], \"713\": [\"n03924679\", \"photocopier\"], \"714\": [\"n03929660\", \"pick\"], \"715\": [\"n03929855\", \"pickelhaube\"], \"716\": [\"n03930313\", \"picket_fence\"], \"717\": [\"n03930630\", \"pickup\"], \"718\": [\"n03933933\", \"pier\"], \"719\": [\"n03935335\", \"piggy_bank\"], \"720\": [\"n03937543\", \"pill_bottle\"], \"721\": [\"n03938244\", \"pillow\"], \"722\": [\"n03942813\", \"ping-pong_ball\"], \"723\": [\"n03944341\", \"pinwheel\"], \"724\": [\"n03947888\", \"pirate\"], \"725\": [\"n03950228\", \"pitcher\"], \"726\": [\"n03954731\", \"plane\"], \"727\": [\"n03956157\", \"planetarium\"], \"728\": [\"n03958227\", \"plastic_bag\"], \"729\": [\"n03961711\", \"plate_rack\"], \"730\": [\"n03967562\", \"plow\"], \"731\": [\"n03970156\", \"plunger\"], \"732\": [\"n03976467\", \"Polaroid_camera\"], \"733\": [\"n03976657\", \"pole\"], \"734\": [\"n03977966\", \"police_van\"], \"735\": [\"n03980874\", \"poncho\"], \"736\": [\"n03982430\", \"pool_table\"], \"737\": [\"n03983396\", \"pop_bottle\"], \"738\": [\"n03991062\", \"pot\"], \"739\": [\"n03992509\", \"potter's_wheel\"], \"740\": [\"n03995372\", \"power_drill\"], \"741\": [\"n03998194\", \"prayer_rug\"], \"742\": [\"n04004767\", \"printer\"], \"743\": [\"n04005630\", \"prison\"], \"744\": [\"n04008634\", \"projectile\"], \"745\": [\"n04009552\", \"projector\"], \"746\": [\"n04019541\", \"puck\"], \"747\": [\"n04023962\", \"punching_bag\"], \"748\": [\"n04026417\", \"purse\"], \"749\": [\"n04033901\", \"quill\"], \"750\": [\"n04033995\", \"quilt\"], \"751\": [\"n04037443\", \"racer\"], \"752\": [\"n04039381\", \"racket\"], \"753\": [\"n04040759\", \"radiator\"], \"754\": [\"n04041544\", \"radio\"], \"755\": [\"n04044716\", \"radio_telescope\"], \"756\": [\"n04049303\", \"rain_barrel\"], \"757\": [\"n04065272\", \"recreational_vehicle\"], \"758\": [\"n04067472\", \"reel\"], \"759\": [\"n04069434\", \"reflex_camera\"], \"760\": [\"n04070727\", \"refrigerator\"], \"761\": [\"n04074963\", \"remote_control\"], \"762\": [\"n04081281\", \"restaurant\"], \"763\": [\"n04086273\", \"revolver\"], \"764\": [\"n04090263\", \"rifle\"], \"765\": [\"n04099969\", \"rocking_chair\"], \"766\": [\"n04111531\", \"rotisserie\"], \"767\": [\"n04116512\", \"rubber_eraser\"], \"768\": [\"n04118538\", \"rugby_ball\"], \"769\": [\"n04118776\", \"rule\"], \"770\": [\"n04120489\", \"running_shoe\"], \"771\": [\"n04125021\", \"safe\"], \"772\": [\"n04127249\", \"safety_pin\"], \"773\": [\"n04131690\", \"saltshaker\"], \"774\": [\"n04133789\", \"sandal\"], \"775\": [\"n04136333\", \"sarong\"], \"776\": [\"n04141076\", \"sax\"], \"777\": [\"n04141327\", \"scabbard\"], \"778\": [\"n04141975\", \"scale\"], \"779\": [\"n04146614\", \"school_bus\"], \"780\": [\"n04147183\", \"schooner\"], \"781\": [\"n04149813\", \"scoreboard\"], \"782\": [\"n04152593\", \"screen\"], \"783\": [\"n04153751\", \"screw\"], \"784\": [\"n04154565\", \"screwdriver\"], \"785\": [\"n04162706\", \"seat_belt\"], \"786\": [\"n04179913\", \"sewing_machine\"], \"787\": [\"n04192698\", \"shield\"], \"788\": [\"n04200800\", \"shoe_shop\"], \"789\": [\"n04201297\", \"shoji\"], \"790\": [\"n04204238\", \"shopping_basket\"], \"791\": [\"n04204347\", \"shopping_cart\"], \"792\": [\"n04208210\", \"shovel\"], \"793\": [\"n04209133\", \"shower_cap\"], \"794\": [\"n04209239\", \"shower_curtain\"], \"795\": [\"n04228054\", \"ski\"], \"796\": [\"n04229816\", \"ski_mask\"], \"797\": [\"n04235860\", \"sleeping_bag\"], \"798\": [\"n04238763\", \"slide_rule\"], \"799\": [\"n04239074\", \"sliding_door\"], \"800\": [\"n04243546\", \"slot\"], \"801\": [\"n04251144\", \"snorkel\"], \"802\": [\"n04252077\", \"snowmobile\"], \"803\": [\"n04252225\", \"snowplow\"], \"804\": [\"n04254120\", \"soap_dispenser\"], \"805\": [\"n04254680\", \"soccer_ball\"], \"806\": [\"n04254777\", \"sock\"], \"807\": [\"n04258138\", \"solar_dish\"], \"808\": [\"n04259630\", \"sombrero\"], \"809\": [\"n04263257\", \"soup_bowl\"], \"810\": [\"n04264628\", \"space_bar\"], \"811\": [\"n04265275\", \"space_heater\"], \"812\": [\"n04266014\", \"space_shuttle\"], \"813\": [\"n04270147\", \"spatula\"], \"814\": [\"n04273569\", \"speedboat\"], \"815\": [\"n04275548\", \"spider_web\"], \"816\": [\"n04277352\", \"spindle\"], \"817\": [\"n04285008\", \"sports_car\"], \"818\": [\"n04286575\", \"spotlight\"], \"819\": [\"n04296562\", \"stage\"], \"820\": [\"n04310018\", \"steam_locomotive\"], \"821\": [\"n04311004\", \"steel_arch_bridge\"], \"822\": [\"n04311174\", \"steel_drum\"], \"823\": [\"n04317175\", \"stethoscope\"], \"824\": [\"n04325704\", \"stole\"], \"825\": [\"n04326547\", \"stone_wall\"], \"826\": [\"n04328186\", \"stopwatch\"], \"827\": [\"n04330267\", \"stove\"], \"828\": [\"n04332243\", \"strainer\"], \"829\": [\"n04335435\", \"streetcar\"], \"830\": [\"n04336792\", \"stretcher\"], \"831\": [\"n04344873\", \"studio_couch\"], \"832\": [\"n04346328\", \"stupa\"], \"833\": [\"n04347754\", \"submarine\"], \"834\": [\"n04350905\", \"suit\"], \"835\": [\"n04355338\", \"sundial\"], \"836\": [\"n04355933\", \"sunglass\"], \"837\": [\"n04356056\", \"sunglasses\"], \"838\": [\"n04357314\", \"sunscreen\"], \"839\": [\"n04366367\", \"suspension_bridge\"], \"840\": [\"n04367480\", \"swab\"], \"841\": [\"n04370456\", \"sweatshirt\"], \"842\": [\"n04371430\", \"swimming_trunks\"], \"843\": [\"n04371774\", \"swing\"], \"844\": [\"n04372370\", \"switch\"], \"845\": [\"n04376876\", \"syringe\"], \"846\": [\"n04380533\", \"table_lamp\"], \"847\": [\"n04389033\", \"tank\"], \"848\": [\"n04392985\", \"tape_player\"], \"849\": [\"n04398044\", \"teapot\"], \"850\": [\"n04399382\", \"teddy\"], \"851\": [\"n04404412\", \"television\"], \"852\": [\"n04409515\", \"tennis_ball\"], \"853\": [\"n04417672\", \"thatch\"], \"854\": [\"n04418357\", \"theater_curtain\"], \"855\": [\"n04423845\", \"thimble\"], \"856\": [\"n04428191\", \"thresher\"], \"857\": [\"n04429376\", \"throne\"], \"858\": [\"n04435653\", \"tile_roof\"], \"859\": [\"n04442312\", \"toaster\"], \"860\": [\"n04443257\", \"tobacco_shop\"], \"861\": [\"n04447861\", \"toilet_seat\"], \"862\": [\"n04456115\", \"torch\"], \"863\": [\"n04458633\", \"totem_pole\"], \"864\": [\"n04461696\", \"tow_truck\"], \"865\": [\"n04462240\", \"toyshop\"], \"866\": [\"n04465501\", \"tractor\"], \"867\": [\"n04467665\", \"trailer_truck\"], \"868\": [\"n04476259\", \"tray\"], \"869\": [\"n04479046\", \"trench_coat\"], \"870\": [\"n04482393\", \"tricycle\"], \"871\": [\"n04483307\", \"trimaran\"], \"872\": [\"n04485082\", \"tripod\"], \"873\": [\"n04486054\", \"triumphal_arch\"], \"874\": [\"n04487081\", \"trolleybus\"], \"875\": [\"n04487394\", \"trombone\"], \"876\": [\"n04493381\", \"tub\"], \"877\": [\"n04501370\", \"turnstile\"], \"878\": [\"n04505470\", \"typewriter_keyboard\"], \"879\": [\"n04507155\", \"umbrella\"], \"880\": [\"n04509417\", \"unicycle\"], \"881\": [\"n04515003\", \"upright\"], \"882\": [\"n04517823\", \"vacuum\"], \"883\": [\"n04522168\", \"vase\"], \"884\": [\"n04523525\", \"vault\"], \"885\": [\"n04525038\", \"velvet\"], \"886\": [\"n04525305\", \"vending_machine\"], \"887\": [\"n04532106\", \"vestment\"], \"888\": [\"n04532670\", \"viaduct\"], \"889\": [\"n04536866\", \"violin\"], \"890\": [\"n04540053\", \"volleyball\"], \"891\": [\"n04542943\", \"waffle_iron\"], \"892\": [\"n04548280\", \"wall_clock\"], \"893\": [\"n04548362\", \"wallet\"], \"894\": [\"n04550184\", \"wardrobe\"], \"895\": [\"n04552348\", \"warplane\"], \"896\": [\"n04553703\", \"washbasin\"], \"897\": [\"n04554684\", \"washer\"], \"898\": [\"n04557648\", \"water_bottle\"], \"899\": [\"n04560804\", \"water_jug\"], \"900\": [\"n04562935\", \"water_tower\"], \"901\": [\"n04579145\", \"whiskey_jug\"], \"902\": [\"n04579432\", \"whistle\"], \"903\": [\"n04584207\", \"wig\"], \"904\": [\"n04589890\", \"window_screen\"], \"905\": [\"n04590129\", \"window_shade\"], \"906\": [\"n04591157\", \"Windsor_tie\"], \"907\": [\"n04591713\", \"wine_bottle\"], \"908\": [\"n04592741\", \"wing\"], \"909\": [\"n04596742\", \"wok\"], \"910\": [\"n04597913\", \"wooden_spoon\"], \"911\": [\"n04599235\", \"wool\"], \"912\": [\"n04604644\", \"worm_fence\"], \"913\": [\"n04606251\", \"wreck\"], \"914\": [\"n04612504\", \"yawl\"], \"915\": [\"n04613696\", \"yurt\"], \"916\": [\"n06359193\", \"web_site\"], \"917\": [\"n06596364\", \"comic_book\"], \"918\": [\"n06785654\", \"crossword_puzzle\"], \"919\": [\"n06794110\", \"street_sign\"], \"920\": [\"n06874185\", \"traffic_light\"], \"921\": [\"n07248320\", \"book_jacket\"], \"922\": [\"n07565083\", \"menu\"], \"923\": [\"n07579787\", \"plate\"], \"924\": [\"n07583066\", \"guacamole\"], \"925\": [\"n07584110\", \"consomme\"], \"926\": [\"n07590611\", \"hot_pot\"], \"927\": [\"n07613480\", \"trifle\"], \"928\": [\"n07614500\", \"ice_cream\"], \"929\": [\"n07615774\", \"ice_lolly\"], \"930\": [\"n07684084\", \"French_loaf\"], \"931\": [\"n07693725\", \"bagel\"], \"932\": [\"n07695742\", \"pretzel\"], \"933\": [\"n07697313\", \"cheeseburger\"], \"934\": [\"n07697537\", \"hotdog\"], \"935\": [\"n07711569\", \"mashed_potato\"], \"936\": [\"n07714571\", \"head_cabbage\"], \"937\": [\"n07714990\", \"broccoli\"], \"938\": [\"n07715103\", \"cauliflower\"], \"939\": [\"n07716358\", \"zucchini\"], \"940\": [\"n07716906\", \"spaghetti_squash\"], \"941\": [\"n07717410\", \"acorn_squash\"], \"942\": [\"n07717556\", \"butternut_squash\"], \"943\": [\"n07718472\", \"cucumber\"], \"944\": [\"n07718747\", \"artichoke\"], \"945\": [\"n07720875\", \"bell_pepper\"], \"946\": [\"n07730033\", \"cardoon\"], \"947\": [\"n07734744\", \"mushroom\"], \"948\": [\"n07742313\", \"Granny_Smith\"], \"949\": [\"n07745940\", \"strawberry\"], \"950\": [\"n07747607\", \"orange\"], \"951\": [\"n07749582\", \"lemon\"], \"952\": [\"n07753113\", \"fig\"], \"953\": [\"n07753275\", \"pineapple\"], \"954\": [\"n07753592\", \"banana\"], \"955\": [\"n07754684\", \"jackfruit\"], \"956\": [\"n07760859\", \"custard_apple\"], \"957\": [\"n07768694\", \"pomegranate\"], \"958\": [\"n07802026\", \"hay\"], \"959\": [\"n07831146\", \"carbonara\"], \"960\": [\"n07836838\", \"chocolate_sauce\"], \"961\": [\"n07860988\", \"dough\"], \"962\": [\"n07871810\", \"meat_loaf\"], \"963\": [\"n07873807\", \"pizza\"], \"964\": [\"n07875152\", \"potpie\"], \"965\": [\"n07880968\", \"burrito\"], \"966\": [\"n07892512\", \"red_wine\"], \"967\": [\"n07920052\", \"espresso\"], \"968\": [\"n07930864\", \"cup\"], \"969\": [\"n07932039\", \"eggnog\"], \"970\": [\"n09193705\", \"alp\"], \"971\": [\"n09229709\", \"bubble\"], \"972\": [\"n09246464\", \"cliff\"], \"973\": [\"n09256479\", \"coral_reef\"], \"974\": [\"n09288635\", \"geyser\"], \"975\": [\"n09332890\", \"lakeside\"], \"976\": [\"n09399592\", \"promontory\"], \"977\": [\"n09421951\", \"sandbar\"], \"978\": [\"n09428293\", \"seashore\"], \"979\": [\"n09468604\", \"valley\"], \"980\": [\"n09472597\", \"volcano\"], \"981\": [\"n09835506\", \"ballplayer\"], \"982\": [\"n10148035\", \"groom\"], \"983\": [\"n10565667\", \"scuba_diver\"], \"984\": [\"n11879895\", \"rapeseed\"], \"985\": [\"n11939491\", \"daisy\"], \"986\": [\"n12057211\", \"yellow_lady's_slipper\"], \"987\": [\"n12144580\", \"corn\"], \"988\": [\"n12267677\", \"acorn\"], \"989\": [\"n12620546\", \"hip\"], \"990\": [\"n12768682\", \"buckeye\"], \"991\": [\"n12985857\", \"coral_fungus\"], \"992\": [\"n12998815\", \"agaric\"], \"993\": [\"n13037406\", \"gyromitra\"], \"994\": [\"n13040303\", \"stinkhorn\"], \"995\": [\"n13044778\", \"earthstar\"], \"996\": [\"n13052670\", \"hen-of-the-woods\"], \"997\": [\"n13054560\", \"bolete\"], \"998\": [\"n13133613\", \"ear\"], \"999\": [\"n15075141\", \"toilet_tissue\"]}\n results = []\n for pred in preds:\n top_indices = pred.argsort()[-top:][::-1]\n result = [tuple(CLASS_INDEX[str(i)]) + (pred[i],) for i in top_indices]\n result.sort(key=lambda x: x[2], reverse=True)\n results.append(result)\n return results\n\n" }, { "path": "imageai_tf_deprecated/__init__.py", "content": "" }, { "path": "requirements.txt", "content": "cython\npillow>=7.0.0\nnumpy>=1.18.1\nopencv-python>=4.1.2\ntorch>=1.9.0 --extra-index-url https://download.pytorch.org/whl/cpu\ntorchvision>=0.10.0 --extra-index-url https://download.pytorch.org/whl/cpu\npytest==7.1.3\ntqdm==4.64.1\nscipy>=1.7.3\nmatplotlib>=3.4.3\nmock==4.0.3" }, { "path": "requirements_extra.txt", "content": "pycocotools@git+https://github.com/gautamchitnis/cocoapi.git@cocodataset-master#subdirectory=PythonAPI" }, { "path": "requirements_gpu.txt", "content": "cython\npillow>=7.0.0\nnumpy>=1.18.1\nopencv-python>=4.1.2\ntorch>=1.9.0 --extra-index-url https://download.pytorch.org/whl/cu102\ntorchvision>=0.10.0 --extra-index-url https://download.pytorch.org/whl/cu102\npytest==7.1.3\ntqdm==4.64.1\nscipy>=1.7.3\nmatplotlib>=3.4.3\nmock==4.0.3" }, { "path": "scripts/pascal_voc_to_yolo.py", "content": "import glob\nimport os\nimport argparse\nimport pickle\nimport xml.etree.ElementTree as ET\nfrom os import listdir, getcwd\nfrom os.path import join\nimport shutil\n\n\ndirs = ['train', 'validation']\nsub_dirs = [\"images\", \"annotations\"]\nclasses = []\n\ndef convert(size, box):\n dw = 1./(size[0])\n dh = 1./(size[1])\n x = (box[0] + box[1])/2.0 - 1\n y = (box[2] + box[3])/2.0 - 1\n w = box[1] - box[0]\n h = box[3] - box[2]\n x = x*dw\n w = w*dw\n y = y*dh\n h = h*dh\n return (x,y,w,h)\n\ndef convert_annotation(input_ann_path):\n\n tree = ET.parse(input_ann_path)\n root = tree.getroot()\n size = root.find('size')\n w = int(size.find('width').text)\n h = int(size.find('height').text)\n\n ann_list = []\n\n for obj in root.iter('object'):\n obj_class = obj.find('name').text\n if obj_class not in classes:\n classes.append(obj_class)\n xmlbox = obj.find('bndbox')\n b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text), float(xmlbox.find('ymax').text))\n bb = convert((w,h), b)\n\n ann_list.append(\n {\n \"class\": obj_class,\n \"bbox\": bb\n }\n )\n\n return ann_list\n\n\ndef main(dataset_dir: str):\n yolo_dataset = os.path.join(\n os.path.dirname(dataset_dir),\n os.path.basename(f\"{dataset_dir}-yolo\")\n )\n for dir in dirs:\n dir_path = os.path.join(\n yolo_dataset,\n dir\n )\n os.makedirs(dir_path, exist_ok=True)\n\n for sub_dir in sub_dirs:\n os.makedirs(\n os.path.join(\n dir_path,\n sub_dir\n ),\n exist_ok=True\n )\n \n train_anns = {}\n validation_anns = {}\n\n for dir in dirs:\n dir_path = os.path.join(\n dataset_dir,\n dir\n )\n\n images = [file for file in os.listdir(\n os.path.join(dir_path, \"images\")\n ) if file.endswith(\".png\") or file.endswith(\".jpg\") or file.endswith(\".jpeg\")]\n\n annotations = [file for file in os.listdir(\n os.path.join(dir_path, \"annotations\")\n ) if file.endswith(\".xml\")]\n\n for image, annotation in zip(images, annotations):\n shutil.copy(\n os.path.join(\n dataset_dir,\n dir,\n \"images\",\n image\n ),\n os.path.join(\n yolo_dataset,\n dir,\n \"images\",\n image\n )\n )\n\n ann_list = convert_annotation(\n os.path.join(\n dataset_dir,\n dir,\n \"annotations\",\n annotation\n ) \n )\n if dir == \"train\":\n train_anns[annotation] = ann_list\n elif dir == \"validation\":\n validation_anns[annotation] = ann_list\n \n all_classes = sorted(classes)\n\n for k,v in {\"train\": train_anns, \"validation\": validation_anns}.items():\n for anns_k, anns_v in v.items():\n output_ann_path = os.path.join(\n yolo_dataset, k, \"annotations\", anns_k.replace(\".xml\", \".txt\")\n )\n anns_str = \"\"\n for ann in anns_v:\n class_idx = all_classes.index(ann[\"class\"])\n bbox = [str(f) for f in ann[\"bbox\"]]\n anns_str += f\"{class_idx} {' '.join(bbox)}\\n\"\n \n with open(output_ann_path, \"w\") as ann_writer:\n ann_writer.write(anns_str)\n \n with open(os.path.join(\n yolo_dataset, k, \"annotations\", \"classes.txt\"\n ), \"w\") as classes_writer:\n classes_writer.write(\"\\n\".join(all_classes))\n \n\n\nif __name__ == \"__main__\":\n\n parse = argparse.ArgumentParser(\n description=\"Convert Pascal VOC dataset to YOLO format\")\n parse.add_argument(\n \"--dataset_dir\",\n help=\"Dataset directory\",\n type=str,\n required=True,\n )\n args = parse.parse_args()\n main(args.dataset_dir)\n" }, { "path": "setup.py", "content": "from setuptools import setup,find_packages\n\nsetup(name=\"imageai\",\n version='3.0.3',\n description='A python library built to empower developers to build applications and systems with self-contained Computer Vision capabilities',\n url=\"https://github.com/OlafenwaMoses/ImageAI\",\n author='Moses Olafenwa',\n author_email='guymodscientist@gmail.com',\n license='MIT',\n packages= find_packages(exclude=[\"*imageai_tf_deprecated*\"]),\n install_requires=[],\n include_package_data=True,\n zip_safe=False)" }, { "path": "test/test_custom_classification.py", "content": "import os, sys\nimport cv2\nfrom PIL import Image\nimport pytest\nfrom os.path import dirname\nsys.path.insert(1, os.path.join(dirname(dirname(os.path.abspath(__file__)))))\nfrom imageai.Classification.Custom import CustomImageClassification\n\ntest_folder = dirname(os.path.abspath(__file__))\n\n\n\n@pytest.mark.parametrize(\n \"image_input\",\n [\n (os.path.join(test_folder, \"data-images\", \"1.jpg\")),\n (cv2.imread(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n (Image.open(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n ]\n)\ndef test_recognition_model_mobilenetv2(image_input):\n\n classifier = CustomImageClassification()\n classifier.setModelTypeAsMobileNetV2()\n classifier.setModelPath(os.path.join(test_folder, \"data-models\", \"mobilenet_v2-idenprof-test_acc_0.85300_epoch-92.pt\"))\n classifier.setJsonPath(os.path.join(test_folder, \"data-json\", \"idenprof_model_classes.json\"))\n classifier.loadModel()\n predictions, probabilities = classifier.classifyImage(image_input=image_input, result_count=5)\n\n assert isinstance(predictions, list)\n assert isinstance(probabilities, list)\n assert isinstance(predictions[0], str)\n assert isinstance(probabilities[0], float)\n\n\n@pytest.mark.parametrize(\n \"image_input\",\n [\n (os.path.join(test_folder, \"data-images\", \"1.jpg\")),\n (cv2.imread(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n (Image.open(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n ]\n)\ndef test_recognition_model_resnet(image_input):\n\n classifier = CustomImageClassification()\n classifier.setModelTypeAsResNet50()\n classifier.setModelPath(os.path.join(test_folder, \"data-models\", \"resnet50-idenprof-test_acc_0.78200_epoch-91.pt\"))\n classifier.setJsonPath(os.path.join(test_folder, \"data-json\", \"idenprof_model_classes.json\"))\n classifier.loadModel()\n predictions, probabilities = classifier.classifyImage(image_input=image_input, result_count=5)\n\n assert isinstance(predictions, list)\n assert isinstance(probabilities, list)\n assert isinstance(predictions[0], str)\n assert isinstance(probabilities[0], float)\n\n@pytest.mark.parametrize(\n \"image_input\",\n [\n (os.path.join(test_folder, \"data-images\", \"1.jpg\")),\n (cv2.imread(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n (Image.open(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n ]\n)\ndef test_recognition_model_inceptionv3(image_input):\n\n classifier = CustomImageClassification()\n classifier.setModelTypeAsInceptionV3()\n classifier.setModelPath(os.path.join(test_folder, \"data-models\", \"inception_v3-idenprof-test_acc_0.81050_epoch-92.pt\"))\n classifier.setJsonPath(os.path.join(test_folder, \"data-json\", \"idenprof_model_classes.json\"))\n classifier.loadModel()\n predictions, probabilities = classifier.classifyImage(image_input=image_input, result_count=5)\n\n assert isinstance(predictions, list)\n assert isinstance(probabilities, list)\n assert isinstance(predictions[0], str)\n assert isinstance(probabilities[0], float)\n\n@pytest.mark.parametrize(\n \"image_input\",\n [\n (os.path.join(test_folder, \"data-images\", \"1.jpg\")),\n (cv2.imread(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n (Image.open(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n ]\n)\ndef test_recognition_model_densenet(image_input):\n\n classifier = CustomImageClassification()\n classifier.setModelTypeAsDenseNet121()\n classifier.setModelPath(os.path.join(test_folder, \"data-models\", \"densenet121-idenprof-test_acc_0.82550_epoch-95.pt\"))\n classifier.setJsonPath(os.path.join(test_folder, \"data-json\", \"idenprof_model_classes.json\"))\n classifier.loadModel()\n predictions, probabilities = classifier.classifyImage(image_input=image_input, result_count=5)\n\n assert isinstance(predictions, list)\n assert isinstance(probabilities, list)\n assert isinstance(predictions[0], str)\n assert isinstance(probabilities[0], float)" }, { "path": "test/test_custom_classification_training.py", "content": "import os, sys\nimport cv2\nimport shutil\nfrom PIL import Image\nimport pytest\nfrom os.path import dirname\nsys.path.insert(1, os.path.join(dirname(dirname(os.path.abspath(__file__)))))\nfrom imageai.Classification.Custom import ClassificationModelTrainer, CustomImageClassification\n\ntest_folder = dirname(os.path.abspath(__file__))\n\n\nclassification_dataset = os.path.join(\n test_folder,\n \"data-datasets\",\n \"idenprof\"\n)\n\npretrained_models_folder = os.path.join(\n test_folder,\n \"data-models\"\n)\n\n\n@pytest.mark.parametrize(\n \"transfer_learning\",\n [\n (os.path.join(\n pretrained_models_folder,\n \"resnet50-19c8e357.pth\"\n )),\n (None),\n ]\n)\ndef test_resnet50_training(transfer_learning):\n\n models_dir = os.path.join(\n classification_dataset,\n \"models\"\n )\n if os.path.isdir(\n models_dir\n ):\n shutil.rmtree(models_dir)\n\n trainer = ClassificationModelTrainer()\n trainer.setModelTypeAsResNet50()\n trainer.setDataDirectory(data_directory=classification_dataset)\n trainer.trainModel(\n num_experiments=1,\n batch_size=2,\n transfer_from_model=transfer_learning)\n\n assert os.path.isdir(models_dir) == True\n assert os.path.isfile(\n os.path.join(\n models_dir, \"idenprof_model_classes.json\"\n )\n ) == True\n \n model_found = False\n for file in os.listdir(models_dir):\n if file.endswith(\".pt\"):\n model_found = True\n assert model_found == True\n\n\n@pytest.mark.parametrize(\n \"transfer_learning\",\n [\n (os.path.join(\n pretrained_models_folder,\n \"densenet121-a639ec97.pth\"\n )),\n (None),\n ]\n)\ndef test_densenet121_training(transfer_learning):\n\n models_dir = os.path.join(\n classification_dataset,\n \"models\"\n )\n if os.path.isdir(\n models_dir\n ):\n shutil.rmtree(models_dir)\n\n trainer = ClassificationModelTrainer()\n trainer.setModelTypeAsDenseNet121()\n trainer.setDataDirectory(data_directory=classification_dataset)\n trainer.trainModel(\n num_experiments=1,\n batch_size=2,\n transfer_from_model=transfer_learning)\n\n assert os.path.isdir(models_dir) == True\n assert os.path.isfile(\n os.path.join(\n models_dir, \"idenprof_model_classes.json\"\n )\n ) == True\n model_found = False\n for file in os.listdir(models_dir):\n if file.endswith(\".pt\"):\n model_found = True\n assert model_found == True\n\n\n\n@pytest.mark.parametrize(\n \"transfer_learning\",\n [\n (os.path.join(\n pretrained_models_folder,\n \"inception_v3_google-1a9a5a14.pth\"\n )),\n (None),\n ]\n)\ndef test_inceptionv3_training(transfer_learning):\n\n models_dir = os.path.join(\n classification_dataset,\n \"models\"\n )\n if os.path.isdir(\n models_dir\n ):\n shutil.rmtree(models_dir)\n\n trainer = ClassificationModelTrainer()\n trainer.setModelTypeAsInceptionV3()\n trainer.setDataDirectory(data_directory=classification_dataset)\n trainer.trainModel(\n num_experiments=1,\n batch_size=2,\n transfer_from_model=transfer_learning)\n\n assert os.path.isdir(models_dir) == True\n assert os.path.isfile(\n os.path.join(\n models_dir, \"idenprof_model_classes.json\"\n )\n ) == True\n model_found = False\n for file in os.listdir(models_dir):\n if file.endswith(\".pt\"):\n model_found = True\n assert model_found == True\n\n\n@pytest.mark.parametrize(\n \"transfer_learning\",\n [\n (os.path.join(\n pretrained_models_folder,\n \"mobilenet_v2-b0353104.pth\"\n )),\n (None),\n ]\n)\ndef test_mobilenetv2_training(transfer_learning):\n\n models_dir = os.path.join(\n classification_dataset,\n \"models\"\n )\n if os.path.isdir(\n models_dir\n ):\n shutil.rmtree(models_dir)\n\n trainer = ClassificationModelTrainer()\n trainer.setModelTypeAsMobileNetV2()\n trainer.setDataDirectory(data_directory=classification_dataset)\n trainer.trainModel(\n num_experiments=1,\n batch_size=2,\n transfer_from_model=transfer_learning)\n\n assert os.path.isdir(models_dir) == True\n assert os.path.isfile(\n os.path.join(\n models_dir, \"idenprof_model_classes.json\"\n )\n ) == True\n model_found = False\n for file in os.listdir(models_dir):\n if file.endswith(\".pt\"):\n model_found = True\n assert model_found == True\n" }, { "path": "test/test_custom_detection_training.py", "content": "import os, sys\nimport shutil\nimport pytest\nfrom os.path import dirname\nsys.path.insert(1, os.path.join(dirname(dirname(os.path.abspath(__file__)))))\nfrom imageai.Detection.Custom import DetectionModelTrainer\n\ntest_folder = dirname(os.path.abspath(__file__))\n\n\ndetection_dataset = os.path.join(\n test_folder,\n \"data-datasets\",\n \"number-plate\"\n)\n\npretrained_models_folder = os.path.join(\n test_folder,\n \"data-models\"\n)\n\ndef delete_cache(dirs: list):\n for dir in dirs:\n if os.path.isdir(dir):\n shutil.rmtree(dir)\n\n@pytest.mark.parametrize(\n \"transfer_learning\",\n [\n (os.path.join(\n pretrained_models_folder,\n \"yolov3.pt\"\n )),\n (None),\n ]\n)\ndef test_yolov3_training(transfer_learning):\n json_dir = os.path.join(detection_dataset, \"json\")\n json_file = os.path.join(json_dir, \"number-plate_yolov3_detection_config.json\")\n models_dir = os.path.join(detection_dataset, \"models\")\n\n delete_cache([json_dir, models_dir])\n\n trainer = DetectionModelTrainer()\n trainer.setModelTypeAsYOLOv3()\n trainer.setDataDirectory(data_directory=detection_dataset)\n trainer.setTrainConfig(object_names_array=[\"number-plate\"], batch_size=2, num_experiments=2, train_from_pretrained_model=transfer_learning)\n trainer.trainModel()\n\n \n assert os.path.isfile(json_file)\n assert len([file for file in os.listdir(models_dir) if file.endswith(\".pt\")]) > 0\n\n delete_cache([json_dir, models_dir])\n\n@pytest.mark.parametrize(\n \"transfer_learning\",\n [\n (os.path.join(\n pretrained_models_folder,\n \"tiny-yolov3.pt\"\n )),\n (None),\n ]\n)\ndef test_tiny_yolov3_training(transfer_learning):\n json_dir = os.path.join(detection_dataset, \"json\")\n json_file = os.path.join(json_dir, \"number-plate_tiny-yolov3_detection_config.json\")\n models_dir = os.path.join(detection_dataset, \"models\")\n\n delete_cache([json_dir, models_dir])\n\n trainer = DetectionModelTrainer()\n trainer.setModelTypeAsTinyYOLOv3()\n trainer.setDataDirectory(data_directory=detection_dataset)\n trainer.setTrainConfig(object_names_array=[\"number-plate\"], batch_size=2, num_experiments=2, train_from_pretrained_model=transfer_learning)\n trainer.trainModel()\n\n \n assert os.path.isfile(json_file)\n assert len([file for file in os.listdir(models_dir) if file.endswith(\".pt\")]) > 0\n\n delete_cache([json_dir, models_dir])" }, { "path": "test/test_custom_object_detection.py", "content": "import os, sys\nfrom typing import List\nimport shutil\nimport cv2\nimport uuid\nfrom PIL import Image\nimport numpy as np\nimport pytest\nfrom os.path import dirname\nsys.path.insert(1, os.path.join(dirname(dirname(os.path.abspath(__file__)))))\nfrom imageai.Detection.Custom import CustomObjectDetection\n\ntest_folder = dirname(os.path.abspath(__file__))\n\n\n\ndef delete_cache(paths: List[str]):\n for path in paths:\n if os.path.isfile(path):\n os.remove(path)\n elif os.path.isdir(path):\n shutil.rmtree(path)\n\n\n@pytest.mark.parametrize(\n \"input_image, output_type, extract_objects\",\n [\n (os.path.join(test_folder, test_folder, \"data-images\", \"15.jpg\"), \"file\", False),\n (os.path.join(test_folder, test_folder, \"data-images\", \"15.jpg\"), \"file\", True),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"15.jpg\")), \"array\", False),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"15.jpg\")), \"array\", True),\n (Image.open(os.path.join(test_folder, test_folder, \"data-images\", \"15.jpg\")), \"array\", True),\n ]\n)\ndef test_object_detection_yolov3(input_image, output_type, extract_objects):\n detector = CustomObjectDetection()\n detector.setModelTypeAsYOLOv3()\n detector.setModelPath(os.path.join(test_folder, \"data-models\", \"yolov3_number-plate-dataset-imageai_mAP-0.57145_epoch-11.pt\"))\n detector.setJsonPath(os.path.join(test_folder, \"data-json\", \"number-plate-dataset-imageai_yolov3_detection_config.json\"))\n detector.loadModel()\n\n output_img_path = os.path.join(test_folder, \"data-images\", str(uuid.uuid4()) + \".jpg\")\n\n if output_type == \"array\":\n if extract_objects:\n output_image_array, detections, extracted_objects = detector.detectObjectsFromImage(input_image=input_image, output_type=output_type, extract_detected_objects=extract_objects)\n\n assert len(detections) > 0\n assert len(extracted_objects) > 0\n for extracted_obj in extracted_objects:\n assert type(extracted_obj) == np.ndarray\n else:\n output_image_array, detections = detector.detectObjectsFromImage(input_image=input_image, output_type=output_type)\n assert type(output_image_array) == np.ndarray\n assert len(detections) > 0\n else:\n if extract_objects:\n detections, extracted_object_paths = detector.detectObjectsFromImage(input_image=input_image, output_image_path=output_img_path, extract_detected_objects=True)\n\n assert len(detections) > 0\n assert os.path.isfile(output_img_path)\n assert len(extracted_object_paths) > 0\n delete_cache(\n extracted_object_paths\n )\n delete_cache(\n [extracted_object_paths[0], output_img_path]\n )\n else:\n detections = detector.detectObjectsFromImage(input_image=input_image, output_image_path=output_img_path)\n assert len(detections) > 0\n delete_cache([output_img_path])\n\n assert type(detections) == list\n \n\n for eachObject in detections:\n assert type(eachObject) == dict\n assert \"name\" in eachObject.keys()\n assert type(eachObject[\"name\"]) == str \n assert \"percentage_probability\" in eachObject.keys()\n assert type(eachObject[\"percentage_probability\"]) == float\n assert \"box_points\" in eachObject.keys()\n assert type(eachObject[\"box_points\"]) == list\n box_points = eachObject[\"box_points\"]\n for point in box_points:\n assert type(point) == int\n assert box_points[0] < box_points[2]\n assert box_points[1] < box_points[3]\n\n\n\n@pytest.mark.parametrize(\n \"input_image, output_type, extract_objects\",\n [\n (os.path.join(test_folder, test_folder, \"data-images\", \"15.jpg\"), \"file\", False),\n (os.path.join(test_folder, test_folder, \"data-images\", \"15.jpg\"), \"file\", True),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"15.jpg\")), \"array\", False),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"15.jpg\")), \"array\", True),\n (Image.open(os.path.join(test_folder, test_folder, \"data-images\", \"15.jpg\")), \"array\", True),\n ]\n)\ndef test_object_detection_tiny_yolov3(input_image, output_type, extract_objects):\n detector = CustomObjectDetection()\n detector.setModelTypeAsTinyYOLOv3()\n detector.setModelPath(os.path.join(test_folder, \"data-models\", \"tiny_yolov3_number-plate-dataset-imageai_mAP-0.22595_epoch-20.pt\"))\n detector.setJsonPath(os.path.join(test_folder, \"data-json\", \"number-plate-dataset-imageai_tiny_yolov3_detection_config.json\"))\n detector.loadModel()\n\n output_img_path = os.path.join(test_folder, \"data-images\", str(uuid.uuid4()) + \".jpg\")\n\n if output_type == \"array\":\n if extract_objects:\n output_image_array, detections, extracted_objects = detector.detectObjectsFromImage(input_image=input_image, output_type=output_type, extract_detected_objects=extract_objects)\n\n assert len(detections) > 0\n assert len(extracted_objects) == len(detections)\n for extracted_obj in extracted_objects:\n assert type(extracted_obj) == np.ndarray\n else:\n output_image_array, detections = detector.detectObjectsFromImage(input_image=input_image, output_type=output_type)\n assert type(output_image_array) == np.ndarray\n assert len(detections) > 0\n else:\n if extract_objects:\n detections, extracted_object_paths = detector.detectObjectsFromImage(input_image=input_image, output_image_path=output_img_path, extract_detected_objects=True)\n\n assert len(detections) > 0\n assert os.path.isfile(output_img_path)\n assert len(extracted_object_paths) == len(detections)\n delete_cache(\n extracted_object_paths\n )\n delete_cache(\n [extracted_object_paths[0], output_img_path]\n )\n else:\n detections = detector.detectObjectsFromImage(input_image=input_image, output_image_path=output_img_path)\n assert len(detections) > 0\n delete_cache([output_img_path])\n\n assert type(detections) == list\n \n\n for eachObject in detections:\n assert type(eachObject) == dict\n assert \"name\" in eachObject.keys()\n assert type(eachObject[\"name\"]) == str \n assert \"percentage_probability\" in eachObject.keys()\n assert type(eachObject[\"percentage_probability\"]) == float\n assert \"box_points\" in eachObject.keys()\n assert type(eachObject[\"box_points\"]) == list\n box_points = eachObject[\"box_points\"]\n for point in box_points:\n assert type(point) == int\n assert box_points[0] < box_points[2]\n assert box_points[1] < box_points[3]\n\n\n" }, { "path": "test/test_custom_video_detection.py", "content": "import os, sys\nfrom typing import List\nfrom numpy import ndarray\nfrom os.path import dirname\nfrom mock import patch\nsys.path.insert(1, os.path.join(dirname(dirname(os.path.abspath(__file__)))))\n\nfrom imageai.Detection.Custom import CustomVideoObjectDetection\n\n\ntest_folder = dirname(os.path.abspath(__file__))\n\nvideo_file = os.path.join(test_folder, \"data-videos\", \"dashcam.mp4\")\nvideo_file_output = os.path.join(test_folder, \"data-videos\", \"dashcam-detected\")\n\n\n\nclass CallbackFunctions:\n def forFrame(frame_number, output_array, output_count, detected_frame):\n assert isinstance(detected_frame, ndarray)\n assert isinstance(frame_number, int)\n assert isinstance(output_array, list)\n assert isinstance(output_array[0], dict)\n assert isinstance(output_array[0][\"name\"], str)\n assert isinstance(output_array[0][\"percentage_probability\"], float)\n assert isinstance(output_array[0][\"box_points\"], list)\n\n assert isinstance(output_count, dict)\n for a_key in dict(output_count).keys():\n assert isinstance(a_key, str)\n assert isinstance(output_count[a_key], int)\n\n def forSecond(second_number, output_arrays, count_arrays, average_output_count, detected_frame):\n assert isinstance(detected_frame, ndarray)\n assert isinstance(second_number, int)\n assert isinstance(output_arrays, list)\n assert isinstance(output_arrays[0], list)\n\n assert isinstance(output_arrays[0][0], dict)\n assert isinstance(output_arrays[0][0][\"name\"], str)\n assert isinstance(output_arrays[0][0][\"percentage_probability\"], float)\n assert isinstance(output_arrays[0][0][\"box_points\"], list)\n\n assert isinstance(count_arrays, list)\n assert isinstance(count_arrays[0], dict)\n for a_key in dict(count_arrays[0]).keys():\n assert isinstance(a_key, str)\n assert isinstance(count_arrays[0][a_key], int)\n\n assert isinstance(average_output_count, dict)\n for a_key2 in dict(average_output_count).keys():\n assert isinstance(a_key2, str)\n assert isinstance(average_output_count[a_key2], int)\n\n\n\ndef delete_cache(files: List[str]):\n for file in files:\n if os.path.isfile(file):\n os.remove(file)\n\n\n\n\ndef test_video_detection_yolov3():\n delete_cache([video_file_output + \".mp4\"])\n\n detector = CustomVideoObjectDetection()\n detector.setModelTypeAsYOLOv3()\n detector.setModelPath(model_path=os.path.join(test_folder, \"data-models\", \"yolov3_number-plate-dataset-imageai_mAP-0.57145_epoch-11.pt\"))\n detector.setJsonPath(os.path.join(test_folder, \"data-json\", \"number-plate-dataset-imageai_yolov3_detection_config.json\"))\n detector.loadModel()\n video_path = detector.detectObjectsFromVideo(input_file_path=video_file, output_file_path=video_file_output, save_detected_video=True, frames_per_second=30, log_progress=True)\n\n assert os.path.exists(video_file_output + \".mp4\")\n assert isinstance(video_path, str)\n \n delete_cache([video_file_output + \".mp4\"])\n\n\ndef test_video_detection_tiny_yolov3():\n delete_cache([video_file_output + \".mp4\"])\n\n detector = CustomVideoObjectDetection()\n detector.setModelTypeAsTinyYOLOv3()\n detector.setModelPath(model_path=os.path.join(test_folder, \"data-models\", \"tiny_yolov3_number-plate-dataset-imageai_mAP-0.22595_epoch-20.pt\"))\n detector.setJsonPath(os.path.join(test_folder, \"data-json\", \"number-plate-dataset-imageai_tiny_yolov3_detection_config.json\"))\n detector.loadModel()\n video_path = detector.detectObjectsFromVideo(input_file_path=video_file, output_file_path=video_file_output, save_detected_video=True, frames_per_second=30, log_progress=True)\n\n assert os.path.exists(video_file_output + \".mp4\")\n assert isinstance(video_path, str)\n\n delete_cache([video_file_output + \".mp4\"])\n\n\ndef test_video_detection_yolo_analysis():\n delete_cache([video_file_output + \".mp4\"])\n\n detector = CustomVideoObjectDetection()\n detector.setModelTypeAsYOLOv3()\n detector.setModelPath(model_path=os.path.join(test_folder, \"data-models\", \"yolov3_number-plate-dataset-imageai_mAP-0.57145_epoch-11.pt\"))\n detector.setJsonPath(os.path.join(test_folder, \"data-json\", \"number-plate-dataset-imageai_yolov3_detection_config.json\"))\n detector.loadModel()\n\n with patch.object(CallbackFunctions, 'forFrame') as frameFunc:\n with patch.object(CallbackFunctions, 'forSecond') as secondFunc:\n\n video_path = detector.detectObjectsFromVideo(input_file_path=video_file, output_file_path=video_file_output, save_detected_video=True, frames_per_second=30, log_progress=True, per_frame_function=frameFunc, per_second_function=secondFunc, return_detected_frame=True)\n\n assert os.path.exists(video_file_output + \".mp4\")\n assert isinstance(video_path, str)\n\n frameFunc.assert_called()\n secondFunc.assert_called()\n\n delete_cache([video_file_output + \".mp4\"])\n\n\n" }, { "path": "test/test_image_classification.py", "content": "import os, sys\nimport cv2\nfrom PIL import Image\nimport pytest\nfrom os.path import dirname\nsys.path.insert(1, os.path.join(dirname(dirname(os.path.abspath(__file__)))))\nfrom imageai.Classification import ImageClassification\n\ntest_folder = dirname(os.path.abspath(__file__))\n\n\n\n@pytest.mark.parametrize(\n \"image_input\",\n [\n (os.path.join(test_folder, \"data-images\", \"1.jpg\")),\n (cv2.imread(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n (Image.open(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n ]\n)\ndef test_recognition_model_mobilenetv2(image_input):\n\n classifier = ImageClassification()\n classifier.setModelTypeAsMobileNetV2()\n classifier.setModelPath(os.path.join(test_folder, \"data-models\", \"mobilenet_v2-b0353104.pth\"))\n classifier.loadModel()\n predictions, probabilities = classifier.classifyImage(image_input=image_input)\n\n assert isinstance(predictions, list)\n assert isinstance(probabilities, list)\n assert isinstance(predictions[0], str)\n assert isinstance(probabilities[0], float)\n\n\n@pytest.mark.parametrize(\n \"image_input\",\n [\n (os.path.join(test_folder, \"data-images\", \"1.jpg\")),\n (cv2.imread(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n (Image.open(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n ]\n)\ndef test_recognition_model_resnet(image_input):\n\n classifier = ImageClassification()\n classifier.setModelTypeAsResNet50()\n classifier.setModelPath(os.path.join(test_folder, \"data-models\", \"resnet50-19c8e357.pth\"))\n classifier.loadModel()\n predictions, probabilities = classifier.classifyImage(image_input=image_input)\n\n assert isinstance(predictions, list)\n assert isinstance(probabilities, list)\n assert isinstance(predictions[0], str)\n assert isinstance(probabilities[0], float)\n\n@pytest.mark.parametrize(\n \"image_input\",\n [\n (os.path.join(test_folder, \"data-images\", \"1.jpg\")),\n (cv2.imread(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n (Image.open(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n ]\n)\ndef test_recognition_model_inceptionv3(image_input):\n\n classifier = ImageClassification()\n classifier.setModelTypeAsInceptionV3()\n classifier.setModelPath(os.path.join(test_folder, \"data-models\", \"inception_v3_google-1a9a5a14.pth\"))\n classifier.loadModel()\n predictions, probabilities = classifier.classifyImage(image_input=image_input)\n\n assert isinstance(predictions, list)\n assert isinstance(probabilities, list)\n assert isinstance(predictions[0], str)\n assert isinstance(probabilities[0], float)\n\n@pytest.mark.parametrize(\n \"image_input\",\n [\n (os.path.join(test_folder, \"data-images\", \"1.jpg\")),\n (cv2.imread(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n (Image.open(os.path.join(test_folder, \"data-images\", \"1.jpg\"))),\n ]\n)\ndef test_recognition_model_densenet(image_input):\n\n classifier = ImageClassification()\n classifier.setModelTypeAsDenseNet121()\n classifier.setModelPath(os.path.join(test_folder, \"data-models\", \"densenet121-a639ec97.pth\"))\n classifier.loadModel()\n predictions, probabilities = classifier.classifyImage(image_input=image_input)\n\n assert isinstance(predictions, list)\n assert isinstance(probabilities, list)\n assert isinstance(predictions[0], str)\n assert isinstance(probabilities[0], float)" }, { "path": "test/test_object_detection.py", "content": "import os, sys\nfrom typing import List\nimport shutil\nimport cv2\nimport uuid\nfrom PIL import Image\nimport numpy as np\nimport pytest\nfrom os.path import dirname\nsys.path.insert(1, os.path.join(dirname(dirname(os.path.abspath(__file__)))))\nfrom imageai.Detection import ObjectDetection\n\ntest_folder = dirname(os.path.abspath(__file__))\n\n\ndef delete_cache(paths: List[str]):\n for path in paths:\n if os.path.isfile(path):\n os.remove(path)\n elif os.path.isdir(path):\n shutil.rmtree(path)\n\n\n@pytest.mark.parametrize(\n \"input_image, output_type, extract_objects\",\n [\n (os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\"), \"file\", False),\n (os.path.join(test_folder, test_folder, \"data-images\", \"4.jpg\"), \"file\", False),\n (os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\"), \"file\", True),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\")), \"array\", False),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\")), \"array\", True),\n (Image.open(os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\")), \"array\", True),\n ]\n)\ndef test_object_detection_retinanet(input_image, output_type, extract_objects):\n detector = ObjectDetection()\n detector.setModelTypeAsRetinaNet()\n detector.setModelPath(os.path.join(test_folder, \"data-models\", \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\n detector.loadModel()\n\n output_img_path = os.path.join(test_folder, \"data-images\", str(uuid.uuid4()) + \".jpg\")\n\n if output_type == \"array\":\n if extract_objects:\n output_image_array, detections, extracted_objects = detector.detectObjectsFromImage(input_image=input_image, output_type=output_type, extract_detected_objects=extract_objects)\n\n assert len(extracted_objects) > 1\n for extracted_obj in extracted_objects:\n assert type(extracted_obj) == np.ndarray\n assert type(detections) == list\n else:\n output_image_array, detections = detector.detectObjectsFromImage(input_image=input_image, output_type=output_type)\n assert type(output_image_array) == np.ndarray\n assert type(detections) == list\n else:\n if extract_objects:\n detections, extracted_object_paths = detector.detectObjectsFromImage(input_image=input_image, output_image_path=output_img_path, extract_detected_objects=True)\n\n assert type(detections) == list\n assert os.path.isfile(output_img_path)\n assert len(extracted_object_paths) > 3\n delete_cache(\n extracted_object_paths\n )\n delete_cache(\n [extracted_object_paths[0], output_img_path]\n )\n else:\n detections = detector.detectObjectsFromImage(input_image=input_image, output_image_path=output_img_path)\n assert type(detections) == list\n delete_cache(\n [output_img_path]\n )\n \n\n for eachObject in detections:\n assert type(eachObject) == dict\n assert \"name\" in eachObject.keys()\n assert type(eachObject[\"name\"]) == str \n assert \"percentage_probability\" in eachObject.keys()\n assert type(eachObject[\"percentage_probability\"]) == float\n assert \"box_points\" in eachObject.keys()\n assert type(eachObject[\"box_points\"]) == list\n box_points = eachObject[\"box_points\"]\n for point in box_points:\n assert type(point) == int\n assert box_points[0] < box_points[2]\n assert box_points[1] < box_points[3]\n\n\n@pytest.mark.parametrize(\n \"input_image, output_type, extract_objects\",\n [\n (os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\"), \"file\", False),\n (os.path.join(test_folder, test_folder, \"data-images\", \"4.jpg\"), \"file\", False),\n (os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\"), \"file\", True),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\")), \"array\", False),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\")), \"array\", True),\n (Image.open(os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\")), \"array\", True),\n ]\n)\ndef test_object_detection_yolov3(input_image, output_type, extract_objects):\n detector = ObjectDetection()\n detector.setModelTypeAsYOLOv3()\n detector.setModelPath(os.path.join(test_folder, \"data-models\", \"yolov3.pt\"))\n detector.loadModel()\n\n output_img_path = os.path.join(test_folder, \"data-images\", str(uuid.uuid4()) + \".jpg\")\n\n if output_type == \"array\":\n if extract_objects:\n output_image_array, detections, extracted_objects = detector.detectObjectsFromImage(input_image=input_image, output_type=output_type, extract_detected_objects=extract_objects)\n\n assert len(extracted_objects) > 1\n assert type(detections) == list\n for extracted_obj in extracted_objects:\n assert type(extracted_obj) == np.ndarray\n else:\n output_image_array, detections = detector.detectObjectsFromImage(input_image=input_image, output_type=output_type)\n assert type(output_image_array) == np.ndarray\n assert type(detections) == list\n else:\n if extract_objects:\n detections, extracted_object_paths = detector.detectObjectsFromImage(input_image=input_image, output_image_path=output_img_path, extract_detected_objects=True)\n\n assert os.path.isfile(output_img_path)\n assert len(extracted_object_paths) > 3\n assert type(detections) == list\n delete_cache(\n extracted_object_paths\n )\n delete_cache(\n [extracted_object_paths[0], output_img_path]\n )\n else:\n detections = detector.detectObjectsFromImage(input_image=input_image, output_image_path=output_img_path)\n assert type(detections) == list\n delete_cache(\n [output_img_path]\n )\n\n \n\n for eachObject in detections:\n assert type(eachObject) == dict\n assert \"name\" in eachObject.keys()\n assert type(eachObject[\"name\"]) == str \n assert \"percentage_probability\" in eachObject.keys()\n assert type(eachObject[\"percentage_probability\"]) == float\n assert \"box_points\" in eachObject.keys()\n assert type(eachObject[\"box_points\"]) == list\n box_points = eachObject[\"box_points\"]\n for point in box_points:\n assert type(point) == int\n assert box_points[0] < box_points[2]\n assert box_points[1] < box_points[3]\n\n\n@pytest.mark.parametrize(\n \"input_image, output_type, extract_objects\",\n [\n (os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\"), \"file\", False),\n (os.path.join(test_folder, test_folder, \"data-images\", \"4.jpg\"), \"file\", False),\n (os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\"), \"file\", True),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\")), \"array\", False),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"1.jpg\")), \"array\", True),\n (Image.open(os.path.join(test_folder, test_folder, \"data-images\", \"11.jpg\")), \"array\", True),\n ]\n)\ndef test_object_detection_tiny_yolov3(input_image, output_type, extract_objects):\n detector = ObjectDetection()\n detector.setModelTypeAsTinyYOLOv3()\n detector.setModelPath(os.path.join(test_folder, \"data-models\", \"tiny-yolov3.pt\"))\n detector.loadModel()\n\n\n output_img_path = os.path.join(test_folder, \"data-images\", str(uuid.uuid4()) + \".jpg\")\n\n if output_type == \"array\":\n if extract_objects:\n output_image_array, detections, extracted_objects = detector.detectObjectsFromImage(input_image=input_image, output_type=output_type, extract_detected_objects=extract_objects)\n\n assert len(extracted_objects) > 1\n assert type(detections) == list\n for extracted_obj in extracted_objects:\n assert type(extracted_obj) == np.ndarray\n else:\n output_image_array, detections = detector.detectObjectsFromImage(input_image=input_image, output_type=output_type)\n assert type(output_image_array) == np.ndarray\n assert type(detections) == list\n else:\n if extract_objects:\n detections, extracted_object_paths = detector.detectObjectsFromImage(input_image=input_image, output_image_path=output_img_path, extract_detected_objects=True)\n\n assert os.path.isfile(output_img_path)\n assert len(extracted_object_paths) > 1\n assert type(detections) == list\n delete_cache(\n extracted_object_paths\n )\n delete_cache(\n [extracted_object_paths[0], output_img_path]\n )\n\n else:\n detections = detector.detectObjectsFromImage(input_image=input_image, output_image_path=output_img_path)\n assert type(detections) == list\n delete_cache(\n [output_img_path]\n )\n \n \n\n for eachObject in detections:\n assert type(eachObject) == dict\n assert \"name\" in eachObject.keys()\n assert type(eachObject[\"name\"]) == str \n assert \"percentage_probability\" in eachObject.keys()\n assert type(eachObject[\"percentage_probability\"]) == float\n assert \"box_points\" in eachObject.keys()\n assert type(eachObject[\"box_points\"]) == list\n box_points = eachObject[\"box_points\"]\n for point in box_points:\n assert type(point) == int\n assert box_points[0] < box_points[2]\n assert box_points[1] < box_points[3]\n\n\n@pytest.mark.parametrize(\n \"input_image\",\n [\n (os.path.join(test_folder, test_folder, \"data-images\", \"11.jpg\")),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"11.jpg\"))),\n (Image.open(os.path.join(test_folder, test_folder, \"data-images\", \"11.jpg\"))),\n ]\n)\ndef test_object_detection_retinanet_custom_objects(input_image):\n detector = ObjectDetection()\n detector.setModelTypeAsRetinaNet()\n detector.setModelPath(os.path.join(test_folder, \"data-models\", \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\n detector.loadModel()\n\n custom = detector.CustomObjects(person=True, cell_phone=True)\n\n custom_detections = detector.detectObjectsFromImage(input_image=input_image, custom_objects=custom)\n \n for custom_detection in custom_detections:\n assert custom_detection[\"name\"] in [\"person\", \"cell phone\"]\n\n detections = detector.detectObjectsFromImage(input_image=input_image)\n\n assert len(detections) > len(custom_detections)\n\n\n@pytest.mark.parametrize(\n \"input_image\",\n [\n (os.path.join(test_folder, test_folder, \"data-images\", \"11.jpg\")),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"11.jpg\"))),\n (Image.open(os.path.join(test_folder, test_folder, \"data-images\", \"11.jpg\"))),\n ]\n)\ndef test_object_detection_yolov3_custom_objects(input_image):\n detector = ObjectDetection()\n detector.setModelTypeAsYOLOv3()\n detector.setModelPath(os.path.join(test_folder, \"data-models\", \"yolov3.pt\"))\n detector.loadModel()\n\n custom = detector.CustomObjects(person=True, cell_phone=True)\n\n custom_detections = detector.detectObjectsFromImage(input_image=input_image, custom_objects=custom)\n \n for custom_detection in custom_detections:\n assert custom_detection[\"name\"] in [\"person\", \"cell phone\"]\n\n detections = detector.detectObjectsFromImage(input_image=input_image)\n\n assert len(detections) > len(custom_detections)\n\n\n@pytest.mark.parametrize(\n \"input_image\",\n [\n (os.path.join(test_folder, test_folder, \"data-images\", \"11.jpg\")),\n (cv2.imread(os.path.join(test_folder, test_folder, \"data-images\", \"11.jpg\"))),\n (Image.open(os.path.join(test_folder, test_folder, \"data-images\", \"11.jpg\"))),\n ]\n)\ndef test_object_detection_tiny_yolov3_custom_objects(input_image):\n detector = ObjectDetection()\n detector.setModelTypeAsTinyYOLOv3()\n detector.setModelPath(os.path.join(test_folder, \"data-models\", \"tiny-yolov3.pt\"))\n detector.loadModel()\n\n custom = detector.CustomObjects(person=True, cell_phone=True)\n\n custom_detections = detector.detectObjectsFromImage(input_image=input_image, custom_objects=custom)\n \n for custom_detection in custom_detections:\n assert custom_detection[\"name\"] in [\"person\", \"cell phone\"]\n\n detections = detector.detectObjectsFromImage(input_image=input_image)\n\n assert len(detections) > len(custom_detections)\n\n" }, { "path": "test/test_video_object_detection.py", "content": "import os, sys\nfrom typing import List\nfrom numpy import ndarray\nfrom os.path import dirname\nfrom mock import patch\nsys.path.insert(1, os.path.join(dirname(dirname(os.path.abspath(__file__)))))\n\nfrom imageai.Detection import VideoObjectDetection\n\n\ntest_folder = dirname(os.path.abspath(__file__))\n\nvideo_file = os.path.join(test_folder, \"data-videos\", \"traffic-micro.mp4\")\nvideo_file_output = os.path.join(test_folder, \"data-videos\", \"traffic-micro-detected\")\n\n\n\nclass CallbackFunctions:\n def forFrame(frame_number, output_array, output_count, detected_frame):\n assert isinstance(detected_frame, ndarray)\n assert isinstance(frame_number, int)\n assert isinstance(output_array, list)\n assert isinstance(output_array[0], dict)\n assert isinstance(output_array[0][\"name\"], str)\n assert isinstance(output_array[0][\"percentage_probability\"], float)\n assert isinstance(output_array[0][\"box_points\"], list)\n\n assert isinstance(output_count, dict)\n for a_key in dict(output_count).keys():\n assert isinstance(a_key, str)\n assert isinstance(output_count[a_key], int)\n\n def forSecond(second_number, output_arrays, count_arrays, average_output_count, detected_frame):\n assert isinstance(detected_frame, ndarray)\n assert isinstance(second_number, int)\n assert isinstance(output_arrays, list)\n assert isinstance(output_arrays[0], list)\n\n assert isinstance(output_arrays[0][0], dict)\n assert isinstance(output_arrays[0][0][\"name\"], str)\n assert isinstance(output_arrays[0][0][\"percentage_probability\"], float)\n assert isinstance(output_arrays[0][0][\"box_points\"], list)\n\n assert isinstance(count_arrays, list)\n assert isinstance(count_arrays[0], dict)\n for a_key in dict(count_arrays[0]).keys():\n assert isinstance(a_key, str)\n assert isinstance(count_arrays[0][a_key], int)\n\n assert isinstance(average_output_count, dict)\n for a_key2 in dict(average_output_count).keys():\n assert isinstance(a_key2, str)\n assert isinstance(average_output_count[a_key2], int)\n\n\n\ndef delete_cache(files: List[str]):\n for file in files:\n if os.path.isfile(file):\n os.remove(file)\n\n\ndef test_video_detection_retinanet():\n\n delete_cache([video_file_output + \".mp4\"])\n\n detector = VideoObjectDetection()\n detector.setModelTypeAsRetinaNet()\n detector.setModelPath(model_path=os.path.join(test_folder, \"data-models\", \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\n detector.loadModel()\n video_path = detector.detectObjectsFromVideo(input_file_path=video_file, output_file_path=video_file_output, save_detected_video=True, frames_per_second=30, log_progress=True)\n\n assert os.path.exists(video_file_output + \".mp4\")\n assert isinstance(video_path, str)\n\n delete_cache([video_file_output + \".mp4\"])\n\n\ndef test_video_detection_retinanet_custom_objects():\n\n delete_cache([video_file_output + \".mp4\"])\n\n detector = VideoObjectDetection()\n detector.setModelTypeAsRetinaNet()\n detector.setModelPath(model_path=os.path.join(test_folder, \"data-models\", \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\n detector.loadModel()\n\n custom_objects = detector.CustomObjects(\n person=True,\n bus=True\n )\n\n video_path = detector.detectObjectsFromVideo(input_file_path=video_file, output_file_path=video_file_output, save_detected_video=True, frames_per_second=30, log_progress=True, custom_objects=custom_objects)\n\n assert os.path.exists(video_file_output + \".mp4\")\n assert isinstance(video_path, str)\n\n delete_cache([video_file_output + \".mp4\"])\n\n\n\n\ndef test_video_detection_yolov3():\n delete_cache([video_file_output + \".mp4\"])\n\n detector = VideoObjectDetection()\n detector.setModelTypeAsYOLOv3()\n detector.setModelPath(model_path=os.path.join(test_folder, \"data-models\", \"yolov3.pt\"))\n detector.loadModel()\n video_path = detector.detectObjectsFromVideo(input_file_path=video_file, output_file_path=video_file_output, save_detected_video=True, frames_per_second=30, log_progress=True)\n\n assert os.path.exists(video_file_output + \".mp4\")\n assert isinstance(video_path, str)\n \n delete_cache([video_file_output + \".mp4\"])\n\n\n\ndef test_video_detection_tiny_yolov3():\n delete_cache([video_file_output + \".mp4\"])\n\n detector = VideoObjectDetection()\n detector.setModelTypeAsTinyYOLOv3()\n detector.setModelPath(model_path=os.path.join(test_folder, \"data-models\", \"tiny-yolov3.pt\"))\n detector.loadModel()\n video_path = detector.detectObjectsFromVideo(input_file_path=video_file, output_file_path=video_file_output, save_detected_video=True, frames_per_second=30, log_progress=True)\n\n assert os.path.exists(video_file_output + \".mp4\")\n assert isinstance(video_path, str)\n\n delete_cache([video_file_output + \".mp4\"])\n\n\ndef test_video_detection_retinanet_analysis():\n delete_cache([video_file_output + \".mp4\"])\n\n detector = VideoObjectDetection()\n detector.setModelTypeAsRetinaNet()\n detector.setModelPath(model_path=os.path.join(test_folder, \"data-models\", \"retinanet_resnet50_fpn_coco-eeacb38b.pth\"))\n detector.loadModel()\n\n with patch.object(CallbackFunctions, 'forFrame') as frameFunc:\n with patch.object(CallbackFunctions, 'forSecond') as secondFunc:\n\n video_path = detector.detectObjectsFromVideo(input_file_path=video_file, output_file_path=video_file_output, save_detected_video=True, frames_per_second=30, log_progress=True, per_frame_function=frameFunc, per_second_function=secondFunc, return_detected_frame=True)\n\n assert os.path.exists(video_file_output + \".mp4\")\n assert isinstance(video_path, str)\n\n frameFunc.assert_called()\n secondFunc.assert_called()\n\n delete_cache([video_file_output + \".mp4\"])\n\n\n" } ]