Showing preview only (1,288K chars total). Download the full file or copy to clipboard to get everything.
Repository: facebookresearch/SlowFast
Branch: main
Commit: 287ec0076846
Files: 220
Total size: 1.2 MB
Directory structure:
gitextract_bls1cq2k/
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── GETTING_STARTED.md
├── INSTALL.md
├── LICENSE
├── MODEL_ZOO.md
├── README.md
├── VISUALIZATION_TOOLS.md
├── ava_evaluation/
│ ├── README.md
│ ├── ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt
│ ├── label_map_util.py
│ ├── metrics.py
│ ├── np_box_list.py
│ ├── np_box_list_ops.py
│ ├── np_box_mask_list.py
│ ├── np_box_mask_list_ops.py
│ ├── np_box_ops.py
│ ├── np_mask_ops.py
│ ├── object_detection_evaluation.py
│ ├── per_image_evaluation.py
│ └── standard_fields.py
├── configs/
│ ├── AVA/
│ │ ├── SLOWFAST_32x2_R50_SHORT.yaml
│ │ ├── SLOW_8x8_R50_SHORT.yaml
│ │ └── c2/
│ │ ├── SLOWFAST_32x2_R101_50_50.yaml
│ │ ├── SLOWFAST_32x2_R101_50_50_v2.1.yaml
│ │ ├── SLOWFAST_32x2_R50.yaml
│ │ ├── SLOWFAST_64x2_R101_50_50.yaml
│ │ └── SLOW_8x8_R50.yaml
│ ├── Charades/
│ │ ├── SLOWFAST_16x8_R50.yaml
│ │ ├── SLOWFAST_16x8_R50_multigrid.yaml
│ │ └── pytorchvideo/
│ │ ├── SLOWFAST_8x8_R50.yaml
│ │ └── SLOW_8x8_R50.yaml
│ ├── ImageNet/
│ │ ├── MVIT_B_16_CONV.yaml
│ │ ├── MVITv2_B.yaml
│ │ ├── MVITv2_S.yaml
│ │ ├── MVITv2_T.yaml
│ │ ├── RES_R50.yaml
│ │ ├── REV_MVIT_B_16_CONV.yaml
│ │ ├── REV_VIT_B.yaml
│ │ └── REV_VIT_S.yaml
│ ├── Kinetics/
│ │ ├── C2D_8x8_R50.yaml
│ │ ├── C2D_8x8_R50_IN1K.yaml
│ │ ├── C2D_NLN_8x8_R50.yaml
│ │ ├── C2D_NLN_8x8_R50_IN1K.yaml
│ │ ├── I3D_8x8_R101.yaml
│ │ ├── I3D_8x8_R50.yaml
│ │ ├── I3D_8x8_R50_IN1K.yaml
│ │ ├── I3D_NLN_8x8_R101.yaml
│ │ ├── I3D_NLN_8x8_R50.yaml
│ │ ├── I3D_NLN_8x8_R50_IN1K.yaml
│ │ ├── MVIT_B_16x4_CONV.yaml
│ │ ├── MVIT_B_32x3_CONV.yaml
│ │ ├── MVITv2_B_32x3.yaml
│ │ ├── MVITv2_L_40x3_test.yaml
│ │ ├── MVITv2_S_16x4.yaml
│ │ ├── REV_MVIT_B_16x4_CONV.yaml
│ │ ├── SLOWFAST_4x16_R50.yaml
│ │ ├── SLOWFAST_8x8_R50.yaml
│ │ ├── SLOWFAST_8x8_R50_stepwise.yaml
│ │ ├── SLOWFAST_8x8_R50_stepwise_multigrid.yaml
│ │ ├── SLOWFAST_NLN_4x16_R50.yaml
│ │ ├── SLOWFAST_NLN_8x8_R50.yaml
│ │ ├── SLOW_4x16_R50.yaml
│ │ ├── SLOW_8x8_R50.yaml
│ │ ├── SLOW_NLN_4x16_R50.yaml
│ │ ├── SLOW_NLN_8x8_R50.yaml
│ │ ├── X3D_L.yaml
│ │ ├── X3D_M.yaml
│ │ ├── X3D_S.yaml
│ │ ├── X3D_XS.yaml
│ │ ├── c2/
│ │ │ ├── C2D_NOPOOL_8x8_R50.yaml
│ │ │ ├── I3D_8x8_R50.yaml
│ │ │ ├── I3D_NLN_8x8_R50.yaml
│ │ │ ├── SLOWFAST_16x8_R101_50_50.yaml
│ │ │ ├── SLOWFAST_4x16_R50.yaml
│ │ │ ├── SLOWFAST_8x8_R101_101_101.yaml
│ │ │ ├── SLOWFAST_8x8_R101_50_101.yaml
│ │ │ ├── SLOWFAST_8x8_R101_50_50.yaml
│ │ │ ├── SLOWFAST_8x8_R50.yaml
│ │ │ ├── SLOWFAST_NLN_16x8_R101_50_50.yaml
│ │ │ ├── SLOW_4x16_R50.yaml
│ │ │ └── SLOW_8x8_R50.yaml
│ │ └── pytorchvideo/
│ │ ├── C2D_8x8_R50.yaml
│ │ ├── CSN_32x2_R101.yaml
│ │ ├── I3D_8x8_R50.yaml
│ │ ├── MVIT_B_16x4_CONV.yaml
│ │ ├── R2PLUS1D_16x4_R50.yaml
│ │ ├── SLOWFAST_16x8_R101_50_50.yaml
│ │ ├── SLOWFAST_4x16_R50.yaml
│ │ ├── SLOWFAST_8x8_R101.yaml
│ │ ├── SLOWFAST_8x8_R50.yaml
│ │ ├── SLOW_4x16_R50.yaml
│ │ ├── SLOW_8x8_R50.yaml
│ │ ├── X3D_L.yaml
│ │ ├── X3D_M.yaml
│ │ ├── X3D_S.yaml
│ │ └── X3D_XS.yaml
│ ├── SSv2/
│ │ ├── MVITv2_B_32x3.yaml
│ │ ├── MVITv2_L_40x3.yaml
│ │ ├── MVITv2_S_16x4.yaml
│ │ ├── SLOWFAST_16x8_R50.yaml
│ │ ├── SLOWFAST_16x8_R50_multigrid.yaml
│ │ └── pytorchvideo/
│ │ ├── SLOWFAST_8x8_R50.yaml
│ │ └── SLOW_8x8_R50.yaml
│ ├── contrastive_ssl/
│ │ ├── BYOL_SlowR50_8x8.yaml
│ │ ├── MoCo_SlowR50_8x8.yaml
│ │ ├── SimCLR_SlowR50_8x8.yaml
│ │ ├── SwAV_Slow_R50_8x8.yaml
│ │ ├── finetune_SSv2_Slow_R50_syn0.yaml
│ │ ├── finetune_SSv2_Slow_R50_syn8.yaml
│ │ ├── finetune_ava_Slow_R50_syn0.yaml
│ │ ├── finetune_ava_Slow_R50_syn8.yaml
│ │ ├── finetune_ucf_Slow_R50_syn0.yaml
│ │ ├── finetune_ucf_Slow_R50_syn8.yaml
│ │ ├── linear_k400_Slow_8x8_R50_syn0.yaml
│ │ └── linear_k400_Slow_8x8_R50_syn8.yaml
│ └── masked_ssl/
│ ├── in1k_VIT_B_MaskFeat_FT.yaml
│ ├── in1k_VIT_B_MaskFeat_PT.yaml
│ ├── in1k_VIT_L_MaskFeat_FT.yaml
│ ├── in1k_VIT_L_MaskFeat_PT.yaml
│ ├── k400_MVITv2_L_16x4_FT.yaml
│ ├── k400_MVITv2_L_16x4_MaskFeat_PT.yaml
│ ├── k400_MVITv2_S_16x4_FT.yaml
│ ├── k400_MVITv2_S_16x4_MaskFeat_PT.yaml
│ ├── k400_VIT_B_16x4_FT.yaml
│ ├── k400_VIT_B_16x4_MAE_PT.yaml
│ ├── k400_VIT_H_16x4_FT.yaml
│ ├── k400_VIT_H_16x4_MAE_PT.yaml
│ ├── k400_VIT_L_16x4_FT.yaml
│ └── k400_VIT_L_16x4_MAE_PT.yaml
├── demo/
│ ├── AVA/
│ │ └── SLOWFAST_32x2_R101_50_50.yaml
│ └── Kinetics/
│ └── SLOWFAST_8x8_R50.yaml
├── linter.sh
├── projects/
│ ├── contrastive_ssl/
│ │ └── README.md
│ ├── mae/
│ │ └── README.md
│ ├── maskfeat/
│ │ └── README.md
│ ├── multigrid/
│ │ └── README.md
│ ├── mvit/
│ │ └── README.md
│ ├── mvitv2/
│ │ └── README.md
│ ├── pytorchvideo/
│ │ └── README.md
│ ├── rev/
│ │ └── README.md
│ └── x3d/
│ └── README.md
├── setup.cfg
├── setup.py
├── slowfast/
│ ├── __init__.py
│ ├── config/
│ │ ├── __init__.py
│ │ ├── custom_config.py
│ │ └── defaults.py
│ ├── datasets/
│ │ ├── DATASET.md
│ │ ├── __init__.py
│ │ ├── ava_dataset.py
│ │ ├── ava_helper.py
│ │ ├── build.py
│ │ ├── charades.py
│ │ ├── cv2_transform.py
│ │ ├── decoder.py
│ │ ├── imagenet.py
│ │ ├── kinetics.py
│ │ ├── loader.py
│ │ ├── mixup.py
│ │ ├── multigrid_helper.py
│ │ ├── ptv_datasets.py
│ │ ├── rand_augment.py
│ │ ├── random_erasing.py
│ │ ├── ssv2.py
│ │ ├── transform.py
│ │ ├── utils.py
│ │ └── video_container.py
│ ├── models/
│ │ ├── __init__.py
│ │ ├── attention.py
│ │ ├── batchnorm_helper.py
│ │ ├── build.py
│ │ ├── common.py
│ │ ├── contrastive.py
│ │ ├── custom_video_model_builder.py
│ │ ├── head_helper.py
│ │ ├── losses.py
│ │ ├── masked.py
│ │ ├── nonlocal_helper.py
│ │ ├── operators.py
│ │ ├── optimizer.py
│ │ ├── ptv_model_builder.py
│ │ ├── resnet_helper.py
│ │ ├── reversible_mvit.py
│ │ ├── stem_helper.py
│ │ ├── utils.py
│ │ └── video_model_builder.py
│ ├── utils/
│ │ ├── __init__.py
│ │ ├── ava_eval_helper.py
│ │ ├── benchmark.py
│ │ ├── bn_helper.py
│ │ ├── c2_model_loading.py
│ │ ├── checkpoint.py
│ │ ├── distributed.py
│ │ ├── env.py
│ │ ├── logging.py
│ │ ├── lr_policy.py
│ │ ├── meters.py
│ │ ├── metrics.py
│ │ ├── misc.py
│ │ ├── multigrid.py
│ │ ├── multiprocessing.py
│ │ ├── parser.py
│ │ └── weight_init_helper.py
│ └── visualization/
│ ├── __init__.py
│ ├── async_predictor.py
│ ├── ava_demo_precomputed_boxes.py
│ ├── demo_loader.py
│ ├── gradcam_utils.py
│ ├── prediction_vis.py
│ ├── predictor.py
│ ├── tensorboard_vis.py
│ ├── utils.py
│ └── video_visualizer.py
└── tools/
├── benchmark.py
├── demo_net.py
├── run_net.py
├── test_net.py
├── train_net.py
└── visualization.py
================================================
FILE CONTENTS
================================================
================================================
FILE: CODE_OF_CONDUCT.md
================================================
# Code of Conduct
Facebook has adopted a Code of Conduct that we expect project participants to adhere to.
Please read the [full text](https://code.fb.com/codeofconduct/)
so that you can understand what actions will and will not be tolerated.
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing to PySlowFast
We want to make contributing to this project as easy and transparent as
possible.
## Pull Requests
We actively welcome your pull requests.
1. Fork the repo and create your branch from `master`.
2. If you've changed APIs, update the documentation.
3. Ensure the test suite passes.
4. Make sure your code lints.
5. Ensure no regressions in baseline model speed and accuracy.
6. If you haven't already, complete the Contributor License Agreement ("CLA").
## Contributor License Agreement ("CLA")
In order to accept your pull request, we need you to submit a CLA. You only need
to do this once to work on any of Facebook's open source projects.
Complete your CLA here: <https://code.facebook.com/cla>
## Issues
Please ensure your description is clear and has sufficient instructions to be able to reproduce the issue. The recommended issue format is:
------
#### To Reproduce
```How to reproduce the issue.```
#### Expected behavior
```Expected output.```
#### Environment
```Your environment.```
------
## Coding Style
* 4 spaces for indentation rather than tabs
* 80 character line length
* PEP8 formatting
## License
By contributing to PySlowFast, you agree that your contributions will be licensed under the LICENSE file in the root directory of this source tree.
================================================
FILE: GETTING_STARTED.md
================================================
# Getting Started with PySlowFast
This document provides a brief intro of launching jobs in PySlowFast for training and testing. Before launching any job, make sure you have properly installed the PySlowFast following the instruction in [README.md](README.md) and you have prepared the dataset following [DATASET.md](slowfast/datasets/DATASET.md) with the correct format.
## Train a Standard Model from Scratch
Here we can start with training a simple C2D models by running:
```
python tools/run_net.py \
--cfg configs/Kinetics/C2D_8x8_R50.yaml \
DATA.PATH_TO_DATA_DIR path_to_your_dataset \
NUM_GPUS 2 \
TRAIN.BATCH_SIZE 16 \
```
You may need to pass location of your dataset in the command line by adding `DATA.PATH_TO_DATA_DIR path_to_your_dataset`, or you can simply add
```
DATA:
PATH_TO_DATA_DIR: path_to_your_dataset
```
To the yaml configs file, then you do not need to pass it to the command line every time.
You may also want to add:
```
DATA_LOADER.NUM_WORKERS 0 \
NUM_GPUS 2 \
TRAIN.BATCH_SIZE 16 \
```
If you want to launch a quick job for debugging on your local machine.
## Resume from an Existing Checkpoint
If your checkpoint is trained by PyTorch, then you can add the following line in the command line, or you can also add it in the YAML config:
```
TRAIN.CHECKPOINT_FILE_PATH path_to_your_PyTorch_checkpoint
```
If the checkpoint in trained by Caffe2, then you can do the following:
```
TRAIN.CHECKPOINT_FILE_PATH path_to_your_Caffe2_checkpoint \
TRAIN.CHECKPOINT_TYPE caffe2
```
If you need to performance inflation on the checkpoint, remember to set `TRAIN.CHECKPOINT_INFLATE` to True.
## Perform Test
We have `TRAIN.ENABLE` and `TEST.ENABLE` to control whether training or testing is required for the current job. If only testing is preferred, you can set the `TRAIN.ENABLE` to False, and do not forget to pass the path to the model you want to test to TEST.CHECKPOINT_FILE_PATH.
```
python tools/run_net.py \
--cfg configs/Kinetics/C2D_8x8_R50.yaml \
DATA.PATH_TO_DATA_DIR path_to_your_dataset \
TEST.CHECKPOINT_FILE_PATH path_to_your_checkpoint \
TRAIN.ENABLE False \
```
### Run command
```
python \tools\run_net.py --cfg path/to/<pretrained_model_config_file>.yaml
```
================================================
FILE: INSTALL.md
================================================
# Installation
## Requirements
- Python >= 3.8
- Numpy
- PyTorch >= 1.3
- [fvcore](https://github.com/facebookresearch/fvcore/): `pip install 'git+https://github.com/facebookresearch/fvcore'`
- [torchvision](https://github.com/pytorch/vision/) that matches the PyTorch installation.
You can install them together at [pytorch.org](https://pytorch.org) to make sure of this.
- simplejson: `pip install simplejson`
- GCC >= 4.9
- PyAV: `conda install av -c conda-forge`
- ffmpeg (4.0 is prefereed, will be installed along with PyAV)
- PyYaml: (will be installed along with fvcore)
- tqdm: (will be installed along with fvcore)
- iopath: `pip install -U iopath` or `conda install -c iopath iopath`
- psutil: `pip install psutil`
- OpenCV: `pip install opencv-python`
- torchvision: `pip install torchvision` or `conda install torchvision -c pytorch`
- tensorboard: `pip install tensorboard`
- moviepy: (optional, for visualizing video on tensorboard) `conda install -c conda-forge moviepy` or `pip install moviepy`
- PyTorchVideo: `pip install pytorchvideo`
- [Detectron2](https://github.com/facebookresearch/detectron2):
- FairScale: `pip install 'git+https://github.com/facebookresearch/fairscale'`
```
pip install -U torch torchvision cython
pip install -U 'git+https://github.com/facebookresearch/fvcore.git' 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'
git clone https://github.com/facebookresearch/detectron2 detectron2_repo
pip install -e detectron2_repo
# You can find more details at https://github.com/facebookresearch/detectron2/blob/master/INSTALL.md
```
## Pytorch
Please follow PyTorch official instructions to install from source:
```
git clone --recursive https://github.com/pytorch/pytorch
```
## PySlowFast
Clone the PySlowFast Video Understanding repository.
```
git clone https://github.com/facebookresearch/slowfast
```
Add this repository to $PYTHONPATH.
```
export PYTHONPATH=/path/to/SlowFast/slowfast:$PYTHONPATH
```
### Build PySlowFast
After having the above dependencies, run:
```
git clone https://github.com/facebookresearch/slowfast
cd SlowFast
python setup.py build develop
```
Now the installation is finished, run the pipeline with:
```
python tools/run_net.py --cfg configs/Kinetics/C2D_8x8_R50.yaml NUM_GPUS 1 TRAIN.BATCH_SIZE 8 SOLVER.BASE_LR 0.0125 DATA.PATH_TO_DATA_DIR path_to_your_data_folder
```
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2019, Facebook, Inc
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: MODEL_ZOO.md
================================================
# PySlowFast Model Zoo and Baselines
## Kinetics 400 and 600
| architecture | size | crops x clips | frame length x sample rate | top1 | top5 | model | config | dataset |
| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |
| C2D | R50 | 3 x 10 | 8 x 8 | 67.2 | 87.8 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/C2D_NOPOOL_8x8_R50.pkl) | Kinetics/c2/C2D_NOPOOL_8x8_R50 | K400 |
| I3D | R50 | 3 x 10 | 8 x 8 | 73.5 | 90.8 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/I3D_8x8_R50.pkl) | Kinetics/c2/I3D_8x8_R50 | K400 |
| I3D NLN | R50 | 3 x 10 | 8 x 8 | 74.0 | 91.1 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/I3D_NLN_8x8_R50.pkl) | Kinetics/c2/I3D_NLN_8x8_R50 | K400 |
| Slow | R50 | 3 x 10 | 4 x 16 | 72.7 | 90.3 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/SLOWONLY_4x16_R50.pkl) | Kinetics/c2/SLOW_4x16_R50 | K400 |
| Slow | R50 | 3 x 10 | 8 x 8 | 74.8 | 91.6 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/SLOWONLY_8x8_R50.pkl) | Kinetics/c2/SLOW_8x8_R50 | K400 |
| SlowFast | R50 | 3 x 10 | 4 x 16 | 75.6 | 92.0 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/SLOWFAST_4x16_R50.pkl) | Kinetics/c2/SLOWFAST_4x16_R50 | K400 |
| SlowFast | R50 | 3 x 10 | 8 x 8 | 77.0 | 92.6 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/SLOWFAST_8x8_R50.pkl) | Kinetics/c2/SLOWFAST_8x8_R50 | K400 |
| MViTv1 | B-Conv | 1 x 5 | 16 x 4 | 78.4 | 93.5 | [`link`](https://drive.google.com/file/d/194gJinVejq6A1FmySNKQ8vAN5-FOY-QL/view?usp=sharing) | Kinetics/MVIT_B_16x4_CONV | K400 |
| rev-MViT | B-Conv | 1 x 5 | 16 x 4 | 78.4 | 93.4 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_MVIT_B_16x4.pyth) | Kinetics/REV_MVIT_B_16x4_CONV | K400 |
| MViTv1 | B-Conv | 1 x 5 | 32 x 3 | 80.4 | 94.8 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvit/k400.pyth) | Kinetics/MVIT_B_32x3_CONV | K400 |
| MViTv1 | B-Conv | 1 x 5 | 32 x 3 | 83.9 | 96.5 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvit/k600.pyth) | Kinetics/MVIT_B_32x3_CONV_K600 | K600 |
| MViTv2 | S | 1 x 5 | 16 x 4 | 81.0 | 94.6 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvitv2/pysf_video_models/MViTv2_S_16x4_k400_f302660347.pyth) | Kinetics/MVITv2_S_16x4 | K400 |
| MViTv2 | B | 1 x 5 | 32 x 3 | 82.9 | 95.7 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/mvitv2/pysf_video_models/MViTv2_B_32x3_k400_f304025456.pyth) | Kinetics/MVITv2_B_32x3 | K400 |
## X3D models (details in projects/x3d)
| architecture | size | pretrain | frame length x sample rate | top1 10-view | top1 30-view | parameters (M) | FLOPs (G) | model | config |
| :-------------: | :-----: | :-----: | :-------------: | :------: | :------: | :------------: | :----: | :------: | :------: |
| X3D | XS | - | 4 x 12 | 68.7 | 69.5 | 3.8 | 0.60 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/x3d_models/x3d_xs.pyth) | Kinetics/X3D_XS |
| X3D | S | - | 13 x 6 | 73.1 | 73.5 | 3.8 | 1.96 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/x3d_models/x3d_s.pyth) | Kinetics/X3D_S |
| X3D | M | - | 16 x 5 | 75.1 | 76.2 | 3.8 | 4.73 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/x3d_models/x3d_m.pyth) | Kinetics/X3D_M |
| X3D | L | - | 16 x 5 | 76.9 | 77.5 | 6.2 | 18.37 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/x3d_models/x3d_l.pyth) | Kinetics/X3D_L |
## AVA
| architecture | size | Pretrain Model | frame length x sample rate | MAP | AVA version | model |
| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |------------- |
| Slow | R50 | [Kinetics 400](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/ava/pretrain/C2D_8x8_R50.pkl) | 4 x 16 | 19.5 | 2.2 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/ava/C2D_8x8_R50.pkl) |
| SlowFast | R101 | [Kinetics 600](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/ava/pretrain/SLOWFAST_32x2_R101_50_50_v2.1.pkl) | 8 x 8 | 28.2 | 2.1 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/ava/SLOWFAST_32x2_R101_50_50_v2.1.pkl) |
| SlowFast | R101 | [Kinetics 600](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/ava/pretrain/SLOWFAST_32x2_R101_50_50.pkl) | 8 x 8 | 29.1 | 2.2 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/ava/SLOWFAST_32x2_R101_50_50.pkl) |
| SlowFast | R101 | [Kinetics 600](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/ava/pretrain/SLOWFAST_64x2_R101_50_50.pkl) | 16 x 8 | 29.4 | 2.2 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/ava/SLOWFAST_64x2_R101_50_50.pkl) |
## Multigrid Training
***Update June, 2020:*** In the following we provide (reimplemented) models from "[A Multigrid Method for Efficiently Training Video Models
](https://arxiv.org/abs/1912.00998)" paper. The multigrid method trains about 3-6x faster than the original training on multiple datasets. See [projects/multigrid](projects/multigrid/README.md) for more information. The following provides models, results, and example config files.
#### Kinetics:
| architecture | size | pretrain | frame length x sample rate | training | top1 | top5 | model | config |
| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |
| SlowFast | R50 | - | 8 x 8 | Standard | 76.8 | 92.7 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/pyslowfast/model_zoo/multigrid/model_zoo/Kinetics/SLOWFAST_8x8_R50_stepwise.pkl) | Kinetics/SLOWFAST_8x8_R50_stepwise |
| SlowFast | R50 | - | 8 x 8 | Multigrid | 76.6 | 92.7 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/pyslowfast/model_zoo/multigrid/model_zoo/Kinetics/SLOWFAST_8x8_R50_stepwise_multigrid.pkl) | Kinetics/SLOWFAST_8x8_R50_stepwise_multigrid |
(Here we use stepwise learning rate schedule.)
#### Something-Something V2:
| architecture | size | pretrain | frame length x sample rate | training | top1 | top5 | model | config |
| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |
| SlowFast | R50 | [Kinetics 400](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/SLOWFAST_8x8_R50.pkl) | 16 x 8 | Standard | 63.0 | 88.5 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/pyslowfast/model_zoo/multigrid/model_zoo/SSv2/SLOWFAST_16x8_R50.pkl) | SSv2/SLOWFAST_16x8_R50 |
| SlowFast | R50 | [Kinetics 400](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/SLOWFAST_8x8_R50.pkl) | 16 x 8 | Multigrid | 63.5 | 88.7 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/pyslowfast/model_zoo/multigrid/model_zoo/SSv2/SLOWFAST_16x8_R50_multigrid.pkl) | SSv2/SLOWFAST_16x8_R50_multigrid |
#### Charades
| architecture | size | pretrain | frame length x sample rate | training | mAP | model | config |
| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |
| SlowFast | R50 | [Kinetics 400](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/SLOWFAST_8x8_R50.pkl) | 16 x 8 | Standard | 38.9 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/pyslowfast/model_zoo/multigrid/model_zoo/Charades/SLOWFAST_16x8_R50.pkl) | SSv2/SLOWFAST_16x8_R50 |
| SlowFast | R50 | [Kinetics 400](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/SLOWFAST_8x8_R50.pkl) | 16 x 8 | Multigrid | 38.6 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/pyslowfast/model_zoo/multigrid/model_zoo/Charades/SLOWFAST_16x8_R50_multigrid.pkl) | SSv2/SLOWFAST_16x8_R50_multigrid |
## ImageNet
We also release the imagenet pretrained model if finetuning from ImageNet is preferred. The reported accuracy is obtained by center crop testing on the validation set.
| architecture | size | Top1 | Top5 | model | Config |
| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |
| ResNet | R50 | 76.4 | 93.2 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/model_zoo/kinetics400/R50_IN1K.pyth) | ImageNet/RES_R50 |
| MVIT | B-16-Conv | 82.9 | 96.3 | [`link`](https://drive.google.com/file/d/1dYYqUB-3DSgBVc9d6o-rW8ojtVsrFLgp/view?usp=sharing) | ImageNet/MVIT_B_16_CONV |
| rev-VIT | Small | 79.9 | 94.9 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_VIT_S.pyth) | ImageNet/REV_VIT_S.yaml |
| rev-VIT | Base | 81.8 | 95.6 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_VIT_B.pyth) | ImageNet/REV_VIT_B.yaml |
| rev-MVIT | Base | 82.9* | 96.3 | [`link`](https://dl.fbaipublicfiles.com/pyslowfast/rev/REV_MVIT_B.pyth) | ImageNet/REV_MVIT_B_16_CONV.yaml |
*please refer to [Reversible Model Zoo](projects/rev/README.md).
## PyTorchVideo
We support and benchmark PyTorchVideo models and datasets in PySlowFast. See [projects/pytorchvideo](projects/pytorchvideo/README.md) for more information about PyTorchVideo Model Zoo.
================================================
FILE: README.md
================================================
# PySlowFast
PySlowFast is an open source video understanding codebase from FAIR that provides state-of-the-art video classification models with efficient training. This repository includes implementations of the following methods:
- [SlowFast Networks for Video Recognition](https://arxiv.org/abs/1812.03982)
- [Non-local Neural Networks](https://arxiv.org/abs/1711.07971)
- [A Multigrid Method for Efficiently Training Video Models](https://arxiv.org/abs/1912.00998)
- [X3D: Progressive Network Expansion for Efficient Video Recognition](https://arxiv.org/abs/2004.04730)
- [Multiscale Vision Transformers](https://arxiv.org/abs/2104.11227)
- [A Large-Scale Study on Unsupervised Spatiotemporal Representation Learning](https://arxiv.org/abs/2104.14558)
- [MViTv2: Improved Multiscale Vision Transformers for Classification and Detection](https://arxiv.org/abs/2112.01526)
- [Masked Feature Prediction for Self-Supervised Visual Pre-Training](https://arxiv.org/abs/2112.09133)
- [Masked Autoencoders As Spatiotemporal Learners](https://arxiv.org/abs/2205.09113)
- [Reversible Vision Transformers](https://openaccess.thecvf.com/content/CVPR2022/papers/Mangalam_Reversible_Vision_Transformers_CVPR_2022_paper.pdf)
<div align="center">
<img src="demo/ava_demo.gif" width="600px"/>
</div>
## Introduction
The goal of PySlowFast is to provide a high-performance, light-weight pytorch codebase provides state-of-the-art video backbones for video understanding research on different tasks (classification, detection, and etc). It is designed in order to support rapid implementation and evaluation of novel video research ideas. PySlowFast includes implementations of the following backbone network architectures:
- SlowFast
- Slow
- C2D
- I3D
- Non-local Network
- X3D
- MViTv1 and MViTv2
- Rev-ViT and Rev-MViT
## Updates
- We now [Reversible Vision Transformers](https://openaccess.thecvf.com/content/CVPR2022/papers/Mangalam_Reversible_Vision_Transformers_CVPR_2022_paper.pdf). Both Reversible ViT and MViT models released. See [`projects/rev`](./projects/rev/README.md).
- We now support [MAE for Video](https://arxiv.org/abs/2104.11227.pdf). See [`projects/mae`](./projects/mae/README.md) for more information.
- We now support [MaskFeat](https://arxiv.org/abs/2112.09133). See [`projects/maskfeat`](./projects/maskfeat/README.md) for more information.
- We now support [MViTv2](https://arxiv.org/abs/2104.11227.pdf) in PySlowFast. See [`projects/mvitv2`](./projects/mvitv2/README.md) for more information.
- We now support [A Large-Scale Study on Unsupervised Spatiotemporal Representation Learning](https://arxiv.org/abs/2104.14558). See [`projects/contrastive_ssl`](./projects/contrastive_ssl/README.md) for more information.
- We now support [Multiscale Vision Transformers](https://arxiv.org/abs/2104.11227.pdf) on Kinetics and ImageNet. See [`projects/mvit`](./projects/mvit/README.md) for more information.
- We now support [PyTorchVideo](https://github.com/facebookresearch/pytorchvideo) models and datasets. See [`projects/pytorchvideo`](./projects/pytorchvideo/README.md) for more information.
- We now support [X3D Models](https://arxiv.org/abs/2004.04730). See [`projects/x3d`](./projects/x3d/README.md) for more information.
- We now support [Multigrid Training](https://arxiv.org/abs/1912.00998) for efficiently training video models. See [`projects/multigrid`](./projects/multigrid/README.md) for more information.
- PySlowFast is released in conjunction with our [ICCV 2019 Tutorial](https://alexander-kirillov.github.io/tutorials/visual-recognition-iccv19/).
## License
PySlowFast is released under the [Apache 2.0 license](LICENSE).
## Model Zoo and Baselines
We provide a large set of baseline results and trained models available for download in the PySlowFast [Model Zoo](MODEL_ZOO.md).
## Installation
Please find installation instructions for PyTorch and PySlowFast in [INSTALL.md](INSTALL.md). You may follow the instructions in [DATASET.md](slowfast/datasets/DATASET.md) to prepare the datasets.
## Quick Start
Follow the example in [GETTING_STARTED.md](GETTING_STARTED.md) to start playing video models with PySlowFast.
## Visualization Tools
We offer a range of visualization tools for the train/eval/test processes, model analysis, and for running inference with trained model.
More information at [Visualization Tools](VISUALIZATION_TOOLS.md).
## Contributors
PySlowFast is written and maintained by [Haoqi Fan](https://haoqifan.github.io/), [Yanghao Li](https://lyttonhao.github.io/), [Bo Xiong](https://www.cs.utexas.edu/~bxiong/), [Wan-Yen Lo](https://www.linkedin.com/in/wanyenlo/), [Christoph Feichtenhofer](https://feichtenhofer.github.io/).
## Citing PySlowFast
If you find PySlowFast useful in your research, please use the following BibTeX entry for citation.
```BibTeX
@misc{fan2020pyslowfast,
author = {Haoqi Fan and Yanghao Li and Bo Xiong and Wan-Yen Lo and
Christoph Feichtenhofer},
title = {PySlowFast},
howpublished = {\url{https://github.com/facebookresearch/slowfast}},
year = {2020}
}
```
================================================
FILE: VISUALIZATION_TOOLS.md
================================================
# Visualization Tools for PySlowFast
This document provides a brief intro for running various visualization tools provided with PySlowFast. Before launching any job, make sure you have properly installed the PySlowFast following the instruction in [README.md](README.md) and you have prepared the dataset following [DATASET.md](slowfast/datasets/DATASET.md) with the correct format.
## Tensorboard Support for Train/Eval/Test
We provide Tensorboard support during the train/eval/test pipeline to assist live monitoring various metrics, and class-level performance
with loss/error graphs, confusion matrices and histograms. Enable Tensorboard support by adding the following to your yaml config file:
```
TENSORBOARD:
ENABLE: True
LOG_DIR: # Leave empty to use cfg.OUTPUT_DIR/runs-{cfg.TRAIN.DATASET} as path.
CLASS_NAMES_PATH: # Path to json file providing class_name - id mapping.
CONFUSION_MATRIX:
ENABLE: True
SUBSET_PATH: # Path to txt file contains class names separated by newline characters.
# Only classes in this file will be visualized in the confusion matrix.
HISTOGRAM:
ENABLE: True
TOP_K: 10 # Top-k most frequently predicted classes for each class in the dataset.
SUBSET_PATH: # Path to txt file contains class names separated by newline characters.
# Only classes in this file will be visualized with histograms.
```
More details can be found at [defaults.py](slowfast/config/defaults.py)
### Loss & Error Graphs on Tensorboard:
<div align="center">
<img src="demo/visualization/metrics/loss.png" width="800px"/>
</div>
### Confusion matrices:
<div align="center">
<img src="demo/visualization/metrics/cf_subset.png" width="367px" style="margin:10px;"/>
<img src="demo/visualization/metrics/cf_parent.png" width="350px" style="margin:10px;"/>
</div>
<div align="center">
</div>
To enable this mode, set:
```
TENSORBOARD:
ENABLE: True
CATEGORIES_PATH: # Path to a json file for categories -> classes mapping
# in the format {"parent_class": ["child_class1", "child_class2",...], ...}.
CONFUSION_MATRIX:
ENABLE: True
```
### Histograms of top-k most frequent predictions:
<div align="center">
<img src="demo/visualization/metrics/hist1.png" width="400px" style="margin:10px;"/>
<img src="demo/visualization/metrics/hist2.png" width="406px" style="margin:10px;"/>
</div>
## Model Analysis
In addition, we provide tools to help with understanding your trained model(s), more options at [defaults.py](slowfast/config/defaults.py)
Adding the following to your yaml config file:
```
TENSORBOARD:
ENABLE: True
MODEL_VIS:
ENABLE: True
MODEL_WEIGHTS: # Set to True to visualize model weights.
ACTIVATIONS: # Set to True to visualize feature maps.
INPUT_VIDEO: # Set to True to visualize the input video(s) for the corresponding feature maps.
LAYER_LIST: # List of layer names to visualize weights and activations for.
GRAD_CAM:
ENABLE: True
LAYER_LIST: # List of CNN layers to use for Grad-CAM visualization method.
# The number of layer must be equal to the number of pathway(s).
```
### Weights Visualization on Tensorboard:
<div align="center">
<img src="demo/visualization/analysis/weights1.png" width="300px" style="margin:10px;"/>
<img src="demo/visualization/analysis/weights2.png" width="328px" style="margin:18px;"/>
</div>
### Feature Maps & Inputs Visualization:
<div align="center">
<img src="demo/visualization/analysis/activations.gif" width="800px"/>
</div>
### Input Videos Visualization with Grad-CAM:
<div align="center">
<img src="demo/visualization/analysis/gradcam.gif" width="400px" style="margin:10px;"/>
<img src="demo/visualization/analysis/gradcam2.gif" width="400px" style="margin:10px;"/>
</div>
## Run the Demo on Videos/Camera
To run inference with PySlowFast model(s) on wild video(s), add the following to your yaml config file:
```
DEMO:
ENABLE: True
LABEL_FILE_PATH: # Path to json file providing class_name - id mapping.
INPUT_VIDEO: # Path to input video file.
OUTPUT_FILE: # Path to output video file to write results to.
# Leave an empty string if you would like to display results to a window.
THREAD_ENABLE: # Run video reader/writer in the background with multi-threading.
NUM_VIS_INSTANCES: # Number of CPU(s)/processes use to run video visualizer.
NUM_CLIPS_SKIP: # Number of clips to skip prediction/visualization
# (mostly to smoothen/improve display quality with wecam input).
```
If you would like to use webcam as an input, in place of `DEMO.INPUT_VIDEO`, set `DEMO.WEBCAM` to the index of the webcam for input. Please check for more options at [defaults.py](slowfast/config/defaults.py)
### Action Recognition Demo:
<div align="center">
<img src="demo/visualization/demo_gifs/recognition.gif" width="600px"/>
</div>
### Action Detection Demo:
<div align="center">
<img src="demo/visualization/demo_gifs/detection.gif" width="600px" style="margin:10px;"/>
</div>
### Demo with AVA video(s):
We also offer an option to use trained models to create and visualize prediction results and ground-truth labels on AVA-format videos and metadata. An example config is:
```
DEMO:
ENABLE: True
OUTPUT_FILE: yourPath/output.mp4
LABEL_FILE_PATH: yourPath/ava_classnames.json
INPUT_VIDEO: yourPath/frames/HVAmkvLrthQ # Path to a video file or image folder
PREDS_BOXES: yourPath/ava_detection_train_boxes_and_labels_include_negative.csv # Path to pre-computed bouding boxes in AVA format.
GT_BOXES: yourPath/ava_train_v2.2.csv # Path to ground-truth boxes and labels in AVA format (optional).
```
<div align="center">
<img src="demo/visualization/demo_gifs/ava_demo2.gif" width="600px" style="margin:10px;"/>
</div>
### Run command
```
python \tools\run_net.py --cfg path/to/<pretrained_model_config_file>.yaml
```
### Download class name files
- [AVA class names json file](https://dl.fbaipublicfiles.com/pyslowfast/dataset/class_names/ava_classids.json)
- [Kinetics class names json file](https://dl.fbaipublicfiles.com/pyslowfast/dataset/class_names/kinetics_classnames.json)
- [Kinetics parent-child class mapping](https://dl.fbaipublicfiles.com/pyslowfast/dataset/class_names/parents.json)
================================================
FILE: ava_evaluation/README.md
================================================
The code under this folder is from the official [ActivityNet repo](https://github.com/activitynet/ActivityNet).
================================================
FILE: ava_evaluation/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt
================================================
item {
name: "bend/bow (at the waist)"
id: 1
}
item {
name: "crouch/kneel"
id: 3
}
item {
name: "dance"
id: 4
}
item {
name: "fall down"
id: 5
}
item {
name: "get up"
id: 6
}
item {
name: "jump/leap"
id: 7
}
item {
name: "lie/sleep"
id: 8
}
item {
name: "martial art"
id: 9
}
item {
name: "run/jog"
id: 10
}
item {
name: "sit"
id: 11
}
item {
name: "stand"
id: 12
}
item {
name: "swim"
id: 13
}
item {
name: "walk"
id: 14
}
item {
name: "answer phone"
id: 15
}
item {
name: "carry/hold (an object)"
id: 17
}
item {
name: "climb (e.g., a mountain)"
id: 20
}
item {
name: "close (e.g., a door, a box)"
id: 22
}
item {
name: "cut"
id: 24
}
item {
name: "dress/put on clothing"
id: 26
}
item {
name: "drink"
id: 27
}
item {
name: "drive (e.g., a car, a truck)"
id: 28
}
item {
name: "eat"
id: 29
}
item {
name: "enter"
id: 30
}
item {
name: "hit (an object)"
id: 34
}
item {
name: "lift/pick up"
id: 36
}
item {
name: "listen (e.g., to music)"
id: 37
}
item {
name: "open (e.g., a window, a car door)"
id: 38
}
item {
name: "play musical instrument"
id: 41
}
item {
name: "point to (an object)"
id: 43
}
item {
name: "pull (an object)"
id: 45
}
item {
name: "push (an object)"
id: 46
}
item {
name: "put down"
id: 47
}
item {
name: "read"
id: 48
}
item {
name: "ride (e.g., a bike, a car, a horse)"
id: 49
}
item {
name: "sail boat"
id: 51
}
item {
name: "shoot"
id: 52
}
item {
name: "smoke"
id: 54
}
item {
name: "take a photo"
id: 56
}
item {
name: "text on/look at a cellphone"
id: 57
}
item {
name: "throw"
id: 58
}
item {
name: "touch (an object)"
id: 59
}
item {
name: "turn (e.g., a screwdriver)"
id: 60
}
item {
name: "watch (e.g., TV)"
id: 61
}
item {
name: "work on a computer"
id: 62
}
item {
name: "write"
id: 63
}
item {
name: "fight/hit (a person)"
id: 64
}
item {
name: "give/serve (an object) to (a person)"
id: 65
}
item {
name: "grab (a person)"
id: 66
}
item {
name: "hand clap"
id: 67
}
item {
name: "hand shake"
id: 68
}
item {
name: "hand wave"
id: 69
}
item {
name: "hug (a person)"
id: 70
}
item {
name: "kiss (a person)"
id: 72
}
item {
name: "lift (a person)"
id: 73
}
item {
name: "listen to (a person)"
id: 74
}
item {
name: "push (another person)"
id: 76
}
item {
name: "sing to (e.g., self, a person, a group)"
id: 77
}
item {
name: "take (an object) from (a person)"
id: 78
}
item {
name: "talk to (e.g., self, a person, a group)"
id: 79
}
item {
name: "watch (a person)"
id: 80
}
================================================
FILE: ava_evaluation/label_map_util.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Label map utility functions."""
from __future__ import absolute_import, division, print_function, unicode_literals
import logging
# from google.protobuf import text_format
# from google3.third_party.tensorflow_models.object_detection.protos import string_int_label_map_pb2
def _validate_label_map(label_map):
"""Checks if a label map is valid.
Args:
label_map: StringIntLabelMap to validate.
Raises:
ValueError: if label map is invalid.
"""
for item in label_map.item:
if item.id < 1:
raise ValueError("Label map ids should be >= 1.")
def create_category_index(categories):
"""Creates dictionary of COCO compatible categories keyed by category id.
Args:
categories: a list of dicts, each of which has the following keys:
'id': (required) an integer id uniquely identifying this category.
'name': (required) string representing category name
e.g., 'cat', 'dog', 'pizza'.
Returns:
category_index: a dict containing the same entries as categories, but keyed
by the 'id' field of each category.
"""
category_index = {}
for cat in categories:
category_index[cat["id"]] = cat
return category_index
def get_max_label_map_index(label_map):
"""Get maximum index in label map.
Args:
label_map: a StringIntLabelMapProto
Returns:
an integer
"""
return max([item.id for item in label_map.item])
def convert_label_map_to_categories(label_map, max_num_classes, use_display_name=True):
"""Loads label map proto and returns categories list compatible with eval.
This function loads a label map and returns a list of dicts, each of which
has the following keys:
'id': (required) an integer id uniquely identifying this category.
'name': (required) string representing category name
e.g., 'cat', 'dog', 'pizza'.
We only allow class into the list if its id-label_id_offset is
between 0 (inclusive) and max_num_classes (exclusive).
If there are several items mapping to the same id in the label map,
we will only keep the first one in the categories list.
Args:
label_map: a StringIntLabelMapProto or None. If None, a default categories
list is created with max_num_classes categories.
max_num_classes: maximum number of (consecutive) label indices to include.
use_display_name: (boolean) choose whether to load 'display_name' field
as category name. If False or if the display_name field does not exist,
uses 'name' field as category names instead.
Returns:
categories: a list of dictionaries representing all possible categories.
"""
categories = []
list_of_ids_already_added = []
if not label_map:
label_id_offset = 1
for class_id in range(max_num_classes):
categories.append(
{
"id": class_id + label_id_offset,
"name": "category_{}".format(class_id + label_id_offset),
}
)
return categories
for item in label_map.item:
if not 0 < item.id <= max_num_classes:
logging.info(
"Ignore item %d since it falls outside of requested label range.",
item.id,
)
continue
if use_display_name and item.HasField("display_name"):
name = item.display_name
else:
name = item.name
if item.id not in list_of_ids_already_added:
list_of_ids_already_added.append(item.id)
categories.append({"id": item.id, "name": name})
return categories
def load_labelmap(path):
"""Loads label map proto.
Args:
path: path to StringIntLabelMap proto text file.
Returns:
a StringIntLabelMapProto
"""
with open(path, "r") as fid:
label_map_string = fid.read()
label_map = string_int_label_map_pb2.StringIntLabelMap() # noqa
try:
text_format.Merge(label_map_string, label_map) # noqa
except text_format.ParseError: # noqa
label_map.ParseFromString(label_map_string)
_validate_label_map(label_map)
return label_map
def get_label_map_dict(label_map_path, use_display_name=False):
"""Reads a label map and returns a dictionary of label names to id.
Args:
label_map_path: path to label_map.
use_display_name: whether to use the label map items' display names as keys.
Returns:
A dictionary mapping label names to id.
"""
label_map = load_labelmap(label_map_path)
label_map_dict = {}
for item in label_map.item:
if use_display_name:
label_map_dict[item.display_name] = item.id
else:
label_map_dict[item.name] = item.id
return label_map_dict
def create_category_index_from_labelmap(label_map_path):
"""Reads a label map and returns a category index.
Args:
label_map_path: Path to `StringIntLabelMap` proto text file.
Returns:
A category index, which is a dictionary that maps integer ids to dicts
containing categories, e.g.
{1: {'id': 1, 'name': 'dog'}, 2: {'id': 2, 'name': 'cat'}, ...}
"""
label_map = load_labelmap(label_map_path)
max_num_classes = max(item.id for item in label_map.item)
categories = convert_label_map_to_categories(label_map, max_num_classes)
return create_category_index(categories)
def create_class_agnostic_category_index():
"""Creates a category index with a single `object` class."""
return {1: {"id": 1, "name": "object"}}
================================================
FILE: ava_evaluation/metrics.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functions for computing metrics like precision, recall, CorLoc and etc."""
from __future__ import division
import numpy as np
def compute_precision_recall(scores, labels, num_gt):
"""Compute precision and recall.
Args:
scores: A float numpy array representing detection score
labels: A boolean numpy array representing true/false positive labels
num_gt: Number of ground truth instances
Raises:
ValueError: if the input is not of the correct format
Returns:
precision: Fraction of positive instances over detected ones. This value is
None if no ground truth labels are present.
recall: Fraction of detected positive instance over all positive instances.
This value is None if no ground truth labels are present.
"""
if (
not isinstance(labels, np.ndarray)
or labels.dtype != bool
or len(labels.shape) != 1
):
raise ValueError("labels must be single dimension bool numpy array")
if not isinstance(scores, np.ndarray) or len(scores.shape) != 1:
raise ValueError("scores must be single dimension numpy array")
if num_gt < np.sum(labels):
raise ValueError("Number of true positives must be smaller than num_gt.")
if len(scores) != len(labels):
raise ValueError("scores and labels must be of the same size.")
if num_gt == 0:
return None, None
sorted_indices = np.argsort(scores)
sorted_indices = sorted_indices[::-1]
labels = labels.astype(int)
true_positive_labels = labels[sorted_indices]
false_positive_labels = 1 - true_positive_labels
cum_true_positives = np.cumsum(true_positive_labels)
cum_false_positives = np.cumsum(false_positive_labels)
precision = cum_true_positives.astype(float) / (
cum_true_positives + cum_false_positives
)
recall = cum_true_positives.astype(float) / num_gt
return precision, recall
def compute_average_precision(precision, recall):
"""Compute Average Precision according to the definition in VOCdevkit.
Precision is modified to ensure that it does not decrease as recall
decrease.
Args:
precision: A float [N, 1] numpy array of precisions
recall: A float [N, 1] numpy array of recalls
Raises:
ValueError: if the input is not of the correct format
Returns:
average_precison: The area under the precision recall curve. NaN if
precision and recall are None.
"""
if precision is None:
if recall is not None:
raise ValueError("If precision is None, recall must also be None")
return np.nan
if not isinstance(precision, np.ndarray) or not isinstance(recall, np.ndarray):
raise ValueError("precision and recall must be numpy array")
if precision.dtype != float or recall.dtype != float:
raise ValueError("input must be float numpy array.")
if len(precision) != len(recall):
raise ValueError("precision and recall must be of the same size.")
if not precision.size:
return 0.0
if np.amin(precision) < 0 or np.amax(precision) > 1:
raise ValueError("Precision must be in the range of [0, 1].")
if np.amin(recall) < 0 or np.amax(recall) > 1:
raise ValueError("recall must be in the range of [0, 1].")
if not all(recall[i] <= recall[i + 1] for i in range(len(recall) - 1)):
raise ValueError("recall must be a non-decreasing array")
recall = np.concatenate([[0], recall, [1]])
precision = np.concatenate([[0], precision, [0]])
# Preprocess precision to be a non-decreasing array
for i in range(len(precision) - 2, -1, -1):
precision[i] = np.maximum(precision[i], precision[i + 1])
indices = np.where(recall[1:] != recall[:-1])[0] + 1
average_precision = np.sum(
(recall[indices] - recall[indices - 1]) * precision[indices]
)
return average_precision
def compute_cor_loc(num_gt_imgs_per_class, num_images_correctly_detected_per_class):
"""Compute CorLoc according to the definition in the following paper.
https://www.robots.ox.ac.uk/~vgg/rg/papers/deselaers-eccv10.pdf
Returns nans if there are no ground truth images for a class.
Args:
num_gt_imgs_per_class: 1D array, representing number of images containing
at least one object instance of a particular class
num_images_correctly_detected_per_class: 1D array, representing number of
images that are correctly detected at least one object instance of a
particular class
Returns:
corloc_per_class: A float numpy array represents the corloc score of each
class
"""
# Divide by zero expected for classes with no gt examples.
with np.errstate(divide="ignore", invalid="ignore"):
return np.where(
num_gt_imgs_per_class == 0,
np.nan,
num_images_correctly_detected_per_class / num_gt_imgs_per_class,
)
================================================
FILE: ava_evaluation/np_box_list.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Numpy BoxList classes and functions."""
from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
class BoxList:
"""Box collection.
BoxList represents a list of bounding boxes as numpy array, where each
bounding box is represented as a row of 4 numbers,
[y_min, x_min, y_max, x_max]. It is assumed that all bounding boxes within a
given list correspond to a single image.
Optionally, users can add additional related fields (such as
objectness/classification scores).
"""
def __init__(self, data):
"""Constructs box collection.
Args:
data: a numpy array of shape [N, 4] representing box coordinates
Raises:
ValueError: if bbox data is not a numpy array
ValueError: if invalid dimensions for bbox data
"""
if not isinstance(data, np.ndarray):
raise ValueError("data must be a numpy array.")
if len(data.shape) != 2 or data.shape[1] != 4:
raise ValueError("Invalid dimensions for box data.")
if data.dtype != np.float32 and data.dtype != np.float64:
raise ValueError("Invalid data type for box data: float is required.")
if not self._is_valid_boxes(data):
raise ValueError(
"Invalid box data. data must be a numpy array of "
"N*[y_min, x_min, y_max, x_max]"
)
self.data = {"boxes": data}
def num_boxes(self):
"""Return number of boxes held in collections."""
return self.data["boxes"].shape[0]
def get_extra_fields(self):
"""Return all non-box fields."""
return [k for k in self.data.keys() if k != "boxes"]
def has_field(self, field):
return field in self.data
def add_field(self, field, field_data):
"""Add data to a specified field.
Args:
field: a string parameter used to speficy a related field to be accessed.
field_data: a numpy array of [N, ...] representing the data associated
with the field.
Raises:
ValueError: if the field is already exist or the dimension of the field
data does not matches the number of boxes.
"""
if self.has_field(field):
raise ValueError("Field " + field + "already exists")
if len(field_data.shape) < 1 or field_data.shape[0] != self.num_boxes():
raise ValueError("Invalid dimensions for field data")
self.data[field] = field_data
def get(self):
"""Convenience function for accesssing box coordinates.
Returns:
a numpy array of shape [N, 4] representing box corners
"""
return self.get_field("boxes")
def get_field(self, field):
"""Accesses data associated with the specified field in the box collection.
Args:
field: a string parameter used to speficy a related field to be accessed.
Returns:
a numpy 1-d array representing data of an associated field
Raises:
ValueError: if invalid field
"""
if not self.has_field(field):
raise ValueError("field {} does not exist".format(field))
return self.data[field]
def get_coordinates(self):
"""Get corner coordinates of boxes.
Returns:
a list of 4 1-d numpy arrays [y_min, x_min, y_max, x_max]
"""
box_coordinates = self.get()
y_min = box_coordinates[:, 0]
x_min = box_coordinates[:, 1]
y_max = box_coordinates[:, 2]
x_max = box_coordinates[:, 3]
return [y_min, x_min, y_max, x_max]
def _is_valid_boxes(self, data):
"""Check whether data fullfills the format of N*[ymin, xmin, ymax, xmin].
Args:
data: a numpy array of shape [N, 4] representing box coordinates
Returns:
a boolean indicating whether all ymax of boxes are equal or greater than
ymin, and all xmax of boxes are equal or greater than xmin.
"""
if data.shape[0] > 0:
for i in range(data.shape[0]):
if data[i, 0] > data[i, 2] or data[i, 1] > data[i, 3]:
return False
return True
================================================
FILE: ava_evaluation/np_box_list_ops.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Bounding Box List operations for Numpy BoxLists.
Example box operations that are supported:
* Areas: compute bounding box areas
* IOU: pairwise intersection-over-union scores
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
from . import np_box_list, np_box_ops
class SortOrder:
"""Enum class for sort order.
Attributes:
ascend: ascend order.
descend: descend order.
"""
ASCEND = 1
DESCEND = 2
def area(boxlist):
"""Computes area of boxes.
Args:
boxlist: BoxList holding N boxes
Returns:
a numpy array with shape [N*1] representing box areas
"""
y_min, x_min, y_max, x_max = boxlist.get_coordinates()
return (y_max - y_min) * (x_max - x_min)
def intersection(boxlist1, boxlist2):
"""Compute pairwise intersection areas between boxes.
Args:
boxlist1: BoxList holding N boxes
boxlist2: BoxList holding M boxes
Returns:
a numpy array with shape [N*M] representing pairwise intersection area
"""
return np_box_ops.intersection(boxlist1.get(), boxlist2.get())
def iou(boxlist1, boxlist2):
"""Computes pairwise intersection-over-union between box collections.
Args:
boxlist1: BoxList holding N boxes
boxlist2: BoxList holding M boxes
Returns:
a numpy array with shape [N, M] representing pairwise iou scores.
"""
return np_box_ops.iou(boxlist1.get(), boxlist2.get())
def ioa(boxlist1, boxlist2):
"""Computes pairwise intersection-over-area between box collections.
Intersection-over-area (ioa) between two boxes box1 and box2 is defined as
their intersection area over box2's area. Note that ioa is not symmetric,
that is, IOA(box1, box2) != IOA(box2, box1).
Args:
boxlist1: BoxList holding N boxes
boxlist2: BoxList holding M boxes
Returns:
a numpy array with shape [N, M] representing pairwise ioa scores.
"""
return np_box_ops.ioa(boxlist1.get(), boxlist2.get())
def gather(boxlist, indices, fields=None):
"""Gather boxes from BoxList according to indices and return new BoxList.
By default, gather returns boxes corresponding to the input index list, as
well as all additional fields stored in the boxlist (indexing into the
first dimension). However one can optionally only gather from a
subset of fields.
Args:
boxlist: BoxList holding N boxes
indices: a 1-d numpy array of type int_
fields: (optional) list of fields to also gather from. If None (default),
all fields are gathered from. Pass an empty fields list to only gather
the box coordinates.
Returns:
subboxlist: a BoxList corresponding to the subset of the input BoxList
specified by indices
Raises:
ValueError: if specified field is not contained in boxlist or if the
indices are not of type int_
"""
if indices.size:
if np.amax(indices) >= boxlist.num_boxes() or np.amin(indices) < 0:
raise ValueError("indices are out of valid range.")
subboxlist = np_box_list.BoxList(boxlist.get()[indices, :])
if fields is None:
fields = boxlist.get_extra_fields()
for field in fields:
extra_field_data = boxlist.get_field(field)
subboxlist.add_field(field, extra_field_data[indices, ...])
return subboxlist
def sort_by_field(boxlist, field, order=SortOrder.DESCEND):
"""Sort boxes and associated fields according to a scalar field.
A common use case is reordering the boxes according to descending scores.
Args:
boxlist: BoxList holding N boxes.
field: A BoxList field for sorting and reordering the BoxList.
order: (Optional) 'descend' or 'ascend'. Default is descend.
Returns:
sorted_boxlist: A sorted BoxList with the field in the specified order.
Raises:
ValueError: if specified field does not exist or is not of single dimension.
ValueError: if the order is not either descend or ascend.
"""
if not boxlist.has_field(field):
raise ValueError("Field " + field + " does not exist")
if len(boxlist.get_field(field).shape) != 1:
raise ValueError("Field " + field + "should be single dimension.")
if order != SortOrder.DESCEND and order != SortOrder.ASCEND:
raise ValueError("Invalid sort order")
field_to_sort = boxlist.get_field(field)
sorted_indices = np.argsort(field_to_sort)
if order == SortOrder.DESCEND:
sorted_indices = sorted_indices[::-1]
return gather(boxlist, sorted_indices)
def non_max_suppression(
boxlist, max_output_size=10000, iou_threshold=1.0, score_threshold=-10.0
):
"""Non maximum suppression.
This op greedily selects a subset of detection bounding boxes, pruning
away boxes that have high IOU (intersection over union) overlap (> thresh)
with already selected boxes. In each iteration, the detected bounding box with
highest score in the available pool is selected.
Args:
boxlist: BoxList holding N boxes. Must contain a 'scores' field
representing detection scores. All scores belong to the same class.
max_output_size: maximum number of retained boxes
iou_threshold: intersection over union threshold.
score_threshold: minimum score threshold. Remove the boxes with scores
less than this value. Default value is set to -10. A very
low threshold to pass pretty much all the boxes, unless
the user sets a different score threshold.
Returns:
a BoxList holding M boxes where M <= max_output_size
Raises:
ValueError: if 'scores' field does not exist
ValueError: if threshold is not in [0, 1]
ValueError: if max_output_size < 0
"""
if not boxlist.has_field("scores"):
raise ValueError("Field scores does not exist")
if iou_threshold < 0.0 or iou_threshold > 1.0:
raise ValueError("IOU threshold must be in [0, 1]")
if max_output_size < 0:
raise ValueError("max_output_size must be bigger than 0.")
boxlist = filter_scores_greater_than(boxlist, score_threshold)
if boxlist.num_boxes() == 0:
return boxlist
boxlist = sort_by_field(boxlist, "scores")
# Prevent further computation if NMS is disabled.
if iou_threshold == 1.0:
if boxlist.num_boxes() > max_output_size:
selected_indices = np.arange(max_output_size)
return gather(boxlist, selected_indices)
else:
return boxlist
boxes = boxlist.get()
num_boxes = boxlist.num_boxes()
# is_index_valid is True only for all remaining valid boxes,
is_index_valid = np.full(num_boxes, 1, dtype=bool)
selected_indices = []
num_output = 0
for i in range(num_boxes):
if num_output < max_output_size:
if is_index_valid[i]:
num_output += 1
selected_indices.append(i)
is_index_valid[i] = False
valid_indices = np.where(is_index_valid)[0]
if valid_indices.size == 0:
break
intersect_over_union = np_box_ops.iou(
np.expand_dims(boxes[i, :], axis=0), boxes[valid_indices, :]
)
intersect_over_union = np.squeeze(intersect_over_union, axis=0)
is_index_valid[valid_indices] = np.logical_and(
is_index_valid[valid_indices],
intersect_over_union <= iou_threshold,
)
return gather(boxlist, np.array(selected_indices))
def multi_class_non_max_suppression(boxlist, score_thresh, iou_thresh, max_output_size):
"""Multi-class version of non maximum suppression.
This op greedily selects a subset of detection bounding boxes, pruning
away boxes that have high IOU (intersection over union) overlap (> thresh)
with already selected boxes. It operates independently for each class for
which scores are provided (via the scores field of the input box_list),
pruning boxes with score less than a provided threshold prior to
applying NMS.
Args:
boxlist: BoxList holding N boxes. Must contain a 'scores' field
representing detection scores. This scores field is a tensor that can
be 1 dimensional (in the case of a single class) or 2-dimensional, which
which case we assume that it takes the shape [num_boxes, num_classes].
We further assume that this rank is known statically and that
scores.shape[1] is also known (i.e., the number of classes is fixed
and known at graph construction time).
score_thresh: scalar threshold for score (low scoring boxes are removed).
iou_thresh: scalar threshold for IOU (boxes that that high IOU overlap
with previously selected boxes are removed).
max_output_size: maximum number of retained boxes per class.
Returns:
a BoxList holding M boxes with a rank-1 scores field representing
corresponding scores for each box with scores sorted in decreasing order
and a rank-1 classes field representing a class label for each box.
Raises:
ValueError: if iou_thresh is not in [0, 1] or if input boxlist does not have
a valid scores field.
"""
if not 0 <= iou_thresh <= 1.0:
raise ValueError("thresh must be between 0 and 1")
if not isinstance(boxlist, np_box_list.BoxList):
raise ValueError("boxlist must be a BoxList")
if not boxlist.has_field("scores"):
raise ValueError("input boxlist must have 'scores' field")
scores = boxlist.get_field("scores")
if len(scores.shape) == 1:
scores = np.reshape(scores, [-1, 1])
elif len(scores.shape) == 2:
if scores.shape[1] is None:
raise ValueError(
"scores field must have statically defined second dimension"
)
else:
raise ValueError("scores field must be of rank 1 or 2")
num_boxes = boxlist.num_boxes()
num_scores = scores.shape[0]
num_classes = scores.shape[1]
if num_boxes != num_scores:
raise ValueError("Incorrect scores field length: actual vs expected.")
selected_boxes_list = []
for class_idx in range(num_classes):
boxlist_and_class_scores = np_box_list.BoxList(boxlist.get())
class_scores = np.reshape(scores[0:num_scores, class_idx], [-1])
boxlist_and_class_scores.add_field("scores", class_scores)
boxlist_filt = filter_scores_greater_than(
boxlist_and_class_scores, score_thresh
)
nms_result = non_max_suppression(
boxlist_filt,
max_output_size=max_output_size,
iou_threshold=iou_thresh,
score_threshold=score_thresh,
)
nms_result.add_field(
"classes", np.zeros_like(nms_result.get_field("scores")) + class_idx
)
selected_boxes_list.append(nms_result)
selected_boxes = concatenate(selected_boxes_list)
sorted_boxes = sort_by_field(selected_boxes, "scores")
return sorted_boxes
def scale(boxlist, y_scale, x_scale):
"""Scale box coordinates in x and y dimensions.
Args:
boxlist: BoxList holding N boxes
y_scale: float
x_scale: float
Returns:
boxlist: BoxList holding N boxes
"""
y_min, x_min, y_max, x_max = np.array_split(boxlist.get(), 4, axis=1)
y_min = y_scale * y_min
y_max = y_scale * y_max
x_min = x_scale * x_min
x_max = x_scale * x_max
scaled_boxlist = np_box_list.BoxList(np.hstack([y_min, x_min, y_max, x_max]))
fields = boxlist.get_extra_fields()
for field in fields:
extra_field_data = boxlist.get_field(field)
scaled_boxlist.add_field(field, extra_field_data)
return scaled_boxlist
def clip_to_window(boxlist, window):
"""Clip bounding boxes to a window.
This op clips input bounding boxes (represented by bounding box
corners) to a window, optionally filtering out boxes that do not
overlap at all with the window.
Args:
boxlist: BoxList holding M_in boxes
window: a numpy array of shape [4] representing the
[y_min, x_min, y_max, x_max] window to which the op
should clip boxes.
Returns:
a BoxList holding M_out boxes where M_out <= M_in
"""
y_min, x_min, y_max, x_max = np.array_split(boxlist.get(), 4, axis=1)
win_y_min = window[0]
win_x_min = window[1]
win_y_max = window[2]
win_x_max = window[3]
y_min_clipped = np.fmax(np.fmin(y_min, win_y_max), win_y_min)
y_max_clipped = np.fmax(np.fmin(y_max, win_y_max), win_y_min)
x_min_clipped = np.fmax(np.fmin(x_min, win_x_max), win_x_min)
x_max_clipped = np.fmax(np.fmin(x_max, win_x_max), win_x_min)
clipped = np_box_list.BoxList(
np.hstack([y_min_clipped, x_min_clipped, y_max_clipped, x_max_clipped])
)
clipped = _copy_extra_fields(clipped, boxlist)
areas = area(clipped)
nonzero_area_indices = np.reshape(np.nonzero(np.greater(areas, 0.0)), [-1]).astype(
np.int32
)
return gather(clipped, nonzero_area_indices)
def prune_non_overlapping_boxes(boxlist1, boxlist2, minoverlap=0.0):
"""Prunes the boxes in boxlist1 that overlap less than thresh with boxlist2.
For each box in boxlist1, we want its IOA to be more than minoverlap with
at least one of the boxes in boxlist2. If it does not, we remove it.
Args:
boxlist1: BoxList holding N boxes.
boxlist2: BoxList holding M boxes.
minoverlap: Minimum required overlap between boxes, to count them as
overlapping.
Returns:
A pruned boxlist with size [N', 4].
"""
intersection_over_area = ioa(boxlist2, boxlist1) # [M, N] tensor
intersection_over_area = np.amax(intersection_over_area, axis=0) # [N] tensor
keep_bool = np.greater_equal(intersection_over_area, np.array(minoverlap))
keep_inds = np.nonzero(keep_bool)[0]
new_boxlist1 = gather(boxlist1, keep_inds)
return new_boxlist1
def prune_outside_window(boxlist, window):
"""Prunes bounding boxes that fall outside a given window.
This function prunes bounding boxes that even partially fall outside the given
window. See also ClipToWindow which only prunes bounding boxes that fall
completely outside the window, and clips any bounding boxes that partially
overflow.
Args:
boxlist: a BoxList holding M_in boxes.
window: a numpy array of size 4, representing [ymin, xmin, ymax, xmax]
of the window.
Returns:
pruned_corners: a tensor with shape [M_out, 4] where M_out <= M_in.
valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes
in the input tensor.
"""
y_min, x_min, y_max, x_max = np.array_split(boxlist.get(), 4, axis=1)
win_y_min = window[0]
win_x_min = window[1]
win_y_max = window[2]
win_x_max = window[3]
coordinate_violations = np.hstack(
[
np.less(y_min, win_y_min),
np.less(x_min, win_x_min),
np.greater(y_max, win_y_max),
np.greater(x_max, win_x_max),
]
)
valid_indices = np.reshape(
np.where(np.logical_not(np.max(coordinate_violations, axis=1))), [-1]
)
return gather(boxlist, valid_indices), valid_indices
def concatenate(boxlists, fields=None):
"""Concatenate list of BoxLists.
This op concatenates a list of input BoxLists into a larger BoxList. It also
handles concatenation of BoxList fields as long as the field tensor shapes
are equal except for the first dimension.
Args:
boxlists: list of BoxList objects
fields: optional list of fields to also concatenate. By default, all
fields from the first BoxList in the list are included in the
concatenation.
Returns:
a BoxList with number of boxes equal to
sum([boxlist.num_boxes() for boxlist in BoxList])
Raises:
ValueError: if boxlists is invalid (i.e., is not a list, is empty, or
contains non BoxList objects), or if requested fields are not contained in
all boxlists
"""
if not isinstance(boxlists, list):
raise ValueError("boxlists should be a list")
if not boxlists:
raise ValueError("boxlists should have nonzero length")
for boxlist in boxlists:
if not isinstance(boxlist, np_box_list.BoxList):
raise ValueError("all elements of boxlists should be BoxList objects")
concatenated = np_box_list.BoxList(
np.vstack([boxlist.get() for boxlist in boxlists])
)
if fields is None:
fields = boxlists[0].get_extra_fields()
for field in fields:
first_field_shape = boxlists[0].get_field(field).shape
first_field_shape = first_field_shape[1:]
for boxlist in boxlists:
if not boxlist.has_field(field):
raise ValueError("boxlist must contain all requested fields")
field_shape = boxlist.get_field(field).shape
field_shape = field_shape[1:]
if field_shape != first_field_shape:
raise ValueError(
"field %s must have same shape for all boxlists "
"except for the 0th dimension." % field
)
concatenated_field = np.concatenate(
[boxlist.get_field(field) for boxlist in boxlists], axis=0
)
concatenated.add_field(field, concatenated_field)
return concatenated
def filter_scores_greater_than(boxlist, thresh):
"""Filter to keep only boxes with score exceeding a given threshold.
This op keeps the collection of boxes whose corresponding scores are
greater than the input threshold.
Args:
boxlist: BoxList holding N boxes. Must contain a 'scores' field
representing detection scores.
thresh: scalar threshold
Returns:
a BoxList holding M boxes where M <= N
Raises:
ValueError: if boxlist not a BoxList object or if it does not
have a scores field
"""
if not isinstance(boxlist, np_box_list.BoxList):
raise ValueError("boxlist must be a BoxList")
if not boxlist.has_field("scores"):
raise ValueError("input boxlist must have 'scores' field")
scores = boxlist.get_field("scores")
if len(scores.shape) > 2:
raise ValueError("Scores should have rank 1 or 2")
if len(scores.shape) == 2 and scores.shape[1] != 1:
raise ValueError(
"Scores should have rank 1 or have shape consistent with [None, 1]"
)
high_score_indices = np.reshape(np.where(np.greater(scores, thresh)), [-1]).astype(
np.int32
)
return gather(boxlist, high_score_indices)
def change_coordinate_frame(boxlist, window):
"""Change coordinate frame of the boxlist to be relative to window's frame.
Given a window of the form [ymin, xmin, ymax, xmax],
changes bounding box coordinates from boxlist to be relative to this window
(e.g., the min corner maps to (0,0) and the max corner maps to (1,1)).
An example use case is data augmentation: where we are given groundtruth
boxes (boxlist) and would like to randomly crop the image to some
window (window). In this case we need to change the coordinate frame of
each groundtruth box to be relative to this new window.
Args:
boxlist: A BoxList object holding N boxes.
window: a size 4 1-D numpy array.
Returns:
Returns a BoxList object with N boxes.
"""
win_height = window[2] - window[0]
win_width = window[3] - window[1]
boxlist_new = scale(
np_box_list.BoxList(
boxlist.get() - [window[0], window[1], window[0], window[1]]
),
1.0 / win_height,
1.0 / win_width,
)
_copy_extra_fields(boxlist_new, boxlist)
return boxlist_new
def _copy_extra_fields(boxlist_to_copy_to, boxlist_to_copy_from):
"""Copies the extra fields of boxlist_to_copy_from to boxlist_to_copy_to.
Args:
boxlist_to_copy_to: BoxList to which extra fields are copied.
boxlist_to_copy_from: BoxList from which fields are copied.
Returns:
boxlist_to_copy_to with extra fields.
"""
for field in boxlist_to_copy_from.get_extra_fields():
boxlist_to_copy_to.add_field(field, boxlist_to_copy_from.get_field(field))
return boxlist_to_copy_to
def _update_valid_indices_by_removing_high_iou_boxes(
selected_indices, is_index_valid, intersect_over_union, threshold
):
max_iou = np.max(intersect_over_union[:, selected_indices], axis=1)
return np.logical_and(is_index_valid, max_iou <= threshold)
================================================
FILE: ava_evaluation/np_box_mask_list.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Numpy BoxMaskList classes and functions."""
from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
from . import np_box_list
class BoxMaskList(np_box_list.BoxList):
"""Convenience wrapper for BoxList with masks.
BoxMaskList extends the np_box_list.BoxList to contain masks as well.
In particular, its constructor receives both boxes and masks. Note that the
masks correspond to the full image.
"""
def __init__(self, box_data, mask_data):
"""Constructs box collection.
Args:
box_data: a numpy array of shape [N, 4] representing box coordinates
mask_data: a numpy array of shape [N, height, width] representing masks
with values are in {0,1}. The masks correspond to the full
image. The height and the width will be equal to image height and width.
Raises:
ValueError: if bbox data is not a numpy array
ValueError: if invalid dimensions for bbox data
ValueError: if mask data is not a numpy array
ValueError: if invalid dimension for mask data
"""
super(BoxMaskList, self).__init__(box_data)
if not isinstance(mask_data, np.ndarray):
raise ValueError("Mask data must be a numpy array.")
if len(mask_data.shape) != 3:
raise ValueError("Invalid dimensions for mask data.")
if mask_data.dtype != np.uint8:
raise ValueError("Invalid data type for mask data: uint8 is required.")
if mask_data.shape[0] != box_data.shape[0]:
raise ValueError("There should be the same number of boxes and masks.")
self.data["masks"] = mask_data
def get_masks(self):
"""Convenience function for accessing masks.
Returns:
a numpy array of shape [N, height, width] representing masks
"""
return self.get_field("masks")
================================================
FILE: ava_evaluation/np_box_mask_list_ops.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Operations for np_box_mask_list.BoxMaskList.
Example box operations that are supported:
* Areas: compute bounding box areas
* IOU: pairwise intersection-over-union scores
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
from . import np_box_list_ops, np_box_mask_list, np_mask_ops
def box_list_to_box_mask_list(boxlist):
"""Converts a BoxList containing 'masks' into a BoxMaskList.
Args:
boxlist: An np_box_list.BoxList object.
Returns:
An np_box_mask_list.BoxMaskList object.
Raises:
ValueError: If boxlist does not contain `masks` as a field.
"""
if not boxlist.has_field("masks"):
raise ValueError("boxlist does not contain mask field.")
box_mask_list = np_box_mask_list.BoxMaskList(
box_data=boxlist.get(), mask_data=boxlist.get_field("masks")
)
extra_fields = boxlist.get_extra_fields()
for key in extra_fields:
if key != "masks":
box_mask_list.data[key] = boxlist.get_field(key)
return box_mask_list
def area(box_mask_list):
"""Computes area of masks.
Args:
box_mask_list: np_box_mask_list.BoxMaskList holding N boxes and masks
Returns:
a numpy array with shape [N*1] representing mask areas
"""
return np_mask_ops.area(box_mask_list.get_masks())
def intersection(box_mask_list1, box_mask_list2):
"""Compute pairwise intersection areas between masks.
Args:
box_mask_list1: BoxMaskList holding N boxes and masks
box_mask_list2: BoxMaskList holding M boxes and masks
Returns:
a numpy array with shape [N*M] representing pairwise intersection area
"""
return np_mask_ops.intersection(
box_mask_list1.get_masks(), box_mask_list2.get_masks()
)
def iou(box_mask_list1, box_mask_list2):
"""Computes pairwise intersection-over-union between box and mask collections.
Args:
box_mask_list1: BoxMaskList holding N boxes and masks
box_mask_list2: BoxMaskList holding M boxes and masks
Returns:
a numpy array with shape [N, M] representing pairwise iou scores.
"""
return np_mask_ops.iou(box_mask_list1.get_masks(), box_mask_list2.get_masks())
def ioa(box_mask_list1, box_mask_list2):
"""Computes pairwise intersection-over-area between box and mask collections.
Intersection-over-area (ioa) between two masks mask1 and mask2 is defined as
their intersection area over mask2's area. Note that ioa is not symmetric,
that is, IOA(mask1, mask2) != IOA(mask2, mask1).
Args:
box_mask_list1: np_box_mask_list.BoxMaskList holding N boxes and masks
box_mask_list2: np_box_mask_list.BoxMaskList holding M boxes and masks
Returns:
a numpy array with shape [N, M] representing pairwise ioa scores.
"""
return np_mask_ops.ioa(box_mask_list1.get_masks(), box_mask_list2.get_masks())
def gather(box_mask_list, indices, fields=None):
"""Gather boxes from np_box_mask_list.BoxMaskList according to indices.
By default, gather returns boxes corresponding to the input index list, as
well as all additional fields stored in the box_mask_list (indexing into the
first dimension). However one can optionally only gather from a
subset of fields.
Args:
box_mask_list: np_box_mask_list.BoxMaskList holding N boxes
indices: a 1-d numpy array of type int_
fields: (optional) list of fields to also gather from. If None (default),
all fields are gathered from. Pass an empty fields list to only gather
the box coordinates.
Returns:
subbox_mask_list: a np_box_mask_list.BoxMaskList corresponding to the subset
of the input box_mask_list specified by indices
Raises:
ValueError: if specified field is not contained in box_mask_list or if the
indices are not of type int_
"""
if fields is not None:
if "masks" not in fields:
fields.append("masks")
return box_list_to_box_mask_list(
np_box_list_ops.gather(boxlist=box_mask_list, indices=indices, fields=fields)
)
def sort_by_field(box_mask_list, field, order=np_box_list_ops.SortOrder.DESCEND):
"""Sort boxes and associated fields according to a scalar field.
A common use case is reordering the boxes according to descending scores.
Args:
box_mask_list: BoxMaskList holding N boxes.
field: A BoxMaskList field for sorting and reordering the BoxMaskList.
order: (Optional) 'descend' or 'ascend'. Default is descend.
Returns:
sorted_box_mask_list: A sorted BoxMaskList with the field in the specified
order.
"""
return box_list_to_box_mask_list(
np_box_list_ops.sort_by_field(boxlist=box_mask_list, field=field, order=order)
)
def non_max_suppression(
box_mask_list,
max_output_size=10000,
iou_threshold=1.0,
score_threshold=-10.0,
):
"""Non maximum suppression.
This op greedily selects a subset of detection bounding boxes, pruning
away boxes that have high IOU (intersection over union) overlap (> thresh)
with already selected boxes. In each iteration, the detected bounding box with
highest score in the available pool is selected.
Args:
box_mask_list: np_box_mask_list.BoxMaskList holding N boxes. Must contain
a 'scores' field representing detection scores. All scores belong to the
same class.
max_output_size: maximum number of retained boxes
iou_threshold: intersection over union threshold.
score_threshold: minimum score threshold. Remove the boxes with scores
less than this value. Default value is set to -10. A very
low threshold to pass pretty much all the boxes, unless
the user sets a different score threshold.
Returns:
an np_box_mask_list.BoxMaskList holding M boxes where M <= max_output_size
Raises:
ValueError: if 'scores' field does not exist
ValueError: if threshold is not in [0, 1]
ValueError: if max_output_size < 0
"""
if not box_mask_list.has_field("scores"):
raise ValueError("Field scores does not exist")
if iou_threshold < 0.0 or iou_threshold > 1.0:
raise ValueError("IOU threshold must be in [0, 1]")
if max_output_size < 0:
raise ValueError("max_output_size must be bigger than 0.")
box_mask_list = filter_scores_greater_than(box_mask_list, score_threshold)
if box_mask_list.num_boxes() == 0:
return box_mask_list
box_mask_list = sort_by_field(box_mask_list, "scores")
# Prevent further computation if NMS is disabled.
if iou_threshold == 1.0:
if box_mask_list.num_boxes() > max_output_size:
selected_indices = np.arange(max_output_size)
return gather(box_mask_list, selected_indices)
else:
return box_mask_list
masks = box_mask_list.get_masks()
num_masks = box_mask_list.num_boxes()
# is_index_valid is True only for all remaining valid boxes,
is_index_valid = np.full(num_masks, 1, dtype=bool)
selected_indices = []
num_output = 0
for i in range(num_masks):
if num_output < max_output_size:
if is_index_valid[i]:
num_output += 1
selected_indices.append(i)
is_index_valid[i] = False
valid_indices = np.where(is_index_valid)[0]
if valid_indices.size == 0:
break
intersect_over_union = np_mask_ops.iou(
np.expand_dims(masks[i], axis=0), masks[valid_indices]
)
intersect_over_union = np.squeeze(intersect_over_union, axis=0)
is_index_valid[valid_indices] = np.logical_and(
is_index_valid[valid_indices],
intersect_over_union <= iou_threshold,
)
return gather(box_mask_list, np.array(selected_indices))
def multi_class_non_max_suppression(
box_mask_list, score_thresh, iou_thresh, max_output_size
):
"""Multi-class version of non maximum suppression.
This op greedily selects a subset of detection bounding boxes, pruning
away boxes that have high IOU (intersection over union) overlap (> thresh)
with already selected boxes. It operates independently for each class for
which scores are provided (via the scores field of the input box_list),
pruning boxes with score less than a provided threshold prior to
applying NMS.
Args:
box_mask_list: np_box_mask_list.BoxMaskList holding N boxes. Must contain a
'scores' field representing detection scores. This scores field is a
tensor that can be 1 dimensional (in the case of a single class) or
2-dimensional, in which case we assume that it takes the
shape [num_boxes, num_classes]. We further assume that this rank is known
statically and that scores.shape[1] is also known (i.e., the number of
classes is fixed and known at graph construction time).
score_thresh: scalar threshold for score (low scoring boxes are removed).
iou_thresh: scalar threshold for IOU (boxes that that high IOU overlap
with previously selected boxes are removed).
max_output_size: maximum number of retained boxes per class.
Returns:
a box_mask_list holding M boxes with a rank-1 scores field representing
corresponding scores for each box with scores sorted in decreasing order
and a rank-1 classes field representing a class label for each box.
Raises:
ValueError: if iou_thresh is not in [0, 1] or if input box_mask_list does
not have a valid scores field.
"""
if not 0 <= iou_thresh <= 1.0:
raise ValueError("thresh must be between 0 and 1")
if not isinstance(box_mask_list, np_box_mask_list.BoxMaskList):
raise ValueError("box_mask_list must be a box_mask_list")
if not box_mask_list.has_field("scores"):
raise ValueError("input box_mask_list must have 'scores' field")
scores = box_mask_list.get_field("scores")
if len(scores.shape) == 1:
scores = np.reshape(scores, [-1, 1])
elif len(scores.shape) == 2:
if scores.shape[1] is None:
raise ValueError(
"scores field must have statically defined second dimension"
)
else:
raise ValueError("scores field must be of rank 1 or 2")
num_boxes = box_mask_list.num_boxes()
num_scores = scores.shape[0]
num_classes = scores.shape[1]
if num_boxes != num_scores:
raise ValueError("Incorrect scores field length: actual vs expected.")
selected_boxes_list = []
for class_idx in range(num_classes):
box_mask_list_and_class_scores = np_box_mask_list.BoxMaskList(
box_data=box_mask_list.get(), mask_data=box_mask_list.get_masks()
)
class_scores = np.reshape(scores[0:num_scores, class_idx], [-1])
box_mask_list_and_class_scores.add_field("scores", class_scores)
box_mask_list_filt = filter_scores_greater_than(
box_mask_list_and_class_scores, score_thresh
)
nms_result = non_max_suppression(
box_mask_list_filt,
max_output_size=max_output_size,
iou_threshold=iou_thresh,
score_threshold=score_thresh,
)
nms_result.add_field(
"classes", np.zeros_like(nms_result.get_field("scores")) + class_idx
)
selected_boxes_list.append(nms_result)
selected_boxes = np_box_list_ops.concatenate(selected_boxes_list)
sorted_boxes = np_box_list_ops.sort_by_field(selected_boxes, "scores")
return box_list_to_box_mask_list(boxlist=sorted_boxes)
def prune_non_overlapping_masks(box_mask_list1, box_mask_list2, minoverlap=0.0):
"""Prunes the boxes in list1 that overlap less than thresh with list2.
For each mask in box_mask_list1, we want its IOA to be more than minoverlap
with at least one of the masks in box_mask_list2. If it does not, we remove
it. If the masks are not full size image, we do the pruning based on boxes.
Args:
box_mask_list1: np_box_mask_list.BoxMaskList holding N boxes and masks.
box_mask_list2: np_box_mask_list.BoxMaskList holding M boxes and masks.
minoverlap: Minimum required overlap between boxes, to count them as
overlapping.
Returns:
A pruned box_mask_list with size [N', 4].
"""
intersection_over_area = ioa(box_mask_list2, box_mask_list1) # [M, N] tensor
intersection_over_area = np.amax(intersection_over_area, axis=0) # [N] tensor
keep_bool = np.greater_equal(intersection_over_area, np.array(minoverlap))
keep_inds = np.nonzero(keep_bool)[0]
new_box_mask_list1 = gather(box_mask_list1, keep_inds)
return new_box_mask_list1
def concatenate(box_mask_lists, fields=None):
"""Concatenate list of box_mask_lists.
This op concatenates a list of input box_mask_lists into a larger
box_mask_list. It also
handles concatenation of box_mask_list fields as long as the field tensor
shapes are equal except for the first dimension.
Args:
box_mask_lists: list of np_box_mask_list.BoxMaskList objects
fields: optional list of fields to also concatenate. By default, all
fields from the first BoxMaskList in the list are included in the
concatenation.
Returns:
a box_mask_list with number of boxes equal to
sum([box_mask_list.num_boxes() for box_mask_list in box_mask_list])
Raises:
ValueError: if box_mask_lists is invalid (i.e., is not a list, is empty, or
contains non box_mask_list objects), or if requested fields are not
contained in all box_mask_lists
"""
if fields is not None:
if "masks" not in fields:
fields.append("masks")
return box_list_to_box_mask_list(
np_box_list_ops.concatenate(boxlists=box_mask_lists, fields=fields)
)
def filter_scores_greater_than(box_mask_list, thresh):
"""Filter to keep only boxes and masks with score exceeding a given threshold.
This op keeps the collection of boxes and masks whose corresponding scores are
greater than the input threshold.
Args:
box_mask_list: BoxMaskList holding N boxes and masks. Must contain a
'scores' field representing detection scores.
thresh: scalar threshold
Returns:
a BoxMaskList holding M boxes and masks where M <= N
Raises:
ValueError: if box_mask_list not a np_box_mask_list.BoxMaskList object or
if it does not have a scores field
"""
if not isinstance(box_mask_list, np_box_mask_list.BoxMaskList):
raise ValueError("box_mask_list must be a BoxMaskList")
if not box_mask_list.has_field("scores"):
raise ValueError("input box_mask_list must have 'scores' field")
scores = box_mask_list.get_field("scores")
if len(scores.shape) > 2:
raise ValueError("Scores should have rank 1 or 2")
if len(scores.shape) == 2 and scores.shape[1] != 1:
raise ValueError(
"Scores should have rank 1 or have shape consistent with [None, 1]"
)
high_score_indices = np.reshape(np.where(np.greater(scores, thresh)), [-1]).astype(
np.int32
)
return gather(box_mask_list, high_score_indices)
================================================
FILE: ava_evaluation/np_box_ops.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Operations for [N, 4] numpy arrays representing bounding boxes.
Example box operations that are supported:
* Areas: compute bounding box areas
* IOU: pairwise intersection-over-union scores
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
def area(boxes):
"""Computes area of boxes.
Args:
boxes: Numpy array with shape [N, 4] holding N boxes
Returns:
a numpy array with shape [N*1] representing box areas
"""
return (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
def intersection(boxes1, boxes2):
"""Compute pairwise intersection areas between boxes.
Args:
boxes1: a numpy array with shape [N, 4] holding N boxes
boxes2: a numpy array with shape [M, 4] holding M boxes
Returns:
a numpy array with shape [N*M] representing pairwise intersection area
"""
[y_min1, x_min1, y_max1, x_max1] = np.split(boxes1, 4, axis=1)
[y_min2, x_min2, y_max2, x_max2] = np.split(boxes2, 4, axis=1)
all_pairs_min_ymax = np.minimum(y_max1, np.transpose(y_max2))
all_pairs_max_ymin = np.maximum(y_min1, np.transpose(y_min2))
intersect_heights = np.maximum(
np.zeros(all_pairs_max_ymin.shape),
all_pairs_min_ymax - all_pairs_max_ymin,
)
all_pairs_min_xmax = np.minimum(x_max1, np.transpose(x_max2))
all_pairs_max_xmin = np.maximum(x_min1, np.transpose(x_min2))
intersect_widths = np.maximum(
np.zeros(all_pairs_max_xmin.shape),
all_pairs_min_xmax - all_pairs_max_xmin,
)
return intersect_heights * intersect_widths
def iou(boxes1, boxes2):
"""Computes pairwise intersection-over-union between box collections.
Args:
boxes1: a numpy array with shape [N, 4] holding N boxes.
boxes2: a numpy array with shape [M, 4] holding N boxes.
Returns:
a numpy array with shape [N, M] representing pairwise iou scores.
"""
intersect = intersection(boxes1, boxes2)
area1 = area(boxes1)
area2 = area(boxes2)
union = np.expand_dims(area1, axis=1) + np.expand_dims(area2, axis=0) - intersect
return intersect / union
def ioa(boxes1, boxes2):
"""Computes pairwise intersection-over-area between box collections.
Intersection-over-area (ioa) between two boxes box1 and box2 is defined as
their intersection area over box2's area. Note that ioa is not symmetric,
that is, IOA(box1, box2) != IOA(box2, box1).
Args:
boxes1: a numpy array with shape [N, 4] holding N boxes.
boxes2: a numpy array with shape [M, 4] holding N boxes.
Returns:
a numpy array with shape [N, M] representing pairwise ioa scores.
"""
intersect = intersection(boxes1, boxes2)
areas = np.expand_dims(area(boxes2), axis=0)
return intersect / areas
================================================
FILE: ava_evaluation/np_mask_ops.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Operations for [N, height, width] numpy arrays representing masks.
Example mask operations that are supported:
* Areas: compute mask areas
* IOU: pairwise intersection-over-union scores
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
EPSILON = 1e-7
def area(masks):
"""Computes area of masks.
Args:
masks: Numpy array with shape [N, height, width] holding N masks. Masks
values are of type np.uint8 and values are in {0,1}.
Returns:
a numpy array with shape [N*1] representing mask areas.
Raises:
ValueError: If masks.dtype is not np.uint8
"""
if masks.dtype != np.uint8:
raise ValueError("Masks type should be np.uint8")
return np.sum(masks, axis=(1, 2), dtype=np.float32)
def intersection(masks1, masks2):
"""Compute pairwise intersection areas between masks.
Args:
masks1: a numpy array with shape [N, height, width] holding N masks. Masks
values are of type np.uint8 and values are in {0,1}.
masks2: a numpy array with shape [M, height, width] holding M masks. Masks
values are of type np.uint8 and values are in {0,1}.
Returns:
a numpy array with shape [N*M] representing pairwise intersection area.
Raises:
ValueError: If masks1 and masks2 are not of type np.uint8.
"""
if masks1.dtype != np.uint8 or masks2.dtype != np.uint8:
raise ValueError("masks1 and masks2 should be of type np.uint8")
n = masks1.shape[0]
m = masks2.shape[0]
answer = np.zeros([n, m], dtype=np.float32)
for i in np.arange(n):
for j in np.arange(m):
answer[i, j] = np.sum(np.minimum(masks1[i], masks2[j]), dtype=np.float32)
return answer
def iou(masks1, masks2):
"""Computes pairwise intersection-over-union between mask collections.
Args:
masks1: a numpy array with shape [N, height, width] holding N masks. Masks
values are of type np.uint8 and values are in {0,1}.
masks2: a numpy array with shape [M, height, width] holding N masks. Masks
values are of type np.uint8 and values are in {0,1}.
Returns:
a numpy array with shape [N, M] representing pairwise iou scores.
Raises:
ValueError: If masks1 and masks2 are not of type np.uint8.
"""
if masks1.dtype != np.uint8 or masks2.dtype != np.uint8:
raise ValueError("masks1 and masks2 should be of type np.uint8")
intersect = intersection(masks1, masks2)
area1 = area(masks1)
area2 = area(masks2)
union = np.expand_dims(area1, axis=1) + np.expand_dims(area2, axis=0) - intersect
return intersect / np.maximum(union, EPSILON)
def ioa(masks1, masks2):
"""Computes pairwise intersection-over-area between box collections.
Intersection-over-area (ioa) between two masks, mask1 and mask2 is defined as
their intersection area over mask2's area. Note that ioa is not symmetric,
that is, IOA(mask1, mask2) != IOA(mask2, mask1).
Args:
masks1: a numpy array with shape [N, height, width] holding N masks. Masks
values are of type np.uint8 and values are in {0,1}.
masks2: a numpy array with shape [M, height, width] holding N masks. Masks
values are of type np.uint8 and values are in {0,1}.
Returns:
a numpy array with shape [N, M] representing pairwise ioa scores.
Raises:
ValueError: If masks1 and masks2 are not of type np.uint8.
"""
if masks1.dtype != np.uint8 or masks2.dtype != np.uint8:
raise ValueError("masks1 and masks2 should be of type np.uint8")
intersect = intersection(masks1, masks2)
areas = np.expand_dims(area(masks2), axis=0)
return intersect / (areas + EPSILON)
================================================
FILE: ava_evaluation/object_detection_evaluation.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""object_detection_evaluation module.
ObjectDetectionEvaluation is a class which manages ground truth information of a
object detection dataset, and computes frequently used detection metrics such as
Precision, Recall, CorLoc of the provided detection results.
It supports the following operations:
1) Add ground truth information of images sequentially.
2) Add detection result of images sequentially.
3) Evaluate detection metrics on already inserted detection results.
4) Write evaluation result into a pickle file for future processing or
visualization.
Note: This module operates on numpy boxes and box lists.
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import collections
import logging
from abc import ABCMeta, abstractmethod
import numpy as np
from . import label_map_util, metrics, per_image_evaluation, standard_fields
class DetectionEvaluator:
"""Interface for object detection evalution classes.
Example usage of the Evaluator:
------------------------------
evaluator = DetectionEvaluator(categories)
# Detections and groundtruth for image 1.
evaluator.add_single_groundtruth_image_info(...)
evaluator.add_single_detected_image_info(...)
# Detections and groundtruth for image 2.
evaluator.add_single_groundtruth_image_info(...)
evaluator.add_single_detected_image_info(...)
metrics_dict = evaluator.evaluate()
"""
__metaclass__ = ABCMeta
def __init__(self, categories):
"""Constructor.
Args:
categories: A list of dicts, each of which has the following keys -
'id': (required) an integer id uniquely identifying this category.
'name': (required) string representing category name e.g., 'cat', 'dog'.
"""
self._categories = categories
@abstractmethod
def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
"""Adds groundtruth for a single image to be used for evaluation.
Args:
image_id: A unique string/integer identifier for the image.
groundtruth_dict: A dictionary of groundtruth numpy arrays required
for evaluations.
"""
pass
@abstractmethod
def add_single_detected_image_info(self, image_id, detections_dict):
"""Adds detections for a single image to be used for evaluation.
Args:
image_id: A unique string/integer identifier for the image.
detections_dict: A dictionary of detection numpy arrays required
for evaluation.
"""
pass
@abstractmethod
def evaluate(self):
"""Evaluates detections and returns a dictionary of metrics."""
pass
@abstractmethod
def clear(self):
"""Clears the state to prepare for a fresh evaluation."""
pass
class ObjectDetectionEvaluator(DetectionEvaluator):
"""A class to evaluate detections."""
def __init__(
self,
categories,
matching_iou_threshold=0.5,
evaluate_corlocs=False,
metric_prefix=None,
use_weighted_mean_ap=False,
evaluate_masks=False,
):
"""Constructor.
Args:
categories: A list of dicts, each of which has the following keys -
'id': (required) an integer id uniquely identifying this category.
'name': (required) string representing category name e.g., 'cat', 'dog'.
matching_iou_threshold: IOU threshold to use for matching groundtruth
boxes to detection boxes.
evaluate_corlocs: (optional) boolean which determines if corloc scores
are to be returned or not.
metric_prefix: (optional) string prefix for metric name; if None, no
prefix is used.
use_weighted_mean_ap: (optional) boolean which determines if the mean
average precision is computed directly from the scores and tp_fp_labels
of all classes.
evaluate_masks: If False, evaluation will be performed based on boxes.
If True, mask evaluation will be performed instead.
Raises:
ValueError: If the category ids are not 1-indexed.
"""
super(ObjectDetectionEvaluator, self).__init__(categories)
self._num_classes = max([cat["id"] for cat in categories])
if min(cat["id"] for cat in categories) < 1:
raise ValueError("Classes should be 1-indexed.")
self._matching_iou_threshold = matching_iou_threshold
self._use_weighted_mean_ap = use_weighted_mean_ap
self._label_id_offset = 1
self._evaluate_masks = evaluate_masks
self._evaluation = ObjectDetectionEvaluation(
num_groundtruth_classes=self._num_classes,
matching_iou_threshold=self._matching_iou_threshold,
use_weighted_mean_ap=self._use_weighted_mean_ap,
label_id_offset=self._label_id_offset,
)
self._image_ids = set([])
self._evaluate_corlocs = evaluate_corlocs
self._metric_prefix = (metric_prefix + "_") if metric_prefix else ""
def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
"""Adds groundtruth for a single image to be used for evaluation.
Args:
image_id: A unique string/integer identifier for the image.
groundtruth_dict: A dictionary containing -
standard_fields.InputDataFields.groundtruth_boxes: float32 numpy array
of shape [num_boxes, 4] containing `num_boxes` groundtruth boxes of
the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
standard_fields.InputDataFields.groundtruth_classes: integer numpy array
of shape [num_boxes] containing 1-indexed groundtruth classes for the
boxes.
standard_fields.InputDataFields.groundtruth_difficult: Optional length
M numpy boolean array denoting whether a ground truth box is a
difficult instance or not. This field is optional to support the case
that no boxes are difficult.
standard_fields.InputDataFields.groundtruth_instance_masks: Optional
numpy array of shape [num_boxes, height, width] with values in {0, 1}.
Raises:
ValueError: On adding groundtruth for an image more than once. Will also
raise error if instance masks are not in groundtruth dictionary.
"""
if image_id in self._image_ids:
raise ValueError("Image with id {} already added.".format(image_id))
groundtruth_classes = (
groundtruth_dict[standard_fields.InputDataFields.groundtruth_classes]
- self._label_id_offset
)
# If the key is not present in the groundtruth_dict or the array is empty
# (unless there are no annotations for the groundtruth on this image)
# use values from the dictionary or insert None otherwise.
if (
standard_fields.InputDataFields.groundtruth_difficult
in groundtruth_dict.keys()
and (
groundtruth_dict[
standard_fields.InputDataFields.groundtruth_difficult
].size
or not groundtruth_classes.size
)
):
groundtruth_difficult = groundtruth_dict[
standard_fields.InputDataFields.groundtruth_difficult
]
else:
groundtruth_difficult = None
if not len(self._image_ids) % 1000:
logging.warning(
"image %s does not have groundtruth difficult flag specified",
image_id,
)
groundtruth_masks = None
if self._evaluate_masks:
if (
standard_fields.InputDataFields.groundtruth_instance_masks
not in groundtruth_dict
):
raise ValueError("Instance masks not in groundtruth dictionary.")
groundtruth_masks = groundtruth_dict[
standard_fields.InputDataFields.groundtruth_instance_masks
]
self._evaluation.add_single_ground_truth_image_info(
image_key=image_id,
groundtruth_boxes=groundtruth_dict[
standard_fields.InputDataFields.groundtruth_boxes
],
groundtruth_class_labels=groundtruth_classes,
groundtruth_is_difficult_list=groundtruth_difficult,
groundtruth_masks=groundtruth_masks,
)
self._image_ids.update([image_id])
def add_single_detected_image_info(self, image_id, detections_dict):
"""Adds detections for a single image to be used for evaluation.
Args:
image_id: A unique string/integer identifier for the image.
detections_dict: A dictionary containing -
standard_fields.DetectionResultFields.detection_boxes: float32 numpy
array of shape [num_boxes, 4] containing `num_boxes` detection boxes
of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
standard_fields.DetectionResultFields.detection_scores: float32 numpy
array of shape [num_boxes] containing detection scores for the boxes.
standard_fields.DetectionResultFields.detection_classes: integer numpy
array of shape [num_boxes] containing 1-indexed detection classes for
the boxes.
standard_fields.DetectionResultFields.detection_masks: uint8 numpy
array of shape [num_boxes, height, width] containing `num_boxes` masks
of values ranging between 0 and 1.
Raises:
ValueError: If detection masks are not in detections dictionary.
"""
detection_classes = (
detections_dict[standard_fields.DetectionResultFields.detection_classes]
- self._label_id_offset
)
detection_masks = None
if self._evaluate_masks:
if (
standard_fields.DetectionResultFields.detection_masks
not in detections_dict
):
raise ValueError("Detection masks not in detections dictionary.")
detection_masks = detections_dict[
standard_fields.DetectionResultFields.detection_masks
]
self._evaluation.add_single_detected_image_info(
image_key=image_id,
detected_boxes=detections_dict[
standard_fields.DetectionResultFields.detection_boxes
],
detected_scores=detections_dict[
standard_fields.DetectionResultFields.detection_scores
],
detected_class_labels=detection_classes,
detected_masks=detection_masks,
)
def evaluate(self):
"""Compute evaluation result.
Returns:
A dictionary of metrics with the following fields -
1. summary_metrics:
'Precision/mAP@<matching_iou_threshold>IOU': mean average precision at
the specified IOU threshold.
2. per_category_ap: category specific results with keys of the form
'PerformanceByCategory/mAP@<matching_iou_threshold>IOU/category'.
"""
(
per_class_ap,
mean_ap,
_,
_,
per_class_corloc,
mean_corloc,
) = self._evaluation.evaluate()
pascal_metrics = {
self._metric_prefix
+ "Precision/mAP@{}IOU".format(self._matching_iou_threshold): mean_ap
}
if self._evaluate_corlocs:
pascal_metrics[
self._metric_prefix
+ "Precision/meanCorLoc@{}IOU".format(self._matching_iou_threshold)
] = mean_corloc
category_index = label_map_util.create_category_index(self._categories)
for idx in range(per_class_ap.size):
if idx + self._label_id_offset in category_index:
display_name = (
self._metric_prefix
+ "PerformanceByCategory/AP@{}IOU/{}".format(
self._matching_iou_threshold,
category_index[idx + self._label_id_offset]["name"],
)
)
pascal_metrics[display_name] = per_class_ap[idx]
# Optionally add CorLoc metrics.classes
if self._evaluate_corlocs:
display_name = (
self._metric_prefix
+ "PerformanceByCategory/CorLoc@{}IOU/{}".format(
self._matching_iou_threshold,
category_index[idx + self._label_id_offset]["name"],
)
)
pascal_metrics[display_name] = per_class_corloc[idx]
return pascal_metrics
def clear(self):
"""Clears the state to prepare for a fresh evaluation."""
self._evaluation = ObjectDetectionEvaluation(
num_groundtruth_classes=self._num_classes,
matching_iou_threshold=self._matching_iou_threshold,
use_weighted_mean_ap=self._use_weighted_mean_ap,
label_id_offset=self._label_id_offset,
)
self._image_ids.clear()
class PascalDetectionEvaluator(ObjectDetectionEvaluator):
"""A class to evaluate detections using PASCAL metrics."""
def __init__(self, categories, matching_iou_threshold=0.5):
super(PascalDetectionEvaluator, self).__init__(
categories,
matching_iou_threshold=matching_iou_threshold,
evaluate_corlocs=False,
metric_prefix="PascalBoxes",
use_weighted_mean_ap=False,
)
class WeightedPascalDetectionEvaluator(ObjectDetectionEvaluator):
"""A class to evaluate detections using weighted PASCAL metrics.
Weighted PASCAL metrics computes the mean average precision as the average
precision given the scores and tp_fp_labels of all classes. In comparison,
PASCAL metrics computes the mean average precision as the mean of the
per-class average precisions.
This definition is very similar to the mean of the per-class average
precisions weighted by class frequency. However, they are typically not the
same as the average precision is not a linear function of the scores and
tp_fp_labels.
"""
def __init__(self, categories, matching_iou_threshold=0.5):
super(WeightedPascalDetectionEvaluator, self).__init__(
categories,
matching_iou_threshold=matching_iou_threshold,
evaluate_corlocs=False,
metric_prefix="WeightedPascalBoxes",
use_weighted_mean_ap=True,
)
class PascalInstanceSegmentationEvaluator(ObjectDetectionEvaluator):
"""A class to evaluate instance masks using PASCAL metrics."""
def __init__(self, categories, matching_iou_threshold=0.5):
super(PascalInstanceSegmentationEvaluator, self).__init__(
categories,
matching_iou_threshold=matching_iou_threshold,
evaluate_corlocs=False,
metric_prefix="PascalMasks",
use_weighted_mean_ap=False,
evaluate_masks=True,
)
class WeightedPascalInstanceSegmentationEvaluator(ObjectDetectionEvaluator):
"""A class to evaluate instance masks using weighted PASCAL metrics.
Weighted PASCAL metrics computes the mean average precision as the average
precision given the scores and tp_fp_labels of all classes. In comparison,
PASCAL metrics computes the mean average precision as the mean of the
per-class average precisions.
This definition is very similar to the mean of the per-class average
precisions weighted by class frequency. However, they are typically not the
same as the average precision is not a linear function of the scores and
tp_fp_labels.
"""
def __init__(self, categories, matching_iou_threshold=0.5):
super(WeightedPascalInstanceSegmentationEvaluator, self).__init__(
categories,
matching_iou_threshold=matching_iou_threshold,
evaluate_corlocs=False,
metric_prefix="WeightedPascalMasks",
use_weighted_mean_ap=True,
evaluate_masks=True,
)
class OpenImagesDetectionEvaluator(ObjectDetectionEvaluator):
"""A class to evaluate detections using Open Images V2 metrics.
Open Images V2 introduce group_of type of bounding boxes and this metric
handles those boxes appropriately.
"""
def __init__(self, categories, matching_iou_threshold=0.5, evaluate_corlocs=False):
"""Constructor.
Args:
categories: A list of dicts, each of which has the following keys -
'id': (required) an integer id uniquely identifying this category.
'name': (required) string representing category name e.g., 'cat', 'dog'.
matching_iou_threshold: IOU threshold to use for matching groundtruth
boxes to detection boxes.
evaluate_corlocs: if True, additionally evaluates and returns CorLoc.
"""
super(OpenImagesDetectionEvaluator, self).__init__(
categories,
matching_iou_threshold,
evaluate_corlocs,
metric_prefix="OpenImagesV2",
)
def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
"""Adds groundtruth for a single image to be used for evaluation.
Args:
image_id: A unique string/integer identifier for the image.
groundtruth_dict: A dictionary containing -
standard_fields.InputDataFields.groundtruth_boxes: float32 numpy array
of shape [num_boxes, 4] containing `num_boxes` groundtruth boxes of
the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
standard_fields.InputDataFields.groundtruth_classes: integer numpy array
of shape [num_boxes] containing 1-indexed groundtruth classes for the
boxes.
standard_fields.InputDataFields.groundtruth_group_of: Optional length
M numpy boolean array denoting whether a groundtruth box contains a
group of instances.
Raises:
ValueError: On adding groundtruth for an image more than once.
"""
if image_id in self._image_ids:
raise ValueError("Image with id {} already added.".format(image_id))
groundtruth_classes = (
groundtruth_dict[standard_fields.InputDataFields.groundtruth_classes]
- self._label_id_offset
)
# If the key is not present in the groundtruth_dict or the array is empty
# (unless there are no annotations for the groundtruth on this image)
# use values from the dictionary or insert None otherwise.
if (
standard_fields.InputDataFields.groundtruth_group_of
in groundtruth_dict.keys()
and (
groundtruth_dict[
standard_fields.InputDataFields.groundtruth_group_of
].size
or not groundtruth_classes.size
)
):
groundtruth_group_of = groundtruth_dict[
standard_fields.InputDataFields.groundtruth_group_of
]
else:
groundtruth_group_of = None
if not len(self._image_ids) % 1000:
logging.warning(
"image %s does not have groundtruth group_of flag specified",
image_id,
)
self._evaluation.add_single_ground_truth_image_info(
image_id,
groundtruth_dict[standard_fields.InputDataFields.groundtruth_boxes],
groundtruth_classes,
groundtruth_is_difficult_list=None,
groundtruth_is_group_of_list=groundtruth_group_of,
)
self._image_ids.update([image_id])
ObjectDetectionEvalMetrics = collections.namedtuple(
"ObjectDetectionEvalMetrics",
[
"average_precisions",
"mean_ap",
"precisions",
"recalls",
"corlocs",
"mean_corloc",
],
)
class ObjectDetectionEvaluation:
"""Internal implementation of Pascal object detection metrics."""
def __init__(
self,
num_groundtruth_classes,
matching_iou_threshold=0.5,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000,
use_weighted_mean_ap=False,
label_id_offset=0,
):
if num_groundtruth_classes < 1:
raise ValueError("Need at least 1 groundtruth class for evaluation.")
self.per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes=num_groundtruth_classes,
matching_iou_threshold=matching_iou_threshold,
)
self.num_class = num_groundtruth_classes
self.use_weighted_mean_ap = use_weighted_mean_ap
self.label_id_offset = label_id_offset
self.groundtruth_boxes = {}
self.groundtruth_class_labels = {}
self.groundtruth_masks = {}
self.groundtruth_is_difficult_list = {}
self.groundtruth_is_group_of_list = {}
self.num_gt_instances_per_class = np.zeros(self.num_class, dtype=int)
self.num_gt_imgs_per_class = np.zeros(self.num_class, dtype=int)
self._initialize_detections()
def _initialize_detections(self):
self.detection_keys = set()
self.scores_per_class = [[] for _ in range(self.num_class)]
self.tp_fp_labels_per_class = [[] for _ in range(self.num_class)]
self.num_images_correctly_detected_per_class = np.zeros(self.num_class)
self.average_precision_per_class = np.empty(self.num_class, dtype=float)
self.average_precision_per_class.fill(np.nan)
self.precisions_per_class = []
self.recalls_per_class = []
self.corloc_per_class = np.ones(self.num_class, dtype=float)
def clear_detections(self):
self._initialize_detections()
def add_single_ground_truth_image_info(
self,
image_key,
groundtruth_boxes,
groundtruth_class_labels,
groundtruth_is_difficult_list=None,
groundtruth_is_group_of_list=None,
groundtruth_masks=None,
):
"""Adds groundtruth for a single image to be used for evaluation.
Args:
image_key: A unique string/integer identifier for the image.
groundtruth_boxes: float32 numpy array of shape [num_boxes, 4]
containing `num_boxes` groundtruth boxes of the format
[ymin, xmin, ymax, xmax] in absolute image coordinates.
groundtruth_class_labels: integer numpy array of shape [num_boxes]
containing 0-indexed groundtruth classes for the boxes.
groundtruth_is_difficult_list: A length M numpy boolean array denoting
whether a ground truth box is a difficult instance or not. To support
the case that no boxes are difficult, it is by default set as None.
groundtruth_is_group_of_list: A length M numpy boolean array denoting
whether a ground truth box is a group-of box or not. To support
the case that no boxes are groups-of, it is by default set as None.
groundtruth_masks: uint8 numpy array of shape
[num_boxes, height, width] containing `num_boxes` groundtruth masks.
The mask values range from 0 to 1.
"""
if image_key in self.groundtruth_boxes:
logging.warning(
"image %s has already been added to the ground truth database.",
image_key,
)
return
self.groundtruth_boxes[image_key] = groundtruth_boxes
self.groundtruth_class_labels[image_key] = groundtruth_class_labels
self.groundtruth_masks[image_key] = groundtruth_masks
if groundtruth_is_difficult_list is None:
num_boxes = groundtruth_boxes.shape[0]
groundtruth_is_difficult_list = np.zeros(num_boxes, dtype=bool)
self.groundtruth_is_difficult_list[image_key] = (
groundtruth_is_difficult_list.astype(dtype=bool)
)
if groundtruth_is_group_of_list is None:
num_boxes = groundtruth_boxes.shape[0]
groundtruth_is_group_of_list = np.zeros(num_boxes, dtype=bool)
self.groundtruth_is_group_of_list[image_key] = (
groundtruth_is_group_of_list.astype(dtype=bool)
)
self._update_ground_truth_statistics(
groundtruth_class_labels,
groundtruth_is_difficult_list.astype(dtype=bool),
groundtruth_is_group_of_list.astype(dtype=bool),
)
def add_single_detected_image_info(
self,
image_key,
detected_boxes,
detected_scores,
detected_class_labels,
detected_masks=None,
):
"""Adds detections for a single image to be used for evaluation.
Args:
image_key: A unique string/integer identifier for the image.
detected_boxes: float32 numpy array of shape [num_boxes, 4]
containing `num_boxes` detection boxes of the format
[ymin, xmin, ymax, xmax] in absolute image coordinates.
detected_scores: float32 numpy array of shape [num_boxes] containing
detection scores for the boxes.
detected_class_labels: integer numpy array of shape [num_boxes] containing
0-indexed detection classes for the boxes.
detected_masks: np.uint8 numpy array of shape [num_boxes, height, width]
containing `num_boxes` detection masks with values ranging
between 0 and 1.
Raises:
ValueError: if the number of boxes, scores and class labels differ in
length.
"""
if len(detected_boxes) != len(detected_scores) or len(detected_boxes) != len(
detected_class_labels
):
raise ValueError(
"detected_boxes, detected_scores and "
"detected_class_labels should all have same lengths. Got"
"[%d, %d, %d]" % len(detected_boxes),
len(detected_scores),
len(detected_class_labels),
)
if image_key in self.detection_keys:
logging.warning(
"image %s has already been added to the detection result database",
image_key,
)
return
self.detection_keys.add(image_key)
if image_key in self.groundtruth_boxes:
groundtruth_boxes = self.groundtruth_boxes[image_key]
groundtruth_class_labels = self.groundtruth_class_labels[image_key]
# Masks are popped instead of look up. The reason is that we do not want
# to keep all masks in memory which can cause memory overflow.
groundtruth_masks = self.groundtruth_masks.pop(image_key)
groundtruth_is_difficult_list = self.groundtruth_is_difficult_list[
image_key
]
groundtruth_is_group_of_list = self.groundtruth_is_group_of_list[image_key]
else:
groundtruth_boxes = np.empty(shape=[0, 4], dtype=float)
groundtruth_class_labels = np.array([], dtype=int)
if detected_masks is None:
groundtruth_masks = None
else:
groundtruth_masks = np.empty(shape=[0, 1, 1], dtype=float)
groundtruth_is_difficult_list = np.array([], dtype=bool)
groundtruth_is_group_of_list = np.array([], dtype=bool)
(
scores,
tp_fp_labels,
) = self.per_image_eval.compute_object_detection_metrics(
detected_boxes=detected_boxes,
detected_scores=detected_scores,
detected_class_labels=detected_class_labels,
groundtruth_boxes=groundtruth_boxes,
groundtruth_class_labels=groundtruth_class_labels,
groundtruth_is_difficult_list=groundtruth_is_difficult_list,
groundtruth_is_group_of_list=groundtruth_is_group_of_list,
detected_masks=detected_masks,
groundtruth_masks=groundtruth_masks,
)
for i in range(self.num_class):
if scores[i].shape[0] > 0:
self.scores_per_class[i].append(scores[i])
self.tp_fp_labels_per_class[i].append(tp_fp_labels[i])
def _update_ground_truth_statistics(
self,
groundtruth_class_labels,
groundtruth_is_difficult_list,
groundtruth_is_group_of_list,
):
"""Update grouth truth statitistics.
1. Difficult boxes are ignored when counting the number of ground truth
instances as done in Pascal VOC devkit.
2. Difficult boxes are treated as normal boxes when computing CorLoc related
statitistics.
Args:
groundtruth_class_labels: An integer numpy array of length M,
representing M class labels of object instances in ground truth
groundtruth_is_difficult_list: A boolean numpy array of length M denoting
whether a ground truth box is a difficult instance or not
groundtruth_is_group_of_list: A boolean numpy array of length M denoting
whether a ground truth box is a group-of box or not
"""
for class_index in range(self.num_class):
num_gt_instances = np.sum(
groundtruth_class_labels[
~groundtruth_is_difficult_list & ~groundtruth_is_group_of_list
]
== class_index
)
self.num_gt_instances_per_class[class_index] += num_gt_instances
if np.any(groundtruth_class_labels == class_index):
self.num_gt_imgs_per_class[class_index] += 1
def evaluate(self):
"""Compute evaluation result.
Returns:
A named tuple with the following fields -
average_precision: float numpy array of average precision for
each class.
mean_ap: mean average precision of all classes, float scalar
precisions: List of precisions, each precision is a float numpy
array
recalls: List of recalls, each recall is a float numpy array
corloc: numpy float array
mean_corloc: Mean CorLoc score for each class, float scalar
"""
if (self.num_gt_instances_per_class == 0).any():
logging.info(
"The following classes have no ground truth examples: %s",
np.squeeze(np.argwhere(self.num_gt_instances_per_class == 0))
+ self.label_id_offset,
)
if self.use_weighted_mean_ap:
all_scores = np.array([], dtype=float)
all_tp_fp_labels = np.array([], dtype=bool)
for class_index in range(self.num_class):
if self.num_gt_instances_per_class[class_index] == 0:
continue
if not self.scores_per_class[class_index]:
scores = np.array([], dtype=float)
tp_fp_labels = np.array([], dtype=bool)
else:
scores = np.concatenate(self.scores_per_class[class_index])
tp_fp_labels = np.concatenate(self.tp_fp_labels_per_class[class_index])
if self.use_weighted_mean_ap:
all_scores = np.append(all_scores, scores)
all_tp_fp_labels = np.append(all_tp_fp_labels, tp_fp_labels)
precision, recall = metrics.compute_precision_recall(
scores,
tp_fp_labels,
self.num_gt_instances_per_class[class_index],
)
self.precisions_per_class.append(precision)
self.recalls_per_class.append(recall)
average_precision = metrics.compute_average_precision(precision, recall)
self.average_precision_per_class[class_index] = average_precision
self.corloc_per_class = metrics.compute_cor_loc(
self.num_gt_imgs_per_class,
self.num_images_correctly_detected_per_class,
)
if self.use_weighted_mean_ap:
num_gt_instances = np.sum(self.num_gt_instances_per_class)
precision, recall = metrics.compute_precision_recall(
all_scores, all_tp_fp_labels, num_gt_instances
)
mean_ap = metrics.compute_average_precision(precision, recall)
else:
mean_ap = np.nanmean(self.average_precision_per_class)
mean_corloc = np.nanmean(self.corloc_per_class)
return ObjectDetectionEvalMetrics(
self.average_precision_per_class,
mean_ap,
self.precisions_per_class,
self.recalls_per_class,
self.corloc_per_class,
mean_corloc,
)
================================================
FILE: ava_evaluation/per_image_evaluation.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Evaluate Object Detection result on a single image.
Annotate each detected result as true positives or false positive according to
a predefined IOU ratio. Non Maximum Supression is used by default. Multi class
detection is supported by default.
Based on the settings, per image evaluation is either performed on boxes or
on object masks.
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import numpy as np
from . import np_box_list, np_box_list_ops
class PerImageEvaluation:
"""Evaluate detection result of a single image."""
def __init__(self, num_groundtruth_classes, matching_iou_threshold=0.5):
"""Initialized PerImageEvaluation by evaluation parameters.
Args:
num_groundtruth_classes: Number of ground truth object classes
matching_iou_threshold: A ratio of area intersection to union, which is
the threshold to consider whether a detection is true positive or not
"""
self.matching_iou_threshold = matching_iou_threshold
self.num_groundtruth_classes = num_groundtruth_classes
def compute_object_detection_metrics(
self,
detected_boxes,
detected_scores,
detected_class_labels,
groundtruth_boxes,
groundtruth_class_labels,
groundtruth_is_difficult_list,
groundtruth_is_group_of_list,
detected_masks=None,
groundtruth_masks=None,
):
"""Evaluates detections as being tp, fp or ignored from a single image.
The evaluation is done in two stages:
1. All detections are matched to non group-of boxes; true positives are
determined and detections matched to difficult boxes are ignored.
2. Detections that are determined as false positives are matched against
group-of boxes and ignored if matched.
Args:
detected_boxes: A float numpy array of shape [N, 4], representing N
regions of detected object regions.
Each row is of the format [y_min, x_min, y_max, x_max]
detected_scores: A float numpy array of shape [N, 1], representing
the confidence scores of the detected N object instances.
detected_class_labels: A integer numpy array of shape [N, 1], repreneting
the class labels of the detected N object instances.
groundtruth_boxes: A float numpy array of shape [M, 4], representing M
regions of object instances in ground truth
groundtruth_class_labels: An integer numpy array of shape [M, 1],
representing M class labels of object instances in ground truth
groundtruth_is_difficult_list: A boolean numpy array of length M denoting
whether a ground truth box is a difficult instance or not
groundtruth_is_group_of_list: A boolean numpy array of length M denoting
whether a ground truth box has group-of tag
detected_masks: (optional) A uint8 numpy array of shape
[N, height, width]. If not None, the metrics will be computed based
on masks.
groundtruth_masks: (optional) A uint8 numpy array of shape
[M, height, width].
Returns:
scores: A list of C float numpy arrays. Each numpy array is of
shape [K, 1], representing K scores detected with object class
label c
tp_fp_labels: A list of C boolean numpy arrays. Each numpy array
is of shape [K, 1], representing K True/False positive label of
object instances detected with class label c
"""
(
detected_boxes,
detected_scores,
detected_class_labels,
detected_masks,
) = self._remove_invalid_boxes(
detected_boxes,
detected_scores,
detected_class_labels,
detected_masks,
)
scores, tp_fp_labels = self._compute_tp_fp(
detected_boxes=detected_boxes,
detected_scores=detected_scores,
detected_class_labels=detected_class_labels,
groundtruth_boxes=groundtruth_boxes,
groundtruth_class_labels=groundtruth_class_labels,
groundtruth_is_difficult_list=groundtruth_is_difficult_list,
groundtruth_is_group_of_list=groundtruth_is_group_of_list,
detected_masks=detected_masks,
groundtruth_masks=groundtruth_masks,
)
return scores, tp_fp_labels
def _compute_tp_fp(
self,
detected_boxes,
detected_scores,
detected_class_labels,
groundtruth_boxes,
groundtruth_class_labels,
groundtruth_is_difficult_list,
groundtruth_is_group_of_list,
detected_masks=None,
groundtruth_masks=None,
):
"""Labels true/false positives of detections of an image across all classes.
Args:
detected_boxes: A float numpy array of shape [N, 4], representing N
regions of detected object regions.
Each row is of the format [y_min, x_min, y_max, x_max]
detected_scores: A float numpy array of shape [N, 1], representing
the confidence scores of the detected N object instances.
detected_class_labels: A integer numpy array of shape [N, 1], repreneting
the class labels of the detected N object instances.
groundtruth_boxes: A float numpy array of shape [M, 4], representing M
regions of object instances in ground truth
groundtruth_class_labels: An integer numpy array of shape [M, 1],
representing M class labels of object instances in ground truth
groundtruth_is_difficult_list: A boolean numpy array of length M denoting
whether a ground truth box is a difficult instance or not
groundtruth_is_group_of_list: A boolean numpy array of length M denoting
whether a ground truth box has group-of tag
detected_masks: (optional) A np.uint8 numpy array of shape
[N, height, width]. If not None, the scores will be computed based
on masks.
groundtruth_masks: (optional) A np.uint8 numpy array of shape
[M, height, width].
Returns:
result_scores: A list of float numpy arrays. Each numpy array is of
shape [K, 1], representing K scores detected with object class
label c
result_tp_fp_labels: A list of boolean numpy array. Each numpy array is of
shape [K, 1], representing K True/False positive label of object
instances detected with class label c
Raises:
ValueError: If detected masks is not None but groundtruth masks are None,
or the other way around.
"""
if detected_masks is not None and groundtruth_masks is None:
raise ValueError(
"Detected masks is available but groundtruth masks is not."
)
if detected_masks is None and groundtruth_masks is not None:
raise ValueError(
"Groundtruth masks is available but detected masks is not."
)
result_scores = []
result_tp_fp_labels = []
for i in range(self.num_groundtruth_classes):
groundtruth_is_difficult_list_at_ith_class = groundtruth_is_difficult_list[
groundtruth_class_labels == i
]
groundtruth_is_group_of_list_at_ith_class = groundtruth_is_group_of_list[
groundtruth_class_labels == i
]
(
gt_boxes_at_ith_class,
gt_masks_at_ith_class,
detected_boxes_at_ith_class,
detected_scores_at_ith_class,
detected_masks_at_ith_class,
) = self._get_ith_class_arrays(
detected_boxes,
detected_scores,
detected_masks,
detected_class_labels,
groundtruth_boxes,
groundtruth_masks,
groundtruth_class_labels,
i,
)
scores, tp_fp_labels = self._compute_tp_fp_for_single_class(
detected_boxes=detected_boxes_at_ith_class,
detected_scores=detected_scores_at_ith_class,
groundtruth_boxes=gt_boxes_at_ith_class,
groundtruth_is_difficult_list=groundtruth_is_difficult_list_at_ith_class,
groundtruth_is_group_of_list=groundtruth_is_group_of_list_at_ith_class,
detected_masks=detected_masks_at_ith_class,
groundtruth_masks=gt_masks_at_ith_class,
)
result_scores.append(scores)
result_tp_fp_labels.append(tp_fp_labels)
return result_scores, result_tp_fp_labels
def _get_overlaps_and_scores_box_mode(
self,
detected_boxes,
detected_scores,
groundtruth_boxes,
groundtruth_is_group_of_list,
):
"""Computes overlaps and scores between detected and groudntruth boxes.
Args:
detected_boxes: A numpy array of shape [N, 4] representing detected box
coordinates
detected_scores: A 1-d numpy array of length N representing classification
score
groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth
box coordinates
groundtruth_is_group_of_list: A boolean numpy array of length M denoting
whether a ground truth box has group-of tag. If a groundtruth box
is group-of box, every detection matching this box is ignored.
Returns:
iou: A float numpy array of size [num_detected_boxes, num_gt_boxes]. If
gt_non_group_of_boxlist.num_boxes() == 0 it will be None.
ioa: A float numpy array of size [num_detected_boxes, num_gt_boxes]. If
gt_group_of_boxlist.num_boxes() == 0 it will be None.
scores: The score of the detected boxlist.
num_boxes: Number of non-maximum suppressed detected boxes.
"""
detected_boxlist = np_box_list.BoxList(detected_boxes)
detected_boxlist.add_field("scores", detected_scores)
gt_non_group_of_boxlist = np_box_list.BoxList(
groundtruth_boxes[~groundtruth_is_group_of_list]
)
iou = np_box_list_ops.iou(detected_boxlist, gt_non_group_of_boxlist)
scores = detected_boxlist.get_field("scores")
num_boxes = detected_boxlist.num_boxes()
return iou, None, scores, num_boxes
def _compute_tp_fp_for_single_class(
self,
detected_boxes,
detected_scores,
groundtruth_boxes,
groundtruth_is_difficult_list,
groundtruth_is_group_of_list,
detected_masks=None,
groundtruth_masks=None,
):
"""Labels boxes detected with the same class from the same image as tp/fp.
Args:
detected_boxes: A numpy array of shape [N, 4] representing detected box
coordinates
detected_scores: A 1-d numpy array of length N representing classification
score
groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth
box coordinates
groundtruth_is_difficult_list: A boolean numpy array of length M denoting
whether a ground truth box is a difficult instance or not. If a
groundtruth box is difficult, every detection matching this box
is ignored.
groundtruth_is_group_of_list: A boolean numpy array of length M denoting
whether a ground truth box has group-of tag. If a groundtruth box
is group-of box, every detection matching this box is ignored.
detected_masks: (optional) A uint8 numpy array of shape
[N, height, width]. If not None, the scores will be computed based
on masks.
groundtruth_masks: (optional) A uint8 numpy array of shape
[M, height, width].
Returns:
Two arrays of the same size, containing all boxes that were evaluated as
being true positives or false positives; if a box matched to a difficult
box or to a group-of box, it is ignored.
scores: A numpy array representing the detection scores.
tp_fp_labels: a boolean numpy array indicating whether a detection is a
true positive.
"""
if detected_boxes.size == 0:
return np.array([], dtype=float), np.array([], dtype=bool)
(
iou,
_,
scores,
num_detected_boxes,
) = self._get_overlaps_and_scores_box_mode(
detected_boxes=detected_boxes,
detected_scores=detected_scores,
groundtruth_boxes=groundtruth_boxes,
groundtruth_is_group_of_list=groundtruth_is_group_of_list,
)
if groundtruth_boxes.size == 0:
return scores, np.zeros(num_detected_boxes, dtype=bool)
tp_fp_labels = np.zeros(num_detected_boxes, dtype=bool)
is_matched_to_difficult_box = np.zeros(num_detected_boxes, dtype=bool)
is_matched_to_group_of_box = np.zeros(num_detected_boxes, dtype=bool)
# The evaluation is done in two stages:
# 1. All detections are matched to non group-of boxes; true positives are
# determined and detections matched to difficult boxes are ignored.
# 2. Detections that are determined as false positives are matched against
# group-of boxes and ignored if matched.
# Tp-fp evaluation for non-group of boxes (if any).
if iou.shape[1] > 0:
groundtruth_nongroup_of_is_difficult_list = groundtruth_is_difficult_list[
~groundtruth_is_group_of_list
]
max_overlap_gt_ids = np.argmax(iou, axis=1)
is_gt_box_detected = np.zeros(iou.shape[1], dtype=bool)
for i in range(num_detected_boxes):
gt_id = max_overlap_gt_ids[i]
if iou[i, gt_id] >= self.matching_iou_threshold:
if not groundtruth_nongroup_of_is_difficult_list[gt_id]:
if not is_gt_box_detected[gt_id]:
tp_fp_labels[i] = True
is_gt_box_detected[gt_id] = True
else:
is_matched_to_difficult_box[i] = True
return (
scores[~is_matched_to_difficult_box & ~is_matched_to_group_of_box],
tp_fp_labels[~is_matched_to_difficult_box & ~is_matched_to_group_of_box],
)
def _get_ith_class_arrays(
self,
detected_boxes,
detected_scores,
detected_masks,
detected_class_labels,
groundtruth_boxes,
groundtruth_masks,
groundtruth_class_labels,
class_index,
):
"""Returns numpy arrays belonging to class with index `class_index`.
Args:
detected_boxes: A numpy array containing detected boxes.
detected_scores: A numpy array containing detected scores.
detected_masks: A numpy array containing detected masks.
detected_class_labels: A numpy array containing detected class labels.
groundtruth_boxes: A numpy array containing groundtruth boxes.
groundtruth_masks: A numpy array containing groundtruth masks.
groundtruth_class_labels: A numpy array containing groundtruth class
labels.
class_index: An integer index.
Returns:
gt_boxes_at_ith_class: A numpy array containing groundtruth boxes labeled
as ith class.
gt_masks_at_ith_class: A numpy array containing groundtruth masks labeled
as ith class.
detected_boxes_at_ith_class: A numpy array containing detected boxes
corresponding to the ith class.
detected_scores_at_ith_class: A numpy array containing detected scores
corresponding to the ith class.
detected_masks_at_ith_class: A numpy array containing detected masks
corresponding to the ith class.
"""
selected_groundtruth = groundtruth_class_labels == class_index
gt_boxes_at_ith_class = groundtruth_boxes[selected_groundtruth]
if groundtruth_masks is not None:
gt_masks_at_ith_class = groundtruth_masks[selected_groundtruth]
else:
gt_masks_at_ith_class = None
selected_detections = detected_class_labels == class_index
detected_boxes_at_ith_class = detected_boxes[selected_detections]
detected_scores_at_ith_class = detected_scores[selected_detections]
if detected_masks is not None:
detected_masks_at_ith_class = detected_masks[selected_detections]
else:
detected_masks_at_ith_class = None
return (
gt_boxes_at_ith_class,
gt_masks_at_ith_class,
detected_boxes_at_ith_class,
detected_scores_at_ith_class,
detected_masks_at_ith_class,
)
def _remove_invalid_boxes(
self,
detected_boxes,
detected_scores,
detected_class_labels,
detected_masks=None,
):
"""Removes entries with invalid boxes.
A box is invalid if either its xmax is smaller than its xmin, or its ymax
is smaller than its ymin.
Args:
detected_boxes: A float numpy array of size [num_boxes, 4] containing box
coordinates in [ymin, xmin, ymax, xmax] format.
detected_scores: A float numpy array of size [num_boxes].
detected_class_labels: A int32 numpy array of size [num_boxes].
detected_masks: A uint8 numpy array of size [num_boxes, height, width].
Returns:
valid_detected_boxes: A float numpy array of size [num_valid_boxes, 4]
containing box coordinates in [ymin, xmin, ymax, xmax] format.
valid_detected_scores: A float numpy array of size [num_valid_boxes].
valid_detected_class_labels: A int32 numpy array of size
[num_valid_boxes].
valid_detected_masks: A uint8 numpy array of size
[num_valid_boxes, height, width].
"""
valid_indices = np.logical_and(
detected_boxes[:, 0] < detected_boxes[:, 2],
detected_boxes[:, 1] < detected_boxes[:, 3],
)
detected_boxes = detected_boxes[valid_indices]
detected_scores = detected_scores[valid_indices]
detected_class_labels = detected_class_labels[valid_indices]
if detected_masks is not None:
detected_masks = detected_masks[valid_indices]
return [
detected_boxes,
detected_scores,
detected_class_labels,
detected_masks,
]
================================================
FILE: ava_evaluation/standard_fields.py
================================================
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Contains classes specifying naming conventions used for object detection.
Specifies:
InputDataFields: standard fields used by reader/preprocessor/batcher.
DetectionResultFields: standard fields returned by object detector.
BoxListFields: standard field used by BoxList
TfExampleFields: standard fields for tf-example data format (go/tf-example).
"""
from __future__ import absolute_import, division, print_function, unicode_literals
class InputDataFields:
"""Names for the input tensors.
Holds the standard data field names to use for identifying input tensors. This
should be used by the decoder to identify keys for the returned tensor_dict
containing input tensors. And it should be used by the model to identify the
tensors it needs.
Attributes:
image: image.
original_image: image in the original input size.
key: unique key corresponding to image.
source_id: source of the original image.
filename: original filename of the dataset (without common path).
groundtruth_image_classes: image-level class labels.
groundtruth_boxes: coordinates of the ground truth boxes in the image.
groundtruth_classes: box-level class labels.
groundtruth_label_types: box-level label types (e.g. explicit negative).
groundtruth_is_crowd: [DEPRECATED, use groundtruth_group_of instead]
is the groundtruth a single object or a crowd.
groundtruth_area: area of a groundtruth segment.
groundtruth_difficult: is a `difficult` object
groundtruth_group_of: is a `group_of` objects, e.g. multiple objects of the
same class, forming a connected group, where instances are heavily
occluding each other.
proposal_boxes: coordinates of object proposal boxes.
proposal_objectness: objectness score of each proposal.
groundtruth_instance_masks: ground truth instance masks.
groundtruth_instance_boundaries: ground truth instance boundaries.
groundtruth_instance_classes: instance mask-level class labels.
groundtruth_keypoints: ground truth keypoints.
groundtruth_keypoint_visibilities: ground truth keypoint visibilities.
groundtruth_label_scores: groundtruth label scores.
groundtruth_weights: groundtruth weight factor for bounding boxes.
num_groundtruth_boxes: number of groundtruth boxes.
true_image_shapes: true shapes of images in the resized images, as resized
images can be padded with zeros.
"""
image = "image"
original_image = "original_image"
key = "key"
source_id = "source_id"
filename = "filename"
groundtruth_image_classes = "groundtruth_image_classes"
groundtruth_boxes = "groundtruth_boxes"
groundtruth_classes = "groundtruth_classes"
groundtruth_label_types = "groundtruth_label_types"
groundtruth_is_crowd = "groundtruth_is_crowd"
groundtruth_area = "groundtruth_area"
groundtruth_difficult = "groundtruth_difficult"
groundtruth_group_of = "groundtruth_group_of"
proposal_boxes = "proposal_boxes"
proposal_objectness = "proposal_objectness"
groundtruth_instance_masks = "groundtruth_instance_masks"
groundtruth_instance_boundaries = "groundtruth_instance_boundaries"
groundtruth_instance_classes = "groundtruth_instance_classes"
groundtruth_keypoints = "groundtruth_keypoints"
groundtruth_keypoint_visibilities = "groundtruth_keypoint_visibilities"
groundtruth_label_scores = "groundtruth_label_scores"
groundtruth_weights = "groundtruth_weights"
num_groundtruth_boxes = "num_groundtruth_boxes"
true_image_shape = "true_image_shape"
class DetectionResultFields:
"""Naming conventions for storing the output of the detector.
Attributes:
source_id: source of the original image.
key: unique key corresponding to image.
detection_boxes: coordinates of the detection boxes in the image.
detection_scores: detection scores for the detection boxes in the image.
detection_classes: detection-level class labels.
detection_masks: contains a segmentation mask for each detection box.
detection_boundaries: contains an object boundary for each detection box.
detection_keypoints: contains detection keypoints for each detection box.
num_detections: number of detections in the batch.
"""
source_id = "source_id"
key = "key"
detection_boxes = "detection_boxes"
detection_scores = "detection_scores"
detection_classes = "detection_classes"
detection_masks = "detection_masks"
detection_boundaries = "detection_boundaries"
detection_keypoints = "detection_keypoints"
num_detections = "num_detections"
class BoxListFields:
"""Naming conventions for BoxLists.
Attributes:
boxes: bounding box coordinates.
classes: classes per bounding box.
scores: scores per bounding box.
weights: sample weights per bounding box.
objectness: objectness score per bounding box.
masks: masks per bounding box.
boundaries: boundaries per bounding box.
keypoints: keypoints per bounding box.
keypoint_heatmaps: keypoint heatmaps per bounding box.
"""
boxes = "boxes"
classes = "classes"
scores = "scores"
weights = "weights"
objectness = "objectness"
masks = "masks"
boundaries = "boundaries"
keypoints = "keypoints"
keypoint_heatmaps = "keypoint_heatmaps"
class TfExampleFields:
"""TF-example proto feature names for object detection.
Holds the standard feature names to load from an Example proto for object
detection.
Attributes:
image_encoded: JPEG encoded string
image_format: image format, e.g. "JPEG"
filename: filename
channels: number of channels of image
colorspace: colorspace, e.g. "RGB"
height: height of image in pixels, e.g. 462
width: width of image in pixels, e.g. 581
source_id: original source of the image
object_class_text: labels in text format, e.g. ["person", "cat"]
object_class_label: labels in numbers, e.g. [16, 8]
object_bbox_xmin: xmin coordinates of groundtruth box, e.g. 10, 30
object_bbox_xmax: xmax coordinates of groundtruth box, e.g. 50, 40
object_bbox_ymin: ymin coordinates of groundtruth box, e.g. 40, 50
object_bbox_ymax: ymax coordinates of groundtruth box, e.g. 80, 70
object_view: viewpoint of object, e.g. ["frontal", "left"]
object_truncated: is object truncated, e.g. [true, false]
object_occluded: is object occluded, e.g. [true, false]
object_difficult: is object difficult, e.g. [true, false]
object_group_of: is object a single object or a group of objects
object_depiction: is object a depiction
object_is_crowd: [DEPRECATED, use object_group_of instead]
is the object a single object or a crowd
object_segment_area: the area of the segment.
object_weight: a weight factor for the object's bounding box.
instance_masks: instance segmentation masks.
instance_boundaries: instance boundaries.
instance_classes: Classes for each instance segmentation mask.
detection_class_label: class label in numbers.
detection_bbox_ymin: ymin coordinates of a detection box.
detection_bbox_xmin: xmin coordinates of a detection box.
detection_bbox_ymax: ymax coordinates of a detection box.
detection_bbox_xmax: xmax coordinates of a detection box.
detection_score: detection score for the class label and box.
"""
image_encoded = "image/encoded"
image_format = "image/format" # format is reserved keyword
filename = "image/filename"
channels = "image/channels"
colorspace = "image/colorspace"
height = "image/height"
width = "image/width"
source_id = "image/source_id"
object_class_text = "image/object/class/text"
object_class_label = "image/object/class/label"
object_bbox_ymin = "image/object/bbox/ymin"
object_bbox_xmin = "image/object/bbox/xmin"
object_bbox_ymax = "image/object/bbox/ymax"
object_bbox_xmax = "image/object/bbox/xmax"
object_view = "image/object/view"
object_truncated = "image/object/truncated"
object_occluded = "image/object/occluded"
object_difficult = "image/object/difficult"
object_group_of = "image/object/group_of"
object_depiction = "image/object/depiction"
object_is_crowd = "image/object/is_crowd"
object_segment_area = "image/object/segment/area"
object_weight = "image/object/weight"
instance_masks = "image/segmentation/object"
instance_boundaries = "image/boundaries/object"
instance_classes = "image/segmentation/object/class"
detection_class_label = "image/detection/label"
detection_bbox_ymin = "image/detection/bbox/ymin"
detection_bbox_xmin = "image/detection/bbox/xmin"
detection_bbox_ymax = "image/detection/bbox/ymax"
detection_bbox_xmax = "image/detection/bbox/xmax"
detection_score = "image/detection/score"
================================================
FILE: configs/AVA/SLOWFAST_32x2_R50_SHORT.yaml
================================================
TRAIN:
ENABLE: True
DATASET: ava
BATCH_SIZE: 64
EVAL_PERIOD: 5
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
# CHECKPOINT_FILE_PATH: path to the pretrain checkpoint file.
CHECKPOINT_TYPE: caffe2
DATA:
NUM_FRAMES: 32
SAMPLING_RATE: 2
TRAIN_JITTER_SCALES: [256, 320]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 224
INPUT_CHANNEL_NUM: [3, 3]
DETECTION:
ENABLE: True
ALIGNED: True
AVA:
DETECTION_SCORE_THRESH: 0.8
TRAIN_PREDICT_BOX_LISTS: [
"ava_train_v2.2.csv",
"person_box_67091280_iou90/ava_detection_train_boxes_and_labels_include_negative_v2.2.csv",
]
TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"]
SLOWFAST:
ALPHA: 4
BETA_INV: 8
FUSION_CONV_CHANNEL_RATIO: 2
FUSION_KERNEL_SZ: 7
RESNET:
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 50
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]]
SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]]
SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]]
NONLOCAL:
LOCATION: [[[], []], [[], []], [[], []], [[], []]]
GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]]
INSTANTIATION: dot_product
POOL: [[[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]]]
BN:
USE_PRECISE_STATS: False
NUM_BATCHES_PRECISE: 200
SOLVER:
BASE_LR: 0.1
LR_POLICY: steps_with_relative_lrs
STEPS: [0, 10, 15, 20]
LRS: [1, 0.1, 0.01, 0.001]
MAX_EPOCH: 20
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-7
WARMUP_EPOCHS: 5.0
WARMUP_START_LR: 0.000125
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 80
ARCH: slowfast
MODEL_NAME: SlowFast
LOSS_FUNC: bce
DROPOUT_RATE: 0.5
HEAD_ACT: sigmoid
TEST:
ENABLE: True
DATASET: ava
BATCH_SIZE: 8
DATA_LOADER:
NUM_WORKERS: 2
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/AVA/SLOW_8x8_R50_SHORT.yaml
================================================
TRAIN:
ENABLE: True
DATASET: ava
BATCH_SIZE: 64
EVAL_PERIOD: 5
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
# CHECKPOINT_FILE_PATH: path to the pretrain checkpoint file.
CHECKPOINT_TYPE: caffe2
DATA:
NUM_FRAMES: 4
SAMPLING_RATE: 16
TRAIN_JITTER_SCALES: [256, 320]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3]
DETECTION:
ENABLE: True
ALIGNED: True
AVA:
DETECTION_SCORE_THRESH: 0.9
TRAIN_PREDICT_BOX_LISTS: [
"ava_train_v2.2.csv",
"person_box_67091280_iou90/ava_detection_train_boxes_and_labels_include_negative_v2.2.csv",
]
TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"]
RESNET:
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 50
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]]
SPATIAL_DILATIONS: [[1], [1], [1], [2]]
SPATIAL_STRIDES: [[1], [2], [2], [1]]
NONLOCAL:
LOCATION: [[[]], [[]], [[]], [[]]]
GROUP: [[1], [1], [1], [1]]
INSTANTIATION: softmax
BN:
USE_PRECISE_STATS: False
NUM_BATCHES_PRECISE: 200
SOLVER:
BASE_LR: 0.1
LR_POLICY: steps_with_relative_lrs
STEPS: [0, 10, 15, 20]
LRS: [1, 0.1, 0.01, 0.001]
MAX_EPOCH: 20
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-7
WARMUP_EPOCHS: 5.0
WARMUP_START_LR: 0.000125
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 80
ARCH: slow
MODEL_NAME: ResNet
LOSS_FUNC: bce
DROPOUT_RATE: 0.5
HEAD_ACT: sigmoid
TEST:
ENABLE: True
DATASET: ava
BATCH_SIZE: 8
DATA_LOADER:
NUM_WORKERS: 2
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/AVA/c2/SLOWFAST_32x2_R101_50_50.yaml
================================================
TRAIN:
ENABLE: False
DATASET: ava
BATCH_SIZE: 16
EVAL_PERIOD: 1
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
# CHECKPOINT_FILE_PATH: path to pretrain model
CHECKPOINT_TYPE: pytorch
DATA:
NUM_FRAMES: 32
SAMPLING_RATE: 2
TRAIN_JITTER_SCALES: [256, 320]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3, 3]
DETECTION:
ENABLE: True
ALIGNED: False
AVA:
BGR: False
DETECTION_SCORE_THRESH: 0.8
TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"]
SLOWFAST:
ALPHA: 4
BETA_INV: 8
FUSION_CONV_CHANNEL_RATIO: 2
FUSION_KERNEL_SZ: 5
RESNET:
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 101
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]]
SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]]
SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]]
NONLOCAL:
LOCATION: [[[], []], [[], []], [[6, 13, 20], []], [[], []]]
GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]]
INSTANTIATION: dot_product
POOL: [[[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]]]
BN:
USE_PRECISE_STATS: False
NUM_BATCHES_PRECISE: 200
SOLVER:
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-7
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 80
ARCH: slowfast
MODEL_NAME: SlowFast
LOSS_FUNC: bce
DROPOUT_RATE: 0.5
HEAD_ACT: sigmoid
TEST:
ENABLE: True
DATASET: ava
BATCH_SIZE: 8
DATA_LOADER:
NUM_WORKERS: 2
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/AVA/c2/SLOWFAST_32x2_R101_50_50_v2.1.yaml
================================================
TRAIN:
ENABLE: False
DATASET: ava
BATCH_SIZE: 16
EVAL_PERIOD: 1
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
# CHECKPOINT_FILE_PATH: path to pretrain model
CHECKPOINT_TYPE: caffe2
DATA:
NUM_FRAMES: 32
SAMPLING_RATE: 2
TRAIN_JITTER_SCALES: [256, 320]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3, 3]
DETECTION:
ENABLE: True
ALIGNED: False
AVA:
BGR: False
DETECTION_SCORE_THRESH: 0.8
TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"]
TRAIN_GT_BOX_LISTS: ["ava_train_v2.1.csv"]
LABEL_MAP_FILE: ava_action_list_v2.1_for_activitynet_2018.pbtxt
EXCLUSION_FILE: ava_val_excluded_timestamps_v2.1.csv
GROUNDTRUTH_FILE: ava_val_v2.1.csv
SLOWFAST:
ALPHA: 4
BETA_INV: 8
FUSION_CONV_CHANNEL_RATIO: 2
FUSION_KERNEL_SZ: 5
RESNET:
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 101
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]]
SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]]
SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]]
NONLOCAL:
LOCATION: [[[], []], [[], []], [[6, 13, 20], []], [[], []]]
GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]]
INSTANTIATION: dot_product
POOL: [[[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]]]
BN:
USE_PRECISE_STATS: False
NUM_BATCHES_PRECISE: 200
SOLVER:
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-7
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 80
ARCH: slowfast
MODEL_NAME: SlowFast
LOSS_FUNC: bce
DROPOUT_RATE: 0.5
HEAD_ACT: sigmoid
TEST:
ENABLE: True
DATASET: ava
BATCH_SIZE: 8
DATA_LOADER:
NUM_WORKERS: 2
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/AVA/c2/SLOWFAST_32x2_R50.yaml
================================================
TRAIN:
ENABLE: False
DATASET: ava
BATCH_SIZE: 16
EVAL_PERIOD: 1
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
# CHECKPOINT_FILE_PATH: path to pretrain model
CHECKPOINT_TYPE: caffe2
DATA:
NUM_FRAMES: 32
SAMPLING_RATE: 2
TRAIN_JITTER_SCALES: [256, 320]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3, 3]
DETECTION:
ENABLE: True
ALIGNED: False
AVA:
BGR: False
DETECTION_SCORE_THRESH: 0.8
TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"]
SLOWFAST:
ALPHA: 4
BETA_INV: 8
FUSION_CONV_CHANNEL_RATIO: 2
FUSION_KERNEL_SZ: 7
RESNET:
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 50
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]]
SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]]
SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]]
NONLOCAL:
LOCATION: [[[], []], [[], []], [[], []], [[], []]]
GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]]
INSTANTIATION: dot_product
POOL: [[[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]]]
BN:
USE_PRECISE_STATS: False
NUM_BATCHES_PRECISE: 200
SOLVER:
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-7
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 80
ARCH: slowfast
MODEL_NAME: SlowFast
LOSS_FUNC: bce
DROPOUT_RATE: 0.5
HEAD_ACT: sigmoid
TEST:
ENABLE: True
DATASET: ava
BATCH_SIZE: 8
DATA_LOADER:
NUM_WORKERS: 2
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/AVA/c2/SLOWFAST_64x2_R101_50_50.yaml
================================================
TRAIN:
ENABLE: False
DATASET: ava
BATCH_SIZE: 16
EVAL_PERIOD: 1
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
# CHECKPOINT_FILE_PATH: path to pretrain model
CHECKPOINT_TYPE: caffe2
DATA:
NUM_FRAMES: 64
SAMPLING_RATE: 2
TRAIN_JITTER_SCALES: [256, 320]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3, 3]
DETECTION:
ENABLE: True
ALIGNED: False
AVA:
BGR: False
DETECTION_SCORE_THRESH: 0.8
TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"]
SLOWFAST:
ALPHA: 4
BETA_INV: 8
FUSION_CONV_CHANNEL_RATIO: 2
FUSION_KERNEL_SZ: 5
RESNET:
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 101
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]]
SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]]
SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]]
NONLOCAL:
LOCATION: [[[], []], [[], []], [[6, 13, 20], []], [[], []]]
GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]]
INSTANTIATION: dot_product
POOL: [[[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]]]
BN:
USE_PRECISE_STATS: False
NUM_BATCHES_PRECISE: 200
SOLVER:
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-7
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 80
ARCH: slowfast
MODEL_NAME: SlowFast
LOSS_FUNC: bce
DROPOUT_RATE: 0.5
HEAD_ACT: sigmoid
TEST:
ENABLE: True
DATASET: ava
BATCH_SIZE: 8
DATA_LOADER:
NUM_WORKERS: 0
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/AVA/c2/SLOW_8x8_R50.yaml
================================================
TRAIN:
ENABLE: False
DATASET: ava
BATCH_SIZE: 64
EVAL_PERIOD: 1
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
# CHECKPOINT_FILE_PATH: path to pretrain model
CHECKPOINT_TYPE: caffe2
DATA:
NUM_FRAMES: 4
SAMPLING_RATE: 16
TRAIN_JITTER_SCALES: [256, 320]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3]
DETECTION:
ENABLE: True
ALIGNED: False
AVA:
BGR: False
DETECTION_SCORE_THRESH: 0.9
TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou75/ava_detection_val_boxes_and_labels.csv"]
RESNET:
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 50
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]]
SPATIAL_DILATIONS: [[1], [1], [1], [2]]
SPATIAL_STRIDES: [[1], [2], [2], [1]]
NONLOCAL:
LOCATION: [[[]], [[]], [[]], [[]]]
GROUP: [[1], [1], [1], [1]]
INSTANTIATION: softmax
BN:
USE_PRECISE_STATS: False
NUM_BATCHES_PRECISE: 200
SOLVER:
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-7
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 80
ARCH: slow
MODEL_NAME: ResNet
LOSS_FUNC: bce
DROPOUT_RATE: 0.5
HEAD_ACT: sigmoid
TEST:
ENABLE: True
DATASET: ava
BATCH_SIZE: 8
DATA_LOADER:
NUM_WORKERS: 2
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/Charades/SLOWFAST_16x8_R50.yaml
================================================
TRAIN:
ENABLE: True
DATASET: charades
BATCH_SIZE: 16
EVAL_PERIOD: 6
CHECKPOINT_PERIOD: 6
AUTO_RESUME: True
CHECKPOINT_FILE_PATH: SLOWFAST_8x8_R50.pkl # please download from the model zoo.
CHECKPOINT_TYPE: caffe2
DATA:
NUM_FRAMES: 64
SAMPLING_RATE: 2
TRAIN_JITTER_SCALES: [256, 340]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3, 3]
MULTI_LABEL: True
INV_UNIFORM_SAMPLE: True
ENSEMBLE_METHOD: max
REVERSE_INPUT_CHANNEL: True
SLOWFAST:
ALPHA: 4
BETA_INV: 8
FUSION_CONV_CHANNEL_RATIO: 2
FUSION_KERNEL_SZ: 7
RESNET:
SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]]
SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]]
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 50
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]]
NONLOCAL:
LOCATION: [[[], []], [[], []], [[], []], [[], []]]
GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]]
INSTANTIATION: dot_product
BN:
USE_PRECISE_STATS: True
NUM_BATCHES_PRECISE: 200
NORM_TYPE: sync_batchnorm
NUM_SYNC_DEVICES: 4
SOLVER:
BASE_LR: 0.0375
LR_POLICY: steps_with_relative_lrs
LRS: [1, 0.1, 0.01, 0.001, 0.0001, 0.00001]
STEPS: [0, 41, 49]
MAX_EPOCH: 57
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-4
WARMUP_EPOCHS: 4.0
WARMUP_START_LR: 0.0001
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 157
ARCH: slowfast
LOSS_FUNC: bce_logit
HEAD_ACT: sigmoid
DROPOUT_RATE: 0.5
TEST:
ENABLE: True
DATASET: charades
BATCH_SIZE: 16
NUM_ENSEMBLE_VIEWS: 10
NUM_SPATIAL_CROPS: 3
DATA_LOADER:
NUM_WORKERS: 8
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
LOG_MODEL_INFO: False
================================================
FILE: configs/Charades/SLOWFAST_16x8_R50_multigrid.yaml
================================================
MULTIGRID:
SHORT_CYCLE: True
LONG_CYCLE: True
TRAIN:
ENABLE: True
DATASET: charades
BATCH_SIZE: 16
EVAL_PERIOD: 6
CHECKPOINT_PERIOD: 6
AUTO_RESUME: True
CHECKPOINT_FILE_PATH: SLOWFAST_8x8_R50.pkl # please download from the model zoo.
CHECKPOINT_TYPE: caffe2
DATA:
NUM_FRAMES: 64
SAMPLING_RATE: 2
TRAIN_JITTER_SCALES: [256, 340]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3, 3]
MULTI_LABEL: True
INV_UNIFORM_SAMPLE: True
ENSEMBLE_METHOD: max
REVERSE_INPUT_CHANNEL: True
SLOWFAST:
ALPHA: 4
BETA_INV: 8
FUSION_CONV_CHANNEL_RATIO: 2
FUSION_KERNEL_SZ: 7
RESNET:
SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]]
SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]]
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 50
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]]
NONLOCAL:
LOCATION: [[[], []], [[], []], [[], []], [[], []]]
GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]]
INSTANTIATION: dot_product
BN:
USE_PRECISE_STATS: True
NUM_BATCHES_PRECISE: 200
NORM_TYPE: sync_batchnorm
NUM_SYNC_DEVICES: 4
SOLVER:
BASE_LR: 0.0375
LR_POLICY: steps_with_relative_lrs
LRS: [1, 0.1, 0.01, 0.001, 0.0001, 0.00001]
STEPS: [0, 41, 49]
MAX_EPOCH: 57
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-4
WARMUP_EPOCHS: 4.0
WARMUP_START_LR: 0.0001
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 157
ARCH: slowfast
LOSS_FUNC: bce_logit
HEAD_ACT: sigmoid
DROPOUT_RATE: 0.5
TEST:
ENABLE: True
DATASET: charades
BATCH_SIZE: 16
NUM_ENSEMBLE_VIEWS: 10
NUM_SPATIAL_CROPS: 3
DATA_LOADER:
NUM_WORKERS: 8
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
LOG_MODEL_INFO: False
================================================
FILE: configs/Charades/pytorchvideo/SLOWFAST_8x8_R50.yaml
================================================
TRAIN:
ENABLE: True
DATASET: ptvcharades
BATCH_SIZE: 64
EVAL_PERIOD: 6
CHECKPOINT_PERIOD: 6
AUTO_RESUME: True
# CHECKPOINT_FILE_PATH: # please download from the model zoo.
CHECKPOINT_EPOCH_RESET: True
DATA:
NUM_FRAMES: 32
SAMPLING_RATE: 2
TRAIN_JITTER_SCALES: [256, 320]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3, 3]
MULTI_LABEL: True
INV_UNIFORM_SAMPLE: True
ENSEMBLE_METHOD: max
REVERSE_INPUT_CHANNEL: True
SLOWFAST:
ALPHA: 4
BETA_INV: 8
FUSION_CONV_CHANNEL_RATIO: 2
FUSION_KERNEL_SZ: 7
RESNET:
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 50
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]]
SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]]
SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]]
NONLOCAL:
LOCATION: [[[], []], [[], []], [[], []], [[], []]]
GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]]
INSTANTIATION: dot_product
POOL: [[[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]]]
BN:
USE_PRECISE_STATS: False
NUM_BATCHES_PRECISE: 200
SOLVER:
BASE_LR: 0.25
BASE_LR_SCALE_NUM_SHARDS: True
LR_POLICY: cosine
COSINE_END_LR: 1e-6
MAX_EPOCH: 60
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-4
WARMUP_EPOCHS: 5.0
WARMUP_START_LR: 0.0001
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 157
ARCH: slowfast
MODEL_NAME: PTVSlowFast
LOSS_FUNC: bce_logit
HEAD_ACT: sigmoid
DROPOUT_RATE: 0.5
TEST:
ENABLE: True
DATASET: ptvcharades
BATCH_SIZE: 64
NUM_ENSEMBLE_VIEWS: 10
NUM_SPATIAL_CROPS: 3
DATA_LOADER:
NUM_WORKERS: 1
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/Charades/pytorchvideo/SLOW_8x8_R50.yaml
================================================
TRAIN:
ENABLE: True
DATASET: ptvcharades
BATCH_SIZE: 64
EVAL_PERIOD: 6
CHECKPOINT_PERIOD: 6
AUTO_RESUME: True
# CHECKPOINT_FILE_PATH: # please download from the model zoo.
CHECKPOINT_EPOCH_RESET: True
DATA:
NUM_FRAMES: 8
SAMPLING_RATE: 8
TRAIN_JITTER_SCALES: [256, 340]
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3]
MULTI_LABEL: True
INV_UNIFORM_SAMPLE: True
ENSEMBLE_METHOD: max
REVERSE_INPUT_CHANNEL: True
RESNET:
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 50
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]]
NONLOCAL:
LOCATION: [[[]], [[]], [[]], [[]]]
GROUP: [[1], [1], [1], [1]]
INSTANTIATION: softmax
BN:
USE_PRECISE_STATS: False
NUM_BATCHES_PRECISE: 200
SOLVER:
BASE_LR: 0.25
BASE_LR_SCALE_NUM_SHARDS: True
LR_POLICY: cosine
COSINE_END_LR: 1e-6
MAX_EPOCH: 60
MOMENTUM: 0.9
WEIGHT_DECAY: 1e-4
WARMUP_EPOCHS: 5.0
WARMUP_START_LR: 0.0001
OPTIMIZING_METHOD: sgd
MODEL:
NUM_CLASSES: 157
ARCH: slow
MODEL_NAME: PTVResNet
LOSS_FUNC: bce_logit
HEAD_ACT: sigmoid
DROPOUT_RATE: 0.5
TEST:
ENABLE: True
DATASET: ptvcharades
BATCH_SIZE: 64
NUM_ENSEMBLE_VIEWS: 10
NUM_SPATIAL_CROPS: 3
DATA_LOADER:
NUM_WORKERS: 1
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/ImageNet/MVIT_B_16_CONV.yaml
================================================
TRAIN:
ENABLE: True
DATASET: imagenet
BATCH_SIZE: 256
EVAL_PERIOD: 10
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
DATA:
# PATH_TO_DATA_DIR: path-to-imagenet-dir
MEAN: [0.485, 0.456, 0.406]
STD: [0.229, 0.224, 0.225]
NUM_FRAMES: 1
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 224
INPUT_CHANNEL_NUM: [3]
MVIT:
PATCH_2D: True
ZERO_DECAY_POS_CLS: False
MODE: "conv"
CLS_EMBED_ON: True
PATCH_KERNEL: [7, 7]
PATCH_STRIDE: [4, 4]
PATCH_PADDING: [3, 3]
EMBED_DIM: 96
NUM_HEADS: 1
MLP_RATIO: 4.0
QKV_BIAS: True
DROPPATH_RATE: 0.1
DEPTH: 16
NORM: "layernorm"
DIM_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]]
HEAD_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]]
POOL_KVQ_KERNEL: [1, 3, 3]
POOL_KV_STRIDE_ADAPTIVE: [1, 4, 4]
POOL_Q_STRIDE: [[1, 1, 2, 2], [3, 1, 2, 2], [14, 1, 2, 2]]
AUG:
ENABLE: True
COLOR_JITTER: 0.4
AA_TYPE: rand-m9-n6-mstd0.5-inc1
INTERPOLATION: bicubic
RE_PROB: 0.25
RE_MODE: pixel
RE_COUNT: 1
RE_SPLIT: False
MIXUP:
ENABLE: True
ALPHA: 0.8
CUTMIX_ALPHA: 1.0
PROB: 1.0
SWITCH_PROB: 0.5
LABEL_SMOOTH_VALUE: 0.1
SOLVER:
BASE_LR_SCALE_NUM_SHARDS: True
BASE_LR: 0.00025
LR_POLICY: cosine
MAX_EPOCH: 300
MOMENTUM: 0.9
WEIGHT_DECAY: 0.05
WARMUP_EPOCHS: 70.0
WARMUP_START_LR: 1e-8
OPTIMIZING_METHOD: adamw
COSINE_AFTER_WARMUP: True
COSINE_END_LR: 1e-6
ZERO_WD_1D_PARAM: True
CLIP_GRAD_L2NORM: 1.0
MODEL:
NUM_CLASSES: 1000
ARCH: mvit
MODEL_NAME: MViT
LOSS_FUNC: soft_cross_entropy
DROPOUT_RATE: 0.0
TEST:
ENABLE: False
DATASET: imagenet
BATCH_SIZE: 256
DATA_LOADER:
NUM_WORKERS: 8
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/ImageNet/MVITv2_B.yaml
================================================
TRAIN:
ENABLE: True
DATASET: imagenet
BATCH_SIZE: 256
EVAL_PERIOD: 10
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
DATA:
# PATH_TO_DATA_DIR: path-to-imagenet-dir
MEAN: [0.485, 0.456, 0.406]
STD: [0.229, 0.224, 0.225]
NUM_FRAMES: 1
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 224
INPUT_CHANNEL_NUM: [3]
MVIT:
PATCH_2D: True
ZERO_DECAY_POS_CLS: False
MODE: "conv"
CLS_EMBED_ON: False
PATCH_KERNEL: [7, 7]
PATCH_STRIDE: [4, 4]
PATCH_PADDING: [3, 3]
EMBED_DIM: 96
NUM_HEADS: 1
MLP_RATIO: 4.0
QKV_BIAS: True
DROPPATH_RATE: 0.3
DEPTH: 24
NORM: "layernorm"
DIM_MUL: [[2, 2.0], [5, 2.0], [21, 2.0]]
HEAD_MUL: [[2, 2.0], [5, 2.0], [21, 2.0]]
POOL_KVQ_KERNEL: [1, 3, 3]
POOL_KV_STRIDE_ADAPTIVE: [1, 4, 4]
POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 1, 1], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 2, 2], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], [15, 1, 1, 1], [16, 1, 1, 1], [17, 1, 1, 1], [18, 1, 1, 1], [19, 1, 1, 1], [20, 1, 1, 1], [21, 1, 2, 2], [22, 1, 1, 1], [23, 1, 1, 1]]
RESIDUAL_POOLING: True
USE_ABS_POS: False
REL_POS_SPATIAL: True
DIM_MUL_IN_ATT: True
AUG:
ENABLE: True
COLOR_JITTER: 0.4
AA_TYPE: rand-m9-n6-mstd0.5-inc1
INTERPOLATION: bicubic
RE_PROB: 0.25
RE_MODE: pixel
RE_COUNT: 1
RE_SPLIT: False
MIXUP:
ENABLE: True
ALPHA: 0.8
CUTMIX_ALPHA: 1.0
PROB: 1.0
SWITCH_PROB: 0.5
LABEL_SMOOTH_VALUE: 0.1
SOLVER:
BASE_LR_SCALE_NUM_SHARDS: True
BASE_LR: 0.00025
LR_POLICY: cosine
MAX_EPOCH: 300
MOMENTUM: 0.9
WEIGHT_DECAY: 0.05
WARMUP_EPOCHS: 70.0
WARMUP_START_LR: 1e-8
OPTIMIZING_METHOD: adamw
COSINE_AFTER_WARMUP: True
COSINE_END_LR: 1e-6
ZERO_WD_1D_PARAM: True
CLIP_GRAD_L2NORM: 1.0
MODEL:
NUM_CLASSES: 1000
ARCH: mvit
MODEL_NAME: MViT
LOSS_FUNC: soft_cross_entropy
DROPOUT_RATE: 0.0
TEST:
ENABLE: True
DATASET: imagenet
BATCH_SIZE: 256
NUM_ENSEMBLE_VIEWS: 1
NUM_SPATIAL_CROPS: 1
DATA_LOADER:
NUM_WORKERS: 8
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/ImageNet/MVITv2_S.yaml
================================================
TRAIN:
ENABLE: True
DATASET: imagenet
BATCH_SIZE: 256
EVAL_PERIOD: 10
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
DATA:
# PATH_TO_DATA_DIR: path-to-imagenet-dir
MEAN: [0.485, 0.456, 0.406]
STD: [0.229, 0.224, 0.225]
NUM_FRAMES: 1
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 224
INPUT_CHANNEL_NUM: [3]
MVIT:
PATCH_2D: True
ZERO_DECAY_POS_CLS: False
MODE: "conv"
CLS_EMBED_ON: False
PATCH_KERNEL: [7, 7]
PATCH_STRIDE: [4, 4]
PATCH_PADDING: [3, 3]
EMBED_DIM: 96
NUM_HEADS: 1
MLP_RATIO: 4.0
QKV_BIAS: True
DROPPATH_RATE: 0.1
DEPTH: 16
NORM: "layernorm"
DIM_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]]
HEAD_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]]
POOL_KVQ_KERNEL: [1, 3, 3]
POOL_KV_STRIDE: [[0, 1, 4, 4], [1, 1, 2, 2], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], [15, 1, 1, 1]]
POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 2, 2], [2, 1, 1, 1], [3, 1, 2, 2], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 2, 2], [15, 1, 1, 1]]
RESIDUAL_POOLING: True
USE_ABS_POS: False
REL_POS_SPATIAL: True
DIM_MUL_IN_ATT: True
AUG:
ENABLE: True
COLOR_JITTER: 0.4
AA_TYPE: rand-m9-n6-mstd0.5-inc1
INTERPOLATION: bicubic
RE_PROB: 0.25
RE_MODE: pixel
RE_COUNT: 1
RE_SPLIT: False
MIXUP:
ENABLE: True
ALPHA: 0.8
CUTMIX_ALPHA: 1.0
PROB: 1.0
SWITCH_PROB: 0.5
LABEL_SMOOTH_VALUE: 0.1
SOLVER:
BASE_LR_SCALE_NUM_SHARDS: True
BASE_LR: 0.00025
LR_POLICY: cosine
MAX_EPOCH: 300
MOMENTUM: 0.9
WEIGHT_DECAY: 0.05
WARMUP_EPOCHS: 70.0
WARMUP_START_LR: 1e-8
OPTIMIZING_METHOD: adamw
COSINE_AFTER_WARMUP: True
COSINE_END_LR: 1e-6
ZERO_WD_1D_PARAM: True
CLIP_GRAD_L2NORM: 1.0
MODEL:
NUM_CLASSES: 1000
ARCH: mvit
MODEL_NAME: MViT
LOSS_FUNC: soft_cross_entropy
DROPOUT_RATE: 0.0
TEST:
ENABLE: True
DATASET: imagenet
BATCH_SIZE: 256
NUM_ENSEMBLE_VIEWS: 1
NUM_SPATIAL_CROPS: 1
DATA_LOADER:
NUM_WORKERS: 8
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/ImageNet/MVITv2_T.yaml
================================================
TRAIN:
ENABLE: True
DATASET: imagenet
BATCH_SIZE: 256
EVAL_PERIOD: 10
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
DATA:
# PATH_TO_DATA_DIR: path-to-imagenet-dir
MEAN: [0.485, 0.456, 0.406]
STD: [0.229, 0.224, 0.225]
NUM_FRAMES: 1
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 224
INPUT_CHANNEL_NUM: [3]
MVIT:
PATCH_2D: True
ZERO_DECAY_POS_CLS: False
MODE: "conv"
CLS_EMBED_ON: False
PATCH_KERNEL: [7, 7]
PATCH_STRIDE: [4, 4]
PATCH_PADDING: [3, 3]
EMBED_DIM: 96
NUM_HEADS: 1
MLP_RATIO: 4.0
QKV_BIAS: True
DROPPATH_RATE: 0.1
DEPTH: 10
NORM: "layernorm"
DIM_MUL: [[1, 2.0], [3, 2.0], [8, 2.0]]
HEAD_MUL: [[1, 2.0], [3, 2.0], [8, 2.0]]
POOL_KVQ_KERNEL: [1, 3, 3]
POOL_KV_STRIDE: [[0, 1, 4, 4], [1, 1, 2, 2], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1]]
POOL_Q_STRIDE: [[0, 1, 1, 1], [1, 1, 2, 2], [2, 1, 1, 1], [3, 1, 2, 2], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 2, 2], [9, 1, 1, 1]]
RESIDUAL_POOLING: True
USE_ABS_POS: False
REL_POS_SPATIAL: True
DIM_MUL_IN_ATT: True
AUG:
ENABLE: True
COLOR_JITTER: 0.4
AA_TYPE: rand-m9-n6-mstd0.5-inc1
INTERPOLATION: bicubic
RE_PROB: 0.25
RE_MODE: pixel
RE_COUNT: 1
RE_SPLIT: False
MIXUP:
ENABLE: True
ALPHA: 0.8
CUTMIX_ALPHA: 1.0
PROB: 1.0
SWITCH_PROB: 0.5
LABEL_SMOOTH_VALUE: 0.1
SOLVER:
BASE_LR_SCALE_NUM_SHARDS: True
BASE_LR: 0.00025
LR_POLICY: cosine
MAX_EPOCH: 300
MOMENTUM: 0.9
WEIGHT_DECAY: 0.05
WARMUP_EPOCHS: 70.0
WARMUP_START_LR: 1e-8
OPTIMIZING_METHOD: adamw
COSINE_AFTER_WARMUP: True
COSINE_END_LR: 1e-6
ZERO_WD_1D_PARAM: True
CLIP_GRAD_L2NORM: 1.0
MODEL:
NUM_CLASSES: 1000
ARCH: mvit
MODEL_NAME: MViT
LOSS_FUNC: soft_cross_entropy
DROPOUT_RATE: 0.0
TEST:
ENABLE: True
DATASET: imagenet
BATCH_SIZE: 256
NUM_ENSEMBLE_VIEWS: 1
NUM_SPATIAL_CROPS: 1
DATA_LOADER:
NUM_WORKERS: 8
PIN_MEMORY: True
NUM_GPUS: 8
NUM_SHARDS: 1
RNG_SEED: 0
OUTPUT_DIR: .
================================================
FILE: configs/ImageNet/RES_R50.yaml
================================================
TRAIN:
ENABLE: True
DATASET: imagenet
BATCH_SIZE: 256
EVAL_PERIOD: 10
CHECKPOINT_PERIOD: 1
AUTO_RESUME: True
DATA:
# PATH_TO_DATA_DIR: path-to-imagenet-dir
MEAN: [0.485, 0.456, 0.406]
STD: [0.229, 0.224, 0.225]
NUM_FRAMES: 1
TRAIN_CROP_SIZE: 224
TEST_CROP_SIZE: 256
INPUT_CHANNEL_NUM: [3]
TRAIN_JITTER_SCALES_RELATIVE: [0.08, 1.0]
TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333333333333333]
RESNET:
ZERO_INIT_FINAL_BN: True
WIDTH_PER_GROUP: 64
NUM_GROUPS: 1
DEPTH: 50
TRANS_FUNC: bottleneck_transform
STRIDE_1X1: False
NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]]
NONLOCAL:
LOCATION: [[[]], [[]], [[]], [[]]]
GROUP: [[1], [1], [1], [1]]
INSTANTIATION: softmax
BN:
USE_PRECISE_STATS: True
NUM_BATCHES_PRECISE: 200
SOLVER:
BASE_LR: 0.1
L
gitextract_bls1cq2k/
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── GETTING_STARTED.md
├── INSTALL.md
├── LICENSE
├── MODEL_ZOO.md
├── README.md
├── VISUALIZATION_TOOLS.md
├── ava_evaluation/
│ ├── README.md
│ ├── ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt
│ ├── label_map_util.py
│ ├── metrics.py
│ ├── np_box_list.py
│ ├── np_box_list_ops.py
│ ├── np_box_mask_list.py
│ ├── np_box_mask_list_ops.py
│ ├── np_box_ops.py
│ ├── np_mask_ops.py
│ ├── object_detection_evaluation.py
│ ├── per_image_evaluation.py
│ └── standard_fields.py
├── configs/
│ ├── AVA/
│ │ ├── SLOWFAST_32x2_R50_SHORT.yaml
│ │ ├── SLOW_8x8_R50_SHORT.yaml
│ │ └── c2/
│ │ ├── SLOWFAST_32x2_R101_50_50.yaml
│ │ ├── SLOWFAST_32x2_R101_50_50_v2.1.yaml
│ │ ├── SLOWFAST_32x2_R50.yaml
│ │ ├── SLOWFAST_64x2_R101_50_50.yaml
│ │ └── SLOW_8x8_R50.yaml
│ ├── Charades/
│ │ ├── SLOWFAST_16x8_R50.yaml
│ │ ├── SLOWFAST_16x8_R50_multigrid.yaml
│ │ └── pytorchvideo/
│ │ ├── SLOWFAST_8x8_R50.yaml
│ │ └── SLOW_8x8_R50.yaml
│ ├── ImageNet/
│ │ ├── MVIT_B_16_CONV.yaml
│ │ ├── MVITv2_B.yaml
│ │ ├── MVITv2_S.yaml
│ │ ├── MVITv2_T.yaml
│ │ ├── RES_R50.yaml
│ │ ├── REV_MVIT_B_16_CONV.yaml
│ │ ├── REV_VIT_B.yaml
│ │ └── REV_VIT_S.yaml
│ ├── Kinetics/
│ │ ├── C2D_8x8_R50.yaml
│ │ ├── C2D_8x8_R50_IN1K.yaml
│ │ ├── C2D_NLN_8x8_R50.yaml
│ │ ├── C2D_NLN_8x8_R50_IN1K.yaml
│ │ ├── I3D_8x8_R101.yaml
│ │ ├── I3D_8x8_R50.yaml
│ │ ├── I3D_8x8_R50_IN1K.yaml
│ │ ├── I3D_NLN_8x8_R101.yaml
│ │ ├── I3D_NLN_8x8_R50.yaml
│ │ ├── I3D_NLN_8x8_R50_IN1K.yaml
│ │ ├── MVIT_B_16x4_CONV.yaml
│ │ ├── MVIT_B_32x3_CONV.yaml
│ │ ├── MVITv2_B_32x3.yaml
│ │ ├── MVITv2_L_40x3_test.yaml
│ │ ├── MVITv2_S_16x4.yaml
│ │ ├── REV_MVIT_B_16x4_CONV.yaml
│ │ ├── SLOWFAST_4x16_R50.yaml
│ │ ├── SLOWFAST_8x8_R50.yaml
│ │ ├── SLOWFAST_8x8_R50_stepwise.yaml
│ │ ├── SLOWFAST_8x8_R50_stepwise_multigrid.yaml
│ │ ├── SLOWFAST_NLN_4x16_R50.yaml
│ │ ├── SLOWFAST_NLN_8x8_R50.yaml
│ │ ├── SLOW_4x16_R50.yaml
│ │ ├── SLOW_8x8_R50.yaml
│ │ ├── SLOW_NLN_4x16_R50.yaml
│ │ ├── SLOW_NLN_8x8_R50.yaml
│ │ ├── X3D_L.yaml
│ │ ├── X3D_M.yaml
│ │ ├── X3D_S.yaml
│ │ ├── X3D_XS.yaml
│ │ ├── c2/
│ │ │ ├── C2D_NOPOOL_8x8_R50.yaml
│ │ │ ├── I3D_8x8_R50.yaml
│ │ │ ├── I3D_NLN_8x8_R50.yaml
│ │ │ ├── SLOWFAST_16x8_R101_50_50.yaml
│ │ │ ├── SLOWFAST_4x16_R50.yaml
│ │ │ ├── SLOWFAST_8x8_R101_101_101.yaml
│ │ │ ├── SLOWFAST_8x8_R101_50_101.yaml
│ │ │ ├── SLOWFAST_8x8_R101_50_50.yaml
│ │ │ ├── SLOWFAST_8x8_R50.yaml
│ │ │ ├── SLOWFAST_NLN_16x8_R101_50_50.yaml
│ │ │ ├── SLOW_4x16_R50.yaml
│ │ │ └── SLOW_8x8_R50.yaml
│ │ └── pytorchvideo/
│ │ ├── C2D_8x8_R50.yaml
│ │ ├── CSN_32x2_R101.yaml
│ │ ├── I3D_8x8_R50.yaml
│ │ ├── MVIT_B_16x4_CONV.yaml
│ │ ├── R2PLUS1D_16x4_R50.yaml
│ │ ├── SLOWFAST_16x8_R101_50_50.yaml
│ │ ├── SLOWFAST_4x16_R50.yaml
│ │ ├── SLOWFAST_8x8_R101.yaml
│ │ ├── SLOWFAST_8x8_R50.yaml
│ │ ├── SLOW_4x16_R50.yaml
│ │ ├── SLOW_8x8_R50.yaml
│ │ ├── X3D_L.yaml
│ │ ├── X3D_M.yaml
│ │ ├── X3D_S.yaml
│ │ └── X3D_XS.yaml
│ ├── SSv2/
│ │ ├── MVITv2_B_32x3.yaml
│ │ ├── MVITv2_L_40x3.yaml
│ │ ├── MVITv2_S_16x4.yaml
│ │ ├── SLOWFAST_16x8_R50.yaml
│ │ ├── SLOWFAST_16x8_R50_multigrid.yaml
│ │ └── pytorchvideo/
│ │ ├── SLOWFAST_8x8_R50.yaml
│ │ └── SLOW_8x8_R50.yaml
│ ├── contrastive_ssl/
│ │ ├── BYOL_SlowR50_8x8.yaml
│ │ ├── MoCo_SlowR50_8x8.yaml
│ │ ├── SimCLR_SlowR50_8x8.yaml
│ │ ├── SwAV_Slow_R50_8x8.yaml
│ │ ├── finetune_SSv2_Slow_R50_syn0.yaml
│ │ ├── finetune_SSv2_Slow_R50_syn8.yaml
│ │ ├── finetune_ava_Slow_R50_syn0.yaml
│ │ ├── finetune_ava_Slow_R50_syn8.yaml
│ │ ├── finetune_ucf_Slow_R50_syn0.yaml
│ │ ├── finetune_ucf_Slow_R50_syn8.yaml
│ │ ├── linear_k400_Slow_8x8_R50_syn0.yaml
│ │ └── linear_k400_Slow_8x8_R50_syn8.yaml
│ └── masked_ssl/
│ ├── in1k_VIT_B_MaskFeat_FT.yaml
│ ├── in1k_VIT_B_MaskFeat_PT.yaml
│ ├── in1k_VIT_L_MaskFeat_FT.yaml
│ ├── in1k_VIT_L_MaskFeat_PT.yaml
│ ├── k400_MVITv2_L_16x4_FT.yaml
│ ├── k400_MVITv2_L_16x4_MaskFeat_PT.yaml
│ ├── k400_MVITv2_S_16x4_FT.yaml
│ ├── k400_MVITv2_S_16x4_MaskFeat_PT.yaml
│ ├── k400_VIT_B_16x4_FT.yaml
│ ├── k400_VIT_B_16x4_MAE_PT.yaml
│ ├── k400_VIT_H_16x4_FT.yaml
│ ├── k400_VIT_H_16x4_MAE_PT.yaml
│ ├── k400_VIT_L_16x4_FT.yaml
│ └── k400_VIT_L_16x4_MAE_PT.yaml
├── demo/
│ ├── AVA/
│ │ └── SLOWFAST_32x2_R101_50_50.yaml
│ └── Kinetics/
│ └── SLOWFAST_8x8_R50.yaml
├── linter.sh
├── projects/
│ ├── contrastive_ssl/
│ │ └── README.md
│ ├── mae/
│ │ └── README.md
│ ├── maskfeat/
│ │ └── README.md
│ ├── multigrid/
│ │ └── README.md
│ ├── mvit/
│ │ └── README.md
│ ├── mvitv2/
│ │ └── README.md
│ ├── pytorchvideo/
│ │ └── README.md
│ ├── rev/
│ │ └── README.md
│ └── x3d/
│ └── README.md
├── setup.cfg
├── setup.py
├── slowfast/
│ ├── __init__.py
│ ├── config/
│ │ ├── __init__.py
│ │ ├── custom_config.py
│ │ └── defaults.py
│ ├── datasets/
│ │ ├── DATASET.md
│ │ ├── __init__.py
│ │ ├── ava_dataset.py
│ │ ├── ava_helper.py
│ │ ├── build.py
│ │ ├── charades.py
│ │ ├── cv2_transform.py
│ │ ├── decoder.py
│ │ ├── imagenet.py
│ │ ├── kinetics.py
│ │ ├── loader.py
│ │ ├── mixup.py
│ │ ├── multigrid_helper.py
│ │ ├── ptv_datasets.py
│ │ ├── rand_augment.py
│ │ ├── random_erasing.py
│ │ ├── ssv2.py
│ │ ├── transform.py
│ │ ├── utils.py
│ │ └── video_container.py
│ ├── models/
│ │ ├── __init__.py
│ │ ├── attention.py
│ │ ├── batchnorm_helper.py
│ │ ├── build.py
│ │ ├── common.py
│ │ ├── contrastive.py
│ │ ├── custom_video_model_builder.py
│ │ ├── head_helper.py
│ │ ├── losses.py
│ │ ├── masked.py
│ │ ├── nonlocal_helper.py
│ │ ├── operators.py
│ │ ├── optimizer.py
│ │ ├── ptv_model_builder.py
│ │ ├── resnet_helper.py
│ │ ├── reversible_mvit.py
│ │ ├── stem_helper.py
│ │ ├── utils.py
│ │ └── video_model_builder.py
│ ├── utils/
│ │ ├── __init__.py
│ │ ├── ava_eval_helper.py
│ │ ├── benchmark.py
│ │ ├── bn_helper.py
│ │ ├── c2_model_loading.py
│ │ ├── checkpoint.py
│ │ ├── distributed.py
│ │ ├── env.py
│ │ ├── logging.py
│ │ ├── lr_policy.py
│ │ ├── meters.py
│ │ ├── metrics.py
│ │ ├── misc.py
│ │ ├── multigrid.py
│ │ ├── multiprocessing.py
│ │ ├── parser.py
│ │ └── weight_init_helper.py
│ └── visualization/
│ ├── __init__.py
│ ├── async_predictor.py
│ ├── ava_demo_precomputed_boxes.py
│ ├── demo_loader.py
│ ├── gradcam_utils.py
│ ├── prediction_vis.py
│ ├── predictor.py
│ ├── tensorboard_vis.py
│ ├── utils.py
│ └── video_visualizer.py
└── tools/
├── benchmark.py
├── demo_net.py
├── run_net.py
├── test_net.py
├── train_net.py
└── visualization.py
SYMBOL INDEX (891 symbols across 79 files)
FILE: ava_evaluation/label_map_util.py
function _validate_label_map (line 25) | def _validate_label_map(label_map):
function create_category_index (line 39) | def create_category_index(categories):
function get_max_label_map_index (line 58) | def get_max_label_map_index(label_map):
function convert_label_map_to_categories (line 70) | def convert_label_map_to_categories(label_map, max_num_classes, use_disp...
function load_labelmap (line 122) | def load_labelmap(path):
function get_label_map_dict (line 141) | def get_label_map_dict(label_map_path, use_display_name=False):
function create_category_index_from_labelmap (line 161) | def create_category_index_from_labelmap(label_map_path):
function create_class_agnostic_category_index (line 178) | def create_class_agnostic_category_index():
FILE: ava_evaluation/metrics.py
function compute_precision_recall (line 23) | def compute_precision_recall(scores, labels, num_gt):
function compute_average_precision (line 74) | def compute_average_precision(precision, recall):
function compute_cor_loc (line 126) | def compute_cor_loc(num_gt_imgs_per_class, num_images_correctly_detected...
FILE: ava_evaluation/np_box_list.py
class BoxList (line 23) | class BoxList:
method __init__ (line 35) | def __init__(self, data):
method num_boxes (line 58) | def num_boxes(self):
method get_extra_fields (line 62) | def get_extra_fields(self):
method has_field (line 66) | def has_field(self, field):
method add_field (line 69) | def add_field(self, field, field_data):
method get (line 86) | def get(self):
method get_field (line 94) | def get_field(self, field):
method get_coordinates (line 110) | def get_coordinates(self):
method _is_valid_boxes (line 123) | def _is_valid_boxes(self, data):
FILE: ava_evaluation/np_box_list_ops.py
class SortOrder (line 30) | class SortOrder:
function area (line 42) | def area(boxlist):
function intersection (line 55) | def intersection(boxlist1, boxlist2):
function iou (line 68) | def iou(boxlist1, boxlist2):
function ioa (line 81) | def ioa(boxlist1, boxlist2):
function gather (line 98) | def gather(boxlist, indices, fields=None):
function sort_by_field (line 133) | def sort_by_field(boxlist, field, order=SortOrder.DESCEND):
function non_max_suppression (line 164) | def non_max_suppression(
function multi_class_non_max_suppression (line 239) | def multi_class_non_max_suppression(boxlist, score_thresh, iou_thresh, m...
function scale (line 316) | def scale(boxlist, y_scale, x_scale):
function clip_to_window (line 342) | def clip_to_window(boxlist, window):
function prune_non_overlapping_boxes (line 378) | def prune_non_overlapping_boxes(boxlist1, boxlist2, minoverlap=0.0):
function prune_outside_window (line 401) | def prune_outside_window(boxlist, window):
function concatenate (line 439) | def concatenate(boxlists, fields=None):
function filter_scores_greater_than (line 492) | def filter_scores_greater_than(boxlist, thresh):
function change_coordinate_frame (line 527) | def change_coordinate_frame(boxlist, window):
function _copy_extra_fields (line 560) | def _copy_extra_fields(boxlist_to_copy_to, boxlist_to_copy_from):
function _update_valid_indices_by_removing_high_iou_boxes (line 575) | def _update_valid_indices_by_removing_high_iou_boxes(
FILE: ava_evaluation/np_box_mask_list.py
class BoxMaskList (line 25) | class BoxMaskList(np_box_list.BoxList):
method __init__ (line 33) | def __init__(self, box_data, mask_data):
method get_masks (line 59) | def get_masks(self):
FILE: ava_evaluation/np_box_mask_list_ops.py
function box_list_to_box_mask_list (line 30) | def box_list_to_box_mask_list(boxlist):
function area (line 54) | def area(box_mask_list):
function intersection (line 66) | def intersection(box_mask_list1, box_mask_list2):
function iou (line 81) | def iou(box_mask_list1, box_mask_list2):
function ioa (line 94) | def ioa(box_mask_list1, box_mask_list2):
function gather (line 111) | def gather(box_mask_list, indices, fields=None):
function sort_by_field (line 142) | def sort_by_field(box_mask_list, field, order=np_box_list_ops.SortOrder....
function non_max_suppression (line 161) | def non_max_suppression(
function multi_class_non_max_suppression (line 242) | def multi_class_non_max_suppression(
function prune_non_overlapping_masks (line 324) | def prune_non_overlapping_masks(box_mask_list1, box_mask_list2, minoverl...
function concatenate (line 348) | def concatenate(box_mask_lists, fields=None):
function filter_scores_greater_than (line 378) | def filter_scores_greater_than(box_mask_list, thresh):
FILE: ava_evaluation/np_box_ops.py
function area (line 28) | def area(boxes):
function intersection (line 40) | def intersection(boxes1, boxes2):
function iou (line 68) | def iou(boxes1, boxes2):
function ioa (line 85) | def ioa(boxes1, boxes2):
FILE: ava_evaluation/np_mask_ops.py
function area (line 30) | def area(masks):
function intersection (line 48) | def intersection(masks1, masks2):
function iou (line 74) | def iou(masks1, masks2):
function ioa (line 98) | def ioa(masks1, masks2):
FILE: ava_evaluation/object_detection_evaluation.py
class DetectionEvaluator (line 41) | class DetectionEvaluator:
method __init__ (line 61) | def __init__(self, categories):
method add_single_ground_truth_image_info (line 72) | def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
method add_single_detected_image_info (line 83) | def add_single_detected_image_info(self, image_id, detections_dict):
method evaluate (line 94) | def evaluate(self):
method clear (line 99) | def clear(self):
class ObjectDetectionEvaluator (line 104) | class ObjectDetectionEvaluator(DetectionEvaluator):
method __init__ (line 107) | def __init__(
method add_single_ground_truth_image_info (line 155) | def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
method add_single_detected_image_info (line 229) | def add_single_detected_image_info(self, image_id, detections_dict):
method evaluate (line 276) | def evaluate(self):
method clear (line 331) | def clear(self):
class PascalDetectionEvaluator (line 342) | class PascalDetectionEvaluator(ObjectDetectionEvaluator):
method __init__ (line 345) | def __init__(self, categories, matching_iou_threshold=0.5):
class WeightedPascalDetectionEvaluator (line 355) | class WeightedPascalDetectionEvaluator(ObjectDetectionEvaluator):
method __init__ (line 369) | def __init__(self, categories, matching_iou_threshold=0.5):
class PascalInstanceSegmentationEvaluator (line 379) | class PascalInstanceSegmentationEvaluator(ObjectDetectionEvaluator):
method __init__ (line 382) | def __init__(self, categories, matching_iou_threshold=0.5):
class WeightedPascalInstanceSegmentationEvaluator (line 393) | class WeightedPascalInstanceSegmentationEvaluator(ObjectDetectionEvaluat...
method __init__ (line 407) | def __init__(self, categories, matching_iou_threshold=0.5):
class OpenImagesDetectionEvaluator (line 418) | class OpenImagesDetectionEvaluator(ObjectDetectionEvaluator):
method __init__ (line 425) | def __init__(self, categories, matching_iou_threshold=0.5, evaluate_co...
method add_single_ground_truth_image_info (line 443) | def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
class ObjectDetectionEvaluation (line 515) | class ObjectDetectionEvaluation:
method __init__ (line 518) | def __init__(
method _initialize_detections (line 548) | def _initialize_detections(self):
method clear_detections (line 559) | def clear_detections(self):
method add_single_ground_truth_image_info (line 562) | def add_single_ground_truth_image_info(
method add_single_detected_image_info (line 619) | def add_single_detected_image_info(
method _update_ground_truth_statistics (line 704) | def _update_ground_truth_statistics(
method evaluate (line 736) | def evaluate(self):
FILE: ava_evaluation/per_image_evaluation.py
class PerImageEvaluation (line 31) | class PerImageEvaluation:
method __init__ (line 34) | def __init__(self, num_groundtruth_classes, matching_iou_threshold=0.5):
method compute_object_detection_metrics (line 45) | def compute_object_detection_metrics(
method _compute_tp_fp (line 120) | def _compute_tp_fp(
method _get_overlaps_and_scores_box_mode (line 215) | def _get_overlaps_and_scores_box_mode(
method _compute_tp_fp_for_single_class (line 253) | def _compute_tp_fp_for_single_class(
method _get_ith_class_arrays (line 344) | def _get_ith_class_arrays(
method _remove_invalid_boxes (line 401) | def _remove_invalid_boxes(
FILE: ava_evaluation/standard_fields.py
class InputDataFields (line 29) | class InputDataFields:
class DetectionResultFields (line 94) | class DetectionResultFields:
class BoxListFields (line 120) | class BoxListFields:
class TfExampleFields (line 146) | class TfExampleFields:
FILE: slowfast/config/custom_config.py
function add_custom_config (line 7) | def add_custom_config(_C):
FILE: slowfast/config/defaults.py
function assert_and_infer_cfg (line 1263) | def assert_and_infer_cfg(cfg):
function get_cfg (line 1291) | def get_cfg():
FILE: slowfast/datasets/ava_dataset.py
class Ava (line 21) | class Ava(torch.utils.data.Dataset):
method __init__ (line 26) | def __init__(self, cfg, split):
method _load_data (line 52) | def _load_data(self, cfg):
method print_summary (line 88) | def print_summary(self):
method __len__ (line 99) | def __len__(self):
method num_videos (line 107) | def num_videos(self):
method _images_and_boxes_preprocessing_cv2 (line 114) | def _images_and_boxes_preprocessing_cv2(self, imgs, boxes):
method _images_and_boxes_preprocessing (line 238) | def _images_and_boxes_preprocessing(self, imgs, boxes):
method __getitem__ (line 339) | def __getitem__(self, idx):
FILE: slowfast/datasets/ava_helper.py
function load_image_lists (line 16) | def load_image_lists(cfg, is_train):
function load_boxes_and_labels (line 63) | def load_boxes_and_labels(cfg, mode):
function get_keyframe_data (line 107) | def get_keyframe_data(boxes_and_labels):
function get_num_boxes_used (line 151) | def get_num_boxes_used(keyframe_indices, keyframe_boxes_and_labels):
function parse_bboxes_file (line 170) | def parse_bboxes_file(ann_filenames, ann_is_gt_box, detect_thresh, boxes...
FILE: slowfast/datasets/build.py
function build_dataset (line 15) | def build_dataset(dataset_name, cfg, split):
FILE: slowfast/datasets/charades.py
class Charades (line 20) | class Charades(torch.utils.data.Dataset):
method __init__ (line 31) | def __init__(self, cfg, mode, num_retries=10):
method _construct_loader (line 72) | def _construct_loader(self):
method get_seq_frames (line 107) | def get_seq_frames(self, index):
method __getitem__ (line 147) | def __getitem__(self, index):
method __len__ (line 233) | def __len__(self):
method num_videos (line 241) | def num_videos(self):
FILE: slowfast/datasets/cv2_transform.py
function clip_boxes_to_image (line 10) | def clip_boxes_to_image(boxes, height, width):
function random_short_side_scale_jitter_list (line 26) | def random_short_side_scale_jitter_list(images, min_size, max_size, boxe...
function scale (line 70) | def scale(size, image):
function scale_boxes (line 95) | def scale_boxes(size, boxes, height, width):
function horizontal_flip_list (line 121) | def horizontal_flip_list(prob, images, order="CHW", boxes=None):
function spatial_shift_crop_list (line 153) | def spatial_shift_crop_list(size, images, spatial_shift_pos, boxes=None):
function CHW2HWC (line 203) | def CHW2HWC(image):
function HWC2CHW (line 215) | def HWC2CHW(image):
function color_jitter_list (line 227) | def color_jitter_list(images, img_brightness=0, img_contrast=0, img_satu...
function lighting_list (line 258) | def lighting_list(imgs, alphastd, eigval, eigvec, alpha=None):
function color_normalization (line 287) | def color_normalization(image, mean, stddev):
function pad_image (line 304) | def pad_image(image, pad_size, order="CHW"):
function horizontal_flip (line 329) | def horizontal_flip(prob, image, order="CHW"):
function flip_boxes (line 350) | def flip_boxes(boxes, im_width):
function crop_boxes (line 366) | def crop_boxes(boxes, x_offset, y_offset):
function random_crop_list (line 379) | def random_crop_list(images, size, pad_size=0, order="CHW", boxes=None):
function center_crop (line 442) | def center_crop(size, image):
function random_scale_jitter (line 461) | def random_scale_jitter(image, min_size, max_size):
function random_scale_jitter_list (line 477) | def random_scale_jitter_list(images, min_size, max_size):
function random_sized_crop (line 493) | def random_sized_crop(image, size, area_frac=0.08):
function lighting (line 532) | def lighting(img, alphastd, eigval, eigvec):
function random_sized_crop_list (line 558) | def random_sized_crop_list(images, size, crop_area_fraction=0.08):
function blend (line 606) | def blend(image1, image2, alpha):
function grayscale (line 610) | def grayscale(image):
function saturation (line 628) | def saturation(var, image):
function brightness (line 642) | def brightness(var, image):
function contrast (line 656) | def contrast(var, image):
function saturation_list (line 671) | def saturation_list(var, images):
function brightness_list (line 689) | def brightness_list(var, images):
function contrast_list (line 707) | def contrast_list(var, images):
function color_jitter (line 726) | def color_jitter(image, img_brightness=0, img_contrast=0, img_saturation...
function revert_scaled_boxes (line 757) | def revert_scaled_boxes(size, boxes, img_height, img_width):
FILE: slowfast/datasets/decoder.py
function temporal_sampling (line 17) | def temporal_sampling(frames, start_idx, end_idx, num_samples):
function get_start_end_idx (line 37) | def get_start_end_idx(
function get_multiple_start_end_idx (line 79) | def get_multiple_start_end_idx(
function pyav_decode_stream (line 186) | def pyav_decode_stream(
function torchvision_decode (line 228) | def torchvision_decode(
function pyav_decode (line 373) | def pyav_decode(
function decode (line 452) | def decode(
FILE: slowfast/datasets/imagenet.py
class Imagenet (line 27) | class Imagenet(torch.utils.data.Dataset):
method __init__ (line 30) | def __init__(self, cfg, mode, num_retries=10):
method _load_imdb (line 49) | def _load_imdb(self):
method _construct_imdb (line 57) | def _construct_imdb(self):
method load_image (line 78) | def load_image(self, im_path):
method _prepare_im_res (line 88) | def _prepare_im_res(self, im_path):
method _prepare_im_tf (line 121) | def _prepare_im_tf(self, im_path):
method _prepare_im_masked (line 168) | def _prepare_im_masked(self, im_path):
method __load__ (line 211) | def __load__(self, index):
method __getitem__ (line 234) | def __getitem__(self, index):
method __len__ (line 267) | def __len__(self):
FILE: slowfast/datasets/kinetics.py
class Kinetics (line 29) | class Kinetics(torch.utils.data.Dataset):
method __init__ (line 40) | def __init__(self, cfg, mode, num_retries=100):
method _construct_loader (line 102) | def _construct_loader(self):
method _set_epoch_num (line 156) | def _set_epoch_num(self, epoch):
method _get_chunk (line 159) | def _get_chunk(self, path_to_file, chunksize):
method __getitem__ (line 174) | def __getitem__(self, index):
method _gen_mask (line 470) | def _gen_mask(self):
method _frame_to_list_img (line 506) | def _frame_to_list_img(self, frames):
method _list_img_to_frames (line 510) | def _list_img_to_frames(self, img_list):
method __len__ (line 514) | def __len__(self):
method num_videos (line 522) | def num_videos(self):
FILE: slowfast/datasets/loader.py
function multiple_samples_collate (line 20) | def multiple_samples_collate(batch, fold=False):
function detection_collate (line 48) | def detection_collate(batch):
function construct_loader (line 86) | def construct_loader(cfg, split, is_precise_bn=False):
function shuffle_dataset (line 174) | def shuffle_dataset(loader, cur_epoch):
FILE: slowfast/datasets/mixup.py
function convert_to_one_hot (line 22) | def convert_to_one_hot(targets, num_classes, on_value=1.0, off_value=0.0):
function mixup_target (line 40) | def mixup_target(target, num_classes, lam=1.0, smoothing=0.0):
function rand_bbox (line 67) | def rand_bbox(img_shape, lam, margin=0.0, count=None):
function get_cutmix_bbox (line 90) | def get_cutmix_bbox(img_shape, lam, correct_lam=True, count=None):
class MixUp (line 109) | class MixUp:
method __init__ (line 117) | def __init__(
method _get_mixup_params (line 148) | def _get_mixup_params(self):
method _mix_batch (line 167) | def _mix_batch(self, x):
method __call__ (line 183) | def __call__(self, x, target):
FILE: slowfast/datasets/multigrid_helper.py
class ShortCycleBatchSampler (line 19) | class ShortCycleBatchSampler(Sampler):
method __init__ (line 26) | def __init__(self, sampler, batch_size, drop_last, cfg):
method __iter__ (line 66) | def __iter__(self):
method __len__ (line 80) | def __len__(self):
FILE: slowfast/datasets/ptv_datasets.py
class PTVDatasetWrapper (line 32) | class PTVDatasetWrapper(torch.utils.data.IterableDataset):
method __init__ (line 37) | def __init__(self, num_videos, clips_per_video, crops_per_clip, dataset):
method __next__ (line 51) | def __next__(self):
method sampler (line 58) | def sampler(self):
method __len__ (line 65) | def __len__(self):
method num_videos (line 73) | def num_videos(self):
method __iter__ (line 80) | def __iter__(self):
class PackPathway (line 84) | class PackPathway(torch.nn.Module):
method __init__ (line 90) | def __init__(self, cfg):
method forward (line 94) | def forward(self, x: torch.Tensor):
class DictToTuple (line 98) | class DictToTuple(torch.nn.Module):
method __init__ (line 104) | def __init__(self, num_clips, num_crops):
method forward (line 109) | def forward(self, x: Dict[str, torch.Tensor]):
function div255 (line 119) | def div255(x):
function Ptvkinetics (line 133) | def Ptvkinetics(cfg, mode):
function process_charades_label (line 255) | def process_charades_label(x, mode, num_classes):
function rgb2bgr (line 274) | def rgb2bgr(x):
function Ptvcharades (line 288) | def Ptvcharades(cfg, mode):
function Ptvssv2 (line 427) | def Ptvssv2(cfg, mode):
FILE: slowfast/datasets/rand_augment.py
function _interpolation (line 53) | def _interpolation(kwargs):
function _check_args_tf (line 61) | def _check_args_tf(kwargs):
function shear_x (line 67) | def shear_x(img, factor, **kwargs):
function shear_y (line 72) | def shear_y(img, factor, **kwargs):
function translate_x_rel (line 77) | def translate_x_rel(img, pct, **kwargs):
function translate_y_rel (line 83) | def translate_y_rel(img, pct, **kwargs):
function translate_x_abs (line 89) | def translate_x_abs(img, pixels, **kwargs):
function translate_y_abs (line 94) | def translate_y_abs(img, pixels, **kwargs):
function rotate (line 99) | def rotate(img, degrees, **kwargs):
function auto_contrast (line 133) | def auto_contrast(img, **__):
function invert (line 137) | def invert(img, **__):
function equalize (line 141) | def equalize(img, **__):
function solarize (line 145) | def solarize(img, thresh, **__):
function solarize_add (line 149) | def solarize_add(img, add, thresh=128, **__):
function posterize (line 164) | def posterize(img, bits_to_keep, **__):
function contrast (line 170) | def contrast(img, factor, **__):
function color (line 174) | def color(img, factor, **__):
function brightness (line 178) | def brightness(img, factor, **__):
function sharpness (line 182) | def sharpness(img, factor, **__):
function _randomly_negate (line 186) | def _randomly_negate(v):
function _rotate_level_to_arg (line 191) | def _rotate_level_to_arg(level, _hparams):
function _enhance_level_to_arg (line 198) | def _enhance_level_to_arg(level, _hparams):
function _enhance_increasing_level_to_arg (line 203) | def _enhance_increasing_level_to_arg(level, _hparams):
function _shear_level_to_arg (line 211) | def _shear_level_to_arg(level, _hparams):
function _translate_abs_level_to_arg (line 218) | def _translate_abs_level_to_arg(level, hparams):
function _translate_rel_level_to_arg (line 225) | def _translate_rel_level_to_arg(level, hparams):
function _posterize_level_to_arg (line 233) | def _posterize_level_to_arg(level, _hparams):
function _posterize_increasing_level_to_arg (line 240) | def _posterize_increasing_level_to_arg(level, hparams):
function _posterize_original_level_to_arg (line 247) | def _posterize_original_level_to_arg(level, _hparams):
function _solarize_level_to_arg (line 254) | def _solarize_level_to_arg(level, _hparams):
function _solarize_increasing_level_to_arg (line 260) | def _solarize_increasing_level_to_arg(level, _hparams):
function _solarize_add_level_to_arg (line 266) | def _solarize_add_level_to_arg(level, _hparams):
class AugmentOp (line 328) | class AugmentOp:
method __init__ (line 333) | def __init__(self, name, prob=0.5, magnitude=10, hparams=None):
method __call__ (line 355) | def __call__(self, img_list):
function _select_rand_weights (line 431) | def _select_rand_weights(weight_idx=0, transforms=None):
function rand_augment_ops (line 440) | def rand_augment_ops(magnitude=10, hparams=None, transforms=None):
class RandAugment (line 449) | class RandAugment:
method __init__ (line 450) | def __init__(self, ops, num_layers=2, choice_weights=None):
method __call__ (line 455) | def __call__(self, img):
function rand_augment_transform (line 468) | def rand_augment_transform(config_str, hparams):
FILE: slowfast/datasets/random_erasing.py
function _get_pixels (line 20) | def _get_pixels(per_pixel, rand_color, patch_size, dtype=torch.float32, ...
class RandomErasing (line 32) | class RandomErasing:
method __init__ (line 51) | def __init__(
method _erase (line 85) | def _erase(self, img, chan, img_h, img_w, dtype):
method _erase_cube (line 114) | def _erase_cube(
method __call__ (line 154) | def __call__(self, input):
FILE: slowfast/datasets/ssv2.py
class Ssv2 (line 22) | class Ssv2(torch.utils.data.Dataset):
method __init__ (line 33) | def __init__(self, cfg, mode, num_retries=10):
method _construct_loader (line 81) | def _construct_loader(self):
method get_seq_frames (line 159) | def get_seq_frames(self, index):
method __getitem__ (line 182) | def __getitem__(self, index):
method __len__ (line 305) | def __len__(self):
method num_videos (line 313) | def num_videos(self):
FILE: slowfast/datasets/transform.py
function _pil_interp (line 34) | def _pil_interp(method):
function random_short_side_scale_jitter (line 48) | def random_short_side_scale_jitter(
function crop_boxes (line 101) | def crop_boxes(boxes, x_offset, y_offset):
function random_crop (line 120) | def random_crop(images, size, boxes=None):
function horizontal_flip (line 152) | def horizontal_flip(prob, images, boxes=None):
function uniform_crop (line 187) | def uniform_crop(images, size, spatial_idx, boxes=None, scale_size=None):
function clip_boxes_to_image (line 246) | def clip_boxes_to_image(boxes, height, width):
function blend (line 268) | def blend(images1, images2, alpha):
function grayscale (line 284) | def grayscale(images):
function color_jitter (line 304) | def color_jitter(images, img_brightness=0, img_contrast=0, img_saturatio...
function brightness_jitter (line 339) | def brightness_jitter(var, images):
function contrast_jitter (line 358) | def contrast_jitter(var, images):
function saturation_jitter (line 378) | def saturation_jitter(var, images):
function lighting_jitter (line 397) | def lighting_jitter(images, alphastd, eigval, eigvec):
function color_normalization (line 443) | def color_normalization(images, mean, stddev):
function _get_param_spatial_crop (line 477) | def _get_param_spatial_crop(
function random_resized_crop (line 519) | def random_resized_crop(
function random_resized_crop_with_shift (line 554) | def random_resized_crop_with_shift(
function create_random_augment (line 599) | def create_random_augment(
function random_sized_crop_img (line 633) | def random_sized_crop_img(
class RandomResizedCropAndInterpolation (line 666) | class RandomResizedCropAndInterpolation:
method __init__ (line 679) | def __init__(
method get_params (line 701) | def get_params(img, scale, ratio):
method __call__ (line 741) | def __call__(self, img):
method __repr__ (line 755) | def __repr__(self):
class MaskingGenerator (line 776) | class MaskingGenerator:
method __init__ (line 777) | def __init__(
method __repr__ (line 807) | def __repr__(self):
method get_shape (line 819) | def get_shape(self):
method _mask (line 822) | def _mask(self, mask, max_mask_patches):
method __call__ (line 846) | def __call__(self):
class MaskingGenerator3D (line 869) | class MaskingGenerator3D:
method __init__ (line 870) | def __init__(
method __repr__ (line 888) | def __repr__(self):
method get_shape (line 901) | def get_shape(self):
method _mask (line 904) | def _mask(self, mask, max_mask_patches):
method __call__ (line 933) | def __call__(self):
function transforms_imagenet_train (line 948) | def transforms_imagenet_train(
function temporal_difference (line 1047) | def temporal_difference(
function color_jitter_video_ssl (line 1073) | def color_jitter_video_ssl(
function augment_raw_frames (line 1124) | def augment_raw_frames(frames, time_diff_prob=0.0, gaussian_prob=0.0):
class GaussianBlur (line 1145) | class GaussianBlur:
method __init__ (line 1148) | def __init__(self, sigma=[0.1, 2.0]):
method __call__ (line 1151) | def __call__(self, x):
class GaussianBlurVideo (line 1160) | class GaussianBlurVideo:
method __init__ (line 1161) | def __init__(self, sigma_min=[0.0, 0.1], sigma_max=[0.0, 2.0], use_PIL...
method __call__ (line 1165) | def __call__(self, frames):
FILE: slowfast/datasets/utils.py
function retry_load_images (line 24) | def retry_load_images(image_paths, retry=10, backend="pytorch"):
function get_sequence (line 55) | def get_sequence(center_idx, half_len, sample_rate, num_frames):
function pack_pathway_output (line 78) | def pack_pathway_output(cfg, frames):
function spatial_sampling (line 114) | def spatial_sampling(
function as_binary_vector (line 188) | def as_binary_vector(labels, num_classes):
function aggregate_labels (line 204) | def aggregate_labels(label_list):
function convert_to_video_level_labels (line 219) | def convert_to_video_level_labels(labels):
function load_image_lists (line 235) | def load_image_lists(frame_list_file, prefix="", return_list=False):
function tensor_normalize (line 278) | def tensor_normalize(tensor, mean, std, func=None):
function get_random_sampling_rate (line 300) | def get_random_sampling_rate(long_cycle_sampling_rate, sampling_rate):
function revert_tensor_normalize (line 312) | def revert_tensor_normalize(tensor, mean, std):
function create_sampler (line 329) | def create_sampler(dataset, shuffle, cfg):
function loader_worker_init_fn (line 346) | def loader_worker_init_fn(dataset):
function aug_frame (line 355) | def aug_frame(
function _frame_to_list_img (line 437) | def _frame_to_list_img(frames):
function _list_img_to_frames (line 442) | def _list_img_to_frames(img_list):
FILE: slowfast/datasets/video_container.py
function get_video_container (line 7) | def get_video_container(path_to_vid, multi_thread_decode=False, backend=...
FILE: slowfast/models/attention.py
function attention_pool (line 13) | def attention_pool(tensor, pool, thw_shape, has_cls_embed=True, norm=None):
function get_rel_pos (line 48) | def get_rel_pos(rel_pos, d):
function cal_rel_pos_spatial (line 64) | def cal_rel_pos_spatial(
function cal_rel_pos_temporal (line 111) | def cal_rel_pos_temporal(attn, q, has_cls_embed, q_shape, k_shape, rel_p...
class MultiScaleAttention (line 150) | class MultiScaleAttention(nn.Module):
method __init__ (line 151) | def __init__(
method forward (line 293) | def forward(self, x, thw_shape):
class MultiScaleBlock (line 395) | class MultiScaleBlock(nn.Module):
method __init__ (line 396) | def __init__(
method forward (line 491) | def forward(self, x, thw_shape=None):
FILE: slowfast/models/batchnorm_helper.py
function get_norm (line 16) | def get_norm(cfg):
class SubBatchNorm3d (line 40) | class SubBatchNorm3d(nn.Module):
method __init__ (line 51) | def __init__(self, num_splits, **args):
method _get_aggregated_mean_std (line 72) | def _get_aggregated_mean_std(self, means, stds, n):
method aggregate_stats (line 87) | def aggregate_stats(self):
method forward (line 101) | def forward(self, x):
FILE: slowfast/models/build.py
function build_model (line 22) | def build_model(cfg, gpu_id=None):
FILE: slowfast/models/common.py
class Mlp (line 7) | class Mlp(nn.Module):
method __init__ (line 8) | def __init__(
method forward (line 26) | def forward(self, x):
class Permute (line 37) | class Permute(nn.Module):
method __init__ (line 38) | def __init__(self, dims):
method forward (line 42) | def forward(self, x):
function drop_path (line 46) | def drop_path(x, drop_prob: float = 0.0, training: bool = False):
class DropPath (line 62) | class DropPath(nn.Module):
method __init__ (line 65) | def __init__(self, drop_prob=None):
method forward (line 69) | def forward(self, x):
class TwoStreamFusion (line 73) | class TwoStreamFusion(nn.Module):
method __init__ (line 74) | def __init__(self, mode, dim=None, kernel=3, padding=1):
method forward (line 141) | def forward(self, x):
FILE: slowfast/models/contrastive.py
class ContrastiveModel (line 32) | class ContrastiveModel(nn.Module):
method __init__ (line 37) | def __init__(self, cfg):
method knn_mem_update (line 126) | def knn_mem_update(self, q_knn, index):
method init_knn_labels (line 137) | def init_knn_labels(self, train_loader):
method _update_history (line 153) | def _update_history(self):
method _batch_shuffle (line 169) | def _batch_shuffle(self, x):
method _batch_unshuffle (line 211) | def _batch_unshuffle(self, x, idx_restore):
method eval_knn (line 227) | def eval_knn(self, q_knn, knn_k=200):
method sim_loss (line 237) | def sim_loss(self, q, k):
method momentum_anneal_cosine (line 246) | def momentum_anneal_cosine(self, epoch_exact):
method _dequeue_and_enqueue (line 255) | def _dequeue_and_enqueue(self, keys, extra_keys=None):
method batch_clips (line 284) | def batch_clips(self, clips):
method compute_key_feat (line 296) | def compute_key_feat(
method forward (line 358) | def forward(self, clips, index=None, time=None, epoch_exact=None, keys...
method _simclr_precompute_pos_neg_mask_multi (line 758) | def _simclr_precompute_pos_neg_mask_multi(self):
method run_swav_encoder_q (line 800) | def run_swav_encoder_q(self, im):
method get_code (line 808) | def get_code(self, out):
method run_swav_orig_encoder_q (line 817) | def run_swav_orig_encoder_q(self, x):
method sinkhorn (line 825) | def sinkhorn(self, Q, iters):
method distributed_sinkhorn (line 841) | def distributed_sinkhorn(self, Q, nmb_iters):
method KLDivLoss (line 864) | def KLDivLoss(self, out, code):
function l2_loss (line 871) | def l2_loss(x, y):
class Normalize (line 875) | class Normalize(nn.Module):
method __init__ (line 876) | def __init__(self, power=2, dim=1):
method forward (line 881) | def forward(self, x):
class Memory (line 887) | class Memory(nn.Module):
method __init__ (line 888) | def __init__(self, length, duration, dim, cfg):
method resize (line 903) | def resize(self, length, duration, dim):
method get (line 916) | def get(self, ind, time, interp=False):
method update (line 939) | def update(self, mem, momentum, ind, time, interp=False):
method forward (line 986) | def forward(self, inputs):
class Memory1D (line 990) | class Memory1D(nn.Module):
method __init__ (line 991) | def __init__(self, length, duration, dim, cfg):
method get (line 1005) | def get(self, ind, time, interp=False):
method update (line 1017) | def update(self, mem, momentum, ind, time, interp=False):
function contrastive_parameter_surgery (line 1031) | def contrastive_parameter_surgery(model, cfg, epoch_exact, cur_iter):
function contrastive_forward (line 1058) | def contrastive_forward(model, cfg, inputs, index, time, epoch_exact, sc...
FILE: slowfast/models/head_helper.py
class ResNetRoIHead (line 20) | class ResNetRoIHead(nn.Module):
method __init__ (line 25) | def __init__(
method forward (line 115) | def forward(self, inputs, bboxes):
class MLPHead (line 147) | class MLPHead(nn.Module):
method __init__ (line 148) | def __init__(
method forward (line 189) | def forward(self, x):
class ResNetBasicHead (line 198) | class ResNetBasicHead(nn.Module):
method __init__ (line 207) | def __init__(
method forward (line 305) | def forward(self, inputs):
class X3DHead (line 353) | class X3DHead(nn.Module):
method __init__ (line 362) | def __init__(
method _construct_head (line 412) | def _construct_head(self, dim_in, dim_inner, dim_out, norm_module):
method forward (line 461) | def forward(self, inputs):
class TransformerBasicHead (line 491) | class TransformerBasicHead(nn.Module):
method __init__ (line 496) | def __init__(
method forward (line 547) | def forward(self, x):
class MSSeparateHead (line 566) | class MSSeparateHead(nn.Module):
method __init__ (line 576) | def __init__(
method _init_weights (line 645) | def _init_weights(self, m):
method forward (line 656) | def forward(self, block_outputs, output_masks, return_all, thw):
FILE: slowfast/models/losses.py
class ContrastiveLoss (line 13) | class ContrastiveLoss(nn.Module):
method __init__ (line 14) | def __init__(self, reduction="mean"):
method forward (line 18) | def forward(self, inputs, dummy_labels=None):
class MultipleMSELoss (line 24) | class MultipleMSELoss(nn.Module):
method __init__ (line 29) | def __init__(self, reduction="mean"):
method forward (line 38) | def forward(self, x, y):
function get_loss_func (line 72) | def get_loss_func(loss_name):
FILE: slowfast/models/masked.py
class MaskMViT (line 24) | class MaskMViT(MViT):
method __init__ (line 25) | def __init__(self, cfg):
method no_weight_decay (line 130) | def no_weight_decay(self):
method fix_init_weight (line 149) | def fix_init_weight(self):
method _get_multiscale_mask (line 165) | def _get_multiscale_mask(self, mask):
method _patchify (line 178) | def _patchify(self, imgs, p=16, time_stride_loss=True):
method _unpatchify (line 190) | def _unpatchify(self, x):
method _get_pixel_label_2d (line 197) | def _get_pixel_label_2d(self, input_img, output_masks, norm=True):
method _get_pixel_label_3d (line 212) | def _get_pixel_label_3d(
method _get_hog_label_2d (line 232) | def _get_hog_label_2d(self, input_frames, output_masks):
method _get_hog_label_3d (line 254) | def _get_hog_label_3d(self, input_frames, output_masks):
method _mae_random_masking (line 283) | def _mae_random_masking(self, x, mask_ratio, mask_in=None):
method _mae_forward_encoder (line 319) | def _mae_forward_encoder(self, x, mask_ratio, mask=None):
method _mae_forward_decoder (line 394) | def _mae_forward_decoder(self, x, ids_restore, mask, thw):
method _mae_forward (line 447) | def _mae_forward(self, imgs, mask_ratio=0.75, mask=None):
method _mae_visualize (line 478) | def _mae_visualize(self, imgs, pred, mask):
method _maskfeat_forward (line 519) | def _maskfeat_forward(self, x, mask, return_all=False):
method forward (line 614) | def forward(self, x, return_all=False):
FILE: slowfast/models/nonlocal_helper.py
class Nonlocal (line 10) | class Nonlocal(nn.Module):
method __init__ (line 20) | def __init__(
method _construct_nonlocal (line 65) | def _construct_nonlocal(
method forward (line 103) | def forward(self, x):
FILE: slowfast/models/operators.py
class SE (line 15) | class SE(nn.Module):
method _round_width (line 18) | def _round_width(self, width, multiplier, min_width=8, divisor=8):
method __init__ (line 37) | def __init__(self, dim_in, ratio, relu_act=True):
method forward (line 55) | def forward(self, x):
class HOGLayerC (line 62) | class HOGLayerC(nn.Module):
method __init__ (line 63) | def __init__(self, nbins=9, pool=7, gaussian_window=0):
method forward (line 80) | def forward(self, x):
FILE: slowfast/models/optimizer.py
function construct_optimizer (line 10) | def construct_optimizer(model, cfg):
function get_param_groups (line 146) | def get_param_groups(model, cfg):
function get_epoch_lr (line 240) | def get_epoch_lr(cur_epoch, cfg):
function set_lr (line 251) | def set_lr(optimizer, new_lr):
class LARS (line 262) | class LARS:
method __init__ (line 274) | def __init__(
method __getstate__ (line 288) | def __getstate__(self):
method __setstate__ (line 291) | def __setstate__(self, state):
method state (line 295) | def state(self):
method __repr__ (line 298) | def __repr__(self):
method param_groups (line 302) | def param_groups(self):
method param_groups (line 306) | def param_groups(self, value):
method state_dict (line 309) | def state_dict(self):
method load_state_dict (line 312) | def load_state_dict(self, state_dict):
method zero_grad (line 315) | def zero_grad(self):
method add_param_group (line 318) | def add_param_group(self, param_group):
method step (line 321) | def step(self):
function get_grad_norm_ (line 362) | def get_grad_norm_(parameters, norm_type=2.0):
FILE: slowfast/models/ptv_model_builder.py
function get_head_act (line 29) | def get_head_act(act_func):
class PTVResNet (line 50) | class PTVResNet(nn.Module):
method __init__ (line 55) | def __init__(self, cfg):
method _construct_network (line 79) | def _construct_network(self, cfg):
method forward (line 198) | def forward(self, x, bboxes=None):
class PTVSlowFast (line 214) | class PTVSlowFast(nn.Module):
method __init__ (line 215) | def __init__(self, cfg):
method _construct_network (line 234) | def _construct_network(self, cfg):
method forward (line 390) | def forward(self, x, bboxes=None):
class PTVX3D (line 405) | class PTVX3D(nn.Module):
method __init__ (line 406) | def __init__(self, cfg):
method _construct_network (line 427) | def _construct_network(self, cfg):
method forward (line 483) | def forward(self, x, bboxes=None):
class PTVCSN (line 496) | class PTVCSN(nn.Module):
method __init__ (line 501) | def __init__(self, cfg):
method _construct_network (line 518) | def _construct_network(self, cfg):
method forward (line 568) | def forward(self, x, bboxes=None):
class PTVR2plus1D (line 581) | class PTVR2plus1D(nn.Module):
method __init__ (line 586) | def __init__(self, cfg):
method _construct_network (line 603) | def _construct_network(self, cfg):
method forward (line 669) | def forward(self, x, bboxes=None):
class PTVMViT (line 682) | class PTVMViT(nn.Module):
method __init__ (line 687) | def __init__(self, cfg):
method _construct_network (line 704) | def _construct_network(self, cfg):
method forward (line 748) | def forward(self, x, bboxes=None):
FILE: slowfast/models/resnet_helper.py
function get_trans_func (line 12) | def get_trans_func(name):
class BasicTransform (line 27) | class BasicTransform(nn.Module):
method __init__ (line 32) | def __init__(
method _construct (line 75) | def _construct(self, dim_in, dim_out, stride, dilation, norm_module):
method forward (line 108) | def forward(self, x):
class X3DTransform (line 118) | class X3DTransform(nn.Module):
method __init__ (line 125) | def __init__(
method _construct (line 188) | def _construct(
method forward (line 253) | def forward(self, x):
class BottleneckTransform (line 259) | class BottleneckTransform(nn.Module):
method __init__ (line 265) | def __init__(
method _construct (line 319) | def _construct(
method forward (line 377) | def forward(self, x):
class ResBlock (line 395) | class ResBlock(nn.Module):
method __init__ (line 400) | def __init__(
method _construct (line 468) | def _construct(
method forward (line 512) | def forward(self, x):
class ResStage (line 524) | class ResStage(nn.Module):
method __init__ (line 535) | def __init__(
method _construct (line 650) | def _construct(
method forward (line 697) | def forward(self, inputs):
FILE: slowfast/models/reversible_mvit.py
class ReversibleMViT (line 12) | class ReversibleMViT(nn.Module):
method __init__ (line 24) | def __init__(self, config, model):
method vanilla_backward (line 128) | def vanilla_backward(h, layers, buffer):
method forward (line 141) | def forward(self, x):
class RevBackProp (line 177) | class RevBackProp(Function):
method forward (line 186) | def forward(
method backward (line 221) | def backward(ctx, dx):
class StageTransitionBlock (line 266) | class StageTransitionBlock(nn.Module):
method __init__ (line 272) | def __init__(
method forward (line 350) | def forward(
class ReversibleBlock (line 412) | class ReversibleBlock(nn.Module):
method __init__ (line 419) | def __init__(
method seed_cuda (line 468) | def seed_cuda(self, key):
method forward (line 491) | def forward(self, X_1, X_2):
method backward_pass (line 528) | def backward_pass(
class MLPSubblock (line 593) | class MLPSubblock(nn.Module):
method __init__ (line 599) | def __init__(
method forward (line 616) | def forward(self, x):
class AttentionSubBlock (line 620) | class AttentionSubBlock(nn.Module):
method __init__ (line 626) | def __init__(
method forward (line 670) | def forward(self, x):
FILE: slowfast/models/stem_helper.py
function get_stem_func (line 9) | def get_stem_func(name):
class VideoModelStem (line 20) | class VideoModelStem(nn.Module):
method __init__ (line 26) | def __init__(
method _construct_stem (line 98) | def _construct_stem(self, dim_in, dim_out, norm_module, stem_func_name):
method forward (line 115) | def forward(self, x):
class ResNetBasicStem (line 127) | class ResNetBasicStem(nn.Module):
method __init__ (line 134) | def __init__(
method _construct_stem (line 181) | def _construct_stem(self, dim_in, dim_out, norm_module):
method forward (line 196) | def forward(self, x):
class X3DStem (line 204) | class X3DStem(nn.Module):
method __init__ (line 211) | def __init__(
method _construct_stem (line 258) | def _construct_stem(self, dim_in, dim_out, norm_module):
method forward (line 280) | def forward(self, x):
class PatchEmbed (line 288) | class PatchEmbed(nn.Module):
method __init__ (line 293) | def __init__(
method forward (line 315) | def forward(self, x, keep_spatial=False):
FILE: slowfast/models/utils.py
function round_width (line 10) | def round_width(width, multiplier, min_width=1, divisor=1, verbose=False):
function validate_checkpoint_wrapper_import (line 26) | def validate_checkpoint_wrapper_import(checkpoint_wrapper):
function get_gkern (line 34) | def get_gkern(kernlen, std):
function get_3d_sincos_pos_embed (line 55) | def get_3d_sincos_pos_embed(embed_dim, grid_size, t_size, cls_token=False):
function get_2d_sincos_pos_embed (line 97) | def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
function get_2d_sincos_pos_embed_from_grid (line 115) | def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
function get_1d_sincos_pos_embed_from_grid (line 126) | def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
function interpolate_pos_embed (line 152) | def interpolate_pos_embed(model, checkpoint_model):
function calc_mvit_feature_geometry (line 185) | def calc_mvit_feature_geometry(cfg):
FILE: slowfast/models/video_model_builder.py
class FuseFastToSlow (line 112) | class FuseFastToSlow(nn.Module):
method __init__ (line 119) | def __init__(
method forward (line 162) | def forward(self, x):
class SlowFast (line 173) | class SlowFast(nn.Module):
method __init__ (line 182) | def __init__(self, cfg):
method _construct_network (line 203) | def _construct_network(self, cfg):
method forward (line 423) | def forward(self, x, bboxes=None):
class ResNet (line 445) | class ResNet(nn.Module):
method __init__ (line 459) | def __init__(self, cfg):
method _construct_network (line 480) | def _construct_network(self, cfg):
method forward (line 645) | def forward(self, x, bboxes=None):
class X3D (line 664) | class X3D(nn.Module):
method __init__ (line 673) | def __init__(self, cfg):
method _round_repeats (line 711) | def _round_repeats(self, repeats, multiplier):
method _construct_network (line 718) | def _construct_network(self, cfg):
method forward (line 799) | def forward(self, x, bboxes=None):
class MViT (line 806) | class MViT(nn.Module):
method __init__ (line 818) | def __init__(self, cfg):
method _init_weights (line 1085) | def _init_weights(self, m):
method no_weight_decay (line 1095) | def no_weight_decay(self):
method _get_pos_embed (line 1118) | def _get_pos_embed(self, pos_embed, bcthw):
method _forward_reversible (line 1143) | def _forward_reversible(self, x):
method forward (line 1166) | def forward(self, x, bboxes=None, return_attn=False):
FILE: slowfast/utils/ava_eval_helper.py
function make_image_key (line 45) | def make_image_key(video_id, timestamp):
function read_csv (line 50) | def read_csv(csv_file, class_whitelist=None, load_score=False):
function read_exclusions (line 87) | def read_exclusions(exclusions_file):
function read_labelmap (line 105) | def read_labelmap(labelmap_file):
function evaluate_ava_from_files (line 123) | def evaluate_ava_from_files(labelmap, groundtruth, detections, exclusions):
function evaluate_ava (line 133) | def evaluate_ava(
function run_evaluation (line 168) | def run_evaluation(categories, groundtruth, detections, excluded_keys, v...
function get_ava_eval_data (line 235) | def get_ava_eval_data(
function write_results (line 274) | def write_results(detections, filename):
FILE: slowfast/utils/benchmark.py
function benchmark_data_loading (line 20) | def benchmark_data_loading(cfg):
FILE: slowfast/utils/bn_helper.py
function compute_and_update_bn_stats (line 12) | def compute_and_update_bn_stats(model, data_loader, num_batches=200):
FILE: slowfast/utils/c2_model_loading.py
function get_name_convert_func (line 9) | def get_name_convert_func():
FILE: slowfast/utils/checkpoint.py
function make_checkpoint_dir (line 22) | def make_checkpoint_dir(path_to_job):
function get_checkpoint_dir (line 38) | def get_checkpoint_dir(path_to_job):
function get_path_to_checkpoint (line 47) | def get_path_to_checkpoint(path_to_job, epoch, task=""):
function get_last_checkpoint (line 61) | def get_last_checkpoint(path_to_job, task):
function has_checkpoint (line 81) | def has_checkpoint(path_to_job):
function is_checkpoint_epoch (line 92) | def is_checkpoint_epoch(cfg, cur_epoch, multigrid_schedule=None):
function save_checkpoint (line 113) | def save_checkpoint(path_to_job, model, optimizer, epoch, cfg, scaler=No...
function inflate_weight (line 148) | def inflate_weight(state_dict_2d, state_dict_3d):
function load_checkpoint (line 181) | def load_checkpoint(
function sub_to_normal_bn (line 526) | def sub_to_normal_bn(sd):
function c2_normal_to_sub_bn (line 567) | def c2_normal_to_sub_bn(key, model_keys):
function normal_to_sub_bn (line 587) | def normal_to_sub_bn(checkpoint_sd, model_sd):
function load_test_checkpoint (line 625) | def load_test_checkpoint(cfg, model):
function load_train_checkpoint (line 663) | def load_train_checkpoint(cfg, model, optimizer, scaler=None):
FILE: slowfast/utils/distributed.py
function init_distributed_training (line 22) | def init_distributed_training(cfg):
function all_gather (line 26) | def all_gather(tensors):
function all_reduce (line 46) | def all_reduce(tensors, average=True):
function init_process_group (line 65) | def init_process_group(
function is_master_proc (line 103) | def is_master_proc(num_gpus=8):
function is_root_proc (line 113) | def is_root_proc():
function get_rank (line 123) | def get_rank():
function synchronize (line 134) | def synchronize():
function _get_global_gloo_group (line 150) | def _get_global_gloo_group():
function _serialize_to_tensor (line 163) | def _serialize_to_tensor(data, group):
function _pad_to_largest_tensor (line 191) | def _pad_to_largest_tensor(tensor, group):
function all_gather_unaligned (line 225) | def all_gather_unaligned(data, group=None):
class GatherLayer (line 264) | class GatherLayer(torch.autograd.Function):
method forward (line 268) | def forward(ctx, input):
method backward (line 275) | def backward(ctx, *grads):
class AllGatherWithGradient (line 282) | class AllGatherWithGradient(torch.autograd.Function):
method forward (line 286) | def forward(ctx, input):
method backward (line 294) | def backward(ctx, grad_output):
FILE: slowfast/utils/env.py
function setup_environment (line 13) | def setup_environment():
FILE: slowfast/utils/logging.py
function _suppress_print (line 19) | def _suppress_print():
function _cached_log_stream (line 31) | def _cached_log_stream(filename):
function setup_logging (line 38) | def setup_logging(output_dir=None):
function get_logger (line 75) | def get_logger(name):
function log_json_stats (line 85) | def log_json_stats(stats, output_dir=None):
FILE: slowfast/utils/lr_policy.py
function get_lr_at_epoch (line 9) | def get_lr_at_epoch(cfg, cur_epoch):
function lr_func_cosine (line 28) | def lr_func_cosine(cfg, cur_epoch):
function lr_func_steps_with_relative_lrs (line 52) | def lr_func_steps_with_relative_lrs(cfg, cur_epoch):
function get_step_index (line 65) | def get_step_index(cfg, cur_epoch):
function get_lr_func (line 80) | def get_lr_func(lr_policy):
FILE: slowfast/utils/meters.py
function get_ava_mini_groundtruth (line 28) | def get_ava_mini_groundtruth(full_groundtruth):
class AVAMeter (line 46) | class AVAMeter:
method __init__ (line 51) | def __init__(self, overall_iters, cfg, mode):
method log_iter_stats (line 88) | def log_iter_stats(self, cur_epoch, cur_iter):
method iter_tic (line 140) | def iter_tic(self):
method iter_toc (line 147) | def iter_toc(self):
method data_toc (line 154) | def data_toc(self):
method reset (line 158) | def reset(self):
method update_stats (line 168) | def update_stats(self, preds, ori_boxes, metadata, loss=None, lr=None):
method finalize_metrics (line 187) | def finalize_metrics(self, log=True):
method log_epoch_stats (line 220) | def log_epoch_stats(self, cur_epoch):
class TestMeter (line 239) | class TestMeter:
method __init__ (line 247) | def __init__(
method reset (line 295) | def reset(self):
method update_stats (line 305) | def update_stats(self, preds, labels, clip_ids):
method log_iter_stats (line 338) | def log_iter_stats(self, cur_iter):
method iter_tic (line 354) | def iter_tic(self):
method iter_toc (line 361) | def iter_toc(self):
method data_toc (line 368) | def data_toc(self):
method finalize_metrics (line 372) | def finalize_metrics(self, ks=(1, 5)):
class ScalarMeter (line 409) | class ScalarMeter:
method __init__ (line 416) | def __init__(self, window_size):
method reset (line 425) | def reset(self):
method add_value (line 433) | def add_value(self, value):
method get_win_median (line 441) | def get_win_median(self):
method get_current_value (line 447) | def get_current_value(self):
method get_win_avg (line 450) | def get_win_avg(self):
method get_global_avg (line 456) | def get_global_avg(self):
class ListMeter (line 463) | class ListMeter:
method __init__ (line 464) | def __init__(self, list_size):
method reset (line 473) | def reset(self):
method add_value (line 481) | def add_value(self, value):
method get_value (line 489) | def get_value(self):
method get_global_avg (line 492) | def get_global_avg(self):
class TrainMeter (line 499) | class TrainMeter:
method __init__ (line 504) | def __init__(self, epoch_iters, cfg):
method reset (line 530) | def reset(self):
method iter_tic (line 546) | def iter_tic(self):
method iter_toc (line 553) | def iter_toc(self):
method data_toc (line 560) | def data_toc(self):
method update_stats (line 564) | def update_stats(
method log_iter_stats (line 608) | def log_iter_stats(self, cur_epoch, cur_iter):
method log_epoch_stats (line 643) | def log_epoch_stats(self, cur_epoch):
class ValMeter (line 679) | class ValMeter:
method __init__ (line 684) | def __init__(self, max_iter, cfg):
method reset (line 709) | def reset(self):
method iter_tic (line 724) | def iter_tic(self):
method iter_toc (line 731) | def iter_toc(self):
method data_toc (line 738) | def data_toc(self):
method update_stats (line 742) | def update_stats(self, top1_err, top5_err, mb_size):
method update_predictions (line 756) | def update_predictions(self, preds, labels):
method log_iter_stats (line 767) | def log_iter_stats(self, cur_epoch, cur_iter):
method log_epoch_stats (line 791) | def log_epoch_stats(self, cur_epoch):
function get_map (line 823) | def get_map(preds, labels):
class EpochTimer (line 850) | class EpochTimer:
method __init__ (line 855) | def __init__(self) -> None:
method reset (line 860) | def reset(self) -> None:
method epoch_tic (line 867) | def epoch_tic(self):
method epoch_toc (line 873) | def epoch_toc(self):
method last_epoch_time (line 880) | def last_epoch_time(self):
method avg_epoch_time (line 888) | def avg_epoch_time(self):
method median_epoch_time (line 896) | def median_epoch_time(self):
FILE: slowfast/utils/metrics.py
function topks_correct (line 9) | def topks_correct(preds, labels, ks):
function topk_errors (line 43) | def topk_errors(preds, labels, ks):
function topk_accuracies (line 55) | def topk_accuracies(preds, labels, ks):
FILE: slowfast/utils/misc.py
function check_nan_losses (line 27) | def check_nan_losses(loss):
function params_count (line 37) | def params_count(model, ignore_bn=False):
function gpu_mem_usage (line 54) | def gpu_mem_usage():
function cpu_mem_usage (line 65) | def cpu_mem_usage():
function _get_model_analysis_input (line 79) | def _get_model_analysis_input(cfg, use_train_input):
function get_model_stats (line 138) | def get_model_stats(model, cfg, mode, use_train_input):
function log_model_info (line 173) | def log_model_info(model, cfg, use_train_input=True):
function is_eval_epoch (line 200) | def is_eval_epoch(cfg, cur_epoch, multigrid_schedule):
function plot_input (line 222) | def plot_input(tensor, bboxes=(), texts=(), path="./tmp_vis.png"):
function plot_input_normed (line 252) | def plot_input_normed(
function convert_normalized_images (line 352) | def convert_normalized_images(tensor):
function frozen_bn_stats (line 361) | def frozen_bn_stats(model):
function aggregate_sub_bn_stats (line 372) | def aggregate_sub_bn_stats(module):
function launch_job (line 390) | def launch_job(cfg, init_method, func, daemon=False):
function get_class_names (line 421) | def get_class_names(path, parent_path=None, subset_path=None):
FILE: slowfast/utils/multigrid.py
class MultigridSchedule (line 12) | class MultigridSchedule:
method init_multigrid (line 17) | def init_multigrid(self, cfg):
method update_long_cycle (line 58) | def update_long_cycle(self, cfg, cur_epoch):
method get_long_cycle_schedule (line 110) | def get_long_cycle_schedule(self, cfg):
function print_schedule (line 196) | def print_schedule(schedule):
function get_current_long_cycle_shape (line 205) | def get_current_long_cycle_shape(schedule, epoch):
FILE: slowfast/utils/multiprocessing.py
function run (line 9) | def run(
FILE: slowfast/utils/parser.py
function parse_args (line 13) | def parse_args():
function load_config (line 67) | def load_config(args, path_to_config=None):
FILE: slowfast/utils/weight_init_helper.py
function init_weights (line 10) | def init_weights(
FILE: slowfast/visualization/async_predictor.py
class AsycnActionPredictor (line 17) | class AsycnActionPredictor:
class _Predictor (line 18) | class _Predictor(mp.Process):
method __init__ (line 19) | def __init__(self, cfg, task_queue, result_queue, gpu_id=None):
method run (line 41) | def run(self):
method __init__ (line 54) | def __init__(self, cfg, result_queue=None):
method put (line 78) | def put(self, task):
method get (line 88) | def get(self):
method __call__ (line 106) | def __call__(self, task):
method shutdown (line 110) | def shutdown(self):
method result_available (line 115) | def result_available(self):
method default_buffer_size (line 122) | def default_buffer_size(self):
class AsyncVis (line 126) | class AsyncVis:
class _VisWorker (line 127) | class _VisWorker(mp.Process):
method __init__ (line 128) | def __init__(self, video_vis, task_queue, result_queue):
method run (line 141) | def run(self):
method __init__ (line 154) | def __init__(self, video_vis, n_workers=None):
method put (line 181) | def put(self, task):
method get (line 191) | def get(self):
method __call__ (line 211) | def __call__(self, task):
method shutdown (line 218) | def shutdown(self):
method result_available (line 223) | def result_available(self):
method default_buffer_size (line 227) | def default_buffer_size(self):
class _StopToken (line 231) | class _StopToken:
class AsyncDemo (line 235) | class AsyncDemo:
method __init__ (line 240) | def __init__(self, cfg, async_vis):
method put (line 250) | def put(self, task):
method get (line 260) | def get(self):
function draw_predictions (line 272) | def draw_predictions(task, video_vis):
FILE: slowfast/visualization/ava_demo_precomputed_boxes.py
class AVAVisualizerWithPrecomputedBox (line 24) | class AVAVisualizerWithPrecomputedBox:
method __init__ (line 30) | def __init__(self, cfg):
method get_output_file (line 77) | def get_output_file(self, path):
method get_input_clip (line 91) | def get_input_clip(self, keyframe_idx):
method get_predictions (line 122) | def get_predictions(self):
method draw_video (line 187) | def draw_video(self):
method __call__ (line 304) | def __call__(self):
method display (line 308) | def display(self, frame):
method _get_keyframe_clip (line 318) | def _get_keyframe_clip(self, keyframe_idx):
method _get_frame_range (line 330) | def _get_frame_range(self, start_idx, num_frames):
function merge_pred_gt_boxes (line 353) | def merge_pred_gt_boxes(pred_dict, gt_dict=None):
function load_boxes_labels (line 381) | def load_boxes_labels(cfg, video_name, fps, img_width, img_height):
FILE: slowfast/visualization/demo_loader.py
class VideoManager (line 17) | class VideoManager:
method __init__ (line 22) | def __init__(self, cfg):
method __iter__ (line 65) | def __iter__(self):
method __next__ (line 68) | def __next__(self):
method get_output_file (line 98) | def get_output_file(self, path, fps=30):
method display (line 113) | def display(self, task):
method clean (line 128) | def clean(self):
method start (line 138) | def start(self):
method join (line 141) | def join(self):
class ThreadVideoManager (line 145) | class ThreadVideoManager:
method __init__ (line 151) | def __init__(self, cfg):
method get_output_file (line 207) | def get_output_file(self, path, fps=30):
method __iter__ (line 222) | def __iter__(self):
method put_fn (line 225) | def put_fn(self):
method __next__ (line 262) | def __next__(self):
method get_fn (line 289) | def get_fn(self):
method display (line 319) | def display(self, task):
method start (line 329) | def start(self):
method join (line 344) | def join(self):
method clean (line 347) | def clean(self):
FILE: slowfast/visualization/gradcam_utils.py
class GradCAM (line 11) | class GradCAM:
method __init__ (line 18) | def __init__(self, model, target_layers, data_mean, data_std, colormap...
method _register_single_hook (line 42) | def _register_single_hook(self, layer_name):
method _register_hooks (line 60) | def _register_hooks(self):
method _calculate_localization_map (line 67) | def _calculate_localization_map(self, inputs, labels=None):
method __call__ (line 133) | def __call__(self, inputs, labels=None, alpha=0.5):
FILE: slowfast/visualization/prediction_vis.py
class WrongPredictionVis (line 15) | class WrongPredictionVis:
method __init__ (line 21) | def __init__(self, cfg):
method _pick_wrong_preds (line 45) | def _pick_wrong_preds(self, labels, preds):
method visualize_vid (line 71) | def visualize_vid(self, video_input, labels, preds, batch_idx):
method wrong_class_prediction (line 118) | def wrong_class_prediction(self):
method clean (line 127) | def clean(self):
FILE: slowfast/visualization/predictor.py
class Predictor (line 20) | class Predictor:
method __init__ (line 25) | def __init__(self, cfg, gpu_id=None):
method __call__ (line 47) | def __call__(self, task):
class ActionPredictor (line 114) | class ActionPredictor:
method __init__ (line 119) | def __init__(self, cfg, async_vis=None, gpu_id=None):
method put (line 130) | def put(self, task):
method get (line 141) | def get(self):
class Detectron2Predictor (line 153) | class Detectron2Predictor:
method __init__ (line 159) | def __init__(self, cfg, gpu_id=None):
method __call__ (line 180) | def __call__(self, task):
FILE: slowfast/visualization/tensorboard_vis.py
class TensorboardWriter (line 20) | class TensorboardWriter:
method __init__ (line 25) | def __init__(self, cfg):
method add_scalars (line 86) | def add_scalars(self, data_dict, global_step=None):
method plot_eval (line 97) | def plot_eval(self, preds, labels, global_step=None):
method add_video (line 167) | def add_video(self, vid_tensor, tag="Video Input", global_step=None, f...
method plot_weights_and_activations (line 179) | def plot_weights_and_activations(
method flush (line 223) | def flush(self):
method close (line 226) | def close(self):
function add_confusion_matrix (line 231) | def add_confusion_matrix(
function plot_hist (line 276) | def plot_hist(
function add_ndim_array (line 326) | def add_ndim_array(
function add_heatmap (line 399) | def add_heatmap(tensor):
FILE: slowfast/visualization/utils.py
function get_confusion_matrix (line 16) | def get_confusion_matrix(preds, labels, num_classes, normalize="true"):
function plot_confusion_matrix (line 48) | def plot_confusion_matrix(cmtx, num_classes, class_names=None, figsize=N...
function plot_topk_histogram (line 92) | def plot_topk_histogram(tag, array, k=10, class_names=None, figsize=None):
class GetWeightAndActivation (line 158) | class GetWeightAndActivation:
method __init__ (line 163) | def __init__(self, model, layers):
method _get_layer (line 178) | def _get_layer(self, layer_name):
method _register_single_hook (line 191) | def _register_single_hook(self, layer_name):
method _register_hooks (line 204) | def _register_hooks(self):
method get_activations (line 211) | def get_activations(self, input, bboxes=None):
method get_weights (line 236) | def get_weights(self):
function get_indexing (line 253) | def get_indexing(string):
function process_layer_index_data (line 276) | def process_layer_index_data(layer_ls, layer_name_prefix=""):
function process_cv2_inputs (line 302) | def process_cv2_inputs(frames, cfg):
function get_layer (line 323) | def get_layer(model, layer_name):
class TaskInfo (line 341) | class TaskInfo:
method __init__ (line 342) | def __init__(self):
method add_frames (line 353) | def add_frames(self, idx, frames):
method add_bboxes (line 363) | def add_bboxes(self, bboxes):
method add_action_preds (line 369) | def add_action_preds(self, preds):
FILE: slowfast/visualization/video_visualizer.py
function _create_text_labels (line 18) | def _create_text_labels(classes, scores, class_names, ground_truth=False):
class ImgVisualizer (line 43) | class ImgVisualizer(Visualizer):
method __init__ (line 44) | def __init__(self, img_rgb, meta, **kwargs):
method draw_text (line 59) | def draw_text(
method draw_multiple_text (line 107) | def draw_multiple_text(
method draw_multiple_text_upward (line 182) | def draw_multiple_text_upward(
method draw_multiple_text_downward (line 235) | def draw_multiple_text_downward(
method _align_x_coordinate (line 288) | def _align_x_coordinate(self, box_coordinate):
method _align_y_top (line 310) | def _align_y_top(self, box_coordinate, num_text, textbox_width):
method _align_y_bottom (line 328) | def _align_y_bottom(self, box_coordinate, num_text, textbox_width):
class VideoVisualizer (line 347) | class VideoVisualizer:
method __init__ (line 348) | def __init__(
method _get_color (line 392) | def _get_color(self, class_id):
method draw_one_frame (line 400) | def draw_one_frame(
method draw_clip_range (line 508) | def draw_clip_range(
method draw_clip (line 562) | def draw_clip(
method _adjust_frames_type (line 629) | def _adjust_frames_type(self, frames):
method _get_thres_array (line 651) | def _get_thres_array(self, common_class_names=None):
FILE: tools/benchmark.py
function main (line 15) | def main():
FILE: tools/demo_net.py
function run_demo (line 21) | def run_demo(cfg, frame_provider):
function demo (line 94) | def demo(cfg):
FILE: tools/run_net.py
function main (line 15) | def main():
FILE: tools/test_net.py
function perform_test (line 25) | def perform_test(test_loader, model, test_meter, cfg, writer=None):
function test (line 154) | def test(cfg):
FILE: tools/train_net.py
function train_epoch (line 33) | def train_epoch(
function eval_epoch (line 282) | def eval_epoch(val_loader, model, val_meter, cur_epoch, cfg, train_loade...
function calculate_and_update_precise_bn (line 425) | def calculate_and_update_precise_bn(loader, model, num_iters=200, use_gp...
function build_trainer (line 449) | def build_trainer(cfg):
function train (line 493) | def train(cfg):
FILE: tools/visualization.py
function run_visualization (line 29) | def run_visualization(vis_loader, model, cfg, writer=None):
function perform_wrong_prediction_vis (line 183) | def perform_wrong_prediction_vis(vis_loader, model, cfg):
function visualize (line 243) | def visualize(cfg):
Condensed preview — 220 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,309K chars).
[
{
"path": "CODE_OF_CONDUCT.md",
"chars": 244,
"preview": "# Code of Conduct\n\nFacebook has adopted a Code of Conduct that we expect project participants to adhere to.\nPlease read "
},
{
"path": "CONTRIBUTING.md",
"chars": 1305,
"preview": "# Contributing to PySlowFast\nWe want to make contributing to this project as easy and transparent as\npossible.\n\n## Pull "
},
{
"path": "GETTING_STARTED.md",
"chars": 2237,
"preview": "# Getting Started with PySlowFast\n\nThis document provides a brief intro of launching jobs in PySlowFast for training and"
},
{
"path": "INSTALL.md",
"chars": 2391,
"preview": "# Installation\n\n## Requirements\n- Python >= 3.8\n- Numpy\n- PyTorch >= 1.3\n- [fvcore](https://github.com/facebookresearch/"
},
{
"path": "LICENSE",
"chars": 10242,
"preview": "Apache License\nVersion 2.0, January 2004\nhttp://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AN"
},
{
"path": "MODEL_ZOO.md",
"chars": 9137,
"preview": "# PySlowFast Model Zoo and Baselines\n\n## Kinetics 400 and 600\n\n| architecture | size | crops x clips | frame length x "
},
{
"path": "README.md",
"chars": 5116,
"preview": "# PySlowFast\n\nPySlowFast is an open source video understanding codebase from FAIR that provides state-of-the-art video c"
},
{
"path": "VISUALIZATION_TOOLS.md",
"chars": 6309,
"preview": "# Visualization Tools for PySlowFast\n\nThis document provides a brief intro for running various visualization tools provi"
},
{
"path": "ava_evaluation/README.md",
"chars": 112,
"preview": "The code under this folder is from the official [ActivityNet repo](https://github.com/activitynet/ActivityNet).\n"
},
{
"path": "ava_evaluation/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt",
"chars": 2649,
"preview": "item {\n name: \"bend/bow (at the waist)\"\n id: 1\n}\nitem {\n name: \"crouch/kneel\"\n id: 3\n}\nitem {\n name: \"dance\"\n id: "
},
{
"path": "ava_evaluation/label_map_util.py",
"chars": 6329,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "ava_evaluation/metrics.py",
"chars": 5661,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "ava_evaluation/np_box_list.py",
"chars": 4970,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "ava_evaluation/np_box_list_ops.py",
"chars": 21514,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "ava_evaluation/np_box_mask_list.py",
"chars": 2625,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "ava_evaluation/np_box_mask_list_ops.py",
"chars": 16164,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "ava_evaluation/np_box_ops.py",
"chars": 3514,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "ava_evaluation/np_mask_ops.py",
"chars": 4444,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "ava_evaluation/object_detection_evaluation.py",
"chars": 33844,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "ava_evaluation/per_image_evaluation.py",
"chars": 19758,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "ava_evaluation/standard_fields.py",
"chars": 9726,
"preview": "# Copyright 2017 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"L"
},
{
"path": "configs/AVA/SLOWFAST_32x2_R50_SHORT.yaml",
"chars": 1885,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ava\n BATCH_SIZE: 64\n EVAL_PERIOD: 5\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n # CH"
},
{
"path": "configs/AVA/SLOW_8x8_R50_SHORT.yaml",
"chars": 1615,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ava\n BATCH_SIZE: 64\n EVAL_PERIOD: 5\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n # CH"
},
{
"path": "configs/AVA/c2/SLOWFAST_32x2_R101_50_50.yaml",
"chars": 1570,
"preview": "TRAIN:\n ENABLE: False\n DATASET: ava\n BATCH_SIZE: 16\n EVAL_PERIOD: 1\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n # C"
},
{
"path": "configs/AVA/c2/SLOWFAST_32x2_R101_50_50_v2.1.yaml",
"chars": 1772,
"preview": "TRAIN:\n ENABLE: False\n DATASET: ava\n BATCH_SIZE: 16\n EVAL_PERIOD: 1\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n # C"
},
{
"path": "configs/AVA/c2/SLOWFAST_32x2_R50.yaml",
"chars": 1559,
"preview": "TRAIN:\n ENABLE: False\n DATASET: ava\n BATCH_SIZE: 16\n EVAL_PERIOD: 1\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n # C"
},
{
"path": "configs/AVA/c2/SLOWFAST_64x2_R101_50_50.yaml",
"chars": 1569,
"preview": "TRAIN:\n ENABLE: False\n DATASET: ava\n BATCH_SIZE: 16\n EVAL_PERIOD: 1\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n # C"
},
{
"path": "configs/AVA/c2/SLOW_8x8_R50.yaml",
"chars": 1289,
"preview": "TRAIN:\n ENABLE: False\n DATASET: ava\n BATCH_SIZE: 64\n EVAL_PERIOD: 1\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n # C"
},
{
"path": "configs/Charades/SLOWFAST_16x8_R50.yaml",
"chars": 1700,
"preview": "TRAIN:\n ENABLE: True\n DATASET: charades\n BATCH_SIZE: 16\n EVAL_PERIOD: 6\n CHECKPOINT_PERIOD: 6\n AUTO_RESUME: True\n "
},
{
"path": "configs/Charades/SLOWFAST_16x8_R50_multigrid.yaml",
"chars": 1750,
"preview": "MULTIGRID:\n SHORT_CYCLE: True\n LONG_CYCLE: True\nTRAIN:\n ENABLE: True\n DATASET: charades\n BATCH_SIZE: 16\n EVAL_PERI"
},
{
"path": "configs/Charades/pytorchvideo/SLOWFAST_8x8_R50.yaml",
"chars": 1721,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvcharades\n BATCH_SIZE: 64\n EVAL_PERIOD: 6\n CHECKPOINT_PERIOD: 6\n AUTO_RESUME: Tru"
},
{
"path": "configs/Charades/pytorchvideo/SLOW_8x8_R50.yaml",
"chars": 1368,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvcharades\n BATCH_SIZE: 64\n EVAL_PERIOD: 6\n CHECKPOINT_PERIOD: 6\n AUTO_RESUME: Tru"
},
{
"path": "configs/ImageNet/MVIT_B_16_CONV.yaml",
"chars": 1659,
"preview": "TRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 256\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/ImageNet/MVITv2_B.yaml",
"chars": 2105,
"preview": "TRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 256\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/ImageNet/MVITv2_S.yaml",
"chars": 2198,
"preview": "TRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 256\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/ImageNet/MVITv2_T.yaml",
"chars": 2016,
"preview": "TRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 256\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/ImageNet/RES_R50.yaml",
"chars": 1267,
"preview": "TRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 256\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/ImageNet/REV_MVIT_B_16_CONV.yaml",
"chars": 1822,
"preview": "TRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 256\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/ImageNet/REV_VIT_B.yaml",
"chars": 1644,
"preview": "TRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 512\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/ImageNet/REV_VIT_S.yaml",
"chars": 1587,
"preview": "TRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 512\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/C2D_8x8_R50.yaml",
"chars": 1043,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/C2D_8x8_R50_IN1K.yaml",
"chars": 1216,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/C2D_NLN_8x8_R50.yaml",
"chars": 1054,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/C2D_NLN_8x8_R50_IN1K.yaml",
"chars": 1227,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/I3D_8x8_R101.yaml",
"chars": 1045,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/I3D_8x8_R50.yaml",
"chars": 1043,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/I3D_8x8_R50_IN1K.yaml",
"chars": 1216,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/I3D_NLN_8x8_R101.yaml",
"chars": 1056,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/I3D_NLN_8x8_R50.yaml",
"chars": 1054,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/I3D_NLN_8x8_R50_IN1K.yaml",
"chars": 1227,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/MVIT_B_16x4_CONV.yaml",
"chars": 1929,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 16\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/MVIT_B_32x3_CONV.yaml",
"chars": 1828,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 16\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/MVITv2_B_32x3.yaml",
"chars": 2325,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 16\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/MVITv2_L_40x3_test.yaml",
"chars": 2403,
"preview": "TRAIN:\n ENABLE: False\nDATA:\n USE_OFFSET_SAMPLING: True\n DECODING_BACKEND: torchvision\n NUM_FRAMES: 40\n SAMPLING_RAT"
},
{
"path": "configs/Kinetics/MVITv2_S_16x4.yaml",
"chars": 2205,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 16\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/REV_MVIT_B_16x4_CONV.yaml",
"chars": 1967,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 16\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/SLOWFAST_4x16_R50.yaml",
"chars": 1292,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/SLOWFAST_8x8_R50.yaml",
"chars": 1292,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/SLOWFAST_8x8_R50_stepwise.yaml",
"chars": 1382,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/SLOWFAST_8x8_R50_stepwise_multigrid.yaml",
"chars": 1432,
"preview": "MULTIGRID:\n SHORT_CYCLE: True\n LONG_CYCLE: True\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERI"
},
{
"path": "configs/Kinetics/SLOWFAST_NLN_4x16_R50.yaml",
"chars": 1303,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/SLOWFAST_NLN_8x8_R50.yaml",
"chars": 1303,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/SLOW_4x16_R50.yaml",
"chars": 1049,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/SLOW_8x8_R50.yaml",
"chars": 1048,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/SLOW_NLN_4x16_R50.yaml",
"chars": 1060,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/SLOW_NLN_8x8_R50.yaml",
"chars": 1059,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n"
},
{
"path": "configs/Kinetics/X3D_L.yaml",
"chars": 1277,
"preview": "TRAIN:\n # ENABLE: False # default True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n "
},
{
"path": "configs/Kinetics/X3D_M.yaml",
"chars": 1277,
"preview": "TRAIN:\n # ENABLE: False # default True\n DATASET: kinetics\n BATCH_SIZE: 128\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n"
},
{
"path": "configs/Kinetics/X3D_S.yaml",
"chars": 1278,
"preview": "TRAIN:\n # ENABLE: False # default True\n DATASET: kinetics\n BATCH_SIZE: 128\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n"
},
{
"path": "configs/Kinetics/X3D_XS.yaml",
"chars": 1278,
"preview": "TRAIN:\n # ENABLE: False # default True\n DATASET: kinetics\n BATCH_SIZE: 128\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n"
},
{
"path": "configs/Kinetics/c2/C2D_NOPOOL_8x8_R50.yaml",
"chars": 1129,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/I3D_8x8_R50.yaml",
"chars": 1122,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/I3D_NLN_8x8_R50.yaml",
"chars": 1133,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/SLOWFAST_16x8_R101_50_50.yaml",
"chars": 1372,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/SLOWFAST_4x16_R50.yaml",
"chars": 1371,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/SLOWFAST_8x8_R101_101_101.yaml",
"chars": 1374,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/SLOWFAST_8x8_R101_50_101.yaml",
"chars": 1373,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/SLOWFAST_8x8_R101_50_50.yaml",
"chars": 1372,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/SLOWFAST_8x8_R50.yaml",
"chars": 1371,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/SLOWFAST_NLN_16x8_R101_50_50.yaml",
"chars": 1381,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/SLOW_4x16_R50.yaml",
"chars": 1128,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/c2/SLOW_8x8_R50.yaml",
"chars": 1127,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/pytorchvideo/C2D_8x8_R50.yaml",
"chars": 1149,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: Tr"
},
{
"path": "configs/Kinetics/pytorchvideo/CSN_32x2_R101.yaml",
"chars": 1064,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: Tr"
},
{
"path": "configs/Kinetics/pytorchvideo/I3D_8x8_R50.yaml",
"chars": 1151,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: Tr"
},
{
"path": "configs/Kinetics/pytorchvideo/MVIT_B_16x4_CONV.yaml",
"chars": 1920,
"preview": "TRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 16\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: True"
},
{
"path": "configs/Kinetics/pytorchvideo/R2PLUS1D_16x4_R50.yaml",
"chars": 1078,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: Tr"
},
{
"path": "configs/Kinetics/pytorchvideo/SLOWFAST_16x8_R101_50_50.yaml",
"chars": 1395,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 2\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: Tr"
},
{
"path": "configs/Kinetics/pytorchvideo/SLOWFAST_4x16_R50.yaml",
"chars": 1396,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: Tr"
},
{
"path": "configs/Kinetics/pytorchvideo/SLOWFAST_8x8_R101.yaml",
"chars": 1399,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 2\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: Tr"
},
{
"path": "configs/Kinetics/pytorchvideo/SLOWFAST_8x8_R50.yaml",
"chars": 1396,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: Tr"
},
{
"path": "configs/Kinetics/pytorchvideo/SLOW_4x16_R50.yaml",
"chars": 1153,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: Tr"
},
{
"path": "configs/Kinetics/pytorchvideo/SLOW_8x8_R50.yaml",
"chars": 1152,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: Tr"
},
{
"path": "configs/Kinetics/pytorchvideo/X3D_L.yaml",
"chars": 1107,
"preview": "TRAIN:\n DATASET: ptvkinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 2\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: True\nX3D:\n WIDTH"
},
{
"path": "configs/Kinetics/pytorchvideo/X3D_M.yaml",
"chars": 1256,
"preview": "TRAIN:\n # ENABLE: False # default True\n DATASET: ptvkinetics\n BATCH_SIZE: 128\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: "
},
{
"path": "configs/Kinetics/pytorchvideo/X3D_S.yaml",
"chars": 1291,
"preview": "TRAIN:\n # ENABLE: False # default True\n DATASET: ptvkinetics\n BATCH_SIZE: 128\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: "
},
{
"path": "configs/Kinetics/pytorchvideo/X3D_XS.yaml",
"chars": 1256,
"preview": "TRAIN:\n # ENABLE: False # default True\n DATASET: ptvkinetics\n BATCH_SIZE: 128\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: "
},
{
"path": "configs/SSv2/MVITv2_B_32x3.yaml",
"chars": 2399,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ssv2\n BATCH_SIZE: 16\n EVAL_PERIOD: 2\n CHECKPOINT_PERIOD: 2\n AUTO_RESUME: True\n # C"
},
{
"path": "configs/SSv2/MVITv2_L_40x3.yaml",
"chars": 3072,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ssv2\n BATCH_SIZE: 8\n EVAL_PERIOD: 2\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n #CHE"
},
{
"path": "configs/SSv2/MVITv2_S_16x4.yaml",
"chars": 2220,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ssv2\n BATCH_SIZE: 16\n EVAL_PERIOD: 2\n CHECKPOINT_PERIOD: 2\n AUTO_RESUME: True\n # C"
},
{
"path": "configs/SSv2/SLOWFAST_16x8_R50.yaml",
"chars": 1652,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ssv2\n BATCH_SIZE: 16\n EVAL_PERIOD: 2\n CHECKPOINT_PERIOD: 2\n AUTO_RESUME: True\n CHE"
},
{
"path": "configs/SSv2/SLOWFAST_16x8_R50_multigrid.yaml",
"chars": 1702,
"preview": "MULTIGRID:\n SHORT_CYCLE: True\n LONG_CYCLE: True\nTRAIN:\n ENABLE: True\n DATASET: ssv2\n BATCH_SIZE: 16\n EVAL_PERIOD: "
},
{
"path": "configs/SSv2/pytorchvideo/SLOWFAST_8x8_R50.yaml",
"chars": 1648,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvssv2\n BATCH_SIZE: 64\n EVAL_PERIOD: 2\n CHECKPOINT_PERIOD: 2\n AUTO_RESUME: True\n "
},
{
"path": "configs/SSv2/pytorchvideo/SLOW_8x8_R50.yaml",
"chars": 1299,
"preview": "TRAIN:\n ENABLE: True\n DATASET: ptvssv2\n BATCH_SIZE: 64\n EVAL_PERIOD: 2\n CHECKPOINT_PERIOD: 2\n AUTO_RESUME: True\n "
},
{
"path": "configs/contrastive_ssl/BYOL_SlowR50_8x8.yaml",
"chars": 1789,
"preview": "TASK: ssl\nTRAIN:\n DATASET: kinetics\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 5\n AUTO_RESUME: True\nMODEL:\n NUM_CLASSES: "
},
{
"path": "configs/contrastive_ssl/MoCo_SlowR50_8x8.yaml",
"chars": 1483,
"preview": "TASK: ssl\nTRAIN:\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 5\n AUTO_RESUME: True\n MIX"
},
{
"path": "configs/contrastive_ssl/SimCLR_SlowR50_8x8.yaml",
"chars": 1598,
"preview": "TASK: ssl\nTRAIN:\n DATASET: kinetics\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: True\nMODEL:\n NUM_CLASSES:"
},
{
"path": "configs/contrastive_ssl/SwAV_Slow_R50_8x8.yaml",
"chars": 1487,
"preview": "TASK: ssl\nTRAIN:\n DATASET: kinetics\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: True\nMODEL:\n NUM_CLASSES:"
},
{
"path": "configs/contrastive_ssl/finetune_SSv2_Slow_R50_syn0.yaml",
"chars": 1181,
"preview": "TASK: ssl_eval_ssv2\nTRAIN:\n ENABLE: True\n DATASET: ssv2\n BATCH_SIZE: 64\n EVAL_PERIOD: 4\n CHECKPOINT_PERIOD: 4\n AUT"
},
{
"path": "configs/contrastive_ssl/finetune_SSv2_Slow_R50_syn8.yaml",
"chars": 1262,
"preview": "TASK: ssl_eval_ssv2\nTRAIN:\n ENABLE: True\n DATASET: ssv2\n BATCH_SIZE: 64\n EVAL_PERIOD: 4\n CHECKPOINT_PERIOD: 4\n AUT"
},
{
"path": "configs/contrastive_ssl/finetune_ava_Slow_R50_syn0.yaml",
"chars": 1601,
"preview": "TASK: ssl_eval_ava\nTRAIN:\n DATASET: ava\n BATCH_SIZE: 64\n EVAL_PERIOD: 4\n CHECKPOINT_PERIOD: 4\n AUTO_RESUME: True\n "
},
{
"path": "configs/contrastive_ssl/finetune_ava_Slow_R50_syn8.yaml",
"chars": 1690,
"preview": "TASK: ssl_eval_ava\nTRAIN:\n DATASET: ava\n BATCH_SIZE: 64\n EVAL_PERIOD: 4\n CHECKPOINT_PERIOD: 4\n AUTO_RESUME: True\n "
},
{
"path": "configs/contrastive_ssl/finetune_ucf_Slow_R50_syn0.yaml",
"chars": 1124,
"preview": "TASK: ssl_eval_ucf\nTRAIN:\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 20\n CHECKPOINT_PERIOD: 20\n AUTO_RESUME: "
},
{
"path": "configs/contrastive_ssl/finetune_ucf_Slow_R50_syn8.yaml",
"chars": 1189,
"preview": "TASK: ssl_eval_ucf\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 20\n CHECKPOINT_PERIOD: 10\n"
},
{
"path": "configs/contrastive_ssl/linear_k400_Slow_8x8_R50_syn0.yaml",
"chars": 1002,
"preview": "TASK: ssl_eval_k400\nTRAIN:\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 20\n CHECKPOINT_PERIOD: 20\n AUTO_RESUME:"
},
{
"path": "configs/contrastive_ssl/linear_k400_Slow_8x8_R50_syn8.yaml",
"chars": 1099,
"preview": "TASK: ssl_eval_k400\nTRAIN:\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 20\n CHECKPOINT_PERIOD: 20\n AUTO_RESUME:"
},
{
"path": "configs/masked_ssl/in1k_VIT_B_MaskFeat_FT.yaml",
"chars": 1583,
"preview": "TASK: ssl_eval\nTRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 512\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 10\n C"
},
{
"path": "configs/masked_ssl/in1k_VIT_B_MaskFeat_PT.yaml",
"chars": 1521,
"preview": "TASK: ssl\nTRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 256\n EVAL_PERIOD: 10000\n CHECKPOINT_PERIOD: 10\n AUT"
},
{
"path": "configs/masked_ssl/in1k_VIT_L_MaskFeat_FT.yaml",
"chars": 1597,
"preview": "TRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 256\n EVAL_PERIOD: 1\n CHECKPOINT_PERIOD: 5\n CHECKPOINT_EPOCH_R"
},
{
"path": "configs/masked_ssl/in1k_VIT_L_MaskFeat_PT.yaml",
"chars": 1626,
"preview": "TRAIN:\n ENABLE: True\n DATASET: imagenet\n BATCH_SIZE: 256\n EVAL_PERIOD: 10000\n CHECKPOINT_PERIOD: 10\n AUTO_RESUME: "
},
{
"path": "configs/masked_ssl/k400_MVITv2_L_16x4_FT.yaml",
"chars": 3070,
"preview": "TASK: ssl_eval\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 16\n EVAL_PERIOD: 5\n CHECKPOINT_PERIOD: 5\n AUTO"
},
{
"path": "configs/masked_ssl/k400_MVITv2_L_16x4_MaskFeat_PT.yaml",
"chars": 2867,
"preview": "TASK: ssl\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 32\n EVAL_PERIOD: 100000\n CHECKPOINT_PERIOD: 5\n AUTO"
},
{
"path": "configs/masked_ssl/k400_MVITv2_S_16x4_FT.yaml",
"chars": 2584,
"preview": "TASK: ssl_eval\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 16\n EVAL_PERIOD: 10\n CHECKPOINT_EPOCH_RESET: Tr"
},
{
"path": "configs/masked_ssl/k400_MVITv2_S_16x4_MaskFeat_PT.yaml",
"chars": 2365,
"preview": "TASK: ssl\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 32\n EVAL_PERIOD: 1000000\n CHECKPOINT_PERIOD: 10\n AU"
},
{
"path": "configs/masked_ssl/k400_VIT_B_16x4_FT.yaml",
"chars": 1866,
"preview": "TASK: ssl_eval\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 1\n EVAL_PERIOD: 1\n CHECKPOINT_PERIOD: 5\n AUTO_"
},
{
"path": "configs/masked_ssl/k400_VIT_B_16x4_MAE_PT.yaml",
"chars": 1877,
"preview": "TASK: ssl\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 100000\n CHECKPOINT_PERIOD: 5\n AUTO"
},
{
"path": "configs/masked_ssl/k400_VIT_H_16x4_FT.yaml",
"chars": 1802,
"preview": "TASK: ssl_eval\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 16\n EVAL_PERIOD: 5\n CHECKPOINT_PERIOD: 5\n AUTO"
},
{
"path": "configs/masked_ssl/k400_VIT_H_16x4_MAE_PT.yaml",
"chars": 1889,
"preview": "TASK: ssl\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 32\n EVAL_PERIOD: 100000\n CHECKPOINT_PERIOD: 5\n AUTO"
},
{
"path": "configs/masked_ssl/k400_VIT_L_16x4_FT.yaml",
"chars": 1820,
"preview": "TASK: ssl_eval\nTRAIN:\n DATASET: kinetics\n BATCH_SIZE: 16\n EVAL_PERIOD: 5\n CHECKPOINT_PERIOD: 5\n AUTO_RESUME: True\n "
},
{
"path": "configs/masked_ssl/k400_VIT_L_16x4_MAE_PT.yaml",
"chars": 1956,
"preview": "TASK: ssl\nTRAIN:\n ENABLE: True\n DATASET: kinetics\n BATCH_SIZE: 32 # 32 fits 32g\n EVAL_PERIOD: 100000\n CHECKPOINT_PE"
},
{
"path": "demo/AVA/SLOWFAST_32x2_R101_50_50.yaml",
"chars": 1899,
"preview": "TRAIN:\n ENABLE: False\n DATASET: ava\n BATCH_SIZE: 16\n EVAL_PERIOD: 1\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True\n CHE"
},
{
"path": "demo/Kinetics/SLOWFAST_8x8_R50.yaml",
"chars": 1468,
"preview": "TRAIN:\n ENABLE: False\n DATASET: kinetics\n BATCH_SIZE: 64\n EVAL_PERIOD: 10\n CHECKPOINT_PERIOD: 1\n AUTO_RESUME: True"
},
{
"path": "linter.sh",
"chars": 344,
"preview": "#!/bin/bash -e\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n# Run this script at project root"
},
{
"path": "projects/contrastive_ssl/README.md",
"chars": 3397,
"preview": "# A Large-Scale Study on Unsupervised Spatiotemporal Representation Learning\n[Christoph Feichtenhofer](http://feichtenho"
},
{
"path": "projects/mae/README.md",
"chars": 2387,
"preview": "# Masked Autoencoders As Spatiotemporal Learners\n[Christoph Feichtenhofer*](http://feichtenhofer.github.io/), [Haoqi Fan"
},
{
"path": "projects/maskfeat/README.md",
"chars": 3525,
"preview": "# Masked Feature Prediction for Self-Supervised Visual Pre-Training\n[Chen Wei*](https://weichen582.github.io/), [Haoqi F"
},
{
"path": "projects/multigrid/README.md",
"chars": 1740,
"preview": "# A Multigrid Method for Efficiently Training Video Models\n[Chao-Yuan Wu](https://www.cs.utexas.edu/~cywu/),\n[Ross Girsh"
},
{
"path": "projects/mvit/README.md",
"chars": 1431,
"preview": "# Multiscale Vision Transformers\n[Haoqi Fan](https://haoqifan.github.io/)\\*, [Bo Xiong](https://www.cs.utexas.edu/~bxion"
},
{
"path": "projects/mvitv2/README.md",
"chars": 4274,
"preview": "# [MViTv2: Improved Multiscale Vision Transformers for Classification and Detection](https://arxiv.org/abs/2112.01526)\n\n"
},
{
"path": "projects/pytorchvideo/README.md",
"chars": 8408,
"preview": "# Support PyTorchVideo in PySlowFast\n\n## Introduction\n\n[PyTorchVideo](https://pytorchvideo.org/) is a new deeplearning l"
},
{
"path": "projects/rev/README.md",
"chars": 3584,
"preview": "# Reversible Vision Transformers\n\nOfficial PyTorch implementation of **Rev-ViT** and **Rev-MViT** models, from the follo"
},
{
"path": "projects/x3d/README.md",
"chars": 1457,
"preview": "# X3D: Progressive Network Expansion for Efficient Video Recognition\n[Christoph Feichtenhofer](http://feichtenhofer.gith"
},
{
"path": "setup.cfg",
"chars": 767,
"preview": "[isort]\nline_length=100\nmulti_line_output=4\nknown_standard_library=numpy,setuptools\nknown_myself=slowfast\nknown_third_pa"
},
{
"path": "setup.py",
"chars": 792,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nfrom setuptools import f"
},
{
"path": "slowfast/__init__.py",
"chars": 166,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nfrom slowfast.utils.env "
},
{
"path": "slowfast/config/__init__.py",
"chars": 95,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n"
},
{
"path": "slowfast/config/custom_config.py",
"chars": 217,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Add custom configs an"
},
{
"path": "slowfast/config/defaults.py",
"chars": 40938,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Configs.\"\"\"\n\nimport m"
},
{
"path": "slowfast/datasets/DATASET.md",
"chars": 6389,
"preview": "# Dataset Preparation\n\n## Kinetics\n\nThe Kinetics Dataset could be downloaded via the code released by ActivityNet:\n\n1. D"
},
{
"path": "slowfast/datasets/__init__.py",
"chars": 498,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nfrom .ava_dataset import"
},
{
"path": "slowfast/datasets/ava_dataset.py",
"chars": 15011,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport logging\n\nimport n"
},
{
"path": "slowfast/datasets/ava_helper.py",
"chars": 8029,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport logging\nimport os"
},
{
"path": "slowfast/datasets/build.py",
"chars": 1120,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nfrom fvcore.common.regis"
},
{
"path": "slowfast/datasets/charades.py",
"chars": 9643,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport os\nimport random\n"
},
{
"path": "slowfast/datasets/cv2_transform.py",
"chars": 26217,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport math\n\nimport cv2\n"
},
{
"path": "slowfast/datasets/decoder.py",
"chars": 23964,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport logging\nimport ma"
},
{
"path": "slowfast/datasets/imagenet.py",
"chars": 10416,
"preview": "# Copyright (c) Facebook, Inc. and its affiliates.\n\nimport json\nimport os\nimport random\nimport re\n\nimport numpy as np\nim"
},
{
"path": "slowfast/datasets/kinetics.py",
"chars": 22595,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport os\nimport random\n"
},
{
"path": "slowfast/datasets/loader.py",
"chars": 7723,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Data loader.\"\"\"\n\nimpo"
},
{
"path": "slowfast/datasets/mixup.py",
"chars": 6786,
"preview": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"\nThis implementation is based on\nhttps://git"
},
{
"path": "slowfast/datasets/multigrid_helper.py",
"chars": 2702,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Helper functions for "
},
{
"path": "slowfast/datasets/ptv_datasets.py",
"chars": 19087,
"preview": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport functools\nimport os\nfrom typing import D"
},
{
"path": "slowfast/datasets/rand_augment.py",
"chars": 16062,
"preview": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"\nThis implementation is based on\nhttps://git"
},
{
"path": "slowfast/datasets/random_erasing.py",
"chars": 6775,
"preview": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"\nThis implementation is based on\nhttps://git"
},
{
"path": "slowfast/datasets/ssv2.py",
"chars": 11840,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport json\nimport os\nim"
},
{
"path": "slowfast/datasets/transform.py",
"chars": 40041,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport logging\nimport ma"
},
{
"path": "slowfast/datasets/utils.py",
"chars": 15076,
"preview": "#!/usr/bin/env python3\n\nimport logging\nimport os\nimport random\nimport time\nfrom collections import defaultdict\n\nimport c"
},
{
"path": "slowfast/datasets/video_container.py",
"chars": 1033,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport av\n\n\ndef get_vide"
},
{
"path": "slowfast/models/__init__.py",
"chars": 532,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nfrom .build import build"
},
{
"path": "slowfast/models/attention.py",
"chars": 17412,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\nimport numpy\nimport tor"
},
{
"path": "slowfast/models/batchnorm_helper.py",
"chars": 3830,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"BatchNorm (BN) utilit"
},
{
"path": "slowfast/models/build.py",
"chars": 2751,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Model construction fu"
},
{
"path": "slowfast/models/common.py",
"chars": 4797,
"preview": "# Copyright (c) Facebook, Inc. and its affiliates.\n\nimport torch\nimport torch.nn as nn\n\n\nclass Mlp(nn.Module):\n def _"
},
{
"path": "slowfast/models/contrastive.py",
"chars": 42268,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport math\n\nimport nump"
},
{
"path": "slowfast/models/custom_video_model_builder.py",
"chars": 133,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\n\"\"\"A More Flexible Vide"
},
{
"path": "slowfast/models/head_helper.py",
"chars": 24722,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"ResNe(X)t Head helper"
},
{
"path": "slowfast/models/losses.py",
"chars": 2371,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Loss functions.\"\"\"\n\nf"
},
{
"path": "slowfast/models/masked.py",
"chars": 24454,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport math\nfrom functoo"
},
{
"path": "slowfast/models/nonlocal_helper.py",
"chars": 5362,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Non-local helper\"\"\"\n\n"
},
{
"path": "slowfast/models/operators.py",
"chars": 4165,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Custom operators.\"\"\"\n"
},
{
"path": "slowfast/models/optimizer.py",
"chars": 13440,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Optimizer.\"\"\"\n\nimport"
},
{
"path": "slowfast/models/ptv_model_builder.py",
"chars": 26685,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\n\"\"\"Video models using P"
},
{
"path": "slowfast/models/resnet_helper.py",
"chars": 24858,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Video models.\"\"\"\n\nimp"
},
{
"path": "slowfast/models/reversible_mvit.py",
"chars": 20368,
"preview": "import sys\nfrom functools import partial\n\nimport torch\nfrom slowfast.models.attention import attention_pool, MultiScaleA"
},
{
"path": "slowfast/models/stem_helper.py",
"chars": 10815,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"ResNe(X)t 3D stem hel"
},
{
"path": "slowfast/models/utils.py",
"chars": 7694,
"preview": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport numpy as np\nimport slowfast.utils.loggin"
},
{
"path": "slowfast/models/video_model_builder.py",
"chars": 46829,
"preview": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\n\"\"\"Video models.\"\"\"\n\nimport math\nfrom functool"
},
{
"path": "slowfast/utils/__init__.py",
"chars": 95,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n"
},
{
"path": "slowfast/utils/ava_eval_helper.py",
"chars": 9756,
"preview": "# Copyright (c) Facebook, Inc. and its affiliates.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n#"
},
{
"path": "slowfast/utils/benchmark.py",
"chars": 3276,
"preview": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\"\"\"\nFunctions for benchmarks.\n\"\"\"\n\nimport pprint\n"
},
{
"path": "slowfast/utils/bn_helper.py",
"chars": 2841,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"bn helper.\"\"\"\n\nimport"
},
{
"path": "slowfast/utils/c2_model_loading.py",
"chars": 5005,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Caffe2 to PyTorch che"
},
{
"path": "slowfast/utils/checkpoint.py",
"chars": 29086,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Functions that handle"
},
{
"path": "slowfast/utils/distributed.py",
"chars": 9129,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Distributed helpers.\""
},
{
"path": "slowfast/utils/env.py",
"chars": 445,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Set up Environment.\"\""
},
{
"path": "slowfast/utils/logging.py",
"chars": 3187,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Logging.\"\"\"\n\nimport a"
},
{
"path": "slowfast/utils/lr_policy.py",
"chars": 3060,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Learning rate policy."
},
{
"path": "slowfast/utils/meters.py",
"chars": 29554,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Meters.\"\"\"\n\nimport da"
},
{
"path": "slowfast/utils/metrics.py",
"chars": 2383,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\n\"\"\"Functions for computi"
},
{
"path": "slowfast/utils/misc.py",
"chars": 16403,
"preview": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n\nimport json\nimport math\n"
}
]
// ... and 20 more files (download for full content)
About this extraction
This page contains the full source code of the facebookresearch/SlowFast GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 220 files (1.2 MB), approximately 338.0k tokens, and a symbol index with 891 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.