[ { "path": ".gitignore", "content": "**/__pycache__/\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as\ncontributors and maintainers pledge to make participation in our project and\nour community a harassment-free experience for everyone, regardless of age, body\nsize, disability, ethnicity, sex characteristics, gender identity and expression,\nlevel of experience, education, socio-economic status, nationality, personal\nappearance, race, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment\ninclude:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or\nadvances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic\naddress, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\nprofessional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable\nbehavior and are expected to take appropriate and fair corrective action in\nresponse to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or\nreject comments, commits, code, wiki edits, issues, and other contributions\nthat are not aligned to this Code of Conduct, or to ban temporarily or\npermanently any contributor for other behaviors that they deem inappropriate,\nthreatening, offensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies within all project spaces, and it also applies when\nan individual is representing the project or its community in public spaces.\nExamples of representing a project or community include using an official\nproject e-mail address, posting via an official social media account, or acting\nas an appointed representative at an online or offline event. Representation of\na project may be further defined and clarified by project maintainers.\n\nThis Code of Conduct also applies outside the project spaces when there is a\nreasonable belief that an individual's behavior may have a negative impact on\nthe project or its community.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported by contacting the project team at . All\ncomplaints will be reviewed and investigated and will result in a response that\nis deemed necessary and appropriate to the circumstances. The project team is\nobligated to maintain confidentiality with regard to the reporter of an incident.\nFurther details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good\nfaith may face temporary or permanent repercussions as determined by other\nmembers of the project's leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,\navailable at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html\n\n[homepage]: https://www.contributor-covenant.org\n\nFor answers to common questions about this code of conduct, see\nhttps://www.contributor-covenant.org/faq" }, { "path": "CONTRIBUTING.md", "content": "# Contributing to Adaptive Teacher\nWe want to make contributing to this project as easy and transparent as\npossible.\n\n## Our Development Process\nTo be added.\n\n## Pull Requests\nWe actively welcome your pull requests.\n\n1. Fork the repo and create your branch from `main`.\n2. If you've added code that should be tested, add tests.\n3. If you've changed APIs, update the documentation.\n4. Ensure the test suite passes.\n5. Make sure your code lints.\n6. If you haven't already, complete the Contributor License Agreement (\"CLA\").\n\n## Contributor License Agreement (\"CLA\")\nIn order to accept your pull request, we need you to submit a CLA. You only need\nto do this once to work on any of Meta's open source projects.\n\nComplete your CLA here: \n\n## Issues\nWe use GitHub issues to track public bugs. Please ensure your description is\nclear and has sufficient instructions to be able to reproduce the issue.\n\nMeta has a [bounty program](https://www.facebook.com/whitehat/) for the safe\ndisclosure of security bugs. In those cases, please go through the process\noutlined on that page and do not file a public issue.\n\n## Coding Style \n* 2 spaces for indentation rather than tabs\n* 80 character line length\n* ...\n\n## License\nBy contributing to Adaptive Teacher, you agree that your contributions will be licensed\nunder the LICENSE file in the root directory of this source tree." }, { "path": "LICENSE", "content": "\nAttribution-NonCommercial 4.0 International\n\n=======================================================================\n\nCreative Commons Corporation (\"Creative Commons\") is not a law firm and\ndoes not provide legal services or legal advice. Distribution of\nCreative Commons public licenses does not create a lawyer-client or\nother relationship. Creative Commons makes its licenses and related\ninformation available on an \"as-is\" basis. Creative Commons gives no\nwarranties regarding its licenses, any material licensed under their\nterms and conditions, or any related information. Creative Commons\ndisclaims all liability for damages resulting from their use to the\nfullest extent possible.\n\nUsing Creative Commons Public Licenses\n\nCreative Commons public licenses provide a standard set of terms and\nconditions that creators and other rights holders may use to share\noriginal works of authorship and other material subject to copyright\nand certain other rights specified in the public license below. The\nfollowing considerations are for informational purposes only, are not\nexhaustive, and do not form part of our licenses.\n\n Considerations for licensors: Our public licenses are\n intended for use by those authorized to give the public\n permission to use material in ways otherwise restricted by\n copyright and certain other rights. Our licenses are\n irrevocable. Licensors should read and understand the terms\n and conditions of the license they choose before applying it.\n Licensors should also secure all rights necessary before\n applying our licenses so that the public can reuse the\n material as expected. Licensors should clearly mark any\n material not subject to the license. This includes other CC-\n licensed material, or material used under an exception or\n limitation to copyright. More considerations for licensors:\n\twiki.creativecommons.org/Considerations_for_licensors\n\n Considerations for the public: By using one of our public\n licenses, a licensor grants the public permission to use the\n licensed material under specified terms and conditions. If\n the licensor's permission is not necessary for any reason--for\n example, because of any applicable exception or limitation to\n copyright--then that use is not regulated by the license. Our\n licenses grant only permissions under copyright and certain\n other rights that a licensor has authority to grant. Use of\n the licensed material may still be restricted for other\n reasons, including because others have copyright or other\n rights in the material. A licensor may make special requests,\n such as asking that all changes be marked or described.\n Although not required by our licenses, you are encouraged to\n respect those requests where reasonable. More_considerations\n for the public: \n\twiki.creativecommons.org/Considerations_for_licensees\n\n=======================================================================\n\nCreative Commons Attribution-NonCommercial 4.0 International Public\nLicense\n\nBy exercising the Licensed Rights (defined below), You accept and agree\nto be bound by the terms and conditions of this Creative Commons\nAttribution-NonCommercial 4.0 International Public License (\"Public\nLicense\"). To the extent this Public License may be interpreted as a\ncontract, You are granted the Licensed Rights in consideration of Your\nacceptance of these terms and conditions, and the Licensor grants You\nsuch rights in consideration of benefits the Licensor receives from\nmaking the Licensed Material available under these terms and\nconditions.\n\nSection 1 -- Definitions.\n\n a. Adapted Material means material subject to Copyright and Similar\n Rights that is derived from or based upon the Licensed Material\n and in which the Licensed Material is translated, altered,\n arranged, transformed, or otherwise modified in a manner requiring\n permission under the Copyright and Similar Rights held by the\n Licensor. For purposes of this Public License, where the Licensed\n Material is a musical work, performance, or sound recording,\n Adapted Material is always produced where the Licensed Material is\n synched in timed relation with a moving image.\n\n b. Adapter's License means the license You apply to Your Copyright\n and Similar Rights in Your contributions to Adapted Material in\n accordance with the terms and conditions of this Public License.\n\n c. Copyright and Similar Rights means copyright and/or similar rights\n closely related to copyright including, without limitation,\n performance, broadcast, sound recording, and Sui Generis Database\n Rights, without regard to how the rights are labeled or\n categorized. For purposes of this Public License, the rights\n specified in Section 2(b)(1)-(2) are not Copyright and Similar\n Rights.\n d. Effective Technological Measures means those measures that, in the\n absence of proper authority, may not be circumvented under laws\n fulfilling obligations under Article 11 of the WIPO Copyright\n Treaty adopted on December 20, 1996, and/or similar international\n agreements.\n\n e. Exceptions and Limitations means fair use, fair dealing, and/or\n any other exception or limitation to Copyright and Similar Rights\n that applies to Your use of the Licensed Material.\n\n f. Licensed Material means the artistic or literary work, database,\n or other material to which the Licensor applied this Public\n License.\n\n g. Licensed Rights means the rights granted to You subject to the\n terms and conditions of this Public License, which are limited to\n all Copyright and Similar Rights that apply to Your use of the\n Licensed Material and that the Licensor has authority to license.\n\n h. Licensor means the individual(s) or entity(ies) granting rights\n under this Public License.\n\n i. NonCommercial means not primarily intended for or directed towards\n commercial advantage or monetary compensation. For purposes of\n this Public License, the exchange of the Licensed Material for\n other material subject to Copyright and Similar Rights by digital\n file-sharing or similar means is NonCommercial provided there is\n no payment of monetary compensation in connection with the\n exchange.\n\n j. Share means to provide material to the public by any means or\n process that requires permission under the Licensed Rights, such\n as reproduction, public display, public performance, distribution,\n dissemination, communication, or importation, and to make material\n available to the public including in ways that members of the\n public may access the material from a place and at a time\n individually chosen by them.\n\n k. Sui Generis Database Rights means rights other than copyright\n resulting from Directive 96/9/EC of the European Parliament and of\n the Council of 11 March 1996 on the legal protection of databases,\n as amended and/or succeeded, as well as other essentially\n equivalent rights anywhere in the world.\n\n l. You means the individual or entity exercising the Licensed Rights\n under this Public License. Your has a corresponding meaning.\n\nSection 2 -- Scope.\n\n a. License grant.\n\n 1. Subject to the terms and conditions of this Public License,\n the Licensor hereby grants You a worldwide, royalty-free,\n non-sublicensable, non-exclusive, irrevocable license to\n exercise the Licensed Rights in the Licensed Material to:\n\n a. reproduce and Share the Licensed Material, in whole or\n in part, for NonCommercial purposes only; and\n\n b. produce, reproduce, and Share Adapted Material for\n NonCommercial purposes only.\n\n 2. Exceptions and Limitations. For the avoidance of doubt, where\n Exceptions and Limitations apply to Your use, this Public\n License does not apply, and You do not need to comply with\n its terms and conditions.\n\n 3. Term. The term of this Public License is specified in Section\n 6(a).\n\n 4. Media and formats; technical modifications allowed. The\n Licensor authorizes You to exercise the Licensed Rights in\n all media and formats whether now known or hereafter created,\n and to make technical modifications necessary to do so. The\n Licensor waives and/or agrees not to assert any right or\n authority to forbid You from making technical modifications\n necessary to exercise the Licensed Rights, including\n technical modifications necessary to circumvent Effective\n Technological Measures. For purposes of this Public License,\n simply making modifications authorized by this Section 2(a)\n (4) never produces Adapted Material.\n\n 5. Downstream recipients.\n\n a. Offer from the Licensor -- Licensed Material. Every\n recipient of the Licensed Material automatically\n receives an offer from the Licensor to exercise the\n Licensed Rights under the terms and conditions of this\n Public License.\n\n b. No downstream restrictions. You may not offer or impose\n any additional or different terms or conditions on, or\n apply any Effective Technological Measures to, the\n Licensed Material if doing so restricts exercise of the\n Licensed Rights by any recipient of the Licensed\n Material.\n\n 6. No endorsement. Nothing in this Public License constitutes or\n may be construed as permission to assert or imply that You\n are, or that Your use of the Licensed Material is, connected\n with, or sponsored, endorsed, or granted official status by,\n the Licensor or others designated to receive attribution as\n provided in Section 3(a)(1)(A)(i).\n\n b. Other rights.\n\n 1. Moral rights, such as the right of integrity, are not\n licensed under this Public License, nor are publicity,\n privacy, and/or other similar personality rights; however, to\n the extent possible, the Licensor waives and/or agrees not to\n assert any such rights held by the Licensor to the limited\n extent necessary to allow You to exercise the Licensed\n Rights, but not otherwise.\n\n 2. Patent and trademark rights are not licensed under this\n Public License.\n\n 3. To the extent possible, the Licensor waives any right to\n collect royalties from You for the exercise of the Licensed\n Rights, whether directly or through a collecting society\n under any voluntary or waivable statutory or compulsory\n licensing scheme. In all other cases the Licensor expressly\n reserves any right to collect such royalties, including when\n the Licensed Material is used other than for NonCommercial\n purposes.\n\nSection 3 -- License Conditions.\n\nYour exercise of the Licensed Rights is expressly made subject to the\nfollowing conditions.\n\n a. Attribution.\n\n 1. If You Share the Licensed Material (including in modified\n form), You must:\n\n a. retain the following if it is supplied by the Licensor\n with the Licensed Material:\n\n i. identification of the creator(s) of the Licensed\n Material and any others designated to receive\n attribution, in any reasonable manner requested by\n the Licensor (including by pseudonym if\n designated);\n\n ii. a copyright notice;\n\n iii. a notice that refers to this Public License;\n\n iv. a notice that refers to the disclaimer of\n warranties;\n\n v. a URI or hyperlink to the Licensed Material to the\n extent reasonably practicable;\n\n b. indicate if You modified the Licensed Material and\n retain an indication of any previous modifications; and\n\n c. indicate the Licensed Material is licensed under this\n Public License, and include the text of, or the URI or\n hyperlink to, this Public License.\n\n 2. You may satisfy the conditions in Section 3(a)(1) in any\n reasonable manner based on the medium, means, and context in\n which You Share the Licensed Material. For example, it may be\n reasonable to satisfy the conditions by providing a URI or\n hyperlink to a resource that includes the required\n information.\n\n 3. If requested by the Licensor, You must remove any of the\n information required by Section 3(a)(1)(A) to the extent\n reasonably practicable.\n\n 4. If You Share Adapted Material You produce, the Adapter's\n License You apply must not prevent recipients of the Adapted\n Material from complying with this Public License.\n\nSection 4 -- Sui Generis Database Rights.\n\nWhere the Licensed Rights include Sui Generis Database Rights that\napply to Your use of the Licensed Material:\n\n a. for the avoidance of doubt, Section 2(a)(1) grants You the right\n to extract, reuse, reproduce, and Share all or a substantial\n portion of the contents of the database for NonCommercial purposes\n only;\n\n b. if You include all or a substantial portion of the database\n contents in a database in which You have Sui Generis Database\n Rights, then the database in which You have Sui Generis Database\n Rights (but not its individual contents) is Adapted Material; and\n\n c. You must comply with the conditions in Section 3(a) if You Share\n all or a substantial portion of the contents of the database.\n\nFor the avoidance of doubt, this Section 4 supplements and does not\nreplace Your obligations under this Public License where the Licensed\nRights include other Copyright and Similar Rights.\n\nSection 5 -- Disclaimer of Warranties and Limitation of Liability.\n\n a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE\n EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS\n AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF\n ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,\n IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,\n WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR\n PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,\n ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT\n KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT\n ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.\n\n b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE\n TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,\n NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,\n INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,\n COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR\n USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN\n ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR\n DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR\n IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.\n\n c. The disclaimer of warranties and limitation of liability provided\n above shall be interpreted in a manner that, to the extent\n possible, most closely approximates an absolute disclaimer and\n waiver of all liability.\n\nSection 6 -- Term and Termination.\n\n a. This Public License applies for the term of the Copyright and\n Similar Rights licensed here. However, if You fail to comply with\n this Public License, then Your rights under this Public License\n terminate automatically.\n\n b. Where Your right to use the Licensed Material has terminated under\n Section 6(a), it reinstates:\n\n 1. automatically as of the date the violation is cured, provided\n it is cured within 30 days of Your discovery of the\n violation; or\n\n 2. upon express reinstatement by the Licensor.\n\n For the avoidance of doubt, this Section 6(b) does not affect any\n right the Licensor may have to seek remedies for Your violations\n of this Public License.\n\n c. For the avoidance of doubt, the Licensor may also offer the\n Licensed Material under separate terms or conditions or stop\n distributing the Licensed Material at any time; however, doing so\n will not terminate this Public License.\n\n d. Sections 1, 5, 6, 7, and 8 survive termination of this Public\n License.\n\nSection 7 -- Other Terms and Conditions.\n\n a. The Licensor shall not be bound by any additional or different\n terms or conditions communicated by You unless expressly agreed.\n\n b. Any arrangements, understandings, or agreements regarding the\n Licensed Material not stated herein are separate from and\n independent of the terms and conditions of this Public License.\n\nSection 8 -- Interpretation.\n\n a. For the avoidance of doubt, this Public License does not, and\n shall not be interpreted to, reduce, limit, restrict, or impose\n conditions on any use of the Licensed Material that could lawfully\n be made without permission under this Public License.\n\n b. To the extent possible, if any provision of this Public License is\n deemed unenforceable, it shall be automatically reformed to the\n minimum extent necessary to make it enforceable. If the provision\n cannot be reformed, it shall be severed from this Public License\n without affecting the enforceability of the remaining terms and\n conditions.\n\n c. No term or condition of this Public License will be waived and no\n failure to comply consented to unless expressly agreed to by the\n Licensor.\n\n d. Nothing in this Public License constitutes or may be interpreted\n as a limitation upon, or waiver of, any privileges and immunities\n that apply to the Licensor or You, including from the legal\n processes of any jurisdiction or authority.\n\n=======================================================================\n\nCreative Commons is not a party to its public\nlicenses. Notwithstanding, Creative Commons may elect to apply one of\nits public licenses to material it publishes and in those instances\nwill be considered the “Licensor.” The text of the Creative Commons\npublic licenses is dedicated to the public domain under the CC0 Public\nDomain Dedication. Except for the limited purpose of indicating that\nmaterial is shared under a Creative Commons public license or as\notherwise permitted by the Creative Commons policies published at\ncreativecommons.org/policies, Creative Commons does not authorize the\nuse of the trademark \"Creative Commons\" or any other trademark or logo\nof Creative Commons without its prior written consent including,\nwithout limitation, in connection with any unauthorized modifications\nto any of its public licenses or any other arrangements,\nunderstandings, or agreements concerning use of licensed material. For\nthe avoidance of doubt, this paragraph does not form part of the\npublic licenses.\n\nCreative Commons may be contacted at creativecommons.org.\n" }, { "path": "README.md", "content": "# Cross-Domain Adaptive Teacher for Object Detection\n\n[![License: CC BY-NC 4.0](https://img.shields.io/badge/License-CC%20BY--NC%204.0-lightgrey.svg)](https://creativecommons.org/licenses/by-nc/4.0/) \n\n[![License: CC BY-NC 4.0](https://licensebuttons.net/l/by-nc/4.0/80x15.png)](https://creativecommons.org/licenses/by-nc/4.0/)\n\nThis is the PyTorch implementation of our paper:
\n**Cross-Domain Adaptive Teacher for Object Detection**
\n [Yu-Jhe Li](https://yujheli.github.io/), [Xiaoliang Dai](https://sites.google.com/view/xiaoliangdai), [Chih-Yao Ma](https://chihyaoma.github.io/), [Yen-Cheng Liu](https://ycliu93.github.io/), [Kan Chen](https://kanchen.info/), [Bichen Wu](https://scholar.google.com/citations?user=K3QJPdMAAAAJ&hl=en), [Zijian He](https://research.fb.com/people/he-zijian/), [Kris Kitani](http://www.cs.cmu.edu/~kkitani/), [Peter Vajda](https://sites.google.com/site/vajdap)
\nIEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), 2022
\n\n[[Paper](https://openaccess.thecvf.com/content/CVPR2022/papers/Li_Cross-Domain_Adaptive_Teacher_for_Object_Detection_CVPR_2022_paper.pdf)] [[Project](https://yujheli.github.io/projects/adaptiveteacher.html)]\n\n

\n\n

\n\n# Installation\n\n## Prerequisites\n\n- Python ≥ 3.6\n- PyTorch ≥ 1.5 and torchvision that matches the PyTorch installation.\n- Detectron2 == 0.3 (The version I used to run my code)\n\n## Our tested environment\n\n- 8 V100 (16 batch size)\n- 4 2080 Ti (4 batch size)\n\n## Install python env\n\nTo install required dependencies on the virtual environment of the python (e.g., virtualenv for python3), please run the following command at the root of this code:\n```\n$ python3 -m venv /path/to/new/virtual/environment/.\n$ source /path/to/new/virtual/environment/bin/activate\n```\nFor example:\n```\n$ mkdir python_env\n$ python3 -m venv python_env/\n$ source python_env/bin/activate\n```\n \n\n## Build Detectron2 from Source\n\nFollow the [INSTALL.md](https://github.com/facebookresearch/detectron2/blob/master/INSTALL.md) to install Detectron2.\n\n## Dataset download\n\n1. Download the datasets\n\n\n2. Organize the dataset as the Cityscapes and PASCAL VOC format following:\n\n```shell\nadaptive_teacher/\n└── datasets/\n └── cityscapes/\n ├── gtFine/\n ├── train/\n └── test/\n └── val/\n ├── leftImg8bit/\n ├── train/\n └── test/\n └── val/\n └── cityscapes_foggy/\n ├── gtFine/\n ├── train/\n └── test/\n └── val/\n ├── leftImg8bit/\n ├── train/\n └── test/\n └── val/\n └── VOC2012/\n ├── Annotations/\n ├── ImageSets/\n └── JPEGImages/\n └── clipark/\n ├── Annotations/\n ├── ImageSets/\n └── JPEGImages/\n └── watercolor/\n ├── Annotations/\n ├── ImageSets/\n └── JPEGImages/\n \n```\n\n# Training\n\n- Train the Adaptive Teacher under PASCAL VOC (source) and Clipart1k (target)\n\n```shell\npython train_net.py \\\n --num-gpus 8 \\\n --config configs/faster_rcnn_R101_cross_clipart.yaml\\\n OUTPUT_DIR output/exp_clipart\n```\n\n- Train the Adaptive Teacher under cityscapes (source) and foggy cityscapes (target)\n\n```shell\npython train_net.py\\\n --num-gpus 8\\\n --config configs/faster_rcnn_VGG_cross_city.yaml\\\n OUTPUT_DIR output/exp_city\n```\n\n## Resume the training\n\n```shell\npython train_net.py \\\n --resume \\\n --num-gpus 8 \\\n --config configs/faster_rcnn_R101_cross_clipart.yaml MODEL.WEIGHTS .pth\n```\n\n## Evaluation\n\n```shell\npython train_net.py \\\n --eval-only \\\n --num-gpus 8 \\\n --config configs/faster_rcnn_R101_cross_clipart.yaml \\\n MODEL.WEIGHTS .pth\n```\n\n## Results and Model Weights\n\nIf you are urgent with the pre-trained weights, please download our interal prod_weights here at the [Link](https://drive.google.com/drive/folders/17p8oYjhmoA77_hyVZq4WLJezsUiSZhdi?usp=sharing). Please note that the key name in the pre-trained model is slightly different and you will need to align manually. Otherwise, please wait and we will try to release the local weights in the future.\n### Real to Artistic Adaptation:\n| Backbone | Source set (labeled) | Target set (unlabeled) | Batch size | AP@.5 | Model Weights | Comment |\n| :-----: | :---------------: | :----------------: | :---------------------: | :-----: | :----------: |:-----: | \n| R101 | VOC12 | Clipark1k | 16 labeled + 16 unlabeled | 40.1 | [link](https://drive.google.com/file/d/1mzqSlkftJDTj1IWZC0huuMIzWDtfSaL0/view?usp=sharing)| Ours w/o discriminator (dis=0)|\n| R101 | VOC12 | Clipark1k | 4 labeled + 4 unlabeled | 47.2 | [link](https://drive.google.com/file/d/1F72bfPP-5uu4H2rS_OscSLVhvjHeylR-/view?usp=sharing)| lr=0.01, dis_w=0.1, default |\n| R101 | VOC12 | Clipark1k | 16 labeled + 16 unlabeled | 49.6 | [link](https://drive.google.com/file/d/1qbueKiNPLIP4gFJrUQi_1kpCAQmUivFG/view?usp=sharing)| Ours in the paper, unsup_w=0.5|\n| R101+FPN | VOC12 | Clipark1k | 16 labeled + 16 unlabeled | 51.2 |link (coming soon) | For future work|\n\n### Weather Adaptation:\n| Backbone | Source set (labeled) | Target set (unlabeled) | Batch size | AP@.5 | Model Weights | Comment|\n| :-----: | :---------------: | :----------------: | :---------------------: | :-----: | :--------------------------------------------------: |:-----: | \n| VGG16| Cityscapes | Foggy Cityscapes (ALL) | 16 labeled + 16 unlabeled | 48.7 | link (coming soon)|Ours w/o discriminator|\n| VGG16| Cityscapes | Foggy Cityscapes (ALL) | 16 labeled + 16 unlabeled | 50.9 | link (coming soon)|Ours in the paper|\n| VGG16| Cityscapes | Foggy Cityscapes (0.02) | 16 labeled + 16 unlabeled | in progress | link (coming soon)|Ours in the paper|\n| VGG16+FPN | Cityscapes | Foggy Cityscapes (ALL) | 16 labeled + 16 unlabeled | 57.4 |link (coming soon) |For future work|\n\n## Citation\n\nIf you use Adaptive Teacher in your research or wish to refer to the results published in the paper, please use the following BibTeX entry.\n\n```BibTeX\n@inproceedings{li2022cross,\n title={Cross-Domain Adaptive Teacher for Object Detection},\n author={Li, Yu-Jhe and Dai, Xiaoliang and Ma, Chih-Yao and Liu, Yen-Cheng and Chen, Kan and Wu, Bichen and He, Zijian and Kitani, Kris and Vajda, Peter},\n booktitle={IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n year={2022}\n} \n```\n\nAlso, if you use Detectron2 in your research, please use the following BibTeX entry.\n\n```BibTeX\n@misc{wu2019detectron2,\n author = {Yuxin Wu and Alexander Kirillov and Francisco Massa and\n Wan-Yen Lo and Ross Girshick},\n title = {Detectron2},\n howpublished = {\\url{https://github.com/facebookresearch/detectron2}},\n year = {2019}\n}\n```\n\n## License\n\nThis project is licensed under CC-BY-NC 4.0 License, as found in the LICENSE file.\n" }, { "path": "adapteacher/__init__.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom .config import add_ateacher_config\n" }, { "path": "adapteacher/checkpoint/detection_checkpoint.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom detectron2.checkpoint.c2_model_loading import align_and_update_state_dicts\nfrom detectron2.checkpoint import DetectionCheckpointer\n\n# for load_student_model\nfrom typing import Any\nfrom fvcore.common.checkpoint import _strip_prefix_if_present, _IncompatibleKeys\n\n\nclass DetectionTSCheckpointer(DetectionCheckpointer):\n def _load_model(self, checkpoint):\n if checkpoint.get(\"__author__\", None) == \"Caffe2\":\n # pretrained model weight: only update student model\n if checkpoint.get(\"matching_heuristics\", False):\n self._convert_ndarray_to_tensor(checkpoint[\"model\"])\n # convert weights by name-matching heuristics\n model_state_dict = self.model.modelStudent.state_dict()\n align_and_update_state_dicts(\n model_state_dict,\n checkpoint[\"model\"],\n c2_conversion=checkpoint.get(\"__author__\", None) == \"Caffe2\",\n )\n checkpoint[\"model\"] = model_state_dict\n\n # for non-caffe2 models, use standard ways to load it\n incompatible = self._load_student_model(checkpoint)\n\n model_buffers = dict(self.model.modelStudent.named_buffers(recurse=False))\n for k in [\"pixel_mean\", \"pixel_std\"]:\n # Ignore missing key message about pixel_mean/std.\n # Though they may be missing in old checkpoints, they will be correctly\n # initialized from config anyway.\n if k in model_buffers:\n try:\n incompatible.missing_keys.remove(k)\n except ValueError:\n pass\n return incompatible\n\n else: # whole model\n if checkpoint.get(\"matching_heuristics\", False):\n self._convert_ndarray_to_tensor(checkpoint[\"model\"])\n # convert weights by name-matching heuristics\n model_state_dict = self.model.state_dict()\n align_and_update_state_dicts(\n model_state_dict,\n checkpoint[\"model\"],\n c2_conversion=checkpoint.get(\"__author__\", None) == \"Caffe2\",\n )\n checkpoint[\"model\"] = model_state_dict \n # for non-caffe2 models, use standard ways to load it\n incompatible = super()._load_model(checkpoint)\n\n model_buffers = dict(self.model.named_buffers(recurse=False))\n for k in [\"pixel_mean\", \"pixel_std\"]:\n # Ignore missing key message about pixel_mean/std.\n # Though they may be missing in old checkpoints, they will be correctly\n # initialized from config anyway.\n if k in model_buffers:\n try:\n incompatible.missing_keys.remove(k)\n except ValueError:\n pass\n return incompatible\n\n def _load_student_model(self, checkpoint: Any) -> _IncompatibleKeys: # pyre-ignore\n checkpoint_state_dict = checkpoint.pop(\"model\")\n self._convert_ndarray_to_tensor(checkpoint_state_dict)\n\n # if the state_dict comes from a model that was wrapped in a\n # DataParallel or DistributedDataParallel during serialization,\n # remove the \"module\" prefix before performing the matching.\n _strip_prefix_if_present(checkpoint_state_dict, \"module.\")\n\n # work around https://github.com/pytorch/pytorch/issues/24139\n model_state_dict = self.model.modelStudent.state_dict()\n incorrect_shapes = []\n for k in list(checkpoint_state_dict.keys()):\n if k in model_state_dict:\n shape_model = tuple(model_state_dict[k].shape)\n shape_checkpoint = tuple(checkpoint_state_dict[k].shape)\n if shape_model != shape_checkpoint:\n incorrect_shapes.append((k, shape_checkpoint, shape_model))\n checkpoint_state_dict.pop(k)\n # pyre-ignore\n incompatible = self.model.modelStudent.load_state_dict(\n checkpoint_state_dict, strict=False\n )\n return _IncompatibleKeys(\n missing_keys=incompatible.missing_keys,\n unexpected_keys=incompatible.unexpected_keys,\n incorrect_shapes=incorrect_shapes,\n )\n\n\n# class DetectionCheckpointer(Checkpointer):\n# \"\"\"\n# Same as :class:`Checkpointer`, but is able to handle models in detectron & detectron2\n# model zoo, and apply conversions for legacy models.\n# \"\"\"\n\n# def __init__(self, model, save_dir=\"\", *, save_to_disk=None, **checkpointables):\n# is_main_process = comm.is_main_process()\n# super().__init__(\n# model,\n# save_dir,\n# save_to_disk=is_main_process if save_to_disk is None else save_to_disk,\n# **checkpointables,\n# )\n\n# def _load_file(self, filename):\n# if filename.endswith(\".pkl\"):\n# with PathManager.open(filename, \"rb\") as f:\n# data = pickle.load(f, encoding=\"latin1\")\n# if \"model\" in data and \"__author__\" in data:\n# # file is in Detectron2 model zoo format\n# self.logger.info(\"Reading a file from '{}'\".format(data[\"__author__\"]))\n# return data\n# else:\n# # assume file is from Caffe2 / Detectron1 model zoo\n# if \"blobs\" in data:\n# # Detection models have \"blobs\", but ImageNet models don't\n# data = data[\"blobs\"]\n# data = {k: v for k, v in data.items() if not k.endswith(\"_momentum\")}\n# return {\"model\": data, \"__author__\": \"Caffe2\", \"matching_heuristics\": True}\n\n# loaded = super()._load_file(filename) # load native pth checkpoint\n# if \"model\" not in loaded:\n# loaded = {\"model\": loaded}\n# return loaded\n\n# def _load_model(self, checkpoint):\n# if checkpoint.get(\"matching_heuristics\", False):\n# self._convert_ndarray_to_tensor(checkpoint[\"model\"])\n# # convert weights by name-matching heuristics\n# model_state_dict = self.model.state_dict()\n# align_and_update_state_dicts(\n# model_state_dict,\n# checkpoint[\"model\"],\n# c2_conversion=checkpoint.get(\"__author__\", None) == \"Caffe2\",\n# )\n# checkpoint[\"model\"] = model_state_dict\n# # for non-caffe2 models, use standard ways to load it\n# super()._load_model(checkpoint)" }, { "path": "adapteacher/config.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom detectron2.config import CfgNode as CN\n\n\ndef add_ateacher_config(cfg):\n \"\"\"\n Add config for semisupnet.\n \"\"\"\n _C = cfg\n _C.TEST.VAL_LOSS = True\n\n _C.MODEL.RPN.UNSUP_LOSS_WEIGHT = 1.0\n _C.MODEL.RPN.LOSS = \"CrossEntropy\"\n _C.MODEL.ROI_HEADS.LOSS = \"CrossEntropy\"\n\n _C.SOLVER.IMG_PER_BATCH_LABEL = 1\n _C.SOLVER.IMG_PER_BATCH_UNLABEL = 1\n _C.SOLVER.FACTOR_LIST = (1,)\n\n _C.DATASETS.TRAIN_LABEL = (\"coco_2017_train\",)\n _C.DATASETS.TRAIN_UNLABEL = (\"coco_2017_train\",)\n _C.DATASETS.CROSS_DATASET = True\n _C.TEST.EVALUATOR = \"COCOeval\"\n\n _C.SEMISUPNET = CN()\n\n # Output dimension of the MLP projector after `res5` block\n _C.SEMISUPNET.MLP_DIM = 128\n\n # Semi-supervised training\n _C.SEMISUPNET.Trainer = \"ateacher\"\n _C.SEMISUPNET.BBOX_THRESHOLD = 0.7\n _C.SEMISUPNET.PSEUDO_BBOX_SAMPLE = \"thresholding\"\n _C.SEMISUPNET.TEACHER_UPDATE_ITER = 1\n _C.SEMISUPNET.BURN_UP_STEP = 12000\n _C.SEMISUPNET.EMA_KEEP_RATE = 0.0\n _C.SEMISUPNET.UNSUP_LOSS_WEIGHT = 4.0\n _C.SEMISUPNET.SUP_LOSS_WEIGHT = 0.5\n _C.SEMISUPNET.LOSS_WEIGHT_TYPE = \"standard\"\n _C.SEMISUPNET.DIS_TYPE = \"res4\"\n _C.SEMISUPNET.DIS_LOSS_WEIGHT = 0.1\n\n # dataloader\n # supervision level\n _C.DATALOADER.SUP_PERCENT = 100.0 # 5 = 5% dataset as labeled set\n _C.DATALOADER.RANDOM_DATA_SEED = 0 # random seed to read data\n _C.DATALOADER.RANDOM_DATA_SEED_PATH = \"dataseed/COCO_supervision.txt\"\n\n _C.EMAMODEL = CN()\n _C.EMAMODEL.SUP_CONSIST = True\n" }, { "path": "adapteacher/data/__init__.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom .build import (\n build_detection_test_loader,\n build_detection_semisup_train_loader,\n)\n" }, { "path": "adapteacher/data/build.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport logging\nimport numpy as np\nimport operator\nimport json\nimport torch.utils.data\nfrom detectron2.utils.comm import get_world_size\nfrom detectron2.data.common import (\n DatasetFromList,\n MapDataset,\n)\nfrom detectron2.data.dataset_mapper import DatasetMapper\nfrom detectron2.data.samplers import (\n InferenceSampler,\n RepeatFactorTrainingSampler,\n TrainingSampler,\n)\nfrom detectron2.data.build import (\n trivial_batch_collator,\n worker_init_reset_seed,\n get_detection_dataset_dicts,\n build_batch_data_loader,\n)\nfrom adapteacher.data.common import (\n AspectRatioGroupedSemiSupDatasetTwoCrop,\n)\n\n\n\"\"\"\nThis file contains the default logic to build a dataloader for training or testing.\n\"\"\"\n\n\ndef divide_label_unlabel(\n dataset_dicts, SupPercent, random_data_seed, random_data_seed_path\n):\n num_all = len(dataset_dicts)\n num_label = int(SupPercent / 100.0 * num_all)\n\n # read from pre-generated data seed\n with open(random_data_seed_path) as COCO_sup_file:\n coco_random_idx = json.load(COCO_sup_file)\n\n labeled_idx = np.array(coco_random_idx[str(SupPercent)][str(random_data_seed)])\n assert labeled_idx.shape[0] == num_label, \"Number of READ_DATA is mismatched.\"\n\n label_dicts = []\n unlabel_dicts = []\n labeled_idx = set(labeled_idx)\n\n for i in range(len(dataset_dicts)):\n if i in labeled_idx:\n label_dicts.append(dataset_dicts[i])\n else:\n unlabel_dicts.append(dataset_dicts[i])\n\n return label_dicts, unlabel_dicts\n\n\n# uesed by supervised-only baseline trainer\ndef build_detection_semisup_train_loader(cfg, mapper=None):\n\n dataset_dicts = get_detection_dataset_dicts(\n cfg.DATASETS.TRAIN,\n filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS,\n min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE\n if cfg.MODEL.KEYPOINT_ON\n else 0,\n proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN\n if cfg.MODEL.LOAD_PROPOSALS\n else None,\n )\n\n # Divide into labeled and unlabeled sets according to supervision percentage\n label_dicts, unlabel_dicts = divide_label_unlabel(\n dataset_dicts,\n cfg.DATALOADER.SUP_PERCENT,\n cfg.DATALOADER.RANDOM_DATA_SEED,\n cfg.DATALOADER.RANDOM_DATA_SEED_PATH,\n )\n\n dataset = DatasetFromList(label_dicts, copy=False)\n\n if mapper is None:\n mapper = DatasetMapper(cfg, True)\n dataset = MapDataset(dataset, mapper)\n\n sampler_name = cfg.DATALOADER.SAMPLER_TRAIN\n logger = logging.getLogger(__name__)\n logger.info(\"Using training sampler {}\".format(sampler_name))\n\n if sampler_name == \"TrainingSampler\":\n sampler = TrainingSampler(len(dataset))\n elif sampler_name == \"RepeatFactorTrainingSampler\":\n repeat_factors = (\n RepeatFactorTrainingSampler.repeat_factors_from_category_frequency(\n label_dicts, cfg.DATALOADER.REPEAT_THRESHOLD\n )\n )\n sampler = RepeatFactorTrainingSampler(repeat_factors)\n else:\n raise ValueError(\"Unknown training sampler: {}\".format(sampler_name))\n\n # list num of labeled and unlabeled\n logger.info(\"\bNumber of training samples \" + str(len(dataset)))\n logger.info(\"Supervision percentage \" + str(cfg.DATALOADER.SUP_PERCENT))\n\n return build_batch_data_loader(\n dataset,\n sampler,\n cfg.SOLVER.IMS_PER_BATCH,\n aspect_ratio_grouping=cfg.DATALOADER.ASPECT_RATIO_GROUPING,\n num_workers=cfg.DATALOADER.NUM_WORKERS,\n )\n\n\n# uesed by evaluation\ndef build_detection_test_loader(cfg, dataset_name, mapper=None):\n dataset_dicts = get_detection_dataset_dicts(\n [dataset_name],\n filter_empty=False,\n proposal_files=[\n cfg.DATASETS.PROPOSAL_FILES_TEST[\n list(cfg.DATASETS.TEST).index(dataset_name)\n ]\n ]\n if cfg.MODEL.LOAD_PROPOSALS\n else None,\n )\n dataset = DatasetFromList(dataset_dicts)\n if mapper is None:\n mapper = DatasetMapper(cfg, False)\n dataset = MapDataset(dataset, mapper)\n\n sampler = InferenceSampler(len(dataset))\n batch_sampler = torch.utils.data.sampler.BatchSampler(sampler, 1, drop_last=False)\n\n data_loader = torch.utils.data.DataLoader(\n dataset,\n num_workers=cfg.DATALOADER.NUM_WORKERS,\n batch_sampler=batch_sampler,\n collate_fn=trivial_batch_collator,\n )\n return data_loader\n\n\n# uesed by unbiased teacher trainer\ndef build_detection_semisup_train_loader_two_crops(cfg, mapper=None):\n if cfg.DATASETS.CROSS_DATASET: # cross-dataset (e.g., coco-additional)\n label_dicts = get_detection_dataset_dicts(\n cfg.DATASETS.TRAIN_LABEL,\n filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS,\n min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE\n if cfg.MODEL.KEYPOINT_ON\n else 0,\n proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN\n if cfg.MODEL.LOAD_PROPOSALS\n else None,\n )\n unlabel_dicts = get_detection_dataset_dicts(\n cfg.DATASETS.TRAIN_UNLABEL,\n filter_empty=False,\n min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE\n if cfg.MODEL.KEYPOINT_ON\n else 0,\n proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN\n if cfg.MODEL.LOAD_PROPOSALS\n else None,\n )\n else: # different degree of supervision (e.g., COCO-supervision)\n dataset_dicts = get_detection_dataset_dicts(\n cfg.DATASETS.TRAIN,\n filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS,\n min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE\n if cfg.MODEL.KEYPOINT_ON\n else 0,\n proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN\n if cfg.MODEL.LOAD_PROPOSALS\n else None,\n )\n\n # Divide into labeled and unlabeled sets according to supervision percentage\n label_dicts, unlabel_dicts = divide_label_unlabel(\n dataset_dicts,\n cfg.DATALOADER.SUP_PERCENT,\n cfg.DATALOADER.RANDOM_DATA_SEED,\n cfg.DATALOADER.RANDOM_DATA_SEED_PATH,\n )\n\n label_dataset = DatasetFromList(label_dicts, copy=False)\n # exclude the labeled set from unlabeled dataset\n unlabel_dataset = DatasetFromList(unlabel_dicts, copy=False)\n # include the labeled set in unlabel dataset\n # unlabel_dataset = DatasetFromList(dataset_dicts, copy=False)\n\n if mapper is None:\n mapper = DatasetMapper(cfg, True)\n label_dataset = MapDataset(label_dataset, mapper)\n unlabel_dataset = MapDataset(unlabel_dataset, mapper)\n\n sampler_name = cfg.DATALOADER.SAMPLER_TRAIN\n logger = logging.getLogger(__name__)\n logger.info(\"Using training sampler {}\".format(sampler_name))\n if sampler_name == \"TrainingSampler\":\n label_sampler = TrainingSampler(len(label_dataset))\n unlabel_sampler = TrainingSampler(len(unlabel_dataset))\n elif sampler_name == \"RepeatFactorTrainingSampler\":\n raise NotImplementedError(\"{} not yet supported.\".format(sampler_name))\n else:\n raise ValueError(\"Unknown training sampler: {}\".format(sampler_name))\n return build_semisup_batch_data_loader_two_crop(\n (label_dataset, unlabel_dataset),\n (label_sampler, unlabel_sampler),\n cfg.SOLVER.IMG_PER_BATCH_LABEL,\n cfg.SOLVER.IMG_PER_BATCH_UNLABEL,\n aspect_ratio_grouping=cfg.DATALOADER.ASPECT_RATIO_GROUPING,\n num_workers=cfg.DATALOADER.NUM_WORKERS,\n )\n\n\n# batch data loader\ndef build_semisup_batch_data_loader_two_crop(\n dataset,\n sampler,\n total_batch_size_label,\n total_batch_size_unlabel,\n *,\n aspect_ratio_grouping=False,\n num_workers=0\n):\n world_size = get_world_size()\n assert (\n total_batch_size_label > 0 and total_batch_size_label % world_size == 0\n ), \"Total label batch size ({}) must be divisible by the number of gpus ({}).\".format(\n total_batch_size_label, world_size\n )\n\n assert (\n total_batch_size_unlabel > 0 and total_batch_size_unlabel % world_size == 0\n ), \"Total unlabel batch size ({}) must be divisible by the number of gpus ({}).\".format(\n total_batch_size_label, world_size\n )\n\n batch_size_label = total_batch_size_label // world_size\n batch_size_unlabel = total_batch_size_unlabel // world_size\n\n label_dataset, unlabel_dataset = dataset\n label_sampler, unlabel_sampler = sampler\n\n if aspect_ratio_grouping:\n label_data_loader = torch.utils.data.DataLoader(\n label_dataset,\n sampler=label_sampler,\n num_workers=num_workers,\n batch_sampler=None,\n collate_fn=operator.itemgetter(\n 0\n ), # don't batch, but yield individual elements\n worker_init_fn=worker_init_reset_seed,\n ) # yield individual mapped dict\n unlabel_data_loader = torch.utils.data.DataLoader(\n unlabel_dataset,\n sampler=unlabel_sampler,\n num_workers=num_workers,\n batch_sampler=None,\n collate_fn=operator.itemgetter(\n 0\n ), # don't batch, but yield individual elements\n worker_init_fn=worker_init_reset_seed,\n ) # yield individual mapped dict\n return AspectRatioGroupedSemiSupDatasetTwoCrop(\n (label_data_loader, unlabel_data_loader),\n (batch_size_label, batch_size_unlabel),\n )\n else:\n raise NotImplementedError(\"ASPECT_RATIO_GROUPING = False is not supported yet\")" }, { "path": "adapteacher/data/common.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport logging\nfrom detectron2.data.common import MapDataset, AspectRatioGroupedDataset\n\n\nclass MapDatasetTwoCrop(MapDataset):\n \"\"\"\n Map a function over the elements in a dataset.\n\n This customized MapDataset transforms an image with two augmentations\n as two inputs (queue and key).\n\n Args:\n dataset: a dataset where map function is applied.\n map_func: a callable which maps the element in dataset. map_func is\n responsible for error handling, when error happens, it needs to\n return None so the MapDataset will randomly use other\n elements from the dataset.\n \"\"\"\n\n def __getitem__(self, idx):\n retry_count = 0\n cur_idx = int(idx)\n\n while True:\n data = self._map_func(self._dataset[cur_idx])\n if data is not None:\n self._fallback_candidates.add(cur_idx)\n return data\n\n # _map_func fails for this idx, use a random new index from the pool\n retry_count += 1\n self._fallback_candidates.discard(cur_idx)\n cur_idx = self._rng.sample(self._fallback_candidates, k=1)[0]\n\n if retry_count >= 3:\n logger = logging.getLogger(__name__)\n logger.warning(\n \"Failed to apply `_map_func` for idx: {}, retry count: {}\".format(\n idx, retry_count\n )\n )\n\n\nclass AspectRatioGroupedDatasetTwoCrop(AspectRatioGroupedDataset):\n \"\"\"\n Batch data that have similar aspect ratio together.\n In this implementation, images whose aspect ratio < (or >) 1 will\n be batched together.\n This improves training speed because the images then need less padding\n to form a batch.\n\n It assumes the underlying dataset produces dicts with \"width\" and \"height\" keys.\n It will then produce a list of original dicts with length = batch_size,\n all with similar aspect ratios.\n \"\"\"\n\n def __init__(self, dataset, batch_size):\n \"\"\"\n Args:\n dataset: an iterable. Each element must be a dict with keys\n \"width\" and \"height\", which will be used to batch data.\n batch_size (int):\n \"\"\"\n self.dataset = dataset\n self.batch_size = batch_size\n self._buckets = [[] for _ in range(2)]\n self._buckets_key = [[] for _ in range(2)]\n # Hard-coded two aspect ratio groups: w > h and w < h.\n # Can add support for more aspect ratio groups, but doesn't seem useful\n\n def __iter__(self):\n for d in self.dataset:\n # d is a tuple with len = 2\n # It's two images (same size) from the same image instance\n w, h = d[0][\"width\"], d[0][\"height\"]\n bucket_id = 0 if w > h else 1\n\n # bucket = bucket for normal images\n bucket = self._buckets[bucket_id]\n bucket.append(d[0])\n\n # buckets_key = bucket for augmented images\n buckets_key = self._buckets_key[bucket_id]\n buckets_key.append(d[1])\n if len(bucket) == self.batch_size:\n yield (bucket[:], buckets_key[:])\n del bucket[:]\n del buckets_key[:]\n\n\nclass AspectRatioGroupedSemiSupDatasetTwoCrop(AspectRatioGroupedDataset):\n \"\"\"\n Batch data that have similar aspect ratio together.\n In this implementation, images whose aspect ratio < (or >) 1 will\n be batched together.\n This improves training speed because the images then need less padding\n to form a batch.\n\n It assumes the underlying dataset produces dicts with \"width\" and \"height\" keys.\n It will then produce a list of original dicts with length = batch_size,\n all with similar aspect ratios.\n \"\"\"\n\n def __init__(self, dataset, batch_size):\n \"\"\"\n Args:\n dataset: a tuple containing two iterable generators. （labeled and unlabeled data)\n Each element must be a dict with keys \"width\" and \"height\", which will be used\n to batch data.\n batch_size (int):\n \"\"\"\n\n self.label_dataset, self.unlabel_dataset = dataset\n self.batch_size_label = batch_size[0]\n self.batch_size_unlabel = batch_size[1]\n\n self._label_buckets = [[] for _ in range(2)]\n self._label_buckets_key = [[] for _ in range(2)]\n self._unlabel_buckets = [[] for _ in range(2)]\n self._unlabel_buckets_key = [[] for _ in range(2)]\n # Hard-coded two aspect ratio groups: w > h and w < h.\n # Can add support for more aspect ratio groups, but doesn't seem useful\n\n def __iter__(self):\n label_bucket, unlabel_bucket = [], []\n for d_label, d_unlabel in zip(self.label_dataset, self.unlabel_dataset):\n # d is a tuple with len = 2\n # It's two images (same size) from the same image instance\n # d[0] is with strong augmentation, d[1] is with weak augmentation\n\n # because we are grouping images with their aspect ratio\n # label and unlabel buckets might not have the same number of data\n # i.e., one could reach batch_size, while the other is still not\n if len(label_bucket) != self.batch_size_label:\n w, h = d_label[0][\"width\"], d_label[0][\"height\"]\n label_bucket_id = 0 if w > h else 1\n label_bucket = self._label_buckets[label_bucket_id]\n label_bucket.append(d_label[0])\n label_buckets_key = self._label_buckets_key[label_bucket_id]\n label_buckets_key.append(d_label[1])\n\n if len(unlabel_bucket) != self.batch_size_unlabel:\n w, h = d_unlabel[0][\"width\"], d_unlabel[0][\"height\"]\n unlabel_bucket_id = 0 if w > h else 1\n unlabel_bucket = self._unlabel_buckets[unlabel_bucket_id]\n unlabel_bucket.append(d_unlabel[0])\n unlabel_buckets_key = self._unlabel_buckets_key[unlabel_bucket_id]\n unlabel_buckets_key.append(d_unlabel[1])\n\n # yield the batch of data until all buckets are full\n if (\n len(label_bucket) == self.batch_size_label\n and len(unlabel_bucket) == self.batch_size_unlabel\n ):\n # label_strong, label_weak, unlabed_strong, unlabled_weak\n yield (\n label_bucket[:],\n label_buckets_key[:],\n unlabel_bucket[:],\n unlabel_buckets_key[:],\n )\n del label_bucket[:]\n del label_buckets_key[:]\n del unlabel_bucket[:]\n del unlabel_buckets_key[:]\n" }, { "path": "adapteacher/data/dataset_mapper.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport copy\nimport logging\nimport numpy as np\nfrom PIL import Image\nimport torch\n\nimport detectron2.data.detection_utils as utils\nimport detectron2.data.transforms as T\nfrom detectron2.data.dataset_mapper import DatasetMapper\nfrom adapteacher.data.detection_utils import build_strong_augmentation\n\n\nclass DatasetMapperTwoCropSeparate(DatasetMapper):\n \"\"\"\n This customized mapper produces two augmented images from a single image\n instance. This mapper makes sure that the two augmented images have the same\n cropping and thus the same size.\n\n A callable which takes a dataset dict in Detectron2 Dataset format,\n and map it into a format used by the model.\n\n This is the default callable to be used to map your dataset dict into training data.\n You may need to follow it to implement your own one for customized logic,\n such as a different way to read or transform images.\n See :doc:`/tutorials/data_loading` for details.\n\n The callable currently does the following:\n\n 1. Read the image from \"file_name\"\n 2. Applies cropping/geometric transforms to the image and annotations\n 3. Prepare data and annotations to Tensor and :class:`Instances`\n \"\"\"\n\n def __init__(self, cfg, is_train=True):\n self.augmentation = utils.build_augmentation(cfg, is_train)\n # include crop into self.augmentation\n if cfg.INPUT.CROP.ENABLED and is_train:\n self.augmentation.insert(\n 0, T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE)\n )\n logging.getLogger(__name__).info(\n \"Cropping used in training: \" + str(self.augmentation[0])\n )\n self.compute_tight_boxes = True\n else:\n self.compute_tight_boxes = False\n self.strong_augmentation = build_strong_augmentation(cfg, is_train)\n\n # fmt: off\n self.img_format = cfg.INPUT.FORMAT\n self.mask_on = cfg.MODEL.MASK_ON\n self.mask_format = cfg.INPUT.MASK_FORMAT\n self.keypoint_on = cfg.MODEL.KEYPOINT_ON\n self.load_proposals = cfg.MODEL.LOAD_PROPOSALS\n # fmt: on\n if self.keypoint_on and is_train:\n self.keypoint_hflip_indices = utils.create_keypoint_hflip_indices(\n cfg.DATASETS.TRAIN\n )\n else:\n self.keypoint_hflip_indices = None\n\n if self.load_proposals:\n self.proposal_min_box_size = cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE\n self.proposal_topk = (\n cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TRAIN\n if is_train\n else cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TEST\n )\n self.is_train = is_train\n\n def __call__(self, dataset_dict):\n \"\"\"\n Args:\n dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format.\n\n Returns:\n dict: a format that builtin models in detectron2 accept\n \"\"\"\n dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below\n image = utils.read_image(dataset_dict[\"file_name\"], format=self.img_format)\n # utils.check_image_size(dataset_dict, image)\n\n if \"sem_seg_file_name\" in dataset_dict:\n sem_seg_gt = utils.read_image(\n dataset_dict.pop(\"sem_seg_file_name\"), \"L\"\n ).squeeze(2)\n else:\n sem_seg_gt = None\n\n aug_input = T.StandardAugInput(image, sem_seg=sem_seg_gt)\n transforms = aug_input.apply_augmentations(self.augmentation)\n image_weak_aug, sem_seg_gt = aug_input.image, aug_input.sem_seg\n image_shape = image_weak_aug.shape[:2] # h, w\n\n if sem_seg_gt is not None:\n dataset_dict[\"sem_seg\"] = torch.as_tensor(sem_seg_gt.astype(\"long\"))\n\n if self.load_proposals:\n utils.transform_proposals(\n dataset_dict,\n image_shape,\n transforms,\n proposal_topk=self.proposal_topk,\n min_box_size=self.proposal_min_box_size,\n )\n\n if not self.is_train:\n dataset_dict.pop(\"annotations\", None)\n dataset_dict.pop(\"sem_seg_file_name\", None)\n return dataset_dict\n\n if \"annotations\" in dataset_dict:\n for anno in dataset_dict[\"annotations\"]:\n if not self.mask_on:\n anno.pop(\"segmentation\", None)\n if not self.keypoint_on:\n anno.pop(\"keypoints\", None)\n\n annos = [\n utils.transform_instance_annotations(\n obj,\n transforms,\n image_shape,\n keypoint_hflip_indices=self.keypoint_hflip_indices,\n )\n for obj in dataset_dict.pop(\"annotations\")\n if obj.get(\"iscrowd\", 0) == 0\n ]\n instances = utils.annotations_to_instances(\n annos, image_shape, mask_format=self.mask_format\n )\n\n if self.compute_tight_boxes and instances.has(\"gt_masks\"):\n instances.gt_boxes = instances.gt_masks.get_bounding_boxes()\n\n bboxes_d2_format = utils.filter_empty_instances(instances)\n dataset_dict[\"instances\"] = bboxes_d2_format\n\n # apply strong augmentation\n # We use torchvision augmentation, which is not compatiable with\n # detectron2, which use numpy format for images. Thus, we need to\n # convert to PIL format first.\n image_pil = Image.fromarray(image_weak_aug.astype(\"uint8\"), \"RGB\")\n image_strong_aug = np.array(self.strong_augmentation(image_pil))\n dataset_dict[\"image\"] = torch.as_tensor(\n np.ascontiguousarray(image_strong_aug.transpose(2, 0, 1))\n )\n\n dataset_dict_key = copy.deepcopy(dataset_dict)\n dataset_dict_key[\"image\"] = torch.as_tensor(\n np.ascontiguousarray(image_weak_aug.transpose(2, 0, 1))\n )\n assert dataset_dict[\"image\"].size(1) == dataset_dict_key[\"image\"].size(1)\n assert dataset_dict[\"image\"].size(2) == dataset_dict_key[\"image\"].size(2)\n return (dataset_dict, dataset_dict_key)\n" }, { "path": "adapteacher/data/datasets/builtin.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport os\nimport contextlib\nfrom detectron2.data import DatasetCatalog, MetadataCatalog\nfrom fvcore.common.timer import Timer\n# from fvcore.common.file_io import PathManager\nfrom iopath.common.file_io import PathManager\n\nfrom detectron2.data.datasets.pascal_voc import register_pascal_voc\nfrom detectron2.data.datasets.builtin_meta import _get_builtin_metadata\nfrom .cityscapes_foggy import load_cityscapes_instances\nimport io\nimport logging\n\nlogger = logging.getLogger(__name__)\n\nJSON_ANNOTATIONS_DIR = \"\"\n_SPLITS_COCO_FORMAT = {}\n_SPLITS_COCO_FORMAT[\"coco\"] = {\n \"coco_2017_unlabel\": (\n \"coco/unlabeled2017\",\n \"coco/annotations/image_info_unlabeled2017.json\",\n ),\n \"coco_2017_for_voc20\": (\n \"coco\",\n \"coco/annotations/google/instances_unlabeledtrainval20class.json\",\n ),\n}\n\n\ndef register_coco_unlabel(root):\n for _, splits_per_dataset in _SPLITS_COCO_FORMAT.items():\n for key, (image_root, json_file) in splits_per_dataset.items():\n meta = {}\n register_coco_unlabel_instances(\n key, meta, os.path.join(root, json_file), os.path.join(root, image_root)\n )\n\n\ndef register_coco_unlabel_instances(name, metadata, json_file, image_root):\n \"\"\"\n Register a dataset in COCO's json annotation format for\n instance detection, instance segmentation and keypoint detection.\n (i.e., Type 1 and 2 in http://cocodataset.org/#format-data.\n `instances*.json` and `person_keypoints*.json` in the dataset).\n\n This is an example of how to register a new dataset.\n You can do something similar to this function, to register new datasets.\n\n Args:\n name (str): the name that identifies a dataset, e.g. \"coco_2014_train\".\n metadata (dict): extra metadata associated with this dataset. You can\n leave it as an empty dict.\n json_file (str): path to the json instance annotation file.\n image_root (str or path-like): directory which contains all the images.\n \"\"\"\n assert isinstance(name, str), name\n assert isinstance(json_file, (str, os.PathLike)), json_file\n assert isinstance(image_root, (str, os.PathLike)), image_root\n\n # 1. register a function which returns dicts\n DatasetCatalog.register(\n name, lambda: load_coco_unlabel_json(json_file, image_root, name)\n )\n\n # 2. Optionally, add metadata about this dataset,\n # since they might be useful in evaluation, visualization or logging\n MetadataCatalog.get(name).set(\n json_file=json_file, image_root=image_root, evaluator_type=\"coco\", **metadata\n )\n\n\ndef load_coco_unlabel_json(\n json_file, image_root, dataset_name=None, extra_annotation_keys=None\n):\n from pycocotools.coco import COCO\n\n timer = Timer()\n json_file = PathManager.get_local_path(json_file)\n with contextlib.redirect_stdout(io.StringIO()):\n coco_api = COCO(json_file)\n if timer.seconds() > 1:\n logger.info(\n \"Loading {} takes {:.2f} seconds.\".format(json_file, timer.seconds())\n )\n\n id_map = None\n # sort indices for reproducible results\n img_ids = sorted(coco_api.imgs.keys())\n\n imgs = coco_api.loadImgs(img_ids)\n\n logger.info(\"Loaded {} images in COCO format from {}\".format(len(imgs), json_file))\n\n dataset_dicts = []\n\n for img_dict in imgs:\n record = {}\n record[\"file_name\"] = os.path.join(image_root, img_dict[\"file_name\"])\n record[\"height\"] = img_dict[\"height\"]\n record[\"width\"] = img_dict[\"width\"]\n image_id = record[\"image_id\"] = img_dict[\"id\"]\n\n dataset_dicts.append(record)\n\n return dataset_dicts\n\n\n_root = os.getenv(\"DETECTRON2_DATASETS\", \"datasets\")\nregister_coco_unlabel(_root)\n\n\n# ==== Predefined splits for raw cityscapes foggy images ===========\n_RAW_CITYSCAPES_SPLITS = {\n # \"cityscapes_foggy_{task}_train\": (\"cityscape_foggy/leftImg8bit/train/\", \"cityscape_foggy/gtFine/train/\"),\n # \"cityscapes_foggy_{task}_val\": (\"cityscape_foggy/leftImg8bit/val/\", \"cityscape_foggy/gtFine/val/\"),\n # \"cityscapes_foggy_{task}_test\": (\"cityscape_foggy/leftImg8bit/test/\", \"cityscape_foggy/gtFine/test/\"),\n \"cityscapes_foggy_train\": (\"cityscapes_foggy/leftImg8bit/train/\", \"cityscapes_foggy/gtFine/train/\"),\n \"cityscapes_foggy_val\": (\"cityscapes_foggy/leftImg8bit/val/\", \"cityscapes_foggy/gtFine/val/\"),\n \"cityscapes_foggy_test\": (\"cityscapes_foggy/leftImg8bit/test/\", \"cityscapes_foggy/gtFine/test/\"),\n}\n\n\ndef register_all_cityscapes_foggy(root):\n # root = \"manifold://mobile_vision_dataset/tree/yujheli/dataset\"\n for key, (image_dir, gt_dir) in _RAW_CITYSCAPES_SPLITS.items():\n meta = _get_builtin_metadata(\"cityscapes\")\n image_dir = os.path.join(root, image_dir)\n gt_dir = os.path.join(root, gt_dir)\n\n # inst_key = key.format(task=\"instance_seg\")\n inst_key = key\n # DatasetCatalog.register(\n # inst_key,\n # lambda x=image_dir, y=gt_dir: load_cityscapes_instances(\n # x, y, from_json=True, to_polygons=True\n # ),\n # )\n DatasetCatalog.register(\n inst_key,\n lambda x=image_dir, y=gt_dir: load_cityscapes_instances(\n x, y, from_json=False, to_polygons=False\n ),\n )\n # MetadataCatalog.get(inst_key).set(\n # image_dir=image_dir, gt_dir=gt_dir, evaluator_type=\"cityscapes_instance\", **meta\n # )\n # MetadataCatalog.get(inst_key).set(\n # image_dir=image_dir, gt_dir=gt_dir, evaluator_type=\"pascal_voc\", **meta\n # )\n MetadataCatalog.get(inst_key).set(\n image_dir=image_dir, gt_dir=gt_dir, evaluator_type=\"coco\", **meta\n )\n\n# ==== Predefined splits for Clipart (PASCAL VOC format) ===========\ndef register_all_clipart(root):\n # root = \"manifold://mobile_vision_dataset/tree/yujheli/dataset\"\n SPLITS = [\n (\"Clipart1k_train\", \"clipart\", \"train\"),\n (\"Clipart1k_test\", \"clipart\", \"test\"),\n ]\n for name, dirname, split in SPLITS:\n year = 2012\n register_pascal_voc(name, os.path.join(root, dirname), split, year)\n MetadataCatalog.get(name).evaluator_type = \"pascal_voc\"\n # MetadataCatalog.get(name).evaluator_type = \"coco\"\n\n# ==== Predefined splits for Watercolor (PASCAL VOC format) ===========\ndef register_all_water(root):\n # root = \"manifold://mobile_vision_dataset/tree/yujheli/dataset\"\n SPLITS = [\n (\"Watercolor_train\", \"watercolor\", \"train\"),\n (\"Watercolor_test\", \"watercolor\", \"test\"),\n ]\n for name, dirname, split in SPLITS:\n year = 2012\n # register_pascal_voc(name, os.path.join(root, dirname), split, year, class_names=[\"person\", \"dog\",\"bicycle\", \"bird\", \"car\", \"cat\"])\n register_pascal_voc(name, os.path.join(root, dirname), split, year)\n MetadataCatalog.get(name).evaluator_type = \"pascal_voc_water\"\n # MetadataCatalog.get(name).thing_classes = [\"person\", \"dog\",\"bike\", \"bird\", \"car\", \"cat\"]\n # MetadataCatalog.get(name).thing_classes = [\"person\", \"dog\",\"bicycle\", \"bird\", \"car\", \"cat\"]\n # MetadataCatalog.get(name).evaluator_type = \"coco\"\n\nregister_all_cityscapes_foggy(_root)\nregister_all_clipart(_root)\nregister_all_water(_root)\n\n" }, { "path": "adapteacher/data/datasets/cityscapes_foggy.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates.\nimport functools\nimport json\nimport logging\nimport multiprocessing as mp\nimport numpy as np\nimport os\nfrom itertools import chain\nimport pycocotools.mask as mask_util\nfrom PIL import Image\n\nfrom detectron2.structures import BoxMode\nfrom detectron2.utils.comm import get_world_size\nfrom detectron2.utils.file_io import PathManager\nfrom detectron2.utils.logger import setup_logger\n\ntry:\n import cv2 # noqa\nexcept ImportError:\n # OpenCV is an optional dependency at the moment\n pass\n\n\nlogger = logging.getLogger(__name__)\n\nload_only_002 = False\n\ndef _get_cityscapes_files(image_dir, gt_dir):\n files = []\n # scan through the directory\n cities = PathManager.ls(image_dir)\n logger.info(f\"{len(cities)} cities found in '{image_dir}'.\")\n for city in cities:\n city_img_dir = os.path.join(image_dir, city)\n city_gt_dir = os.path.join(gt_dir, city)\n for basename in PathManager.ls(city_img_dir):\n if load_only_002 and '0.02.png' not in basename:\n continue\n image_file = os.path.join(city_img_dir, basename)\n\n # suffix = \"leftImg8bit.png\"\n # assert basename.endswith(suffix), basename\n # basename = basename[: -len(suffix)]\n\n suffix = 'leftImg8bit_foggy'\n basename = basename.split(suffix)[0]\n\n instance_file = os.path.join(city_gt_dir, basename + \"gtFine_instanceIds.png\")\n label_file = os.path.join(city_gt_dir, basename + \"gtFine_labelIds.png\")\n json_file = os.path.join(city_gt_dir, basename + \"gtFine_polygons.json\")\n\n files.append((image_file, instance_file, label_file, json_file))\n assert len(files), \"No images found in {}\".format(image_dir)\n for f in files[0]:\n assert PathManager.isfile(f), f\n return files\n\n\ndef load_cityscapes_instances(image_dir, gt_dir, from_json=True, to_polygons=True):\n \"\"\"\n Args:\n image_dir (str): path to the raw dataset. e.g., \"~/cityscapes/leftImg8bit/train\".\n gt_dir (str): path to the raw annotations. e.g., \"~/cityscapes/gtFine/train\".\n from_json (bool): whether to read annotations from the raw json file or the png files.\n to_polygons (bool): whether to represent the segmentation as polygons\n (COCO's format) instead of masks (cityscapes's format).\n\n Returns:\n list[dict]: a list of dicts in Detectron2 standard format. (See\n `Using Custom Datasets `_ )\n \"\"\"\n if from_json:\n assert to_polygons, (\n \"Cityscapes's json annotations are in polygon format. \"\n \"Converting to mask format is not supported now.\"\n )\n files = _get_cityscapes_files(image_dir, gt_dir)\n\n logger.info(\"Preprocessing cityscapes annotations ...\")\n # This is still not fast: all workers will execute duplicate works and will\n # take up to 10m on a 8GPU server.\n pool = mp.Pool(processes=max(mp.cpu_count() // get_world_size() // 2, 4))\n\n ret = pool.map(\n functools.partial(_cityscapes_files_to_dict, from_json=from_json, to_polygons=to_polygons),\n files,\n )\n logger.info(\"Loaded {} images from {}\".format(len(ret), image_dir))\n pool.close()\n\n # Map cityscape ids to contiguous ids\n from cityscapesscripts.helpers.labels import labels\n\n labels = [l for l in labels if l.hasInstances and not l.ignoreInEval]\n dataset_id_to_contiguous_id = {l.id: idx for idx, l in enumerate(labels)}\n for dict_per_image in ret:\n for anno in dict_per_image[\"annotations\"]:\n anno[\"category_id\"] = dataset_id_to_contiguous_id[anno[\"category_id\"]]\n return ret\n\n\ndef load_cityscapes_semantic(image_dir, gt_dir):\n \"\"\"\n Args:\n image_dir (str): path to the raw dataset. e.g., \"~/cityscapes/leftImg8bit/train\".\n gt_dir (str): path to the raw annotations. e.g., \"~/cityscapes/gtFine/train\".\n\n Returns:\n list[dict]: a list of dict, each has \"file_name\" and\n \"sem_seg_file_name\".\n \"\"\"\n ret = []\n # gt_dir is small and contain many small files. make sense to fetch to local first\n gt_dir = PathManager.get_local_path(gt_dir)\n for image_file, _, label_file, json_file in _get_cityscapes_files(image_dir, gt_dir):\n label_file = label_file.replace(\"labelIds\", \"labelTrainIds\")\n\n with PathManager.open(json_file, \"r\") as f:\n jsonobj = json.load(f)\n ret.append(\n {\n \"file_name\": image_file,\n \"sem_seg_file_name\": label_file,\n \"height\": jsonobj[\"imgHeight\"],\n \"width\": jsonobj[\"imgWidth\"],\n }\n )\n assert len(ret), f\"No images found in {image_dir}!\"\n assert PathManager.isfile(\n ret[0][\"sem_seg_file_name\"]\n ), \"Please generate labelTrainIds.png with cityscapesscripts/preparation/createTrainIdLabelImgs.py\" # noqa\n return ret\n\n\ndef _cityscapes_files_to_dict(files, from_json, to_polygons):\n \"\"\"\n Parse cityscapes annotation files to a instance segmentation dataset dict.\n\n Args:\n files (tuple): consists of (image_file, instance_id_file, label_id_file, json_file)\n from_json (bool): whether to read annotations from the raw json file or the png files.\n to_polygons (bool): whether to represent the segmentation as polygons\n (COCO's format) instead of masks (cityscapes's format).\n\n Returns:\n A dict in Detectron2 Dataset format.\n \"\"\"\n from cityscapesscripts.helpers.labels import id2label, name2label\n\n image_file, instance_id_file, _, json_file = files\n\n annos = []\n\n if from_json:\n from shapely.geometry import MultiPolygon, Polygon\n\n with PathManager.open(json_file, \"r\") as f:\n jsonobj = json.load(f)\n ret = {\n \"file_name\": image_file,\n \"image_id\": os.path.basename(image_file),\n \"height\": jsonobj[\"imgHeight\"],\n \"width\": jsonobj[\"imgWidth\"],\n }\n\n # `polygons_union` contains the union of all valid polygons.\n polygons_union = Polygon()\n\n # CityscapesScripts draw the polygons in sequential order\n # and each polygon *overwrites* existing ones. See\n # (https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/preparation/json2instanceImg.py) # noqa\n # We use reverse order, and each polygon *avoids* early ones.\n # This will resolve the ploygon overlaps in the same way as CityscapesScripts.\n for obj in jsonobj[\"objects\"][::-1]:\n if \"deleted\" in obj: # cityscapes data format specific\n continue\n label_name = obj[\"label\"]\n\n try:\n label = name2label[label_name]\n except KeyError:\n if label_name.endswith(\"group\"): # crowd area\n label = name2label[label_name[: -len(\"group\")]]\n else:\n raise\n if label.id < 0: # cityscapes data format\n continue\n\n # Cityscapes's raw annotations uses integer coordinates\n # Therefore +0.5 here\n poly_coord = np.asarray(obj[\"polygon\"], dtype=\"f4\") + 0.5\n # CityscapesScript uses PIL.ImageDraw.polygon to rasterize\n # polygons for evaluation. This function operates in integer space\n # and draws each pixel whose center falls into the polygon.\n # Therefore it draws a polygon which is 0.5 \"fatter\" in expectation.\n # We therefore dilate the input polygon by 0.5 as our input.\n poly = Polygon(poly_coord).buffer(0.5, resolution=4)\n\n if not label.hasInstances or label.ignoreInEval:\n # even if we won't store the polygon it still contributes to overlaps resolution\n polygons_union = polygons_union.union(poly)\n continue\n\n # Take non-overlapping part of the polygon\n poly_wo_overlaps = poly.difference(polygons_union)\n if poly_wo_overlaps.is_empty:\n continue\n polygons_union = polygons_union.union(poly)\n\n anno = {}\n anno[\"iscrowd\"] = label_name.endswith(\"group\")\n anno[\"category_id\"] = label.id\n\n if isinstance(poly_wo_overlaps, Polygon):\n poly_list = [poly_wo_overlaps]\n elif isinstance(poly_wo_overlaps, MultiPolygon):\n poly_list = poly_wo_overlaps.geoms\n else:\n raise NotImplementedError(\"Unknown geometric structure {}\".format(poly_wo_overlaps))\n\n poly_coord = []\n for poly_el in poly_list:\n # COCO API can work only with exterior boundaries now, hence we store only them.\n # TODO: store both exterior and interior boundaries once other parts of the\n # codebase support holes in polygons.\n poly_coord.append(list(chain(*poly_el.exterior.coords)))\n anno[\"segmentation\"] = poly_coord\n (xmin, ymin, xmax, ymax) = poly_wo_overlaps.bounds\n\n anno[\"bbox\"] = (xmin, ymin, xmax, ymax)\n anno[\"bbox_mode\"] = BoxMode.XYXY_ABS\n\n annos.append(anno)\n else:\n # See also the official annotation parsing scripts at\n # https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/instances2dict.py # noqa\n with PathManager.open(instance_id_file, \"rb\") as f:\n inst_image = np.asarray(Image.open(f), order=\"F\")\n # ids < 24 are stuff labels (filtering them first is about 5% faster)\n flattened_ids = np.unique(inst_image[inst_image >= 24])\n\n ret = {\n \"file_name\": image_file,\n \"image_id\": os.path.basename(image_file),\n \"height\": inst_image.shape[0],\n \"width\": inst_image.shape[1],\n }\n\n for instance_id in flattened_ids:\n # For non-crowd annotations, instance_id // 1000 is the label_id\n # Crowd annotations have <1000 instance ids\n label_id = instance_id // 1000 if instance_id >= 1000 else instance_id\n label = id2label[label_id]\n if not label.hasInstances or label.ignoreInEval:\n continue\n\n anno = {}\n anno[\"iscrowd\"] = instance_id < 1000\n anno[\"category_id\"] = label.id\n\n mask = np.asarray(inst_image == instance_id, dtype=np.uint8, order=\"F\")\n\n inds = np.nonzero(mask)\n ymin, ymax = inds[0].min(), inds[0].max()\n xmin, xmax = inds[1].min(), inds[1].max()\n anno[\"bbox\"] = (xmin, ymin, xmax, ymax)\n if xmax <= xmin or ymax <= ymin:\n continue\n anno[\"bbox_mode\"] = BoxMode.XYXY_ABS\n if to_polygons:\n # This conversion comes from D4809743 and D5171122,\n # when Mask-RCNN was first developed.\n contours = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[\n -2\n ]\n polygons = [c.reshape(-1).tolist() for c in contours if len(c) >= 3]\n # opencv's can produce invalid polygons\n if len(polygons) == 0:\n continue\n anno[\"segmentation\"] = polygons\n else:\n anno[\"segmentation\"] = mask_util.encode(mask[:, :, None])[0]\n annos.append(anno)\n ret[\"annotations\"] = annos\n return ret\n" }, { "path": "adapteacher/data/detection_utils.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport logging\nimport torchvision.transforms as transforms\nfrom adapteacher.data.transforms.augmentation_impl import (\n GaussianBlur,\n)\n\n\ndef build_strong_augmentation(cfg, is_train):\n \"\"\"\n Create a list of :class:`Augmentation` from config.\n Now it includes resizing and flipping.\n\n Returns:\n list[Augmentation]\n \"\"\"\n\n logger = logging.getLogger(__name__)\n augmentation = []\n if is_train:\n # This is simialr to SimCLR https://arxiv.org/abs/2002.05709\n augmentation.append(\n transforms.RandomApply([transforms.ColorJitter(0.4, 0.4, 0.4, 0.1)], p=0.8)\n )\n augmentation.append(transforms.RandomGrayscale(p=0.2))\n augmentation.append(transforms.RandomApply([GaussianBlur([0.1, 2.0])], p=0.5))\n\n randcrop_transform = transforms.Compose(\n [\n transforms.ToTensor(),\n transforms.RandomErasing(\n p=0.7, scale=(0.05, 0.2), ratio=(0.3, 3.3), value=\"random\"\n ),\n transforms.RandomErasing(\n p=0.5, scale=(0.02, 0.2), ratio=(0.1, 6), value=\"random\"\n ),\n transforms.RandomErasing(\n p=0.3, scale=(0.02, 0.2), ratio=(0.05, 8), value=\"random\"\n ),\n transforms.ToPILImage(),\n ]\n )\n augmentation.append(randcrop_transform)\n\n logger.info(\"Augmentations used in training: \" + str(augmentation))\n return transforms.Compose(augmentation)" }, { "path": "adapteacher/data/transforms/augmentation_impl.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport random\nfrom PIL import ImageFilter\n\n\nclass GaussianBlur:\n \"\"\"\n Gaussian blur augmentation in SimCLR https://arxiv.org/abs/2002.05709\n Adapted from MoCo:\n https://github.com/facebookresearch/moco/blob/master/moco/loader.py\n Note that this implementation does not seem to be exactly the same as\n described in SimCLR.\n \"\"\"\n\n def __init__(self, sigma=[0.1, 2.0]):\n self.sigma = sigma\n\n def __call__(self, x):\n sigma = random.uniform(self.sigma[0], self.sigma[1])\n x = x.filter(ImageFilter.GaussianBlur(radius=sigma))\n return x\n" }, { "path": "adapteacher/engine/hooks.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom detectron2.engine.hooks import HookBase\nimport detectron2.utils.comm as comm\n\nimport torch\nimport numpy as np\nfrom contextlib import contextmanager\n\n\nclass LossEvalHook(HookBase):\n def __init__(self, eval_period, model, data_loader, model_output, model_name=\"\"):\n self._model = model\n self._period = eval_period\n self._data_loader = data_loader\n self._model_output = model_output\n self._model_name = model_name\n\n def _do_loss_eval(self):\n record_acc_dict = {}\n with inference_context(self._model), torch.no_grad():\n for _, inputs in enumerate(self._data_loader):\n record_dict = self._get_loss(inputs, self._model)\n # accumulate the losses\n for loss_type in record_dict.keys():\n if loss_type not in record_acc_dict.keys():\n record_acc_dict[loss_type] = record_dict[loss_type]\n else:\n record_acc_dict[loss_type] += record_dict[loss_type]\n # average\n for loss_type in record_acc_dict.keys():\n record_acc_dict[loss_type] = record_acc_dict[loss_type] / len(\n self._data_loader\n )\n\n # divide loss and other metrics\n loss_acc_dict = {}\n for key in record_acc_dict.keys():\n if key[:4] == \"loss\":\n loss_acc_dict[key] = record_acc_dict[key]\n\n # only output the results of major node\n if comm.is_main_process():\n total_losses_reduced = sum(loss for loss in loss_acc_dict.values())\n self.trainer.storage.put_scalar(\n \"val_total_loss_val\" + self._model_name, total_losses_reduced\n )\n\n record_acc_dict = {\n \"val_\" + k + self._model_name: record_acc_dict[k]\n for k in record_acc_dict.keys()\n }\n\n if len(record_acc_dict) > 1:\n self.trainer.storage.put_scalars(**record_acc_dict)\n\n def _get_loss(self, data, model):\n if self._model_output == \"loss_only\":\n record_dict = model(data)\n\n elif self._model_output == \"loss_proposal\":\n record_dict, _, _, _ = model(data, branch=\"val_loss\", val_mode=True)\n\n elif self._model_output == \"meanteacher\":\n record_dict, _, _, _, _ = model(data)\n\n metrics_dict = {\n k: v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v)\n for k, v in record_dict.items()\n }\n\n return metrics_dict\n\n def _write_losses(self, metrics_dict):\n # gather metrics among all workers for logging\n # This assumes we do DDP-style training, which is currently the only\n # supported method in detectron2.\n comm.synchronize()\n all_metrics_dict = comm.gather(metrics_dict, dst=0)\n\n if comm.is_main_process():\n # average the rest metrics\n metrics_dict = {\n \"val_\" + k: np.mean([x[k] for x in all_metrics_dict])\n for k in all_metrics_dict[0].keys()\n }\n total_losses_reduced = sum(loss for loss in metrics_dict.values())\n\n self.trainer.storage.put_scalar(\"val_total_loss_val\", total_losses_reduced)\n if len(metrics_dict) > 1:\n self.trainer.storage.put_scalars(**metrics_dict)\n\n def _detect_anomaly(self, losses, loss_dict):\n if not torch.isfinite(losses).all():\n raise FloatingPointError(\n \"Loss became infinite or NaN at iteration={}!\\nloss_dict = {}\".format(\n self.trainer.iter, loss_dict\n )\n )\n\n def after_step(self):\n next_iter = self.trainer.iter + 1\n is_final = next_iter == self.trainer.max_iter\n if is_final or (self._period > 0 and next_iter % self._period == 0):\n self._do_loss_eval()\n\n\n@contextmanager\ndef inference_context(model):\n \"\"\"\n A context where the model is temporarily changed to eval mode,\n and restored to previous mode afterwards.\n\n Args:\n model: a torch Module\n \"\"\"\n training_mode = model.training\n model.eval()\n yield\n model.train(training_mode)\n" }, { "path": "adapteacher/engine/probe.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom detectron2.structures import pairwise_iou\n\nclass OpenMatchTrainerProbe:\n def __init__(self, cfg):\n self.BOX_AP = 0.5\n self.NUM_CLASSES = cfg.MODEL.ROI_HEADS.NUM_CLASSES\n # self.bbox_stat_list = ['compute_fp_gtoutlier', 'compute_num_box', 'compute_ood_acc']\n \n def bbox_stat(self, unlabel_gt, unlabel_pseudo, name, bbox_stat_list):\n stats = {}\n \n sum_gpu_names = []\n for metric in bbox_stat_list:\n stats_per, sum_gpu_names_per = getattr(\n self, metric)(unlabel_gt, unlabel_pseudo, name)\n stats.update(stats_per)\n sum_gpu_names.extend(sum_gpu_names_per)\n \n return stats, sum_gpu_names\n \n def compute_fp_gtoutlier(self, unlabel_gt, unlabel_pseudo, name):\n num_gt_ood_object = 0\n num_gt_fp_ood_object = 0\n sum_iou = 0.0\n sum_gpu_names = []\n results = {}\n \n if len(unlabel_gt) != 0:\n for gt, pseudo in zip(unlabel_gt, unlabel_pseudo):\n # import pdb; pdb. set_trace() \n if name == \"pred\":\n pp_boxes = pseudo.pred_boxes\n elif name == \"pseudo_conf\" or name == \"pseudo_ood\":\n # filter predicted ood box when evaluating this metric\n pseudo = pseudo[pseudo.gt_classes != -1]\n pp_boxes = pseudo.gt_boxes\n \n else:\n raise ValueError(\"Unknown name for probe roi bbox.\")\n \n if len(gt) != 0 and len(pseudo) != 0:\n max_iou, max_idx = pairwise_iou(\n gt.gt_boxes.to('cuda'), pp_boxes).max(1)\n ood_idx = (gt.gt_classes == -1)\n \n num_gt_ood_object += ood_idx.sum().item()\n num_gt_fp_ood_object += (max_iou[ood_idx]\n > self.BOX_AP).sum().item()\n sum_iou += max_iou[ood_idx].sum().item()\n \n elif len(gt) != 0 and len(pseudo) == 0:\n ood_idx = (gt.gt_classes == -1)\n num_gt_ood_object += ood_idx.shape[0]\n \n results = {'Analysis_'+name+'/num_gt_ood_object': num_gt_ood_object,\n 'Analysis_'+name+'/num_gt_fp_ood_object': num_gt_fp_ood_object,\n 'Analysis_'+name+'/sum_iou': sum_iou}\n \n sum_gpu_names.extend(list(results.keys()))\n \n return results, sum_gpu_names\n \n def compute_num_box(self, unlabel_gt, unlabel_pseudo, name, processed=False):\n num_bbox = 0.0\n size_bbox = 0.0\n avg_conf = 0.0\n \n # measure in and out box for openset SS-OD\n num_bbox_in = 0.0\n num_bbox_out = 0.0\n num_bg = 0.0\n \n # when ground-truth is missing in unlabeled data\n if len(unlabel_gt) == 0:\n for pp_roi in unlabel_pseudo:\n if name == \"pred\":\n pp_boxes = pp_roi.pred_boxes\n pp_classes = pp_roi.pred_classes\n pp_scores = pp_roi.scores\n elif name == \"pseudo_conf\" or name == \"pseudo_ood\":\n pp_boxes = pp_roi.gt_boxes\n pp_classes = pp_roi.gt_classes\n pp_scores = pp_roi.scores\n elif name == \"gt\":\n pp_boxes = pp_roi.gt_boxes\n pp_classes = pp_roi.gt_classes\n else:\n raise ValueError(\"Unknown name for probe roi bbox.\")\n # all boxes (in + out boxes)\n if len(pp_roi) != 0:\n # bbox number and size\n num_bbox += len(pp_roi)\n size_bbox += pp_boxes.area().mean().item()\n # average box confidence\n if name != \"gt\":\n avg_conf += pp_scores.mean()\n else:\n num_bbox += 0\n size_bbox += torch.tensor(0).cuda()\n num_valid_img = len(unlabel_pseudo)\n else:\n # with ground-truth\n num_valid_img = 0\n for gt, pp_roi in zip(unlabel_gt, unlabel_pseudo):\n \n if name == \"pred\":\n pp_boxes = pp_roi.pred_boxes\n pp_classes = pp_roi.pred_classes\n pp_scores = pp_roi.scores\n elif name == \"pseudo_conf\" or name == \"pseudo_ood\":\n # filter out ood pseudo-box when doing analysis\n pp_roi = pp_roi[pp_roi.gt_classes != -1]\n \n pp_boxes = pp_roi.gt_boxes\n pp_classes = pp_roi.gt_classes\n pp_scores = pp_roi.scores\n elif name == \"gt\":\n pp_boxes = pp_roi.gt_boxes\n pp_classes = pp_roi.gt_classes\n else:\n raise ValueError(\"Unknown name for probe roi bbox.\")\n \n # all boxes (in + out boxes)\n if len(pp_roi) != 0:\n # bbox number and size\n num_bbox += len(pp_roi)\n size_bbox += pp_boxes.area().mean().item()\n \n # average box confidence\n if name != \"gt\":\n avg_conf += pp_scores.mean()\n else:\n num_bbox += 0\n size_bbox += torch.tensor(0).cuda()\n \n # in and out class\n if name == \"gt\":\n pp_roi_in = pp_roi[pp_classes != -1]\n num_bbox_in += len(pp_roi_in)\n \n pp_roi_out = pp_roi[pp_classes == -1]\n num_bbox_out += len(pp_roi_out)\n num_valid_img += 1\n \n \n elif name == \"pred\" or name == \"pseudo_conf\" or name == \"pseudo_ood\":\n \n if len(gt.gt_boxes.to('cuda'))>0 and len(pp_boxes) > 0:\n max_iou, max_idx = pairwise_iou(gt.gt_boxes.to('cuda'), pp_boxes).max(0)\n \n # for the ground-truth label for each pseudo-box\n gtclass4pseudo = gt.gt_classes[max_idx]\n matchgtbox = max_iou > 0.5\n \n # compute the number of boxes (background, inlier, outlier)\n num_bg += (~matchgtbox).sum().item()\n num_bbox_in += (gtclass4pseudo[matchgtbox]\n != -1).sum().item()\n num_bbox_out += (gtclass4pseudo[matchgtbox]\n == -1).sum().item()\n num_valid_img += 1\n \n else:\n raise ValueError(\"Unknown name for probe roi bbox.\")\n \n box_probe = {}\n if processed == True:\n name = name+\"processed\"\n\n if num_bbox == 0:\n return box_probe, []\n if num_valid_img >0 :\n box_probe[\"Analysis_\" + name + \"/Num_bbox\"] = num_bbox / \\\n num_valid_img\n box_probe[\"Analysis_\" + name + \"/Size_bbox\"] = size_bbox / \\\n num_valid_img\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_inlier\"] = num_bbox_in / num_valid_img\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_outlier\"] = num_bbox_out / num_valid_img\n \n if name != \"gt\": # prediciton, background number\n box_probe[\"Analysis_\" + name + \"/Conf\"] = avg_conf / \\\n num_valid_img\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_background\"] = num_bg / num_valid_img\n box_probe[\"Analysis_\" + name +\n \"/background_fp_ratio\"] = num_bg / num_bbox\n box_probe[\"Analysis_\" + name +\n \"/background_tp_ratio\"] = num_bbox_in / num_bbox\n else:\n box_probe[\"Analysis_\" + name + \"/Num_bbox\"] = 0.0\n box_probe[\"Analysis_\" + name + \"/Size_bbox\"] = 0.0\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_inlier\"] = 0.0\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_outlier\"] = 0.0\n \n if name != \"gt\": # prediciton, background number\n box_probe[\"Analysis_\" + name + \"/Conf\"] = 0.0\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_background\"] = 0.0\n box_probe[\"Analysis_\" + name +\n \"/background_fp_ratio\"] = num_bg / num_bbox\n box_probe[\"Analysis_\" + name +\n \"/background_tp_ratio\"] = num_bbox_in / num_bbox\n\n return box_probe, []\n \n def compute_ood_acc(self, unlabel_gt, unlabel_pseudo, name, BOX_IOU=0.5):\n results = {}\n sum_gpu_names = []\n \n if len(unlabel_gt) != 0:\n for metric in ['acc_outlier', 'recall_outlier']:\n for samples in ['_fg', '_all']:\n for fraction_part in ['_nume', '_deno']:\n results[metric+samples+fraction_part] = 0.0\n \n for gt, pred in zip(unlabel_gt, unlabel_pseudo):\n if name == \"pred\":\n pp_boxes = pred.pred_boxes\n pp_ood_scores = pred.ood_scores\n \n elif name == \"pseudo_conf\" or name == \"pseudo_ood\":\n # assume these outlier are suppressed\n pred = pred[pred.gt_classes != -1]\n \n pp_boxes = pred.gt_boxes\n pp_ood_scores = pred.ood_scores\n \n else:\n raise ValueError(\"Unknown name for probe roi bbox.\")\n \n if len(gt) != 0 and len(pred) != 0:\n # find the most overlapped ground-truth box for each pseudo-box\n max_iou, max_idx = pairwise_iou(\n gt.gt_boxes.to('cuda'), pp_boxes).max(0)\n \n # ignore background instances\n find_fg_mask = max_iou > BOX_IOU\n if find_fg_mask.sum() > 0:\n gt_corres = gt[max_idx].gt_classes.to(\"cuda\")\n \n gt_outlier = (gt_corres[find_fg_mask] == -1)\n pred_outlier = pp_ood_scores[find_fg_mask][:, 0] > 0.5\n \n # accurcay of ood detection (foreground)\n # acc_outlier_fg = (pred_outlier == gt_outlier).sum() /find_fg_mask.sum()\n results['acc_outlier_fg_nume'] += (\n pred_outlier == gt_outlier).sum()\n results['acc_outlier_fg_deno'] += find_fg_mask.sum()\n \n # recall of ood detection (foreground)\n # recall_outlier_fg = (pred_outlier[gt_outlier] == gt_outlier[gt_outlier]).sum() /gt_outlier.sum()\n results['recall_outlier_fg_nume'] += (\n pred_outlier[gt_outlier] == gt_outlier[gt_outlier]).sum()\n results['recall_outlier_fg_deno'] += gt_outlier.sum()\n \n # Regard backgound gt as outlier\n gt_corres = gt[max_idx].gt_classes.to(\"cuda\")\n # convert all background gt as outlier\n gt_corres[~find_fg_mask] = -1\n gt_outlier = gt_corres == -1\n \n pred_outlier = pp_ood_scores[:, 0] > 0.5\n \n # accurcay of ood detection (all)\n # acc_outlier_all = (pred_outlier == gt_outlier).sum() /len(pred)\n results['acc_outlier_all_nume'] += (\n pred_outlier == gt_outlier).sum()\n results['acc_outlier_all_deno'] += len(pred)\n \n # recall of ood detection (all)\n # recall_outlier_all = (pred_outlier[gt_outlier] == gt_outlier[gt_outlier]).sum() /gt_outlier.sum()\n results['recall_outlier_all_nume'] += (\n pred_outlier[gt_outlier] == gt_outlier[gt_outlier]).sum()\n results['recall_outlier_all_deno'] += gt_outlier.sum()\n \n results = {'Analysis_'+name+'/'+k: v for k, v in results.items()}\n \n sum_gpu_names.extend(list(results.keys()))\n \n return results, sum_gpu_names\n\n\n\nimport torch\n\n\ndef probe(\n cfg,\n proposals_roih_unsup_k,\n unlabel_data_k,\n pesudo_proposals_roih_unsup_k,\n record_dict,\n):\n \"\"\"\n Probe for research development\n \"\"\"\n # [probe] roi result from weak branch (before pseudo-labeling)\n record_roih = probe_roih_bbox(\n proposals_roih_unsup_k, cfg.MODEL.ROI_HEADS.NUM_CLASSES, \"roih\"\n )\n record_dict.update(record_roih)\n\n # [probe] roi result after pseudo-labeling from weak branch\n record_roih_pseudo = probe_roih_bbox(\n pesudo_proposals_roih_unsup_k, cfg.MODEL.ROI_HEADS.NUM_CLASSES, \"roih_pseudo\"\n )\n record_dict.update(record_roih_pseudo)\n\n return record_dict\n\n\ndef probe_roih_bbox(proposals_roih, num_cls, name=\"\"):\n num_bbox = 0.0\n size_bbox = 0.0\n avg_conf = 0.0\n\n pred_cls_list = []\n for pp_roi in proposals_roih:\n if name == \"roih\":\n pp_boxes = pp_roi.pred_boxes\n pp_classes = pp_roi.pred_classes\n pp_scores = pp_roi.scores\n elif name == \"roih_pseudo\":\n pp_boxes = pp_roi.gt_boxes\n pp_classes = pp_roi.gt_classes\n pp_scores = pp_roi.scores\n elif name == \"gt\":\n pp_boxes = pp_roi.gt_boxes\n pp_classes = pp_roi.gt_classes\n else:\n raise ValueError(f\"Unknown name for probe roi bbox '{name}'\")\n\n device = pp_classes.device\n\n if pp_roi:\n # bbox number and size\n num_bbox += len(pp_roi)\n size_bbox += pp_boxes.area().mean()\n\n if name != \"gt\":\n avg_conf += pp_scores.mean()\n\n # ratio of majority class\n all_idx, cls_count = torch.unique(pp_classes, return_counts=True)\n major_cls_idx = all_idx[torch.argmax(cls_count)]\n major_cls_ratio = torch.max(cls_count).float() / pp_classes.numel()\n\n # cls_sum\n pred_cls_list.append(pp_classes)\n else:\n num_bbox += 0\n size_bbox += torch.tensor(0).to(device)\n major_cls_idx = torch.tensor(0).to(device)\n major_cls_ratio = torch.tensor(0).to(device)\n\n # boxes monitor\n box_probe = {}\n box_probe[\"bbox_probe_\" + name + \"/Num_bbox\"] = num_bbox / len(proposals_roih)\n box_probe[\"bbox_probe_\" + name + \"/Size_bbox\"] = size_bbox.item() / len(\n proposals_roih\n )\n\n if name != \"gt\":\n box_probe[\"bbox_probe_\" + name + \"/Conf\"] = avg_conf / len(proposals_roih)\n\n box_probe[\"bbox_probe_\" + name + \"/Ratio_major_cls_idx\"] = major_cls_idx.item()\n box_probe[\"bbox_probe_\" + name + \"/Ratio_major_cls\"] = major_cls_ratio.item()\n\n return box_probe" }, { "path": "adapteacher/engine/trainer.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport os\nimport time\nimport logging\nimport torch\nfrom torch.nn.parallel import DistributedDataParallel\nfrom fvcore.nn.precise_bn import get_bn_modules\nimport numpy as np\nfrom collections import OrderedDict\n\nimport detectron2.utils.comm as comm\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.engine import DefaultTrainer, SimpleTrainer, TrainerBase\nfrom detectron2.engine.train_loop import AMPTrainer\nfrom detectron2.utils.events import EventStorage\nfrom detectron2.evaluation import verify_results, DatasetEvaluators\n# from detectron2.evaluation import COCOEvaluator, verify_results, DatasetEvaluators\n\nfrom detectron2.data.dataset_mapper import DatasetMapper\nfrom detectron2.engine import hooks\nfrom detectron2.structures.boxes import Boxes\nfrom detectron2.structures.instances import Instances\nfrom detectron2.utils.env import TORCH_VERSION\nfrom detectron2.data import MetadataCatalog\n\nfrom adapteacher.data.build import (\n build_detection_semisup_train_loader,\n build_detection_test_loader,\n build_detection_semisup_train_loader_two_crops,\n)\nfrom adapteacher.data.dataset_mapper import DatasetMapperTwoCropSeparate\nfrom adapteacher.engine.hooks import LossEvalHook\nfrom adapteacher.modeling.meta_arch.ts_ensemble import EnsembleTSModel\nfrom adapteacher.checkpoint.detection_checkpoint import DetectionTSCheckpointer\nfrom adapteacher.solver.build import build_lr_scheduler\nfrom adapteacher.evaluation import PascalVOCDetectionEvaluator, COCOEvaluator\n\nfrom .probe import OpenMatchTrainerProbe\nimport copy\n\n\n# Supervised-only Trainer\nclass BaselineTrainer(DefaultTrainer):\n def __init__(self, cfg):\n \"\"\"\n Args:\n cfg (CfgNode):\n Use the custom checkpointer, which loads other backbone models\n with matching heuristics.\n \"\"\"\n cfg = DefaultTrainer.auto_scale_workers(cfg, comm.get_world_size())\n model = self.build_model(cfg)\n optimizer = self.build_optimizer(cfg, model)\n data_loader = self.build_train_loader(cfg)\n\n if comm.get_world_size() > 1:\n model = DistributedDataParallel(\n model, device_ids=[comm.get_local_rank()], broadcast_buffers=False\n )\n\n TrainerBase.__init__(self)\n self._trainer = (AMPTrainer if cfg.SOLVER.AMP.ENABLED else SimpleTrainer)(\n model, data_loader, optimizer\n )\n\n self.scheduler = self.build_lr_scheduler(cfg, optimizer)\n self.checkpointer = DetectionCheckpointer(\n model,\n cfg.OUTPUT_DIR,\n optimizer=optimizer,\n scheduler=self.scheduler,\n )\n self.start_iter = 0\n self.max_iter = cfg.SOLVER.MAX_ITER\n self.cfg = cfg\n\n self.register_hooks(self.build_hooks())\n\n def resume_or_load(self, resume=True):\n \"\"\"\n If `resume==True` and `cfg.OUTPUT_DIR` contains the last checkpoint (defined by\n a `last_checkpoint` file), resume from the file. Resuming means loading all\n available states (eg. optimizer and scheduler) and update iteration counter\n from the checkpoint. ``cfg.MODEL.WEIGHTS`` will not be used.\n Otherwise, this is considered as an independent training. The method will load model\n weights from the file `cfg.MODEL.WEIGHTS` (but will not load other states) and start\n from iteration 0.\n Args:\n resume (bool): whether to do resume or not\n \"\"\"\n checkpoint = self.checkpointer.resume_or_load(\n self.cfg.MODEL.WEIGHTS, resume=resume\n )\n if resume and self.checkpointer.has_checkpoint():\n self.start_iter = checkpoint.get(\"iteration\", -1) + 1\n # The checkpoint stores the training iteration that just finished, thus we start\n # at the next iteration (or iter zero if there's no checkpoint).\n if isinstance(self.model, DistributedDataParallel):\n # broadcast loaded data/model from the first rank, because other\n # machines may not have access to the checkpoint file\n if TORCH_VERSION >= (1, 7):\n self.model._sync_params_and_buffers()\n self.start_iter = comm.all_gather(self.start_iter)[0]\n\n def train_loop(self, start_iter: int, max_iter: int):\n \"\"\"\n Args:\n start_iter, max_iter (int): See docs above\n \"\"\"\n logger = logging.getLogger(__name__)\n logger.info(\"Starting training from iteration {}\".format(start_iter))\n\n self.iter = self.start_iter = start_iter\n self.max_iter = max_iter\n\n with EventStorage(start_iter) as self.storage:\n try:\n self.before_train()\n for self.iter in range(start_iter, max_iter):\n self.before_step()\n self.run_step()\n self.after_step()\n except Exception:\n logger.exception(\"Exception during training:\")\n raise\n finally:\n self.after_train()\n\n def run_step(self):\n self._trainer.iter = self.iter\n\n assert self.model.training, \"[SimpleTrainer] model was changed to eval mode!\"\n start = time.perf_counter()\n\n data = next(self._trainer._data_loader_iter)\n data_time = time.perf_counter() - start\n\n record_dict, _, _, _ = self.model(data, branch=\"supervised\")\n\n num_gt_bbox = 0.0\n for element in data:\n num_gt_bbox += len(element[\"instances\"])\n num_gt_bbox = num_gt_bbox / len(data)\n record_dict[\"bbox_num/gt_bboxes\"] = num_gt_bbox\n\n loss_dict = {}\n for key in record_dict.keys():\n if key[:4] == \"loss\" and key[-3:] != \"val\":\n loss_dict[key] = record_dict[key]\n\n losses = sum(loss_dict.values())\n\n metrics_dict = record_dict\n metrics_dict[\"data_time\"] = data_time\n self._write_metrics(metrics_dict)\n\n self.optimizer.zero_grad()\n losses.backward()\n self.optimizer.step()\n\n @classmethod\n def build_evaluator(cls, cfg, dataset_name, output_folder=None):\n if output_folder is None:\n output_folder = os.path.join(cfg.OUTPUT_DIR, \"inference\")\n evaluator_list = []\n evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type\n\n if evaluator_type == \"coco\":\n evaluator_list.append(COCOEvaluator(\n dataset_name, output_dir=output_folder))\n elif evaluator_type == \"pascal_voc\":\n return PascalVOCDetectionEvaluator(dataset_name)\n elif evaluator_type == \"pascal_voc_water\":\n return PascalVOCDetectionEvaluator(dataset_name, target_classnames=[\"bicycle\", \"bird\", \"car\", \"cat\", \"dog\", \"person\"])\n if len(evaluator_list) == 0:\n raise NotImplementedError(\n \"no Evaluator for the dataset {} with the type {}\".format(\n dataset_name, evaluator_type\n )\n )\n elif len(evaluator_list) == 1:\n return evaluator_list[0]\n\n return DatasetEvaluators(evaluator_list)\n\n @classmethod\n def build_train_loader(cls, cfg):\n return build_detection_semisup_train_loader(cfg, mapper=None)\n\n @classmethod\n def build_test_loader(cls, cfg, dataset_name):\n \"\"\"\n Returns:\n iterable\n \"\"\"\n return build_detection_test_loader(cfg, dataset_name)\n\n def build_hooks(self):\n \"\"\"\n Build a list of default hooks, including timing, evaluation,\n checkpointing, lr scheduling, precise BN, writing events.\n\n Returns:\n list[HookBase]:\n \"\"\"\n cfg = self.cfg.clone()\n cfg.defrost()\n cfg.DATALOADER.NUM_WORKERS = 0\n\n ret = [\n hooks.IterationTimer(),\n hooks.LRScheduler(self.optimizer, self.scheduler),\n hooks.PreciseBN(\n cfg.TEST.EVAL_PERIOD,\n self.model,\n self.build_train_loader(cfg),\n cfg.TEST.PRECISE_BN.NUM_ITER,\n )\n if cfg.TEST.PRECISE_BN.ENABLED and get_bn_modules(self.model)\n else None,\n ]\n\n if comm.is_main_process():\n ret.append(\n hooks.PeriodicCheckpointer(\n self.checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD\n )\n )\n\n def test_and_save_results():\n self._last_eval_results = self.test(self.cfg, self.model)\n return self._last_eval_results\n\n ret.append(hooks.EvalHook(cfg.TEST.EVAL_PERIOD, test_and_save_results))\n\n if comm.is_main_process():\n ret.append(hooks.PeriodicWriter(self.build_writers(), period=20))\n return ret\n\n def _write_metrics(self, metrics_dict: dict):\n \"\"\"\n Args:\n metrics_dict (dict): dict of scalar metrics\n \"\"\"\n metrics_dict = {\n k: v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v)\n for k, v in metrics_dict.items()\n }\n # gather metrics among all workers for logging\n # This assumes we do DDP-style training, which is currently the only\n # supported method in detectron2.\n all_metrics_dict = comm.gather(metrics_dict)\n\n if comm.is_main_process():\n if \"data_time\" in all_metrics_dict[0]:\n data_time = np.max([x.pop(\"data_time\")\n for x in all_metrics_dict])\n self.storage.put_scalar(\"data_time\", data_time)\n\n metrics_dict = {\n k: np.mean([x[k] for x in all_metrics_dict])\n for k in all_metrics_dict[0].keys()\n }\n\n loss_dict = {}\n for key in metrics_dict.keys():\n if key[:4] == \"loss\":\n loss_dict[key] = metrics_dict[key]\n\n total_losses_reduced = sum(loss for loss in loss_dict.values())\n\n self.storage.put_scalar(\"total_loss\", total_losses_reduced)\n if len(metrics_dict) > 1:\n self.storage.put_scalars(**metrics_dict)\n\n\n# Adaptive Teacher Trainer\nclass ATeacherTrainer(DefaultTrainer):\n def __init__(self, cfg):\n \"\"\"\n Args:\n cfg (CfgNode):\n Use the custom checkpointer, which loads other backbone models\n with matching heuristics.\n \"\"\"\n cfg = DefaultTrainer.auto_scale_workers(cfg, comm.get_world_size())\n data_loader = self.build_train_loader(cfg)\n\n # create an student model\n model = self.build_model(cfg)\n optimizer = self.build_optimizer(cfg, model)\n\n # create an teacher model\n model_teacher = self.build_model(cfg)\n self.model_teacher = model_teacher\n\n # For training, wrap with DDP. But don't need this for inference.\n if comm.get_world_size() > 1:\n model = DistributedDataParallel(\n model, device_ids=[comm.get_local_rank()], broadcast_buffers=False\n )\n\n TrainerBase.__init__(self)\n self._trainer = (AMPTrainer if cfg.SOLVER.AMP.ENABLED else SimpleTrainer)(\n model, data_loader, optimizer\n )\n self.scheduler = self.build_lr_scheduler(cfg, optimizer)\n\n # Ensemble teacher and student model is for model saving and loading\n ensem_ts_model = EnsembleTSModel(model_teacher, model)\n\n self.checkpointer = DetectionTSCheckpointer(\n ensem_ts_model,\n cfg.OUTPUT_DIR,\n optimizer=optimizer,\n scheduler=self.scheduler,\n )\n self.start_iter = 0\n self.max_iter = cfg.SOLVER.MAX_ITER\n self.cfg = cfg\n\n self.probe = OpenMatchTrainerProbe(cfg)\n self.register_hooks(self.build_hooks()) \n\n def resume_or_load(self, resume=True):\n \"\"\"\n If `resume==True` and `cfg.OUTPUT_DIR` contains the last checkpoint (defined by\n a `last_checkpoint` file), resume from the file. Resuming means loading all\n available states (eg. optimizer and scheduler) and update iteration counter\n from the checkpoint. ``cfg.MODEL.WEIGHTS`` will not be used.\n Otherwise, this is considered as an independent training. The method will load model\n weights from the file `cfg.MODEL.WEIGHTS` (but will not load other states) and start\n from iteration 0.\n Args:\n resume (bool): whether to do resume or not\n \"\"\"\n checkpoint = self.checkpointer.resume_or_load(\n self.cfg.MODEL.WEIGHTS, resume=resume\n )\n if resume and self.checkpointer.has_checkpoint():\n self.start_iter = checkpoint.get(\"iteration\", -1) + 1\n # The checkpoint stores the training iteration that just finished, thus we start\n # at the next iteration (or iter zero if there's no checkpoint).\n if isinstance(self.model, DistributedDataParallel):\n # broadcast loaded data/model from the first rank, because other\n # machines may not have access to the checkpoint file\n if TORCH_VERSION >= (1, 7):\n self.model._sync_params_and_buffers()\n self.start_iter = comm.all_gather(self.start_iter)[0]\n\n @classmethod\n def build_evaluator(cls, cfg, dataset_name, output_folder=None):\n if output_folder is None:\n output_folder = os.path.join(cfg.OUTPUT_DIR, \"inference\")\n evaluator_list = []\n evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type\n\n if evaluator_type == \"coco\":\n evaluator_list.append(COCOEvaluator(\n dataset_name, output_dir=output_folder))\n elif evaluator_type == \"pascal_voc\":\n return PascalVOCDetectionEvaluator(dataset_name)\n elif evaluator_type == \"pascal_voc_water\":\n return PascalVOCDetectionEvaluator(dataset_name, target_classnames=[\"bicycle\", \"bird\", \"car\", \"cat\", \"dog\", \"person\"])\n if len(evaluator_list) == 0:\n raise NotImplementedError(\n \"no Evaluator for the dataset {} with the type {}\".format(\n dataset_name, evaluator_type\n )\n )\n elif len(evaluator_list) == 1:\n return evaluator_list[0]\n\n return DatasetEvaluators(evaluator_list)\n\n @classmethod\n def build_train_loader(cls, cfg):\n mapper = DatasetMapperTwoCropSeparate(cfg, True)\n return build_detection_semisup_train_loader_two_crops(cfg, mapper)\n\n @classmethod\n def build_lr_scheduler(cls, cfg, optimizer):\n return build_lr_scheduler(cfg, optimizer)\n\n def train(self):\n self.train_loop(self.start_iter, self.max_iter)\n if hasattr(self, \"_last_eval_results\") and comm.is_main_process():\n verify_results(self.cfg, self._last_eval_results)\n return self._last_eval_results\n\n def train_loop(self, start_iter: int, max_iter: int):\n logger = logging.getLogger(__name__)\n logger.info(\"Starting training from iteration {}\".format(start_iter))\n\n self.iter = self.start_iter = start_iter\n self.max_iter = max_iter\n\n with EventStorage(start_iter) as self.storage:\n try:\n self.before_train()\n\n for self.iter in range(start_iter, max_iter):\n self.before_step()\n self.run_step_full_semisup()\n self.after_step()\n except Exception:\n logger.exception(\"Exception during training:\")\n raise\n finally:\n self.after_train()\n\n # =====================================================\n # ================== Pseduo-labeling ==================\n # =====================================================\n def threshold_bbox(self, proposal_bbox_inst, thres=0.7, proposal_type=\"roih\"):\n if proposal_type == \"rpn\":\n valid_map = proposal_bbox_inst.objectness_logits > thres\n\n # create instances containing boxes and gt_classes\n image_shape = proposal_bbox_inst.image_size\n new_proposal_inst = Instances(image_shape)\n\n # create box\n new_bbox_loc = proposal_bbox_inst.proposal_boxes.tensor[valid_map, :]\n new_boxes = Boxes(new_bbox_loc)\n\n # add boxes to instances\n new_proposal_inst.gt_boxes = new_boxes\n new_proposal_inst.objectness_logits = proposal_bbox_inst.objectness_logits[\n valid_map\n ]\n elif proposal_type == \"roih\":\n valid_map = proposal_bbox_inst.scores > thres\n\n # create instances containing boxes and gt_classes\n image_shape = proposal_bbox_inst.image_size\n new_proposal_inst = Instances(image_shape)\n\n # create box\n new_bbox_loc = proposal_bbox_inst.pred_boxes.tensor[valid_map, :]\n new_boxes = Boxes(new_bbox_loc)\n\n # add boxes to instances\n new_proposal_inst.gt_boxes = new_boxes\n new_proposal_inst.gt_classes = proposal_bbox_inst.pred_classes[valid_map]\n new_proposal_inst.scores = proposal_bbox_inst.scores[valid_map]\n\n return new_proposal_inst\n\n def process_pseudo_label(\n self, proposals_rpn_unsup_k, cur_threshold, proposal_type, psedo_label_method=\"\"\n ):\n list_instances = []\n num_proposal_output = 0.0\n for proposal_bbox_inst in proposals_rpn_unsup_k:\n # thresholding\n if psedo_label_method == \"thresholding\":\n proposal_bbox_inst = self.threshold_bbox(\n proposal_bbox_inst, thres=cur_threshold, proposal_type=proposal_type\n )\n else:\n raise ValueError(\"Unkown pseudo label boxes methods\")\n num_proposal_output += len(proposal_bbox_inst)\n list_instances.append(proposal_bbox_inst)\n num_proposal_output = num_proposal_output / len(proposals_rpn_unsup_k)\n return list_instances, num_proposal_output\n\n def remove_label(self, label_data):\n for label_datum in label_data:\n if \"instances\" in label_datum.keys():\n del label_datum[\"instances\"]\n return label_data\n\n def add_label(self, unlabled_data, label):\n for unlabel_datum, lab_inst in zip(unlabled_data, label):\n unlabel_datum[\"instances\"] = lab_inst\n return unlabled_data\n \n def get_label(self, label_data):\n label_list = []\n for label_datum in label_data:\n if \"instances\" in label_datum.keys():\n label_list.append(copy.deepcopy(label_datum[\"instances\"]))\n \n return label_list\n \n # def get_label_test(self, label_data):\n # label_list = []\n # for label_datum in label_data:\n # if \"instances\" in label_datum.keys():\n # label_list.append(label_datum[\"instances\"])\n\n # =====================================================\n # =================== Training Flow ===================\n # =====================================================\n\n def run_step_full_semisup(self):\n self._trainer.iter = self.iter\n assert self.model.training, \"[UBTeacherTrainer] model was changed to eval mode!\"\n start = time.perf_counter()\n data = next(self._trainer._data_loader_iter)\n # data_q and data_k from different augmentations (q:strong, k:weak)\n # label_strong, label_weak, unlabed_strong, unlabled_weak\n label_data_q, label_data_k, unlabel_data_q, unlabel_data_k = data\n data_time = time.perf_counter() - start\n\n # burn-in stage (supervised training with labeled data)\n if self.iter < self.cfg.SEMISUPNET.BURN_UP_STEP:\n\n # input both strong and weak supervised data into model\n label_data_q.extend(label_data_k)\n record_dict, _, _, _ = self.model(\n label_data_q, branch=\"supervised\")\n\n # weight losses\n loss_dict = {}\n for key in record_dict.keys():\n if key[:4] == \"loss\":\n loss_dict[key] = record_dict[key] * 1\n losses = sum(loss_dict.values())\n\n else:\n if self.iter == self.cfg.SEMISUPNET.BURN_UP_STEP:\n # update copy the the whole model\n self._update_teacher_model(keep_rate=0.00)\n # self.model.build_discriminator()\n\n elif (\n self.iter - self.cfg.SEMISUPNET.BURN_UP_STEP\n ) % self.cfg.SEMISUPNET.TEACHER_UPDATE_ITER == 0:\n self._update_teacher_model(\n keep_rate=self.cfg.SEMISUPNET.EMA_KEEP_RATE)\n\n record_dict = {}\n\n ######################## For probe #################################\n # import pdb; pdb. set_trace() \n gt_unlabel_k = self.get_label(unlabel_data_k)\n # gt_unlabel_q = self.get_label_test(unlabel_data_q)\n \n\n # 0. remove unlabeled data labels\n unlabel_data_q = self.remove_label(unlabel_data_q)\n unlabel_data_k = self.remove_label(unlabel_data_k)\n\n # 1. generate the pseudo-label using teacher model\n with torch.no_grad():\n (\n _,\n proposals_rpn_unsup_k,\n proposals_roih_unsup_k,\n _,\n ) = self.model_teacher(unlabel_data_k, branch=\"unsup_data_weak\")\n\n ######################## For probe #################################\n # import pdb; pdb. set_trace() \n\n # probe_metrics = ['compute_fp_gtoutlier', 'compute_num_box']\n # probe_metrics = ['compute_num_box'] \n # analysis_pred, _ = self.probe.compute_num_box(gt_unlabel_k,proposals_roih_unsup_k,'pred')\n # record_dict.update(analysis_pred)\n ######################## For probe END #################################\n\n # 2. Pseudo-labeling\n cur_threshold = self.cfg.SEMISUPNET.BBOX_THRESHOLD\n\n joint_proposal_dict = {}\n joint_proposal_dict[\"proposals_rpn\"] = proposals_rpn_unsup_k\n #Process pseudo labels and thresholding\n (\n pesudo_proposals_rpn_unsup_k,\n nun_pseudo_bbox_rpn,\n ) = self.process_pseudo_label(\n proposals_rpn_unsup_k, cur_threshold, \"rpn\", \"thresholding\"\n )\n # analysis_pred, _ = self.probe.compute_num_box(gt_unlabel_k,pesudo_proposals_rpn_unsup_k,'pred',True)\n # record_dict.update(analysis_pred)\n\n joint_proposal_dict[\"proposals_pseudo_rpn\"] = pesudo_proposals_rpn_unsup_k\n # Pseudo_labeling for ROI head (bbox location/objectness)\n pesudo_proposals_roih_unsup_k, _ = self.process_pseudo_label(\n proposals_roih_unsup_k, cur_threshold, \"roih\", \"thresholding\"\n )\n joint_proposal_dict[\"proposals_pseudo_roih\"] = pesudo_proposals_roih_unsup_k\n\n # 3. add pseudo-label to unlabeled data\n\n unlabel_data_q = self.add_label(\n unlabel_data_q, joint_proposal_dict[\"proposals_pseudo_roih\"]\n )\n unlabel_data_k = self.add_label(\n unlabel_data_k, joint_proposal_dict[\"proposals_pseudo_roih\"]\n )\n\n all_label_data = label_data_q + label_data_k\n all_unlabel_data = unlabel_data_q\n\n # 4. input both strongly and weakly augmented labeled data into student model\n record_all_label_data, _, _, _ = self.model(\n all_label_data, branch=\"supervised\"\n )\n record_dict.update(record_all_label_data)\n\n # 5. input strongly augmented unlabeled data into model\n record_all_unlabel_data, _, _, _ = self.model(\n all_unlabel_data, branch=\"supervised_target\"\n )\n new_record_all_unlabel_data = {}\n for key in record_all_unlabel_data.keys():\n new_record_all_unlabel_data[key + \"_pseudo\"] = record_all_unlabel_data[\n key\n ]\n record_dict.update(new_record_all_unlabel_data)\n\n # 6. input weakly labeled data (source) and weakly unlabeled data (target) to student model\n # give sign to the target data\n\n for i_index in range(len(unlabel_data_k)):\n # unlabel_data_item = {}\n for k, v in unlabel_data_k[i_index].items():\n # label_data_k[i_index][k + \"_unlabeled\"] = v\n label_data_k[i_index][k + \"_unlabeled\"] = v\n # unlabel_data_k[i_index] = unlabel_data_item\n\n all_domain_data = label_data_k\n # all_domain_data = label_data_k + unlabel_data_k\n record_all_domain_data, _, _, _ = self.model(all_domain_data, branch=\"domain\")\n record_dict.update(record_all_domain_data)\n\n\n # weight losses\n loss_dict = {}\n for key in record_dict.keys():\n if key.startswith(\"loss\"):\n if key == \"loss_rpn_loc_pseudo\" or key == \"loss_box_reg_pseudo\":\n # pseudo bbox regression <- 0\n loss_dict[key] = record_dict[key] * 0\n elif key[-6:] == \"pseudo\": # unsupervised loss\n loss_dict[key] = (\n record_dict[key] *\n self.cfg.SEMISUPNET.UNSUP_LOSS_WEIGHT\n )\n elif (\n key == \"loss_D_img_s\" or key == \"loss_D_img_t\"\n ): # set weight for discriminator\n # import pdb\n # pdb.set_trace()\n loss_dict[key] = record_dict[key] * self.cfg.SEMISUPNET.DIS_LOSS_WEIGHT #Need to modify defaults and yaml\n else: # supervised loss\n loss_dict[key] = record_dict[key] * 1\n\n losses = sum(loss_dict.values())\n\n metrics_dict = record_dict\n metrics_dict[\"data_time\"] = data_time\n self._write_metrics(metrics_dict)\n\n self.optimizer.zero_grad()\n losses.backward()\n self.optimizer.step()\n\n def _write_metrics(self, metrics_dict: dict):\n metrics_dict = {\n k: v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v)\n for k, v in metrics_dict.items()\n }\n\n # gather metrics among all workers for logging\n # This assumes we do DDP-style training, which is currently the only\n # supported method in detectron2.\n all_metrics_dict = comm.gather(metrics_dict)\n # all_hg_dict = comm.gather(hg_dict)\n\n if comm.is_main_process():\n if \"data_time\" in all_metrics_dict[0]:\n # data_time among workers can have high variance. The actual latency\n # caused by data_time is the maximum among workers.\n data_time = np.max([x.pop(\"data_time\")\n for x in all_metrics_dict])\n self.storage.put_scalar(\"data_time\", data_time)\n\n # average the rest metrics\n metrics_dict = {\n k: np.mean([x[k] for x in all_metrics_dict])\n for k in all_metrics_dict[0].keys()\n }\n\n # append the list\n loss_dict = {}\n for key in metrics_dict.keys():\n if key[:4] == \"loss\":\n loss_dict[key] = metrics_dict[key]\n\n total_losses_reduced = sum(loss for loss in loss_dict.values())\n\n self.storage.put_scalar(\"total_loss\", total_losses_reduced)\n if len(metrics_dict) > 1:\n self.storage.put_scalars(**metrics_dict)\n\n @torch.no_grad()\n def _update_teacher_model(self, keep_rate=0.9996):\n if comm.get_world_size() > 1:\n student_model_dict = {\n key[7:]: value for key, value in self.model.state_dict().items()\n }\n else:\n student_model_dict = self.model.state_dict()\n\n new_teacher_dict = OrderedDict()\n for key, value in self.model_teacher.state_dict().items():\n if key in student_model_dict.keys():\n new_teacher_dict[key] = (\n student_model_dict[key] *\n (1 - keep_rate) + value * keep_rate\n )\n else:\n raise Exception(\"{} is not found in student model\".format(key))\n\n self.model_teacher.load_state_dict(new_teacher_dict)\n\n @torch.no_grad()\n def _copy_main_model(self):\n # initialize all parameters\n if comm.get_world_size() > 1:\n rename_model_dict = {\n key[7:]: value for key, value in self.model.state_dict().items()\n }\n self.model_teacher.load_state_dict(rename_model_dict)\n else:\n self.model_teacher.load_state_dict(self.model.state_dict())\n\n @classmethod\n def build_test_loader(cls, cfg, dataset_name):\n return build_detection_test_loader(cfg, dataset_name)\n\n def build_hooks(self):\n cfg = self.cfg.clone()\n cfg.defrost()\n cfg.DATALOADER.NUM_WORKERS = 0 # save some memory and time for PreciseBN\n\n ret = [\n hooks.IterationTimer(),\n hooks.LRScheduler(self.optimizer, self.scheduler),\n hooks.PreciseBN(\n # Run at the same freq as (but before) evaluation.\n cfg.TEST.EVAL_PERIOD,\n self.model,\n # Build a new data loader to not affect training\n self.build_train_loader(cfg),\n cfg.TEST.PRECISE_BN.NUM_ITER,\n )\n if cfg.TEST.PRECISE_BN.ENABLED and get_bn_modules(self.model)\n else None,\n ]\n\n # Do PreciseBN before checkpointer, because it updates the model and need to\n # be saved by checkpointer.\n # This is not always the best: if checkpointing has a different frequency,\n # some checkpoints may have more precise statistics than others.\n if comm.is_main_process():\n ret.append(\n hooks.PeriodicCheckpointer(\n self.checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD\n )\n )\n\n def test_and_save_results_student():\n self._last_eval_results_student = self.test(self.cfg, self.model)\n _last_eval_results_student = {\n k + \"_student\": self._last_eval_results_student[k]\n for k in self._last_eval_results_student.keys()\n }\n return _last_eval_results_student\n\n def test_and_save_results_teacher():\n self._last_eval_results_teacher = self.test(\n self.cfg, self.model_teacher)\n return self._last_eval_results_teacher\n\n ret.append(hooks.EvalHook(cfg.TEST.EVAL_PERIOD,\n test_and_save_results_student))\n ret.append(hooks.EvalHook(cfg.TEST.EVAL_PERIOD,\n test_and_save_results_teacher))\n\n if comm.is_main_process():\n # run writers in the end, so that evaluation metrics are written\n ret.append(hooks.PeriodicWriter(self.build_writers(), period=20))\n return ret\n" }, { "path": "adapteacher/evaluation/__init__.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates.\nfrom .coco_evaluation import COCOEvaluator\nfrom .pascal_voc_evaluation import PascalVOCDetectionEvaluator\n\n# __all__ = [k for k in globals().keys() if not k.startswith(\"_\")]\n\n__all__ = [\n \"COCOEvaluator\",\n \"PascalVOCDetectionEvaluator\"\n]\n" }, { "path": "adapteacher/evaluation/coco_evaluation.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates.\nimport contextlib\nimport copy\nimport io\nimport itertools\nimport json\nimport logging\nimport numpy as np\nimport os\nimport pickle\nfrom collections import OrderedDict\nimport pycocotools.mask as mask_util\nimport torch\nfrom pycocotools.coco import COCO\nfrom pycocotools.cocoeval import COCOeval\nfrom tabulate import tabulate\n\nimport detectron2.utils.comm as comm\nfrom detectron2.config import CfgNode\nfrom detectron2.data import MetadataCatalog\nfrom detectron2.data.datasets.coco import convert_to_coco_dict\nfrom detectron2.evaluation.fast_eval_api import COCOeval_opt\nfrom detectron2.structures import Boxes, BoxMode, pairwise_iou\nfrom detectron2.utils.file_io import PathManager\nfrom detectron2.utils.logger import create_small_table\n\nfrom detectron2.evaluation import DatasetEvaluator\nfrom iopath.common.file_io import file_lock\n\nlogger = logging.getLogger(__name__)\n\ndef convert_to_coco_json(dataset_name, output_file, allow_cached=True):\n \"\"\"\n Converts dataset into COCO format and saves it to a json file.\n dataset_name must be registered in DatasetCatalog and in detectron2's standard format.\n\n Args:\n dataset_name:\n reference from the config file to the catalogs\n must be registered in DatasetCatalog and in detectron2's standard format\n output_file: path of json file that will be saved to\n allow_cached: if json file is already present then skip conversion\n \"\"\"\n\n # TODO: The dataset or the conversion script *may* change,\n # a checksum would be useful for validating the cached data\n\n PathManager.mkdirs(os.path.dirname(output_file))\n with file_lock(output_file):\n if PathManager.exists(output_file) and allow_cached:\n logger.warning(\n f\"Using previously cached COCO format annotations at '{output_file}'. \"\n \"You need to clear the cache file if your dataset has been modified.\"\n )\n else:\n logger.info(f\"Converting annotations of dataset '{dataset_name}' to COCO format ...)\")\n coco_dict = convert_to_coco_dict(dataset_name)\n\n logger.info(f\"Caching COCO format annotations at '{output_file}' ...\")\n tmp_file = output_file #+ \".tmp\"\n # with PathManager.open(tmp_file, \"w\") as f:\n # json.dump(coco_dict, f)\n # shutil.move(tmp_file, output_file)\n with PathManager.open(tmp_file, \"w\") as f:\n json.dump(coco_dict, f)\n\nclass COCOEvaluator(DatasetEvaluator):\n \"\"\"\n Evaluate AR for object proposals, AP for instance detection/segmentation, AP\n for keypoint detection outputs using COCO's metrics.\n See http://cocodataset.org/#detection-eval and\n http://cocodataset.org/#keypoints-eval to understand its metrics.\n The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means\n the metric cannot be computed (e.g. due to no predictions made).\n\n In addition to COCO, this evaluator is able to support any bounding box detection,\n instance segmentation, or keypoint detection dataset.\n \"\"\"\n\n def __init__(\n self,\n dataset_name,\n tasks=None,\n distributed=True,\n output_dir=None,\n *,\n use_fast_impl=True,\n kpt_oks_sigmas=(),\n ):\n \"\"\"\n Args:\n dataset_name (str): name of the dataset to be evaluated.\n It must have either the following corresponding metadata:\n\n \"json_file\": the path to the COCO format annotation\n\n Or it must be in detectron2's standard dataset format\n so it can be converted to COCO format automatically.\n tasks (tuple[str]): tasks that can be evaluated under the given\n configuration. A task is one of \"bbox\", \"segm\", \"keypoints\".\n By default, will infer this automatically from predictions.\n distributed (True): if True, will collect results from all ranks and run evaluation\n in the main process.\n Otherwise, will only evaluate the results in the current process.\n output_dir (str): optional, an output directory to dump all\n results predicted on the dataset. The dump contains two files:\n\n 1. \"instances_predictions.pth\" a file that can be loaded with `torch.load` and\n contains all the results in the format they are produced by the model.\n 2. \"coco_instances_results.json\" a json file in COCO's result format.\n use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP.\n Although the results should be very close to the official implementation in COCO\n API, it is still recommended to compute results with the official API for use in\n papers. The faster implementation also uses more RAM.\n kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS.\n See http://cocodataset.org/#keypoints-eval\n When empty, it will use the defaults in COCO.\n Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS.\n \"\"\"\n self._logger = logging.getLogger(__name__)\n self._distributed = distributed\n self._output_dir = output_dir\n self._use_fast_impl = use_fast_impl\n\n if tasks is not None and isinstance(tasks, CfgNode):\n kpt_oks_sigmas = (\n tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas\n )\n self._logger.warn(\n \"COCO Evaluator instantiated using config, this is deprecated behavior.\"\n \" Please pass in explicit arguments instead.\"\n )\n self._tasks = None # Infering it from predictions should be better\n else:\n self._tasks = tasks\n\n self._cpu_device = torch.device(\"cpu\")\n\n self._metadata = MetadataCatalog.get(dataset_name)\n if not hasattr(self._metadata, \"json_file\"):\n self._logger.info(\n f\"'{dataset_name}' is not registered by `register_coco_instances`.\"\n \" Therefore trying to convert it to COCO format ...\"\n )\n\n cache_path = os.path.join(output_dir, f\"{dataset_name}_coco_format.json\")\n self._metadata.json_file = cache_path\n convert_to_coco_json(dataset_name, cache_path)\n\n json_file = PathManager.get_local_path(self._metadata.json_file)\n with contextlib.redirect_stdout(io.StringIO()):\n self._coco_api = COCO(json_file)\n\n # Test set json files do not contain annotations (evaluation must be\n # performed using the COCO evaluation server).\n self._do_evaluation = \"annotations\" in self._coco_api.dataset\n if self._do_evaluation:\n self._kpt_oks_sigmas = kpt_oks_sigmas\n\n def reset(self):\n self._predictions = []\n\n def process(self, inputs, outputs):\n \"\"\"\n Args:\n inputs: the inputs to a COCO model (e.g., GeneralizedRCNN).\n It is a list of dict. Each dict corresponds to an image and\n contains keys like \"height\", \"width\", \"file_name\", \"image_id\".\n outputs: the outputs of a COCO model. It is a list of dicts with key\n \"instances\" that contains :class:`Instances`.\n \"\"\"\n for input, output in zip(inputs, outputs):\n prediction = {\"image_id\": input[\"image_id\"]}\n\n if \"instances\" in output:\n instances = output[\"instances\"].to(self._cpu_device)\n prediction[\"instances\"] = instances_to_coco_json(instances, input[\"image_id\"])\n if \"proposals\" in output:\n prediction[\"proposals\"] = output[\"proposals\"].to(self._cpu_device)\n if len(prediction) > 1:\n self._predictions.append(prediction)\n\n def evaluate(self, img_ids=None):\n \"\"\"\n Args:\n img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset\n \"\"\"\n if self._distributed:\n comm.synchronize()\n predictions = comm.gather(self._predictions, dst=0)\n predictions = list(itertools.chain(*predictions))\n\n if not comm.is_main_process():\n return {}\n else:\n predictions = self._predictions\n\n if len(predictions) == 0:\n self._logger.warning(\"[COCOEvaluator] Did not receive valid predictions.\")\n return {}\n\n if self._output_dir:\n PathManager.mkdirs(self._output_dir)\n file_path = os.path.join(self._output_dir, \"instances_predictions.pth\")\n with PathManager.open(file_path, \"wb\") as f:\n torch.save(predictions, f)\n\n self._results = OrderedDict()\n if \"proposals\" in predictions[0]:\n self._eval_box_proposals(predictions)\n if \"instances\" in predictions[0]:\n self._eval_predictions(predictions, img_ids=img_ids)\n # Copy so the caller can do whatever with results\n return copy.deepcopy(self._results)\n\n def _tasks_from_predictions(self, predictions):\n \"\"\"\n Get COCO API \"tasks\" (i.e. iou_type) from COCO-format predictions.\n \"\"\"\n tasks = {\"bbox\"}\n for pred in predictions:\n if \"segmentation\" in pred:\n tasks.add(\"segm\")\n if \"keypoints\" in pred:\n tasks.add(\"keypoints\")\n return sorted(tasks)\n\n def _eval_predictions(self, predictions, img_ids=None):\n \"\"\"\n Evaluate predictions. Fill self._results with the metrics of the tasks.\n \"\"\"\n self._logger.info(\"Preparing results for COCO format ...\")\n coco_results = list(itertools.chain(*[x[\"instances\"] for x in predictions]))\n tasks = self._tasks or self._tasks_from_predictions(coco_results)\n\n # unmap the category ids for COCO\n if hasattr(self._metadata, \"thing_dataset_id_to_contiguous_id\"):\n dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id\n all_contiguous_ids = list(dataset_id_to_contiguous_id.values())\n num_classes = len(all_contiguous_ids)\n assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1\n\n reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()}\n for result in coco_results:\n category_id = result[\"category_id\"]\n assert category_id < num_classes, (\n f\"A prediction has class={category_id}, \"\n f\"but the dataset only has {num_classes} classes and \"\n f\"predicted class id should be in [0, {num_classes - 1}].\"\n )\n result[\"category_id\"] = reverse_id_mapping[category_id]\n\n if self._output_dir:\n file_path = os.path.join(self._output_dir, \"coco_instances_results.json\")\n self._logger.info(\"Saving results to {}\".format(file_path))\n with PathManager.open(file_path, \"w\") as f:\n f.write(json.dumps(coco_results))\n f.flush()\n\n if not self._do_evaluation:\n self._logger.info(\"Annotations are not available for evaluation.\")\n return\n\n self._logger.info(\n \"Evaluating predictions with {} COCO API...\".format(\n \"unofficial\" if self._use_fast_impl else \"official\"\n )\n )\n for task in sorted(tasks):\n assert task in {\"bbox\", \"segm\", \"keypoints\"}, f\"Got unknown task: {task}!\"\n coco_eval = (\n _evaluate_predictions_on_coco(\n self._coco_api,\n coco_results,\n task,\n kpt_oks_sigmas=self._kpt_oks_sigmas,\n use_fast_impl=self._use_fast_impl,\n img_ids=img_ids,\n )\n if len(coco_results) > 0\n else None # cocoapi does not handle empty results very well\n )\n\n res = self._derive_coco_results(\n coco_eval, task, class_names=self._metadata.get(\"thing_classes\")\n )\n self._results[task] = res\n\n def _eval_box_proposals(self, predictions):\n \"\"\"\n Evaluate the box proposals in predictions.\n Fill self._results with the metrics for \"box_proposals\" task.\n \"\"\"\n if self._output_dir:\n # Saving generated box proposals to file.\n # Predicted box_proposals are in XYXY_ABS mode.\n bbox_mode = BoxMode.XYXY_ABS.value\n ids, boxes, objectness_logits = [], [], []\n for prediction in predictions:\n ids.append(prediction[\"image_id\"])\n boxes.append(prediction[\"proposals\"].proposal_boxes.tensor.numpy())\n objectness_logits.append(prediction[\"proposals\"].objectness_logits.numpy())\n\n proposal_data = {\n \"boxes\": boxes,\n \"objectness_logits\": objectness_logits,\n \"ids\": ids,\n \"bbox_mode\": bbox_mode,\n }\n with PathManager.open(os.path.join(self._output_dir, \"box_proposals.pkl\"), \"wb\") as f:\n pickle.dump(proposal_data, f)\n\n if not self._do_evaluation:\n self._logger.info(\"Annotations are not available for evaluation.\")\n return\n\n self._logger.info(\"Evaluating bbox proposals ...\")\n res = {}\n areas = {\"all\": \"\", \"small\": \"s\", \"medium\": \"m\", \"large\": \"l\"}\n for limit in [100, 1000]:\n for area, suffix in areas.items():\n stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit)\n key = \"AR{}@{:d}\".format(suffix, limit)\n res[key] = float(stats[\"ar\"].item() * 100)\n self._logger.info(\"Proposal metrics: \\n\" + create_small_table(res))\n self._results[\"box_proposals\"] = res\n\n def _derive_coco_results(self, coco_eval, iou_type, class_names=None):\n \"\"\"\n Derive the desired score numbers from summarized COCOeval.\n\n Args:\n coco_eval (None or COCOEval): None represents no predictions from model.\n iou_type (str):\n class_names (None or list[str]): if provided, will use it to predict\n per-category AP.\n\n Returns:\n a dict of {metric name: score}\n \"\"\"\n\n metrics = {\n \"bbox\": [\"AP\", \"AP50\", \"AP75\", \"APs\", \"APm\", \"APl\"],\n \"segm\": [\"AP\", \"AP50\", \"AP75\", \"APs\", \"APm\", \"APl\"],\n \"keypoints\": [\"AP\", \"AP50\", \"AP75\", \"APm\", \"APl\"],\n }[iou_type]\n\n if coco_eval is None:\n self._logger.warn(\"No predictions from the model!\")\n return {metric: float(\"nan\") for metric in metrics}\n\n # the standard metrics\n results = {\n metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else \"nan\")\n for idx, metric in enumerate(metrics)\n }\n self._logger.info(\n \"Evaluation results for {}: \\n\".format(iou_type) + create_small_table(results)\n )\n if not np.isfinite(sum(results.values())):\n self._logger.info(\"Some metrics cannot be computed and is shown as NaN.\")\n\n if class_names is None or len(class_names) <= 1:\n return results\n # Compute per-category AP\n # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa\n precisions = coco_eval.eval[\"precision\"]\n # precision has dims (iou, recall, cls, area range, max dets)\n assert len(class_names) == precisions.shape[2]\n\n results_per_category = []\n for idx, name in enumerate(class_names):\n # area range index 0: all area ranges\n # max dets index -1: typically 100 per image\n precision = precisions[:, :, idx, 0, -1]\n precision = precision[precision > -1]\n ap = np.mean(precision) if precision.size else float(\"nan\")\n results_per_category.append((\"{}\".format(name), float(ap * 100)))\n\n # tabulate it\n N_COLS = min(6, len(results_per_category) * 2)\n results_flatten = list(itertools.chain(*results_per_category))\n results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)])\n table = tabulate(\n results_2d,\n tablefmt=\"pipe\",\n floatfmt=\".3f\",\n headers=[\"category\", \"AP\"] * (N_COLS // 2),\n numalign=\"left\",\n )\n self._logger.info(\"Per-category {} AP: \\n\".format(iou_type) + table)\n # results.update({\"AP-\" + name: ap for name, ap in results_per_category})\n\n results_per_category_AP50 = []\n for idx, name in enumerate(class_names):\n # area range index 0: all area ranges\n # max dets index -1: typically 100 per image\n\n t = np.where(.5 == coco_eval.params.iouThrs)[0]\n precisions_50 = precisions[t]\n precisions_50 = precisions_50[:, :, idx, 0, -1]\n precisions_50 = precisions_50[precisions_50 > -1]\n ap = np.mean(precisions_50) if precisions_50.size else float(\"nan\")\n results_per_category_AP50.append((\"{}\".format(name), float(ap * 100)))\n\n # tabulate it\n N_COLS = min(6, len(results_per_category_AP50) * 2)\n results_flatten = list(itertools.chain(*results_per_category_AP50))\n results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)])\n table = tabulate(\n results_2d,\n tablefmt=\"pipe\",\n floatfmt=\".3f\",\n headers=[\"category\", \"AP50\"] * (N_COLS // 2),\n numalign=\"left\",\n )\n self._logger.info(\"Per-category {} AP50: \\n\".format(iou_type) + table)\n results.update({\"AP50-\" + name: ap for name, ap in results_per_category_AP50})\n\n return results\n\n\ndef instances_to_coco_json(instances, img_id):\n \"\"\"\n Dump an \"Instances\" object to a COCO-format json that's used for evaluation.\n\n Args:\n instances (Instances):\n img_id (int): the image id\n\n Returns:\n list[dict]: list of json annotations in COCO format.\n \"\"\"\n num_instance = len(instances)\n if num_instance == 0:\n return []\n\n boxes = instances.pred_boxes.tensor.numpy()\n boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS)\n boxes = boxes.tolist()\n scores = instances.scores.tolist()\n classes = instances.pred_classes.tolist()\n\n has_mask = instances.has(\"pred_masks\")\n if has_mask:\n # use RLE to encode the masks, because they are too large and takes memory\n # since this evaluator stores outputs of the entire dataset\n rles = [\n mask_util.encode(np.array(mask[:, :, None], order=\"F\", dtype=\"uint8\"))[0]\n for mask in instances.pred_masks\n ]\n for rle in rles:\n # \"counts\" is an array encoded by mask_util as a byte-stream. Python3's\n # json writer which always produces strings cannot serialize a bytestream\n # unless you decode it. Thankfully, utf-8 works out (which is also what\n # the pycocotools/_mask.pyx does).\n rle[\"counts\"] = rle[\"counts\"].decode(\"utf-8\")\n\n has_keypoints = instances.has(\"pred_keypoints\")\n if has_keypoints:\n keypoints = instances.pred_keypoints\n\n results = []\n for k in range(num_instance):\n result = {\n \"image_id\": img_id,\n \"category_id\": classes[k],\n \"bbox\": boxes[k],\n \"score\": scores[k],\n }\n if has_mask:\n result[\"segmentation\"] = rles[k]\n if has_keypoints:\n # In COCO annotations,\n # keypoints coordinates are pixel indices.\n # However our predictions are floating point coordinates.\n # Therefore we subtract 0.5 to be consistent with the annotation format.\n # This is the inverse of data loading logic in `datasets/coco.py`.\n keypoints[k][:, :2] -= 0.5\n result[\"keypoints\"] = keypoints[k].flatten().tolist()\n results.append(result)\n return results\n\n\n# inspired from Detectron:\n# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa\ndef _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area=\"all\", limit=None):\n \"\"\"\n Evaluate detection proposal recall metrics. This function is a much\n faster alternative to the official COCO API recall evaluation code. However,\n it produces slightly different results.\n \"\"\"\n # Record max overlap value for each gt box\n # Return vector of overlap values\n areas = {\n \"all\": 0,\n \"small\": 1,\n \"medium\": 2,\n \"large\": 3,\n \"96-128\": 4,\n \"128-256\": 5,\n \"256-512\": 6,\n \"512-inf\": 7,\n }\n area_ranges = [\n [0 ** 2, 1e5 ** 2], # all\n [0 ** 2, 32 ** 2], # small\n [32 ** 2, 96 ** 2], # medium\n [96 ** 2, 1e5 ** 2], # large\n [96 ** 2, 128 ** 2], # 96-128\n [128 ** 2, 256 ** 2], # 128-256\n [256 ** 2, 512 ** 2], # 256-512\n [512 ** 2, 1e5 ** 2],\n ] # 512-inf\n assert area in areas, \"Unknown area range: {}\".format(area)\n area_range = area_ranges[areas[area]]\n gt_overlaps = []\n num_pos = 0\n\n for prediction_dict in dataset_predictions:\n predictions = prediction_dict[\"proposals\"]\n\n # sort predictions in descending order\n # TODO maybe remove this and make it explicit in the documentation\n inds = predictions.objectness_logits.sort(descending=True)[1]\n predictions = predictions[inds]\n\n ann_ids = coco_api.getAnnIds(imgIds=prediction_dict[\"image_id\"])\n anno = coco_api.loadAnns(ann_ids)\n gt_boxes = [\n BoxMode.convert(obj[\"bbox\"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS)\n for obj in anno\n if obj[\"iscrowd\"] == 0\n ]\n gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes\n gt_boxes = Boxes(gt_boxes)\n gt_areas = torch.as_tensor([obj[\"area\"] for obj in anno if obj[\"iscrowd\"] == 0])\n\n if len(gt_boxes) == 0 or len(predictions) == 0:\n continue\n\n valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1])\n gt_boxes = gt_boxes[valid_gt_inds]\n\n num_pos += len(gt_boxes)\n\n if len(gt_boxes) == 0:\n continue\n\n if limit is not None and len(predictions) > limit:\n predictions = predictions[:limit]\n\n overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes)\n\n _gt_overlaps = torch.zeros(len(gt_boxes))\n for j in range(min(len(predictions), len(gt_boxes))):\n # find which proposal box maximally covers each gt box\n # and get the iou amount of coverage for each gt box\n max_overlaps, argmax_overlaps = overlaps.max(dim=0)\n\n # find which gt box is 'best' covered (i.e. 'best' = most iou)\n gt_ovr, gt_ind = max_overlaps.max(dim=0)\n assert gt_ovr >= 0\n # find the proposal box that covers the best covered gt box\n box_ind = argmax_overlaps[gt_ind]\n # record the iou coverage of this gt box\n _gt_overlaps[j] = overlaps[box_ind, gt_ind]\n assert _gt_overlaps[j] == gt_ovr\n # mark the proposal box and the gt box as used\n overlaps[box_ind, :] = -1\n overlaps[:, gt_ind] = -1\n\n # append recorded iou coverage level\n gt_overlaps.append(_gt_overlaps)\n gt_overlaps = (\n torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32)\n )\n gt_overlaps, _ = torch.sort(gt_overlaps)\n\n if thresholds is None:\n step = 0.05\n thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32)\n recalls = torch.zeros_like(thresholds)\n # compute recall for each iou threshold\n for i, t in enumerate(thresholds):\n recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos)\n # ar = 2 * np.trapz(recalls, thresholds)\n ar = recalls.mean()\n return {\n \"ar\": ar,\n \"recalls\": recalls,\n \"thresholds\": thresholds,\n \"gt_overlaps\": gt_overlaps,\n \"num_pos\": num_pos,\n }\n\n\ndef _evaluate_predictions_on_coco(\n coco_gt, coco_results, iou_type, kpt_oks_sigmas=None, use_fast_impl=True, img_ids=None\n):\n \"\"\"\n Evaluate the coco results using COCOEval API.\n \"\"\"\n assert len(coco_results) > 0\n\n if iou_type == \"segm\":\n coco_results = copy.deepcopy(coco_results)\n # When evaluating mask AP, if the results contain bbox, cocoapi will\n # use the box area as the area of the instance, instead of the mask area.\n # This leads to a different definition of small/medium/large.\n # We remove the bbox field to let mask AP use mask area.\n for c in coco_results:\n c.pop(\"bbox\", None)\n\n coco_dt = coco_gt.loadRes(coco_results)\n coco_eval = (COCOeval_opt if use_fast_impl else COCOeval)(coco_gt, coco_dt, iou_type)\n if img_ids is not None:\n coco_eval.params.imgIds = img_ids\n\n if iou_type == \"keypoints\":\n # Use the COCO default keypoint OKS sigmas unless overrides are specified\n if kpt_oks_sigmas:\n assert hasattr(coco_eval.params, \"kpt_oks_sigmas\"), \"pycocotools is too old!\"\n coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas)\n # COCOAPI requires every detection and every gt to have keypoints, so\n # we just take the first entry from both\n num_keypoints_dt = len(coco_results[0][\"keypoints\"]) // 3\n num_keypoints_gt = len(next(iter(coco_gt.anns.values()))[\"keypoints\"]) // 3\n num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas)\n assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, (\n f\"[COCOEvaluator] Prediction contain {num_keypoints_dt} keypoints. \"\n f\"Ground truth contains {num_keypoints_gt} keypoints. \"\n f\"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. \"\n \"They have to agree with each other. For meaning of OKS, please refer to \"\n \"http://cocodataset.org/#keypoints-eval.\"\n )\n\n coco_eval.evaluate()\n coco_eval.accumulate()\n coco_eval.summarize()\n\n return coco_eval\n" }, { "path": "adapteacher/evaluation/pascal_voc_evaluation.py", "content": "# -*- coding: utf-8 -*-\n# Copyright (c) Facebook, Inc. and its affiliates.\n\nimport logging\nimport numpy as np\nimport os\nimport tempfile\nimport xml.etree.ElementTree as ET\nfrom collections import OrderedDict, defaultdict\nfrom functools import lru_cache\nimport torch\n\nfrom detectron2.data import MetadataCatalog\nfrom detectron2.utils import comm\nfrom detectron2.utils.file_io import PathManager\n\nfrom detectron2.evaluation import DatasetEvaluator\n\nclass PascalVOCDetectionEvaluator(DatasetEvaluator):\n \"\"\"\n Evaluate Pascal VOC style AP for Pascal VOC dataset.\n It contains a synchronization, therefore has to be called from all ranks.\n\n Note that the concept of AP can be implemented in different ways and may not\n produce identical results. This class mimics the implementation of the official\n Pascal VOC Matlab API, and should produce similar but not identical results to the\n official API.\n \"\"\"\n\n def __init__(self, dataset_name, target_classnames=None):\n \"\"\"\n Args:\n dataset_name (str): name of the dataset, e.g., \"voc_2007_test\"\n \"\"\"\n self._dataset_name = dataset_name\n meta = MetadataCatalog.get(dataset_name)\n\n # Too many tiny files, download all to local for speed.\n annotation_dir_local = PathManager.get_local_path(\n os.path.join(meta.dirname, \"Annotations/\")\n )\n self._anno_file_template = os.path.join(annotation_dir_local, \"{}.xml\")\n self._image_set_path = os.path.join(meta.dirname, \"ImageSets\", \"Main\", meta.split + \".txt\")\n self._class_names = meta.thing_classes\n assert meta.year in [2007, 2012], meta.year\n self._is_2007 = meta.year == 2007\n self._cpu_device = torch.device(\"cpu\")\n self._logger = logging.getLogger(__name__)\n\n if target_classnames == None:\n self.target_classnames = self._class_names\n else:\n self.target_classnames = target_classnames\n\n def reset(self):\n self._predictions = defaultdict(list) # class name -> list of prediction strings\n\n def process(self, inputs, outputs):\n for input, output in zip(inputs, outputs):\n image_id = input[\"image_id\"]\n instances = output[\"instances\"].to(self._cpu_device)\n boxes = instances.pred_boxes.tensor.numpy()\n scores = instances.scores.tolist()\n classes = instances.pred_classes.tolist()\n for box, score, cls in zip(boxes, scores, classes):\n xmin, ymin, xmax, ymax = box\n # The inverse of data loading logic in `datasets/pascal_voc.py`\n xmin += 1\n ymin += 1\n self._predictions[cls].append(\n f\"{image_id} {score:.3f} {xmin:.1f} {ymin:.1f} {xmax:.1f} {ymax:.1f}\"\n )\n\n def evaluate(self):\n \"\"\"\n Returns:\n dict: has a key \"segm\", whose value is a dict of \"AP\", \"AP50\", and \"AP75\".\n \"\"\"\n all_predictions = comm.gather(self._predictions, dst=0)\n if not comm.is_main_process():\n return\n predictions = defaultdict(list)\n for predictions_per_rank in all_predictions:\n for clsid, lines in predictions_per_rank.items():\n predictions[clsid].extend(lines)\n del all_predictions\n\n self._logger.info(\n \"Evaluating {} using {} metric. \"\n \"Note that results do not use the official Matlab API.\".format(\n self._dataset_name, 2007 if self._is_2007 else 2012\n )\n )\n\n with tempfile.TemporaryDirectory(prefix=\"pascal_voc_eval_\") as dirname:\n res_file_template = os.path.join(dirname, \"{}.txt\")\n\n aps = defaultdict(list) # iou -> ap per class\n for cls_id, cls_name in enumerate(self._class_names):\n if cls_name not in self.target_classnames:\n continue\n lines = predictions.get(cls_id, [\"\"])\n\n with open(res_file_template.format(cls_name), \"w\") as f:\n f.write(\"\\n\".join(lines))\n\n for thresh in range(50, 100, 5):\n rec, prec, ap = voc_eval(\n res_file_template,\n self._anno_file_template,\n self._image_set_path,\n cls_name,\n ovthresh=thresh / 100.0,\n use_07_metric=self._is_2007,\n )\n aps[thresh].append(ap * 100)\n\n ret = OrderedDict()\n mAP = {iou: np.mean(x) for iou, x in aps.items()}\n ret[\"bbox\"] = {\"AP\": np.mean(list(mAP.values())), \"AP50\": mAP[50], \"AP75\": mAP[75]}\n\n #Add the codes for AP50\n for idx, name in enumerate(self.target_classnames):\n ret[\"bbox\"].update({\"AP50-\" + name: aps[50][idx]})\n\n return ret\n\n\n##############################################################################\n#\n# Below code is modified from\n# https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/datasets/voc_eval.py\n# --------------------------------------------------------\n# Fast/er R-CNN\n# Licensed under The MIT License [see LICENSE for details]\n# Written by Bharath Hariharan\n# --------------------------------------------------------\n\n\"\"\"Python implementation of the PASCAL VOC devkit's AP evaluation code.\"\"\"\n\n\n@lru_cache(maxsize=None)\ndef parse_rec(filename):\n \"\"\"Parse a PASCAL VOC xml file.\"\"\"\n with PathManager.open(filename) as f:\n tree = ET.parse(f)\n objects = []\n for obj in tree.findall(\"object\"):\n obj_struct = {}\n obj_struct[\"name\"] = obj.find(\"name\").text\n obj_struct[\"pose\"] = obj.find(\"pose\").text\n obj_struct[\"truncated\"] = int(obj.find(\"truncated\").text)\n obj_struct[\"difficult\"] = int(obj.find(\"difficult\").text)\n bbox = obj.find(\"bndbox\")\n obj_struct[\"bbox\"] = [\n int(bbox.find(\"xmin\").text),\n int(bbox.find(\"ymin\").text),\n int(bbox.find(\"xmax\").text),\n int(bbox.find(\"ymax\").text),\n ]\n objects.append(obj_struct)\n\n return objects\n\n\ndef voc_ap(rec, prec, use_07_metric=False):\n \"\"\"Compute VOC AP given precision and recall. If use_07_metric is true, uses\n the VOC 07 11-point method (default:False).\n \"\"\"\n if use_07_metric:\n # 11 point metric\n ap = 0.0\n for t in np.arange(0.0, 1.1, 0.1):\n if np.sum(rec >= t) == 0:\n p = 0\n else:\n p = np.max(prec[rec >= t])\n ap = ap + p / 11.0\n else:\n # correct AP calculation\n # first append sentinel values at the end\n mrec = np.concatenate(([0.0], rec, [1.0]))\n mpre = np.concatenate(([0.0], prec, [0.0]))\n\n # compute the precision envelope\n for i in range(mpre.size - 1, 0, -1):\n mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])\n\n # to calculate area under PR curve, look for points\n # where X axis (recall) changes value\n i = np.where(mrec[1:] != mrec[:-1])[0]\n\n # and sum (\\Delta recall) * prec\n ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])\n return ap\n\n\ndef voc_eval(detpath, annopath, imagesetfile, classname, ovthresh=0.5, use_07_metric=False):\n \"\"\"rec, prec, ap = voc_eval(detpath,\n annopath,\n imagesetfile,\n classname,\n [ovthresh],\n [use_07_metric])\n\n Top level function that does the PASCAL VOC evaluation.\n\n detpath: Path to detections\n detpath.format(classname) should produce the detection results file.\n annopath: Path to annotations\n annopath.format(imagename) should be the xml annotations file.\n imagesetfile: Text file containing the list of images, one image per line.\n classname: Category name (duh)\n [ovthresh]: Overlap threshold (default = 0.5)\n [use_07_metric]: Whether to use VOC07's 11 point AP computation\n (default False)\n \"\"\"\n # assumes detections are in detpath.format(classname)\n # assumes annotations are in annopath.format(imagename)\n # assumes imagesetfile is a text file with each line an image name\n\n # first load gt\n # read list of images\n with PathManager.open(imagesetfile, \"r\") as f:\n lines = f.readlines()\n imagenames = [x.strip() for x in lines]\n\n # load annots\n recs = {}\n for imagename in imagenames:\n recs[imagename] = parse_rec(annopath.format(imagename))\n\n # extract gt objects for this class\n class_recs = {}\n npos = 0\n for imagename in imagenames:\n R = [obj for obj in recs[imagename] if obj[\"name\"] == classname]\n bbox = np.array([x[\"bbox\"] for x in R])\n difficult = np.array([x[\"difficult\"] for x in R]).astype(np.bool)\n # difficult = np.array([False for x in R]).astype(np.bool) # treat all \"difficult\" as GT\n det = [False] * len(R)\n npos = npos + sum(~difficult)\n class_recs[imagename] = {\"bbox\": bbox, \"difficult\": difficult, \"det\": det}\n\n # read dets\n detfile = detpath.format(classname)\n with open(detfile, \"r\") as f:\n lines = f.readlines()\n\n splitlines = [x.strip().split(\" \") for x in lines]\n image_ids = [x[0] for x in splitlines]\n confidence = np.array([float(x[1]) for x in splitlines])\n BB = np.array([[float(z) for z in x[2:]] for x in splitlines]).reshape(-1, 4)\n\n # sort by confidence\n sorted_ind = np.argsort(-confidence)\n BB = BB[sorted_ind, :]\n image_ids = [image_ids[x] for x in sorted_ind]\n\n # go down dets and mark TPs and FPs\n nd = len(image_ids)\n tp = np.zeros(nd)\n fp = np.zeros(nd)\n for d in range(nd):\n R = class_recs[image_ids[d]]\n bb = BB[d, :].astype(float)\n ovmax = -np.inf\n BBGT = R[\"bbox\"].astype(float)\n\n if BBGT.size > 0:\n # compute overlaps\n # intersection\n ixmin = np.maximum(BBGT[:, 0], bb[0])\n iymin = np.maximum(BBGT[:, 1], bb[1])\n ixmax = np.minimum(BBGT[:, 2], bb[2])\n iymax = np.minimum(BBGT[:, 3], bb[3])\n iw = np.maximum(ixmax - ixmin + 1.0, 0.0)\n ih = np.maximum(iymax - iymin + 1.0, 0.0)\n inters = iw * ih\n\n # union\n uni = (\n (bb[2] - bb[0] + 1.0) * (bb[3] - bb[1] + 1.0)\n + (BBGT[:, 2] - BBGT[:, 0] + 1.0) * (BBGT[:, 3] - BBGT[:, 1] + 1.0)\n - inters\n )\n\n overlaps = inters / uni\n ovmax = np.max(overlaps)\n jmax = np.argmax(overlaps)\n\n if ovmax > ovthresh:\n if not R[\"difficult\"][jmax]:\n if not R[\"det\"][jmax]:\n tp[d] = 1.0\n R[\"det\"][jmax] = 1\n else:\n fp[d] = 1.0\n else:\n fp[d] = 1.0\n\n # compute precision recall\n fp = np.cumsum(fp)\n tp = np.cumsum(tp)\n rec = tp / float(npos)\n # avoid divide by zero in case the first detection matches a difficult\n # ground truth\n prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)\n ap = voc_ap(rec, prec, use_07_metric)\n\n return rec, prec, ap\n" }, { "path": "adapteacher/modeling/meta_arch/rcnn.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom detectron2.modeling.meta_arch.build import META_ARCH_REGISTRY\nfrom detectron2.modeling.meta_arch.rcnn import GeneralizedRCNN\nfrom detectron2.config import configurable\n# from detectron2.modeling.meta_arch.build import META_ARCH_REGISTRY\n# from detectron2.modeling.meta_arch.rcnn import GeneralizedRCNN\nimport logging\nfrom typing import Dict, Tuple, List, Optional\nfrom collections import OrderedDict\nfrom detectron2.modeling.proposal_generator import build_proposal_generator\nfrom detectron2.modeling.backbone import build_backbone, Backbone\nfrom detectron2.modeling.roi_heads import build_roi_heads\nfrom detectron2.utils.events import get_event_storage\nfrom detectron2.structures import ImageList\n\n############### Image discriminator ##############\nclass FCDiscriminator_img(nn.Module):\n def __init__(self, num_classes, ndf1=256, ndf2=128):\n super(FCDiscriminator_img, self).__init__()\n\n self.conv1 = nn.Conv2d(num_classes, ndf1, kernel_size=3, padding=1)\n self.conv2 = nn.Conv2d(ndf1, ndf2, kernel_size=3, padding=1)\n self.conv3 = nn.Conv2d(ndf2, ndf2, kernel_size=3, padding=1)\n self.classifier = nn.Conv2d(ndf2, 1, kernel_size=3, padding=1)\n\n self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.leaky_relu(x)\n x = self.conv2(x)\n x = self.leaky_relu(x)\n x = self.conv3(x)\n x = self.leaky_relu(x)\n x = self.classifier(x)\n return x\n#################################\n\n################ Gradient reverse function\nclass GradReverse(torch.autograd.Function):\n @staticmethod\n def forward(ctx, x):\n return x.view_as(x)\n\n @staticmethod\n def backward(ctx, grad_output):\n return grad_output.neg()\n\ndef grad_reverse(x):\n return GradReverse.apply(x)\n\n#######################\n\n@META_ARCH_REGISTRY.register()\nclass DAobjTwoStagePseudoLabGeneralizedRCNN(GeneralizedRCNN):\n\n @configurable\n def __init__(\n self,\n *,\n backbone: Backbone,\n proposal_generator: nn.Module,\n roi_heads: nn.Module,\n pixel_mean: Tuple[float],\n pixel_std: Tuple[float],\n input_format: Optional[str] = None,\n vis_period: int = 0,\n dis_type: str,\n # dis_loss_weight: float = 0,\n ):\n \"\"\"\n Args:\n backbone: a backbone module, must follow detectron2's backbone interface\n proposal_generator: a module that generates proposals using backbone features\n roi_heads: a ROI head that performs per-region computation\n pixel_mean, pixel_std: list or tuple with #channels element, representing\n the per-channel mean and std to be used to normalize the input image\n input_format: describe the meaning of channels of input. Needed by visualization\n vis_period: the period to run visualization. Set to 0 to disable.\n \"\"\"\n super(GeneralizedRCNN, self).__init__()\n self.backbone = backbone\n self.proposal_generator = proposal_generator\n self.roi_heads = roi_heads\n\n self.input_format = input_format\n self.vis_period = vis_period\n if vis_period > 0:\n assert input_format is not None, \"input_format is required for visualization!\"\n\n self.register_buffer(\"pixel_mean\", torch.tensor(pixel_mean).view(-1, 1, 1), False)\n self.register_buffer(\"pixel_std\", torch.tensor(pixel_std).view(-1, 1, 1), False)\n assert (\n self.pixel_mean.shape == self.pixel_std.shape\n ), f\"{self.pixel_mean} and {self.pixel_std} have different shapes!\"\n # @yujheli: you may need to build your discriminator here\n\n self.dis_type = dis_type\n self.D_img = None\n # self.D_img = FCDiscriminator_img(self.backbone._out_feature_channels['res4']) # Need to know the channel\n \n # self.D_img = None\n self.D_img = FCDiscriminator_img(self.backbone._out_feature_channels[self.dis_type]) # Need to know the channel\n # self.bceLoss_func = nn.BCEWithLogitsLoss()\n def build_discriminator(self):\n self.D_img = FCDiscriminator_img(self.backbone._out_feature_channels[self.dis_type]).to(self.device) # Need to know the channel\n\n @classmethod\n def from_config(cls, cfg):\n backbone = build_backbone(cfg)\n return {\n \"backbone\": backbone,\n \"proposal_generator\": build_proposal_generator(cfg, backbone.output_shape()),\n \"roi_heads\": build_roi_heads(cfg, backbone.output_shape()),\n \"input_format\": cfg.INPUT.FORMAT,\n \"vis_period\": cfg.VIS_PERIOD,\n \"pixel_mean\": cfg.MODEL.PIXEL_MEAN,\n \"pixel_std\": cfg.MODEL.PIXEL_STD,\n \"dis_type\": cfg.SEMISUPNET.DIS_TYPE,\n # \"dis_loss_ratio\": cfg.xxx,\n }\n\n def preprocess_image_train(self, batched_inputs: List[Dict[str, torch.Tensor]]):\n \"\"\"\n Normalize, pad and batch the input images.\n \"\"\"\n images = [x[\"image\"].to(self.device) for x in batched_inputs]\n images = [(x - self.pixel_mean) / self.pixel_std for x in images]\n images = ImageList.from_tensors(images, self.backbone.size_divisibility)\n\n images_t = [x[\"image_unlabeled\"].to(self.device) for x in batched_inputs]\n images_t = [(x - self.pixel_mean) / self.pixel_std for x in images_t]\n images_t = ImageList.from_tensors(images_t, self.backbone.size_divisibility)\n\n return images, images_t\n\n def forward(\n self, batched_inputs, branch=\"supervised\", given_proposals=None, val_mode=False\n ):\n \"\"\"\n Args:\n batched_inputs: a list, batched outputs of :class:`DatasetMapper` .\n Each item in the list contains the inputs for one image.\n For now, each item in the list is a dict that contains:\n\n * image: Tensor, image in (C, H, W) format.\n * instances (optional): groundtruth :class:`Instances`\n * proposals (optional): :class:`Instances`, precomputed proposals.\n\n Other information that's included in the original dicts, such as:\n\n * \"height\", \"width\" (int): the output resolution of the model, used in inference.\n See :meth:`postprocess` for details.\n\n Returns:\n list[dict]:\n Each dict is the output for one input image.\n The dict contains one key \"instances\" whose value is a :class:`Instances`.\n The :class:`Instances` object has the following keys:\n \"pred_boxes\", \"pred_classes\", \"scores\", \"pred_masks\", \"pred_keypoints\"\n \"\"\"\n if self.D_img == None:\n self.build_discriminator()\n if (not self.training) and (not val_mode): # only conduct when testing mode\n return self.inference(batched_inputs)\n\n source_label = 0\n target_label = 1\n\n if branch == \"domain\":\n # self.D_img.train()\n # source_label = 0\n # target_label = 1\n # images = self.preprocess_image(batched_inputs)\n images_s, images_t = self.preprocess_image_train(batched_inputs)\n\n features = self.backbone(images_s.tensor)\n\n # import pdb\n # pdb.set_trace()\n \n features_s = grad_reverse(features[self.dis_type])\n D_img_out_s = self.D_img(features_s)\n loss_D_img_s = F.binary_cross_entropy_with_logits(D_img_out_s, torch.FloatTensor(D_img_out_s.data.size()).fill_(source_label).to(self.device))\n\n features_t = self.backbone(images_t.tensor)\n \n features_t = grad_reverse(features_t[self.dis_type])\n # features_t = grad_reverse(features_t['p2'])\n D_img_out_t = self.D_img(features_t)\n loss_D_img_t = F.binary_cross_entropy_with_logits(D_img_out_t, torch.FloatTensor(D_img_out_t.data.size()).fill_(target_label).to(self.device))\n\n # import pdb\n # pdb.set_trace()\n\n losses = {}\n losses[\"loss_D_img_s\"] = loss_D_img_s\n losses[\"loss_D_img_t\"] = loss_D_img_t\n return losses, [], [], None\n\n # self.D_img.eval()\n images = self.preprocess_image(batched_inputs)\n\n if \"instances\" in batched_inputs[0]:\n gt_instances = [x[\"instances\"].to(self.device) for x in batched_inputs]\n else:\n gt_instances = None\n\n features = self.backbone(images.tensor)\n\n # TODO: remove the usage of if else here. This needs to be re-organized\n if branch == \"supervised\":\n features_s = grad_reverse(features[self.dis_type])\n D_img_out_s = self.D_img(features_s)\n loss_D_img_s = F.binary_cross_entropy_with_logits(D_img_out_s, torch.FloatTensor(D_img_out_s.data.size()).fill_(source_label).to(self.device))\n\n \n # Region proposal network\n proposals_rpn, proposal_losses = self.proposal_generator(\n images, features, gt_instances\n )\n\n # roi_head lower branch\n _, detector_losses = self.roi_heads(\n images,\n features,\n proposals_rpn,\n compute_loss=True,\n targets=gt_instances,\n branch=branch,\n )\n\n # visualization\n if self.vis_period > 0:\n storage = get_event_storage()\n if storage.iter % self.vis_period == 0:\n self.visualize_training(batched_inputs, proposals_rpn, branch)\n\n losses = {}\n losses.update(detector_losses)\n losses.update(proposal_losses)\n losses[\"loss_D_img_s\"] = loss_D_img_s*0.001\n return losses, [], [], None\n\n elif branch == \"supervised_target\":\n\n # features_t = grad_reverse(features_t[self.dis_type])\n # D_img_out_t = self.D_img(features_t)\n # loss_D_img_t = F.binary_cross_entropy_with_logits(D_img_out_t, torch.FloatTensor(D_img_out_t.data.size()).fill_(target_label).to(self.device))\n\n \n # Region proposal network\n proposals_rpn, proposal_losses = self.proposal_generator(\n images, features, gt_instances\n )\n\n # roi_head lower branch\n _, detector_losses = self.roi_heads(\n images,\n features,\n proposals_rpn,\n compute_loss=True,\n targets=gt_instances,\n branch=branch,\n )\n\n # visualization\n if self.vis_period > 0:\n storage = get_event_storage()\n if storage.iter % self.vis_period == 0:\n self.visualize_training(batched_inputs, proposals_rpn, branch)\n\n losses = {}\n losses.update(detector_losses)\n losses.update(proposal_losses)\n # losses[\"loss_D_img_t\"] = loss_D_img_t*0.001\n # losses[\"loss_D_img_s\"] = loss_D_img_s*0.001\n return losses, [], [], None\n\n elif branch == \"unsup_data_weak\":\n \"\"\"\n unsupervised weak branch: input image without any ground-truth label; output proposals of rpn and roi-head\n \"\"\"\n # Region proposal network\n proposals_rpn, _ = self.proposal_generator(\n images, features, None, compute_loss=False\n )\n\n # roi_head lower branch (keep this for further production)\n # notice that we do not use any target in ROI head to do inference!\n proposals_roih, ROI_predictions = self.roi_heads(\n images,\n features,\n proposals_rpn,\n targets=None,\n compute_loss=False,\n branch=branch,\n )\n\n # if self.vis_period > 0:\n # storage = get_event_storage()\n # if storage.iter % self.vis_period == 0:\n # self.visualize_training(batched_inputs, proposals_rpn, branch)\n\n return {}, proposals_rpn, proposals_roih, ROI_predictions\n elif branch == \"unsup_data_strong\":\n raise NotImplementedError()\n elif branch == \"val_loss\":\n raise NotImplementedError()\n\n def visualize_training(self, batched_inputs, proposals, branch=\"\"):\n \"\"\"\n This function different from the original one:\n - it adds \"branch\" to the `vis_name`.\n\n A function used to visualize images and proposals. It shows ground truth\n bounding boxes on the original image and up to 20 predicted object\n proposals on the original image. Users can implement different\n visualization functions for different models.\n\n Args:\n batched_inputs (list): a list that contains input to the model.\n proposals (list): a list that contains predicted proposals. Both\n batched_inputs and proposals should have the same length.\n \"\"\"\n from detectron2.utils.visualizer import Visualizer\n\n storage = get_event_storage()\n max_vis_prop = 20\n\n for input, prop in zip(batched_inputs, proposals):\n img = input[\"image\"]\n img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format)\n v_gt = Visualizer(img, None)\n v_gt = v_gt.overlay_instances(boxes=input[\"instances\"].gt_boxes)\n anno_img = v_gt.get_image()\n box_size = min(len(prop.proposal_boxes), max_vis_prop)\n v_pred = Visualizer(img, None)\n v_pred = v_pred.overlay_instances(\n boxes=prop.proposal_boxes[0:box_size].tensor.cpu().numpy()\n )\n prop_img = v_pred.get_image()\n vis_img = np.concatenate((anno_img, prop_img), axis=1)\n vis_img = vis_img.transpose(2, 0, 1)\n vis_name = (\n \"Left: GT bounding boxes \"\n + branch\n + \"; Right: Predicted proposals \"\n + branch\n )\n storage.put_image(vis_name, vis_img)\n break # only visualize one image in a batch\n\n\n\n@META_ARCH_REGISTRY.register()\nclass TwoStagePseudoLabGeneralizedRCNN(GeneralizedRCNN):\n def forward(\n self, batched_inputs, branch=\"supervised\", given_proposals=None, val_mode=False\n ):\n if (not self.training) and (not val_mode):\n return self.inference(batched_inputs)\n\n images = self.preprocess_image(batched_inputs)\n\n if \"instances\" in batched_inputs[0]:\n gt_instances = [x[\"instances\"].to(self.device) for x in batched_inputs]\n else:\n gt_instances = None\n\n features = self.backbone(images.tensor)\n\n if branch == \"supervised\":\n # Region proposal network\n proposals_rpn, proposal_losses = self.proposal_generator(\n images, features, gt_instances\n )\n\n # # roi_head lower branch\n _, detector_losses = self.roi_heads(\n images, features, proposals_rpn, gt_instances, branch=branch\n )\n\n losses = {}\n losses.update(detector_losses)\n losses.update(proposal_losses)\n return losses, [], [], None\n\n elif branch == \"unsup_data_weak\":\n # Region proposal network\n proposals_rpn, _ = self.proposal_generator(\n images, features, None, compute_loss=False\n )\n\n # roi_head lower branch (keep this for further production) # notice that we do not use any target in ROI head to do inference !\n proposals_roih, ROI_predictions = self.roi_heads(\n images,\n features,\n proposals_rpn,\n targets=None,\n compute_loss=False,\n branch=branch,\n )\n\n return {}, proposals_rpn, proposals_roih, ROI_predictions\n\n elif branch == \"val_loss\":\n\n # Region proposal network\n proposals_rpn, proposal_losses = self.proposal_generator(\n images, features, gt_instances, compute_val_loss=True\n )\n\n # roi_head lower branch\n _, detector_losses = self.roi_heads(\n images,\n features,\n proposals_rpn,\n gt_instances,\n branch=branch,\n compute_val_loss=True,\n )\n\n losses = {}\n losses.update(detector_losses)\n losses.update(proposal_losses)\n return losses, [], [], None\n\n\n" }, { "path": "adapteacher/modeling/meta_arch/ts_ensemble.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom torch.nn.parallel import DataParallel, DistributedDataParallel\nimport torch.nn as nn\n\n\nclass EnsembleTSModel(nn.Module):\n def __init__(self, modelTeacher, modelStudent):\n super(EnsembleTSModel, self).__init__()\n\n if isinstance(modelTeacher, (DistributedDataParallel, DataParallel)):\n modelTeacher = modelTeacher.module\n if isinstance(modelStudent, (DistributedDataParallel, DataParallel)):\n modelStudent = modelStudent.module\n\n self.modelTeacher = modelTeacher\n self.modelStudent = modelStudent" }, { "path": "adapteacher/modeling/meta_arch/vgg.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport torch.nn as nn\nimport copy\nimport torch\nfrom typing import Union, List, Dict, Any, cast\nfrom detectron2.modeling.backbone import (\n ResNet,\n Backbone,\n build_resnet_backbone,\n BACKBONE_REGISTRY\n)\nfrom detectron2.modeling.backbone.fpn import FPN, LastLevelMaxPool, LastLevelP6P7\n\n\n\ndef make_layers(cfg: List[Union[str, int]], batch_norm: bool = False) -> nn.Sequential:\n layers: List[nn.Module] = []\n in_channels = 3\n for v in cfg:\n if v == 'M':\n layers += [nn.MaxPool2d(kernel_size=2, stride=2)]\n else:\n v = cast(int, v)\n conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)\n if batch_norm:\n layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]\n else:\n layers += [conv2d, nn.ReLU(inplace=True)]\n in_channels = v\n return nn.Sequential(*layers)\n\ncfgs: Dict[str, List[Union[str, int]]] = {\n 'vgg11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],\n 'vgg13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],\n 'vgg16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],\n 'vgg19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'],\n}\n\n\nclass vgg_backbone(Backbone):\n \"\"\"\n Backbone (bottom-up) for FBNet.\n\n Hierarchy:\n trunk0:\n xif0_0\n xif0_1\n ...\n trunk1:\n xif1_0\n xif1_1\n ...\n ...\n\n Output features:\n The outputs from each \"stage\", i.e. trunkX.\n \"\"\"\n\n def __init__(self, cfg):\n super().__init__()\n\n self.vgg = make_layers(cfgs['vgg16'],batch_norm=True)\n\n self._initialize_weights()\n # self.stage_names_index = {'vgg1':3, 'vgg2':8 , 'vgg3':15, 'vgg4':22, 'vgg5':29}\n _out_feature_channels = [64, 128, 256, 512, 512]\n _out_feature_strides = [2, 4, 8, 16, 32]\n # stages, shape_specs = build_fbnet(\n # cfg,\n # name=\"trunk\",\n # in_channels=cfg.MODEL.FBNET_V2.STEM_IN_CHANNELS\n # )\n\n # nn.Sequential(*list(self.vgg.features._modules.values())[:14])\n\n self.stages = [nn.Sequential(*list(self.vgg._modules.values())[0:7]),\\\n nn.Sequential(*list(self.vgg._modules.values())[7:14]),\\\n nn.Sequential(*list(self.vgg._modules.values())[14:24]),\\\n nn.Sequential(*list(self.vgg._modules.values())[24:34]),\\\n nn.Sequential(*list(self.vgg._modules.values())[34:]),]\n self._out_feature_channels = {}\n self._out_feature_strides = {}\n self._stage_names = []\n\n for i, stage in enumerate(self.stages):\n name = \"vgg{}\".format(i)\n self.add_module(name, stage)\n self._stage_names.append(name)\n self._out_feature_channels[name] = _out_feature_channels[i]\n self._out_feature_strides[name] = _out_feature_strides[i]\n\n self._out_features = self._stage_names\n\n del self.vgg\n\n def forward(self, x):\n features = {}\n for name, stage in zip(self._stage_names, self.stages):\n x = stage(x)\n # if name in self._out_features:\n # outputs[name] = x\n features[name] = x\n # import pdb\n # pdb.set_trace()\n\n return features\n\n def _initialize_weights(self) -> None:\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n nn.init.constant_(m.bias, 0)\n\n\n@BACKBONE_REGISTRY.register() #already register in baseline model\ndef build_vgg_backbone(cfg, _):\n return vgg_backbone(cfg)\n\n\n@BACKBONE_REGISTRY.register() #already register in baseline model\ndef build_vgg_fpn_backbone(cfg, _):\n # backbone = FPN(\n # bottom_up=build_vgg_backbone(cfg),\n # in_features=cfg.MODEL.FPN.IN_FEATURES,\n # out_channels=cfg.MODEL.FPN.OUT_CHANNELS,\n # norm=cfg.MODEL.FPN.NORM,\n # top_block=LastLevelMaxPool(),\n # )\n\n bottom_up = vgg_backbone(cfg)\n in_features = cfg.MODEL.FPN.IN_FEATURES\n out_channels = cfg.MODEL.FPN.OUT_CHANNELS\n backbone = FPN(\n bottom_up=bottom_up,\n in_features=in_features,\n out_channels=out_channels,\n norm=cfg.MODEL.FPN.NORM,\n top_block=LastLevelMaxPool(),\n # fuse_type=cfg.MODEL.FPN.FUSE_TYPE,\n )\n # return backbone\n\n return backbone\n" }, { "path": "adapteacher/modeling/proposal_generator/rpn.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom typing import Dict, Optional\nimport torch\n\nfrom detectron2.structures import ImageList, Instances\nfrom detectron2.modeling.proposal_generator import RPN\nfrom detectron2.modeling.proposal_generator.build import PROPOSAL_GENERATOR_REGISTRY\n\n\n@PROPOSAL_GENERATOR_REGISTRY.register()\nclass PseudoLabRPN(RPN):\n \"\"\"\n Region Proposal Network, introduced by :paper:`Faster R-CNN`.\n \"\"\"\n\n def forward(\n self,\n images: ImageList,\n features: Dict[str, torch.Tensor],\n gt_instances: Optional[Instances] = None,\n compute_loss: bool = True,\n compute_val_loss: bool = False,\n ):\n features = [features[f] for f in self.in_features]\n anchors = self.anchor_generator(features)\n\n pred_objectness_logits, pred_anchor_deltas = self.rpn_head(features)\n pred_objectness_logits = [\n # (N, A, Hi, Wi) -> (N, Hi, Wi, A) -> (N, Hi*Wi*A)\n score.permute(0, 2, 3, 1).flatten(1)\n for score in pred_objectness_logits\n ]\n pred_anchor_deltas = [\n # (N, A*B, Hi, Wi) -> (N, A, B, Hi, Wi) -> (N, Hi, Wi, A, B) -> (N, Hi*Wi*A, B)\n x.view(\n x.shape[0], -1, self.anchor_generator.box_dim, x.shape[-2], x.shape[-1]\n )\n .permute(0, 3, 4, 1, 2)\n .flatten(1, -2)\n for x in pred_anchor_deltas\n ]\n\n if (self.training and compute_loss) or compute_val_loss:\n gt_labels, gt_boxes = self.label_and_sample_anchors(anchors, gt_instances)\n losses = self.losses(\n anchors, pred_objectness_logits, gt_labels, pred_anchor_deltas, gt_boxes\n )\n losses = {k: v * self.loss_weight.get(k, 1.0) for k, v in losses.items()}\n else: # inference\n losses = {}\n\n proposals = self.predict_proposals(\n anchors, pred_objectness_logits, pred_anchor_deltas, images.image_sizes\n )\n\n return proposals, losses" }, { "path": "adapteacher/modeling/roi_heads/fast_rcnn.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\nfrom detectron2.modeling.roi_heads.fast_rcnn import (\n FastRCNNOutputLayers,\n FastRCNNOutputs,\n)\n\n# focal loss\nclass FastRCNNFocaltLossOutputLayers(FastRCNNOutputLayers):\n def __init__(self, cfg, input_shape):\n super(FastRCNNFocaltLossOutputLayers, self).__init__(cfg, input_shape)\n self.num_classes = cfg.MODEL.ROI_HEADS.NUM_CLASSES\n\n def losses(self, predictions, proposals):\n \"\"\"\n Args:\n predictions: return values of :meth:`forward()`.\n proposals (list[Instances]): proposals that match the features\n that were used to compute predictions.\n \"\"\"\n scores, proposal_deltas = predictions\n losses = FastRCNNFocalLoss(\n self.box2box_transform,\n scores,\n proposal_deltas,\n proposals,\n self.smooth_l1_beta,\n self.box_reg_loss_type,\n num_classes=self.num_classes,\n ).losses()\n\n return losses\n\n\nclass FastRCNNFocalLoss(FastRCNNOutputs):\n \"\"\"\n A class that stores information about outputs of a Fast R-CNN head.\n It provides methods that are used to decode the outputs of a Fast R-CNN head.\n \"\"\"\n\n def __init__(\n self,\n box2box_transform,\n pred_class_logits,\n pred_proposal_deltas,\n proposals,\n smooth_l1_beta=0.0,\n box_reg_loss_type=\"smooth_l1\",\n num_classes=80,\n ):\n super(FastRCNNFocalLoss, self).__init__(\n box2box_transform,\n pred_class_logits,\n pred_proposal_deltas,\n proposals,\n smooth_l1_beta,\n box_reg_loss_type,\n )\n self.num_classes = num_classes\n\n def losses(self):\n return {\n \"loss_cls\": self.comput_focal_loss(),\n \"loss_box_reg\": self.box_reg_loss(),\n }\n\n def comput_focal_loss(self):\n if self._no_instances:\n return 0.0 * self.pred_class_logits.sum()\n else:\n FC_loss = FocalLoss(\n gamma=1.5,\n num_classes=self.num_classes,\n )\n total_loss = FC_loss(input=self.pred_class_logits, target=self.gt_classes)\n total_loss = total_loss / self.gt_classes.shape[0]\n\n return total_loss\n\n\nclass FocalLoss(nn.Module):\n def __init__(\n self,\n weight=None,\n gamma=1.0,\n num_classes=80,\n ):\n super(FocalLoss, self).__init__()\n assert gamma >= 0\n self.gamma = gamma\n self.weight = weight\n\n self.num_classes = num_classes\n\n def forward(self, input, target):\n # focal loss\n CE = F.cross_entropy(input, target, reduction=\"none\")\n p = torch.exp(-CE)\n loss = (1 - p) ** self.gamma * CE\n return loss.sum()\n" }, { "path": "adapteacher/modeling/roi_heads/roi_heads.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport torch\nfrom typing import Dict, List, Optional, Tuple, Union\nfrom detectron2.structures import Boxes, ImageList, Instances, pairwise_iou\nfrom detectron2.modeling.proposal_generator.proposal_utils import (\n add_ground_truth_to_proposals,\n)\nfrom detectron2.utils.events import get_event_storage\nfrom detectron2.modeling.roi_heads.box_head import build_box_head\nfrom detectron2.layers import ShapeSpec\nfrom detectron2.modeling.roi_heads import (\n ROI_HEADS_REGISTRY,\n StandardROIHeads,\n)\nfrom detectron2.modeling.roi_heads.fast_rcnn import FastRCNNOutputLayers\nfrom adapteacher.modeling.roi_heads.fast_rcnn import FastRCNNFocaltLossOutputLayers\n\nimport numpy as np\nfrom detectron2.modeling.poolers import ROIPooler\n\n\n@ROI_HEADS_REGISTRY.register()\nclass StandardROIHeadsPseudoLab(StandardROIHeads):\n @classmethod\n def _init_box_head(cls, cfg, input_shape):\n # fmt: off\n in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES\n pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION\n pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features)\n sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO\n pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE\n # fmt: on\n\n in_channels = [input_shape[f].channels for f in in_features]\n # Check all channel counts are equal\n assert len(set(in_channels)) == 1, in_channels\n in_channels = in_channels[0]\n\n box_pooler = ROIPooler(\n output_size=pooler_resolution,\n scales=pooler_scales,\n sampling_ratio=sampling_ratio,\n pooler_type=pooler_type,\n )\n box_head = build_box_head(\n cfg,\n ShapeSpec(\n channels=in_channels, height=pooler_resolution, width=pooler_resolution\n ),\n )\n if cfg.MODEL.ROI_HEADS.LOSS == \"CrossEntropy\":\n box_predictor = FastRCNNOutputLayers(cfg, box_head.output_shape)\n elif cfg.MODEL.ROI_HEADS.LOSS == \"FocalLoss\":\n box_predictor = FastRCNNFocaltLossOutputLayers(cfg, box_head.output_shape)\n else:\n raise ValueError(\"Unknown ROI head loss.\")\n\n return {\n \"box_in_features\": in_features,\n \"box_pooler\": box_pooler,\n \"box_head\": box_head,\n \"box_predictor\": box_predictor,\n }\n\n def forward(\n self,\n images: ImageList,\n features: Dict[str, torch.Tensor],\n proposals: List[Instances],\n targets: Optional[List[Instances]] = None,\n compute_loss=True,\n branch=\"\",\n compute_val_loss=False,\n ) -> Tuple[List[Instances], Dict[str, torch.Tensor]]:\n\n del images\n if self.training and compute_loss: # apply if training loss\n assert targets\n # 1000 --> 512\n proposals = self.label_and_sample_proposals(\n proposals, targets, branch=branch\n )\n elif compute_val_loss: # apply if val loss\n assert targets\n # 1000 --> 512\n temp_proposal_append_gt = self.proposal_append_gt\n self.proposal_append_gt = False\n proposals = self.label_and_sample_proposals(\n proposals, targets, branch=branch\n ) # do not apply target on proposals\n self.proposal_append_gt = temp_proposal_append_gt\n del targets\n\n if (self.training and compute_loss) or compute_val_loss:\n losses, _ = self._forward_box(\n features, proposals, compute_loss, compute_val_loss, branch\n )\n return proposals, losses\n else:\n pred_instances, predictions = self._forward_box(\n features, proposals, compute_loss, compute_val_loss, branch\n )\n\n return pred_instances, predictions\n\n def _forward_box(\n self,\n features: Dict[str, torch.Tensor],\n proposals: List[Instances],\n compute_loss: bool = True,\n compute_val_loss: bool = False,\n branch: str = \"\",\n ) -> Union[Dict[str, torch.Tensor], List[Instances]]:\n features = [features[f] for f in self.box_in_features]\n box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])\n box_features = self.box_head(box_features)\n predictions = self.box_predictor(box_features)\n del box_features\n\n if (\n self.training and compute_loss\n ) or compute_val_loss: # apply if training loss or val loss\n losses = self.box_predictor.losses(predictions, proposals)\n\n if self.train_on_pred_boxes:\n with torch.no_grad():\n pred_boxes = self.box_predictor.predict_boxes_for_gt_classes(\n predictions, proposals\n )\n for proposals_per_image, pred_boxes_per_image in zip(\n proposals, pred_boxes\n ):\n proposals_per_image.proposal_boxes = Boxes(pred_boxes_per_image)\n return losses, predictions\n else:\n\n pred_instances, _ = self.box_predictor.inference(predictions, proposals)\n return pred_instances, predictions\n\n @torch.no_grad()\n def label_and_sample_proposals(\n self, proposals: List[Instances], targets: List[Instances], branch: str = \"\"\n ) -> List[Instances]:\n gt_boxes = [x.gt_boxes for x in targets]\n if self.proposal_append_gt:\n proposals = add_ground_truth_to_proposals(gt_boxes, proposals)\n\n proposals_with_gt = []\n\n num_fg_samples = []\n num_bg_samples = []\n for proposals_per_image, targets_per_image in zip(proposals, targets):\n has_gt = len(targets_per_image) > 0\n match_quality_matrix = pairwise_iou(\n targets_per_image.gt_boxes, proposals_per_image.proposal_boxes\n )\n matched_idxs, matched_labels = self.proposal_matcher(match_quality_matrix)\n sampled_idxs, gt_classes = self._sample_proposals(\n matched_idxs, matched_labels, targets_per_image.gt_classes\n )\n\n proposals_per_image = proposals_per_image[sampled_idxs]\n proposals_per_image.gt_classes = gt_classes\n\n if has_gt:\n sampled_targets = matched_idxs[sampled_idxs]\n for (trg_name, trg_value) in targets_per_image.get_fields().items():\n if trg_name.startswith(\"gt_\") and not proposals_per_image.has(\n trg_name\n ):\n proposals_per_image.set(trg_name, trg_value[sampled_targets])\n else:\n gt_boxes = Boxes(\n targets_per_image.gt_boxes.tensor.new_zeros((len(sampled_idxs), 4))\n )\n proposals_per_image.gt_boxes = gt_boxes\n\n num_bg_samples.append((gt_classes == self.num_classes).sum().item())\n num_fg_samples.append(gt_classes.numel() - num_bg_samples[-1])\n proposals_with_gt.append(proposals_per_image)\n\n storage = get_event_storage()\n storage.put_scalar(\n \"roi_head/num_target_fg_samples_\" + branch, np.mean(num_fg_samples)\n )\n storage.put_scalar(\n \"roi_head/num_target_bg_samples_\" + branch, np.mean(num_bg_samples)\n )\n\n return proposals_with_gt\n" }, { "path": "adapteacher/solver/build.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport torch\nfrom detectron2.config import CfgNode\n\nfrom detectron2.solver.lr_scheduler import WarmupCosineLR, WarmupMultiStepLR\nfrom .lr_scheduler import WarmupTwoStageMultiStepLR\n\ndef build_lr_scheduler(\n cfg: CfgNode, optimizer: torch.optim.Optimizer\n) -> torch.optim.lr_scheduler._LRScheduler:\n \"\"\"\n Build a LR scheduler from config.\n \"\"\"\n name = cfg.SOLVER.LR_SCHEDULER_NAME\n if name == \"WarmupMultiStepLR\":\n return WarmupMultiStepLR(\n optimizer,\n cfg.SOLVER.STEPS,\n cfg.SOLVER.GAMMA,\n warmup_factor=cfg.SOLVER.WARMUP_FACTOR,\n warmup_iters=cfg.SOLVER.WARMUP_ITERS,\n warmup_method=cfg.SOLVER.WARMUP_METHOD,\n )\n elif name == \"WarmupCosineLR\":\n return WarmupCosineLR(\n optimizer,\n cfg.SOLVER.MAX_ITER,\n warmup_factor=cfg.SOLVER.WARMUP_FACTOR,\n warmup_iters=cfg.SOLVER.WARMUP_ITERS,\n warmup_method=cfg.SOLVER.WARMUP_METHOD,\n )\n elif name == \"WarmupTwoStageMultiStepLR\":\n return WarmupTwoStageMultiStepLR(\n optimizer,\n cfg.SOLVER.STEPS,\n factor_list=cfg.SOLVER.FACTOR_LIST, \n gamma=cfg.SOLVER.GAMMA,\n warmup_factor=cfg.SOLVER.WARMUP_FACTOR,\n warmup_iters=cfg.SOLVER.WARMUP_ITERS,\n warmup_method=cfg.SOLVER.WARMUP_METHOD,\n )\n else:\n raise ValueError(\"Unknown LR scheduler: {}\".format(name))\n" }, { "path": "adapteacher/solver/lr_scheduler.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom bisect import bisect_right\nfrom typing import List\nimport torch\nfrom detectron2.solver.lr_scheduler import _get_warmup_factor_at_iter\n\n\nclass WarmupTwoStageMultiStepLR(torch.optim.lr_scheduler._LRScheduler):\n def __init__(\n self,\n optimizer: torch.optim.Optimizer,\n milestones: List[int],\n factor_list: List[int],\n gamma: float = 0.1,\n warmup_factor: float = 0.001,\n warmup_iters: int = 1000,\n warmup_method: str = \"linear\",\n last_epoch: int = -1,\n ):\n if not list(milestones) == sorted(milestones):\n raise ValueError(\n \"Milestones should be a list of\" \" increasing integers. Got {}\",\n milestones,\n )\n if len(milestones) + 1 != len(factor_list):\n raise ValueError(\"Length of milestones should match length of factor_list.\")\n\n self.milestones = milestones\n self.gamma = gamma\n self.warmup_factor = warmup_factor\n self.warmup_iters = warmup_iters\n self.warmup_method = warmup_method\n self.factor_list = factor_list\n\n super().__init__(optimizer, last_epoch)\n\n def get_lr(self) -> List[float]:\n\n warmup_factor = _get_warmup_factor_at_iter(\n self.warmup_method, self.last_epoch, self.warmup_iters, self.warmup_factor\n )\n\n return [\n base_lr\n * warmup_factor\n * self.factor_list[bisect_right(self.milestones, self.last_epoch)]\n for base_lr in self.base_lrs\n ]\n\n def _compute_values(self) -> List[float]:\n # The new interface\n return self.get_lr()\n" }, { "path": "configs/Base-RCNN-C4.yaml", "content": "MODEL:\n META_ARCHITECTURE: \"GeneralizedRCNN\"\n RPN:\n PRE_NMS_TOPK_TEST: 6000\n POST_NMS_TOPK_TEST: 1000\n ROI_HEADS:\n NAME: \"Res5ROIHeads\"\nDATASETS:\n TRAIN: (\"coco_2017_train\",)\n TEST: (\"coco_2017_val\",)\nSOLVER:\n IMS_PER_BATCH: 16\n BASE_LR: 0.02\n STEPS: (60000, 80000)\n MAX_ITER: 90000\nINPUT:\n MIN_SIZE_TRAIN: (600,)\nVERSION: 2\n" }, { "path": "configs/faster_rcnn_R101_cross_clipart.yaml", "content": "_BASE_: \"./Base-RCNN-C4.yaml\"\nMODEL:\n # META_ARCHITECTURE: \"TwoStagePseudoLabGeneralizedRCNN\"\n META_ARCHITECTURE: \"DAobjTwoStagePseudoLabGeneralizedRCNN\"\n # WEIGHTS: \"detectron2://ImageNetPretrained/MSRA/R-101.pkl\"\n WEIGHTS: \"detectron2://ImageNetPretrained/MSRA/R-101.pkl\"\n MASK_ON: False\n RESNETS:\n DEPTH: 101\n PROPOSAL_GENERATOR:\n NAME: \"PseudoLabRPN\"\n # RPN:\n # POSITIVE_FRACTION: 0.25\n ROI_HEADS:\n NAME: \"StandardROIHeadsPseudoLab\"\n LOSS: \"CrossEntropy\" # variant: \"CrossEntropy\"\n NUM_CLASSES: 20\n ROI_BOX_HEAD:\n NAME: \"FastRCNNConvFCHead\"\n NUM_FC: 2\n POOLER_RESOLUTION: 7\nSOLVER:\n LR_SCHEDULER_NAME: \"WarmupTwoStageMultiStepLR\"\n STEPS: (60000, 80000, 90000, 360000)\n FACTOR_LIST: (1, 1, 1, 1, 1)\n MAX_ITER: 100000\n IMG_PER_BATCH_LABEL: 16\n IMG_PER_BATCH_UNLABEL: 16\n BASE_LR: 0.01\nDATALOADER:\n SUP_PERCENT: 100.0\nDATASETS:\n CROSS_DATASET: True\n TRAIN_LABEL: (\"voc_2012_trainval\",) #voc_2012_train\n # TRAIN_UNLABEL: (\"voc_2012_val\",) #Clipart1k_train\n # TEST: (\"voc_2012_val\",) #Clipart1k_test\n TRAIN_UNLABEL: (\"Clipart1k_train\",) #Clipart1k_train\n TEST: (\"Clipart1k_test\",) #Clipart1k_test\nSEMISUPNET:\n Trainer: \"ateacher\"\n BBOX_THRESHOLD: 0.8\n TEACHER_UPDATE_ITER: 1\n BURN_UP_STEP: 60000\n EMA_KEEP_RATE: 0.9996\n UNSUP_LOSS_WEIGHT: 0.5 # 1.0 is suboptimal\n SUP_LOSS_WEIGHT: 1.0\n DIS_TYPE: \"res4\" #[\"concate\",\"p2\",\"multi\"]\n DIS_LOSS_WEIGHT: 0.1\nTEST:\n EVAL_PERIOD: 2000\nOUTPUT_DIR: ./output/faster_rcnn_R101_cross_clipart_test\n\n" }, { "path": "configs/faster_rcnn_R101_cross_clipart_b4.yaml", "content": "_BASE_: \"./Base-RCNN-C4.yaml\"\nMODEL:\n META_ARCHITECTURE: \"DAobjTwoStagePseudoLabGeneralizedRCNN\"\n WEIGHTS: \"detectron2://ImageNetPretrained/MSRA/R-101.pkl\"\n MASK_ON: False\n RESNETS:\n DEPTH: 101\n PROPOSAL_GENERATOR:\n NAME: \"PseudoLabRPN\"\n # RPN:\n # POSITIVE_FRACTION: 0.25\n ROI_HEADS:\n NAME: \"StandardROIHeadsPseudoLab\"\n LOSS: \"CrossEntropy\" # variant: \"CrossEntropy\"\n NUM_CLASSES: 20\n ROI_BOX_HEAD:\n NAME: \"FastRCNNConvFCHead\"\n NUM_FC: 2\n POOLER_RESOLUTION: 7\nSOLVER:\n LR_SCHEDULER_NAME: \"WarmupTwoStageMultiStepLR\"\n STEPS: (240000, 320000, 360000, 1440000)\n FACTOR_LIST: (1, 1, 1, 1, 1)\n MAX_ITER: 400000\n IMG_PER_BATCH_LABEL: 4\n IMG_PER_BATCH_UNLABEL: 4\n IMS_PER_BATCH: 4\n BASE_LR: 0.01\nDATALOADER:\n SUP_PERCENT: 100.0\nDATASETS:\n CROSS_DATASET: True\n TRAIN_LABEL: (\"voc_2012_trainval\",) #(\"voc_2012_trainval\",\"voc_2007_trainval\")\n TRAIN_UNLABEL: (\"Clipart1k_train\",)\n TEST: (\"Clipart1k_test\",)\nSEMISUPNET:\n Trainer: \"ateacher\"\n BBOX_THRESHOLD: 0.8\n TEACHER_UPDATE_ITER: 1\n BURN_UP_STEP: 80000\n EMA_KEEP_RATE: 0.9996\n UNSUP_LOSS_WEIGHT: 1.0\n SUP_LOSS_WEIGHT: 1.0\n DIS_TYPE: \"res4\" #[\"concate\",\"p2\",\"multi\"]\nTEST:\n EVAL_PERIOD: 4000\nOUTPUT_DIR: ./output/faster_rcnn_R101_cross_clipart_mod\n" }, { "path": "configs/faster_rcnn_R101_cross_water.yaml", "content": "_BASE_: \"./Base-RCNN-C4.yaml\"\nMODEL:\n # META_ARCHITECTURE: \"TwoStagePseudoLabGeneralizedRCNN\"\n META_ARCHITECTURE: \"DAobjTwoStagePseudoLabGeneralizedRCNN\"\n # WEIGHTS: \"detectron2://ImageNetPretrained/MSRA/R-101.pkl\"\n WEIGHTS: \"detectron2://ImageNetPretrained/MSRA/R-101.pkl\"\n MASK_ON: False\n RESNETS:\n DEPTH: 101\n PROPOSAL_GENERATOR:\n NAME: \"PseudoLabRPN\"\n # RPN:\n # POSITIVE_FRACTION: 0.25\n ROI_HEADS:\n NAME: \"StandardROIHeadsPseudoLab\"\n LOSS: \"CrossEntropy\" # variant: \"CrossEntropy\"\n NUM_CLASSES: 20\n ROI_BOX_HEAD:\n NAME: \"FastRCNNConvFCHead\"\n NUM_FC: 2\n POOLER_RESOLUTION: 7\nSOLVER:\n LR_SCHEDULER_NAME: \"WarmupTwoStageMultiStepLR\"\n STEPS: (60000, 80000, 90000, 360000)\n FACTOR_LIST: (1, 1, 1, 1, 1)\n MAX_ITER: 360000\n IMG_PER_BATCH_LABEL: 16\n IMG_PER_BATCH_UNLABEL: 16\n BASE_LR: 0.04\nDATALOADER:\n SUP_PERCENT: 100.0\nDATASETS:\n CROSS_DATASET: True\n TRAIN_LABEL: (\"voc_2012_trainval\",) #voc_2012_train\n TRAIN_UNLABEL: (\"Watercolor_train\",) #Clipart1k_train\n TEST: (\"Watercolor_test\",) #Clipart1k_test\nSEMISUPNET:\n Trainer: \"ateacher\"\n BBOX_THRESHOLD: 0.8\n TEACHER_UPDATE_ITER: 1\n BURN_UP_STEP: 20000\n EMA_KEEP_RATE: 0.9996\n UNSUP_LOSS_WEIGHT: 1.0\n SUP_LOSS_WEIGHT: 1.0\n DIS_TYPE: \"res4\" #[\"concate\",\"p2\",\"multi\"]\nTEST:\n EVAL_PERIOD: 1000\n" }, { "path": "configs/faster_rcnn_VGG_cross_city.yaml", "content": "_BASE_: \"./Base-RCNN-C4.yaml\"\nMODEL:\n # META_ARCHITECTURE: \"TwoStagePseudoLabGeneralizedRCNN\"\n META_ARCHITECTURE: \"DAobjTwoStagePseudoLabGeneralizedRCNN\"\n BACKBONE:\n NAME: \"build_vgg_backbone\"\n MASK_ON: False\n RESNETS:\n DEPTH: 101\n PROPOSAL_GENERATOR:\n NAME: \"PseudoLabRPN\"\n # RPN:\n # POSITIVE_FRACTION: 0.25\n RPN:\n IN_FEATURES: [\"vgg4\"]\n ROI_HEADS:\n NAME: \"StandardROIHeadsPseudoLab\"\n LOSS: \"CrossEntropy\" # variant: \"CrossEntropy\"\n NUM_CLASSES: 8\n IN_FEATURES: [\"vgg4\"]\n ROI_BOX_HEAD:\n NAME: \"FastRCNNConvFCHead\"\n NUM_FC: 2\n POOLER_RESOLUTION: 7\nSOLVER:\n LR_SCHEDULER_NAME: \"WarmupTwoStageMultiStepLR\"\n STEPS: (60000, 80000, 90000, 360000)\n FACTOR_LIST: (1, 1, 1, 1, 1)\n MAX_ITER: 100000\n IMG_PER_BATCH_LABEL: 16\n IMG_PER_BATCH_UNLABEL: 16\n BASE_LR: 0.04\nDATALOADER:\n SUP_PERCENT: 100.0\nDATASETS:\nDATASETS:\n CROSS_DATASET: True\n TRAIN_LABEL: (\"cityscapes_fine_instance_seg_train\",)\n TRAIN_UNLABEL: (\"cityscapes_foggy_train\",)\n TEST: (\"cityscapes_foggy_val\",)\nSEMISUPNET:\n Trainer: \"ateacher\"\n BBOX_THRESHOLD: 0.8\n TEACHER_UPDATE_ITER: 1\n BURN_UP_STEP: 20000\n EMA_KEEP_RATE: 0.9996\n UNSUP_LOSS_WEIGHT: 1.0\n SUP_LOSS_WEIGHT: 1.0\n DIS_TYPE: \"vgg4\" #[\"concate\",\"p2\",\"multi\"]\nTEST:\n EVAL_PERIOD: 1000\n" }, { "path": "prod_lib/TARGETS", "content": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved" }, { "path": "prod_lib/config/defaults.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\n# from d2go.config import CfgNode as CN\n\ndef add_aut_config(cfg):\n \"\"\"\n Add config for SemiSupSegRunner.\n \"\"\"\n _C = cfg\n #New added for discriminator\n _C.UNBIASEDTEACHER.DIS_LOSS_WEIGHT = 0.1\n _C.UNBIASEDTEACHER.DIS_TYPE = \"concate\" #[\"concate\",\"p2\",\"multi\"]\n _C.UNBIASEDTEACHER.ISAUG = \"Yes\"\n" }, { "path": "prod_lib/data/builtin.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\nimport contextlib\nimport io\nimport logging\nimport os\nimport json\n\nfrom detectron2.data import DatasetCatalog, MetadataCatalog\nfrom d2go.data.utils import CallFuncWithJsonFile\nfrom detectron2.utils.file_io import PathManager\nfrom fvcore.common.timer import Timer\n\nfrom detectron2.data.datasets.pascal_voc import register_pascal_voc\nfrom detectron2.data.datasets.builtin_meta import _get_builtin_metadata\nfrom .cityscapes_foggy import load_cityscapes_instances\n\nlogger = logging.getLogger(__name__)\n\n_SPLITS_COCO_FORMAT = {}\n_SPLITS_COCO_FORMAT[\"coco\"] = {\n \"coco_2017_unlabel\": (\n \"memcache_manifold://mobile_vision_dataset/tree/coco_unlabel2017\",\n \"memcache_manifold://mobile_vision_dataset/tree/coco_unlabel2017/coco_jsons/image_info_unlabeled2017.json\",\n ),\n \"goi_v5_unlabel\": (\n \"memcache_manifold://portal_ai_data/tree/goi_v5/train\",\n \"memcache_manifold://mobile_vision_dataset/tree/goi/v5/coco_jsons/openimages_v5_train_unlabel.json\",\n ),\n}\n\n\ndef register_coco_unlabel():\n for _, splits_per_dataset in _SPLITS_COCO_FORMAT.items():\n for key, (image_root, json_file) in splits_per_dataset.items():\n meta = {}\n register_coco_unlabel_instances(key, meta, json_file, image_root)\n\n\ndef register_coco_unlabel_instances(name, metadata, json_file, image_root):\n \"\"\"\n Register a dataset in COCO's json annotation format for\n instance detection, instance segmentation and keypoint detection.\n (i.e., Type 1 and 2 in http://cocodataset.org/#format-data.\n `instances*.json` and `person_keypoints*.json` in the dataset).\n\n This is an example of how to register a new dataset.\n You can do something similar to this function, to register new datasets.\n\n Args:\n name (str): the name that identifies a dataset, e.g. \"coco_2014_train\".\n metadata (dict): extra metadata associated with this dataset. You can\n leave it as an empty dict.\n json_file (str): path to the json instance annotation file.\n image_root (str or path-like): directory which contains all the images.\n \"\"\"\n assert isinstance(name, str), name\n assert isinstance(json_file, (str, os.PathLike)), json_file\n assert isinstance(image_root, (str, os.PathLike)), image_root\n\n # 1. register a function which returns dicts\n DatasetCatalog.register(\n name, lambda: load_coco_unlabel_json(json_file, image_root, name)\n )\n\n # 2. Optionally, add metadata about this dataset,\n # since they might be useful in evaluation, visualization or logging\n MetadataCatalog.get(name).set(\n json_file=json_file, image_root=image_root, evaluator_type=\"coco\", **metadata\n )\n\n\ndef load_coco_unlabel_json(\n json_file, image_root, dataset_name=None, extra_annotation_keys=None\n):\n \"\"\"\n Load a json file with COCO's instances annotation format.\n Currently supports instance detection, instance segmentation,\n and person keypoints annotations.\n\n Args:\n json_file (str): full path to the json file in COCO instances annotation format.\n image_root (str or path-like): the directory where the images in this json file exists.\n dataset_name (str): the name of the dataset (e.g., coco_2017_train).\n If provided, this function will also put \"thing_classes\" into\n the metadata associated with this dataset.\n extra_annotation_keys (list[str]): list of per-annotation keys that should also be\n loaded into the dataset dict (besides \"iscrowd\", \"bbox\", \"keypoints\",\n \"category_id\", \"segmentation\"). The values for these keys will be returned as-is.\n For example, the densepose annotations are loaded in this way.\n\n Returns:\n list[dict]: a list of dicts in Detectron2 standard dataset dicts format. (See\n `Using Custom Datasets `_ )\n\n Notes:\n 1. This function does not read the image files.\n The results do not have the \"image\" field.\n \"\"\"\n from pycocotools.coco import COCO\n\n timer = Timer()\n json_file = PathManager.get_local_path(json_file)\n with contextlib.redirect_stdout(io.StringIO()):\n coco_api = COCO(json_file)\n if timer.seconds() > 1:\n logger.info(\n \"Loading {} takes {:.2f} seconds.\".format(json_file, timer.seconds())\n )\n\n # sort indices for reproducible results\n img_ids = sorted(coco_api.imgs.keys())\n imgs = coco_api.loadImgs(img_ids)\n logger.info(\n \"Loaded {} unlabeled images in COCO format from {}\".format(len(imgs), json_file)\n )\n\n dataset_dicts = []\n for img_dict in imgs:\n record = {}\n record[\"file_name\"] = os.path.join(image_root, img_dict[\"file_name\"])\n record[\"height\"] = img_dict[\"height\"]\n record[\"width\"] = img_dict[\"width\"]\n record[\"image_id\"] = img_dict[\"id\"]\n\n dataset_dicts.append(record)\n\n return dataset_dicts\n\n\n_UNLABELED_DATASETS = {\n # 1-2 people images extracted from UGC ig images or ig profiles using fetch_image flow\n \"UGC_unlabel_ig_1M_20210514_1or2people\": \"manifold://pai_mobile/tree/datasets/semi_supervised/unlabeled_UGC/sweep_4m_20210514_20210515_1or2people.json\",\n # hand non-UGC long range frames extracted from collected videos\n \"hand_nonUGC_long_range_384K_20210521\": \"manifold://pai_mobile/tree/datasets/hand_unlabeled_nonUGC/long_range.json\",\n # hand non-UGC short range images cropped from the annotated bounding boxes in long-range videos\n \"hand_nonUGC_short_range_183K_20210521\": \"manifold://pai_mobile/tree/datasets/hand_unlabeled_nonUGC/short_range.json\",\n}\n\ndef load_json(json_file):\n \"\"\"\n Simply load and return the json_file\n \"\"\"\n with PathManager.open(json_file, \"r\") as f:\n json_data = json.load(f)\n return json_data\n\ndef register_unlabeled():\n \"\"\"\n Register the unlabeled datasets\n The json_file needs to be in D2's format\n \"\"\"\n for name, json_file in _UNLABELED_DATASETS.items():\n # 1. register a function which returns dicts\n DatasetCatalog.register(\n name,\n CallFuncWithJsonFile(\n func=load_json,\n json_file=json_file\n )\n )\n\n # 2. Optionally, add metadata about this dataset,\n # since they might be useful in evaluation, visualization or logging\n MetadataCatalog.get(name).set(\n json_file=json_file, image_root=\"\", evaluator_type=\"coco\"\n )\n\n\n# ==== Predefined splits for raw cityscapes foggy images ===========\n_RAW_CITYSCAPES_SPLITS = {\n # \"cityscapes_foggy_{task}_train\": (\"cityscape_foggy/leftImg8bit/train/\", \"cityscape_foggy/gtFine/train/\"),\n # \"cityscapes_foggy_{task}_val\": (\"cityscape_foggy/leftImg8bit/val/\", \"cityscape_foggy/gtFine/val/\"),\n # \"cityscapes_foggy_{task}_test\": (\"cityscape_foggy/leftImg8bit/test/\", \"cityscape_foggy/gtFine/test/\"),\n \"cityscapes_foggy_train\": (\"cityscape_foggy/leftImg8bit/train/\", \"cityscape_foggy/gtFine/train/\"),\n \"cityscapes_foggy_val\": (\"cityscape_foggy/leftImg8bit/val/\", \"cityscape_foggy/gtFine/val/\"),\n \"cityscapes_foggy_test\": (\"cityscape_foggy/leftImg8bit/test/\", \"cityscape_foggy/gtFine/test/\"),\n}\n\n\ndef register_all_cityscapes_foggy():\n root = \"manifold://mobile_vision_dataset/tree/yujheli/dataset\"\n for key, (image_dir, gt_dir) in _RAW_CITYSCAPES_SPLITS.items():\n meta = _get_builtin_metadata(\"cityscapes\")\n image_dir = os.path.join(root, image_dir)\n gt_dir = os.path.join(root, gt_dir)\n\n # inst_key = key.format(task=\"instance_seg\")\n inst_key = key\n # DatasetCatalog.register(\n # inst_key,\n # lambda x=image_dir, y=gt_dir: load_cityscapes_instances(\n # x, y, from_json=True, to_polygons=True\n # ),\n # )\n DatasetCatalog.register(\n inst_key,\n lambda x=image_dir, y=gt_dir: load_cityscapes_instances(\n x, y, from_json=False, to_polygons=False\n ),\n )\n # MetadataCatalog.get(inst_key).set(\n # image_dir=image_dir, gt_dir=gt_dir, evaluator_type=\"cityscapes_instance\", **meta\n # )\n # MetadataCatalog.get(inst_key).set(\n # image_dir=image_dir, gt_dir=gt_dir, evaluator_type=\"pascal_voc\", **meta\n # )\n MetadataCatalog.get(inst_key).set(\n image_dir=image_dir, gt_dir=gt_dir, evaluator_type=\"coco\", **meta\n )\n\n# ==== Predefined splits for Clipart (PASCAL VOC format) ===========\ndef register_all_clipart():\n root = \"manifold://mobile_vision_dataset/tree/yujheli/dataset\"\n SPLITS = [\n (\"Clipart1k_train\", \"clipart\", \"train\"),\n (\"Clipart1k_test\", \"clipart\", \"test\"),\n ]\n for name, dirname, split in SPLITS:\n year = 2012\n register_pascal_voc(name, os.path.join(root, dirname), split, year)\n MetadataCatalog.get(name).evaluator_type = \"pascal_voc\"\n # MetadataCatalog.get(name).evaluator_type = \"coco\"\n\nregister_all_cityscapes_foggy()\nregister_all_clipart()\n\n# register_coco_unlabel()\n# register_unlabeled()\n\ndef register_all_water():\n root = \"manifold://mobile_vision_dataset/tree/yujheli/dataset\" #Need to modify to the correct folder containing the dataset.\n SPLITS = [\n (\"Watercolor_train\", \"watercolor\", \"train\"),\n (\"Watercolor_test\", \"watercolor\", \"test\"),\n ]\n for name, dirname, split in SPLITS:\n year = 2012\n # register_pascal_voc(name, os.path.join(root, dirname), split, year, class_names=[\"person\", \"dog\",\"bicycle\", \"bird\", \"car\", \"cat\"])\n register_pascal_voc(name, os.path.join(root, dirname), split, year)\n MetadataCatalog.get(name).evaluator_type = \"pascal_voc_water\"\n\nregister_all_water()\n\ndef register_all_clipart_ws():\n root = \"manifold://mobile_vision_dataset/tree/yujheli/dataset\"\n SPLITS = [\n (\"Clipart1k_train_w\", \"clipart\", \"train\"),\n (\"Clipart1k_test_w\", \"clipart\", \"test\"),\n ]\n for name, dirname, split in SPLITS:\n year = 2012\n register_pascal_voc(name, os.path.join(root, dirname), split, year)\n MetadataCatalog.get(name).evaluator_type = \"pascal_voc_water\"\n # MetadataCatalog.get(name).evaluator_type = \"coco\"\n\nregister_all_clipart_ws()" }, { "path": "prod_lib/data/cityscapes_foggy.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates.\nimport functools\nimport json\nimport logging\nimport multiprocessing as mp\nimport numpy as np\nimport os\nfrom itertools import chain\nimport pycocotools.mask as mask_util\nfrom PIL import Image\n\nfrom detectron2.structures import BoxMode\nfrom detectron2.utils.comm import get_world_size\nfrom detectron2.utils.file_io import PathManager\nfrom detectron2.utils.logger import setup_logger\n\ntry:\n import cv2 # noqa\nexcept ImportError:\n # OpenCV is an optional dependency at the moment\n pass\n\n\nlogger = logging.getLogger(__name__)\n\n\ndef _get_cityscapes_files(image_dir, gt_dir):\n files = []\n # scan through the directory\n cities = PathManager.ls(image_dir)\n logger.info(f\"{len(cities)} cities found in '{image_dir}'.\")\n for city in cities:\n city_img_dir = os.path.join(image_dir, city)\n city_gt_dir = os.path.join(gt_dir, city)\n for basename in PathManager.ls(city_img_dir):\n image_file = os.path.join(city_img_dir, basename)\n\n # suffix = \"leftImg8bit.png\"\n # assert basename.endswith(suffix), basename\n # basename = basename[: -len(suffix)]\n\n suffix = 'leftImg8bit_foggy'\n basename = basename.split(suffix)[0]\n\n instance_file = os.path.join(city_gt_dir, basename + \"gtFine_instanceIds.png\")\n label_file = os.path.join(city_gt_dir, basename + \"gtFine_labelIds.png\")\n json_file = os.path.join(city_gt_dir, basename + \"gtFine_polygons.json\")\n\n files.append((image_file, instance_file, label_file, json_file))\n assert len(files), \"No images found in {}\".format(image_dir)\n for f in files[0]:\n assert PathManager.isfile(f), f\n return files\n\n\ndef load_cityscapes_instances(image_dir, gt_dir, from_json=True, to_polygons=True):\n \"\"\"\n Args:\n image_dir (str): path to the raw dataset. e.g., \"~/cityscapes/leftImg8bit/train\".\n gt_dir (str): path to the raw annotations. e.g., \"~/cityscapes/gtFine/train\".\n from_json (bool): whether to read annotations from the raw json file or the png files.\n to_polygons (bool): whether to represent the segmentation as polygons\n (COCO's format) instead of masks (cityscapes's format).\n\n Returns:\n list[dict]: a list of dicts in Detectron2 standard format. (See\n `Using Custom Datasets `_ )\n \"\"\"\n if from_json:\n assert to_polygons, (\n \"Cityscapes's json annotations are in polygon format. \"\n \"Converting to mask format is not supported now.\"\n )\n files = _get_cityscapes_files(image_dir, gt_dir)\n\n logger.info(\"Preprocessing cityscapes annotations ...\")\n # This is still not fast: all workers will execute duplicate works and will\n # take up to 10m on a 8GPU server.\n pool = mp.Pool(processes=max(mp.cpu_count() // get_world_size() // 2, 4))\n\n ret = pool.map(\n functools.partial(_cityscapes_files_to_dict, from_json=from_json, to_polygons=to_polygons),\n files,\n )\n logger.info(\"Loaded {} images from {}\".format(len(ret), image_dir))\n\n # Map cityscape ids to contiguous ids\n from cityscapesscripts.helpers.labels import labels\n\n labels = [l for l in labels if l.hasInstances and not l.ignoreInEval]\n dataset_id_to_contiguous_id = {l.id: idx for idx, l in enumerate(labels)}\n for dict_per_image in ret:\n for anno in dict_per_image[\"annotations\"]:\n anno[\"category_id\"] = dataset_id_to_contiguous_id[anno[\"category_id\"]]\n return ret\n\n\ndef load_cityscapes_semantic(image_dir, gt_dir):\n \"\"\"\n Args:\n image_dir (str): path to the raw dataset. e.g., \"~/cityscapes/leftImg8bit/train\".\n gt_dir (str): path to the raw annotations. e.g., \"~/cityscapes/gtFine/train\".\n\n Returns:\n list[dict]: a list of dict, each has \"file_name\" and\n \"sem_seg_file_name\".\n \"\"\"\n ret = []\n # gt_dir is small and contain many small files. make sense to fetch to local first\n gt_dir = PathManager.get_local_path(gt_dir)\n for image_file, _, label_file, json_file in _get_cityscapes_files(image_dir, gt_dir):\n label_file = label_file.replace(\"labelIds\", \"labelTrainIds\")\n\n with PathManager.open(json_file, \"r\") as f:\n jsonobj = json.load(f)\n ret.append(\n {\n \"file_name\": image_file,\n \"sem_seg_file_name\": label_file,\n \"height\": jsonobj[\"imgHeight\"],\n \"width\": jsonobj[\"imgWidth\"],\n }\n )\n assert len(ret), f\"No images found in {image_dir}!\"\n assert PathManager.isfile(\n ret[0][\"sem_seg_file_name\"]\n ), \"Please generate labelTrainIds.png with cityscapesscripts/preparation/createTrainIdLabelImgs.py\" # noqa\n return ret\n\n\ndef _cityscapes_files_to_dict(files, from_json, to_polygons):\n \"\"\"\n Parse cityscapes annotation files to a instance segmentation dataset dict.\n\n Args:\n files (tuple): consists of (image_file, instance_id_file, label_id_file, json_file)\n from_json (bool): whether to read annotations from the raw json file or the png files.\n to_polygons (bool): whether to represent the segmentation as polygons\n (COCO's format) instead of masks (cityscapes's format).\n\n Returns:\n A dict in Detectron2 Dataset format.\n \"\"\"\n from cityscapesscripts.helpers.labels import id2label, name2label\n\n image_file, instance_id_file, _, json_file = files\n\n annos = []\n\n if from_json:\n from shapely.geometry import MultiPolygon, Polygon\n\n with PathManager.open(json_file, \"r\") as f:\n jsonobj = json.load(f)\n ret = {\n \"file_name\": image_file,\n \"image_id\": os.path.basename(image_file),\n \"height\": jsonobj[\"imgHeight\"],\n \"width\": jsonobj[\"imgWidth\"],\n }\n\n # `polygons_union` contains the union of all valid polygons.\n polygons_union = Polygon()\n\n # CityscapesScripts draw the polygons in sequential order\n # and each polygon *overwrites* existing ones. See\n # (https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/preparation/json2instanceImg.py) # noqa\n # We use reverse order, and each polygon *avoids* early ones.\n # This will resolve the ploygon overlaps in the same way as CityscapesScripts.\n for obj in jsonobj[\"objects\"][::-1]:\n if \"deleted\" in obj: # cityscapes data format specific\n continue\n label_name = obj[\"label\"]\n\n try:\n label = name2label[label_name]\n except KeyError:\n if label_name.endswith(\"group\"): # crowd area\n label = name2label[label_name[: -len(\"group\")]]\n else:\n raise\n if label.id < 0: # cityscapes data format\n continue\n\n # Cityscapes's raw annotations uses integer coordinates\n # Therefore +0.5 here\n poly_coord = np.asarray(obj[\"polygon\"], dtype=\"f4\") + 0.5\n # CityscapesScript uses PIL.ImageDraw.polygon to rasterize\n # polygons for evaluation. This function operates in integer space\n # and draws each pixel whose center falls into the polygon.\n # Therefore it draws a polygon which is 0.5 \"fatter\" in expectation.\n # We therefore dilate the input polygon by 0.5 as our input.\n poly = Polygon(poly_coord).buffer(0.5, resolution=4)\n\n if not label.hasInstances or label.ignoreInEval:\n # even if we won't store the polygon it still contributes to overlaps resolution\n polygons_union = polygons_union.union(poly)\n continue\n\n # Take non-overlapping part of the polygon\n poly_wo_overlaps = poly.difference(polygons_union)\n if poly_wo_overlaps.is_empty:\n continue\n polygons_union = polygons_union.union(poly)\n\n anno = {}\n anno[\"iscrowd\"] = label_name.endswith(\"group\")\n anno[\"category_id\"] = label.id\n\n if isinstance(poly_wo_overlaps, Polygon):\n poly_list = [poly_wo_overlaps]\n elif isinstance(poly_wo_overlaps, MultiPolygon):\n poly_list = poly_wo_overlaps.geoms\n else:\n raise NotImplementedError(\"Unknown geometric structure {}\".format(poly_wo_overlaps))\n\n poly_coord = []\n for poly_el in poly_list:\n # COCO API can work only with exterior boundaries now, hence we store only them.\n # TODO: store both exterior and interior boundaries once other parts of the\n # codebase support holes in polygons.\n poly_coord.append(list(chain(*poly_el.exterior.coords)))\n anno[\"segmentation\"] = poly_coord\n (xmin, ymin, xmax, ymax) = poly_wo_overlaps.bounds\n\n anno[\"bbox\"] = (xmin, ymin, xmax, ymax)\n anno[\"bbox_mode\"] = BoxMode.XYXY_ABS\n\n annos.append(anno)\n else:\n # See also the official annotation parsing scripts at\n # https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/instances2dict.py # noqa\n with PathManager.open(instance_id_file, \"rb\") as f:\n inst_image = np.asarray(Image.open(f), order=\"F\")\n # ids < 24 are stuff labels (filtering them first is about 5% faster)\n flattened_ids = np.unique(inst_image[inst_image >= 24])\n\n ret = {\n \"file_name\": image_file,\n \"image_id\": os.path.basename(image_file),\n \"height\": inst_image.shape[0],\n \"width\": inst_image.shape[1],\n }\n\n for instance_id in flattened_ids:\n # For non-crowd annotations, instance_id // 1000 is the label_id\n # Crowd annotations have <1000 instance ids\n label_id = instance_id // 1000 if instance_id >= 1000 else instance_id\n label = id2label[label_id]\n if not label.hasInstances or label.ignoreInEval:\n continue\n\n anno = {}\n anno[\"iscrowd\"] = instance_id < 1000\n anno[\"category_id\"] = label.id\n\n mask = np.asarray(inst_image == instance_id, dtype=np.uint8, order=\"F\")\n\n inds = np.nonzero(mask)\n ymin, ymax = inds[0].min(), inds[0].max()\n xmin, xmax = inds[1].min(), inds[1].max()\n anno[\"bbox\"] = (xmin, ymin, xmax, ymax)\n if xmax <= xmin or ymax <= ymin:\n continue\n anno[\"bbox_mode\"] = BoxMode.XYXY_ABS\n if to_polygons:\n # This conversion comes from D4809743 and D5171122,\n # when Mask-RCNN was first developed.\n contours = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[\n -2\n ]\n polygons = [c.reshape(-1).tolist() for c in contours if len(c) >= 3]\n # opencv's can produce invalid polygons\n if len(polygons) == 0:\n continue\n anno[\"segmentation\"] = polygons\n else:\n anno[\"segmentation\"] = mask_util.encode(mask[:, :, None])[0]\n annos.append(anno)\n ret[\"annotations\"] = annos\n return ret\n" }, { "path": "prod_lib/engine/probe.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\nfrom detectron2.structures import pairwise_iou\n\nclass OpenMatchTrainerProbe:\n def __init__(self, cfg):\n self.BOX_AP = 0.5\n self.NUM_CLASSES = cfg.MODEL.ROI_HEADS.NUM_CLASSES\n # self.bbox_stat_list = ['compute_fp_gtoutlier', 'compute_num_box', 'compute_ood_acc']\n \n def bbox_stat(self, unlabel_gt, unlabel_pseudo, name, bbox_stat_list):\n stats = {}\n \n sum_gpu_names = []\n for metric in bbox_stat_list:\n stats_per, sum_gpu_names_per = getattr(\n self, metric)(unlabel_gt, unlabel_pseudo, name)\n stats.update(stats_per)\n sum_gpu_names.extend(sum_gpu_names_per)\n \n return stats, sum_gpu_names\n \n def compute_fp_gtoutlier(self, unlabel_gt, unlabel_pseudo, name):\n num_gt_ood_object = 0\n num_gt_fp_ood_object = 0\n sum_iou = 0.0\n sum_gpu_names = []\n results = {}\n \n if len(unlabel_gt) != 0:\n for gt, pseudo in zip(unlabel_gt, unlabel_pseudo):\n # import pdb; pdb. set_trace() \n if name == \"pred\":\n pp_boxes = pseudo.pred_boxes\n elif name == \"pseudo_conf\" or name == \"pseudo_ood\":\n # filter predicted ood box when evaluating this metric\n pseudo = pseudo[pseudo.gt_classes != -1]\n pp_boxes = pseudo.gt_boxes\n \n else:\n raise ValueError(\"Unknown name for probe roi bbox.\")\n \n if len(gt) != 0 and len(pseudo) != 0:\n max_iou, max_idx = pairwise_iou(\n gt.gt_boxes.to('cuda'), pp_boxes).max(1)\n ood_idx = (gt.gt_classes == -1)\n \n num_gt_ood_object += ood_idx.sum().item()\n num_gt_fp_ood_object += (max_iou[ood_idx]\n > self.BOX_AP).sum().item()\n sum_iou += max_iou[ood_idx].sum().item()\n \n elif len(gt) != 0 and len(pseudo) == 0:\n ood_idx = (gt.gt_classes == -1)\n num_gt_ood_object += ood_idx.shape[0]\n \n results = {'Analysis_'+name+'/num_gt_ood_object': num_gt_ood_object,\n 'Analysis_'+name+'/num_gt_fp_ood_object': num_gt_fp_ood_object,\n 'Analysis_'+name+'/sum_iou': sum_iou}\n \n sum_gpu_names.extend(list(results.keys()))\n \n return results, sum_gpu_names\n \n def compute_num_box(self, unlabel_gt, unlabel_pseudo, name):\n num_bbox = 0.0\n size_bbox = 0.0\n avg_conf = 0.0\n \n # measure in and out box for openset SS-OD\n num_bbox_in = 0.0\n num_bbox_out = 0.0\n num_bg = 0.0\n \n # when ground-truth is missing in unlabeled data\n if len(unlabel_gt) == 0:\n for pp_roi in unlabel_pseudo:\n if name == \"pred\":\n pp_boxes = pp_roi.pred_boxes\n pp_classes = pp_roi.pred_classes\n pp_scores = pp_roi.scores\n elif name == \"pseudo_conf\" or name == \"pseudo_ood\":\n pp_boxes = pp_roi.gt_boxes\n pp_classes = pp_roi.gt_classes\n pp_scores = pp_roi.scores\n elif name == \"gt\":\n pp_boxes = pp_roi.gt_boxes\n pp_classes = pp_roi.gt_classes\n else:\n raise ValueError(\"Unknown name for probe roi bbox.\")\n # all boxes (in + out boxes)\n if len(pp_roi) != 0:\n # bbox number and size\n num_bbox += len(pp_roi)\n size_bbox += pp_boxes.area().mean().item()\n # average box confidence\n if name != \"gt\":\n avg_conf += pp_scores.mean()\n else:\n num_bbox += 0\n size_bbox += torch.tensor(0).cuda()\n num_valid_img = len(unlabel_pseudo)\n else:\n # with ground-truth\n num_valid_img = 0\n for gt, pp_roi in zip(unlabel_gt, unlabel_pseudo):\n \n if name == \"pred\":\n pp_boxes = pp_roi.pred_boxes\n pp_classes = pp_roi.pred_classes\n pp_scores = pp_roi.scores\n elif name == \"pseudo_conf\" or name == \"pseudo_ood\":\n # filter out ood pseudo-box when doing analysis\n pp_roi = pp_roi[pp_roi.gt_classes != -1]\n \n pp_boxes = pp_roi.gt_boxes\n pp_classes = pp_roi.gt_classes\n pp_scores = pp_roi.scores\n elif name == \"gt\":\n pp_boxes = pp_roi.gt_boxes\n pp_classes = pp_roi.gt_classes\n else:\n raise ValueError(\"Unknown name for probe roi bbox.\")\n \n # all boxes (in + out boxes)\n if len(pp_roi) != 0:\n # bbox number and size\n num_bbox += len(pp_roi)\n size_bbox += pp_boxes.area().mean().item()\n \n # average box confidence\n if name != \"gt\":\n avg_conf += pp_scores.mean()\n else:\n num_bbox += 0\n size_bbox += torch.tensor(0).cuda()\n \n # in and out class\n if name == \"gt\":\n pp_roi_in = pp_roi[pp_classes != -1]\n num_bbox_in += len(pp_roi_in)\n \n pp_roi_out = pp_roi[pp_classes == -1]\n num_bbox_out += len(pp_roi_out)\n num_valid_img += 1\n \n \n elif name == \"pred\" or name == \"pseudo_conf\" or name == \"pseudo_ood\":\n \n if len(gt.gt_boxes.to('cuda'))>0 and len(pp_boxes) > 0:\n max_iou, max_idx = pairwise_iou(gt.gt_boxes.to('cuda'), pp_boxes).max(0)\n \n # for the ground-truth label for each pseudo-box\n gtclass4pseudo = gt.gt_classes[max_idx]\n matchgtbox = max_iou > 0.5\n \n # compute the number of boxes (background, inlier, outlier)\n num_bg += (~matchgtbox).sum().item()\n num_bbox_in += (gtclass4pseudo[matchgtbox]\n != -1).sum().item()\n num_bbox_out += (gtclass4pseudo[matchgtbox]\n == -1).sum().item()\n num_valid_img += 1\n \n else:\n raise ValueError(\"Unknown name for probe roi bbox.\")\n \n box_probe = {}\n if num_valid_img >0 :\n box_probe[\"Analysis_\" + name + \"/Num_bbox\"] = num_bbox / \\\n num_valid_img\n box_probe[\"Analysis_\" + name + \"/Size_bbox\"] = size_bbox / \\\n num_valid_img\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_inlier\"] = num_bbox_in / num_valid_img\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_outlier\"] = num_bbox_out / num_valid_img\n \n if name != \"gt\": # prediciton, background number\n box_probe[\"Analysis_\" + name + \"/Conf\"] = avg_conf / \\\n num_valid_img\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_background\"] = num_bg / num_valid_img\n box_probe[\"Analysis_\" + name +\n \"/background_fp_ratio\"] = num_bg / num_bbox\n box_probe[\"Analysis_\" + name +\n \"/background_tp_ratio\"] = num_bbox_in / num_bbox\n else:\n box_probe[\"Analysis_\" + name + \"/Num_bbox\"] = 0.0\n box_probe[\"Analysis_\" + name + \"/Size_bbox\"] = 0.0\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_inlier\"] = 0.0\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_outlier\"] = 0.0\n \n if name != \"gt\": # prediciton, background number\n box_probe[\"Analysis_\" + name + \"/Conf\"] = 0.0\n box_probe[\"Analysis_\" + name +\n \"/Num_bbox_background\"] = 0.0\n box_probe[\"Analysis_\" + name +\n \"/background_fp_ratio\"] = num_bg / num_bbox\n box_probe[\"Analysis_\" + name +\n \"/background_tp_ratio\"] = num_bbox_in / num_bbox\n\n return box_probe, []\n \n def compute_ood_acc(self, unlabel_gt, unlabel_pseudo, name, BOX_IOU=0.5):\n results = {}\n sum_gpu_names = []\n \n if len(unlabel_gt) != 0:\n for metric in ['acc_outlier', 'recall_outlier']:\n for samples in ['_fg', '_all']:\n for fraction_part in ['_nume', '_deno']:\n results[metric+samples+fraction_part] = 0.0\n \n for gt, pred in zip(unlabel_gt, unlabel_pseudo):\n if name == \"pred\":\n pp_boxes = pred.pred_boxes\n pp_ood_scores = pred.ood_scores\n \n elif name == \"pseudo_conf\" or name == \"pseudo_ood\":\n # assume these outlier are suppressed\n pred = pred[pred.gt_classes != -1]\n \n pp_boxes = pred.gt_boxes\n pp_ood_scores = pred.ood_scores\n \n else:\n raise ValueError(\"Unknown name for probe roi bbox.\")\n \n if len(gt) != 0 and len(pred) != 0:\n # find the most overlapped ground-truth box for each pseudo-box\n max_iou, max_idx = pairwise_iou(\n gt.gt_boxes.to('cuda'), pp_boxes).max(0)\n \n # ignore background instances\n find_fg_mask = max_iou > BOX_IOU\n if find_fg_mask.sum() > 0:\n gt_corres = gt[max_idx].gt_classes.to(\"cuda\")\n \n gt_outlier = (gt_corres[find_fg_mask] == -1)\n pred_outlier = pp_ood_scores[find_fg_mask][:, 0] > 0.5\n \n # accurcay of ood detection (foreground)\n # acc_outlier_fg = (pred_outlier == gt_outlier).sum() /find_fg_mask.sum()\n results['acc_outlier_fg_nume'] += (\n pred_outlier == gt_outlier).sum()\n results['acc_outlier_fg_deno'] += find_fg_mask.sum()\n \n # recall of ood detection (foreground)\n # recall_outlier_fg = (pred_outlier[gt_outlier] == gt_outlier[gt_outlier]).sum() /gt_outlier.sum()\n results['recall_outlier_fg_nume'] += (\n pred_outlier[gt_outlier] == gt_outlier[gt_outlier]).sum()\n results['recall_outlier_fg_deno'] += gt_outlier.sum()\n \n # Regard backgound gt as outlier\n gt_corres = gt[max_idx].gt_classes.to(\"cuda\")\n # convert all background gt as outlier\n gt_corres[~find_fg_mask] = -1\n gt_outlier = gt_corres == -1\n \n pred_outlier = pp_ood_scores[:, 0] > 0.5\n \n # accurcay of ood detection (all)\n # acc_outlier_all = (pred_outlier == gt_outlier).sum() /len(pred)\n results['acc_outlier_all_nume'] += (\n pred_outlier == gt_outlier).sum()\n results['acc_outlier_all_deno'] += len(pred)\n \n # recall of ood detection (all)\n # recall_outlier_all = (pred_outlier[gt_outlier] == gt_outlier[gt_outlier]).sum() /gt_outlier.sum()\n results['recall_outlier_all_nume'] += (\n pred_outlier[gt_outlier] == gt_outlier[gt_outlier]).sum()\n results['recall_outlier_all_deno'] += gt_outlier.sum()\n \n results = {'Analysis_'+name+'/'+k: v for k, v in results.items()}\n \n sum_gpu_names.extend(list(results.keys()))\n \n return results, sum_gpu_names\n" }, { "path": "prod_lib/engine/trainer.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\nimport logging\nimport time\nfrom collections import OrderedDict\nfrom typing import Dict\n\nimport detectron2.utils.comm as comm\nimport numpy as np\nimport torch\nfrom detectron2.engine import SimpleTrainer\nfrom detectron2.structures import BitMasks, Boxes, Instances, Keypoints\nfrom detectron2.utils.events import get_event_storage\nfrom d2go.projects.unbiased_teacher.engine.trainer import UnbiasedTeacherTrainer\nfrom d2go.projects.unbiased_teacher.utils.probe import probe\n\nimport copy\n\nlogger = logging.getLogger(__name__)\n\nclass DAobjTrainer(UnbiasedTeacherTrainer):\n \"\"\"\n A trainer for Teacher-Student mutual learning following this paper:\n \"Unbiased Teacher for Semi-Supervised Object Detection\"\n\n It assumes that every step, you:\n\n For Teacher:\n 1. Perform a forward pass on a weakly augmented unlabeled data from the data_loader.\n 2. Generate pseudo-labels on the weakly augmented unlabeled data\n\n For Student:\n 1. Perform a forward pass on a strongly augmented unlabeled data from the data_loader.\n 2. Perform a forward pass on a labeled data from the data_loader.\n 1. Use pseudo-labels generated from the Teacher as target and compute the\n loss on a strongly augmented unlabeled data\n 2. Compute the gradients with the above losses on labeled and unlabeled data.\n 3. Update the Student model with the optimizer.\n 4. EMA update the Teacher model\n \"\"\"\n\n # def __init__(self, cfg, model, model_teacher, data_loader, optimizer):\n # \"\"\"\n # Args:\n # model: a torch Module. Takes a data from data_loader and returns a\n # dict of losses.\n # data_loader: an iterable. Contains data to be used to call model.\n # optimizer: a torch optimizer.\n # \"\"\"\n # super().__init__(model, data_loader, optimizer)\n\n # self.cfg = cfg\n # self.model_teacher = model_teacher\n\n def run_step(self):\n assert (\n self.model.training\n ), \"Student model was changed to eval mode during training\"\n start = time.perf_counter()\n\n data = next(self._data_loader_iter)\n # q (queue): strongly augmented, k (key): weakly augmented\n #TODO Need to further use the weak samples for domain adaptation\n label_data_q, label_data_k, unlabel_data_q, unlabel_data_k = data\n data_time = time.perf_counter() - start\n\n if (\n self.cfg.UNBIASEDTEACHER.BURN_IN_STEP != 0\n and self.iter < self.cfg.UNBIASEDTEACHER.BURN_IN_STEP\n ):\n # Burn-In stage. Supervisedly train the Student model.\n losses, loss_dict, record_dict = self.burn_in(label_data_q, label_data_k)\n else:\n # Copy the Student model to the Teacher (using keep_rate = 0)\n if self.iter == self.cfg.UNBIASEDTEACHER.BURN_IN_STEP:\n logger.info(\"Copying Student weights to the Teacher .....\")\n self._update_teacher_model(keep_rate=0.0)\n elif (\n self.iter - self.cfg.UNBIASEDTEACHER.BURN_IN_STEP\n ) % self.cfg.UNBIASEDTEACHER.TEACHER_UPDATE_ITER == 0:\n self._update_teacher_model(\n keep_rate=self.cfg.UNBIASEDTEACHER.EMA.KEEP_RATE\n )\n\n # Teacher-Student Mutual Learning\n losses, loss_dict, record_dict = self.teacher_student_learning(\n label_data_q, label_data_k, unlabel_data_q, unlabel_data_k\n )\n\n self.optimizer.zero_grad()\n losses.backward()\n\n self._write_metrics(record_dict, data_time)\n\n \"\"\"\n If you need gradient clipping/scaling or other processing, you can\n wrap the optimizer with your custom `step()` method. But it is\n suboptimal as explained in https://arxiv.org/abs/2006.15704 Sec 3.2.4\n \"\"\"\n self.optimizer.step()\n\n def burn_in(self, label_data_q, label_data_k):\n \"\"\"\n Perform Burn-In stage with labeled data\n \"\"\"\n # combine label_data_q + label_data_k\n label_data_q.extend(label_data_k)\n\n record_dict, _, _, _ = self.model(label_data_q, branch=\"supervised\")\n\n # weight losses\n loss_dict = self.weight_losses(record_dict)\n losses = sum(loss_dict.values())\n return losses, loss_dict, record_dict\n\n def teacher_student_learning(\n self, label_data_q, label_data_k, unlabel_data_q, unlabel_data_k\n ):\n \"\"\"\n Perform Teacher-Student Mutual Learning with labeled and unlabeled data\n \"\"\"\n # q (queue): strongly augmented, k (key): weakly augmented\n record_dict = {}\n\n ######################## For probe #################################\n # import pdb; pdb. set_trace() \n gt_unlabel_k = self.get_label(unlabel_data_k)\n\n # 0. remove potential ground-truth labels in the unlabeled data\n unlabel_data_q = self.remove_label(unlabel_data_q)\n unlabel_data_k = self.remove_label(unlabel_data_k)\n\n # 1. generate the pseudo-label using teacher model\n # TODO: why is the Teacher not in .eval() mode?\n with torch.no_grad():\n (\n _,\n proposals_rpn_unsup_k,\n proposals_roih_unsup_k,\n _,\n ) = self.model_teacher(unlabel_data_k, branch=\"unsup_data_weak\")\n\n \n\n ######################## For probe #################################\n # import pdb; pdb. set_trace()\n # analysis_pred, _ = self.probe.compute_num_box(gt_unlabel_k,proposals_roih_unsup_k,'pred')\n # record_dict.update(analysis_pred)\n # 2. Pseudo-labeling\n # Pseudo-labeling for RPN head (bbox location/objectness)\n joint_proposal_dict = {}\n\n ## No need this\n joint_proposal_dict[\"proposals_rpn\"] = proposals_rpn_unsup_k\n\n (\n pesudo_proposals_rpn_unsup_k,\n nun_pseudo_bbox_rpn,\n ) = self.process_pseudo_label(\n proposals_rpn_unsup_k,\n self.cfg.UNBIASEDTEACHER.BBOX_THRESHOLD,\n self.cfg.UNBIASEDTEACHER.MASK_THRESHOLD,\n self.cfg.UNBIASEDTEACHER.KEYPOINT_THRESHOLD,\n \"rpn\",\n \"thresholding\",\n )\n joint_proposal_dict[\"proposals_pseudo_rpn\"] = pesudo_proposals_rpn_unsup_k\n ## No need this end\n\n\n # Pseudo-labeling for ROI head (bbox location/objectness)\n pesudo_proposals_roih_unsup_k, _ = self.process_pseudo_label(\n proposals_roih_unsup_k,\n self.cfg.UNBIASEDTEACHER.BBOX_THRESHOLD,\n self.cfg.UNBIASEDTEACHER.MASK_THRESHOLD,\n self.cfg.UNBIASEDTEACHER.KEYPOINT_THRESHOLD,\n \"roih\",\n \"thresholding\",\n )\n joint_proposal_dict[\"proposals_pseudo_roih\"] = pesudo_proposals_roih_unsup_k\n\n ######################## For probe #################################\n analysis_pred, _ = self.probe.compute_num_box(gt_unlabel_k,pesudo_proposals_roih_unsup_k,'pred')\n record_dict.update(analysis_pred)\n\n # Probe for analysis (usually for research development)\n if self.cfg.UNBIASEDTEACHER.PROBE:\n record_dict = probe(\n self.cfg,\n proposals_roih_unsup_k,\n unlabel_data_k,\n pesudo_proposals_roih_unsup_k,\n record_dict,\n )\n\n # 3. add pseudo-label to unlabeled data\n unlabel_data_q = self.add_label(\n unlabel_data_q, joint_proposal_dict[\"proposals_pseudo_roih\"]\n )\n unlabel_data_k = self.add_label(\n unlabel_data_k, joint_proposal_dict[\"proposals_pseudo_roih\"]\n )\n\n # all_label_data = label_data_q + label_data_k\n if self.cfg.UNBIASEDTEACHER.ISAUG == \"No\":\n all_label_data = label_data_k\n all_unlabel_data = unlabel_data_k\n else:\n all_label_data = label_data_q + label_data_k\n all_unlabel_data = unlabel_data_q\n\n # 4. input both strongly and weakly augmented labeled data into student model\n # all_unlabel_data = unlabel_data_q\n record_all_label_data, _, _, _ = self.model(all_label_data, branch=\"supervised\")\n record_dict.update(record_all_label_data)\n\n # 5. input strongly augmented unlabeled data into model\n record_all_unlabel_data, _, _, _ = self.model(\n all_unlabel_data, branch=\"supervised-pseudo\"\n )\n\n # rename unsupervised loss\n # NOTE: names of the recorded output from model are hard-coded\n # we rename them accordingly for unlabeled data\n new_record_all_unlabel_data = {}\n for key in record_all_unlabel_data.keys():\n new_record_all_unlabel_data[key + \"_pseudo\"] = record_all_unlabel_data[key]\n record_dict.update(new_record_all_unlabel_data)\n\n # 6. input weakly labeled data (source) and weakly unlabeled data (target) to student model\n # give sign to the target data\n\n for i_index in range(len(unlabel_data_k)):\n # unlabel_data_item = {}\n for k, v in unlabel_data_k[i_index].items():\n # label_data_k[i_index][k + \"_unlabeled\"] = v\n label_data_k[i_index][k + \"_unlabeled\"] = v\n # unlabel_data_k[i_index] = unlabel_data_item\n\n all_domain_data = label_data_k\n # all_domain_data = label_data_k + unlabel_data_k\n record_all_domain_data, _, _, _ = self.model(all_domain_data, branch=\"domain\")\n record_dict.update(record_all_domain_data)\n\n # 7. distill teacher\n # for distill back to teacher\n with torch.no_grad():\n (\n _,\n proposals_rpn_unsup_dis,\n proposals_roih_unsup_dis,\n _,\n ) = self.model(unlabel_data_k, branch=\"unsup_data_weak\")\n\n\n pesudo_proposals_roih_unsup_k, _ = self.process_pseudo_label(\n proposals_roih_unsup_dis,\n self.cfg.UNBIASEDTEACHER.BBOX_THRESHOLD,\n self.cfg.UNBIASEDTEACHER.MASK_THRESHOLD,\n self.cfg.UNBIASEDTEACHER.KEYPOINT_THRESHOLD,\n \"roih\",\n \"thresholding\",\n )\n unlabel_data_k = self.remove_label(unlabel_data_k)\n unlabel_data_k = self.add_label(\n unlabel_data_k, pesudo_proposals_roih_unsup_k\n )\n record_distill_data, _, _, _ = self.model_teacher(\n unlabel_data_k, branch=\"supervised-pseudo\"\n )\n new_record_all_distill_data = {}\n for key in record_distill_data.keys():\n new_record_all_distill_data[key + \"_distill\"] = record_distill_data[key]\n record_dict.update(new_record_all_distill_data)\n\n\n\n # weighting losses\n loss_dict = self.weight_losses(record_dict)\n\n #Add discriminator loss here\n #loss_dict.update(...)\n losses = sum(loss_dict.values())\n return losses, loss_dict, record_dict\n\n def weight_losses(self, record_dict):\n loss_dict = {}\n REGRESSION_LOSS_WEIGHT = 0\n for key in record_dict.keys():\n if key.startswith(\"loss\"):\n if key == \"loss_rpn_cls_pseudo\":\n loss_dict[key] = (\n record_dict[key]\n * self.cfg.UNBIASEDTEACHER.UNSUP_LOSS_WEIGHT_RPN_CLS\n )\n elif (\n key == \"loss_rpn_loc_pseudo\" or key == \"loss_box_reg_pseudo\"\n ): # set pseudo bbox regression to 0\n loss_dict[key] = record_dict[key] * REGRESSION_LOSS_WEIGHT\n elif (\n key == \"loss_rpn_loc_distill\" or key == \"loss_box_reg_distill\"\n ): # set pseudo bbox regression to 0\n loss_dict[key] = record_dict[key] * REGRESSION_LOSS_WEIGHT\n elif key.endswith(\"mask_pseudo\"): # unsupervised loss for segmentation\n loss_dict[key] = (\n record_dict[key]\n * self.cfg.UNBIASEDTEACHER.UNSUP_LOSS_WEIGHT_MASK\n )\n elif key.endswith(\"keypoint_pseudo\"): # unsupervised loss for keypoint\n loss_dict[key] = (\n record_dict[key]\n * self.cfg.UNBIASEDTEACHER.UNSUP_LOSS_WEIGHT_KEYPOINT\n )\n elif key.endswith(\"pseudo\"): # unsupervised loss\n loss_dict[key] = (\n record_dict[key] * self.cfg.UNBIASEDTEACHER.UNSUP_LOSS_WEIGHT\n )\n elif (\n key == \"loss_D_img_s\" or key == \"loss_D_img_t\"\n ): # set weight for discriminator\n # import pdb\n # pdb.set_trace()\n loss_dict[key] = record_dict[key] * self.cfg.UNBIASEDTEACHER.DIS_LOSS_WEIGHT #Need to modify defaults and yaml\n else: # supervised loss\n loss_dict[key] = record_dict[key] * 1\n return loss_dict\n\n def threshold_bbox(\n self,\n proposal_bbox_inst,\n thres=0.7,\n mask_thres=0.5,\n keypoint_thres=0.5,\n proposal_type=\"roih\",\n ):\n if proposal_type == \"rpn\":\n valid_map = proposal_bbox_inst.objectness_logits > thres\n\n # create instances containing boxes and gt_classes\n image_shape = proposal_bbox_inst.image_size\n new_proposal_inst = Instances(image_shape)\n\n # create box\n new_bbox_loc = proposal_bbox_inst.proposal_boxes.tensor[valid_map, :]\n new_boxes = Boxes(new_bbox_loc)\n\n # add boxes to instances\n new_proposal_inst.gt_boxes = new_boxes\n new_proposal_inst.pred_boxes = new_boxes\n new_proposal_inst.objectness_logits = proposal_bbox_inst.objectness_logits[\n valid_map\n ]\n elif proposal_type == \"roih\":\n valid_map = proposal_bbox_inst.scores > thres\n\n # create instances containing boxes and gt_classes\n image_shape = proposal_bbox_inst.image_size\n new_proposal_inst = Instances(image_shape)\n\n # create box\n new_bbox_loc = proposal_bbox_inst.pred_boxes.tensor[valid_map, :]\n new_boxes = Boxes(new_bbox_loc)\n\n # add boxes to instances\n new_proposal_inst.gt_boxes = new_boxes\n new_proposal_inst.pred_boxes = new_boxes\n new_proposal_inst.gt_classes = proposal_bbox_inst.pred_classes[valid_map]\n new_proposal_inst.pred_classes = proposal_bbox_inst.pred_classes[valid_map]\n new_proposal_inst.scores = proposal_bbox_inst.scores[valid_map]\n\n if self.cfg.MODEL.MASK_ON and new_boxes:\n # put predicted output into gt_masks with thresholding\n new_masks = proposal_bbox_inst.pred_masks[valid_map].squeeze(1)\n new_masks = new_masks >= mask_thres\n new_proposal_inst.gt_masks = BitMasks(new_masks)\n if self.cfg.MODEL.KEYPOINT_ON and new_boxes:\n # we use the keypoint score as the basis for thresholding\n new_keypoints = proposal_bbox_inst.pred_keypoints[valid_map, :]\n invalid_keypoints = new_keypoints[:, :, 2] < keypoint_thres\n # (x, y, visibility): visibility flag = 0 -> not labeled (in which case x=y=0)\n new_keypoints[invalid_keypoints] = torch.FloatTensor([0, 0, 0]).to(\n new_keypoints.device\n )\n new_proposal_inst.gt_keypoints = Keypoints(new_keypoints)\n\n return new_proposal_inst\n \n" }, { "path": "prod_lib/evaluation/__init__.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates.\nfrom .coco_evaluation import COCOEvaluator\nfrom .pascal_voc_evaluation import PascalVOCDetectionEvaluator\n\n# __all__ = [k for k in globals().keys() if not k.startswith(\"_\")]\n\n__all__ = [\n \"COCOEvaluator\",\n \"PascalVOCDetectionEvaluator\"\n]\n" }, { "path": "prod_lib/evaluation/coco_evaluation.py", "content": "# Copyright (c) Facebook, Inc. and its affiliates.\nimport contextlib\nimport copy\nimport io\nimport itertools\nimport json\nimport logging\nimport numpy as np\nimport os\nimport pickle\nfrom collections import OrderedDict\nimport pycocotools.mask as mask_util\nimport torch\nfrom pycocotools.coco import COCO\nfrom pycocotools.cocoeval import COCOeval\nfrom tabulate import tabulate\n\nimport detectron2.utils.comm as comm\nfrom detectron2.config import CfgNode\nfrom detectron2.data import MetadataCatalog\nfrom detectron2.data.datasets.coco import convert_to_coco_dict\nfrom detectron2.evaluation.fast_eval_api import COCOeval_opt\nfrom detectron2.structures import Boxes, BoxMode, pairwise_iou\nfrom detectron2.utils.file_io import PathManager\nfrom detectron2.utils.logger import create_small_table\n\nfrom detectron2.evaluation import DatasetEvaluator\nfrom iopath.common.file_io import file_lock\n\nlogger = logging.getLogger(__name__)\n\ndef convert_to_coco_json(dataset_name, output_file, allow_cached=True):\n \"\"\"\n Converts dataset into COCO format and saves it to a json file.\n dataset_name must be registered in DatasetCatalog and in detectron2's standard format.\n\n Args:\n dataset_name:\n reference from the config file to the catalogs\n must be registered in DatasetCatalog and in detectron2's standard format\n output_file: path of json file that will be saved to\n allow_cached: if json file is already present then skip conversion\n \"\"\"\n\n # TODO: The dataset or the conversion script *may* change,\n # a checksum would be useful for validating the cached data\n\n PathManager.mkdirs(os.path.dirname(output_file))\n with file_lock(output_file):\n if PathManager.exists(output_file) and allow_cached:\n logger.warning(\n f\"Using previously cached COCO format annotations at '{output_file}'. \"\n \"You need to clear the cache file if your dataset has been modified.\"\n )\n else:\n logger.info(f\"Converting annotations of dataset '{dataset_name}' to COCO format ...)\")\n coco_dict = convert_to_coco_dict(dataset_name)\n\n logger.info(f\"Caching COCO format annotations at '{output_file}' ...\")\n tmp_file = output_file #+ \".tmp\"\n # with PathManager.open(tmp_file, \"w\") as f:\n # json.dump(coco_dict, f)\n # shutil.move(tmp_file, output_file)\n with PathManager.open(tmp_file, \"w\") as f:\n json.dump(coco_dict, f)\n\nclass COCOEvaluator(DatasetEvaluator):\n \"\"\"\n Evaluate AR for object proposals, AP for instance detection/segmentation, AP\n for keypoint detection outputs using COCO's metrics.\n See http://cocodataset.org/#detection-eval and\n http://cocodataset.org/#keypoints-eval to understand its metrics.\n The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means\n the metric cannot be computed (e.g. due to no predictions made).\n\n In addition to COCO, this evaluator is able to support any bounding box detection,\n instance segmentation, or keypoint detection dataset.\n \"\"\"\n\n def __init__(\n self,\n dataset_name,\n tasks=None,\n distributed=True,\n output_dir=None,\n *,\n use_fast_impl=True,\n kpt_oks_sigmas=(),\n ):\n \"\"\"\n Args:\n dataset_name (str): name of the dataset to be evaluated.\n It must have either the following corresponding metadata:\n\n \"json_file\": the path to the COCO format annotation\n\n Or it must be in detectron2's standard dataset format\n so it can be converted to COCO format automatically.\n tasks (tuple[str]): tasks that can be evaluated under the given\n configuration. A task is one of \"bbox\", \"segm\", \"keypoints\".\n By default, will infer this automatically from predictions.\n distributed (True): if True, will collect results from all ranks and run evaluation\n in the main process.\n Otherwise, will only evaluate the results in the current process.\n output_dir (str): optional, an output directory to dump all\n results predicted on the dataset. The dump contains two files:\n\n 1. \"instances_predictions.pth\" a file that can be loaded with `torch.load` and\n contains all the results in the format they are produced by the model.\n 2. \"coco_instances_results.json\" a json file in COCO's result format.\n use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP.\n Although the results should be very close to the official implementation in COCO\n API, it is still recommended to compute results with the official API for use in\n papers. The faster implementation also uses more RAM.\n kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS.\n See http://cocodataset.org/#keypoints-eval\n When empty, it will use the defaults in COCO.\n Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS.\n \"\"\"\n self._logger = logging.getLogger(__name__)\n self._distributed = distributed\n self._output_dir = output_dir\n self._use_fast_impl = use_fast_impl\n\n if tasks is not None and isinstance(tasks, CfgNode):\n kpt_oks_sigmas = (\n tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas\n )\n self._logger.warn(\n \"COCO Evaluator instantiated using config, this is deprecated behavior.\"\n \" Please pass in explicit arguments instead.\"\n )\n self._tasks = None # Infering it from predictions should be better\n else:\n self._tasks = tasks\n\n self._cpu_device = torch.device(\"cpu\")\n\n self._metadata = MetadataCatalog.get(dataset_name)\n if not hasattr(self._metadata, \"json_file\"):\n self._logger.info(\n f\"'{dataset_name}' is not registered by `register_coco_instances`.\"\n \" Therefore trying to convert it to COCO format ...\"\n )\n\n cache_path = os.path.join(output_dir, f\"{dataset_name}_coco_format.json\")\n self._metadata.json_file = cache_path\n convert_to_coco_json(dataset_name, cache_path)\n\n json_file = PathManager.get_local_path(self._metadata.json_file)\n with contextlib.redirect_stdout(io.StringIO()):\n self._coco_api = COCO(json_file)\n\n # Test set json files do not contain annotations (evaluation must be\n # performed using the COCO evaluation server).\n self._do_evaluation = \"annotations\" in self._coco_api.dataset\n if self._do_evaluation:\n self._kpt_oks_sigmas = kpt_oks_sigmas\n\n def reset(self):\n self._predictions = []\n\n def process(self, inputs, outputs):\n \"\"\"\n Args:\n inputs: the inputs to a COCO model (e.g., GeneralizedRCNN).\n It is a list of dict. Each dict corresponds to an image and\n contains keys like \"height\", \"width\", \"file_name\", \"image_id\".\n outputs: the outputs of a COCO model. It is a list of dicts with key\n \"instances\" that contains :class:`Instances`.\n \"\"\"\n for input, output in zip(inputs, outputs):\n prediction = {\"image_id\": input[\"image_id\"]}\n\n if \"instances\" in output:\n instances = output[\"instances\"].to(self._cpu_device)\n prediction[\"instances\"] = instances_to_coco_json(instances, input[\"image_id\"])\n if \"proposals\" in output:\n prediction[\"proposals\"] = output[\"proposals\"].to(self._cpu_device)\n if len(prediction) > 1:\n self._predictions.append(prediction)\n\n def evaluate(self, img_ids=None):\n \"\"\"\n Args:\n img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset\n \"\"\"\n if self._distributed:\n comm.synchronize()\n predictions = comm.gather(self._predictions, dst=0)\n predictions = list(itertools.chain(*predictions))\n\n if not comm.is_main_process():\n return {}\n else:\n predictions = self._predictions\n\n if len(predictions) == 0:\n self._logger.warning(\"[COCOEvaluator] Did not receive valid predictions.\")\n return {}\n\n if self._output_dir:\n PathManager.mkdirs(self._output_dir)\n file_path = os.path.join(self._output_dir, \"instances_predictions.pth\")\n with PathManager.open(file_path, \"wb\") as f:\n torch.save(predictions, f)\n\n self._results = OrderedDict()\n if \"proposals\" in predictions[0]:\n self._eval_box_proposals(predictions)\n if \"instances\" in predictions[0]:\n self._eval_predictions(predictions, img_ids=img_ids)\n # Copy so the caller can do whatever with results\n return copy.deepcopy(self._results)\n\n def _tasks_from_predictions(self, predictions):\n \"\"\"\n Get COCO API \"tasks\" (i.e. iou_type) from COCO-format predictions.\n \"\"\"\n tasks = {\"bbox\"}\n for pred in predictions:\n if \"segmentation\" in pred:\n tasks.add(\"segm\")\n if \"keypoints\" in pred:\n tasks.add(\"keypoints\")\n return sorted(tasks)\n\n def _eval_predictions(self, predictions, img_ids=None):\n \"\"\"\n Evaluate predictions. Fill self._results with the metrics of the tasks.\n \"\"\"\n self._logger.info(\"Preparing results for COCO format ...\")\n coco_results = list(itertools.chain(*[x[\"instances\"] for x in predictions]))\n tasks = self._tasks or self._tasks_from_predictions(coco_results)\n\n # unmap the category ids for COCO\n if hasattr(self._metadata, \"thing_dataset_id_to_contiguous_id\"):\n dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id\n all_contiguous_ids = list(dataset_id_to_contiguous_id.values())\n num_classes = len(all_contiguous_ids)\n assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1\n\n reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()}\n for result in coco_results:\n category_id = result[\"category_id\"]\n assert category_id < num_classes, (\n f\"A prediction has class={category_id}, \"\n f\"but the dataset only has {num_classes} classes and \"\n f\"predicted class id should be in [0, {num_classes - 1}].\"\n )\n result[\"category_id\"] = reverse_id_mapping[category_id]\n\n if self._output_dir:\n file_path = os.path.join(self._output_dir, \"coco_instances_results.json\")\n self._logger.info(\"Saving results to {}\".format(file_path))\n with PathManager.open(file_path, \"w\") as f:\n f.write(json.dumps(coco_results))\n f.flush()\n\n if not self._do_evaluation:\n self._logger.info(\"Annotations are not available for evaluation.\")\n return\n\n self._logger.info(\n \"Evaluating predictions with {} COCO API...\".format(\n \"unofficial\" if self._use_fast_impl else \"official\"\n )\n )\n for task in sorted(tasks):\n assert task in {\"bbox\", \"segm\", \"keypoints\"}, f\"Got unknown task: {task}!\"\n coco_eval = (\n _evaluate_predictions_on_coco(\n self._coco_api,\n coco_results,\n task,\n kpt_oks_sigmas=self._kpt_oks_sigmas,\n use_fast_impl=self._use_fast_impl,\n img_ids=img_ids,\n )\n if len(coco_results) > 0\n else None # cocoapi does not handle empty results very well\n )\n\n res = self._derive_coco_results(\n coco_eval, task, class_names=self._metadata.get(\"thing_classes\")\n )\n self._results[task] = res\n\n def _eval_box_proposals(self, predictions):\n \"\"\"\n Evaluate the box proposals in predictions.\n Fill self._results with the metrics for \"box_proposals\" task.\n \"\"\"\n if self._output_dir:\n # Saving generated box proposals to file.\n # Predicted box_proposals are in XYXY_ABS mode.\n bbox_mode = BoxMode.XYXY_ABS.value\n ids, boxes, objectness_logits = [], [], []\n for prediction in predictions:\n ids.append(prediction[\"image_id\"])\n boxes.append(prediction[\"proposals\"].proposal_boxes.tensor.numpy())\n objectness_logits.append(prediction[\"proposals\"].objectness_logits.numpy())\n\n proposal_data = {\n \"boxes\": boxes,\n \"objectness_logits\": objectness_logits,\n \"ids\": ids,\n \"bbox_mode\": bbox_mode,\n }\n with PathManager.open(os.path.join(self._output_dir, \"box_proposals.pkl\"), \"wb\") as f:\n pickle.dump(proposal_data, f)\n\n if not self._do_evaluation:\n self._logger.info(\"Annotations are not available for evaluation.\")\n return\n\n self._logger.info(\"Evaluating bbox proposals ...\")\n res = {}\n areas = {\"all\": \"\", \"small\": \"s\", \"medium\": \"m\", \"large\": \"l\"}\n for limit in [100, 1000]:\n for area, suffix in areas.items():\n stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit)\n key = \"AR{}@{:d}\".format(suffix, limit)\n res[key] = float(stats[\"ar\"].item() * 100)\n self._logger.info(\"Proposal metrics: \\n\" + create_small_table(res))\n self._results[\"box_proposals\"] = res\n\n def _derive_coco_results(self, coco_eval, iou_type, class_names=None):\n \"\"\"\n Derive the desired score numbers from summarized COCOeval.\n\n Args:\n coco_eval (None or COCOEval): None represents no predictions from model.\n iou_type (str):\n class_names (None or list[str]): if provided, will use it to predict\n per-category AP.\n\n Returns:\n a dict of {metric name: score}\n \"\"\"\n\n metrics = {\n \"bbox\": [\"AP\", \"AP50\", \"AP75\", \"APs\", \"APm\", \"APl\"],\n \"segm\": [\"AP\", \"AP50\", \"AP75\", \"APs\", \"APm\", \"APl\"],\n \"keypoints\": [\"AP\", \"AP50\", \"AP75\", \"APm\", \"APl\"],\n }[iou_type]\n\n if coco_eval is None:\n self._logger.warn(\"No predictions from the model!\")\n return {metric: float(\"nan\") for metric in metrics}\n\n # the standard metrics\n results = {\n metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else \"nan\")\n for idx, metric in enumerate(metrics)\n }\n self._logger.info(\n \"Evaluation results for {}: \\n\".format(iou_type) + create_small_table(results)\n )\n if not np.isfinite(sum(results.values())):\n self._logger.info(\"Some metrics cannot be computed and is shown as NaN.\")\n\n if class_names is None or len(class_names) <= 1:\n return results\n # Compute per-category AP\n # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa\n precisions = coco_eval.eval[\"precision\"]\n # precision has dims (iou, recall, cls, area range, max dets)\n assert len(class_names) == precisions.shape[2]\n\n results_per_category = []\n for idx, name in enumerate(class_names):\n # area range index 0: all area ranges\n # max dets index -1: typically 100 per image\n precision = precisions[:, :, idx, 0, -1]\n precision = precision[precision > -1]\n ap = np.mean(precision) if precision.size else float(\"nan\")\n results_per_category.append((\"{}\".format(name), float(ap * 100)))\n\n # tabulate it\n N_COLS = min(6, len(results_per_category) * 2)\n results_flatten = list(itertools.chain(*results_per_category))\n results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)])\n table = tabulate(\n results_2d,\n tablefmt=\"pipe\",\n floatfmt=\".3f\",\n headers=[\"category\", \"AP\"] * (N_COLS // 2),\n numalign=\"left\",\n )\n self._logger.info(\"Per-category {} AP: \\n\".format(iou_type) + table)\n # results.update({\"AP-\" + name: ap for name, ap in results_per_category})\n\n results_per_category_AP50 = []\n for idx, name in enumerate(class_names):\n # area range index 0: all area ranges\n # max dets index -1: typically 100 per image\n\n t = np.where(.5 == coco_eval.params.iouThrs)[0]\n precisions_50 = precisions[t]\n precisions_50 = precisions_50[:, :, idx, 0, -1]\n precisions_50 = precisions_50[precisions_50 > -1]\n ap = np.mean(precisions_50) if precisions_50.size else float(\"nan\")\n results_per_category_AP50.append((\"{}\".format(name), float(ap * 100)))\n\n # tabulate it\n N_COLS = min(6, len(results_per_category_AP50) * 2)\n results_flatten = list(itertools.chain(*results_per_category_AP50))\n results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)])\n table = tabulate(\n results_2d,\n tablefmt=\"pipe\",\n floatfmt=\".3f\",\n headers=[\"category\", \"AP50\"] * (N_COLS // 2),\n numalign=\"left\",\n )\n self._logger.info(\"Per-category {} AP50: \\n\".format(iou_type) + table)\n results.update({\"AP50-\" + name: ap for name, ap in results_per_category_AP50})\n\n return results\n\n\ndef instances_to_coco_json(instances, img_id):\n \"\"\"\n Dump an \"Instances\" object to a COCO-format json that's used for evaluation.\n\n Args:\n instances (Instances):\n img_id (int): the image id\n\n Returns:\n list[dict]: list of json annotations in COCO format.\n \"\"\"\n num_instance = len(instances)\n if num_instance == 0:\n return []\n\n boxes = instances.pred_boxes.tensor.numpy()\n boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS)\n boxes = boxes.tolist()\n scores = instances.scores.tolist()\n classes = instances.pred_classes.tolist()\n\n has_mask = instances.has(\"pred_masks\")\n if has_mask:\n # use RLE to encode the masks, because they are too large and takes memory\n # since this evaluator stores outputs of the entire dataset\n rles = [\n mask_util.encode(np.array(mask[:, :, None], order=\"F\", dtype=\"uint8\"))[0]\n for mask in instances.pred_masks\n ]\n for rle in rles:\n # \"counts\" is an array encoded by mask_util as a byte-stream. Python3's\n # json writer which always produces strings cannot serialize a bytestream\n # unless you decode it. Thankfully, utf-8 works out (which is also what\n # the pycocotools/_mask.pyx does).\n rle[\"counts\"] = rle[\"counts\"].decode(\"utf-8\")\n\n has_keypoints = instances.has(\"pred_keypoints\")\n if has_keypoints:\n keypoints = instances.pred_keypoints\n\n results = []\n for k in range(num_instance):\n result = {\n \"image_id\": img_id,\n \"category_id\": classes[k],\n \"bbox\": boxes[k],\n \"score\": scores[k],\n }\n if has_mask:\n result[\"segmentation\"] = rles[k]\n if has_keypoints:\n # In COCO annotations,\n # keypoints coordinates are pixel indices.\n # However our predictions are floating point coordinates.\n # Therefore we subtract 0.5 to be consistent with the annotation format.\n # This is the inverse of data loading logic in `datasets/coco.py`.\n keypoints[k][:, :2] -= 0.5\n result[\"keypoints\"] = keypoints[k].flatten().tolist()\n results.append(result)\n return results\n\n\n# inspired from Detectron:\n# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa\ndef _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area=\"all\", limit=None):\n \"\"\"\n Evaluate detection proposal recall metrics. This function is a much\n faster alternative to the official COCO API recall evaluation code. However,\n it produces slightly different results.\n \"\"\"\n # Record max overlap value for each gt box\n # Return vector of overlap values\n areas = {\n \"all\": 0,\n \"small\": 1,\n \"medium\": 2,\n \"large\": 3,\n \"96-128\": 4,\n \"128-256\": 5,\n \"256-512\": 6,\n \"512-inf\": 7,\n }\n area_ranges = [\n [0 ** 2, 1e5 ** 2], # all\n [0 ** 2, 32 ** 2], # small\n [32 ** 2, 96 ** 2], # medium\n [96 ** 2, 1e5 ** 2], # large\n [96 ** 2, 128 ** 2], # 96-128\n [128 ** 2, 256 ** 2], # 128-256\n [256 ** 2, 512 ** 2], # 256-512\n [512 ** 2, 1e5 ** 2],\n ] # 512-inf\n assert area in areas, \"Unknown area range: {}\".format(area)\n area_range = area_ranges[areas[area]]\n gt_overlaps = []\n num_pos = 0\n\n for prediction_dict in dataset_predictions:\n predictions = prediction_dict[\"proposals\"]\n\n # sort predictions in descending order\n # TODO maybe remove this and make it explicit in the documentation\n inds = predictions.objectness_logits.sort(descending=True)[1]\n predictions = predictions[inds]\n\n ann_ids = coco_api.getAnnIds(imgIds=prediction_dict[\"image_id\"])\n anno = coco_api.loadAnns(ann_ids)\n gt_boxes = [\n BoxMode.convert(obj[\"bbox\"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS)\n for obj in anno\n if obj[\"iscrowd\"] == 0\n ]\n gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes\n gt_boxes = Boxes(gt_boxes)\n gt_areas = torch.as_tensor([obj[\"area\"] for obj in anno if obj[\"iscrowd\"] == 0])\n\n if len(gt_boxes) == 0 or len(predictions) == 0:\n continue\n\n valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1])\n gt_boxes = gt_boxes[valid_gt_inds]\n\n num_pos += len(gt_boxes)\n\n if len(gt_boxes) == 0:\n continue\n\n if limit is not None and len(predictions) > limit:\n predictions = predictions[:limit]\n\n overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes)\n\n _gt_overlaps = torch.zeros(len(gt_boxes))\n for j in range(min(len(predictions), len(gt_boxes))):\n # find which proposal box maximally covers each gt box\n # and get the iou amount of coverage for each gt box\n max_overlaps, argmax_overlaps = overlaps.max(dim=0)\n\n # find which gt box is 'best' covered (i.e. 'best' = most iou)\n gt_ovr, gt_ind = max_overlaps.max(dim=0)\n assert gt_ovr >= 0\n # find the proposal box that covers the best covered gt box\n box_ind = argmax_overlaps[gt_ind]\n # record the iou coverage of this gt box\n _gt_overlaps[j] = overlaps[box_ind, gt_ind]\n assert _gt_overlaps[j] == gt_ovr\n # mark the proposal box and the gt box as used\n overlaps[box_ind, :] = -1\n overlaps[:, gt_ind] = -1\n\n # append recorded iou coverage level\n gt_overlaps.append(_gt_overlaps)\n gt_overlaps = (\n torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32)\n )\n gt_overlaps, _ = torch.sort(gt_overlaps)\n\n if thresholds is None:\n step = 0.05\n thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32)\n recalls = torch.zeros_like(thresholds)\n # compute recall for each iou threshold\n for i, t in enumerate(thresholds):\n recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos)\n # ar = 2 * np.trapz(recalls, thresholds)\n ar = recalls.mean()\n return {\n \"ar\": ar,\n \"recalls\": recalls,\n \"thresholds\": thresholds,\n \"gt_overlaps\": gt_overlaps,\n \"num_pos\": num_pos,\n }\n\n\ndef _evaluate_predictions_on_coco(\n coco_gt, coco_results, iou_type, kpt_oks_sigmas=None, use_fast_impl=True, img_ids=None\n):\n \"\"\"\n Evaluate the coco results using COCOEval API.\n \"\"\"\n assert len(coco_results) > 0\n\n if iou_type == \"segm\":\n coco_results = copy.deepcopy(coco_results)\n # When evaluating mask AP, if the results contain bbox, cocoapi will\n # use the box area as the area of the instance, instead of the mask area.\n # This leads to a different definition of small/medium/large.\n # We remove the bbox field to let mask AP use mask area.\n for c in coco_results:\n c.pop(\"bbox\", None)\n\n coco_dt = coco_gt.loadRes(coco_results)\n coco_eval = (COCOeval_opt if use_fast_impl else COCOeval)(coco_gt, coco_dt, iou_type)\n if img_ids is not None:\n coco_eval.params.imgIds = img_ids\n\n if iou_type == \"keypoints\":\n # Use the COCO default keypoint OKS sigmas unless overrides are specified\n if kpt_oks_sigmas:\n assert hasattr(coco_eval.params, \"kpt_oks_sigmas\"), \"pycocotools is too old!\"\n coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas)\n # COCOAPI requires every detection and every gt to have keypoints, so\n # we just take the first entry from both\n num_keypoints_dt = len(coco_results[0][\"keypoints\"]) // 3\n num_keypoints_gt = len(next(iter(coco_gt.anns.values()))[\"keypoints\"]) // 3\n num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas)\n assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, (\n f\"[COCOEvaluator] Prediction contain {num_keypoints_dt} keypoints. \"\n f\"Ground truth contains {num_keypoints_gt} keypoints. \"\n f\"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. \"\n \"They have to agree with each other. For meaning of OKS, please refer to \"\n \"http://cocodataset.org/#keypoints-eval.\"\n )\n\n coco_eval.evaluate()\n coco_eval.accumulate()\n coco_eval.summarize()\n\n return coco_eval\n" }, { "path": "prod_lib/evaluation/pascal_voc_evaluation.py", "content": "# -*- coding: utf-8 -*-\n# Copyright (c) Facebook, Inc. and its affiliates.\n\nimport logging\nimport numpy as np\nimport os\nimport tempfile\nimport xml.etree.ElementTree as ET\nfrom collections import OrderedDict, defaultdict\nfrom functools import lru_cache\nimport torch\n\nfrom detectron2.data import MetadataCatalog\nfrom detectron2.utils import comm\nfrom detectron2.utils.file_io import PathManager\n\nfrom detectron2.evaluation import DatasetEvaluator\n\nclass PascalVOCDetectionEvaluator(DatasetEvaluator):\n \"\"\"\n Evaluate Pascal VOC style AP for Pascal VOC dataset.\n It contains a synchronization, therefore has to be called from all ranks.\n\n Note that the concept of AP can be implemented in different ways and may not\n produce identical results. This class mimics the implementation of the official\n Pascal VOC Matlab API, and should produce similar but not identical results to the\n official API.\n \"\"\"\n\n def __init__(self, dataset_name, target_classnames=None):\n \"\"\"\n Args:\n dataset_name (str): name of the dataset, e.g., \"voc_2007_test\"\n \"\"\"\n self._dataset_name = dataset_name\n meta = MetadataCatalog.get(dataset_name)\n\n # Too many tiny files, download all to local for speed.\n annotation_dir_local = PathManager.get_local_path(\n os.path.join(meta.dirname, \"Annotations/\")\n )\n self._anno_file_template = os.path.join(annotation_dir_local, \"{}.xml\")\n self._image_set_path = os.path.join(meta.dirname, \"ImageSets\", \"Main\", meta.split + \".txt\")\n self._class_names = meta.thing_classes\n assert meta.year in [2007, 2012], meta.year\n self._is_2007 = meta.year == 2007\n self._cpu_device = torch.device(\"cpu\")\n self._logger = logging.getLogger(__name__)\n\n if target_classnames == None:\n self.target_classnames = self._class_names\n else:\n self.target_classnames = target_classnames\n\n def reset(self):\n self._predictions = defaultdict(list) # class name -> list of prediction strings\n\n def process(self, inputs, outputs):\n for input, output in zip(inputs, outputs):\n image_id = input[\"image_id\"]\n instances = output[\"instances\"].to(self._cpu_device)\n boxes = instances.pred_boxes.tensor.numpy()\n scores = instances.scores.tolist()\n classes = instances.pred_classes.tolist()\n for box, score, cls in zip(boxes, scores, classes):\n xmin, ymin, xmax, ymax = box\n # The inverse of data loading logic in `datasets/pascal_voc.py`\n xmin += 1\n ymin += 1\n self._predictions[cls].append(\n f\"{image_id} {score:.3f} {xmin:.1f} {ymin:.1f} {xmax:.1f} {ymax:.1f}\"\n )\n\n def evaluate(self):\n \"\"\"\n Returns:\n dict: has a key \"segm\", whose value is a dict of \"AP\", \"AP50\", and \"AP75\".\n \"\"\"\n all_predictions = comm.gather(self._predictions, dst=0)\n if not comm.is_main_process():\n return\n predictions = defaultdict(list)\n for predictions_per_rank in all_predictions:\n for clsid, lines in predictions_per_rank.items():\n predictions[clsid].extend(lines)\n del all_predictions\n\n self._logger.info(\n \"Evaluating {} using {} metric. \"\n \"Note that results do not use the official Matlab API.\".format(\n self._dataset_name, 2007 if self._is_2007 else 2012\n )\n )\n\n with tempfile.TemporaryDirectory(prefix=\"pascal_voc_eval_\") as dirname:\n res_file_template = os.path.join(dirname, \"{}.txt\")\n\n aps = defaultdict(list) # iou -> ap per class\n for cls_id, cls_name in enumerate(self._class_names):\n if cls_name not in self.target_classnames:\n continue\n lines = predictions.get(cls_id, [\"\"])\n\n with open(res_file_template.format(cls_name), \"w\") as f:\n f.write(\"\\n\".join(lines))\n\n for thresh in range(50, 100, 5):\n rec, prec, ap = voc_eval(\n res_file_template,\n self._anno_file_template,\n self._image_set_path,\n cls_name,\n ovthresh=thresh / 100.0,\n use_07_metric=self._is_2007,\n )\n aps[thresh].append(ap * 100)\n\n ret = OrderedDict()\n mAP = {iou: np.mean(x) for iou, x in aps.items()}\n ret[\"bbox\"] = {\"AP\": np.mean(list(mAP.values())), \"AP50\": mAP[50], \"AP75\": mAP[75]}\n\n #Add the codes for AP50\n for idx, name in enumerate(self.target_classnames):\n ret[\"bbox\"].update({\"AP50-\" + name: aps[50][idx]})\n\n return ret\n\n\n##############################################################################\n#\n# Below code is modified from\n# https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/datasets/voc_eval.py\n# --------------------------------------------------------\n# Fast/er R-CNN\n# Licensed under The MIT License [see LICENSE for details]\n# Written by Bharath Hariharan\n# --------------------------------------------------------\n\n\"\"\"Python implementation of the PASCAL VOC devkit's AP evaluation code.\"\"\"\n\n\n@lru_cache(maxsize=None)\ndef parse_rec(filename):\n \"\"\"Parse a PASCAL VOC xml file.\"\"\"\n with PathManager.open(filename) as f:\n tree = ET.parse(f)\n objects = []\n for obj in tree.findall(\"object\"):\n obj_struct = {}\n obj_struct[\"name\"] = obj.find(\"name\").text\n obj_struct[\"pose\"] = obj.find(\"pose\").text\n obj_struct[\"truncated\"] = int(obj.find(\"truncated\").text)\n obj_struct[\"difficult\"] = int(obj.find(\"difficult\").text)\n bbox = obj.find(\"bndbox\")\n obj_struct[\"bbox\"] = [\n int(bbox.find(\"xmin\").text),\n int(bbox.find(\"ymin\").text),\n int(bbox.find(\"xmax\").text),\n int(bbox.find(\"ymax\").text),\n ]\n objects.append(obj_struct)\n\n return objects\n\n\ndef voc_ap(rec, prec, use_07_metric=False):\n \"\"\"Compute VOC AP given precision and recall. If use_07_metric is true, uses\n the VOC 07 11-point method (default:False).\n \"\"\"\n if use_07_metric:\n # 11 point metric\n ap = 0.0\n for t in np.arange(0.0, 1.1, 0.1):\n if np.sum(rec >= t) == 0:\n p = 0\n else:\n p = np.max(prec[rec >= t])\n ap = ap + p / 11.0\n else:\n # correct AP calculation\n # first append sentinel values at the end\n mrec = np.concatenate(([0.0], rec, [1.0]))\n mpre = np.concatenate(([0.0], prec, [0.0]))\n\n # compute the precision envelope\n for i in range(mpre.size - 1, 0, -1):\n mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])\n\n # to calculate area under PR curve, look for points\n # where X axis (recall) changes value\n i = np.where(mrec[1:] != mrec[:-1])[0]\n\n # and sum (\\Delta recall) * prec\n ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])\n return ap\n\n\ndef voc_eval(detpath, annopath, imagesetfile, classname, ovthresh=0.5, use_07_metric=False):\n \"\"\"rec, prec, ap = voc_eval(detpath,\n annopath,\n imagesetfile,\n classname,\n [ovthresh],\n [use_07_metric])\n\n Top level function that does the PASCAL VOC evaluation.\n\n detpath: Path to detections\n detpath.format(classname) should produce the detection results file.\n annopath: Path to annotations\n annopath.format(imagename) should be the xml annotations file.\n imagesetfile: Text file containing the list of images, one image per line.\n classname: Category name (duh)\n [ovthresh]: Overlap threshold (default = 0.5)\n [use_07_metric]: Whether to use VOC07's 11 point AP computation\n (default False)\n \"\"\"\n # assumes detections are in detpath.format(classname)\n # assumes annotations are in annopath.format(imagename)\n # assumes imagesetfile is a text file with each line an image name\n\n # first load gt\n # read list of images\n with PathManager.open(imagesetfile, \"r\") as f:\n lines = f.readlines()\n imagenames = [x.strip() for x in lines]\n\n # load annots\n recs = {}\n for imagename in imagenames:\n recs[imagename] = parse_rec(annopath.format(imagename))\n\n # extract gt objects for this class\n class_recs = {}\n npos = 0\n for imagename in imagenames:\n R = [obj for obj in recs[imagename] if obj[\"name\"] == classname]\n bbox = np.array([x[\"bbox\"] for x in R])\n difficult = np.array([x[\"difficult\"] for x in R]).astype(np.bool)\n # difficult = np.array([False for x in R]).astype(np.bool) # treat all \"difficult\" as GT\n det = [False] * len(R)\n npos = npos + sum(~difficult)\n class_recs[imagename] = {\"bbox\": bbox, \"difficult\": difficult, \"det\": det}\n\n # read dets\n detfile = detpath.format(classname)\n with open(detfile, \"r\") as f:\n lines = f.readlines()\n\n splitlines = [x.strip().split(\" \") for x in lines]\n image_ids = [x[0] for x in splitlines]\n confidence = np.array([float(x[1]) for x in splitlines])\n BB = np.array([[float(z) for z in x[2:]] for x in splitlines]).reshape(-1, 4)\n\n # sort by confidence\n sorted_ind = np.argsort(-confidence)\n BB = BB[sorted_ind, :]\n image_ids = [image_ids[x] for x in sorted_ind]\n\n # go down dets and mark TPs and FPs\n nd = len(image_ids)\n tp = np.zeros(nd)\n fp = np.zeros(nd)\n for d in range(nd):\n R = class_recs[image_ids[d]]\n bb = BB[d, :].astype(float)\n ovmax = -np.inf\n BBGT = R[\"bbox\"].astype(float)\n\n if BBGT.size > 0:\n # compute overlaps\n # intersection\n ixmin = np.maximum(BBGT[:, 0], bb[0])\n iymin = np.maximum(BBGT[:, 1], bb[1])\n ixmax = np.minimum(BBGT[:, 2], bb[2])\n iymax = np.minimum(BBGT[:, 3], bb[3])\n iw = np.maximum(ixmax - ixmin + 1.0, 0.0)\n ih = np.maximum(iymax - iymin + 1.0, 0.0)\n inters = iw * ih\n\n # union\n uni = (\n (bb[2] - bb[0] + 1.0) * (bb[3] - bb[1] + 1.0)\n + (BBGT[:, 2] - BBGT[:, 0] + 1.0) * (BBGT[:, 3] - BBGT[:, 1] + 1.0)\n - inters\n )\n\n overlaps = inters / uni\n ovmax = np.max(overlaps)\n jmax = np.argmax(overlaps)\n\n if ovmax > ovthresh:\n if not R[\"difficult\"][jmax]:\n if not R[\"det\"][jmax]:\n tp[d] = 1.0\n R[\"det\"][jmax] = 1\n else:\n fp[d] = 1.0\n else:\n fp[d] = 1.0\n\n # compute precision recall\n fp = np.cumsum(fp)\n tp = np.cumsum(tp)\n rec = tp / float(npos)\n # avoid divide by zero in case the first detection matches a difficult\n # ground truth\n prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)\n ap = voc_ap(rec, prec, use_07_metric)\n\n return rec, prec, ap\n" }, { "path": "prod_lib/modeling/daobj_rcnn.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom detectron2.data.detection_utils import convert_image_to_rgb\nfrom detectron2.modeling import META_ARCH_REGISTRY, GeneralizedRCNN\nfrom detectron2.utils.events import get_event_storage\n\nimport logging\nfrom typing import Dict, Tuple, List, Optional\nfrom collections import OrderedDict\n\nfrom detectron2.config import configurable\n# from detectron2.modeling.meta_arch.build import META_ARCH_REGISTRY\n# from detectron2.modeling.meta_arch.rcnn import GeneralizedRCNN\nfrom detectron2.modeling.proposal_generator import build_proposal_generator\nfrom detectron2.modeling.backbone import build_backbone, Backbone\nfrom detectron2.modeling.roi_heads import build_roi_heads\nfrom detectron2.utils.events import get_event_storage\nfrom detectron2.structures import ImageList\n\n############### Image discriminator ##############\nclass FCDiscriminator_img(nn.Module):\n def __init__(self, num_classes, ndf1=256, ndf2=128):\n super(FCDiscriminator_img, self).__init__()\n\n self.conv1 = nn.Conv2d(num_classes, ndf1, kernel_size=3, padding=1)\n self.conv2 = nn.Conv2d(ndf1, ndf2, kernel_size=3, padding=1)\n self.conv3 = nn.Conv2d(ndf2, ndf2, kernel_size=3, padding=1)\n self.classifier = nn.Conv2d(ndf2, 1, kernel_size=3, padding=1)\n\n self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.leaky_relu(x)\n x = self.conv2(x)\n x = self.leaky_relu(x)\n x = self.conv3(x)\n x = self.leaky_relu(x)\n x = self.classifier(x)\n return x\n#################################\n\n################ Gradient reverse function\nclass GradReverse(torch.autograd.Function):\n @staticmethod\n def forward(ctx, x):\n return x.view_as(x)\n\n @staticmethod\n def backward(ctx, grad_output):\n return grad_output.neg()\n\ndef grad_reverse(x):\n return GradReverse.apply(x)\n\n#######################\n\n@META_ARCH_REGISTRY.register()\nclass DAobjTwoStagePseudoLabGeneralizedRCNN(GeneralizedRCNN):\n\n @configurable\n def __init__(\n self,\n *,\n backbone: Backbone,\n proposal_generator: nn.Module,\n roi_heads: nn.Module,\n pixel_mean: Tuple[float],\n pixel_std: Tuple[float],\n input_format: Optional[str] = None,\n vis_period: int = 0,\n dis_type: str,\n # dis_loss_weight: float = 0,\n ):\n \"\"\"\n Args:\n backbone: a backbone module, must follow detectron2's backbone interface\n proposal_generator: a module that generates proposals using backbone features\n roi_heads: a ROI head that performs per-region computation\n pixel_mean, pixel_std: list or tuple with #channels element, representing\n the per-channel mean and std to be used to normalize the input image\n input_format: describe the meaning of channels of input. Needed by visualization\n vis_period: the period to run visualization. Set to 0 to disable.\n \"\"\"\n super(GeneralizedRCNN, self).__init__()\n self.backbone = backbone\n self.proposal_generator = proposal_generator\n self.roi_heads = roi_heads\n\n self.input_format = input_format\n self.vis_period = vis_period\n if vis_period > 0:\n assert input_format is not None, \"input_format is required for visualization!\"\n\n self.register_buffer(\"pixel_mean\", torch.tensor(pixel_mean).view(-1, 1, 1), False)\n self.register_buffer(\"pixel_std\", torch.tensor(pixel_std).view(-1, 1, 1), False)\n assert (\n self.pixel_mean.shape == self.pixel_std.shape\n ), f\"{self.pixel_mean} and {self.pixel_std} have different shapes!\"\n # @yujheli: you may need to build your discriminator here\n\n self.dis_type = dis_type\n # self.D_img = FCDiscriminator_img(self.backbone._out_feature_channels['res4']) # Need to know the channel\n if self.dis_type == \"multi\":\n self.D_img_dict = {}\n for k,v in self.backbone._out_feature_channels.items():\n self.D_img_dict[k] = FCDiscriminator_img(v)\n self.add_module(\"D_\"+k, self.D_img_dict[k])\n else:\n self.D_img = FCDiscriminator_img(self.backbone._out_feature_channels[self.dis_type]) # Need to know the channel\n # self.bceLoss_func = nn.BCEWithLogitsLoss()\n\n @classmethod\n def from_config(cls, cfg):\n backbone = build_backbone(cfg)\n return {\n \"backbone\": backbone,\n \"proposal_generator\": build_proposal_generator(cfg, backbone.output_shape()),\n \"roi_heads\": build_roi_heads(cfg, backbone.output_shape()),\n \"input_format\": cfg.INPUT.FORMAT,\n \"vis_period\": cfg.VIS_PERIOD,\n \"pixel_mean\": cfg.MODEL.PIXEL_MEAN,\n \"pixel_std\": cfg.MODEL.PIXEL_STD,\n \"dis_type\": cfg.UNBIASEDTEACHER.DIS_TYPE,\n # \"dis_loss_ratio\": cfg.xxx,\n }\n\n def preprocess_image_train(self, batched_inputs: List[Dict[str, torch.Tensor]]):\n \"\"\"\n Normalize, pad and batch the input images.\n \"\"\"\n images = [x[\"image\"].to(self.device) for x in batched_inputs]\n images = [(x - self.pixel_mean) / self.pixel_std for x in images]\n images = ImageList.from_tensors(images, self.backbone.size_divisibility)\n\n images_t = [x[\"image_unlabeled\"].to(self.device) for x in batched_inputs]\n images_t = [(x - self.pixel_mean) / self.pixel_std for x in images_t]\n images_t = ImageList.from_tensors(images_t, self.backbone.size_divisibility)\n\n return images, images_t\n\n def forward(\n self, batched_inputs, branch=\"supervised\", given_proposals=None, val_mode=False\n ):\n \"\"\"\n Args:\n batched_inputs: a list, batched outputs of :class:`DatasetMapper` .\n Each item in the list contains the inputs for one image.\n For now, each item in the list is a dict that contains:\n\n * image: Tensor, image in (C, H, W) format.\n * instances (optional): groundtruth :class:`Instances`\n * proposals (optional): :class:`Instances`, precomputed proposals.\n\n Other information that's included in the original dicts, such as:\n\n * \"height\", \"width\" (int): the output resolution of the model, used in inference.\n See :meth:`postprocess` for details.\n\n Returns:\n list[dict]:\n Each dict is the output for one input image.\n The dict contains one key \"instances\" whose value is a :class:`Instances`.\n The :class:`Instances` object has the following keys:\n \"pred_boxes\", \"pred_classes\", \"scores\", \"pred_masks\", \"pred_keypoints\"\n \"\"\"\n\n if (not self.training) and (not val_mode): # only conduct when testing mode\n return self.inference(batched_inputs)\n\n\n if branch == \"domain\":\n source_label = 0\n target_label = 1\n # images = self.preprocess_image(batched_inputs)\n images_s, images_t = self.preprocess_image_train(batched_inputs)\n\n features = self.backbone(images_s.tensor)\n\n # import pdb\n # pdb.set_trace()\n if self.dis_type == \"multi\":\n loss_D_img_s = 0\n for k, v in features.items():\n features_s = grad_reverse(v)\n D_img_out_s = self.D_img_dict[k](features_s)\n loss_D_img_s += F.binary_cross_entropy_with_logits(D_img_out_s, torch.FloatTensor(D_img_out_s.data.size()).fill_(source_label).to(self.device))\n loss_D_img_s /= len(features)\n # features_s = grad_reverse(torch.cat((features['p2'],features['p3'],features['p4'],features['p5']),dim=1))\n else:\n features_s = grad_reverse(features[self.dis_type])\n D_img_out_s = self.D_img(features_s)\n loss_D_img_s = F.binary_cross_entropy_with_logits(D_img_out_s, torch.FloatTensor(D_img_out_s.data.size()).fill_(source_label).to(self.device))\n\n features_t = self.backbone(images_t.tensor)\n if self.dis_type == \"multi\":\n loss_D_img_t = 0\n for k, v in features_t.items():\n features_tt = grad_reverse(v)\n D_img_out_t = self.D_img_dict[k](features_tt)\n loss_D_img_t += F.binary_cross_entropy_with_logits(D_img_out_t, torch.FloatTensor(D_img_out_t.data.size()).fill_(target_label).to(self.device))\n loss_D_img_t /= len(features_t)\n else:\n features_t = grad_reverse(features_t[self.dis_type])\n # features_t = grad_reverse(features_t['p2'])\n D_img_out_t = self.D_img(features_t)\n loss_D_img_t = F.binary_cross_entropy_with_logits(D_img_out_t, torch.FloatTensor(D_img_out_t.data.size()).fill_(target_label).to(self.device))\n\n # import pdb\n # pdb.set_trace()\n\n losses = {}\n losses[\"loss_D_img_s\"] = loss_D_img_s\n losses[\"loss_D_img_t\"] = loss_D_img_t\n return losses, [], [], None\n\n\n images = self.preprocess_image(batched_inputs)\n\n if \"instances\" in batched_inputs[0]:\n gt_instances = [x[\"instances\"].to(self.device) for x in batched_inputs]\n else:\n gt_instances = None\n\n features = self.backbone(images.tensor)\n\n # TODO: remove the usage of if else here. This needs to be re-organized\n if branch.startswith(\"supervised\"):\n # Region proposal network\n proposals_rpn, proposal_losses = self.proposal_generator(\n images, features, gt_instances\n )\n\n # roi_head lower branch\n _, detector_losses = self.roi_heads(\n images,\n features,\n proposals_rpn,\n compute_loss=True,\n targets=gt_instances,\n branch=branch,\n )\n\n # visualization\n if self.vis_period > 0:\n storage = get_event_storage()\n if storage.iter % self.vis_period == 0:\n self.visualize_training(batched_inputs, proposals_rpn, branch)\n\n losses = {}\n losses.update(detector_losses)\n losses.update(proposal_losses)\n return losses, [], [], None\n\n elif branch == \"unsup_data_weak\":\n \"\"\"\n unsupervised weak branch: input image without any ground-truth label; output proposals of rpn and roi-head\n \"\"\"\n # Region proposal network\n proposals_rpn, _ = self.proposal_generator(\n images, features, None, compute_loss=False\n )\n\n # roi_head lower branch (keep this for further production)\n # notice that we do not use any target in ROI head to do inference!\n proposals_roih, ROI_predictions = self.roi_heads(\n images,\n features,\n proposals_rpn,\n targets=None,\n compute_loss=False,\n branch=branch,\n )\n\n # if self.vis_period > 0:\n # storage = get_event_storage()\n # if storage.iter % self.vis_period == 0:\n # self.visualize_training(batched_inputs, proposals_rpn, branch)\n\n return {}, proposals_rpn, proposals_roih, ROI_predictions\n elif branch == \"unsup_data_strong\":\n raise NotImplementedError()\n elif branch == \"val_loss\":\n raise NotImplementedError()\n\n def visualize_training(self, batched_inputs, proposals, branch=\"\"):\n \"\"\"\n This function different from the original one:\n - it adds \"branch\" to the `vis_name`.\n\n A function used to visualize images and proposals. It shows ground truth\n bounding boxes on the original image and up to 20 predicted object\n proposals on the original image. Users can implement different\n visualization functions for different models.\n\n Args:\n batched_inputs (list): a list that contains input to the model.\n proposals (list): a list that contains predicted proposals. Both\n batched_inputs and proposals should have the same length.\n \"\"\"\n from detectron2.utils.visualizer import Visualizer\n\n storage = get_event_storage()\n max_vis_prop = 20\n\n for input, prop in zip(batched_inputs, proposals):\n img = input[\"image\"]\n img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format)\n v_gt = Visualizer(img, None)\n v_gt = v_gt.overlay_instances(boxes=input[\"instances\"].gt_boxes)\n anno_img = v_gt.get_image()\n box_size = min(len(prop.proposal_boxes), max_vis_prop)\n v_pred = Visualizer(img, None)\n v_pred = v_pred.overlay_instances(\n boxes=prop.proposal_boxes[0:box_size].tensor.cpu().numpy()\n )\n prop_img = v_pred.get_image()\n vis_img = np.concatenate((anno_img, prop_img), axis=1)\n vis_img = vis_img.transpose(2, 0, 1)\n vis_name = (\n \"Left: GT bounding boxes \"\n + branch\n + \"; Right: Predicted proposals \"\n + branch\n )\n storage.put_image(vis_name, vis_img)\n break # only visualize one image in a batch\n" }, { "path": "prod_lib/modeling/vgg.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\nimport torch.nn as nn\nimport copy\nimport torch\nfrom typing import Union, List, Dict, Any, cast\nfrom detectron2.modeling.backbone import (\n ResNet,\n Backbone,\n build_resnet_backbone,\n BACKBONE_REGISTRY\n)\nfrom detectron2.modeling.backbone.fpn import FPN, LastLevelMaxPool, LastLevelP6P7\n\n\n\ndef make_layers(cfg: List[Union[str, int]], batch_norm: bool = False) -> nn.Sequential:\n layers: List[nn.Module] = []\n in_channels = 3\n for v in cfg:\n if v == 'M':\n layers += [nn.MaxPool2d(kernel_size=2, stride=2)]\n else:\n v = cast(int, v)\n conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)\n if batch_norm:\n layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]\n else:\n layers += [conv2d, nn.ReLU(inplace=True)]\n in_channels = v\n return nn.Sequential(*layers)\n\ncfgs: Dict[str, List[Union[str, int]]] = {\n 'vgg11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],\n 'vgg13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],\n 'vgg16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],\n 'vgg19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'],\n}\n\n\nclass vgg_backbone(Backbone):\n \"\"\"\n Backbone (bottom-up) for FBNet.\n\n Hierarchy:\n trunk0:\n xif0_0\n xif0_1\n ...\n trunk1:\n xif1_0\n xif1_1\n ...\n ...\n\n Output features:\n The outputs from each \"stage\", i.e. trunkX.\n \"\"\"\n\n def __init__(self, cfg):\n super().__init__()\n\n self.vgg = make_layers(cfgs['vgg16'],batch_norm=True)\n\n self._initialize_weights()\n # self.stage_names_index = {'vgg1':3, 'vgg2':8 , 'vgg3':15, 'vgg4':22, 'vgg5':29}\n _out_feature_channels = [64, 128, 256, 512, 512]\n _out_feature_strides = [2, 4, 8, 16, 32]\n # stages, shape_specs = build_fbnet(\n # cfg,\n # name=\"trunk\",\n # in_channels=cfg.MODEL.FBNET_V2.STEM_IN_CHANNELS\n # )\n\n # nn.Sequential(*list(self.vgg.features._modules.values())[:14])\n\n self.stages = [nn.Sequential(*list(self.vgg._modules.values())[0:7]),\\\n nn.Sequential(*list(self.vgg._modules.values())[7:14]),\\\n nn.Sequential(*list(self.vgg._modules.values())[14:24]),\\\n nn.Sequential(*list(self.vgg._modules.values())[24:34]),\\\n nn.Sequential(*list(self.vgg._modules.values())[34:]),]\n self._out_feature_channels = {}\n self._out_feature_strides = {}\n self._stage_names = []\n\n for i, stage in enumerate(self.stages):\n name = \"vgg{}\".format(i)\n self.add_module(name, stage)\n self._stage_names.append(name)\n self._out_feature_channels[name] = _out_feature_channels[i]\n self._out_feature_strides[name] = _out_feature_strides[i]\n\n self._out_features = self._stage_names\n\n del self.vgg\n\n def forward(self, x):\n features = {}\n for name, stage in zip(self._stage_names, self.stages):\n x = stage(x)\n # if name in self._out_features:\n # outputs[name] = x\n features[name] = x\n # import pdb\n # pdb.set_trace()\n\n return features\n\n def _initialize_weights(self) -> None:\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n nn.init.constant_(m.bias, 0)\n\n\n@BACKBONE_REGISTRY.register() #already register in baseline model\ndef build_vgg_backbone(cfg, _):\n return vgg_backbone(cfg)\n\n\n@BACKBONE_REGISTRY.register() #already register in baseline model\ndef build_vgg_fpn_backbone(cfg, _):\n # backbone = FPN(\n # bottom_up=build_vgg_backbone(cfg),\n # in_features=cfg.MODEL.FPN.IN_FEATURES,\n # out_channels=cfg.MODEL.FPN.OUT_CHANNELS,\n # norm=cfg.MODEL.FPN.NORM,\n # top_block=LastLevelMaxPool(),\n # )\n\n bottom_up = vgg_backbone(cfg)\n in_features = cfg.MODEL.FPN.IN_FEATURES\n out_channels = cfg.MODEL.FPN.OUT_CHANNELS\n backbone = FPN(\n bottom_up=bottom_up,\n in_features=in_features,\n out_channels=out_channels,\n norm=cfg.MODEL.FPN.NORM,\n top_block=LastLevelMaxPool(),\n # fuse_type=cfg.MODEL.FPN.FUSE_TYPE,\n )\n # return backbone\n\n return backbone\n" }, { "path": "prod_lib/runner/__init__.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\n# from .runner import SemiSupSegRunner, SemiSupHandTrackingRunner # noqa\nfrom .runner import BaseUnbiasedTeacherRunner # noqa\nfrom .runner import DAobjUnbiasedTeacherRunner # noqa\n" }, { "path": "prod_lib/runner/runner.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\n\nimport logging\nimport os\nfrom collections import OrderedDict\nfrom functools import lru_cache\n\nimport d2go.utils.abnormal_checker as abnormal_checker\nimport detectron2.utils.comm as comm\nfrom d2go.config import CONFIG_SCALING_METHOD_REGISTRY, temp_defrost\nfrom d2go.data.dataset_mappers import D2GoDatasetMapper, build_dataset_mapper\nfrom d2go.data.transforms.build import build_transform_gen\nfrom d2go.data.utils import maybe_subsample_n_images\nfrom d2go.modeling import build_model, kmeans_anchors, model_ema\nfrom d2go.runner import GeneralizedRCNNRunner\nfrom d2go.utils.flop_calculator import add_print_flops_callback\nfrom d2go.utils.misc import get_tensorboard_log_dir\nfrom d2go.utils.helper import TensorboardXWriter, D2Trainer\nfrom detectron2.checkpoint import PeriodicCheckpointer\nfrom detectron2.engine import hooks\nfrom detectron2.utils.events import CommonMetricPrinter, JSONWriter, TensorboardXWriter\nfrom torch.nn.parallel import DataParallel, DistributedDataParallel\nfrom detectron2.evaluation import (\n DatasetEvaluators,\n)\nfrom detectron2.data import (\n MetadataCatalog,\n)\nfrom ..evaluation import (\n COCOEvaluator,\n PascalVOCDetectionEvaluator,\n)\nfrom d2go.projects.unbiased_teacher.checkpoint import EnsembleTSModel\nfrom ..config.defaults import add_aut_config\n# from ..config.defaults import add_ut_config\n# from ..data.build import (\n# build_detection_semisup_train_loader_two_crops,\n# build_uru_detection_semisup_train_loader,\n# inject_uru_dataset,\n# )\nfrom d2go.projects.unbiased_teacher.data.build import (\n build_detection_semisup_train_loader_two_crops,\n build_uru_detection_semisup_train_loader,\n)\nfrom d2go.projects.unbiased_teacher.runner.runner import UnbiasedTeacherRunner\nfrom d2go.projects.unbiased_teacher.data.dataset_mapper import DatasetMapperTwoCropSeparate # noqa\nfrom ..data import builtin # noqa; for registering COCO unlabel dataset\nfrom d2go.projects.unbiased_teacher.engine.trainer import UnbiasedTeacherTrainer\nfrom d2go.projects.unbiased_teacher.modeling.meta_arch.rcnn import TwoStagePseudoLabGeneralizedRCNN # noqa\nfrom d2go.projects.unbiased_teacher.modeling.proposal_generator.rpn import PseudoLabRPN # noqa\nfrom d2go.projects.unbiased_teacher.modeling.roi_heads.roi_heads import StandardROIHeadsPseudoLab # noqa\nfrom d2go.projects.unbiased_teacher.solver.build import ut_build_lr_scheduler\n\n#For DA object detection\nfrom ..engine.trainer import DAobjTrainer\nfrom ..modeling.meta_arch.daobj_rcnn import DAobjTwoStagePseudoLabGeneralizedRCNN # noqa\n#For VGG model architecture\nfrom ..modeling.meta_arch.vgg import build_vgg_backbone,build_vgg_fpn_backbone # noqa\n\nALL_TB_WRITERS = []\n\n\n@lru_cache()\ndef _get_tbx_writer(log_dir):\n ret = TensorboardXWriter(log_dir)\n ALL_TB_WRITERS.append(ret)\n return ret\n\nclass BaseUnbiasedTeacherRunner(UnbiasedTeacherRunner):\n def get_default_cfg(self):\n cfg = super().get_default_cfg()\n add_aut_config(cfg)\n\n # add_pointrend_config(cfg)\n # cfg = CN(cfg) # upgrade from D2's CfgNode to D2Go's CfgNode\n return cfg\n\n @staticmethod\n def get_evaluator(cfg, dataset_name, output_folder):\n evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type\n if evaluator_type in [\"coco\"]:\n # D2 is in the process of reducing the use of cfg.\n dataset_evaluators = COCOEvaluator(\n dataset_name,\n output_dir=output_folder,\n kpt_oks_sigmas=cfg.TEST.KEYPOINT_OKS_SIGMAS,\n )\n elif evaluator_type in [\"pascal_voc\"]:\n dataset_evaluators = PascalVOCDetectionEvaluator(dataset_name)\n elif evaluator_type in [\"pascal_voc_water\"]:\n dataset_evaluators = PascalVOCDetectionEvaluator(dataset_name, target_classnames=[\"bicycle\", \"bird\", \"car\", \"cat\", \"dog\", \"person\"])\n else:\n dataset_evaluators = D2Trainer.build_evaluator(\n cfg, dataset_name, output_folder\n )\n if not isinstance(dataset_evaluators, DatasetEvaluators):\n dataset_evaluators = DatasetEvaluators([dataset_evaluators])\n return dataset_evaluators\n\n# class DAobjUnbiasedTeacherRunner(UnbiasedTeacherRunner):\nclass DAobjUnbiasedTeacherRunner(BaseUnbiasedTeacherRunner):\n def get_default_cfg(self):\n cfg = super().get_default_cfg()\n\n # add_aut_config(cfg)\n # add_pointrend_config(cfg)\n # cfg = CN(cfg) # upgrade from D2's CfgNode to D2Go's CfgNode\n return cfg\n\n def build_model(self, cfg, eval_only=False):\n \"\"\"\n Build both Student and Teacher models\n\n Student: regular model\n Teacher: model that is updated by EMA\n \"\"\"\n # build_model might modify the cfg, thus clone\n cfg = cfg.clone()\n\n model = build_model(cfg)\n model_teacher = build_model(cfg)\n\n if cfg.MODEL.FROZEN_LAYER_REG_EXP:\n raise NotImplementedError()\n\n if cfg.QUANTIZATION.QAT.ENABLED:\n raise NotImplementedError()\n\n if eval_only:\n raise NotImplementedError()\n\n return EnsembleTSModel(model_teacher, model)\n\n\n def do_train(self, cfg, model, resume):\n\n # NOTE: d2go's train_net applies DDP layer by default\n # we need to strip it away and only put DDP on model_student\n if isinstance(model, (DistributedDataParallel, DataParallel)):\n model = model.module\n\n model_teacher, model_student = model.model_teacher, model.model_student\n\n if comm.get_world_size() > 1:\n model_student = DistributedDataParallel(\n model_student,\n device_ids=None\n if cfg.MODEL.DEVICE == \"cpu\"\n else [comm.get_local_rank()],\n broadcast_buffers=False,\n find_unused_parameters=cfg.MODEL.DDP_FIND_UNUSED_PARAMETERS,\n )\n\n add_print_flops_callback(cfg, model_student, disable_after_callback=True)\n\n optimizer = self.build_optimizer(cfg, model_student)\n scheduler = self.build_lr_scheduler(cfg, optimizer)\n\n checkpointer = self.build_checkpointer(\n cfg,\n model,\n save_dir=cfg.OUTPUT_DIR,\n optimizer=optimizer,\n scheduler=scheduler,\n )\n\n checkpoint = checkpointer.resume_or_load(\n cfg.MODEL.WEIGHTS, resume=resume or cfg.UNBIASEDTEACHER.RESUME_FROM_ANOTHER\n )\n start_iter = (\n checkpoint.get(\"iteration\", -1)\n if resume\n and checkpointer.has_checkpoint()\n or cfg.UNBIASEDTEACHER.RESUME_FROM_ANOTHER\n else -1\n )\n # The checkpoint stores the training iteration that just finished, thus we start\n # at the next iteration (or iter zero if there's no checkpoint).\n start_iter += 1\n max_iter = cfg.SOLVER.MAX_ITER\n periodic_checkpointer = PeriodicCheckpointer(\n checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD, max_iter=max_iter\n )\n\n # if resume from a pre-trained checkpoint, we modify the BURN_IN_STEP\n # so that the weights of the Student will be copied to the Teacher\n # at the 1st iteration when the training started\n if cfg.UNBIASEDTEACHER.RESUME_FROM_ANOTHER:\n cfg.defrost()\n cfg.UNBIASEDTEACHER.BURN_IN_STEP = start_iter\n cfg.freeze()\n\n data_loader = self.build_detection_train_loader(cfg)\n\n def _get_model_with_abnormal_checker(model):\n if not cfg.ABNORMAL_CHECKER.ENABLED:\n return model\n\n tbx_writer = _get_tbx_writer(get_tensorboard_log_dir(cfg.OUTPUT_DIR))\n writers = abnormal_checker.get_writers(cfg, tbx_writer)\n checker = abnormal_checker.AbnormalLossChecker(start_iter, writers)\n ret = abnormal_checker.AbnormalLossCheckerWrapper(model, checker)\n return ret\n\n trainer = DAobjTrainer(\n cfg,\n _get_model_with_abnormal_checker(model_student),\n _get_model_with_abnormal_checker(model_teacher),\n data_loader,\n optimizer,\n )\n\n trainer_hooks = [\n hooks.IterationTimer(),\n self._create_after_step_hook(\n cfg, model_student, optimizer, scheduler, periodic_checkpointer\n ),\n hooks.EvalHook(\n cfg.TEST.EVAL_PERIOD,\n lambda: self.do_test(cfg, model, train_iter=trainer.iter),\n ),\n kmeans_anchors.compute_kmeans_anchors_hook(self, cfg),\n self._create_qat_hook(cfg) if cfg.QUANTIZATION.QAT.ENABLED else None,\n ]\n\n if comm.is_main_process():\n tbx_writer = _get_tbx_writer(get_tensorboard_log_dir(cfg.OUTPUT_DIR))\n writers = [\n CommonMetricPrinter(max_iter),\n JSONWriter(os.path.join(cfg.OUTPUT_DIR, \"metrics.json\")),\n tbx_writer,\n ]\n trainer_hooks.append(\n hooks.PeriodicWriter(writers, period=cfg.WRITER_PERIOD)\n )\n trainer.register_hooks(trainer_hooks)\n trainer.train(start_iter, max_iter)\n\n trained_cfg = cfg.clone()\n with temp_defrost(trained_cfg):\n trained_cfg.MODEL.WEIGHTS = checkpointer.get_checkpoint_file()\n return {\"model_final\": trained_cfg}\n" }, { "path": "train_net.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\nimport detectron2.utils.comm as comm\nfrom detectron2.checkpoint import DetectionCheckpointer\nfrom detectron2.config import get_cfg\nfrom detectron2.engine import default_argument_parser, default_setup, launch\n\nfrom adapteacher import add_ateacher_config\nfrom adapteacher.engine.trainer import ATeacherTrainer, BaselineTrainer\n\n# hacky way to register\nfrom adapteacher.modeling.meta_arch.rcnn import TwoStagePseudoLabGeneralizedRCNN, DAobjTwoStagePseudoLabGeneralizedRCNN\nfrom adapteacher.modeling.meta_arch.vgg import build_vgg_backbone # noqa\nfrom adapteacher.modeling.proposal_generator.rpn import PseudoLabRPN\nfrom adapteacher.modeling.roi_heads.roi_heads import StandardROIHeadsPseudoLab\nimport adapteacher.data.datasets.builtin\n\nfrom adapteacher.modeling.meta_arch.ts_ensemble import EnsembleTSModel\n\n\ndef setup(args):\n \"\"\"\n Create configs and perform basic setups.\n \"\"\"\n cfg = get_cfg()\n add_ateacher_config(cfg)\n cfg.merge_from_file(args.config_file)\n cfg.merge_from_list(args.opts)\n cfg.freeze()\n default_setup(cfg, args)\n return cfg\n\n\ndef main(args):\n cfg = setup(args)\n if cfg.SEMISUPNET.Trainer == \"ateacher\":\n Trainer = ATeacherTrainer\n elif cfg.SEMISUPNET.Trainer == \"baseline\":\n Trainer = BaselineTrainer\n else:\n raise ValueError(\"Trainer Name is not found.\")\n\n if args.eval_only:\n if cfg.SEMISUPNET.Trainer == \"ateacher\":\n model = Trainer.build_model(cfg)\n model_teacher = Trainer.build_model(cfg)\n ensem_ts_model = EnsembleTSModel(model_teacher, model)\n\n DetectionCheckpointer(\n ensem_ts_model, save_dir=cfg.OUTPUT_DIR\n ).resume_or_load(cfg.MODEL.WEIGHTS, resume=args.resume)\n res = Trainer.test(cfg, ensem_ts_model.modelTeacher)\n\n else:\n model = Trainer.build_model(cfg)\n DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(\n cfg.MODEL.WEIGHTS, resume=args.resume\n )\n res = Trainer.test(cfg, model)\n return res\n\n trainer = Trainer(cfg)\n trainer.resume_or_load(resume=args.resume)\n\n return trainer.train()\n\n\nif __name__ == \"__main__\":\n args = default_argument_parser().parse_args()\n\n print(\"Command Line Args:\", args)\n launch(\n main,\n args.num_gpus,\n num_machines=args.num_machines,\n machine_rank=args.machine_rank,\n dist_url=args.dist_url,\n args=(args,),\n )\n" } ]