[ { "path": ".dockerignore", "content": ".env/\nresults/\n**__pycache__**\n*.onnx\n*.pt\n*.mlmodel\n**byte_track_results**\n**deep_sort_results**\n**nor_fair_results**\ntest_env/" }, { "path": ".gitignore", "content": ".env/\n**__pycache__**\n*.onnx\n*.pt\n*.pth\n*.mlmodel\n**byte_track_results**\n**deep_sort_results**\n**nor_fair_results**\nbuild/\ndist/\nasone.egg-info/\ntest_custom_tracker.py\ncustom_weights.py\ndata/results/\ntemp/" }, { "path": "Dockerfile", "content": "FROM pytorch/pytorch:latest\n\n# Set Time Zone to prevent issues for installing some apt packages\nENV TZ=Europe/Minsk\nRUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone\n\n# install apt packages\nRUN apt-get update -y \nRUN apt-get install git gcc \\\n g++ python3-opencv \\\n vim -y\n\nRUN mkdir /app\nWORKDIR /app \n\nADD asone asone\n\nADD sample_videos sample_videos\nADD main.py main.py\n# ADD demo.py demo.py\n\nADD setup.py setup.py\nADD requirements.txt requirements.txt\n\n\nRUN pip3 install Cython numpy\nRUN pip3 install cython-bbox\nADD pypi_README.md pypi_README.md\n\nRUN pip3 install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113\nRUN pip3 install .\n\n\nWORKDIR /workspace\n# Entry Point\nCMD /bin/bash\n" }, { "path": "LICENCE", "content": " GNU GENERAL PUBLIC LICENSE\n Version 3, 29 June 2007\n\n Copyright (C) 2007 Free Software Foundation, Inc. \n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n Preamble\n\n The GNU General Public License is a free, copyleft license for\nsoftware and other kinds of works.\n\n The licenses for most software and other practical works are designed\nto take away your freedom to share and change the works. By contrast,\nthe GNU General Public License is intended to guarantee your freedom to\nshare and change all versions of a program--to make sure it remains free\nsoftware for all its users. We, the Free Software Foundation, use the\nGNU General Public License for most of our software; it applies also to\nany other work released this way by its authors. You can apply it to\nyour programs, too.\n\n When we speak of free software, we are referring to freedom, not\nprice. Our General Public Licenses are designed to make sure that you\nhave the freedom to distribute copies of free software (and charge for\nthem if you wish), that you receive source code or can get it if you\nwant it, that you can change the software or use pieces of it in new\nfree programs, and that you know you can do these things.\n\n To protect your rights, we need to prevent others from denying you\nthese rights or asking you to surrender the rights. Therefore, you have\ncertain responsibilities if you distribute copies of the software, or if\nyou modify it: responsibilities to respect the freedom of others.\n\n For example, if you distribute copies of such a program, whether\ngratis or for a fee, you must pass on to the recipients the same\nfreedoms that you received. You must make sure that they, too, receive\nor can get the source code. And you must show them these terms so they\nknow their rights.\n\n Developers that use the GNU GPL protect your rights with two steps:\n(1) assert copyright on the software, and (2) offer you this License\ngiving you legal permission to copy, distribute and/or modify it.\n\n For the developers' and authors' protection, the GPL clearly explains\nthat there is no warranty for this free software. For both users' and\nauthors' sake, the GPL requires that modified versions be marked as\nchanged, so that their problems will not be attributed erroneously to\nauthors of previous versions.\n\n Some devices are designed to deny users access to install or run\nmodified versions of the software inside them, although the manufacturer\ncan do so. This is fundamentally incompatible with the aim of\nprotecting users' freedom to change the software. The systematic\npattern of such abuse occurs in the area of products for individuals to\nuse, which is precisely where it is most unacceptable. Therefore, we\nhave designed this version of the GPL to prohibit the practice for those\nproducts. If such problems arise substantially in other domains, we\nstand ready to extend this provision to those domains in future versions\nof the GPL, as needed to protect the freedom of users.\n\n Finally, every program is threatened constantly by software patents.\nStates should not allow patents to restrict development and use of\nsoftware on general-purpose computers, but in those that do, we wish to\navoid the special danger that patents applied to a free program could\nmake it effectively proprietary. To prevent this, the GPL assures that\npatents cannot be used to render the program non-free.\n\n The precise terms and conditions for copying, distribution and\nmodification follow.\n\n TERMS AND CONDITIONS\n\n 0. Definitions.\n\n \"This License\" refers to version 3 of the GNU General Public License.\n\n \"Copyright\" also means copyright-like laws that apply to other kinds of\nworks, such as semiconductor masks.\n\n \"The Program\" refers to any copyrightable work licensed under this\nLicense. Each licensee is addressed as \"you\". \"Licensees\" and\n\"recipients\" may be individuals or organizations.\n\n To \"modify\" a work means to copy from or adapt all or part of the work\nin a fashion requiring copyright permission, other than the making of an\nexact copy. The resulting work is called a \"modified version\" of the\nearlier work or a work \"based on\" the earlier work.\n\n A \"covered work\" means either the unmodified Program or a work based\non the Program.\n\n To \"propagate\" a work means to do anything with it that, without\npermission, would make you directly or secondarily liable for\ninfringement under applicable copyright law, except executing it on a\ncomputer or modifying a private copy. Propagation includes copying,\ndistribution (with or without modification), making available to the\npublic, and in some countries other activities as well.\n\n To \"convey\" a work means any kind of propagation that enables other\nparties to make or receive copies. Mere interaction with a user through\na computer network, with no transfer of a copy, is not conveying.\n\n An interactive user interface displays \"Appropriate Legal Notices\"\nto the extent that it includes a convenient and prominently visible\nfeature that (1) displays an appropriate copyright notice, and (2)\ntells the user that there is no warranty for the work (except to the\nextent that warranties are provided), that licensees may convey the\nwork under this License, and how to view a copy of this License. If\nthe interface presents a list of user commands or options, such as a\nmenu, a prominent item in the list meets this criterion.\n\n 1. Source Code.\n\n The \"source code\" for a work means the preferred form of the work\nfor making modifications to it. \"Object code\" means any non-source\nform of a work.\n\n A \"Standard Interface\" means an interface that either is an official\nstandard defined by a recognized standards body, or, in the case of\ninterfaces specified for a particular programming language, one that\nis widely used among developers working in that language.\n\n The \"System Libraries\" of an executable work include anything, other\nthan the work as a whole, that (a) is included in the normal form of\npackaging a Major Component, but which is not part of that Major\nComponent, and (b) serves only to enable use of the work with that\nMajor Component, or to implement a Standard Interface for which an\nimplementation is available to the public in source code form. A\n\"Major Component\", in this context, means a major essential component\n(kernel, window system, and so on) of the specific operating system\n(if any) on which the executable work runs, or a compiler used to\nproduce the work, or an object code interpreter used to run it.\n\n The \"Corresponding Source\" for a work in object code form means all\nthe source code needed to generate, install, and (for an executable\nwork) run the object code and to modify the work, including scripts to\ncontrol those activities. However, it does not include the work's\nSystem Libraries, or general-purpose tools or generally available free\nprograms which are used unmodified in performing those activities but\nwhich are not part of the work. For example, Corresponding Source\nincludes interface definition files associated with source files for\nthe work, and the source code for shared libraries and dynamically\nlinked subprograms that the work is specifically designed to require,\nsuch as by intimate data communication or control flow between those\nsubprograms and other parts of the work.\n\n The Corresponding Source need not include anything that users\ncan regenerate automatically from other parts of the Corresponding\nSource.\n\n The Corresponding Source for a work in source code form is that\nsame work.\n\n 2. Basic Permissions.\n\n All rights granted under this License are granted for the term of\ncopyright on the Program, and are irrevocable provided the stated\nconditions are met. This License explicitly affirms your unlimited\npermission to run the unmodified Program. The output from running a\ncovered work is covered by this License only if the output, given its\ncontent, constitutes a covered work. This License acknowledges your\nrights of fair use or other equivalent, as provided by copyright law.\n\n You may make, run and propagate covered works that you do not\nconvey, without conditions so long as your license otherwise remains\nin force. You may convey covered works to others for the sole purpose\nof having them make modifications exclusively for you, or provide you\nwith facilities for running those works, provided that you comply with\nthe terms of this License in conveying all material for which you do\nnot control copyright. Those thus making or running the covered works\nfor you must do so exclusively on your behalf, under your direction\nand control, on terms that prohibit them from making any copies of\nyour copyrighted material outside their relationship with you.\n\n Conveying under any other circumstances is permitted solely under\nthe conditions stated below. Sublicensing is not allowed; section 10\nmakes it unnecessary.\n\n 3. Protecting Users' Legal Rights From Anti-Circumvention Law.\n\n No covered work shall be deemed part of an effective technological\nmeasure under any applicable law fulfilling obligations under article\n11 of the WIPO copyright treaty adopted on 20 December 1996, or\nsimilar laws prohibiting or restricting circumvention of such\nmeasures.\n\n When you convey a covered work, you waive any legal power to forbid\ncircumvention of technological measures to the extent such circumvention\nis effected by exercising rights under this License with respect to\nthe covered work, and you disclaim any intention to limit operation or\nmodification of the work as a means of enforcing, against the work's\nusers, your or third parties' legal rights to forbid circumvention of\ntechnological measures.\n\n 4. Conveying Verbatim Copies.\n\n You may convey verbatim copies of the Program's source code as you\nreceive it, in any medium, provided that you conspicuously and\nappropriately publish on each copy an appropriate copyright notice;\nkeep intact all notices stating that this License and any\nnon-permissive terms added in accord with section 7 apply to the code;\nkeep intact all notices of the absence of any warranty; and give all\nrecipients a copy of this License along with the Program.\n\n You may charge any price or no price for each copy that you convey,\nand you may offer support or warranty protection for a fee.\n\n 5. Conveying Modified Source Versions.\n\n You may convey a work based on the Program, or the modifications to\nproduce it from the Program, in the form of source code under the\nterms of section 4, provided that you also meet all of these conditions:\n\n a) The work must carry prominent notices stating that you modified\n it, and giving a relevant date.\n\n b) The work must carry prominent notices stating that it is\n released under this License and any conditions added under section\n 7. This requirement modifies the requirement in section 4 to\n \"keep intact all notices\".\n\n c) You must license the entire work, as a whole, under this\n License to anyone who comes into possession of a copy. This\n License will therefore apply, along with any applicable section 7\n additional terms, to the whole of the work, and all its parts,\n regardless of how they are packaged. This License gives no\n permission to license the work in any other way, but it does not\n invalidate such permission if you have separately received it.\n\n d) If the work has interactive user interfaces, each must display\n Appropriate Legal Notices; however, if the Program has interactive\n interfaces that do not display Appropriate Legal Notices, your\n work need not make them do so.\n\n A compilation of a covered work with other separate and independent\nworks, which are not by their nature extensions of the covered work,\nand which are not combined with it such as to form a larger program,\nin or on a volume of a storage or distribution medium, is called an\n\"aggregate\" if the compilation and its resulting copyright are not\nused to limit the access or legal rights of the compilation's users\nbeyond what the individual works permit. Inclusion of a covered work\nin an aggregate does not cause this License to apply to the other\nparts of the aggregate.\n\n 6. Conveying Non-Source Forms.\n\n You may convey a covered work in object code form under the terms\nof sections 4 and 5, provided that you also convey the\nmachine-readable Corresponding Source under the terms of this License,\nin one of these ways:\n\n a) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by the\n Corresponding Source fixed on a durable physical medium\n customarily used for software interchange.\n\n b) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by a\n written offer, valid for at least three years and valid for as\n long as you offer spare parts or customer support for that product\n model, to give anyone who possesses the object code either (1) a\n copy of the Corresponding Source for all the software in the\n product that is covered by this License, on a durable physical\n medium customarily used for software interchange, for a price no\n more than your reasonable cost of physically performing this\n conveying of source, or (2) access to copy the\n Corresponding Source from a network server at no charge.\n\n c) Convey individual copies of the object code with a copy of the\n written offer to provide the Corresponding Source. This\n alternative is allowed only occasionally and noncommercially, and\n only if you received the object code with such an offer, in accord\n with subsection 6b.\n\n d) Convey the object code by offering access from a designated\n place (gratis or for a charge), and offer equivalent access to the\n Corresponding Source in the same way through the same place at no\n further charge. You need not require recipients to copy the\n Corresponding Source along with the object code. If the place to\n copy the object code is a network server, the Corresponding Source\n may be on a different server (operated by you or a third party)\n that supports equivalent copying facilities, provided you maintain\n clear directions next to the object code saying where to find the\n Corresponding Source. Regardless of what server hosts the\n Corresponding Source, you remain obligated to ensure that it is\n available for as long as needed to satisfy these requirements.\n\n e) Convey the object code using peer-to-peer transmission, provided\n you inform other peers where the object code and Corresponding\n Source of the work are being offered to the general public at no\n charge under subsection 6d.\n\n A separable portion of the object code, whose source code is excluded\nfrom the Corresponding Source as a System Library, need not be\nincluded in conveying the object code work.\n\n A \"User Product\" is either (1) a \"consumer product\", which means any\ntangible personal property which is normally used for personal, family,\nor household purposes, or (2) anything designed or sold for incorporation\ninto a dwelling. In determining whether a product is a consumer product,\ndoubtful cases shall be resolved in favor of coverage. For a particular\nproduct received by a particular user, \"normally used\" refers to a\ntypical or common use of that class of product, regardless of the status\nof the particular user or of the way in which the particular user\nactually uses, or expects or is expected to use, the product. A product\nis a consumer product regardless of whether the product has substantial\ncommercial, industrial or non-consumer uses, unless such uses represent\nthe only significant mode of use of the product.\n\n \"Installation Information\" for a User Product means any methods,\nprocedures, authorization keys, or other information required to install\nand execute modified versions of a covered work in that User Product from\na modified version of its Corresponding Source. The information must\nsuffice to ensure that the continued functioning of the modified object\ncode is in no case prevented or interfered with solely because\nmodification has been made.\n\n If you convey an object code work under this section in, or with, or\nspecifically for use in, a User Product, and the conveying occurs as\npart of a transaction in which the right of possession and use of the\nUser Product is transferred to the recipient in perpetuity or for a\nfixed term (regardless of how the transaction is characterized), the\nCorresponding Source conveyed under this section must be accompanied\nby the Installation Information. But this requirement does not apply\nif neither you nor any third party retains the ability to install\nmodified object code on the User Product (for example, the work has\nbeen installed in ROM).\n\n The requirement to provide Installation Information does not include a\nrequirement to continue to provide support service, warranty, or updates\nfor a work that has been modified or installed by the recipient, or for\nthe User Product in which it has been modified or installed. Access to a\nnetwork may be denied when the modification itself materially and\nadversely affects the operation of the network or violates the rules and\nprotocols for communication across the network.\n\n Corresponding Source conveyed, and Installation Information provided,\nin accord with this section must be in a format that is publicly\ndocumented (and with an implementation available to the public in\nsource code form), and must require no special password or key for\nunpacking, reading or copying.\n\n 7. Additional Terms.\n\n \"Additional permissions\" are terms that supplement the terms of this\nLicense by making exceptions from one or more of its conditions.\nAdditional permissions that are applicable to the entire Program shall\nbe treated as though they were included in this License, to the extent\nthat they are valid under applicable law. If additional permissions\napply only to part of the Program, that part may be used separately\nunder those permissions, but the entire Program remains governed by\nthis License without regard to the additional permissions.\n\n When you convey a copy of a covered work, you may at your option\nremove any additional permissions from that copy, or from any part of\nit. (Additional permissions may be written to require their own\nremoval in certain cases when you modify the work.) You may place\nadditional permissions on material, added by you to a covered work,\nfor which you have or can give appropriate copyright permission.\n\n Notwithstanding any other provision of this License, for material you\nadd to a covered work, you may (if authorized by the copyright holders of\nthat material) supplement the terms of this License with terms:\n\n a) Disclaiming warranty or limiting liability differently from the\n terms of sections 15 and 16 of this License; or\n\n b) Requiring preservation of specified reasonable legal notices or\n author attributions in that material or in the Appropriate Legal\n Notices displayed by works containing it; or\n\n c) Prohibiting misrepresentation of the origin of that material, or\n requiring that modified versions of such material be marked in\n reasonable ways as different from the original version; or\n\n d) Limiting the use for publicity purposes of names of licensors or\n authors of the material; or\n\n e) Declining to grant rights under trademark law for use of some\n trade names, trademarks, or service marks; or\n\n f) Requiring indemnification of licensors and authors of that\n material by anyone who conveys the material (or modified versions of\n it) with contractual assumptions of liability to the recipient, for\n any liability that these contractual assumptions directly impose on\n those licensors and authors.\n\n All other non-permissive additional terms are considered \"further\nrestrictions\" within the meaning of section 10. If the Program as you\nreceived it, or any part of it, contains a notice stating that it is\ngoverned by this License along with a term that is a further\nrestriction, you may remove that term. If a license document contains\na further restriction but permits relicensing or conveying under this\nLicense, you may add to a covered work material governed by the terms\nof that license document, provided that the further restriction does\nnot survive such relicensing or conveying.\n\n If you add terms to a covered work in accord with this section, you\nmust place, in the relevant source files, a statement of the\nadditional terms that apply to those files, or a notice indicating\nwhere to find the applicable terms.\n\n Additional terms, permissive or non-permissive, may be stated in the\nform of a separately written license, or stated as exceptions;\nthe above requirements apply either way.\n\n 8. Termination.\n\n You may not propagate or modify a covered work except as expressly\nprovided under this License. Any attempt otherwise to propagate or\nmodify it is void, and will automatically terminate your rights under\nthis License (including any patent licenses granted under the third\nparagraph of section 11).\n\n However, if you cease all violation of this License, then your\nlicense from a particular copyright holder is reinstated (a)\nprovisionally, unless and until the copyright holder explicitly and\nfinally terminates your license, and (b) permanently, if the copyright\nholder fails to notify you of the violation by some reasonable means\nprior to 60 days after the cessation.\n\n Moreover, your license from a particular copyright holder is\nreinstated permanently if the copyright holder notifies you of the\nviolation by some reasonable means, this is the first time you have\nreceived notice of violation of this License (for any work) from that\ncopyright holder, and you cure the violation prior to 30 days after\nyour receipt of the notice.\n\n Termination of your rights under this section does not terminate the\nlicenses of parties who have received copies or rights from you under\nthis License. If your rights have been terminated and not permanently\nreinstated, you do not qualify to receive new licenses for the same\nmaterial under section 10.\n\n 9. Acceptance Not Required for Having Copies.\n\n You are not required to accept this License in order to receive or\nrun a copy of the Program. Ancillary propagation of a covered work\noccurring solely as a consequence of using peer-to-peer transmission\nto receive a copy likewise does not require acceptance. However,\nnothing other than this License grants you permission to propagate or\nmodify any covered work. These actions infringe copyright if you do\nnot accept this License. Therefore, by modifying or propagating a\ncovered work, you indicate your acceptance of this License to do so.\n\n 10. Automatic Licensing of Downstream Recipients.\n\n Each time you convey a covered work, the recipient automatically\nreceives a license from the original licensors, to run, modify and\npropagate that work, subject to this License. You are not responsible\nfor enforcing compliance by third parties with this License.\n\n An \"entity transaction\" is a transaction transferring control of an\norganization, or substantially all assets of one, or subdividing an\norganization, or merging organizations. If propagation of a covered\nwork results from an entity transaction, each party to that\ntransaction who receives a copy of the work also receives whatever\nlicenses to the work the party's predecessor in interest had or could\ngive under the previous paragraph, plus a right to possession of the\nCorresponding Source of the work from the predecessor in interest, if\nthe predecessor has it or can get it with reasonable efforts.\n\n You may not impose any further restrictions on the exercise of the\nrights granted or affirmed under this License. For example, you may\nnot impose a license fee, royalty, or other charge for exercise of\nrights granted under this License, and you may not initiate litigation\n(including a cross-claim or counterclaim in a lawsuit) alleging that\nany patent claim is infringed by making, using, selling, offering for\nsale, or importing the Program or any portion of it.\n\n 11. Patents.\n\n A \"contributor\" is a copyright holder who authorizes use under this\nLicense of the Program or a work on which the Program is based. The\nwork thus licensed is called the contributor's \"contributor version\".\n\n A contributor's \"essential patent claims\" are all patent claims\nowned or controlled by the contributor, whether already acquired or\nhereafter acquired, that would be infringed by some manner, permitted\nby this License, of making, using, or selling its contributor version,\nbut do not include claims that would be infringed only as a\nconsequence of further modification of the contributor version. For\npurposes of this definition, \"control\" includes the right to grant\npatent sublicenses in a manner consistent with the requirements of\nthis License.\n\n Each contributor grants you a non-exclusive, worldwide, royalty-free\npatent license under the contributor's essential patent claims, to\nmake, use, sell, offer for sale, import and otherwise run, modify and\npropagate the contents of its contributor version.\n\n In the following three paragraphs, a \"patent license\" is any express\nagreement or commitment, however denominated, not to enforce a patent\n(such as an express permission to practice a patent or covenant not to\nsue for patent infringement). To \"grant\" such a patent license to a\nparty means to make such an agreement or commitment not to enforce a\npatent against the party.\n\n If you convey a covered work, knowingly relying on a patent license,\nand the Corresponding Source of the work is not available for anyone\nto copy, free of charge and under the terms of this License, through a\npublicly available network server or other readily accessible means,\nthen you must either (1) cause the Corresponding Source to be so\navailable, or (2) arrange to deprive yourself of the benefit of the\npatent license for this particular work, or (3) arrange, in a manner\nconsistent with the requirements of this License, to extend the patent\nlicense to downstream recipients. \"Knowingly relying\" means you have\nactual knowledge that, but for the patent license, your conveying the\ncovered work in a country, or your recipient's use of the covered work\nin a country, would infringe one or more identifiable patents in that\ncountry that you have reason to believe are valid.\n\n If, pursuant to or in connection with a single transaction or\narrangement, you convey, or propagate by procuring conveyance of, a\ncovered work, and grant a patent license to some of the parties\nreceiving the covered work authorizing them to use, propagate, modify\nor convey a specific copy of the covered work, then the patent license\nyou grant is automatically extended to all recipients of the covered\nwork and works based on it.\n\n A patent license is \"discriminatory\" if it does not include within\nthe scope of its coverage, prohibits the exercise of, or is\nconditioned on the non-exercise of one or more of the rights that are\nspecifically granted under this License. You may not convey a covered\nwork if you are a party to an arrangement with a third party that is\nin the business of distributing software, under which you make payment\nto the third party based on the extent of your activity of conveying\nthe work, and under which the third party grants, to any of the\nparties who would receive the covered work from you, a discriminatory\npatent license (a) in connection with copies of the covered work\nconveyed by you (or copies made from those copies), or (b) primarily\nfor and in connection with specific products or compilations that\ncontain the covered work, unless you entered into that arrangement,\nor that patent license was granted, prior to 28 March 2007.\n\n Nothing in this License shall be construed as excluding or limiting\nany implied license or other defenses to infringement that may\notherwise be available to you under applicable patent law.\n\n 12. No Surrender of Others' Freedom.\n\n If conditions are imposed on you (whether by court order, agreement or\notherwise) that contradict the conditions of this License, they do not\nexcuse you from the conditions of this License. If you cannot convey a\ncovered work so as to satisfy simultaneously your obligations under this\nLicense and any other pertinent obligations, then as a consequence you may\nnot convey it at all. For example, if you agree to terms that obligate you\nto collect a royalty for further conveying from those to whom you convey\nthe Program, the only way you could satisfy both those terms and this\nLicense would be to refrain entirely from conveying the Program.\n\n 13. Use with the GNU Affero General Public License.\n\n Notwithstanding any other provision of this License, you have\npermission to link or combine any covered work with a work licensed\nunder version 3 of the GNU Affero General Public License into a single\ncombined work, and to convey the resulting work. The terms of this\nLicense will continue to apply to the part which is the covered work,\nbut the special requirements of the GNU Affero General Public License,\nsection 13, concerning interaction through a network will apply to the\ncombination as such.\n\n 14. Revised Versions of this License.\n\n The Free Software Foundation may publish revised and/or new versions of\nthe GNU General Public License from time to time. Such new versions will\nbe similar in spirit to the present version, but may differ in detail to\naddress new problems or concerns.\n\n Each version is given a distinguishing version number. If the\nProgram specifies that a certain numbered version of the GNU General\nPublic License \"or any later version\" applies to it, you have the\noption of following the terms and conditions either of that numbered\nversion or of any later version published by the Free Software\nFoundation. If the Program does not specify a version number of the\nGNU General Public License, you may choose any version ever published\nby the Free Software Foundation.\n\n If the Program specifies that a proxy can decide which future\nversions of the GNU General Public License can be used, that proxy's\npublic statement of acceptance of a version permanently authorizes you\nto choose that version for the Program.\n\n Later license versions may give you additional or different\npermissions. However, no additional obligations are imposed on any\nauthor or copyright holder as a result of your choosing to follow a\nlater version.\n\n 15. Disclaimer of Warranty.\n\n THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY\nAPPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT\nHOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY\nOF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM\nIS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF\nALL NECESSARY SERVICING, REPAIR OR CORRECTION.\n\n 16. Limitation of Liability.\n\n IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\nWILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS\nTHE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY\nGENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE\nUSE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF\nDATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD\nPARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),\nEVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF\nSUCH DAMAGES.\n\n 17. Interpretation of Sections 15 and 16.\n\n If the disclaimer of warranty and limitation of liability provided\nabove cannot be given local legal effect according to their terms,\nreviewing courts shall apply local law that most closely approximates\nan absolute waiver of all civil liability in connection with the\nProgram, unless a warranty or assumption of liability accompanies a\ncopy of the Program in return for a fee.\n\n END OF TERMS AND CONDITIONS\n\n How to Apply These Terms to Your New Programs\n\n If you develop a new program, and you want it to be of the greatest\npossible use to the public, the best way to achieve this is to make it\nfree software which everyone can redistribute and change under these terms.\n\n To do so, attach the following notices to the program. It is safest\nto attach them to the start of each source file to most effectively\nstate the exclusion of warranty; and each file should have at least\nthe \"copyright\" line and a pointer to where the full notice is found.\n\n \n Copyright (C) \n\n This program is free software: you can redistribute it and/or modify\n it under the terms of the GNU General Public License as published by\n the Free Software Foundation, either version 3 of the License, or\n (at your option) any later version.\n\n This program is distributed in the hope that it will be useful,\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n GNU General Public License for more details.\n\n You should have received a copy of the GNU General Public License\n along with this program. If not, see .\n\nAlso add information on how to contact you by electronic and paper mail.\n\n If the program does terminal interaction, make it output a short\nnotice like this when it starts in an interactive mode:\n\n Copyright (C) \n This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.\n This is free software, and you are welcome to redistribute it\n under certain conditions; type `show c' for details.\n\nThe hypothetical commands `show w' and `show c' should show the appropriate\nparts of the General Public License. Of course, your program's commands\nmight be different; for a GUI interface, you would use an \"about box\".\n\n You should also get your employer (if you work as a programmer) or school,\nif any, to sign a \"copyright disclaimer\" for the program, if necessary.\nFor more information on this, and how to apply and follow the GNU GPL, see\n.\n\n The GNU General Public License does not permit incorporating your program\ninto proprietary programs. If your program is a subroutine library, you\nmay consider it more useful to permit linking proprietary applications with\nthe library. If this is what you want to do, use the GNU Lesser General\nPublic License instead of this License. But first, please read\n." }, { "path": "README.md", "content": "# AS-One v2 : A Modular Library for YOLO Object Detection, Segmentation, Tracking & Pose\n\n\n\n
\n

\n \n \n Watch Video\n\n \n

\n\n
\n\n
\n\n[![PyPI version](https://badge.fury.io/py/asone.svg)](https://badge.fury.io/py/asone)\n[![python-version](https://img.shields.io/pypi/pyversions/supervision)](https://badge.fury.io/py/asone)\n[![colab](https://colab.research.google.com/assets/colab-badge.svg)](https://drive.google.com/file/d/1xy5P9WGI19-PzRH3ceOmoCgp63K6J_Ls/view?usp=sharing)\n[![start with why](https://img.shields.io/badge/version-2.0.0-green)](https://github.com/augmentedstartups/AS-One)\n[![GPLv3 License](https://img.shields.io/badge/License-GPL%20v3-yellow.svg)](https://opensource.org/licenses/)\n\n
\n\n## 👋 Hello\n\n==UPDATE: ASOne v2 is now out! We've updated with YOLOV9 and SAM==\n\nAS-One is a python wrapper for multiple detection and tracking algorithms all at one place. Different trackers such as `ByteTrack`, `DeepSORT` or `NorFair` can be integrated with different versions of `YOLO` with minimum lines of code.\nThis python wrapper provides YOLO models in `ONNX`, `PyTorch` & `CoreML` flavors. We plan to offer support for future versions of YOLO when they get released.\n\nThis is One Library for most of your computer vision needs.\n\nIf you would like to dive deeper into YOLO Object Detection and Tracking, then check out our [courses](https://www.augmentedstartups.com/store) and [projects](https://store.augmentedstartups.com)\n\n[](https://www.youtube.com/watch?v=K-VcpPwcM8k)\n\nWatch the step-by-step tutorial 🤝\n\n\n\n## 💻 Install\n
🔥 Prerequisites\n\n- Make sure to install `GPU` drivers in your system if you want to use `GPU` . Follow [driver installation](asone/linux/Instructions/Driver-Installations.md) for further instructions.\n- Make sure you have [MS Build tools](https://aka.ms/vs/17/release/vs_BuildTools.exe) installed in system if using windows.\n- [Download git for windows](https://git-scm.com/download/win) if not installed.\n
\n\n```bash\npip install asone\n```\n\nFor windows machine, you will need to install from source to run `asone` library. Check out instructions in `👉 Install from Source` section below to install on windows.\n
\n 👉 Install from Source\n\n### 💾 Clone the Repository\n\nNavigate to an empty folder of your choice.\n\n`git clone https://github.com/augmentedstartups/AS-One.git`\n\nChange Directory to AS-One\n\n`cd AS-One`\n\n
\n 👉 For Linux\n\n\n```shell\npython3 -m venv .env\nsource .env/bin/activate\n\npip install -r requirements.txt\n\n# for CPU\npip install torch torchvision\n# for GPU\npip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113\n```\n\n\n
\n\n
\n 👉 For Windows 10/11\n\n```shell\npython -m venv .env\n.env\\Scripts\\activate\npip install numpy Cython\npip install lap\npip install -e git+https://github.com/samson-wang/cython_bbox.git#egg=cython-bbox\n\npip install asone onnxruntime-gpu==1.12.1\npip install typing_extensions==4.7.1\npip install super-gradients==3.1.3\n# for CPU\npip install torch torchvision\n\n# for GPU\npip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113\nor\npip install torch==1.10.1+cu113 torchvision==0.11.2+cu113 torchaudio===0.10.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html\n```\n\n
\n
\n 👉 For MacOS\n\n```shell\npython3 -m venv .env\nsource .env/bin/activate\n\n\npip install -r requirements.txt\n\n# for CPU\npip install torch torchvision\n```\n\n
\n
\n\n## Quick Start 🏃‍♂️\n\nUse tracker on sample video.\n\n```python\nimport asone\nfrom asone import ASOne\n\nmodel = ASOne(tracker=asone.BYTETRACK, detector=asone.YOLOV9_C, use_cuda=True)\ntracks = model.video_tracker('data/sample_videos/test.mp4', filter_classes=['car'])\n\nfor model_output in tracks:\n annotations = ASOne.draw(model_output, display=False)\n```\n\n\n### Run in `Google Colab` 💻\n\n\n\"Open\n\n## Sample Code Snippets 📃\n\n
\n6.1 👉 Object Detection\n\n```python\nimport asone\nfrom asone import ASOne\n\nmodel = ASOne(detector=asone.YOLOV9_C, use_cuda=True) # Set use_cuda to False for cpu\nvid = model.read_video('data/sample_videos/test.mp4')\n\nfor img in vid:\n detection = model.detecter(img)\n annotations = ASOne.draw(detection, img=img, display=True)\n```\n\nRun the `asone/demo_detector.py` to test detector.\n\n```shell\n# run on gpu\npython -m asone.demo_detector data/sample_videos/test.mp4\n\n# run on cpu\npython -m asone.demo_detector data/sample_videos/test.mp4 --cpu\n```\n\n\n
\n6.1.1 👉 Use Custom Trained Weights for Detector\n\n\nUse your custom weights of a detector model trained on custom data by simply providing path of the weights file.\n\n```python\nimport asone\nfrom asone import ASOne\n\nmodel = ASOne(detector=asone.YOLOV9_C, weights='data/custom_weights/yolov7_custom.pt', use_cuda=True) # Set use_cuda to False for cpu\nvid = model.read_video('data/sample_videos/license_video.mp4')\n\nfor img in vid:\n detection = model.detecter(img)\n annotations = ASOne.draw(detection, img=img, display=True, class_names=['license_plate'])\n```\n\n
\n\n
\n6.1.2 👉 Changing Detector Models \n\nChange detector by simply changing detector flag. The flags are provided in [benchmark](asone/linux/Instructions/Benchmarking.md) tables.\n\n- Our library now supports YOLOv5, YOLOv7, and YOLOv8 on macOS.\n\n```python\n# Change detector\nmodel = ASOne(detector=asone.YOLOX_S_PYTORCH, use_cuda=True)\n\n# For macOs\n# YOLO5\nmodel = ASOne(detector=asone.YOLOV5X_MLMODEL)\n# YOLO7\nmodel = ASOne(detector=asone.YOLOV7_MLMODEL)\n# YOLO8\nmodel = ASOne(detector=asone.YOLOV8L_MLMODEL)\n```\n\n
\n\n
\n\n
\n6.2 👉 Object Tracking \n\nUse tracker on sample video.\n\n```python\nimport asone\nfrom asone import ASOne\n\n# Instantiate Asone object\nmodel = ASOne(tracker=asone.BYTETRACK, detector=asone.YOLOV9_C, use_cuda=True) #set use_cuda=False to use cpu\ntracks = model.video_tracker('data/sample_videos/test.mp4', filter_classes=['car'])\n\n# Loop over track to retrieve outputs of each frame\nfor model_output in tracks:\n annotations = ASOne.draw(model_output, display=True)\n # Do anything with bboxes here\n```\n\n[Note] Use can use custom weights for a detector model by simply providing path of the weights file. in `ASOne` class.\n\n
\n6.2.1 👉 Changing Detector and Tracking Models\n\n\n\nChange Tracker by simply changing the tracker flag.\n\nThe flags are provided in [benchmark](asone/linux/Instructions/Benchmarking.md) tables.\n\n```python\nmodel = ASOne(tracker=asone.BYTETRACK, detector=asone.YOLOV9_C, use_cuda=True)\n# Change tracker\nmodel = ASOne(tracker=asone.DEEPSORT, detector=asone.YOLOV9_C, use_cuda=True)\n```\n\n```python\n# Change Detector\nmodel = ASOne(tracker=asone.DEEPSORT, detector=asone.YOLOX_S_PYTORCH, use_cuda=True)\n```\n\n
\n\nRun the `asone/demo_tracker.py` to test detector.\n\n```shell\n# run on gpu\npython -m asone.demo_tracker data/sample_videos/test.mp4\n\n# run on cpu\npython -m asone.demo_tracker data/sample_videos/test.mp4 --cpu\n```\n\n
\n\n
\n6.3 👉 Segmentation\n\n\n```python\nimport asone\nfrom asone import ASOne\n\nmodel = ASOne(detector=asone.YOLOV9_C, segmentor=asone.SAM, use_cuda=True) #set use_cuda=False to use cpu\ntracks = model.video_detecter('data/sample_videos/test.mp4', filter_classes=['car'])\n\nfor model_output in tracks:\n annotations = ASOne.draw_masks(model_output, display=True) # Draw masks\n```\n
\n\n
\n6.4 👉 Text Detection\n \nSample code to detect text on an image\n\n```python\n# Detect and recognize text\nimport asone\nfrom asone import ASOne, utils\nimport cv2\n\nmodel = ASOne(detector=asone.CRAFT, recognizer=asone.EASYOCR, use_cuda=True) # Set use_cuda to False for cpu\nimg = cv2.imread('data/sample_imgs/sample_text.jpeg')\nresults = model.detect_text(img)\nannotations = utils.draw_text(img, results, display=True)\n```\n\nUse Tracker on Text\n\n```python\nimport asone\nfrom asone import ASOne\n\n# Instantiate Asone object\nmodel = ASOne(tracker=asone.DEEPSORT, detector=asone.CRAFT, recognizer=asone.EASYOCR, use_cuda=True) #set use_cuda=False to use cpu\ntracks = model.video_tracker('data/sample_videos/GTA_5-Unique_License_Plate.mp4')\n\n# Loop over track to retrieve outputs of each frame\nfor model_output in tracks:\n annotations = ASOne.draw(model_output, display=True)\n\n # Do anything with bboxes here\n```\n\nRun the `asone/demo_ocr.py` to test ocr.\n\n```shell\n# run on gpu\n python -m asone.demo_ocr data/sample_videos/GTA_5-Unique_License_Plate.mp4\n\n# run on cpu\n python -m asone.demo_ocr data/sample_videos/GTA_5-Unique_License_Plate.mp4 --cpu\n```\n\n
\n\n
\n6.5 👉 Pose Estimation\n\n\nSample code to estimate pose on an image\n\n```python\n# Pose Estimation\nimport asone\nfrom asone import PoseEstimator, utils\nimport cv2\n\nmodel = PoseEstimator(estimator_flag=asone.YOLOV8M_POSE, use_cuda=True) #set use_cuda=False to use cpu\nimg = cv2.imread('data/sample_imgs/test2.jpg')\nkpts = model.estimate_image(img)\nannotations = utils.draw_kpts(kpts, image=img, display=True)\n```\n\n- Now you can use Yolov8 and Yolov7-w6 for pose estimation. The flags are provided in [benchmark](asone/linux/Instructions/Benchmarking.md) tables.\n\n```python\n# Pose Estimation on video\nimport asone\nfrom asone import PoseEstimator, utils\n\nmodel = PoseEstimator(estimator_flag=asone.YOLOV7_W6_POSE, use_cuda=True) #set use_cuda=False to use cpu\nestimator = model.video_estimator('data/sample_videos/football1.mp4')\nfor model_output in estimator:\n annotations = utils.draw_kpts(model_output)\n # Do anything with kpts here\n```\n\nRun the `asone/demo_pose_estimator.py` to test Pose estimation.\n\n```shell\n# run on gpu\n python -m asone.demo_pose_estimator data/sample_videos/football1.mp4\n\n# run on cpu\n python -m asone.demo_pose_estimator data/sample_videos/football1.mp4 --cpu\n```\n\n
\n\nTo setup ASOne using Docker follow instructions given in [docker setup](asone/linux/Instructions/Docker-Setup.md)🐳\n\n### ToDo 📝\n\n- [x] First Release\n- [x] Import trained models\n- [x] Simplify code even further\n- [x] Updated for YOLOv8\n- [x] OCR and Counting\n- [x] OCSORT, StrongSORT, MoTPy\n- [x] M1/2 Apple Silicon Compatibility\n- [x] Pose Estimation YOLOv7/v8\n- [x] YOLO-NAS\n- [x] Updated for YOLOv8.1\n- [x] YOLOV9\n- [x] SAM Integration\n\n\n| Offered By 💼 : | Maintained By 👨‍💻 : |\n| ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------- |\n| [![AugmentedStarups](https://user-images.githubusercontent.com/107035454/195115263-d3271ef3-973b-40a4-83c8-0ade8727dd40.png)](https://augmentedstartups.com) | [![AxcelerateAI](https://user-images.githubusercontent.com/107035454/195114870-691c8a52-fcf0-462e-9e02-a720fc83b93f.png)](https://axcelerate.ai/) |\n" }, { "path": "asone/__init__.py", "content": "from .asone import ASOne\nimport asone.detectors\nimport asone.trackers\nimport asone.recognizers\nimport asone.segmentors\nfrom .pose_estimator import PoseEstimator\nfrom asone.utils.video_reader import VideoReader\n\nBYTETRACK = 0\nDEEPSORT = 1\nNORFAIR = 2\nMOTPY = 3\nOCSORT = 4\nSTRONGSORT = 5\n\n\nYOLOV5X6_PYTORCH = 0\nYOLOV5S_PYTORCH = 2\nYOLOV5N_PYTORCH = 4\nYOLOV5M_PYTORCH = 6\nYOLOV5L_PYTORCH = 8\nYOLOV5X_PYTORCH = 10\nYOLOV5N6_PYTORCH = 12\nYOLOV5S6_PYTORCH = 14\nYOLOV5M6_PYTORCH = 16\nYOLOV5L6_PYTORCH = 18\n\n\nYOLOV6N_PYTORCH = 20\nYOLOV6T_PYTORCH = 22\nYOLOV6S_PYTORCH = 24\nYOLOV6M_PYTORCH = 26\nYOLOV6L_PYTORCH = 28 \nYOLOV6L_RELU_PYTORCH = 30\nYOLOV6S_REPOPT_PYTORCH = 32\n\nYOLOV7_TINY_PYTORCH = 34\nYOLOV7_PYTORCH = 36\nYOLOV7_X_PYTORCH = 38\nYOLOV7_W6_PYTORCH = 40\nYOLOV7_E6_PYTORCH = 42\nYOLOV7_D6_PYTORCH = 44\nYOLOV7_E6E_PYTORCH = 46\n\nYOLOR_CSP_X_PYTORCH = 48\nYOLOR_CSP_X_STAR_PYTORCH = 50 \nYOLOR_CSP_STAR_PYTORCH = 52\nYOLOR_CSP_PYTORCH = 54\nYOLOR_P6_PYTORCH = 56\n\n\n\n\nYOLOX_L_PYTORCH = 58\nYOLOX_NANO_PYTORCH = 60 \nYOLOX_TINY_PYTORCH = 62\nYOLOX_DARKNET_PYTORCH = 64 \nYOLOX_S_PYTORCH = 66\nYOLOX_M_PYTORCH = 68\nYOLOX_X_PYTORCH = 70\n\n#ONNX\n\nYOLOV5X6_ONNX = 1\nYOLOV5S_ONNX = 3\nYOLOV5N_ONNX = 5\nYOLOV5M_ONNX = 7\nYOLOV5L_ONNX = 9\nYOLOV5X_ONNX = 11\nYOLOV5N6_ONNX = 13\nYOLOV5S6_ONNX = 15\nYOLOV5M6_ONNX = 17\nYOLOV5L6_ONNX = 19\n\n\nYOLOV6N_ONNX = 21\nYOLOV6T_ONNX = 23\nYOLOV6S_ONNX = 25\nYOLOV6M_ONNX = 27\nYOLOV6L_ONNX = 29 \nYOLOV6L_RELU_ONNX = 31\nYOLOV6S_REPOPT_ONNX = 33\n\nYOLOV7_TINY_ONNX = 35\nYOLOV7_ONNX = 37\nYOLOV7_X_ONNX = 39\nYOLOV7_W6_ONNX = 41\nYOLOV7_E6_ONNX = 43\nYOLOV7_D6_ONNX = 45\nYOLOV7_E6E_ONNX = 47\n\nYOLOR_CSP_X_ONNX = 49\nYOLOR_CSP_X_STAR_ONNX = 51\nYOLOR_CSP_STAR_ONNX = 53\nYOLOR_CSP_ONNX = 55\nYOLOR_P6_ONNX = 57\n\n\nYOLOX_L_ONNX = 59\nYOLOX_NANO_ONNX = 61 \nYOLOX_TINY_ONNX = 63\nYOLOX_DARKNET_ONNX = 65 \nYOLOX_S_ONNX = 67\nYOLOX_M_ONNX = 69\nYOLOX_X_ONNX = 71\n\n# YOLOv8\nYOLOV8N_PYTORCH = 72\nYOLOV8N_ONNX = 73\nYOLOV8S_PYTORCH = 74\nYOLOV8S_ONNX = 75\nYOLOV8M_PYTORCH = 76\nYOLOV8M_ONNX = 77\nYOLOV8L_PYTORCH = 78\nYOLOV8L_ONNX = 79\nYOLOV8X_PYTORCH = 80\nYOLOV8X_ONNX = 81\n\n# coreml\n\nYOLOV5N_MLMODEL = 120\nYOLOV5S_MLMODEL = 121\nYOLOV5X6_MLMODEL = 122\nYOLOV5M_MLMODEL = 123\nYOLOV5L_MLMODEL = 124\nYOLOV5X_MLMODEL = 125\nYOLOV5N6_MLMODEL = 126\nYOLOV5S6_MLMODEL = 127\nYOLOV5M6_MLMODEL = 128\nYOLOV5L6_MLMODEL = 129\n \n\nYOLOV7_TINY_MLMODEL = 130\nYOLOV7_MLMODEL = 131\nYOLOV7_X_MLMODEL = 132\nYOLOV7_W6_MLMODEL = 133\nYOLOV7_E6_MLMODEL = 134\nYOLOV7_D6_MLMODEL = 135\nYOLOV7_E6E_MLMODEL = 136\n\nYOLOV8N_MLMODEL = 139\nYOLOV8S_MLMODEL = 140\nYOLOV8M_MLMODEL = 141\nYOLOV8L_MLMODEL = 142\nYOLOV8X_MLMODEL = 143\n\nYOLOV8N_POSE = 144\nYOLOV8S_POSE = 145\nYOLOV8M_POSE = 146\nYOLOV8L_POSE = 147\nYOLOV8X_POSE = 148\n\nYOLOV7_W6_POSE = 149\n\nYOLONAS_S_PYTORCH = 160\nYOLONAS_M_PYTORCH = 161\nYOLONAS_L_PYTORCH = 162\n\n# YOLOv9\nYOLOV9_C_CONVERTED = 164\nYOLOV9_E_CONVERTED = 165\nYOLOV9_C = 166\nYOLOV9_E = 167\nGELAN_C = 168\nGELAN_E = 169\n\n# Segmentors\nSAM = 171\n\n# Text Detectors\n# easyocr\nCRAFT = 82\nDBNET18 = 83\n\n# Text Recognizers\nEASYOCR = 200\n\n\n\n__all__ = ['ASOne', 'detectors', 'trackers', 'recognizers', 'segmentors', 'PoseEstimator'] \n" }, { "path": "asone/asone.py", "content": "import copy\nimport warnings\nimport cv2\nfrom loguru import logger\nimport os\nimport time\nimport asone.utils as utils\nfrom asone.trackers import Tracker\nfrom asone.detectors import Detector\nfrom asone.recognizers import TextRecognizer\nfrom asone.segmentors import Segmentor\nfrom asone.utils.default_cfg import config\nfrom asone.utils.video_reader import VideoReader\nfrom asone.utils import compute_color_for_labels\nfrom asone.schemas.output_schemas import ModelOutput\n\nimport numpy as np\n\n\nclass ASOne:\n def __init__(self,\n detector: int = 0,\n tracker: int = -1,\n segmentor: int = -1,\n weights: str = None,\n segmentor_weights: str = None,\n use_cuda: bool = True,\n recognizer: int = None,\n languages: list = ['en'],\n num_classes=80\n ) -> None:\n\n self.use_cuda = use_cuda\n self.use_segmentation = False\n self.model_output = ModelOutput()\n \n # Check if user want to use segmentor\n if segmentor != -1:\n self.use_segmentation = True\n\n # Load Segmentation model\n self.segmentor = self.get_segmentor(segmentor, segmentor_weights)\n\n # get detector object\n self.detector = self.get_detector(detector, weights, recognizer, num_classes)\n self.recognizer = self.get_recognizer(recognizer, languages=languages)\n \n if tracker == -1:\n self.tracker = None\n return\n \n self.tracker = self.get_tracker(tracker)\n\n def get_detector(self, detector: int, weights: str, recognizer, num_classes):\n detector = Detector(detector, weights=weights,\n use_cuda=self.use_cuda, recognizer=recognizer, num_classes=num_classes).get_detector()\n return detector\n\n def get_recognizer(self, recognizer: int, languages):\n if recognizer == None:\n return None\n recognizer = TextRecognizer(recognizer,\n use_cuda=self.use_cuda, languages=languages).get_recognizer()\n\n return recognizer\n\n def get_tracker(self, tracker: int):\n tracker = Tracker(tracker, self.detector,\n use_cuda=self.use_cuda)\n return tracker\n \n def get_segmentor(self, segmentor, segmentor_weights):\n segmentor = Segmentor(segmentor, segmentor_weights, self.use_cuda)\n return segmentor\n\n def _update_args(self, kwargs):\n for key, value in kwargs.items():\n if key in config.keys():\n config[key] = value\n else:\n print(f'\"{key}\" argument not found! valid args: {list(config.keys())}')\n exit()\n return config\n\n def track_stream(self,\n stream_url,\n **kwargs\n ):\n \n # Emit the warning for DeprecationWarning\n with warnings.catch_warnings():\n warnings.simplefilter(\"always\", DeprecationWarning)\n warnings.warn(\"track_stream function is deprecated. Kindly use stream_tracker instead\", DeprecationWarning)\n\n output_filename = 'result.mp4'\n kwargs['filename'] = output_filename\n config = self._update_args(kwargs)\n\n for (bbox_details, frame_details) in self._start_tracking(stream_url, config):\n # yeild bbox_details, frame_details to main script\n yield bbox_details, frame_details\n \n def stream_tracker(self,\n stream_url,\n **kwargs\n ):\n\n output_filename = 'result.mp4'\n kwargs['filename'] = output_filename\n config = self._update_args(kwargs)\n\n for (bbox_details, frame_details) in self._start_tracking(stream_url, config):\n # yeild bbox_details, frame_details to main script\n yield self.format_output(bbox_details, frame_details)\n\n def track_video(self,\n video_path,\n **kwargs\n ): \n \n # Emit the warning for DeprecationWarning\n with warnings.catch_warnings():\n warnings.simplefilter(\"always\", DeprecationWarning)\n warnings.warn(\"track_video function is deprecated. Kindly use video_tracker instead\", DeprecationWarning)\n \n output_filename = os.path.basename(video_path)\n kwargs['filename'] = output_filename\n config = self._update_args(kwargs)\n\n for (bbox_details, frame_details) in self._start_tracking(video_path, config):\n # yeild bbox_details, frame_details to main script\n yield bbox_details, frame_details\n \n def video_tracker(self,\n video_path,\n **kwargs\n ): \n output_filename = os.path.basename(video_path)\n kwargs['filename'] = output_filename\n config = self._update_args(kwargs)\n\n for (bbox_details, frame_details) in self._start_tracking(video_path, config):\n # yeild bbox_details, frame_details to main script\n yield self.format_output(bbox_details, frame_details)\n\n def detect_video(self,\n video_path,\n **kwargs\n ): \n \n # Emit the warning for DeprecationWarning\n with warnings.catch_warnings():\n warnings.simplefilter(\"always\", DeprecationWarning)\n warnings.warn(\"detect_video function is deprecated. Kindly use video_detecter instead\", DeprecationWarning)\n \n output_filename = os.path.basename(video_path)\n kwargs['filename'] = output_filename\n config = self._update_args(kwargs)\n\n for (bbox_details, frame_details) in self._start_tracking(video_path, config):\n # yeild bbox_details, frame_details to main script\n yield bbox_details, frame_details\n \n def video_detecter(self,\n video_path,\n **kwargs\n ): \n output_filename = os.path.basename(video_path)\n kwargs['filename'] = output_filename\n config = self._update_args(kwargs)\n\n for (bbox_details, frame_details) in self._start_tracking(video_path, config):\n # yeild bbox_details, frame_details to main script\n yield self.format_output(bbox_details, frame_details)\n \n def detect(self, source, **kwargs)->np.ndarray:\n \"\"\" Function to perform detection on an img\n\n Args:\n source (_type_): if str read the image. if nd.array pass it directly to detect\n\n Returns:\n _type_: ndarray of detection\n \"\"\"\n # Emit the warning for DeprecationWarning\n with warnings.catch_warnings():\n warnings.simplefilter(\"always\", DeprecationWarning)\n warnings.warn(\"detect function is deprecated. Kindly use detecter instead\", DeprecationWarning)\n \n if isinstance(source, str):\n source = cv2.imread(source)\n return self.detector.detect(source, **kwargs)\n \n def detecter(self, source, **kwargs):\n \"\"\" Function to perform detection on an img\n\n Args:\n source (_type_): if str read the image. if nd.array pass it directly to detect\n\n Returns:\n _type_: ndarray of detection\n \"\"\"\n if isinstance(source, str):\n source = cv2.imread(source)\n dets, _ = self.detector.detect(source, **kwargs)\n bboxes_xyxy = dets[:, :4]\n scores = dets[:, 4]\n class_ids = dets[:, 5]\n ids = None\n info = None\n return self.format_output((bboxes_xyxy, ids, scores, class_ids), info)\n\n def detect_and_track(self, frame, **kwargs):\n if self.tracker:\n bboxes_xyxy, ids, scores, class_ids = self.tracker.detect_and_track(\n frame, kwargs)\n info = None\n else:\n dets, info = self.detect(source=frame, **kwargs)\n bboxes_xyxy = dets[:, :4]\n scores = dets[:, 4]\n class_ids = dets[:, 5]\n ids = None\n\n return (bboxes_xyxy, ids, scores, class_ids), info\n \n def detect_track_manager(self, frame, **kwargs):\n if self.tracker:\n bboxes_xyxy, ids, scores, class_ids = self.tracker.detect_and_track(\n frame, kwargs)\n info = None\n else:\n model_output = self.detecter(source=frame, **kwargs)\n \n info = model_output.info \n bboxes_xyxy = model_output.dets.bbox\n scores = model_output.dets.score\n class_ids = model_output.dets.class_ids\n ids = None\n\n return (bboxes_xyxy, ids, scores, class_ids), info\n \n def detect_text(self, image):\n horizontal_list, _ = self.detector.detect(image)\n if self.recognizer is None:\n raise TypeError(\"Recognizer can not be None\")\n \n return self.recognizer.recognize(image, horizontal_list=horizontal_list,\n free_list=[])\n\n def track_webcam(self,\n cam_id=0,\n **kwargs):\n # Emit the warning for DeprecationWarning\n with warnings.catch_warnings():\n warnings.simplefilter(\"always\", DeprecationWarning)\n warnings.warn(\"track_webcam function is deprecated. Kindly use webcam_tracker instead\", DeprecationWarning)\n \n output_filename = 'results.mp4'\n\n kwargs['filename'] = output_filename\n kwargs['fps'] = 29\n config = self._update_args(kwargs)\n\n\n for (bbox_details, frame_details) in self._start_tracking(cam_id, config):\n # yeild bbox_details, frame_details to main script\n yield bbox_details, frame_details\n \n def webcam_tracker(self,\n cam_id=0,\n **kwargs):\n output_filename = 'results.mp4'\n\n kwargs['filename'] = output_filename\n kwargs['fps'] = 29\n config = self._update_args(kwargs)\n\n\n for (bbox_details, frame_details) in self._start_tracking(cam_id, config):\n # yeild bbox_details, frame_details to main script\n yield self.format_output(bbox_details, frame_details)\n \n def _start_tracking(self,\n stream_path: str,\n config: dict) -> tuple:\n\n if not self.tracker:\n warnings.warn(f'No tracker has been selected. Only the detector is operational.')\n\n fps = config.pop('fps')\n output_dir = config.pop('output_dir')\n filename = config.pop('filename')\n save_result = config.pop('save_result')\n display = config.pop('display')\n draw_trails = config.pop('draw_trails')\n class_names = config.pop('class_names')\n\n cap = self.read_video(stream_path)\n width, height = cap.frame_size\n frame_count = cap.frame_counts\n\n if fps is None:\n fps = cap.fps\n\n if save_result:\n os.makedirs(output_dir, exist_ok=True)\n save_path = os.path.join(output_dir, filename)\n logger.info(f\"video save path is {save_path}\")\n\n video_writer = cv2.VideoWriter(\n save_path,\n cv2.VideoWriter_fourcc(*\"mp4v\"),\n fps,\n (int(width), int(height)),\n )\n\n frame_id = 1\n tic = time.time()\n prevTime = 0\n\n for frame in cap:\n start_time = time.time()\n\n im0 = copy.deepcopy(frame)\n try:\n (bboxes_xyxy, ids, scores, class_ids), _ = self.detect_track_manager(frame, **config)\n except:\n (bboxes_xyxy, ids, scores, class_ids), _ = self.detect_and_track(frame, **config)\n\n elapsed_time = time.time() - start_time\n\n logger.info(\n 'frame {}/{} ({:.2f} ms)'.format(frame_id, int(frame_count),\n elapsed_time * 1000))\n\n if self.recognizer:\n res = self.recognizer.recognize(frame, horizontal_list=bboxes_xyxy,\n free_list=[])\n im0 = utils.draw_text(im0, res)\n else:\n im0 = self.draw((bboxes_xyxy, ids, scores, class_ids),\n img=im0,\n draw_trails=draw_trails,\n class_names=class_names,\n display=display)\n\n currTime = time.time()\n fps = 1 / (currTime - prevTime)\n prevTime = currTime\n cv2.line(im0, (20, 25), (127, 25), [85, 45, 255], 30)\n cv2.putText(im0, f'FPS: {int(fps)}', (11, 35), 0, 1, [\n 225, 255, 255], thickness=2, lineType=cv2.LINE_AA)\n \n if self.use_segmentation:\n if len(bboxes_xyxy) > 0: # Check if bounding box is present or not\n # Will generate mask using SAM\n masks = self.segmentor.create_mask(np.array(bboxes_xyxy), frame)\n im0 = self.draw_masks(masks, img=im0, display=display)\n bboxes_xyxy = (bboxes_xyxy, masks) \n \n if save_result:\n video_writer.write(im0)\n\n frame_id += 1\n\n if cv2.waitKey(25) & 0xFF == ord('q'):\n break\n\n # yeild required values in form of (bbox_details, frames_details)\n yield (bboxes_xyxy, ids, scores, class_ids), (im0 if display else frame, frame_id-1, fps)\n\n tac = time.time()\n print(f'Total Time Taken: {tac - tic:.2f}')\n\n @staticmethod\n def draw(dets, display=False, img=None, **kwargs): \n draw_trails = kwargs.get('draw_trails', False)\n class_names = kwargs.get('class_names', None)\n if isinstance(dets, tuple):\n bboxes_xyxy, ids, scores, class_ids = dets\n if isinstance(bboxes_xyxy, tuple):\n bboxes_xyxy, _ = bboxes_xyxy \n \n elif isinstance(dets, np.ndarray):\n bboxes_xyxy = dets[:, :4]\n scores = dets[:, 4]\n class_ids = dets[:, 5]\n ids = None\n \n elif isinstance(dets, ModelOutput):\n bboxes_xyxy = dets.dets.bbox\n ids = dets.dets.ids\n score = dets.dets.score\n class_ids = dets.dets.class_ids\n img = dets.info.image if dets.info.image is not None else img\n frame_no = dets.info.frame_no\n fps = dets.info.fps\n \n img = utils.draw_boxes(img,\n bbox_xyxy=bboxes_xyxy,\n class_ids=class_ids,\n identities=ids,\n draw_trails=draw_trails,\n class_names=class_names)\n \n if display:\n cv2.imshow(' Sample', img)\n \n return img\n \n @staticmethod \n def draw_masks(dets, display, img=None, **kwargs):\n # Check if bounding box are present\n if isinstance(dets, tuple) and len(dets) > 0 and len(dets[0]) == 0:\n return img\n \n elif isinstance(dets, ModelOutput):\n masks = dets.dets.bbox\n ids = dets.dets.ids\n score = dets.dets.score\n class_ids = dets.dets.class_ids\n img = dets.info.image if dets.info.image is not None else img\n frame_no = dets.info.frame_no\n fps = dets.info.fps\n if isinstance(masks, tuple):\n bboxes_xyxy, masks = masks\n if isinstance(masks, np.ndarray):\n return img\n \n elif isinstance(dets, tuple):\n bboxes_xyxy, ids, scores, class_ids = dets\n if isinstance(bboxes_xyxy, tuple):\n bboxes_xyxy, masks = bboxes_xyxy\n else:\n masks = dets\n class_ids = None\n \n color = [0, 255, 0]\n masked_image = img.copy()\n for idx in range(len(masks)):\n mask = masks[idx].squeeze() # Squeeze to remove singleton dimension\n if class_ids is not None:\n color = compute_color_for_labels(int(class_ids[idx]))\n color = np.asarray(color, dtype='uint8')\n mask_color = np.expand_dims(mask, axis=-1) * color # Apply color to the mask\n # Apply the mask to the image\n masked_image = np.where(mask_color > 0, mask_color, masked_image)\n\n masked_image = masked_image.astype(np.uint8)\n img = cv2.addWeighted(img, 0.5, masked_image, 0.5, 0)\n \n if display:\n cv2.imshow(' Sample', img)\n return img\n\n def read_video(self, video_path):\n vid = VideoReader(video_path)\n \n return vid\n \n def format_output(self, bbox_details, frame_details):\n\n # Set detections\n self.model_output.dets.bbox = bbox_details[0]\n self.model_output.dets.ids = bbox_details[1]\n self.model_output.dets.score = bbox_details[2]\n self.model_output.dets.class_ids = bbox_details[3]\n if frame_details:\n # Set image info\n self.model_output.info.image = frame_details[0]\n self.model_output.info.frame_no = frame_details[1]\n self.model_output.info.fps = frame_details[2]\n\n return self.model_output\n \nif __name__ == '__main__':\n # asone = ASOne(tracker='norfair')\n asone = ASOne()\n\n asone.start_tracking('data/sample_videos/video2.mp4',\n save_result=True, display=False)\n" }, { "path": "asone/demo_detector.py", "content": "import sys\nimport argparse\nimport asone\nfrom asone import ASOne\nimport torch\n\n\ndef main(args):\n filter_classes = args.filter_classes\n\n if filter_classes:\n filter_classes = ['person']\n # Check if cuda available\n if args.use_cuda and torch.cuda.is_available():\n args.use_cuda = True\n else:\n args.use_cuda = False\n \n if sys.platform.startswith('darwin'):\n detector = asone.YOLOV7_MLMODEL \n else:\n detector = asone.YOLOV7_PYTORCH\n \n detect = ASOne(\n detector=detector,\n weights=args.weights,\n use_cuda=args.use_cuda\n )\n # Get tracking function\n track = detect.detect_video(args.video_path,\n output_dir=args.output_dir,\n conf_thres=args.conf_thres,\n iou_thres=args.iou_thres,\n display=args.display,\n filter_classes=filter_classes,\n class_names=None) # class_names=['License Plate'] for custom weights\n \n # Loop over track_fn to retrieve outputs of each frame \n for bbox_details, frame_details in track:\n bbox_xyxy, ids, scores, class_ids = bbox_details\n frame, frame_num, fps = frame_details\n print(frame_num)\n \n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n\n parser.add_argument('video_path', help='Path to input video')\n parser.add_argument('--cpu', default=True, action='store_false', dest='use_cuda',\n help='run on cpu if not provided the program will run on gpu.')\n parser.add_argument('--no_save', default=True, action='store_false',\n dest='save_result', help='whether or not save results')\n parser.add_argument('--no_display', default=True, action='store_false',\n dest='display', help='whether or not display results on screen')\n parser.add_argument('--output_dir', default='data/results', help='Path to output directory')\n parser.add_argument('--draw_trails', action='store_true', default=False,\n help='if provided object motion trails will be drawn.')\n parser.add_argument('--filter_classes', default=None, help='Filter class name')\n parser.add_argument('-w', '--weights', default=None, help='Path of trained weights')\n parser.add_argument('-ct', '--conf_thres', default=0.25, type=float, help='confidence score threshold')\n parser.add_argument('-it', '--iou_thres', default=0.45, type=float, help='iou score threshold')\n\n args = parser.parse_args()\n\n main(args)" }, { "path": "asone/demo_ocr.py", "content": "import argparse\nimport asone\nfrom asone import ASOne\n\ndef main(args):\n\n detect = ASOne(\n tracker=asone.DEEPSORT,\n detector=asone.CRAFT,\n weights=args.weights,\n recognizer=asone.EASYOCR,\n use_cuda=args.use_cuda\n )\n # Get tracking function\n track = detect.track_video(args.video_path,\n output_dir=args.output_dir,\n conf_thres=args.conf_thres,\n iou_thres=args.iou_thres,\n display=args.display,\n draw_trails=args.draw_trails) # class_names=['License Plate'] for custom weights\n \n # Loop over track to retrieve outputs of each frame \n for bbox_details, frame_details in track:\n bbox_xyxy, ids, scores, class_ids = bbox_details\n frame, frame_num, fps = frame_details\n print(frame_num)\n \n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n\n parser.add_argument('video_path', help='Path to input video')\n parser.add_argument('--cpu', default=True, action='store_false', dest='use_cuda',\n help='run on cpu if not provided the program will run on gpu.')\n parser.add_argument('--no_save', default=True, action='store_false',\n dest='save_result', help='whether or not save results')\n parser.add_argument('--no_display', default=True, action='store_false',\n dest='display', help='whether or not display results on screen')\n parser.add_argument('--output_dir', default='data/results', help='Path to output directory')\n parser.add_argument('--draw_trails', action='store_true', default=False,\n help='if provided object motion trails will be drawn.')\n parser.add_argument('-w', '--weights', default=None, help='Path of trained weights')\n parser.add_argument('-ct', '--conf_thres', default=0.25, type=float, help='confidence score threshold')\n parser.add_argument('-it', '--iou_thres', default=0.45, type=float, help='iou score threshold')\n\n args = parser.parse_args()\n\n main(args)" }, { "path": "asone/demo_pose_estimator.py", "content": "import asone\nfrom asone import PoseEstimator\nfrom .utils import draw_kpts\nimport cv2\nimport argparse\nimport time\nimport os\n\n\ndef main(args):\n \n video_path = args.video\n os.makedirs(args.output_path, exist_ok=True)\n estimator = PoseEstimator(asone.YOLOV7_W6_POSE, weights=args.weights, use_cuda=args.use_cuda)\n\n cap = cv2.VideoCapture(video_path)\n width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)\n height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)\n FPS = cap.get(cv2.CAP_PROP_FPS)\n\n if args.save:\n video_writer = cv2.VideoWriter(\n os.path.join(args.output_path,os.path.basename(video_path)),\n cv2.VideoWriter_fourcc(*\"mp4v\"),\n FPS,\n (int(width), int(height)),\n )\n \n frame_no = 1\n tic = time.time()\n\n prevTime = 0\n fframe_num = 0\n while True:\n start_time = time.time()\n\n ret, img = cap.read()\n if not ret:\n break\n frame = img.copy()\n \n kpts = estimator.estimate_image(img)\n currTime = time.time()\n fps = 1 / (currTime - prevTime)\n prevTime = currTime\n\n if kpts is not None: \n img = draw_kpts(img, kpts)\n cv2.line(img, (20, 25), (127, 25), [85, 45, 255], 30)\n cv2.putText(img, f'FPS: {int(fps)}', (11, 35), 0, 1, [\n 225, 255, 255], thickness=2, lineType=cv2.LINE_AA)\n\n\n frame_no+=1\n if args.display:\n cv2.imshow('Window', img)\n\n if args.save:\n video_writer.write(img)\n \n # frame_no += 1 \n print(frame_no)\n \n if cv2.waitKey(25) & 0xFF == ord('q'):\n break\n\nif __name__=='__main__':\n \n parser = argparse.ArgumentParser()\n parser.add_argument(\"video\", help=\"Path of video\")\n parser.add_argument('--cpu', default=True, action='store_false', dest='use_cuda', help='If provided the model will run on cpu otherwise it will run on gpu')\n parser.add_argument('-w', '--weights', default=None, help='Path of trained weights')\n parser.add_argument('-o', '--output_path', default='data/results', help='path of output file')\n parser.add_argument('--no_display', action='store_false', default=True, dest='display', help='if provided video will not be displayed')\n parser.add_argument('--no_save', action='store_false', default=True, dest='save', help='if provided video will not be saved')\n\n args = parser.parse_args()\n main(args)" }, { "path": "asone/demo_segmentor.py", "content": "import sys\nimport argparse\nimport asone\nfrom asone import ASOne\nimport torch\n\n\ndef main(args):\n filter_classes = args.filter_classes\n\n if filter_classes:\n filter_classes = ['person']\n # Check if cuda available\n if args.use_cuda and torch.cuda.is_available():\n args.use_cuda = True\n else:\n args.use_cuda = False\n \n if sys.platform.startswith('darwin'):\n detector = asone.YOLOV7_MLMODEL \n else:\n detector = asone.YOLOV7_PYTORCH\n \n detect = ASOne(\n detector=detector,\n segmentor=asone.SAM,\n weights=args.weights,\n segmentor_weights=args.segmentor_weights,\n use_cuda=args.use_cuda\n )\n # Get tracking function\n track = detect.detect_video(args.video_path,\n output_dir=args.output_dir,\n conf_thres=args.conf_thres,\n iou_thres=args.iou_thres,\n display=args.display,\n draw_trails=args.draw_trails,\n filter_classes=filter_classes,\n class_names=None) # class_names=['License Plate'] for custom weights\n \n # Loop over track_fn to retrieve outputs of each frame \n for bbox_details, frame_details in track:\n bbox_xyxy, ids, scores, class_ids = bbox_details\n frame, frame_num, fps = frame_details\n print(frame_num)\n \n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n\n parser.add_argument('video_path', help='Path to input video')\n parser.add_argument('--cpu', default=True, action='store_false', dest='use_cuda',\n help='run on cpu if not provided the program will run on gpu.')\n parser.add_argument('--no_save', default=True, action='store_false',\n dest='save_result', help='whether or not save results')\n parser.add_argument('--no_display', default=True, action='store_false',\n dest='display', help='whether or not display results on screen')\n parser.add_argument('--output_dir', default='data/results', help='Path to output directory')\n parser.add_argument('--draw_trails', action='store_true', default=False,\n help='if provided object motion trails will be drawn.')\n parser.add_argument('--filter_classes', default=None, help='Filter class name')\n parser.add_argument('-w', '--weights', default=None, help='Path of trained weights')\n parser.add_argument('--segmentor_weights', default=None, help='Path of Segmentor weights')\n parser.add_argument('-ct', '--conf_thres', default=0.25, type=float, help='confidence score threshold')\n parser.add_argument('-it', '--iou_thres', default=0.45, type=float, help='iou score threshold')\n\n args = parser.parse_args()\n\n main(args)" }, { "path": "asone/demo_tracker.py", "content": "import asone\nfrom asone import ASOne\nfrom .utils import draw_boxes\nimport cv2\nimport argparse\nimport time\nimport os\n\n\ndef main(args):\n filter_classes = args.filter_classes\n video_path = args.video\n\n os.makedirs(args.output_path, exist_ok=True)\n\n if filter_classes:\n filter_classes = filter_classes.split(',')\n\n\n detect = ASOne(tracker=asone.BYTETRACK, detector=asone.YOLOV7_PYTORCH, \n use_cuda=args.use_cuda)\n\n track = detect.track_video(video_path, output_dir=args.output_path, \n save_result=args.save, display=args.display,\n filter_classes=filter_classes)\n \n for bbox_details, frame_details in track:\n bbox_xyxy, ids, scores, class_ids = bbox_details\n frame, frame_num, fps = frame_details\n\n\nif __name__=='__main__':\n \n parser = argparse.ArgumentParser()\n parser.add_argument(\"video\", help=\"Path of video\")\n parser.add_argument('--cpu', default=True, action='store_false', dest='use_cuda', help='If provided the model will run on cpu otherwise it will run on gpu')\n parser.add_argument('--filter_classes', default=None, help='Class names seperated by comma (,). e.g. person,car ')\n parser.add_argument('-w', '--weights', default=None, help='Path of trained weights')\n parser.add_argument('-o', '--output_path', default='data/results', help='path of output file')\n parser.add_argument('--no_display', action='store_false', default=True, dest='display', help='if provided video will not be displayed')\n parser.add_argument('--no_save', action='store_false', default=True, dest='save', help='if provided video will not be saved')\n\n args = parser.parse_args()\n main(args)" }, { "path": "asone/detectors/__init__.py", "content": "from asone.detectors.yolov5 import YOLOv5Detector\nfrom asone.detectors.yolov6 import YOLOv6Detector\nfrom asone.detectors.yolov7 import YOLOv7Detector\nfrom asone.detectors.yolov9 import YOLOv9Detector\nfrom asone.detectors.yolor import YOLOrDetector\nfrom asone.detectors.yolox import YOLOxDetector\nfrom asone.detectors.easyocr_detector import TextDetector\n\nfrom asone.detectors.detector import Detector\n__all__ = ['Detector'\n 'YOLOv5Detector',\n 'YOLOv6Detector',\n 'YOLOv7Detector',\n 'YOLOv9Detector',\n 'YOLOrDetector',\n 'YOLOxDetector',\n 'TextDetector',\n 'YOLOnasDetector']\n" }, { "path": "asone/detectors/detector.py", "content": "import cv2\n\nfrom asone.detectors.utils.weights_path import get_weight_path\nfrom asone.detectors.utils.cfg_path import get_cfg_path\nfrom asone.detectors.utils.exp_name import get_exp__name\n\nclass Detector:\n def __init__(self,\n model_flag: int,\n weights: str = None,\n use_cuda: bool = True,\n recognizer:int = None,\n num_classes=80):\n \n self.model = self._select_detector(model_flag, weights, use_cuda, recognizer, num_classes)\n def _select_detector(self, model_flag, weights, cuda, recognizer, num_classes):\n # Get required weight using model_flag\n mlmodel = False\n if weights and weights.split('.')[-1] == 'onnx':\n onnx = True\n weight = weights\n elif weights and weights.split('.')[-1] == 'mlmodel':\n onnx = False\n weight = weights\n mlmodel = True \n elif weights:\n onnx = False\n weight = weights\n else:\n mlmodel, onnx, weight = get_weight_path(model_flag)\n \n if model_flag in range(0, 20) or model_flag in range(120, 131):\n from asone.detectors.yolov5 import YOLOv5Detector\n _detector = YOLOv5Detector(weights=weight,\n use_onnx=onnx,\n mlmodel=mlmodel,\n use_cuda=cuda)\n elif model_flag in range(20, 34):\n from asone.detectors.yolov6 import YOLOv6Detector\n _detector = YOLOv6Detector(weights=weight,\n use_onnx=onnx,\n use_cuda=cuda)\n elif model_flag in range(34, 48) or model_flag in range(131, 139):\n from asone.detectors.yolov7 import YOLOv7Detector\n # Get exp file and corresponding model for coreml only\n _detector = YOLOv7Detector(weights=weight,\n use_onnx=onnx,\n mlmodel=mlmodel,\n use_cuda=cuda)\n elif model_flag in range(48, 58):\n from asone.detectors.yolor import YOLOrDetector\n # Get Configuration file for Yolor\n if model_flag in range(48, 57, 2):\n cfg = get_cfg_path(model_flag)\n else:\n cfg = None\n _detector = YOLOrDetector(weights=weight,\n cfg=cfg,\n use_onnx=onnx,\n use_cuda=cuda)\n\n elif model_flag in range(58, 72):\n from asone.detectors.yolox import YOLOxDetector\n # Get exp file and corresponding model for pytorch only\n if model_flag in range(58, 71, 2):\n exp, model_name = get_exp__name(model_flag)\n else:\n exp = model_name = None\n _detector = YOLOxDetector(model_name=model_name,\n exp_file=exp,\n weights=weight,\n use_onnx=onnx,\n use_cuda=cuda)\n elif model_flag in range(72, 82) or model_flag in range(139, 144):\n from asone.detectors.yolov8 import YOLOv8Detector\n # Get exp file and corresponding model for pytorch only\n _detector = YOLOv8Detector(weights=weight,\n use_onnx=onnx,\n mlmodel=mlmodel,\n use_cuda=cuda)\n # Get TextDetector model\n elif model_flag in range(82, 85):\n from asone.detectors.easyocr_detector import TextDetector\n _detector = TextDetector(detect_network=weight, use_cuda=cuda)\n\n elif model_flag in range(160, 163):\n from asone.detectors.yolonas import YOLOnasDetector\n # Get exp file and corresponding model for coreml only\n _detector = YOLOnasDetector(\n model_flag,\n weights=weight,\n use_onnx=onnx,\n use_cuda=cuda,\n num_classes=num_classes)\n \n elif model_flag in range(164, 170):\n from asone.detectors.yolov9 import YOLOv9Detector\n # Get exp file and corresponding model for pytorch only\n _detector = YOLOv9Detector(\n weights=weight,\n use_onnx=onnx,\n use_cuda=cuda)\n \n return _detector\n\n def get_detector(self):\n return self.model\n\n def detect(self,\n image: list,\n return_image=False,\n **kwargs: dict):\n return self.model.detect(image,return_image,**kwargs)\n\n\nif __name__ == '__main__':\n\n # Initialize YOLOv6 object detector\n model_type = 56\n result = Detector(model_flag=model_type, use_cuda=True)\n img = cv2.imread('asone/asone-linux/test.jpeg')\n pred = result.get_detector(img)\n print(pred)\n" }, { "path": "asone/detectors/easyocr_detector/__init__.py", "content": "from .text_detector import TextDetector\n__all__ = ['TextDetector']" }, { "path": "asone/detectors/easyocr_detector/text_detector.py", "content": "import easyocr\nimport numpy as np\n\n\nclass TextDetector:\n def __init__(self, detect_network, languages: list = ['en'], use_cuda=True):\n self.use_cuda = use_cuda\n self.detect_network = detect_network\n self.reader = easyocr.Reader(languages, detect_network=self.detect_network ,gpu=self.use_cuda)\n \n def detect(self, image: list, freelist: bool=False, return_image=False, **config) -> list:\n \"\"\"_summary_\n Args:\n image : Image \n languages (list, optional): List of languages. Defaults to ['en'].\n Returns:\n list: numpy array of extracted text and img info(heigh, width)\n \"\"\"\n \n h, w = image.shape[0:2]\n horizontal_list, free_list = self.reader.detect(image) \n\n if horizontal_list[0] == [] and free_list[0] == []:\n if return_image:\n return horizontal_list, image\n return np.empty((0, 6)), {'width': w, 'height': h}\n \n if freelist:\n return horizontal_list, free_list, {'width': w, 'height': h}\n \n x_list = []\n y_list = []\n new_points = []\n if free_list[0] != []:\n bbox_list = np.array(free_list[0]).astype(int)\n xmin= bbox_list[:, :, 0].min(axis=1, keepdims=True)\n xmax= bbox_list[:, :, 0].max(axis=1, keepdims=True)\n ymin= bbox_list[:, :, 1].min(axis=1, keepdims=True)\n ymax= bbox_list[:, :, 1].max(axis=1, keepdims=True)\n new_points = np.hstack((xmin, xmax, ymin, ymax)).tolist()\n\n if len(horizontal_list[0]) < 1:\n horizontal_list = [new_points]\n else:\n horizontal_list = [horizontal_list[0] + new_points]\n\n horizontal_list = np.array(horizontal_list[0])\n horizontal_list[:, [1, 2]] = horizontal_list[:, [2, 1]]\n horizontal_list = np.hstack((horizontal_list, np.array([[0.7, 80]]*len(horizontal_list))))\n \n if return_image:\n return horizontal_list, image\n \n return horizontal_list, {'width': w, 'height': h}\n" }, { "path": "asone/detectors/utils/__init__.py", "content": "" }, { "path": "asone/detectors/utils/cfg_path.py", "content": "import os\n\ncfg_dir = os.path.dirname(os.path.dirname(__file__))\n\nconfiguration = {'0': os.path.join(cfg_dir, 'yolor','cfg','yolor_csp_x.cfg'),\n '1': os.path.join(cfg_dir, 'yolor','cfg','yolor_csp.cfg'),\n '2': os.path.join(cfg_dir, 'yolor','cfg','yolor_p6.cfg')}\n\ndef get_cfg_path(model_flag):\n if model_flag in [48,50]:\n cfg = configuration['0']\n if model_flag in [52,54]:\n cfg = configuration['1']\n if model_flag == 56:\n cfg = configuration['2']\n return cfg\n \n " }, { "path": "asone/detectors/utils/coreml_utils.py", "content": "import numpy as np\n\n\ndef yolo_to_xyxy(bboxes, img_size):\n w, h = img_size\n \n bboxes = bboxes[:, 0:]\n bboxes[:, 0] = bboxes[:, 0]*w\n bboxes[:, 1] = bboxes[:, 1]*h\n bboxes[:, 2] = bboxes[:, 2]*w\n bboxes[:, 3] = bboxes[:, 3]*h\n\n bboxes[:, 0] = bboxes[:, 0] - bboxes[:, 2]/2\n bboxes[:, 1] = bboxes[:, 1] - bboxes[:, 3]/2\n bboxes[:, 2] = bboxes[:, 0] + bboxes[:, 2]\n bboxes[:, 3] = bboxes[:, 1] + bboxes[:, 3]\n \n return bboxes.astype(int)\n\ndef generalize_output_format(bboxes, confidence_list, conf_thres):\n \"\"\"_summary_\n\n Args:\n bboxes : Bounding boxes in xyxy format\n confidence_list : List containing confidence score of each class\n conf_thres : confidence_threshold\n\n Returns:\n np.array: Array of format [Xmin, Ymin, Xmax, Ymax, confidence, class_id]\n \"\"\"\n \n class_ids = np.argmax(confidence_list, axis=1)\n conf_scr = []\n output = []\n for i, confidence in enumerate(confidence_list):\n if conf_thres < confidence[class_ids[i]]:\n conf_scr = confidence[class_ids[i]]\n res = np.append(np.append(bboxes[i], conf_scr), class_ids[i])\n output.append(res)\n return np.array(output)\n\ndef scale_bboxes(bboxes, org_img_shape, resized_img_shape):\n # Rescaling Bounding Boxes \n bboxes[:, :4] /= np.array([resized_img_shape[1], resized_img_shape[0], resized_img_shape[1], resized_img_shape[0]])\n bboxes[:, :4] *= np.array([org_img_shape[1], org_img_shape[0], org_img_shape[1], org_img_shape[0]]) \n\n return bboxes" }, { "path": "asone/detectors/utils/exp_name.py", "content": "import os\n\nexp_dir = os.path.dirname(os.path.dirname(__file__))\n\nexp_file_name = {'58': (os.path.join(exp_dir, 'yolox','exps','yolox_l.py'),'yolox-l'),\n '60': (os.path.join(exp_dir, 'yolox','exps','yolox_nano.py'),'yolox-nano'),\n '62': (os.path.join(exp_dir, 'yolox','exps','yolox_tiny'),'yolox-tiny'),\n '64': (os.path.join(exp_dir, 'yolox','exps','yolov3.py'),'yolox-darknet'),\n '66': (os.path.join(exp_dir, 'yolox','exps','yolox_s.py'),'yolox-s'),\n '68': (os.path.join(exp_dir, 'yolox','exps','yolox_m.py'),'yolox-m'),\n '70': (os.path.join(exp_dir, 'yolox','exps','yolox_x.py'),'yolox-x')\n }\n\n\ndef get_exp__name(model_flag):\n \n if model_flag == 58:\n exp, model_name = exp_file_name['58'][0], exp_file_name['58'][1]\n elif model_flag == 60:\n exp, model_name = exp_file_name['60'][0], exp_file_name['60'][1]\n elif model_flag == 62:\n exp, model_name = exp_file_name['62'][0], exp_file_name['62'][1]\n elif model_flag == 64:\n exp, model_name = exp_file_name['64'][0], exp_file_name['64'][1]\n elif model_flag == 66:\n exp, model_name = exp_file_name['66'][0], exp_file_name['66'][1]\n elif model_flag == 68:\n exp, model_name = exp_file_name['68'][0], exp_file_name['68'][1] \n elif model_flag == 70:\n exp, model_name = exp_file_name['70'][0], exp_file_name['70'][1] \n\n return exp, model_name " }, { "path": "asone/detectors/utils/weights_path.py", "content": "import os\n\nweights = { '0': os.path.join('yolov5','weights','yolov5x6.pt'),\n '1': os.path.join('yolov5','weights','yolov5x6.onnx'),\n '2': os.path.join('yolov5','weights','yolov5s.pt'),\n '3': os.path.join('yolov5','weights','yolov5s.onnx'),\n '4': os.path.join('yolov5','weights','yolov5n.pt'),\n '5': os.path.join('yolov5','weights','yolov5n.onnx'),\n '6': os.path.join('yolov5','weights','yolov5m.pt'),\n '7': os.path.join('yolov5','weights','yolov5m.onnx'),\n '8': os.path.join('yolov5','weights','yolov5l.pt'),\n '9': os.path.join('yolov5','weights','yolov5l.onnx'),\n '10': os.path.join('yolov5','weights','yolov5x.pt'),\n '11': os.path.join('yolov5','weights','yolov5x.onnx'),\n '12': os.path.join('yolov5','weights','yolov5n6.pt'),\n '13': os.path.join('yolov5','weights','yolov5n6.onnx'),\n '14': os.path.join('yolov5','weights','yolov5s6.pt'),\n '15': os.path.join('yolov5','weights','yolov5s6.onnx'),\n '16': os.path.join('yolov5','weights','yolov5m6.pt'),\n '17': os.path.join('yolov5','weights','yolov5m6.onnx'),\n '18': os.path.join('yolov5','weights','yolov5l6.pt'),\n '19': os.path.join('yolov5','weights','yolov5l6.onnx'),\n \n '120': os.path.join('yolov5','weights','yolov5n.mlmodel'),\n '121': os.path.join('yolov5','weights','yolov5s.mlmodel'),\n '122': os.path.join('yolov5','weights','yolov5x6.mlmodel'),\n '123': os.path.join('yolov5','weights','yolov5m.mlmodel'),\n '124': os.path.join('yolov5','weights','yolov5l.mlmodel'),\n '125': os.path.join('yolov5','weights','yolov5x.mlmodel'),\n '126': os.path.join('yolov5','weights','yolov5n6.mlmodel'),\n '127': os.path.join('yolov5','weights','yolov5s6.mlmodel'),\n '128': os.path.join('yolov5','weights','yolov5m6.mlmodel'),\n '129': os.path.join('yolov5','weights','yolov5l6.mlmodel'),\n \n # YOLOv6\n '20': os.path.join('yolov6','weights','yolov6n.pt'),\n '21': os.path.join('yolov6','weights','yolov6n.onnx'),\n '22': os.path.join('yolov6','weights','yolov6t.pt'),\n '23': os.path.join('yolov6','weights','yolov6t.onnx'),\n '24': os.path.join('yolov6','weights','yolov6s.pt'),\n '25': os.path.join('yolov6','weights','yolov6s.onnx'),\n '26': os.path.join('yolov6','weights','yolov6m.pt'),\n '27': os.path.join('yolov6','weights','yolov6m.onnx'),\n '28': os.path.join('yolov6','weights','yolov6l.pt'),\n '29': os.path.join('yolov6','weights','yolov6l.onnx'),\n '30': os.path.join('yolov6','weights','yolov6l_relu.pt'),\n '31': os.path.join('yolov6','weights','yolov6l_relu.onnx'),\n '32': os.path.join('yolov6','weights','yolov6s_repopt.pt'),\n '33': os.path.join('yolov6','weights','yolov6s_repopt.onnx'),\n # YOLOv7\n '34': os.path.join('yolov7','weights','yolov7-tiny.pt'),\n '35': os.path.join('yolov7','weights','yolov7-tiny.onnx'),\n '36': os.path.join('yolov7','weights','yolov7.pt'),\n '37': os.path.join('yolov7','weights','yolov7.onnx'),\n '38': os.path.join('yolov7','weights','yolov7x.pt'),\n '39': os.path.join('yolov7','weights','yolov7x.onnx'),\n '40': os.path.join('yolov7','weights','yolov7-w6.pt'),\n '41': os.path.join('yolov7','weights','yolov7-w6.onnx'),\n '42': os.path.join('yolov7','weights','yolov7-e6.pt'),\n '43': os.path.join('yolov7','weights','yolov7-e6.onnx'),\n '44': os.path.join('yolov7','weights','yolov7-d6.pt'),\n '45': os.path.join('yolov7','weights','yolov7-d6.onnx'),\n '46': os.path.join('yolov7','weights','yolov7-e6e.pt'),\n '47': os.path.join('yolov7','weights','yolov7-e6e.onnx'),\n \n '130': os.path.join('yolov7','weights','yolov7-tiny.mlmodel'),\n '131': os.path.join('yolov7','weights','yolov7.mlmodel'),\n '132': os.path.join('yolov7','weights','yolov7x.mlmodel'),\n '133': os.path.join('yolov7','weights','yolov7-w6.mlmodel'),\n '134': os.path.join('yolov7','weights','yolov7-e6.mlmodel'),\n '135': os.path.join('yolov7','weights','yolov7-d6.mlmodel'),\n '136': os.path.join('yolov7','weights','yolov7-e6e.mlmodel'),\n # YOLOR\n '48': os.path.join('yolor','weights','yolor_csp_x.pt'),\n '49': os.path.join('yolor','weights','yolor_csp_x.onnx'),\n '50': os.path.join('yolor','weights','yolor_csp_x_star.pt'),\n '51': os.path.join('yolor','weights','yolor_csp_x_star.onnx'),\n '52': os.path.join('yolor','weights','yolor_csp_star.pt'),\n '53': os.path.join('yolor','weights','yolor_csp_star.onnx'),\n '54': os.path.join('yolor','weights','yolor_csp.pt'),\n '55': os.path.join('yolor','weights','yolor_csp.onnx'),\n '56': os.path.join('yolor','weights','yolor_p6.pt'),\n '57': os.path.join('yolor','weights','yolor_p6.onnx'),\n # YOLOX\n '58': os.path.join('yolox','weights','yolox_l.pth'),\n '59': os.path.join('yolox','weights','yolox_l.onnx'),\n '60': os.path.join('yolox','weights','yolox_nano.pth'),\n '61': os.path.join('yolox','weights','yolox_nano.onnx'),\n '62': os.path.join('yolox','weights','yolox_tiny.pth'),\n '63': os.path.join('yolox','weights','yolox_tiny.onnx'),\n '64': os.path.join('yolox','weights','yolox_darknet.pth'),\n '65': os.path.join('yolox','weights','yolox_darknet.onnx'),\n '66': os.path.join('yolox','weights','yolox_s.pth'),\n '67': os.path.join('yolox','weights','yolox_s.onnx'),\n '68': os.path.join('yolox','weights','yolox_m.pth'),\n '69': os.path.join('yolox','weights','yolox_m.onnx'),\n '70': os.path.join('yolox','weights','yolox_x.pth'),\n '71': os.path.join('yolox','weights','yolox_x.onnx'),\n # YOLOv8\n '72': os.path.join('yolov8','weights','yolov8n.pt'),\n '73': os.path.join('yolov8','weights','yolov8n.onnx'),\n '74': os.path.join('yolov8','weights','yolov8s.pt'),\n '75': os.path.join('yolov8','weights','yolov8s.onnx'),\n '76': os.path.join('yolov8','weights','yolov8m.pt'),\n '77': os.path.join('yolov8','weights','yolov8m.onnx'),\n '78': os.path.join('yolov8','weights','yolov8l.pt'),\n '79': os.path.join('yolov8','weights','yolov8l.onnx'),\n '80': os.path.join('yolov8','weights','yolov8x.pt'),\n '81': os.path.join('yolov8','weights','yolov8x.onnx'),\n '139': os.path.join('yolov8','weights','yolov8n.mlmodel'),\n '140': os.path.join('yolov8','weights','yolov8s.mlmodel'),\n '141': os.path.join('yolov8','weights','yolov8m.mlmodel'),\n '142': os.path.join('yolov8','weights','yolov8l.mlmodel'),\n '143': os.path.join('yolov8','weights','yolov8x.mlmodel'),\n \n # Text Detectors\n '82': 'craft',\n '83': 'dbnet18',\n # YOLONAS_S_PYTORCH\n # YOLO NAS\n '160':os.path.join('yolonas','weights','yolo_nas_s.pth'),\n '161':os.path.join('yolonas','weights','yolo_nas_m.pth'),\n '162':os.path.join('yolonas','weights','yolo_nas_l.pth'),\n \n # YOLOv9\n '164':os.path.join('yolov9','weights','yolov9-c-converted.pt'),\n '165':os.path.join('yolov9','weights','yolov9-e-converted.pt'),\n '166':os.path.join('yolov9','weights','yolov9-c.pt'),\n '167':os.path.join('yolov9','weights','yolov9-e.pt'),\n '168':os.path.join('yolov9','weights','gelan-c.pt'),\n '169':os.path.join('yolov9','weights','gelan-e.pt'),\n \n \n # Segmentor\n '171':os.path.join('sam','weights','sam_vit_h_4b8939.pth'),\n}\n\ndef get_weight_path(model_flag):\n coreml= False\n if model_flag in range(0, 20):\n onnx = False if (model_flag % 2 == 0) else True\n weight = weights[str(model_flag)]\n elif model_flag in range(20, 34):\n onnx = False if (model_flag % 2 == 0) else True\n weight = weights[str(model_flag)]\n elif model_flag in range(34, 48):\n onnx = False if (model_flag % 2 == 0) else True\n weight = weights[str(model_flag)]\n elif model_flag in range(48, 58):\n onnx = False if (model_flag % 2 == 0) else True\n weight = weights[str(model_flag)]\n elif model_flag in range(58, 72):\n onnx = False if (model_flag % 2 == 0) else True\n weight = weights[str(model_flag)]\n elif model_flag in range(72, 82):\n onnx = False if (model_flag % 2 == 0) else True\n weight = weights[str(model_flag)]\n elif model_flag in range(82, 85):\n onnx = False\n weight = weights[str(model_flag)]\n \n elif model_flag in range(120, 130):\n weight = weights[str(model_flag)]\n onnx=False\n coreml = True\n elif model_flag in range(130, 137):\n weight = weights[str(model_flag)]\n onnx=False\n coreml = True \n elif model_flag in range(139, 145):\n weight = weights[str(model_flag)]\n onnx=False\n coreml = True\n elif model_flag in range(160, 163):\n weight = weights[str(model_flag)]\n onnx=False\n coreml = True\n elif model_flag in range(164, 170):\n onnx = False\n weight = weights[str(model_flag)]\n\n return coreml, onnx, weight\n \n" }, { "path": "asone/detectors/yolonas/__init__.py", "content": "from .yolonas import YOLOnasDetector\n__all__ = ['YOLOnasDetector']" }, { "path": "asone/detectors/yolonas/yolonas.py", "content": "import os\nfrom asone.utils import get_names\nimport numpy as np\nimport warnings\nimport torch\nimport onnxruntime\nfrom asone import utils\nimport super_gradients\nimport numpy as np\nfrom super_gradients.training.processing import DetectionCenterPadding, StandardizeImage, NormalizeImage, ImagePermute, ComposeProcessing, DetectionLongestMaxSizeRescale\nfrom super_gradients.training import models\nfrom super_gradients.common.object_names import Models\n\nfrom asone.utils.utils import PathResolver\n\n\nclass_names = [\"\"]\n\n\nclass YOLOnasDetector:\n def __init__(self,\n model_flag,\n weights=None,\n cfg=None,\n use_onnx=True,\n use_cuda=True,\n # checkpoint_num_classes=80,\n num_classes=80\n ):\n \n \n self.model_flag = model_flag\n # self.checkpoint_num_classes = checkpoint_num_classes\n if not os.path.exists(weights):\n utils.download_weights(weights)\n \n self.num_classes = num_classes\n self.device = 'cuda' if use_cuda and torch.cuda.is_available() else 'cpu'\n self.use_onnx = use_onnx\n\n with PathResolver():\n # Load Model\n self.model = self.load_model(weights=weights)\n\n def load_model(self, weights):\n \n # model = super_gradients.training.models.get(name, \n # checkpoint_path=weights, \n # checkpoint_num_classes=self.checkpoint_num_classes,\n # num_classes=self.num_classes).to(self.device)\n \n if self.model_flag == 160: \n model = models.get(Models.YOLO_NAS_S,\n checkpoint_path=weights,\n num_classes=self.num_classes).to(self.device)\n elif self.model_flag == 161:\n model = models.get(Models.YOLO_NAS_M,\n checkpoint_path=weights,\n num_classes=self.num_classes).to(self.device)\n elif self.model_flag == 162:\n model = models.get(Models.YOLO_NAS_L,\n checkpoint_path=weights,\n num_classes=self.num_classes).to(self.device)\n return model\n \n def detect(self, image: list,\n input_shape: tuple = (640, 640),\n conf_thres: float = 0.25,\n iou_thres: float = 0.45,\n max_det: int = 1000,\n filter_classes: bool = None,\n agnostic_nms: bool = True,\n with_p6: bool = False,\n return_image=False) -> list:\n\n if self.num_classes==80:\n self.model.set_dataset_processing_params(class_names=class_names,\n image_processor=ComposeProcessing(\n [\n DetectionLongestMaxSizeRescale(output_shape=(636, 636)),\n DetectionCenterPadding(output_shape=(640, 640), pad_value=114),\n StandardizeImage(max_value=255.0),\n ImagePermute(permutation=(2, 0, 1)),\n ]\n ),\n iou=iou_thres,conf=conf_thres,\n )\n original_image = image\n # Inference\n if self.use_onnx:\n pass\n \n else:\n\n detections = self.model.predict(image)\n image_info = {\n 'width': original_image.shape[1],\n 'height': original_image.shape[0],\n }\n detections = list(detections) \n pred = detections[0].prediction\n bboxes_xyxy = pred.bboxes_xyxy\n confidence = pred.confidence\n labels = pred.labels\n\n confidence = confidence.reshape(-1,1)\n labels = labels.reshape(-1,1)\n arr = np.append(bboxes_xyxy, confidence, axis=1)\n predictions = np.append(arr, labels, axis=1)\n \n\n if return_image:\n return predictions, original_image\n else: \n return predictions, image_info\n \n" }, { "path": "asone/detectors/yolor/__init__.py", "content": "from .yolor_detector import YOLOrDetector\n__all__ = ['YOLOrDetector']" }, { "path": "asone/detectors/yolor/cfg/yolor_csp.cfg", "content": "[net]\n# Testing\n#batch=1\n#subdivisions=1\n# Training\nbatch=64\nsubdivisions=8\nwidth=512\nheight=512\nchannels=3\nmomentum=0.949\ndecay=0.0005\nangle=0\nsaturation = 1.5\nexposure = 1.5\nhue=.1\n\nlearning_rate=0.00261\nburn_in=1000\nmax_batches = 500500\npolicy=steps\nsteps=400000,450000\nscales=.1,.1\n\n#cutmix=1\nmosaic=1\n\n\n# ============ Backbone ============ #\n\n# Stem \n\n# 0\n[convolutional]\nbatch_normalize=1\nfilters=32\nsize=3\nstride=1\npad=1\nactivation=silu\n\n# P1\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=32\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=3\nstride=1\npad=1\nactivation=silu\n\n# 4 (previous+1+3k)\n[shortcut]\nfrom=-3\nactivation=linear\n\n# P2\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Merge [-1, -(3k+4)]\n\n[route]\nlayers = -1,-10\n\n# Transition last\n\n# 17 (previous+7+3k)\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# P3\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Merge [-1 -(4+3k)]\n\n[route]\nlayers = -1,-28\n\n# Transition last\n\n# 48 (previous+7+3k)\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# P4\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Merge [-1 -(3k+4)]\n\n[route]\nlayers = -1,-28\n\n# Transition last\n\n# 79 (previous+7+3k)\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# P5\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=1024\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Merge [-1 -(3k+4)]\n\n[route]\nlayers = -1,-16\n\n# Transition last\n\n# 98 (previous+7+3k)\n[convolutional]\nbatch_normalize=1\nfilters=1024\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# ============ End of Backbone ============ #\n\n# ============ Neck ============ #\n\n# CSPSPP\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=512\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n### SPP ###\n[maxpool]\nstride=1\nsize=5\n\n[route]\nlayers=-2\n\n[maxpool]\nstride=1\nsize=9\n\n[route]\nlayers=-4\n\n[maxpool]\nstride=1\nsize=13\n\n[route]\nlayers=-1,-3,-5,-6\n### End SPP ###\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=512\nactivation=silu\n\n[route]\nlayers = -1, -13\n\n# 113 (previous+6+5+2k)\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# End of CSPSPP\n\n\n# FPN-4\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[upsample]\nstride=2\n\n[route]\nlayers = 79\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -1, -3\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n# Merge [-1, -(2k+2)]\n\n[route]\nlayers = -1, -6\n\n# Transition last\n\n# 127 (previous+6+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# FPN-3\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[upsample]\nstride=2\n\n[route]\nlayers = 48\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -1, -3\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=128\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=128\nactivation=silu\n\n# Merge [-1, -(2k+2)]\n\n[route]\nlayers = -1, -6\n\n# Transition last\n\n# 141 (previous+6+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# PAN-4\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=2\npad=1\nfilters=256\nactivation=silu\n\n[route]\nlayers = -1, 127\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n[route]\nlayers = -1,-6\n\n# Transition last\n\n# 152 (previous+3+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# PAN-5\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=2\npad=1\nfilters=512\nactivation=silu\n\n[route]\nlayers = -1, 113\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=512\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=512\nactivation=silu\n\n[route]\nlayers = -1,-6\n\n# Transition last\n\n# 163 (previous+3+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# ============ End of Neck ============ #\n\n# 164\n[implicit_add]\nfilters=256\n\n# 165\n[implicit_add]\nfilters=512\n\n# 166\n[implicit_add]\nfilters=1024\n\n# 167\n[implicit_mul]\nfilters=255\n\n# 168\n[implicit_mul]\nfilters=255\n\n# 169\n[implicit_mul]\nfilters=255\n\n# ============ Head ============ #\n\n# YOLO-3\n\n[route]\nlayers = 141\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n[shift_channels]\nfrom=164\n\n[convolutional]\nsize=1\nstride=1\npad=1\nfilters=255\nactivation=linear\n\n[control_channels]\nfrom=167\n\n[yolo]\nmask = 0,1,2\nanchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401\nclasses=80\nnum=9\njitter=.3\nignore_thresh = .7\ntruth_thresh = 1\nrandom=1\nscale_x_y = 1.05\niou_thresh=0.213\ncls_normalizer=1.0\niou_normalizer=0.07\niou_loss=ciou\nnms_kind=greedynms\nbeta_nms=0.6\n\n\n# YOLO-4\n\n[route]\nlayers = 152\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=512\nactivation=silu\n\n[shift_channels]\nfrom=165\n\n[convolutional]\nsize=1\nstride=1\npad=1\nfilters=255\nactivation=linear\n\n[control_channels]\nfrom=168\n\n[yolo]\nmask = 3,4,5\nanchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401\nclasses=80\nnum=9\njitter=.3\nignore_thresh = .7\ntruth_thresh = 1\nrandom=1\nscale_x_y = 1.05\niou_thresh=0.213\ncls_normalizer=1.0\niou_normalizer=0.07\niou_loss=ciou\nnms_kind=greedynms\nbeta_nms=0.6\n\n\n# YOLO-5\n\n[route]\nlayers = 163\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=1024\nactivation=silu\n\n[shift_channels]\nfrom=166\n\n[convolutional]\nsize=1\nstride=1\npad=1\nfilters=255\nactivation=linear\n\n[control_channels]\nfrom=169\n\n[yolo]\nmask = 6,7,8\nanchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401\nclasses=80\nnum=9\njitter=.3\nignore_thresh = .7\ntruth_thresh = 1\nrandom=1\nscale_x_y = 1.05\niou_thresh=0.213\ncls_normalizer=1.0\niou_normalizer=0.07\niou_loss=ciou\nnms_kind=greedynms\nbeta_nms=0.6\n" }, { "path": "asone/detectors/yolor/cfg/yolor_csp_x.cfg", "content": "[net]\n# Testing\n#batch=1\n#subdivisions=1\n# Training\nbatch=64\nsubdivisions=8\nwidth=512\nheight=512\nchannels=3\nmomentum=0.949\ndecay=0.0005\nangle=0\nsaturation = 1.5\nexposure = 1.5\nhue=.1\n\nlearning_rate=0.00261\nburn_in=1000\nmax_batches = 500500\npolicy=steps\nsteps=400000,450000\nscales=.1,.1\n\n#cutmix=1\nmosaic=1\n\n\n# ============ Backbone ============ #\n\n# Stem \n\n# 0\n[convolutional]\nbatch_normalize=1\nfilters=32\nsize=3\nstride=1\npad=1\nactivation=silu\n\n# P1\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=40\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=3\nstride=1\npad=1\nactivation=silu\n\n# 4 (previous+1+3k)\n[shortcut]\nfrom=-3\nactivation=linear\n\n# P2\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n\n[convolutional]\nbatch_normalize=1\nfilters=80\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Merge [-1, -(3k+4)]\n\n[route]\nlayers = -1,-13\n\n# Transition last\n\n# 20 (previous+7+3k)\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# P3\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Merge [-1 -(4+3k)]\n\n[route]\nlayers = -1,-34\n\n# Transition last\n\n# 57 (previous+7+3k)\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# P4\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Merge [-1 -(3k+4)]\n\n[route]\nlayers = -1,-34\n\n# Transition last\n\n# 94 (previous+7+3k)\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# P5\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=1280\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Merge [-1 -(3k+4)]\n\n[route]\nlayers = -1,-19\n\n# Transition last\n\n# 116 (previous+7+3k)\n[convolutional]\nbatch_normalize=1\nfilters=1280\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# ============ End of Backbone ============ #\n\n# ============ Neck ============ #\n\n# CSPSPP\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=640\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n### SPP ###\n[maxpool]\nstride=1\nsize=5\n\n[route]\nlayers=-2\n\n[maxpool]\nstride=1\nsize=9\n\n[route]\nlayers=-4\n\n[maxpool]\nstride=1\nsize=13\n\n[route]\nlayers=-1,-3,-5,-6\n### End SPP ###\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=640\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=640\nactivation=silu\n\n[route]\nlayers = -1, -15\n\n# 133 (previous+6+5+2k)\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# End of CSPSPP\n\n\n# FPN-4\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[upsample]\nstride=2\n\n[route]\nlayers = 94\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -1, -3\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n# Merge [-1, -(2k+2)]\n\n[route]\nlayers = -1, -8\n\n# Transition last\n\n# 149 (previous+6+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# FPN-3\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[upsample]\nstride=2\n\n[route]\nlayers = 57\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -1, -3\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=160\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=160\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=160\nactivation=silu\n\n# Merge [-1, -(2k+2)]\n\n[route]\nlayers = -1, -8\n\n# Transition last\n\n# 165 (previous+6+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=160\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# PAN-4\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=2\npad=1\nfilters=320\nactivation=silu\n\n[route]\nlayers = -1, 149\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[route]\nlayers = -1,-8\n\n# Transition last\n\n# 178 (previous+3+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# PAN-5\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=2\npad=1\nfilters=640\nactivation=silu\n\n[route]\nlayers = -1, 133\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=640\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=640\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=640\nactivation=silu\n\n[route]\nlayers = -1,-8\n\n# Transition last\n\n# 191 (previous+3+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# ============ End of Neck ============ #\n\n# 192\n[implicit_add]\nfilters=320\n\n# 193\n[implicit_add]\nfilters=640\n\n# 194\n[implicit_add]\nfilters=1280\n\n# 195\n[implicit_mul]\nfilters=255\n\n# 196\n[implicit_mul]\nfilters=255\n\n# 197\n[implicit_mul]\nfilters=255\n\n# ============ Head ============ #\n\n# YOLO-3\n\n[route]\nlayers = 165\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[shift_channels]\nfrom=192\n\n[convolutional]\nsize=1\nstride=1\npad=1\nfilters=255\nactivation=linear\n\n[control_channels]\nfrom=195\n\n[yolo]\nmask = 0,1,2\nanchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401\nclasses=80\nnum=9\njitter=.3\nignore_thresh = .7\ntruth_thresh = 1\nrandom=1\nscale_x_y = 1.05\niou_thresh=0.213\ncls_normalizer=1.0\niou_normalizer=0.07\niou_loss=ciou\nnms_kind=greedynms\nbeta_nms=0.6\n\n\n# YOLO-4\n\n[route]\nlayers = 178\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=640\nactivation=silu\n\n[shift_channels]\nfrom=193\n\n[convolutional]\nsize=1\nstride=1\npad=1\nfilters=255\nactivation=linear\n\n[control_channels]\nfrom=196\n\n[yolo]\nmask = 3,4,5\nanchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401\nclasses=80\nnum=9\njitter=.3\nignore_thresh = .7\ntruth_thresh = 1\nrandom=1\nscale_x_y = 1.05\niou_thresh=0.213\ncls_normalizer=1.0\niou_normalizer=0.07\niou_loss=ciou\nnms_kind=greedynms\nbeta_nms=0.6\n\n\n# YOLO-5\n\n[route]\nlayers = 191\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=1280\nactivation=silu\n\n[shift_channels]\nfrom=194\n\n[convolutional]\nsize=1\nstride=1\npad=1\nfilters=255\nactivation=linear\n\n[control_channels]\nfrom=197\n\n[yolo]\nmask = 6,7,8\nanchors = 12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401\nclasses=80\nnum=9\njitter=.3\nignore_thresh = .7\ntruth_thresh = 1\nrandom=1\nscale_x_y = 1.05\niou_thresh=0.213\ncls_normalizer=1.0\niou_normalizer=0.07\niou_loss=ciou\nnms_kind=greedynms\nbeta_nms=0.6\n" }, { "path": "asone/detectors/yolor/cfg/yolor_p6.cfg", "content": "[net]\nbatch=64\nsubdivisions=8\nwidth=1280\nheight=1280\nchannels=3\nmomentum=0.949\ndecay=0.0005\nangle=0\nsaturation = 1.5\nexposure = 1.5\nhue=.1\n\nlearning_rate=0.00261\nburn_in=1000\nmax_batches = 500500\npolicy=steps\nsteps=400000,450000\nscales=.1,.1\n\nmosaic=1\n\n\n# ============ Backbone ============ #\n\n# Stem \n\n# P1\n\n# Downsample\n\n# 0\n[reorg]\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=3\nstride=1\npad=1\nactivation=silu\n\n\n# P2\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=64\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n#\n#[convolutional]\n#batch_normalize=1\n#filters=64\n#size=1\n#stride=1\n#pad=1\n#activation=silu\n\n# Merge [-1, -(3k+3)]\n\n[route]\nlayers = -1,-12\n\n# Transition last\n\n# 16 (previous+6+3k)\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# P3\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n#\n#[convolutional]\n#batch_normalize=1\n#filters=128\n#size=1\n#stride=1\n#pad=1\n#activation=silu\n\n# Merge [-1, -(3k+3)]\n\n[route]\nlayers = -1,-24\n\n# Transition last\n\n# 43 (previous+6+3k)\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# P4\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=384\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n#\n#[convolutional]\n#batch_normalize=1\n#filters=192\n#size=1\n#stride=1\n#pad=1\n#activation=silu\n\n# Merge [-1, -(3k+3)]\n\n[route]\nlayers = -1,-24\n\n# Transition last\n\n# 70 (previous+6+3k)\n[convolutional]\nbatch_normalize=1\nfilters=384\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# P5\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n#\n#[convolutional]\n#batch_normalize=1\n#filters=256\n#size=1\n#stride=1\n#pad=1\n#activation=silu\n\n# Merge [-1, -(3k+3)]\n\n[route]\nlayers = -1,-12\n\n# Transition last\n\n# 85 (previous+6+3k)\n[convolutional]\nbatch_normalize=1\nfilters=512\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# P6\n\n# Downsample\n\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=3\nstride=2\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Residual Block\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=3\nstride=1\npad=1\nactivation=silu\n\n[shortcut]\nfrom=-3\nactivation=linear\n\n# Transition first\n#\n#[convolutional]\n#batch_normalize=1\n#filters=320\n#size=1\n#stride=1\n#pad=1\n#activation=silu\n\n# Merge [-1, -(3k+3)]\n\n[route]\nlayers = -1,-12\n\n# Transition last\n\n# 100 (previous+6+3k)\n[convolutional]\nbatch_normalize=1\nfilters=640\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# ============ End of Backbone ============ #\n\n# ============ Neck ============ #\n\n# CSPSPP\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n### SPP ###\n[maxpool]\nstride=1\nsize=5\n\n[route]\nlayers=-2\n\n[maxpool]\nstride=1\nsize=9\n\n[route]\nlayers=-4\n\n[maxpool]\nstride=1\nsize=13\n\n[route]\nlayers=-1,-3,-5,-6\n### End SPP ###\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[route]\nlayers = -1, -13\n\n# 115 (previous+6+5+2k)\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# End of CSPSPP\n\n\n# FPN-5\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[upsample]\nstride=2\n\n[route]\nlayers = 85\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -1, -3\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n# Merge [-1, -(2k+2)]\n\n[route]\nlayers = -1, -8\n\n# Transition last\n\n# 131 (previous+6+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# FPN-4\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[upsample]\nstride=2\n\n[route]\nlayers = 70\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -1, -3\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=192\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=192\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=192\nactivation=silu\n\n# Merge [-1, -(2k+2)]\n\n[route]\nlayers = -1, -8\n\n# Transition last\n\n# 147 (previous+6+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# FPN-3\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[upsample]\nstride=2\n\n[route]\nlayers = 43\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -1, -3\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=128\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=128\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=128\nactivation=silu\n\n# Merge [-1, -(2k+2)]\n\n[route]\nlayers = -1, -8\n\n# Transition last\n\n# 163 (previous+6+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=128\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# PAN-4\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=2\npad=1\nfilters=192\nactivation=silu\n\n[route]\nlayers = -1, 147\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=192\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=192\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=192\nactivation=silu\n\n[route]\nlayers = -1,-8\n\n# Transition last\n\n# 176 (previous+3+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=192\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# PAN-5\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=2\npad=1\nfilters=256\nactivation=silu\n\n[route]\nlayers = -1, 131\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n[route]\nlayers = -1,-8\n\n# Transition last\n\n# 189 (previous+3+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=256\nsize=1\nstride=1\npad=1\nactivation=silu\n\n\n# PAN-6\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=2\npad=1\nfilters=320\nactivation=silu\n\n[route]\nlayers = -1, 115\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# Split\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[route]\nlayers = -2\n\n# Plain Block\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=320\nactivation=silu\n\n[route]\nlayers = -1,-8\n\n# Transition last\n\n# 202 (previous+3+4+2k)\n[convolutional]\nbatch_normalize=1\nfilters=320\nsize=1\nstride=1\npad=1\nactivation=silu\n\n# ============ End of Neck ============ #\n\n# 203\n[implicit_add]\nfilters=256\n\n# 204\n[implicit_add]\nfilters=384\n\n# 205\n[implicit_add]\nfilters=512\n\n# 206\n[implicit_add]\nfilters=640\n\n# 207\n[implicit_mul]\nfilters=255\n\n# 208\n[implicit_mul]\nfilters=255\n\n# 209\n[implicit_mul]\nfilters=255\n\n# 210\n[implicit_mul]\nfilters=255\n\n# ============ Head ============ #\n\n# YOLO-3\n\n[route]\nlayers = 163\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=256\nactivation=silu\n\n[shift_channels]\nfrom=203\n\n[convolutional]\nsize=1\nstride=1\npad=1\nfilters=255\nactivation=linear\n\n[control_channels]\nfrom=207\n\n[yolo]\nmask = 0,1,2\nanchors = 19,27, 44,40, 38,94, 96,68, 86,152, 180,137, 140,301, 303,264, 238,542, 436,615, 739,380, 925,792\nclasses=80\nnum=12\njitter=.3\nignore_thresh = .7\ntruth_thresh = 1\nrandom=1\nscale_x_y = 1.05\niou_thresh=0.213\ncls_normalizer=1.0\niou_normalizer=0.07\niou_loss=ciou\nnms_kind=greedynms\nbeta_nms=0.6\n\n\n# YOLO-4\n\n[route]\nlayers = 176\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=384\nactivation=silu\n\n[shift_channels]\nfrom=204\n\n[convolutional]\nsize=1\nstride=1\npad=1\nfilters=255\nactivation=linear\n\n[control_channels]\nfrom=208\n\n[yolo]\nmask = 3,4,5\nanchors = 19,27, 44,40, 38,94, 96,68, 86,152, 180,137, 140,301, 303,264, 238,542, 436,615, 739,380, 925,792\nclasses=80\nnum=12\njitter=.3\nignore_thresh = .7\ntruth_thresh = 1\nrandom=1\nscale_x_y = 1.05\niou_thresh=0.213\ncls_normalizer=1.0\niou_normalizer=0.07\niou_loss=ciou\nnms_kind=greedynms\nbeta_nms=0.6\n\n\n# YOLO-5\n\n[route]\nlayers = 189\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=512\nactivation=silu\n\n[shift_channels]\nfrom=205\n\n[convolutional]\nsize=1\nstride=1\npad=1\nfilters=255\nactivation=linear\n\n[control_channels]\nfrom=209\n\n[yolo]\nmask = 6,7,8\nanchors = 19,27, 44,40, 38,94, 96,68, 86,152, 180,137, 140,301, 303,264, 238,542, 436,615, 739,380, 925,792\nclasses=80\nnum=12\njitter=.3\nignore_thresh = .7\ntruth_thresh = 1\nrandom=1\nscale_x_y = 1.05\niou_thresh=0.213\ncls_normalizer=1.0\niou_normalizer=0.07\niou_loss=ciou\nnms_kind=greedynms\nbeta_nms=0.6\n\n\n# YOLO-6\n\n[route]\nlayers = 202\n\n[convolutional]\nbatch_normalize=1\nsize=3\nstride=1\npad=1\nfilters=640\nactivation=silu\n\n[shift_channels]\nfrom=206\n\n[convolutional]\nsize=1\nstride=1\npad=1\nfilters=255\nactivation=linear\n\n[control_channels]\nfrom=210\n\n[yolo]\nmask = 9,10,11\nanchors = 19,27, 44,40, 38,94, 96,68, 86,152, 180,137, 140,301, 303,264, 238,542, 436,615, 739,380, 925,792\nclasses=80\nnum=12\njitter=.3\nignore_thresh = .7\ntruth_thresh = 1\nrandom=1\nscale_x_y = 1.05\niou_thresh=0.213\ncls_normalizer=1.0\niou_normalizer=0.07\niou_loss=ciou\nnms_kind=greedynms\nbeta_nms=0.6\n\n# ============ End of Head ============ #\n" }, { "path": "asone/detectors/yolor/models/__init__.py", "content": "\n" }, { "path": "asone/detectors/yolor/models/common.py", "content": "# This file contains modules common to various models\n\nimport math\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom PIL import Image, ImageDraw\n\nfrom asone.detectors.yolor.utils.datasets import letterbox\nfrom asone.detectors.yolor.utils.general import non_max_suppression, make_divisible, scale_coords, xyxy2xywh\nfrom asone.detectors.yolor.utils.plots import color_list\n\ntry:\n from pytorch_wavelets import DWTForward, DWTInverse\n\n class DWT(nn.Module):\n def __init__(self):\n super(DWT, self).__init__()\n self.xfm = DWTForward(J=1, wave='db1', mode='zero')\n\n def forward(self, x):\n b,c,w,h = x.shape\n yl, yh = self.xfm(x)\n return torch.cat([yl/2., yh[0].view(b,-1,w//2,h//2)/2.+.5], 1)\nexcept:\n\n class DWT(nn.Module): # use ReOrg instead\n def __init__(self):\n super(DWT, self).__init__()\n\n def forward(self, x):\n return torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1)\n\n\nclass ImplicitA(nn.Module):\n def __init__(self, channel):\n super(ImplicitA, self).__init__()\n self.channel = channel\n self.implicit = nn.Parameter(torch.zeros(1, channel, 1, 1))\n nn.init.normal_(self.implicit, std=.02)\n\n def forward(self, x):\n return self.implicit.expand_as(x) + x\n\n\nclass ImplicitM(nn.Module):\n def __init__(self, channel):\n super(ImplicitM, self).__init__()\n self.channel = channel\n self.implicit = nn.Parameter(torch.ones(1, channel, 1, 1))\n nn.init.normal_(self.implicit, mean=1., std=.02)\n\n def forward(self, x):\n return self.implicit.expand_as(x) * x\n \n \nclass ReOrg(nn.Module):\n def __init__(self):\n super(ReOrg, self).__init__()\n\n def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2)\n return torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1)\n\ndef autopad(k, p=None): # kernel, padding\n # Pad to 'same'\n if p is None:\n p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad\n return p\n\n\ndef DWConv(c1, c2, k=1, s=1, act=True):\n # Depthwise convolution\n return Conv(c1, c2, k, s, g=math.gcd(c1, c2), act=act)\n\n\nclass Conv(nn.Module):\n # Standard convolution\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super(Conv, self).__init__()\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False)\n self.bn = nn.BatchNorm2d(c2)\n self.act = nn.SiLU() if act else nn.Identity()\n\n def forward(self, x):\n return self.act(self.bn(self.conv(x)))\n\n def fuseforward(self, x):\n return self.act(self.conv(x))\n\n\nclass ConvSig(nn.Module):\n # Standard convolution\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super(ConvSig, self).__init__()\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False)\n self.act = nn.Sigmoid() if act else nn.Identity()\n\n def forward(self, x):\n return self.act(self.conv(x))\n\n def fuseforward(self, x):\n return self.act(self.conv(x))\n\n\nclass ConvSqu(nn.Module):\n # Standard convolution\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super(ConvSqu, self).__init__()\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False)\n self.act = nn.SiLU() if act else nn.Identity()\n\n def forward(self, x):\n return self.act(self.conv(x))\n\n def fuseforward(self, x):\n return self.act(self.conv(x))\n\n\nclass Bottleneck(nn.Module):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super(Bottleneck, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c2, 3, 1, g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass BottleneckG(nn.Module):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super(BottleneckG, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1, g=g)\n self.cv2 = Conv(c_, c2, 3, 1, g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass BottleneckCSP(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSP, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass BottleneckCSPF(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPF, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n #self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.m(self.cv1(x))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass BottleneckCSPL(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPL, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n #self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.act(self.bn(torch.cat((y1, y2), dim=1)))\n\n\nclass BottleneckCSPLG(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=3, e=0.25): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPLG, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, g*c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(g*c_, g*c_, 1, 1, groups=g, bias=False)\n #self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d((1+g) * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[BottleneckG(g*c_, g*c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.act(self.bn(torch.cat((y1, y2), dim=1)))\n\n\nclass BottleneckCSPSE(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPSE, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.avg_pool = nn.AdaptiveAvgPool2d(1)\n self.cs = ConvSqu(c1, c1//8, 1, 1)\n self.cvsig = ConvSig(c1//8, c1, 1, 1)\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n x = x * self.cvsig(self.cs(self.avg_pool(x))).expand_as(x)\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass BottleneckCSPSEA(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPSEA, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.avg_pool = nn.AdaptiveAvgPool2d(1)\n self.cs = ConvSqu(c1, c1//8, 1, 1)\n self.cvsig = ConvSig(c1//8, c1, 1, 1)\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n x = x + x * self.cvsig(self.cs(self.avg_pool(x))).expand_as(x)\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass BottleneckCSPSAM(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPSAM, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cvsig = ConvSig(c1, c1, 1, 1)\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n x = x * self.cvsig(x)\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass BottleneckCSPSAMA(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPSAMA, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cvsig = ConvSig(c1, c1, 1, 1)\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n x = x + x * self.cvsig(x)\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass BottleneckCSPSAMB(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPSAMB, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cvsig = ConvSig(c2, c2, 1, 1)\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n y = self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n return y * self.cvsig(y)\n\n\nclass BottleneckCSPGC(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPGC, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n \n self.channel_add_conv = nn.Sequential(\n nn.Conv2d(c2, c2, kernel_size=1),\n nn.LayerNorm([c2, 1, 1]),\n nn.ReLU(inplace=True), # yapf: disable\n nn.Conv2d(c2, c2, kernel_size=1))\n \n self.conv_mask = nn.Conv2d(c2, 1, kernel_size=1)\n self.softmax = nn.Softmax(dim=2)\n \n def spatial_pool(self, x):\n \n batch, channel, height, width = x.size()\n \n input_x = x \n # [N, C, H * W]\n input_x = input_x.view(batch, channel, height * width)\n # [N, 1, C, H * W]\n input_x = input_x.unsqueeze(1)\n # [N, 1, H, W]\n context_mask = self.conv_mask(x)\n # [N, 1, H * W]\n context_mask = context_mask.view(batch, 1, height * width)\n # [N, 1, H * W]\n context_mask = self.softmax(context_mask)\n # [N, 1, H * W, 1]\n context_mask = context_mask.unsqueeze(-1)\n # [N, 1, C, 1]\n context = torch.matmul(input_x, context_mask)\n # [N, C, 1, 1]\n context = context.view(batch, channel, 1, 1)\n\n return context\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n y = self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n return y + self.channel_add_conv(self.spatial_pool(y))\n\n\nclass BottleneckCSPDNL(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPDNL, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n \n \n self.conv_query = nn.Conv2d(c2, c2, kernel_size=1)\n self.conv_key = nn.Conv2d(c2, c2, kernel_size=1) \n self.conv_value = nn.Conv2d(c2, c2, kernel_size=1, bias=False)\n self.conv_out = None \n self.scale = math.sqrt(c2)\n self.temperature = 0.05 \n self.softmax = nn.Softmax(dim=2) \n self.gamma = nn.Parameter(torch.zeros(1)) \n self.conv_mask = nn.Conv2d(c2, 1, kernel_size=1)\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n y = self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n # [N, C, T, H, W]\n residual = y \n # [N, C, T, H', W'] \n input_x = y\n # [N, C', T, H, W]\n query = self.conv_query(y) \n # [N, C', T, H', W']\n key = self.conv_key(input_x)\n value = self.conv_value(input_x)\n # [N, C', H x W]\n query = query.view(query.size(0), query.size(1), -1) \n # [N, C', H' x W']\n key = key.view(key.size(0), key.size(1), -1)\n value = value.view(value.size(0), value.size(1), -1) \n # channel whitening\n key_mean = key.mean(2).unsqueeze(2)\n query_mean = query.mean(2).unsqueeze(2)\n key -= key_mean\n query -= query_mean\n # [N, T x H x W, T x H' x W']\n sim_map = torch.bmm(query.transpose(1, 2), key)\n sim_map = sim_map/self.scale\n sim_map = sim_map/self.temperature\n sim_map = self.softmax(sim_map)\n # [N, T x H x W, C']\n out_sim = torch.bmm(sim_map, value.transpose(1, 2)) \n # [N, C', T x H x W]\n out_sim = out_sim.transpose(1, 2) \n # [N, C', T, H, W]\n out_sim = out_sim.view(out_sim.size(0), out_sim.size(1), *y.size()[2:]).contiguous()\n out_sim = self.gamma * out_sim \n # [N, 1, H', W']\n mask = self.conv_mask(input_x)\n # [N, 1, H'x W']\n mask = mask.view(mask.size(0), mask.size(1), -1)\n mask = self.softmax(mask)\n # [N, C, 1, 1]\n out_gc = torch.bmm(value, mask.permute(0,2,1)).unsqueeze(-1).contiguous()\n\n return out_sim + out_gc + residual\n\n\nclass BottleneckCSP2(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSP2, self).__init__()\n c_ = int(c2) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv3 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) \n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n x1 = self.cv1(x)\n y1 = self.m(x1)\n y2 = self.cv2(x1)\n return self.cv3(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass BottleneckCSP2SAM(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSP2SAM, self).__init__()\n c_ = int(c2) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cvsig = ConvSig(c_, c_, 1, 1)\n self.cv2 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv3 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) \n self.act = nn.SiLU()\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n x1 = self.cv1(x)\n x1 = x1 * self.cvsig(x1).contiguous()\n y1 = self.m(x1)\n y2 = self.cv2(x1)\n return self.cv3(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass VoVCSP(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(VoVCSP, self).__init__()\n c_ = int(c2) # hidden channels\n self.cv1 = Conv(c1//2, c_//2, 3, 1)\n self.cv2 = Conv(c_//2, c_//2, 3, 1)\n self.cv3 = Conv(c_, c2, 1, 1)\n\n def forward(self, x):\n _, x1 = x.chunk(2, dim=1)\n x1 = self.cv1(x1)\n x2 = self.cv2(x1)\n return self.cv3(torch.cat((x1,x2), dim=1))\n\n\nclass SPP(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, k=(5, 9, 13)):\n super(SPP, self).__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1)\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\n\n def forward(self, x):\n x = self.cv1(x)\n return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1))\n\n\nclass SPPCSP(nn.Module):\n # CSP SPP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, k=(5, 9, 13)):\n super(SPPCSP, self).__init__()\n c_ = int(2 * c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = Conv(c_, c_, 3, 1)\n self.cv4 = Conv(c_, c_, 1, 1)\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\n self.cv5 = Conv(4 * c_, c_, 1, 1)\n self.cv6 = Conv(c_, c_, 3, 1)\n self.bn = nn.BatchNorm2d(2 * c_) \n self.act = nn.SiLU()\n self.cv7 = Conv(2 * c_, c2, 1, 1)\n\n def forward(self, x):\n x1 = self.cv4(self.cv3(self.cv1(x)))\n y1 = self.cv6(self.cv5(torch.cat([x1] + [m(x1) for m in self.m], 1)))\n y2 = self.cv2(x)\n return self.cv7(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass Focus(nn.Module):\n # Focus wh information into c-space\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super(Focus, self).__init__()\n self.conv = Conv(c1 * 4, c2, k, s, p, g, act)\n\n def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2)\n return self.conv(torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1))\n\n\nclass MP(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, k=2):\n super(MP, self).__init__()\n self.m = nn.MaxPool2d(kernel_size=k, stride=k)\n\n def forward(self, x):\n return self.m(x)\n\n\nclass DownD(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, n=1, k=2):\n super(DownD, self).__init__()\n c_ = int(c1) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c_, 3, k)\n self.cv3 = Conv(c_, c2, 1, 1)\n self.cv4 = Conv(c1, c2, 1, 1)\n self.ap = nn.AvgPool2d(kernel_size=k, stride=k)\n\n def forward(self, x):\n return self.cv3(self.cv2(self.cv1(x))) + self.cv4(self.ap(x))\n\n\nclass DownC(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, n=1, k=2):\n super(DownC, self).__init__()\n c_ = int(c1) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c2//2, 3, k)\n self.cv3 = Conv(c1, c2//2, 1, 1)\n self.mp = nn.MaxPool2d(kernel_size=k, stride=k)\n\n def forward(self, x):\n return torch.cat((self.cv2(self.cv1(x)), self.cv3(self.mp(x))), dim=1)\n\n\nclass DNL(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, k=3, s=1):\n super(DNL, self).__init__()\n c_ = int(c1) # hidden channels\n \n # \n self.conv_query = nn.Conv2d(c1, c_, kernel_size=1)\n self.conv_key = nn.Conv2d(c1, c_, kernel_size=1)\n \n self.conv_value = nn.Conv2d(c1, c1, kernel_size=1, bias=False)\n self.conv_out = None\n \n self.scale = math.sqrt(c_)\n self.temperature = 0.05\n \n self.softmax = nn.Softmax(dim=2)\n \n self.gamma = nn.Parameter(torch.zeros(1))\n \n self.conv_mask = nn.Conv2d(c1, 1, kernel_size=1)\n \n self.cv = Conv(c1, c2, k, s)\n\n def forward(self, x):\n\n # [N, C, T, H, W]\n residual = x\n \n # [N, C, T, H', W'] \n input_x = x\n\n # [N, C', T, H, W]\n query = self.conv_query(x)\n \n # [N, C', T, H', W']\n key = self.conv_key(input_x)\n value = self.conv_value(input_x)\n\n # [N, C', H x W]\n query = query.view(query.size(0), query.size(1), -1)\n \n # [N, C', H' x W']\n key = key.view(key.size(0), key.size(1), -1)\n value = value.view(value.size(0), value.size(1), -1)\n \n # channel whitening\n key_mean = key.mean(2).unsqueeze(2)\n query_mean = query.mean(2).unsqueeze(2)\n key -= key_mean\n query -= query_mean\n\n # [N, T x H x W, T x H' x W']\n sim_map = torch.bmm(query.transpose(1, 2), key)\n sim_map = sim_map/self.scale\n sim_map = sim_map/self.temperature\n sim_map = self.softmax(sim_map)\n\n # [N, T x H x W, C']\n out_sim = torch.bmm(sim_map, value.transpose(1, 2))\n \n # [N, C', T x H x W]\n out_sim = out_sim.transpose(1, 2)\n \n # [N, C', T, H, W]\n out_sim = out_sim.view(out_sim.size(0), out_sim.size(1), *x.size()[2:])\n out_sim = self.gamma * out_sim\n \n # [N, 1, H', W']\n mask = self.conv_mask(input_x)\n # [N, 1, H'x W']\n mask = mask.view(mask.size(0), mask.size(1), -1)\n mask = self.softmax(mask)\n # [N, C, 1, 1]\n out_gc = torch.bmm(value, mask.permute(0,2,1)).unsqueeze(-1)\n out_sim = out_sim+out_gc\n\n return self.cv(out_sim + residual)\n\n\nclass GC(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, k=3, s=1):\n super(GC, self).__init__()\n c_ = int(c1) # hidden channels\n \n # \n self.channel_add_conv = nn.Sequential(\n nn.Conv2d(c1, c_, kernel_size=1),\n nn.LayerNorm([c_, 1, 1]),\n nn.ReLU(inplace=True), # yapf: disable\n nn.Conv2d(c_, c1, kernel_size=1))\n \n self.conv_mask = nn.Conv2d(c_, 1, kernel_size=1)\n self.softmax = nn.Softmax(dim=2)\n \n self.cv = Conv(c1, c2, k, s)\n \n \n def spatial_pool(self, x):\n \n batch, channel, height, width = x.size()\n \n input_x = x \n # [N, C, H * W]\n input_x = input_x.view(batch, channel, height * width)\n # [N, 1, C, H * W]\n input_x = input_x.unsqueeze(1)\n # [N, 1, H, W]\n context_mask = self.conv_mask(x)\n # [N, 1, H * W]\n context_mask = context_mask.view(batch, 1, height * width)\n # [N, 1, H * W]\n context_mask = self.softmax(context_mask)\n # [N, 1, H * W, 1]\n context_mask = context_mask.unsqueeze(-1)\n # [N, 1, C, 1]\n context = torch.matmul(input_x, context_mask)\n # [N, C, 1, 1]\n context = context.view(batch, channel, 1, 1)\n\n return context\n\n def forward(self, x):\n\n return self.cv(x + self.channel_add_conv(self.spatial_pool(x)))\n\n\nclass SAM(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, k=3, s=1):\n super(SAM, self).__init__()\n c_ = int(c1) # hidden channels \n self.cvsig = ConvSig(c1, c1, 1, 1) \n self.cv = Conv(c1, c2, k, s)\n\n def forward(self, x):\n\n return self.cv(x * self.cvsig(x))\n\n\nclass SAMA(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, k=3, s=1):\n super(SAMA, self).__init__()\n c_ = int(c1) # hidden channels \n self.cvsig = ConvSig(c1, c1, 1, 1) \n self.cv = Conv(c1, c2, k, s)\n\n def forward(self, x):\n\n return self.cv(x + x * self.cvsig(x))\n\n\nclass SAMB(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, k=3, s=1):\n super(SAMB, self).__init__()\n c_ = int(c1) # hidden channels \n self.cv = Conv(c1, c2, k, s) \n self.cvsig = ConvSig(c2, c2, 1, 1) \n\n def forward(self, x):\n \n x = self.cv(x)\n\n return x * self.cvsig(x)\n\n\nclass Concat(nn.Module):\n # Concatenate a list of tensors along dimension\n def __init__(self, dimension=1):\n super(Concat, self).__init__()\n self.d = dimension\n\n def forward(self, x):\n return torch.cat(x, self.d)\n\n\nclass NMS(nn.Module):\n # Non-Maximum Suppression (NMS) module\n conf = 0.25 # confidence threshold\n iou = 0.45 # IoU threshold\n classes = None # (optional list) filter by class\n\n def __init__(self):\n super(NMS, self).__init__()\n\n def forward(self, x):\n return non_max_suppression(x[0], conf_thres=self.conf, iou_thres=self.iou, classes=self.classes)\n\n\nclass autoShape(nn.Module):\n # input-robust model wrapper for passing cv2/np/PIL/torch inputs. Includes preprocessing, inference and NMS\n img_size = 640 # inference size (pixels)\n conf = 0.25 # NMS confidence threshold\n iou = 0.45 # NMS IoU threshold\n classes = None # (optional list) filter by class\n\n def __init__(self, model):\n super(autoShape, self).__init__()\n self.model = model.eval()\n\n def forward(self, imgs, size=640, augment=False, profile=False):\n # supports inference from various sources. For height=720, width=1280, RGB images example inputs are:\n # opencv: imgs = cv2.imread('image.jpg')[:,:,::-1] # HWC BGR to RGB x(720,1280,3)\n # PIL: imgs = Image.open('image.jpg') # HWC x(720,1280,3)\n # numpy: imgs = np.zeros((720,1280,3)) # HWC\n # torch: imgs = torch.zeros(16,3,720,1280) # BCHW\n # multiple: imgs = [Image.open('image1.jpg'), Image.open('image2.jpg'), ...] # list of images\n\n p = next(self.model.parameters()) # for device and type\n if isinstance(imgs, torch.Tensor): # torch\n return self.model(imgs.to(p.device).type_as(p), augment, profile) # inference\n\n # Pre-process\n if not isinstance(imgs, list):\n imgs = [imgs]\n shape0, shape1 = [], [] # image and inference shapes\n batch = range(len(imgs)) # batch size\n for i in batch:\n imgs[i] = np.array(imgs[i]) # to numpy\n imgs[i] = imgs[i][:, :, :3] if imgs[i].ndim == 3 else np.tile(imgs[i][:, :, None], 3) # enforce 3ch input\n s = imgs[i].shape[:2] # HWC\n shape0.append(s) # image shape\n g = (size / max(s)) # gain\n shape1.append([y * g for y in s])\n shape1 = [make_divisible(x, int(self.stride.max())) for x in np.stack(shape1, 0).max(0)] # inference shape\n x = [letterbox(imgs[i], new_shape=shape1, auto=False)[0] for i in batch] # pad\n x = np.stack(x, 0) if batch[-1] else x[0][None] # stack\n x = np.ascontiguousarray(x.transpose((0, 3, 1, 2))) # BHWC to BCHW\n x = torch.from_numpy(x).to(p.device).type_as(p) / 255. # uint8 to fp16/32\n\n # Inference\n with torch.no_grad():\n y = self.model(x, augment, profile)[0] # forward\n y = non_max_suppression(y, conf_thres=self.conf, iou_thres=self.iou, classes=self.classes) # NMS\n\n # Post-process\n for i in batch:\n if y[i] is not None:\n y[i][:, :4] = scale_coords(shape1, y[i][:, :4], shape0[i])\n\n return Detections(imgs, y, self.names)\n\n\nclass Detections:\n # detections class for YOLOv5 inference results\n def __init__(self, imgs, pred, names=None):\n super(Detections, self).__init__()\n self.imgs = imgs # list of images as numpy arrays\n self.pred = pred # list of tensors pred[0] = (xyxy, conf, cls)\n self.names = names # class names\n self.xyxy = pred # xyxy pixels\n self.xywh = [xyxy2xywh(x) for x in pred] # xywh pixels\n gn = [torch.Tensor([*[im.shape[i] for i in [1, 0, 1, 0]], 1., 1.]) for im in imgs] # normalization gains\n self.xyxyn = [x / g for x, g in zip(self.xyxy, gn)] # xyxy normalized\n self.xywhn = [x / g for x, g in zip(self.xywh, gn)] # xywh normalized\n\n def display(self, pprint=False, show=False, save=False):\n colors = color_list()\n for i, (img, pred) in enumerate(zip(self.imgs, self.pred)):\n str = f'Image {i + 1}/{len(self.pred)}: {img.shape[0]}x{img.shape[1]} '\n if pred is not None:\n for c in pred[:, -1].unique():\n n = (pred[:, -1] == c).sum() # detections per class\n str += f'{n} {self.names[int(c)]}s, ' # add to string\n if show or save:\n img = Image.fromarray(img.astype(np.uint8)) if isinstance(img, np.ndarray) else img # from np\n for *box, conf, cls in pred: # xyxy, confidence, class\n # str += '%s %.2f, ' % (names[int(cls)], conf) # label\n ImageDraw.Draw(img).rectangle(box, width=4, outline=colors[int(cls) % 10]) # plot\n if save:\n f = f'results{i}.jpg'\n str += f\"saved to '{f}'\"\n img.save(f) # save\n if show:\n img.show(f'Image {i}') # show\n if pprint:\n print(str)\n\n def print(self):\n self.display(pprint=True) # print results\n\n def show(self):\n self.display(show=True) # show results\n\n def save(self):\n self.display(save=True) # save results\n\n\nclass Flatten(nn.Module):\n # Use after nn.AdaptiveAvgPool2d(1) to remove last 2 dimensions\n @staticmethod\n def forward(x):\n return x.view(x.size(0), -1)\n\n\nclass Classify(nn.Module):\n # Classification head, i.e. x(b,c1,20,20) to x(b,c2)\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1): # ch_in, ch_out, kernel, stride, padding, groups\n super(Classify, self).__init__()\n self.aap = nn.AdaptiveAvgPool2d(1) # to x(b,c1,1,1)\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False) # to x(b,c2,1,1)\n self.flat = Flatten()\n\n def forward(self, x):\n z = torch.cat([self.aap(y) for y in (x if isinstance(x, list) else [x])], 1) # cat if list\n return self.flat(self.conv(z)) # flatten to x(b,c2)\n \n \nclass TransformerLayer(nn.Module):\n def __init__(self, c, num_heads):\n super().__init__()\n\n self.ln1 = nn.LayerNorm(c)\n self.q = nn.Linear(c, c, bias=False)\n self.k = nn.Linear(c, c, bias=False)\n self.v = nn.Linear(c, c, bias=False)\n self.ma = nn.MultiheadAttention(embed_dim=c, num_heads=num_heads)\n self.ln2 = nn.LayerNorm(c)\n self.fc1 = nn.Linear(c, c, bias=False)\n self.fc2 = nn.Linear(c, c, bias=False)\n\n def forward(self, x):\n x_ = self.ln1(x)\n x = self.ma(self.q(x_), self.k(x_), self.v(x_))[0] + x\n x = self.ln2(x)\n x = self.fc2(self.fc1(x)) + x\n return x\n\n\nclass TransformerBlock(nn.Module):\n def __init__(self, c1, c2, num_heads, num_layers):\n super().__init__()\n\n self.conv = None\n if c1 != c2:\n self.conv = Conv(c1, c2)\n self.linear = nn.Linear(c2, c2)\n self.tr = nn.Sequential(*[TransformerLayer(c2, num_heads) for _ in range(num_layers)])\n self.c2 = c2\n\n def forward(self, x):\n if self.conv is not None:\n x = self.conv(x)\n b, _, w, h = x.shape\n p = x.flatten(2)\n p = p.unsqueeze(0)\n p = p.transpose(0, 3)\n p = p.squeeze(3)\n e = self.linear(p)\n x = p + e\n\n x = self.tr(x)\n x = x.unsqueeze(3)\n x = x.transpose(0, 3)\n x = x.reshape(b, self.c2, w, h)\n return x\n\n\n \nclass BottleneckCSPTR(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPTR, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = TransformerBlock(c_, c_, 4, n)\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\nclass BottleneckCSP2TR(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSP2TR, self).__init__()\n c_ = int(c2) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv3 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) \n self.act = nn.SiLU()\n self.m = TransformerBlock(c_, c_, 4, n)\n\n def forward(self, x):\n x1 = self.cv1(x)\n y1 = self.m(x1)\n y2 = self.cv2(x1)\n return self.cv3(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n\n\nclass SPPCSPTR(nn.Module):\n # CSP SPP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, k=(5, 9, 13)):\n super(SPPCSPTR, self).__init__()\n c_ = int(2 * c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = Conv(c_, c_, 3, 1)\n self.cv4 = Conv(c_, c_, 1, 1)\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\n self.cv5 = Conv(4 * c_, c_, 1, 1)\n self.cv6 = TransformerBlock(c_, c_, 4, 1)\n self.bn = nn.BatchNorm2d(2 * c_) \n self.act = nn.SiLU()\n self.cv7 = Conv(2 * c_, c2, 1, 1)\n\n def forward(self, x):\n x1 = self.cv4(self.cv3(self.cv1(x)))\n y1 = self.cv6(self.cv5(torch.cat([x1] + [m(x1) for m in self.m], 1)))\n y2 = self.cv2(x)\n return self.cv7(self.act(self.bn(torch.cat((y1, y2), dim=1))))\n \nclass TR(BottleneckCSPTR):\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5):\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e)\n self.m = TransformerBlock(c_, c_, 4, n)" }, { "path": "asone/detectors/yolor/models/export.py", "content": "import argparse\n\nimport torch\n\nfrom asone.detectors.yolor.utils.google_utils import attempt_download\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument('--weights', type=str, default='./yolov4.pt', help='weights path')\n parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='image size')\n parser.add_argument('--batch-size', type=int, default=1, help='batch size')\n opt = parser.parse_args()\n opt.img_size *= 2 if len(opt.img_size) == 1 else 1 # expand\n print(opt)\n\n # Input\n img = torch.zeros((opt.batch_size, 3, *opt.img_size)) # image size(1,3,320,192) iDetection\n\n # Load PyTorch model\n attempt_download(opt.weights)\n model = torch.load(opt.weights, map_location=torch.device('cpu'))['model'].float()\n model.eval()\n model.model[-1].export = True # set Detect() layer export=True\n y = model(img) # dry run\n\n # TorchScript export\n try:\n print('\\nStarting TorchScript export with torch %s...' % torch.__version__)\n f = opt.weights.replace('.pt', '.torchscript.pt') # filename\n ts = torch.jit.trace(model, img)\n ts.save(f)\n print('TorchScript export success, saved as %s' % f)\n except Exception as e:\n print('TorchScript export failure: %s' % e)\n\n # ONNX export\n try:\n import onnx\n\n print('\\nStarting ONNX export with onnx %s...' % onnx.__version__)\n f = opt.weights.replace('.pt', '.onnx') # filename\n model.fuse() # only for ONNX\n torch.onnx.export(model, img, f, verbose=False, opset_version=12, input_names=['images'],\n output_names=['classes', 'boxes'] if y is None else ['output'])\n\n # Checks\n onnx_model = onnx.load(f) # load onnx model\n onnx.checker.check_model(onnx_model) # check onnx model\n print(onnx.helper.printable_graph(onnx_model.graph)) # print a human readable model\n print('ONNX export success, saved as %s' % f)\n except Exception as e:\n print('ONNX export failure: %s' % e)\n\n # CoreML export\n try:\n import coremltools as ct\n\n print('\\nStarting CoreML export with coremltools %s...' % ct.__version__)\n # convert model from torchscript and apply pixel scaling as per detect.py\n model = ct.convert(ts, inputs=[ct.ImageType(name='images', shape=img.shape, scale=1 / 255.0, bias=[0, 0, 0])])\n f = opt.weights.replace('.pt', '.mlmodel') # filename\n model.save(f)\n print('CoreML export success, saved as %s' % f)\n except Exception as e:\n print('CoreML export failure: %s' % e)\n\n # Finish\n print('\\nExport complete. Visualize with https://github.com/lutzroeder/netron.')\n" }, { "path": "asone/detectors/yolor/models/models.py", "content": "from asone.detectors.yolor.utils.google_utils import *\nfrom asone.detectors.yolor.utils.layers import *\nfrom asone.detectors.yolor.utils.parse_config import *\nfrom asone.detectors.yolor.utils import torch_utils\n\nONNX_EXPORT = False\n\n\ndef create_modules(module_defs, img_size, cfg):\n # Constructs module list of layer blocks from module configuration in module_defs\n\n img_size = [img_size] * 2 if isinstance(img_size, int) else img_size # expand if necessary\n _ = module_defs.pop(0) # cfg training hyperparams (unused)\n output_filters = [3] # input channels\n module_list = nn.ModuleList()\n routs = [] # list of layers which rout to deeper layers\n yolo_index = -1\n\n for i, mdef in enumerate(module_defs):\n modules = nn.Sequential()\n\n if mdef['type'] == 'convolutional':\n bn = mdef['batch_normalize']\n filters = mdef['filters']\n k = mdef['size'] # kernel size\n stride = mdef['stride'] if 'stride' in mdef else (mdef['stride_y'], mdef['stride_x'])\n if isinstance(k, int): # single-size conv\n modules.add_module('Conv2d', nn.Conv2d(in_channels=output_filters[-1],\n out_channels=filters,\n kernel_size=k,\n stride=stride,\n padding=k // 2 if mdef['pad'] else 0,\n groups=mdef['groups'] if 'groups' in mdef else 1,\n bias=not bn))\n else: # multiple-size conv\n modules.add_module('MixConv2d', MixConv2d(in_ch=output_filters[-1],\n out_ch=filters,\n k=k,\n stride=stride,\n bias=not bn))\n\n if bn:\n modules.add_module('BatchNorm2d', nn.BatchNorm2d(filters, momentum=0.03, eps=1E-4))\n else:\n routs.append(i) # detection output (goes into yolo layer)\n\n if mdef['activation'] == 'leaky': # activation study https://github.com/ultralytics/yolov3/issues/441\n modules.add_module('activation', nn.LeakyReLU(0.1, inplace=True))\n elif mdef['activation'] == 'swish':\n modules.add_module('activation', Swish())\n elif mdef['activation'] == 'mish':\n modules.add_module('activation', Mish())\n elif mdef['activation'] == 'emb':\n modules.add_module('activation', F.normalize())\n elif mdef['activation'] == 'logistic':\n modules.add_module('activation', nn.Sigmoid())\n elif mdef['activation'] == 'silu':\n modules.add_module('activation', nn.SiLU())\n\n elif mdef['type'] == 'deformableconvolutional':\n bn = mdef['batch_normalize']\n filters = mdef['filters']\n k = mdef['size'] # kernel size\n stride = mdef['stride'] if 'stride' in mdef else (mdef['stride_y'], mdef['stride_x'])\n if isinstance(k, int): # single-size conv\n modules.add_module('DeformConv2d', DeformConv2d(output_filters[-1],\n filters,\n kernel_size=k,\n padding=k // 2 if mdef['pad'] else 0,\n stride=stride,\n bias=not bn,\n modulation=True))\n else: # multiple-size conv\n modules.add_module('MixConv2d', MixConv2d(in_ch=output_filters[-1],\n out_ch=filters,\n k=k,\n stride=stride,\n bias=not bn))\n\n if bn:\n modules.add_module('BatchNorm2d', nn.BatchNorm2d(filters, momentum=0.03, eps=1E-4))\n else:\n routs.append(i) # detection output (goes into yolo layer)\n\n if mdef['activation'] == 'leaky': # activation study https://github.com/ultralytics/yolov3/issues/441\n modules.add_module('activation', nn.LeakyReLU(0.1, inplace=True))\n elif mdef['activation'] == 'swish':\n modules.add_module('activation', Swish())\n elif mdef['activation'] == 'mish':\n modules.add_module('activation', Mish())\n elif mdef['activation'] == 'silu':\n modules.add_module('activation', nn.SiLU())\n \n elif mdef['type'] == 'dropout':\n p = mdef['probability']\n modules = nn.Dropout(p)\n\n elif mdef['type'] == 'avgpool':\n modules = GAP()\n\n elif mdef['type'] == 'silence':\n filters = output_filters[-1]\n modules = Silence()\n\n elif mdef['type'] == 'scale_channels': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1]\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = ScaleChannel(layers=layers)\n\n elif mdef['type'] == 'shift_channels': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1]\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = ShiftChannel(layers=layers)\n\n elif mdef['type'] == 'shift_channels_2d': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1]\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = ShiftChannel2D(layers=layers)\n\n elif mdef['type'] == 'control_channels': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1]\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = ControlChannel(layers=layers)\n\n elif mdef['type'] == 'control_channels_2d': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1]\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = ControlChannel2D(layers=layers)\n\n elif mdef['type'] == 'alternate_channels': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1] * 2\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = AlternateChannel(layers=layers)\n\n elif mdef['type'] == 'alternate_channels_2d': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1] * 2\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = AlternateChannel2D(layers=layers)\n\n elif mdef['type'] == 'select_channels': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1]\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = SelectChannel(layers=layers)\n\n elif mdef['type'] == 'select_channels_2d': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1]\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = SelectChannel2D(layers=layers)\n\n elif mdef['type'] == 'sam': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1]\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = ScaleSpatial(layers=layers)\n\n elif mdef['type'] == 'BatchNorm2d':\n filters = output_filters[-1]\n modules = nn.BatchNorm2d(filters, momentum=0.03, eps=1E-4)\n if i == 0 and filters == 3: # normalize RGB image\n # imagenet mean and var https://pytorch.org/docs/stable/torchvision/models.html#classification\n modules.running_mean = torch.tensor([0.485, 0.456, 0.406])\n modules.running_var = torch.tensor([0.0524, 0.0502, 0.0506])\n\n elif mdef['type'] == 'maxpool':\n k = mdef['size'] # kernel size\n stride = mdef['stride']\n maxpool = nn.MaxPool2d(kernel_size=k, stride=stride, padding=(k - 1) // 2)\n if k == 2 and stride == 1: # yolov3-tiny\n modules.add_module('ZeroPad2d', nn.ZeroPad2d((0, 1, 0, 1)))\n modules.add_module('MaxPool2d', maxpool)\n else:\n modules = maxpool\n\n elif mdef['type'] == 'local_avgpool':\n k = mdef['size'] # kernel size\n stride = mdef['stride']\n avgpool = nn.AvgPool2d(kernel_size=k, stride=stride, padding=(k - 1) // 2)\n if k == 2 and stride == 1: # yolov3-tiny\n modules.add_module('ZeroPad2d', nn.ZeroPad2d((0, 1, 0, 1)))\n modules.add_module('AvgPool2d', avgpool)\n else:\n modules = avgpool\n\n elif mdef['type'] == 'upsample':\n if ONNX_EXPORT: # explicitly state size, avoid scale_factor\n g = (yolo_index + 1) * 2 / 32 # gain\n modules = nn.Upsample(size=tuple(int(x * g) for x in img_size)) # img_size = (320, 192)\n else:\n modules = nn.Upsample(scale_factor=mdef['stride'])\n\n elif mdef['type'] == 'route': # nn.Sequential() placeholder for 'route' layer\n layers = mdef['layers']\n filters = sum([output_filters[l + 1 if l > 0 else l] for l in layers])\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = FeatureConcat(layers=layers)\n\n elif mdef['type'] == 'route2': # nn.Sequential() placeholder for 'route' layer\n layers = mdef['layers']\n filters = sum([output_filters[l + 1 if l > 0 else l] for l in layers])\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = FeatureConcat2(layers=layers)\n\n elif mdef['type'] == 'route3': # nn.Sequential() placeholder for 'route' layer\n layers = mdef['layers']\n filters = sum([output_filters[l + 1 if l > 0 else l] for l in layers])\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = FeatureConcat3(layers=layers)\n\n elif mdef['type'] == 'route_lhalf': # nn.Sequential() placeholder for 'route' layer\n layers = mdef['layers']\n filters = sum([output_filters[l + 1 if l > 0 else l] for l in layers])//2\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = FeatureConcat_l(layers=layers)\n\n elif mdef['type'] == 'shortcut': # nn.Sequential() placeholder for 'shortcut' layer\n layers = mdef['from']\n filters = output_filters[-1]\n routs.extend([i + l if l < 0 else l for l in layers])\n modules = WeightedFeatureFusion(layers=layers, weight='weights_type' in mdef)\n\n elif mdef['type'] == 'reorg3d': # yolov3-spp-pan-scale\n pass\n\n elif mdef['type'] == 'reorg': # yolov3-spp-pan-scale\n filters = 4 * output_filters[-1]\n modules.add_module('Reorg', Reorg())\n\n elif mdef['type'] == 'dwt': # yolov3-spp-pan-scale\n filters = 4 * output_filters[-1]\n modules.add_module('DWT', DWT())\n\n elif mdef['type'] == 'implicit_add': # yolov3-spp-pan-scale\n filters = mdef['filters']\n modules = ImplicitA(channel=filters)\n\n elif mdef['type'] == 'implicit_mul': # yolov3-spp-pan-scale\n filters = mdef['filters']\n modules = ImplicitM(channel=filters)\n\n elif mdef['type'] == 'implicit_cat': # yolov3-spp-pan-scale\n filters = mdef['filters']\n modules = ImplicitC(channel=filters)\n\n elif mdef['type'] == 'implicit_add_2d': # yolov3-spp-pan-scale\n channels = mdef['filters']\n filters = mdef['atoms']\n modules = Implicit2DA(atom=filters, channel=channels)\n\n elif mdef['type'] == 'implicit_mul_2d': # yolov3-spp-pan-scale\n channels = mdef['filters']\n filters = mdef['atoms']\n modules = Implicit2DM(atom=filters, channel=channels)\n\n elif mdef['type'] == 'implicit_cat_2d': # yolov3-spp-pan-scale\n channels = mdef['filters']\n filters = mdef['atoms']\n modules = Implicit2DC(atom=filters, channel=channels)\n\n elif mdef['type'] == 'yolo':\n yolo_index += 1\n stride = [8, 16, 32, 64, 128] # P3, P4, P5, P6, P7 strides\n if any(x in cfg for x in ['yolov4-tiny', 'fpn', 'yolov3']): # P5, P4, P3 strides\n stride = [32, 16, 8]\n layers = mdef['from'] if 'from' in mdef else []\n modules = YOLOLayer(anchors=mdef['anchors'][mdef['mask']], # anchor list\n nc=mdef['classes'], # number of classes\n img_size=img_size, # (416, 416)\n yolo_index=yolo_index, # 0, 1, 2...\n layers=layers, # output layers\n stride=stride[yolo_index])\n\n # Initialize preceding Conv2d() bias (https://arxiv.org/pdf/1708.02002.pdf section 3.3)\n try:\n j = layers[yolo_index] if 'from' in mdef else -2\n bias_ = module_list[j][0].bias # shape(255,)\n bias = bias_[:modules.no * modules.na].view(modules.na, -1) # shape(3,85)\n #bias[:, 4] += -4.5 # obj\n bias.data[:, 4] += math.log(8 / (640 / stride[yolo_index]) ** 2) # obj (8 objects per 640 image)\n bias.data[:, 5:] += math.log(0.6 / (modules.nc - 0.99)) # cls (sigmoid(p) = 1/nc)\n module_list[j][0].bias = torch.nn.Parameter(bias_, requires_grad=bias_.requires_grad)\n \n #j = [-2, -5, -8]\n #for sj in j:\n # bias_ = module_list[sj][0].bias\n # bias = bias_[:modules.no * 1].view(1, -1)\n # bias.data[:, 4] += math.log(8 / (640 / stride[yolo_index]) ** 2)\n # bias.data[:, 5:] += math.log(0.6 / (modules.nc - 0.99))\n # module_list[sj][0].bias = torch.nn.Parameter(bias_, requires_grad=bias_.requires_grad)\n except:\n print('WARNING: smart bias initialization failure.')\n\n elif mdef['type'] == 'jde':\n yolo_index += 1\n stride = [8, 16, 32, 64, 128] # P3, P4, P5, P6, P7 strides\n if any(x in cfg for x in ['yolov4-tiny', 'fpn', 'yolov3']): # P5, P4, P3 strides\n stride = [32, 16, 8]\n layers = mdef['from'] if 'from' in mdef else []\n modules = JDELayer(anchors=mdef['anchors'][mdef['mask']], # anchor list\n nc=mdef['classes'], # number of classes\n img_size=img_size, # (416, 416)\n yolo_index=yolo_index, # 0, 1, 2...\n layers=layers, # output layers\n stride=stride[yolo_index])\n\n # Initialize preceding Conv2d() bias (https://arxiv.org/pdf/1708.02002.pdf section 3.3)\n try:\n j = layers[yolo_index] if 'from' in mdef else -1\n bias_ = module_list[j][0].bias # shape(255,)\n bias = bias_[:modules.no * modules.na].view(modules.na, -1) # shape(3,85)\n #bias[:, 4] += -4.5 # obj\n bias.data[:, 4] += math.log(8 / (640 / stride[yolo_index]) ** 2) # obj (8 objects per 640 image)\n bias.data[:, 5:] += math.log(0.6 / (modules.nc - 0.99)) # cls (sigmoid(p) = 1/nc)\n module_list[j][0].bias = torch.nn.Parameter(bias_, requires_grad=bias_.requires_grad)\n except:\n print('WARNING: smart bias initialization failure.')\n\n else:\n print('Warning: Unrecognized Layer Type: ' + mdef['type'])\n\n # Register module list and number of output filters\n module_list.append(modules)\n output_filters.append(filters)\n\n routs_binary = [False] * (i + 1)\n for i in routs:\n routs_binary[i] = True\n return module_list, routs_binary\n\n\nclass YOLOLayer(nn.Module):\n def __init__(self, anchors, nc, img_size, yolo_index, layers, stride):\n super(YOLOLayer, self).__init__()\n self.anchors = torch.Tensor(anchors)\n self.index = yolo_index # index of this layer in layers\n self.layers = layers # model output layer indices\n self.stride = stride # layer stride\n self.nl = len(layers) # number of output layers (3)\n self.na = len(anchors) # number of anchors (3)\n self.nc = nc # number of classes (80)\n self.no = nc + 5 # number of outputs (85)\n self.nx, self.ny, self.ng = 0, 0, 0 # initialize number of x, y gridpoints\n self.anchor_vec = self.anchors / self.stride\n self.anchor_wh = self.anchor_vec.view(1, self.na, 1, 1, 2)\n\n if ONNX_EXPORT:\n self.training = False\n self.create_grids((img_size[1] // stride, img_size[0] // stride)) # number x, y grid points\n\n def create_grids(self, ng=(13, 13), device='cpu'):\n self.nx, self.ny = ng # x and y grid size\n self.ng = torch.tensor(ng, dtype=torch.float)\n\n # build xy offsets\n if not self.training:\n yv, xv = torch.meshgrid([torch.arange(self.ny, device=device), torch.arange(self.nx, device=device)])\n self.grid = torch.stack((xv, yv), 2).view((1, 1, self.ny, self.nx, 2)).float()\n\n if self.anchor_vec.device != device:\n self.anchor_vec = self.anchor_vec.to(device)\n self.anchor_wh = self.anchor_wh.to(device)\n\n def forward(self, p, out):\n ASFF = False # https://arxiv.org/abs/1911.09516\n if ASFF:\n i, n = self.index, self.nl # index in layers, number of layers\n p = out[self.layers[i]]\n bs, _, ny, nx = p.shape # bs, 255, 13, 13\n if (self.nx, self.ny) != (nx, ny):\n self.create_grids((nx, ny), p.device)\n\n # outputs and weights\n # w = F.softmax(p[:, -n:], 1) # normalized weights\n w = torch.sigmoid(p[:, -n:]) * (2 / n) # sigmoid weights (faster)\n # w = w / w.sum(1).unsqueeze(1) # normalize across layer dimension\n\n # weighted ASFF sum\n p = out[self.layers[i]][:, :-n] * w[:, i:i + 1]\n for j in range(n):\n if j != i:\n p += w[:, j:j + 1] * \\\n F.interpolate(out[self.layers[j]][:, :-n], size=[ny, nx], mode='bilinear', align_corners=False)\n\n elif ONNX_EXPORT:\n bs = 1 # batch size\n else:\n bs, _, ny, nx = p.shape # bs, 255, 13, 13\n if (self.nx, self.ny) != (nx, ny):\n self.create_grids((nx, ny), p.device)\n\n # p.view(bs, 255, 13, 13) -- > (bs, 3, 13, 13, 85) # (bs, anchors, grid, grid, classes + xywh)\n p = p.view(bs, self.na, self.no, self.ny, self.nx).permute(0, 1, 3, 4, 2).contiguous() # prediction\n\n if self.training:\n return p\n\n elif ONNX_EXPORT:\n # Avoid broadcasting for ANE operations\n m = self.na * self.nx * self.ny\n ng = 1. / self.ng.repeat(m, 1)\n grid = self.grid.repeat(1, self.na, 1, 1, 1).view(m, 2)\n anchor_wh = self.anchor_wh.repeat(1, 1, self.nx, self.ny, 1).view(m, 2) * ng\n\n p = p.view(m, self.no)\n xy = torch.sigmoid(p[:, 0:2]) + grid # x, y\n wh = torch.exp(p[:, 2:4]) * anchor_wh # width, height\n p_cls = torch.sigmoid(p[:, 4:5]) if self.nc == 1 else \\\n torch.sigmoid(p[:, 5:self.no]) * torch.sigmoid(p[:, 4:5]) # conf\n return p_cls, xy * ng, wh\n\n else: # inference\n io = p.sigmoid()\n io[..., :2] = (io[..., :2] * 2. - 0.5 + self.grid)\n io[..., 2:4] = (io[..., 2:4] * 2) ** 2 * self.anchor_wh\n io[..., :4] *= self.stride\n #io = p.clone() # inference output\n #io[..., :2] = torch.sigmoid(io[..., :2]) + self.grid # xy\n #io[..., 2:4] = torch.exp(io[..., 2:4]) * self.anchor_wh # wh yolo method\n #io[..., :4] *= self.stride\n #torch.sigmoid_(io[..., 4:])\n return io.view(bs, -1, self.no), p # view [1, 3, 13, 13, 85] as [1, 507, 85]\n\n\nclass JDELayer(nn.Module):\n def __init__(self, anchors, nc, img_size, yolo_index, layers, stride):\n super(JDELayer, self).__init__()\n self.anchors = torch.Tensor(anchors)\n self.index = yolo_index # index of this layer in layers\n self.layers = layers # model output layer indices\n self.stride = stride # layer stride\n self.nl = len(layers) # number of output layers (3)\n self.na = len(anchors) # number of anchors (3)\n self.nc = nc # number of classes (80)\n self.no = nc + 5 # number of outputs (85)\n self.nx, self.ny, self.ng = 0, 0, 0 # initialize number of x, y gridpoints\n self.anchor_vec = self.anchors / self.stride\n self.anchor_wh = self.anchor_vec.view(1, self.na, 1, 1, 2)\n\n if ONNX_EXPORT:\n self.training = False\n self.create_grids((img_size[1] // stride, img_size[0] // stride)) # number x, y grid points\n\n def create_grids(self, ng=(13, 13), device='cpu'):\n self.nx, self.ny = ng # x and y grid size\n self.ng = torch.tensor(ng, dtype=torch.float)\n\n # build xy offsets\n if not self.training:\n yv, xv = torch.meshgrid([torch.arange(self.ny, device=device), torch.arange(self.nx, device=device)])\n self.grid = torch.stack((xv, yv), 2).view((1, 1, self.ny, self.nx, 2)).float()\n\n if self.anchor_vec.device != device:\n self.anchor_vec = self.anchor_vec.to(device)\n self.anchor_wh = self.anchor_wh.to(device)\n\n def forward(self, p, out):\n ASFF = False # https://arxiv.org/abs/1911.09516\n if ASFF:\n i, n = self.index, self.nl # index in layers, number of layers\n p = out[self.layers[i]]\n bs, _, ny, nx = p.shape # bs, 255, 13, 13\n if (self.nx, self.ny) != (nx, ny):\n self.create_grids((nx, ny), p.device)\n\n # outputs and weights\n # w = F.softmax(p[:, -n:], 1) # normalized weights\n w = torch.sigmoid(p[:, -n:]) * (2 / n) # sigmoid weights (faster)\n # w = w / w.sum(1).unsqueeze(1) # normalize across layer dimension\n\n # weighted ASFF sum\n p = out[self.layers[i]][:, :-n] * w[:, i:i + 1]\n for j in range(n):\n if j != i:\n p += w[:, j:j + 1] * \\\n F.interpolate(out[self.layers[j]][:, :-n], size=[ny, nx], mode='bilinear', align_corners=False)\n\n elif ONNX_EXPORT:\n bs = 1 # batch size\n else:\n bs, _, ny, nx = p.shape # bs, 255, 13, 13\n if (self.nx, self.ny) != (nx, ny):\n self.create_grids((nx, ny), p.device)\n\n # p.view(bs, 255, 13, 13) -- > (bs, 3, 13, 13, 85) # (bs, anchors, grid, grid, classes + xywh)\n p = p.view(bs, self.na, self.no, self.ny, self.nx).permute(0, 1, 3, 4, 2).contiguous() # prediction\n\n if self.training:\n return p\n\n elif ONNX_EXPORT:\n # Avoid broadcasting for ANE operations\n m = self.na * self.nx * self.ny\n ng = 1. / self.ng.repeat(m, 1)\n grid = self.grid.repeat(1, self.na, 1, 1, 1).view(m, 2)\n anchor_wh = self.anchor_wh.repeat(1, 1, self.nx, self.ny, 1).view(m, 2) * ng\n\n p = p.view(m, self.no)\n xy = torch.sigmoid(p[:, 0:2]) + grid # x, y\n wh = torch.exp(p[:, 2:4]) * anchor_wh # width, height\n p_cls = torch.sigmoid(p[:, 4:5]) if self.nc == 1 else \\\n torch.sigmoid(p[:, 5:self.no]) * torch.sigmoid(p[:, 4:5]) # conf\n return p_cls, xy * ng, wh\n\n else: # inference\n #io = p.sigmoid()\n #io[..., :2] = (io[..., :2] * 2. - 0.5 + self.grid)\n #io[..., 2:4] = (io[..., 2:4] * 2) ** 2 * self.anchor_wh\n #io[..., :4] *= self.stride\n io = p.clone() # inference output\n io[..., :2] = torch.sigmoid(io[..., :2]) * 2. - 0.5 + self.grid # xy\n io[..., 2:4] = (torch.sigmoid(io[..., 2:4]) * 2) ** 2 * self.anchor_wh # wh yolo method\n io[..., :4] *= self.stride\n io[..., 4:] = F.softmax(io[..., 4:])\n return io.view(bs, -1, self.no), p # view [1, 3, 13, 13, 85] as [1, 507, 85]\n\nclass Darknet(nn.Module):\n # YOLOv3 object detection model\n\n def __init__(self, cfg, img_size=(416, 416), verbose=False):\n super(Darknet, self).__init__()\n\n self.module_defs = parse_model_cfg(cfg)\n self.module_list, self.routs = create_modules(self.module_defs, img_size, cfg)\n self.yolo_layers = get_yolo_layers(self)\n # torch_utils.initialize_weights(self)\n\n # Darknet Header https://github.com/AlexeyAB/darknet/issues/2914#issuecomment-496675346\n self.version = np.array([0, 2, 5], dtype=np.int32) # (int32) version info: major, minor, revision\n self.seen = np.array([0], dtype=np.int64) # (int64) number of images seen during training\n self.info(verbose) if not ONNX_EXPORT else None # print model description\n\n def forward(self, x, augment=False, verbose=False):\n\n if not augment:\n return self.forward_once(x)\n else: # Augment images (inference and test only) https://github.com/ultralytics/yolov3/issues/931\n img_size = x.shape[-2:] # height, width\n s = [0.83, 0.67] # scales\n y = []\n for i, xi in enumerate((x,\n torch_utils.scale_img(x.flip(3), s[0], same_shape=False), # flip-lr and scale\n torch_utils.scale_img(x, s[1], same_shape=False), # scale\n )):\n # cv2.imwrite('img%g.jpg' % i, 255 * xi[0].numpy().transpose((1, 2, 0))[:, :, ::-1])\n y.append(self.forward_once(xi)[0])\n\n y[1][..., :4] /= s[0] # scale\n y[1][..., 0] = img_size[1] - y[1][..., 0] # flip lr\n y[2][..., :4] /= s[1] # scale\n\n # for i, yi in enumerate(y): # coco small, medium, large = < 32**2 < 96**2 <\n # area = yi[..., 2:4].prod(2)[:, :, None]\n # if i == 1:\n # yi *= (area < 96. ** 2).float()\n # elif i == 2:\n # yi *= (area > 32. ** 2).float()\n # y[i] = yi\n\n y = torch.cat(y, 1)\n return y, None\n\n def forward_once(self, x, augment=False, verbose=False):\n img_size = x.shape[-2:] # height, width\n yolo_out, out = [], []\n if verbose:\n print('0', x.shape)\n str = ''\n\n # Augment images (inference and test only)\n if augment: # https://github.com/ultralytics/yolov3/issues/931\n nb = x.shape[0] # batch size\n s = [0.83, 0.67] # scales\n x = torch.cat((x,\n torch_utils.scale_img(x.flip(3), s[0]), # flip-lr and scale\n torch_utils.scale_img(x, s[1]), # scale\n ), 0)\n\n for i, module in enumerate(self.module_list):\n name = module.__class__.__name__\n #print(name)\n if name in ['WeightedFeatureFusion', 'FeatureConcat', 'FeatureConcat2', 'FeatureConcat3', 'FeatureConcat_l', 'ScaleChannel', 'ShiftChannel', 'ShiftChannel2D', 'ControlChannel', 'ControlChannel2D', 'AlternateChannel', 'AlternateChannel2D', 'SelectChannel', 'SelectChannel2D', 'ScaleSpatial']: # sum, concat\n if verbose:\n l = [i - 1] + module.layers # layers\n sh = [list(x.shape)] + [list(out[i].shape) for i in module.layers] # shapes\n str = ' >> ' + ' + '.join(['layer %g %s' % x for x in zip(l, sh)])\n x = module(x, out) # WeightedFeatureFusion(), FeatureConcat()\n elif name in ['ImplicitA', 'ImplicitM', 'ImplicitC', 'Implicit2DA', 'Implicit2DM', 'Implicit2DC']:\n x = module()\n elif name == 'YOLOLayer':\n yolo_out.append(module(x, out))\n elif name == 'JDELayer':\n yolo_out.append(module(x, out))\n else: # run module directly, i.e. mtype = 'convolutional', 'upsample', 'maxpool', 'batchnorm2d' etc.\n #print(module)\n #print(x.shape)\n x = module(x)\n\n out.append(x if self.routs[i] else [])\n if verbose:\n print('%g/%g %s -' % (i, len(self.module_list), name), list(x.shape), str)\n str = ''\n\n if self.training: # train\n return yolo_out\n elif ONNX_EXPORT: # export\n x = [torch.cat(x, 0) for x in zip(*yolo_out)]\n return x[0], torch.cat(x[1:3], 1) # scores, boxes: 3780x80, 3780x4\n else: # inference or test\n x, p = zip(*yolo_out) # inference output, training output\n x = torch.cat(x, 1) # cat yolo outputs\n if augment: # de-augment results\n x = torch.split(x, nb, dim=0)\n x[1][..., :4] /= s[0] # scale\n x[1][..., 0] = img_size[1] - x[1][..., 0] # flip lr\n x[2][..., :4] /= s[1] # scale\n x = torch.cat(x, 1)\n return x, p\n\n def fuse(self):\n # Fuse Conv2d + BatchNorm2d layers throughout model\n print('Fusing layers...')\n fused_list = nn.ModuleList()\n for a in list(self.children())[0]:\n if isinstance(a, nn.Sequential):\n for i, b in enumerate(a):\n if isinstance(b, nn.modules.batchnorm.BatchNorm2d):\n # fuse this bn layer with the previous conv2d layer\n conv = a[i - 1]\n fused = torch_utils.fuse_conv_and_bn(conv, b)\n a = nn.Sequential(fused, *list(a.children())[i + 1:])\n break\n fused_list.append(a)\n self.module_list = fused_list\n self.info() if not ONNX_EXPORT else None # yolov3-spp reduced from 225 to 152 layers\n\n def info(self, verbose=False):\n torch_utils.model_info(self, verbose)\n\n\ndef get_yolo_layers(model):\n return [i for i, m in enumerate(model.module_list) if m.__class__.__name__ in ['YOLOLayer', 'JDELayer']] # [89, 101, 113]\n\n\ndef load_darknet_weights(self, weights, cutoff=-1):\n # Parses and loads the weights stored in 'weights'\n\n # Establish cutoffs (load layers between 0 and cutoff. if cutoff = -1 all are loaded)\n file = Path(weights).name\n if file == 'darknet53.conv.74':\n cutoff = 75\n elif file == 'yolov3-tiny.conv.15':\n cutoff = 15\n\n # Read weights file\n with open(weights, 'rb') as f:\n # Read Header https://github.com/AlexeyAB/darknet/issues/2914#issuecomment-496675346\n self.version = np.fromfile(f, dtype=np.int32, count=3) # (int32) version info: major, minor, revision\n self.seen = np.fromfile(f, dtype=np.int64, count=1) # (int64) number of images seen during training\n\n weights = np.fromfile(f, dtype=np.float32) # the rest are weights\n\n ptr = 0\n for i, (mdef, module) in enumerate(zip(self.module_defs[:cutoff], self.module_list[:cutoff])):\n if mdef['type'] == 'convolutional':\n conv = module[0]\n if mdef['batch_normalize']:\n # Load BN bias, weights, running mean and running variance\n bn = module[1]\n nb = bn.bias.numel() # number of biases\n # Bias\n bn.bias.data.copy_(torch.from_numpy(weights[ptr:ptr + nb]).view_as(bn.bias))\n ptr += nb\n # Weight\n bn.weight.data.copy_(torch.from_numpy(weights[ptr:ptr + nb]).view_as(bn.weight))\n ptr += nb\n # Running Mean\n bn.running_mean.data.copy_(torch.from_numpy(weights[ptr:ptr + nb]).view_as(bn.running_mean))\n ptr += nb\n # Running Var\n bn.running_var.data.copy_(torch.from_numpy(weights[ptr:ptr + nb]).view_as(bn.running_var))\n ptr += nb\n else:\n # Load conv. bias\n nb = conv.bias.numel()\n conv_b = torch.from_numpy(weights[ptr:ptr + nb]).view_as(conv.bias)\n conv.bias.data.copy_(conv_b)\n ptr += nb\n # Load conv. weights\n nw = conv.weight.numel() # number of weights\n conv.weight.data.copy_(torch.from_numpy(weights[ptr:ptr + nw]).view_as(conv.weight))\n ptr += nw\n\n\ndef save_weights(self, path='model.weights', cutoff=-1):\n # Converts a PyTorch model to Darket format (*.pt to *.weights)\n # Note: Does not work if model.fuse() is applied\n with open(path, 'wb') as f:\n # Write Header https://github.com/AlexeyAB/darknet/issues/2914#issuecomment-496675346\n self.version.tofile(f) # (int32) version info: major, minor, revision\n self.seen.tofile(f) # (int64) number of images seen during training\n\n # Iterate through layers\n for i, (mdef, module) in enumerate(zip(self.module_defs[:cutoff], self.module_list[:cutoff])):\n if mdef['type'] == 'convolutional':\n conv_layer = module[0]\n # If batch norm, load bn first\n if mdef['batch_normalize']:\n bn_layer = module[1]\n bn_layer.bias.data.cpu().numpy().tofile(f)\n bn_layer.weight.data.cpu().numpy().tofile(f)\n bn_layer.running_mean.data.cpu().numpy().tofile(f)\n bn_layer.running_var.data.cpu().numpy().tofile(f)\n # Load conv bias\n else:\n conv_layer.bias.data.cpu().numpy().tofile(f)\n # Load conv weights\n conv_layer.weight.data.cpu().numpy().tofile(f)\n\n\ndef convert(cfg='cfg/yolov3-spp.cfg', weights='weights/yolov3-spp.weights', saveto='converted.weights'):\n # Converts between PyTorch and Darknet format per extension (i.e. *.weights convert to *.pt and vice versa)\n # from models import *; convert('cfg/yolov3-spp.cfg', 'weights/yolov3-spp.weights')\n\n # Initialize model\n model = Darknet(cfg)\n ckpt = torch.load(weights) # load checkpoint\n try:\n ckpt['model'] = {k: v for k, v in ckpt['model'].items() if model.state_dict()[k].numel() == v.numel()}\n model.load_state_dict(ckpt['model'], strict=False)\n save_weights(model, path=saveto, cutoff=-1)\n except KeyError as e:\n print(e)\n\ndef attempt_download(weights):\n # Attempt to download pretrained weights if not found locally\n weights = weights.strip()\n msg = weights + ' missing, try downloading from https://drive.google.com/open?id=1LezFG5g3BCW6iYaV89B2i64cqEUZD7e0'\n\n if len(weights) > 0 and not os.path.isfile(weights):\n d = {''}\n\n file = Path(weights).name\n if file in d:\n r = gdrive_download(id=d[file], name=weights)\n else: # download from pjreddie.com\n url = 'https://pjreddie.com/media/files/' + file\n print('Downloading ' + url)\n r = os.system('curl -f ' + url + ' -o ' + weights)\n\n # Error check\n if not (r == 0 and os.path.exists(weights) and os.path.getsize(weights) > 1E6): # weights exist and > 1MB\n os.system('rm ' + weights) # remove partial downloads\n raise Exception(msg)\n" }, { "path": "asone/detectors/yolor/utils/__init__.py", "content": "\n" }, { "path": "asone/detectors/yolor/utils/activations.py", "content": "# Activation functions\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\n# Swish https://arxiv.org/pdf/1905.02244.pdf ---------------------------------------------------------------------------\nclass Swish(nn.Module): #\n @staticmethod\n def forward(x):\n return x * torch.sigmoid(x)\n\n\nclass Hardswish(nn.Module): # export-friendly version of nn.Hardswish()\n @staticmethod\n def forward(x):\n # return x * F.hardsigmoid(x) # for torchscript and CoreML\n return x * F.hardtanh(x + 3, 0., 6.) / 6. # for torchscript, CoreML and ONNX\n\n\nclass MemoryEfficientSwish(nn.Module):\n class F(torch.autograd.Function):\n @staticmethod\n def forward(ctx, x):\n ctx.save_for_backward(x)\n return x * torch.sigmoid(x)\n\n @staticmethod\n def backward(ctx, grad_output):\n x = ctx.saved_tensors[0]\n sx = torch.sigmoid(x)\n return grad_output * (sx * (1 + x * (1 - sx)))\n\n def forward(self, x):\n return self.F.apply(x)\n\n\n# Mish https://github.com/digantamisra98/Mish --------------------------------------------------------------------------\nclass Mish(nn.Module):\n @staticmethod\n def forward(x):\n return x * F.softplus(x).tanh()\n\n\nclass MemoryEfficientMish(nn.Module):\n class F(torch.autograd.Function):\n @staticmethod\n def forward(ctx, x):\n ctx.save_for_backward(x)\n return x.mul(torch.tanh(F.softplus(x))) # x * tanh(ln(1 + exp(x)))\n\n @staticmethod\n def backward(ctx, grad_output):\n x = ctx.saved_tensors[0]\n sx = torch.sigmoid(x)\n fx = F.softplus(x).tanh()\n return grad_output * (fx + x * sx * (1 - fx * fx))\n\n def forward(self, x):\n return self.F.apply(x)\n\n\n# FReLU https://arxiv.org/abs/2007.11824 -------------------------------------------------------------------------------\nclass FReLU(nn.Module):\n def __init__(self, c1, k=3): # ch_in, kernel\n super().__init__()\n self.conv = nn.Conv2d(c1, c1, k, 1, 1, groups=c1)\n self.bn = nn.BatchNorm2d(c1)\n\n def forward(self, x):\n return torch.max(x, self.bn(self.conv(x)))\n" }, { "path": "asone/detectors/yolor/utils/autoanchor.py", "content": "# Auto-anchor utils\n\nimport numpy as np\nimport torch\nimport yaml\nfrom scipy.cluster.vq import kmeans\nfrom tqdm import tqdm\n\n\ndef check_anchor_order(m):\n # Check anchor order against stride order for YOLOv5 Detect() module m, and correct if necessary\n a = m.anchor_grid.prod(-1).view(-1) # anchor area\n da = a[-1] - a[0] # delta a\n ds = m.stride[-1] - m.stride[0] # delta s\n if da.sign() != ds.sign(): # same order\n print('Reversing anchor order')\n m.anchors[:] = m.anchors.flip(0)\n m.anchor_grid[:] = m.anchor_grid.flip(0)\n\n\ndef check_anchors(dataset, model, thr=4.0, imgsz=640):\n # Check anchor fit to data, recompute if necessary\n print('\\nAnalyzing anchors... ', end='')\n m = model.module.model[-1] if hasattr(model, 'module') else model.model[-1] # Detect()\n shapes = imgsz * dataset.shapes / dataset.shapes.max(1, keepdims=True)\n scale = np.random.uniform(0.9, 1.1, size=(shapes.shape[0], 1)) # augment scale\n wh = torch.tensor(np.concatenate([l[:, 3:5] * s for s, l in zip(shapes * scale, dataset.labels)])).float() # wh\n\n def metric(k): # compute metric\n r = wh[:, None] / k[None]\n x = torch.min(r, 1. / r).min(2)[0] # ratio metric\n best = x.max(1)[0] # best_x\n aat = (x > 1. / thr).float().sum(1).mean() # anchors above threshold\n bpr = (best > 1. / thr).float().mean() # best possible recall\n return bpr, aat\n\n bpr, aat = metric(m.anchor_grid.clone().cpu().view(-1, 2))\n print('anchors/target = %.2f, Best Possible Recall (BPR) = %.4f' % (aat, bpr), end='')\n if bpr < 0.98: # threshold to recompute\n print('. Attempting to improve anchors, please wait...')\n na = m.anchor_grid.numel() // 2 # number of anchors\n new_anchors = kmean_anchors(dataset, n=na, img_size=imgsz, thr=thr, gen=1000, verbose=False)\n new_bpr = metric(new_anchors.reshape(-1, 2))[0]\n if new_bpr > bpr: # replace anchors\n new_anchors = torch.tensor(new_anchors, device=m.anchors.device).type_as(m.anchors)\n m.anchor_grid[:] = new_anchors.clone().view_as(m.anchor_grid) # for inference\n m.anchors[:] = new_anchors.clone().view_as(m.anchors) / m.stride.to(m.anchors.device).view(-1, 1, 1) # loss\n check_anchor_order(m)\n print('New anchors saved to model. Update model *.yaml to use these anchors in the future.')\n else:\n print('Original anchors better than new anchors. Proceeding with original anchors.')\n print('') # newline\n\n\ndef kmean_anchors(path='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen=1000, verbose=True):\n \"\"\" Creates kmeans-evolved anchors from training dataset\n\n Arguments:\n path: path to dataset *.yaml, or a loaded dataset\n n: number of anchors\n img_size: image size used for training\n thr: anchor-label wh ratio threshold hyperparameter hyp['anchor_t'] used for training, default=4.0\n gen: generations to evolve anchors using genetic algorithm\n verbose: print all results\n\n Return:\n k: kmeans evolved anchors\n\n Usage:\n from utils.general import *; _ = kmean_anchors()\n \"\"\"\n thr = 1. / thr\n\n def metric(k, wh): # compute metrics\n r = wh[:, None] / k[None]\n x = torch.min(r, 1. / r).min(2)[0] # ratio metric\n # x = wh_iou(wh, torch.tensor(k)) # iou metric\n return x, x.max(1)[0] # x, best_x\n\n def anchor_fitness(k): # mutation fitness\n _, best = metric(torch.tensor(k, dtype=torch.float32), wh)\n return (best * (best > thr).float()).mean() # fitness\n\n def print_results(k):\n k = k[np.argsort(k.prod(1))] # sort small to large\n x, best = metric(k, wh0)\n bpr, aat = (best > thr).float().mean(), (x > thr).float().mean() * n # best possible recall, anch > thr\n print('thr=%.2f: %.4f best possible recall, %.2f anchors past thr' % (thr, bpr, aat))\n print('n=%g, img_size=%s, metric_all=%.3f/%.3f-mean/best, past_thr=%.3f-mean: ' %\n (n, img_size, x.mean(), best.mean(), x[x > thr].mean()), end='')\n for i, x in enumerate(k):\n print('%i,%i' % (round(x[0]), round(x[1])), end=', ' if i < len(k) - 1 else '\\n') # use in *.cfg\n return k\n\n if isinstance(path, str): # *.yaml file\n with open(path) as f:\n data_dict = yaml.load(f, Loader=yaml.FullLoader) # model dict\n from asone.detectors.yolor.utils.datasets import LoadImagesAndLabels\n dataset = LoadImagesAndLabels(data_dict['train'], augment=True, rect=True)\n else:\n dataset = path # dataset\n\n # Get label wh\n shapes = img_size * dataset.shapes / dataset.shapes.max(1, keepdims=True)\n wh0 = np.concatenate([l[:, 3:5] * s for s, l in zip(shapes, dataset.labels)]) # wh\n\n # Filter\n i = (wh0 < 3.0).any(1).sum()\n if i:\n print('WARNING: Extremely small objects found. '\n '%g of %g labels are < 3 pixels in width or height.' % (i, len(wh0)))\n wh = wh0[(wh0 >= 2.0).any(1)] # filter > 2 pixels\n\n # Kmeans calculation\n print('Running kmeans for %g anchors on %g points...' % (n, len(wh)))\n s = wh.std(0) # sigmas for whitening\n k, dist = kmeans(wh / s, n, iter=30) # points, mean distance\n k *= s\n wh = torch.tensor(wh, dtype=torch.float32) # filtered\n wh0 = torch.tensor(wh0, dtype=torch.float32) # unfiltered\n k = print_results(k)\n\n # Plot\n # k, d = [None] * 20, [None] * 20\n # for i in tqdm(range(1, 21)):\n # k[i-1], d[i-1] = kmeans(wh / s, i) # points, mean distance\n # fig, ax = plt.subplots(1, 2, figsize=(14, 7))\n # ax = ax.ravel()\n # ax[0].plot(np.arange(1, 21), np.array(d) ** 2, marker='.')\n # fig, ax = plt.subplots(1, 2, figsize=(14, 7)) # plot wh\n # ax[0].hist(wh[wh[:, 0]<100, 0],400)\n # ax[1].hist(wh[wh[:, 1]<100, 1],400)\n # fig.tight_layout()\n # fig.savefig('wh.png', dpi=200)\n\n # Evolve\n npr = np.random\n f, sh, mp, s = anchor_fitness(k), k.shape, 0.9, 0.1 # fitness, generations, mutation prob, sigma\n pbar = tqdm(range(gen), desc='Evolving anchors with Genetic Algorithm') # progress bar\n for _ in pbar:\n v = np.ones(sh)\n while (v == 1).all(): # mutate until a change occurs (prevent duplicates)\n v = ((npr.random(sh) < mp) * npr.random() * npr.randn(*sh) * s + 1).clip(0.3, 3.0)\n kg = (k.copy() * v).clip(min=2.0)\n fg = anchor_fitness(kg)\n if fg > f:\n f, k = fg, kg.copy()\n pbar.desc = 'Evolving anchors with Genetic Algorithm: fitness = %.4f' % f\n if verbose:\n print_results(k)\n\n return print_results(k)\n" }, { "path": "asone/detectors/yolor/utils/datasets.py", "content": "# Dataset utils and dataloaders\n\nimport glob\nimport math\nimport os\nimport random\nimport shutil\nimport time\nfrom itertools import repeat\nfrom multiprocessing.pool import ThreadPool\nfrom pathlib import Path\nfrom threading import Thread\n\nimport cv2\nimport numpy as np\nimport torch\nfrom PIL import Image, ExifTags\nfrom torch.utils.data import Dataset\nfrom tqdm import tqdm\n\nimport pickle\nfrom copy import deepcopy\nfrom pycocotools import mask as maskUtils\nfrom torchvision.utils import save_image\n\nfrom asone.detectors.yolor.utils.general import xyxy2xywh, xywh2xyxy\nfrom asone.detectors.yolor.utils.torch_utils import torch_distributed_zero_first\n\n# Parameters\nhelp_url = 'https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data'\nimg_formats = ['bmp', 'jpg', 'jpeg', 'png', 'tif', 'tiff', 'dng'] # acceptable image suffixes\nvid_formats = ['mov', 'avi', 'mp4', 'mpg', 'mpeg', 'm4v', 'wmv', 'mkv'] # acceptable video suffixes\n\n# Get orientation exif tag\nfor orientation in ExifTags.TAGS.keys():\n if ExifTags.TAGS[orientation] == 'Orientation':\n break\n\n\ndef get_hash(files):\n # Returns a single hash value of a list of files\n return sum(os.path.getsize(f) for f in files if os.path.isfile(f))\n\n\ndef exif_size(img):\n # Returns exif-corrected PIL size\n s = img.size # (width, height)\n try:\n rotation = dict(img._getexif().items())[orientation]\n if rotation == 6: # rotation 270\n s = (s[1], s[0])\n elif rotation == 8: # rotation 90\n s = (s[1], s[0])\n except:\n pass\n\n return s\n\n\ndef create_dataloader(path, imgsz, batch_size, stride, opt, hyp=None, augment=False, cache=False, pad=0.0, rect=False,\n rank=-1, world_size=1, workers=8):\n # Make sure only the first process in DDP process the dataset first, and the following others can use the cache\n with torch_distributed_zero_first(rank):\n dataset = LoadImagesAndLabels(path, imgsz, batch_size,\n augment=augment, # augment images\n hyp=hyp, # augmentation hyperparameters\n rect=rect, # rectangular training\n cache_images=cache,\n single_cls=opt.single_cls,\n stride=int(stride),\n pad=pad,\n rank=rank)\n\n batch_size = min(batch_size, len(dataset))\n nw = min([os.cpu_count() // world_size, batch_size if batch_size > 1 else 0, workers]) # number of workers\n sampler = torch.utils.data.distributed.DistributedSampler(dataset) if rank != -1 else None\n dataloader = InfiniteDataLoader(dataset,\n batch_size=batch_size,\n num_workers=nw,\n sampler=sampler,\n pin_memory=True,\n collate_fn=LoadImagesAndLabels.collate_fn) # torch.utils.data.DataLoader()\n return dataloader, dataset\n\n\ndef create_dataloader9(path, imgsz, batch_size, stride, opt, hyp=None, augment=False, cache=False, pad=0.0, rect=False,\n rank=-1, world_size=1, workers=8):\n # Make sure only the first process in DDP process the dataset first, and the following others can use the cache\n with torch_distributed_zero_first(rank):\n dataset = LoadImagesAndLabels9(path, imgsz, batch_size,\n augment=augment, # augment images\n hyp=hyp, # augmentation hyperparameters\n rect=rect, # rectangular training\n cache_images=cache,\n single_cls=opt.single_cls,\n stride=int(stride),\n pad=pad,\n rank=rank)\n\n batch_size = min(batch_size, len(dataset))\n nw = min([os.cpu_count() // world_size, batch_size if batch_size > 1 else 0, workers]) # number of workers\n sampler = torch.utils.data.distributed.DistributedSampler(dataset) if rank != -1 else None\n dataloader = InfiniteDataLoader(dataset,\n batch_size=batch_size,\n num_workers=nw,\n sampler=sampler,\n pin_memory=True,\n collate_fn=LoadImagesAndLabels9.collate_fn) # torch.utils.data.DataLoader()\n return dataloader, dataset\n\n\nclass InfiniteDataLoader(torch.utils.data.dataloader.DataLoader):\n \"\"\" Dataloader that reuses workers\n\n Uses same syntax as vanilla DataLoader\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n object.__setattr__(self, 'batch_sampler', _RepeatSampler(self.batch_sampler))\n self.iterator = super().__iter__()\n\n def __len__(self):\n return len(self.batch_sampler.sampler)\n\n def __iter__(self):\n for i in range(len(self)):\n yield next(self.iterator)\n\n\nclass _RepeatSampler(object):\n \"\"\" Sampler that repeats forever\n\n Args:\n sampler (Sampler)\n \"\"\"\n\n def __init__(self, sampler):\n self.sampler = sampler\n\n def __iter__(self):\n while True:\n yield from iter(self.sampler)\n\n\nclass LoadImages: # for inference\n def __init__(self, path, img_size=640, auto_size=32):\n p = str(Path(path)) # os-agnostic\n p = os.path.abspath(p) # absolute path\n if '*' in p:\n files = sorted(glob.glob(p, recursive=True)) # glob\n elif os.path.isdir(p):\n files = sorted(glob.glob(os.path.join(p, '*.*'))) # dir\n elif os.path.isfile(p):\n files = [p] # files\n else:\n raise Exception('ERROR: %s does not exist' % p)\n\n images = [x for x in files if x.split('.')[-1].lower() in img_formats]\n videos = [x for x in files if x.split('.')[-1].lower() in vid_formats]\n ni, nv = len(images), len(videos)\n\n self.img_size = img_size\n self.auto_size = auto_size\n self.files = images + videos\n self.nf = ni + nv # number of files\n self.video_flag = [False] * ni + [True] * nv\n self.mode = 'images'\n if any(videos):\n self.new_video(videos[0]) # new video\n else:\n self.cap = None\n assert self.nf > 0, 'No images or videos found in %s. Supported formats are:\\nimages: %s\\nvideos: %s' % \\\n (p, img_formats, vid_formats)\n\n def __iter__(self):\n self.count = 0\n return self\n\n def __next__(self):\n if self.count == self.nf:\n raise StopIteration\n path = self.files[self.count]\n\n if self.video_flag[self.count]:\n # Read video\n self.mode = 'video'\n ret_val, img0 = self.cap.read()\n if not ret_val:\n self.count += 1\n self.cap.release()\n if self.count == self.nf: # last video\n raise StopIteration\n else:\n path = self.files[self.count]\n self.new_video(path)\n ret_val, img0 = self.cap.read()\n\n self.frame += 1\n print('video %g/%g (%g/%g) %s: ' % (self.count + 1, self.nf, self.frame, self.nframes, path), end='')\n\n else:\n # Read image\n self.count += 1\n img0 = cv2.imread(path) # BGR\n assert img0 is not None, 'Image Not Found ' + path\n print('image %g/%g %s: ' % (self.count, self.nf, path), end='')\n\n # Padded resize\n img = letterbox(img0, new_shape=self.img_size, auto_size=self.auto_size)[0]\n\n # Convert\n img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n img = np.ascontiguousarray(img)\n\n return path, img, img0, self.cap\n\n def new_video(self, path):\n self.frame = 0\n self.cap = cv2.VideoCapture(path)\n self.nframes = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT))\n\n def __len__(self):\n return self.nf # number of files\n\n\nclass LoadWebcam: # for inference\n def __init__(self, pipe='0', img_size=640):\n self.img_size = img_size\n\n if pipe.isnumeric():\n pipe = eval(pipe) # local camera\n # pipe = 'rtsp://192.168.1.64/1' # IP camera\n # pipe = 'rtsp://username:password@192.168.1.64/1' # IP camera with login\n # pipe = 'http://wmccpinetop.axiscam.net/mjpg/video.mjpg' # IP golf camera\n\n self.pipe = pipe\n self.cap = cv2.VideoCapture(pipe) # video capture object\n self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 3) # set buffer size\n\n def __iter__(self):\n self.count = -1\n return self\n\n def __next__(self):\n self.count += 1\n if cv2.waitKey(1) == ord('q'): # q to quit\n self.cap.release()\n cv2.destroyAllWindows()\n raise StopIteration\n\n # Read frame\n if self.pipe == 0: # local camera\n ret_val, img0 = self.cap.read()\n img0 = cv2.flip(img0, 1) # flip left-right\n else: # IP camera\n n = 0\n while True:\n n += 1\n self.cap.grab()\n if n % 30 == 0: # skip frames\n ret_val, img0 = self.cap.retrieve()\n if ret_val:\n break\n\n # Print\n assert ret_val, 'Camera Error %s' % self.pipe\n img_path = 'webcam.jpg'\n print('webcam %g: ' % self.count, end='')\n\n # Padded resize\n img = letterbox(img0, new_shape=self.img_size)[0]\n\n # Convert\n img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n img = np.ascontiguousarray(img)\n\n return img_path, img, img0, None\n\n def __len__(self):\n return 0\n\n\nclass LoadStreams: # multiple IP or RTSP cameras\n def __init__(self, sources='streams.txt', img_size=640):\n self.mode = 'images'\n self.img_size = img_size\n\n if os.path.isfile(sources):\n with open(sources, 'r') as f:\n sources = [x.strip() for x in f.read().splitlines() if len(x.strip())]\n else:\n sources = [sources]\n\n n = len(sources)\n self.imgs = [None] * n\n self.sources = sources\n for i, s in enumerate(sources):\n # Start the thread to read frames from the video stream\n print('%g/%g: %s... ' % (i + 1, n, s), end='')\n cap = cv2.VideoCapture(eval(s) if s.isnumeric() else s)\n assert cap.isOpened(), 'Failed to open %s' % s\n w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n fps = cap.get(cv2.CAP_PROP_FPS) % 100\n _, self.imgs[i] = cap.read() # guarantee first frame\n thread = Thread(target=self.update, args=([i, cap]), daemon=True)\n print(' success (%gx%g at %.2f FPS).' % (w, h, fps))\n thread.start()\n print('') # newline\n\n # check for common shapes\n s = np.stack([letterbox(x, new_shape=self.img_size)[0].shape for x in self.imgs], 0) # inference shapes\n self.rect = np.unique(s, axis=0).shape[0] == 1 # rect inference if all shapes equal\n if not self.rect:\n print('WARNING: Different stream shapes detected. For optimal performance supply similarly-shaped streams.')\n\n def update(self, index, cap):\n # Read next stream frame in a daemon thread\n n = 0\n while cap.isOpened():\n n += 1\n # _, self.imgs[index] = cap.read()\n cap.grab()\n if n == 4: # read every 4th frame\n _, self.imgs[index] = cap.retrieve()\n n = 0\n time.sleep(0.01) # wait time\n\n def __iter__(self):\n self.count = -1\n return self\n\n def __next__(self):\n self.count += 1\n img0 = self.imgs.copy()\n if cv2.waitKey(1) == ord('q'): # q to quit\n cv2.destroyAllWindows()\n raise StopIteration\n\n # Letterbox\n img = [letterbox(x, new_shape=self.img_size, auto=self.rect)[0] for x in img0]\n\n # Stack\n img = np.stack(img, 0)\n\n # Convert\n img = img[:, :, :, ::-1].transpose(0, 3, 1, 2) # BGR to RGB, to bsx3x416x416\n img = np.ascontiguousarray(img)\n\n return self.sources, img, img0, None\n\n def __len__(self):\n return 0 # 1E12 frames = 32 streams at 30 FPS for 30 years\n\n\nclass LoadImagesAndLabels(Dataset): # for training/testing\n def __init__(self, path, img_size=640, batch_size=16, augment=False, hyp=None, rect=False, image_weights=False,\n cache_images=False, single_cls=False, stride=32, pad=0.0, rank=-1):\n self.img_size = img_size\n self.augment = augment\n self.hyp = hyp\n self.image_weights = image_weights\n self.rect = False if image_weights else rect\n self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training)\n self.mosaic_border = [-img_size // 2, -img_size // 2]\n self.stride = stride\n\n def img2label_paths(img_paths):\n # Define label paths as a function of image paths\n sa, sb = os.sep + 'images' + os.sep, os.sep + 'labels' + os.sep # /images/, /labels/ substrings\n return [x.replace(sa, sb, 1).replace(x.split('.')[-1], 'txt') for x in img_paths]\n\n try:\n f = [] # image files\n for p in path if isinstance(path, list) else [path]:\n p = Path(p) # os-agnostic\n if p.is_dir(): # dir\n f += glob.glob(str(p / '**' / '*.*'), recursive=True)\n elif p.is_file(): # file\n with open(p, 'r') as t:\n t = t.read().splitlines()\n parent = str(p.parent) + os.sep\n f += [x.replace('./', parent) if x.startswith('./') else x for x in t] # local to global path\n else:\n raise Exception('%s does not exist' % p)\n self.img_files = sorted([x.replace('/', os.sep) for x in f if x.split('.')[-1].lower() in img_formats])\n assert self.img_files, 'No images found'\n except Exception as e:\n raise Exception('Error loading data from %s: %s\\nSee %s' % (path, e, help_url))\n\n # Check cache\n self.label_files = img2label_paths(self.img_files) # labels\n cache_path = str(Path(self.label_files[0]).parent) + '.cache3' # cached labels\n if os.path.isfile(cache_path):\n cache = torch.load(cache_path) # load\n if cache['hash'] != get_hash(self.label_files + self.img_files): # dataset changed\n cache = self.cache_labels(cache_path) # re-cache\n else:\n cache = self.cache_labels(cache_path) # cache\n\n # Read cache\n cache.pop('hash') # remove hash\n labels, shapes = zip(*cache.values())\n self.labels = list(labels)\n self.shapes = np.array(shapes, dtype=np.float64)\n self.img_files = list(cache.keys()) # update\n self.label_files = img2label_paths(cache.keys()) # update\n\n n = len(shapes) # number of images\n bi = np.floor(np.arange(n) / batch_size).astype(np.int) # batch index\n nb = bi[-1] + 1 # number of batches\n self.batch = bi # batch index of image\n self.n = n\n\n # Rectangular Training\n if self.rect:\n # Sort by aspect ratio\n s = self.shapes # wh\n ar = s[:, 1] / s[:, 0] # aspect ratio\n irect = ar.argsort()\n self.img_files = [self.img_files[i] for i in irect]\n self.label_files = [self.label_files[i] for i in irect]\n self.labels = [self.labels[i] for i in irect]\n self.shapes = s[irect] # wh\n ar = ar[irect]\n\n # Set training image shapes\n shapes = [[1, 1]] * nb\n for i in range(nb):\n ari = ar[bi == i]\n mini, maxi = ari.min(), ari.max()\n if maxi < 1:\n shapes[i] = [maxi, 1]\n elif mini > 1:\n shapes[i] = [1, 1 / mini]\n\n self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(np.int) * stride\n\n # Check labels\n create_datasubset, extract_bounding_boxes, labels_loaded = False, False, False\n nm, nf, ne, ns, nd = 0, 0, 0, 0, 0 # number missing, found, empty, datasubset, duplicate\n pbar = enumerate(self.label_files)\n if rank in [-1, 0]:\n pbar = tqdm(pbar)\n for i, file in pbar:\n l = self.labels[i] # label\n if l is not None and l.shape[0]:\n assert l.shape[1] == 5, '> 5 label columns: %s' % file\n assert (l >= 0).all(), 'negative labels: %s' % file\n assert (l[:, 1:] <= 1).all(), 'non-normalized or out of bounds coordinate labels: %s' % file\n if np.unique(l, axis=0).shape[0] < l.shape[0]: # duplicate rows\n nd += 1 # print('WARNING: duplicate rows in %s' % self.label_files[i]) # duplicate rows\n if single_cls:\n l[:, 0] = 0 # force dataset into single-class mode\n self.labels[i] = l\n nf += 1 # file found\n\n # Create subdataset (a smaller dataset)\n if create_datasubset and ns < 1E4:\n if ns == 0:\n create_folder(path='./datasubset')\n os.makedirs('./datasubset/images')\n exclude_classes = 43\n if exclude_classes not in l[:, 0]:\n ns += 1\n # shutil.copy(src=self.img_files[i], dst='./datasubset/images/') # copy image\n with open('./datasubset/images.txt', 'a') as f:\n f.write(self.img_files[i] + '\\n')\n\n # Extract object detection boxes for a second stage classifier\n if extract_bounding_boxes:\n p = Path(self.img_files[i])\n img = cv2.imread(str(p))\n h, w = img.shape[:2]\n for j, x in enumerate(l):\n f = '%s%sclassifier%s%g_%g_%s' % (p.parent.parent, os.sep, os.sep, x[0], j, p.name)\n if not os.path.exists(Path(f).parent):\n os.makedirs(Path(f).parent) # make new output folder\n\n b = x[1:] * [w, h, w, h] # box\n b[2:] = b[2:].max() # rectangle to square\n b[2:] = b[2:] * 1.3 + 30 # pad\n b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(np.int)\n\n b[[0, 2]] = np.clip(b[[0, 2]], 0, w) # clip boxes outside of image\n b[[1, 3]] = np.clip(b[[1, 3]], 0, h)\n assert cv2.imwrite(f, img[b[1]:b[3], b[0]:b[2]]), 'Failure extracting classifier boxes'\n else:\n ne += 1 # print('empty labels for image %s' % self.img_files[i]) # file empty\n # os.system(\"rm '%s' '%s'\" % (self.img_files[i], self.label_files[i])) # remove\n\n if rank in [-1, 0]:\n pbar.desc = 'Scanning labels %s (%g found, %g missing, %g empty, %g duplicate, for %g images)' % (\n cache_path, nf, nm, ne, nd, n)\n if nf == 0:\n s = 'WARNING: No labels found in %s. See %s' % (os.path.dirname(file) + os.sep, help_url)\n print(s)\n assert not augment, '%s. Can not train without labels.' % s\n\n # Cache images into memory for faster training (WARNING: large datasets may exceed system RAM)\n self.imgs = [None] * n\n if cache_images:\n gb = 0 # Gigabytes of cached images\n self.img_hw0, self.img_hw = [None] * n, [None] * n\n results = ThreadPool(8).imap(lambda x: load_image(*x), zip(repeat(self), range(n))) # 8 threads\n pbar = tqdm(enumerate(results), total=n)\n for i, x in pbar:\n self.imgs[i], self.img_hw0[i], self.img_hw[i] = x # img, hw_original, hw_resized = load_image(self, i)\n gb += self.imgs[i].nbytes\n pbar.desc = 'Caching images (%.1fGB)' % (gb / 1E9)\n\n def cache_labels(self, path='labels.cache3'):\n # Cache dataset labels, check images and read shapes\n x = {} # dict\n pbar = tqdm(zip(self.img_files, self.label_files), desc='Scanning images', total=len(self.img_files))\n for (img, label) in pbar:\n try:\n l = []\n im = Image.open(img)\n im.verify() # PIL verify\n shape = exif_size(im) # image size\n assert (shape[0] > 9) & (shape[1] > 9), 'image size <10 pixels'\n if os.path.isfile(label):\n with open(label, 'r') as f:\n l = np.array([x.split() for x in f.read().splitlines()], dtype=np.float32) # labels\n if len(l) == 0:\n l = np.zeros((0, 5), dtype=np.float32)\n x[img] = [l, shape]\n except Exception as e:\n print('WARNING: Ignoring corrupted image and/or label %s: %s' % (img, e))\n\n x['hash'] = get_hash(self.label_files + self.img_files)\n torch.save(x, path) # save for next time\n return x\n\n def __len__(self):\n return len(self.img_files)\n\n # def __iter__(self):\n # self.count = -1\n # print('ran dataset iter')\n # #self.shuffled_vector = np.random.permutation(self.nF) if self.augment else np.arange(self.nF)\n # return self\n\n def __getitem__(self, index):\n if self.image_weights:\n index = self.indices[index]\n\n hyp = self.hyp\n mosaic = self.mosaic and random.random() < hyp['mosaic']\n if mosaic:\n # Load mosaic\n img, labels = load_mosaic(self, index)\n #img, labels = load_mosaic9(self, index)\n shapes = None\n\n # MixUp https://arxiv.org/pdf/1710.09412.pdf\n if random.random() < hyp['mixup']:\n img2, labels2 = load_mosaic(self, random.randint(0, len(self.labels) - 1))\n #img2, labels2 = load_mosaic9(self, random.randint(0, len(self.labels) - 1))\n r = np.random.beta(8.0, 8.0) # mixup ratio, alpha=beta=8.0\n img = (img * r + img2 * (1 - r)).astype(np.uint8)\n labels = np.concatenate((labels, labels2), 0)\n\n else:\n # Load image\n img, (h0, w0), (h, w) = load_image(self, index)\n\n # Letterbox\n shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape\n img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment)\n shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling\n\n # Load labels\n labels = []\n x = self.labels[index]\n if x.size > 0:\n # Normalized xywh to pixel xyxy format\n labels = x.copy()\n labels[:, 1] = ratio[0] * w * (x[:, 1] - x[:, 3] / 2) + pad[0] # pad width\n labels[:, 2] = ratio[1] * h * (x[:, 2] - x[:, 4] / 2) + pad[1] # pad height\n labels[:, 3] = ratio[0] * w * (x[:, 1] + x[:, 3] / 2) + pad[0]\n labels[:, 4] = ratio[1] * h * (x[:, 2] + x[:, 4] / 2) + pad[1]\n\n if self.augment:\n # Augment imagespace\n if not mosaic:\n img, labels = random_perspective(img, labels,\n degrees=hyp['degrees'],\n translate=hyp['translate'],\n scale=hyp['scale'],\n shear=hyp['shear'],\n perspective=hyp['perspective'])\n\n # Augment colorspace\n augment_hsv(img, hgain=hyp['hsv_h'], sgain=hyp['hsv_s'], vgain=hyp['hsv_v'])\n\n # Apply cutouts\n # if random.random() < 0.9:\n # labels = cutout(img, labels)\n\n nL = len(labels) # number of labels\n if nL:\n labels[:, 1:5] = xyxy2xywh(labels[:, 1:5]) # convert xyxy to xywh\n labels[:, [2, 4]] /= img.shape[0] # normalized height 0-1\n labels[:, [1, 3]] /= img.shape[1] # normalized width 0-1\n\n if self.augment:\n # flip up-down\n if random.random() < hyp['flipud']:\n img = np.flipud(img)\n if nL:\n labels[:, 2] = 1 - labels[:, 2]\n\n # flip left-right\n if random.random() < hyp['fliplr']:\n img = np.fliplr(img)\n if nL:\n labels[:, 1] = 1 - labels[:, 1]\n\n labels_out = torch.zeros((nL, 6))\n if nL:\n labels_out[:, 1:] = torch.from_numpy(labels)\n\n # Convert\n img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n img = np.ascontiguousarray(img)\n\n return torch.from_numpy(img), labels_out, self.img_files[index], shapes\n\n @staticmethod\n def collate_fn(batch):\n img, label, path, shapes = zip(*batch) # transposed\n for i, l in enumerate(label):\n l[:, 0] = i # add target image index for build_targets()\n return torch.stack(img, 0), torch.cat(label, 0), path, shapes\n\n\nclass LoadImagesAndLabels9(Dataset): # for training/testing\n def __init__(self, path, img_size=640, batch_size=16, augment=False, hyp=None, rect=False, image_weights=False,\n cache_images=False, single_cls=False, stride=32, pad=0.0, rank=-1):\n self.img_size = img_size\n self.augment = augment\n self.hyp = hyp\n self.image_weights = image_weights\n self.rect = False if image_weights else rect\n self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training)\n self.mosaic_border = [-img_size // 2, -img_size // 2]\n self.stride = stride\n\n def img2label_paths(img_paths):\n # Define label paths as a function of image paths\n sa, sb = os.sep + 'images' + os.sep, os.sep + 'labels' + os.sep # /images/, /labels/ substrings\n return [x.replace(sa, sb, 1).replace(x.split('.')[-1], 'txt') for x in img_paths]\n\n try:\n f = [] # image files\n for p in path if isinstance(path, list) else [path]:\n p = Path(p) # os-agnostic\n if p.is_dir(): # dir\n f += glob.glob(str(p / '**' / '*.*'), recursive=True)\n elif p.is_file(): # file\n with open(p, 'r') as t:\n t = t.read().splitlines()\n parent = str(p.parent) + os.sep\n f += [x.replace('./', parent) if x.startswith('./') else x for x in t] # local to global path\n else:\n raise Exception('%s does not exist' % p)\n self.img_files = sorted([x.replace('/', os.sep) for x in f if x.split('.')[-1].lower() in img_formats])\n assert self.img_files, 'No images found'\n except Exception as e:\n raise Exception('Error loading data from %s: %s\\nSee %s' % (path, e, help_url))\n\n # Check cache\n self.label_files = img2label_paths(self.img_files) # labels\n cache_path = str(Path(self.label_files[0]).parent) + '.cache3' # cached labels\n if os.path.isfile(cache_path):\n cache = torch.load(cache_path) # load\n if cache['hash'] != get_hash(self.label_files + self.img_files): # dataset changed\n cache = self.cache_labels(cache_path) # re-cache\n else:\n cache = self.cache_labels(cache_path) # cache\n\n # Read cache\n cache.pop('hash') # remove hash\n labels, shapes = zip(*cache.values())\n self.labels = list(labels)\n self.shapes = np.array(shapes, dtype=np.float64)\n self.img_files = list(cache.keys()) # update\n self.label_files = img2label_paths(cache.keys()) # update\n\n n = len(shapes) # number of images\n bi = np.floor(np.arange(n) / batch_size).astype(np.int) # batch index\n nb = bi[-1] + 1 # number of batches\n self.batch = bi # batch index of image\n self.n = n\n\n # Rectangular Training\n if self.rect:\n # Sort by aspect ratio\n s = self.shapes # wh\n ar = s[:, 1] / s[:, 0] # aspect ratio\n irect = ar.argsort()\n self.img_files = [self.img_files[i] for i in irect]\n self.label_files = [self.label_files[i] for i in irect]\n self.labels = [self.labels[i] for i in irect]\n self.shapes = s[irect] # wh\n ar = ar[irect]\n\n # Set training image shapes\n shapes = [[1, 1]] * nb\n for i in range(nb):\n ari = ar[bi == i]\n mini, maxi = ari.min(), ari.max()\n if maxi < 1:\n shapes[i] = [maxi, 1]\n elif mini > 1:\n shapes[i] = [1, 1 / mini]\n\n self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(np.int) * stride\n\n # Check labels\n create_datasubset, extract_bounding_boxes, labels_loaded = False, False, False\n nm, nf, ne, ns, nd = 0, 0, 0, 0, 0 # number missing, found, empty, datasubset, duplicate\n pbar = enumerate(self.label_files)\n if rank in [-1, 0]:\n pbar = tqdm(pbar)\n for i, file in pbar:\n l = self.labels[i] # label\n if l is not None and l.shape[0]:\n assert l.shape[1] == 5, '> 5 label columns: %s' % file\n assert (l >= 0).all(), 'negative labels: %s' % file\n assert (l[:, 1:] <= 1).all(), 'non-normalized or out of bounds coordinate labels: %s' % file\n if np.unique(l, axis=0).shape[0] < l.shape[0]: # duplicate rows\n nd += 1 # print('WARNING: duplicate rows in %s' % self.label_files[i]) # duplicate rows\n if single_cls:\n l[:, 0] = 0 # force dataset into single-class mode\n self.labels[i] = l\n nf += 1 # file found\n\n # Create subdataset (a smaller dataset)\n if create_datasubset and ns < 1E4:\n if ns == 0:\n create_folder(path='./datasubset')\n os.makedirs('./datasubset/images')\n exclude_classes = 43\n if exclude_classes not in l[:, 0]:\n ns += 1\n # shutil.copy(src=self.img_files[i], dst='./datasubset/images/') # copy image\n with open('./datasubset/images.txt', 'a') as f:\n f.write(self.img_files[i] + '\\n')\n\n # Extract object detection boxes for a second stage classifier\n if extract_bounding_boxes:\n p = Path(self.img_files[i])\n img = cv2.imread(str(p))\n h, w = img.shape[:2]\n for j, x in enumerate(l):\n f = '%s%sclassifier%s%g_%g_%s' % (p.parent.parent, os.sep, os.sep, x[0], j, p.name)\n if not os.path.exists(Path(f).parent):\n os.makedirs(Path(f).parent) # make new output folder\n\n b = x[1:] * [w, h, w, h] # box\n b[2:] = b[2:].max() # rectangle to square\n b[2:] = b[2:] * 1.3 + 30 # pad\n b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(np.int)\n\n b[[0, 2]] = np.clip(b[[0, 2]], 0, w) # clip boxes outside of image\n b[[1, 3]] = np.clip(b[[1, 3]], 0, h)\n assert cv2.imwrite(f, img[b[1]:b[3], b[0]:b[2]]), 'Failure extracting classifier boxes'\n else:\n ne += 1 # print('empty labels for image %s' % self.img_files[i]) # file empty\n # os.system(\"rm '%s' '%s'\" % (self.img_files[i], self.label_files[i])) # remove\n\n if rank in [-1, 0]:\n pbar.desc = 'Scanning labels %s (%g found, %g missing, %g empty, %g duplicate, for %g images)' % (\n cache_path, nf, nm, ne, nd, n)\n if nf == 0:\n s = 'WARNING: No labels found in %s. See %s' % (os.path.dirname(file) + os.sep, help_url)\n print(s)\n assert not augment, '%s. Can not train without labels.' % s\n\n # Cache images into memory for faster training (WARNING: large datasets may exceed system RAM)\n self.imgs = [None] * n\n if cache_images:\n gb = 0 # Gigabytes of cached images\n self.img_hw0, self.img_hw = [None] * n, [None] * n\n results = ThreadPool(8).imap(lambda x: load_image(*x), zip(repeat(self), range(n))) # 8 threads\n pbar = tqdm(enumerate(results), total=n)\n for i, x in pbar:\n self.imgs[i], self.img_hw0[i], self.img_hw[i] = x # img, hw_original, hw_resized = load_image(self, i)\n gb += self.imgs[i].nbytes\n pbar.desc = 'Caching images (%.1fGB)' % (gb / 1E9)\n\n def cache_labels(self, path='labels.cache3'):\n # Cache dataset labels, check images and read shapes\n x = {} # dict\n pbar = tqdm(zip(self.img_files, self.label_files), desc='Scanning images', total=len(self.img_files))\n for (img, label) in pbar:\n try:\n l = []\n im = Image.open(img)\n im.verify() # PIL verify\n shape = exif_size(im) # image size\n assert (shape[0] > 9) & (shape[1] > 9), 'image size <10 pixels'\n if os.path.isfile(label):\n with open(label, 'r') as f:\n l = np.array([x.split() for x in f.read().splitlines()], dtype=np.float32) # labels\n if len(l) == 0:\n l = np.zeros((0, 5), dtype=np.float32)\n x[img] = [l, shape]\n except Exception as e:\n print('WARNING: Ignoring corrupted image and/or label %s: %s' % (img, e))\n\n x['hash'] = get_hash(self.label_files + self.img_files)\n torch.save(x, path) # save for next time\n return x\n\n def __len__(self):\n return len(self.img_files)\n\n # def __iter__(self):\n # self.count = -1\n # print('ran dataset iter')\n # #self.shuffled_vector = np.random.permutation(self.nF) if self.augment else np.arange(self.nF)\n # return self\n\n def __getitem__(self, index):\n if self.image_weights:\n index = self.indices[index]\n\n hyp = self.hyp\n mosaic = self.mosaic and random.random() < hyp['mosaic']\n if mosaic:\n # Load mosaic\n #img, labels = load_mosaic(self, index)\n img, labels = load_mosaic9(self, index)\n shapes = None\n\n # MixUp https://arxiv.org/pdf/1710.09412.pdf\n if random.random() < hyp['mixup']:\n #img2, labels2 = load_mosaic(self, random.randint(0, len(self.labels) - 1))\n img2, labels2 = load_mosaic9(self, random.randint(0, len(self.labels) - 1))\n r = np.random.beta(8.0, 8.0) # mixup ratio, alpha=beta=8.0\n img = (img * r + img2 * (1 - r)).astype(np.uint8)\n labels = np.concatenate((labels, labels2), 0)\n\n else:\n # Load image\n img, (h0, w0), (h, w) = load_image(self, index)\n\n # Letterbox\n shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape\n img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment)\n shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling\n\n # Load labels\n labels = []\n x = self.labels[index]\n if x.size > 0:\n # Normalized xywh to pixel xyxy format\n labels = x.copy()\n labels[:, 1] = ratio[0] * w * (x[:, 1] - x[:, 3] / 2) + pad[0] # pad width\n labels[:, 2] = ratio[1] * h * (x[:, 2] - x[:, 4] / 2) + pad[1] # pad height\n labels[:, 3] = ratio[0] * w * (x[:, 1] + x[:, 3] / 2) + pad[0]\n labels[:, 4] = ratio[1] * h * (x[:, 2] + x[:, 4] / 2) + pad[1]\n\n if self.augment:\n # Augment imagespace\n if not mosaic:\n img, labels = random_perspective(img, labels,\n degrees=hyp['degrees'],\n translate=hyp['translate'],\n scale=hyp['scale'],\n shear=hyp['shear'],\n perspective=hyp['perspective'])\n\n # Augment colorspace\n augment_hsv(img, hgain=hyp['hsv_h'], sgain=hyp['hsv_s'], vgain=hyp['hsv_v'])\n\n # Apply cutouts\n # if random.random() < 0.9:\n # labels = cutout(img, labels)\n\n nL = len(labels) # number of labels\n if nL:\n labels[:, 1:5] = xyxy2xywh(labels[:, 1:5]) # convert xyxy to xywh\n labels[:, [2, 4]] /= img.shape[0] # normalized height 0-1\n labels[:, [1, 3]] /= img.shape[1] # normalized width 0-1\n\n if self.augment:\n # flip up-down\n if random.random() < hyp['flipud']:\n img = np.flipud(img)\n if nL:\n labels[:, 2] = 1 - labels[:, 2]\n\n # flip left-right\n if random.random() < hyp['fliplr']:\n img = np.fliplr(img)\n if nL:\n labels[:, 1] = 1 - labels[:, 1]\n\n labels_out = torch.zeros((nL, 6))\n if nL:\n labels_out[:, 1:] = torch.from_numpy(labels)\n\n # Convert\n img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n img = np.ascontiguousarray(img)\n\n return torch.from_numpy(img), labels_out, self.img_files[index], shapes\n\n @staticmethod\n def collate_fn(batch):\n img, label, path, shapes = zip(*batch) # transposed\n for i, l in enumerate(label):\n l[:, 0] = i # add target image index for build_targets()\n return torch.stack(img, 0), torch.cat(label, 0), path, shapes\n\n\n# Ancillary functions --------------------------------------------------------------------------------------------------\ndef load_image(self, index):\n # loads 1 image from dataset, returns img, original hw, resized hw\n img = self.imgs[index]\n if img is None: # not cached\n path = self.img_files[index]\n img = cv2.imread(path) # BGR\n assert img is not None, 'Image Not Found ' + path\n h0, w0 = img.shape[:2] # orig hw\n r = self.img_size / max(h0, w0) # resize image to img_size\n if r != 1: # always resize down, only resize up if training with augmentation\n interp = cv2.INTER_AREA if r < 1 and not self.augment else cv2.INTER_LINEAR\n img = cv2.resize(img, (int(w0 * r), int(h0 * r)), interpolation=interp)\n return img, (h0, w0), img.shape[:2] # img, hw_original, hw_resized\n else:\n return self.imgs[index], self.img_hw0[index], self.img_hw[index] # img, hw_original, hw_resized\n\n\ndef augment_hsv(img, hgain=0.5, sgain=0.5, vgain=0.5):\n r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains\n hue, sat, val = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV))\n dtype = img.dtype # uint8\n\n x = np.arange(0, 256, dtype=np.int16)\n lut_hue = ((x * r[0]) % 180).astype(dtype)\n lut_sat = np.clip(x * r[1], 0, 255).astype(dtype)\n lut_val = np.clip(x * r[2], 0, 255).astype(dtype)\n\n img_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))).astype(dtype)\n cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR, dst=img) # no return needed\n\n # Histogram equalization\n # if random.random() < 0.2:\n # for i in range(3):\n # img[:, :, i] = cv2.equalizeHist(img[:, :, i])\n\n\ndef load_mosaic(self, index):\n # loads images in a mosaic\n\n labels4 = []\n s = self.img_size\n yc, xc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border] # mosaic center x, y\n indices = [index] + [random.randint(0, len(self.labels) - 1) for _ in range(3)] # 3 additional image indices\n for i, index in enumerate(indices):\n # Load image\n img, _, (h, w) = load_image(self, index)\n\n # place img in img4\n if i == 0: # top left\n img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\n x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image)\n x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image)\n elif i == 1: # top right\n x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n elif i == 2: # bottom left\n x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h)\n elif i == 3: # bottom right\n x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n\n img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]\n padw = x1a - x1b\n padh = y1a - y1b\n\n # Labels\n x = self.labels[index]\n labels = x.copy()\n if x.size > 0: # Normalized xywh to pixel xyxy format\n labels[:, 1] = w * (x[:, 1] - x[:, 3] / 2) + padw\n labels[:, 2] = h * (x[:, 2] - x[:, 4] / 2) + padh\n labels[:, 3] = w * (x[:, 1] + x[:, 3] / 2) + padw\n labels[:, 4] = h * (x[:, 2] + x[:, 4] / 2) + padh\n labels4.append(labels)\n\n # Concat/clip labels\n if len(labels4):\n labels4 = np.concatenate(labels4, 0)\n np.clip(labels4[:, 1:], 0, 2 * s, out=labels4[:, 1:]) # use with random_perspective\n # img4, labels4 = replicate(img4, labels4) # replicate\n\n # Augment\n img4, labels4 = random_perspective(img4, labels4,\n degrees=self.hyp['degrees'],\n translate=self.hyp['translate'],\n scale=self.hyp['scale'],\n shear=self.hyp['shear'],\n perspective=self.hyp['perspective'],\n border=self.mosaic_border) # border to remove\n\n return img4, labels4\n\n\ndef load_mosaic9(self, index):\n # loads images in a 9-mosaic\n\n labels9 = []\n s = self.img_size\n indices = [index] + [random.randint(0, len(self.labels) - 1) for _ in range(8)] # 8 additional image indices\n for i, index in enumerate(indices):\n # Load image\n img, _, (h, w) = load_image(self, index)\n\n # place img in img9\n if i == 0: # center\n img9 = np.full((s * 3, s * 3, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\n h0, w0 = h, w\n c = s, s, s + w, s + h # xmin, ymin, xmax, ymax (base) coordinates\n elif i == 1: # top\n c = s, s - h, s + w, s\n elif i == 2: # top right\n c = s + wp, s - h, s + wp + w, s\n elif i == 3: # right\n c = s + w0, s, s + w0 + w, s + h\n elif i == 4: # bottom right\n c = s + w0, s + hp, s + w0 + w, s + hp + h\n elif i == 5: # bottom\n c = s + w0 - w, s + h0, s + w0, s + h0 + h\n elif i == 6: # bottom left\n c = s + w0 - wp - w, s + h0, s + w0 - wp, s + h0 + h\n elif i == 7: # left\n c = s - w, s + h0 - h, s, s + h0\n elif i == 8: # top left\n c = s - w, s + h0 - hp - h, s, s + h0 - hp\n\n padx, pady = c[:2]\n x1, y1, x2, y2 = [max(x, 0) for x in c] # allocate coords\n\n # Labels\n x = self.labels[index]\n labels = x.copy()\n if x.size > 0: # Normalized xywh to pixel xyxy format\n labels[:, 1] = w * (x[:, 1] - x[:, 3] / 2) + padx\n labels[:, 2] = h * (x[:, 2] - x[:, 4] / 2) + pady\n labels[:, 3] = w * (x[:, 1] + x[:, 3] / 2) + padx\n labels[:, 4] = h * (x[:, 2] + x[:, 4] / 2) + pady\n labels9.append(labels)\n\n # Image\n img9[y1:y2, x1:x2] = img[y1 - pady:, x1 - padx:] # img9[ymin:ymax, xmin:xmax]\n hp, wp = h, w # height, width previous\n\n # Offset\n yc, xc = [int(random.uniform(0, s)) for x in self.mosaic_border] # mosaic center x, y\n img9 = img9[yc:yc + 2 * s, xc:xc + 2 * s]\n\n # Concat/clip labels\n if len(labels9):\n labels9 = np.concatenate(labels9, 0)\n labels9[:, [1, 3]] -= xc\n labels9[:, [2, 4]] -= yc\n\n np.clip(labels9[:, 1:], 0, 2 * s, out=labels9[:, 1:]) # use with random_perspective\n # img9, labels9 = replicate(img9, labels9) # replicate\n\n # Augment\n img9, labels9 = random_perspective(img9, labels9,\n degrees=self.hyp['degrees'],\n translate=self.hyp['translate'],\n scale=self.hyp['scale'],\n shear=self.hyp['shear'],\n perspective=self.hyp['perspective'],\n border=self.mosaic_border) # border to remove\n\n return img9, labels9\n\n\ndef replicate(img, labels):\n # Replicate labels\n h, w = img.shape[:2]\n boxes = labels[:, 1:].astype(int)\n x1, y1, x2, y2 = boxes.T\n s = ((x2 - x1) + (y2 - y1)) / 2 # side length (pixels)\n for i in s.argsort()[:round(s.size * 0.5)]: # smallest indices\n x1b, y1b, x2b, y2b = boxes[i]\n bh, bw = y2b - y1b, x2b - x1b\n yc, xc = int(random.uniform(0, h - bh)), int(random.uniform(0, w - bw)) # offset x, y\n x1a, y1a, x2a, y2a = [xc, yc, xc + bw, yc + bh]\n img[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]\n labels = np.append(labels, [[labels[i, 0], x1a, y1a, x2a, y2a]], axis=0)\n\n return img, labels\n\n\ndef letterbox(img, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, auto_size=32):\n # Resize image to a 32-pixel-multiple rectangle https://github.com/ultralytics/yolov3/issues/232\n shape = img.shape[:2] # current shape [height, width]\n if isinstance(new_shape, int):\n new_shape = (new_shape, new_shape)\n\n # Scale ratio (new / old)\n r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])\n if not scaleup: # only scale down, do not scale up (for better test mAP)\n r = min(r, 1.0)\n\n # Compute padding\n ratio = r, r # width, height ratios\n new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))\n dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding\n if auto: # minimum rectangle\n dw, dh = np.mod(dw, auto_size), np.mod(dh, auto_size) # wh padding\n elif scaleFill: # stretch\n dw, dh = 0.0, 0.0\n new_unpad = (new_shape[1], new_shape[0])\n ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios\n\n dw /= 2 # divide padding into 2 sides\n dh /= 2\n\n if shape[::-1] != new_unpad: # resize\n img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR)\n top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))\n left, right = int(round(dw - 0.1)), int(round(dw + 0.1))\n img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border\n return img, ratio, (dw, dh)\n\n\ndef random_perspective(img, targets=(), degrees=10, translate=.1, scale=.1, shear=10, perspective=0.0, border=(0, 0)):\n # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(.1, .1), scale=(.9, 1.1), shear=(-10, 10))\n # targets = [cls, xyxy]\n\n height = img.shape[0] + border[0] * 2 # shape(h,w,c)\n width = img.shape[1] + border[1] * 2\n\n # Center\n C = np.eye(3)\n C[0, 2] = -img.shape[1] / 2 # x translation (pixels)\n C[1, 2] = -img.shape[0] / 2 # y translation (pixels)\n\n # Perspective\n P = np.eye(3)\n P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y)\n P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x)\n\n # Rotation and Scale\n R = np.eye(3)\n a = random.uniform(-degrees, degrees)\n # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations\n s = random.uniform(1 - scale, 1 + scale)\n # s = 2 ** random.uniform(-scale, scale)\n R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s)\n\n # Shear\n S = np.eye(3)\n S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg)\n S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg)\n\n # Translation\n T = np.eye(3)\n T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width # x translation (pixels)\n T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height # y translation (pixels)\n\n # Combined rotation matrix\n M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT\n if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed\n if perspective:\n img = cv2.warpPerspective(img, M, dsize=(width, height), borderValue=(114, 114, 114))\n else: # affine\n img = cv2.warpAffine(img, M[:2], dsize=(width, height), borderValue=(114, 114, 114))\n\n # Visualize\n # import matplotlib.pyplot as plt\n # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel()\n # ax[0].imshow(img[:, :, ::-1]) # base\n # ax[1].imshow(img2[:, :, ::-1]) # warped\n\n # Transform label coordinates\n n = len(targets)\n if n:\n # warp points\n xy = np.ones((n * 4, 3))\n xy[:, :2] = targets[:, [1, 2, 3, 4, 1, 4, 3, 2]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1\n xy = xy @ M.T # transform\n if perspective:\n xy = (xy[:, :2] / xy[:, 2:3]).reshape(n, 8) # rescale\n else: # affine\n xy = xy[:, :2].reshape(n, 8)\n\n # create new boxes\n x = xy[:, [0, 2, 4, 6]]\n y = xy[:, [1, 3, 5, 7]]\n xy = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T\n\n # # apply angle-based reduction of bounding boxes\n # radians = a * math.pi / 180\n # reduction = max(abs(math.sin(radians)), abs(math.cos(radians))) ** 0.5\n # x = (xy[:, 2] + xy[:, 0]) / 2\n # y = (xy[:, 3] + xy[:, 1]) / 2\n # w = (xy[:, 2] - xy[:, 0]) * reduction\n # h = (xy[:, 3] - xy[:, 1]) * reduction\n # xy = np.concatenate((x - w / 2, y - h / 2, x + w / 2, y + h / 2)).reshape(4, n).T\n\n # clip boxes\n xy[:, [0, 2]] = xy[:, [0, 2]].clip(0, width)\n xy[:, [1, 3]] = xy[:, [1, 3]].clip(0, height)\n\n # filter candidates\n i = box_candidates(box1=targets[:, 1:5].T * s, box2=xy.T)\n targets = targets[i]\n targets[:, 1:5] = xy[i]\n\n return img, targets\n\n\ndef box_candidates(box1, box2, wh_thr=2, ar_thr=20, area_thr=0.1): # box1(4,n), box2(4,n)\n # Compute candidate boxes: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio\n w1, h1 = box1[2] - box1[0], box1[3] - box1[1]\n w2, h2 = box2[2] - box2[0], box2[3] - box2[1]\n ar = np.maximum(w2 / (h2 + 1e-16), h2 / (w2 + 1e-16)) # aspect ratio\n return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + 1e-16) > area_thr) & (ar < ar_thr) # candidates\n\n\ndef cutout(image, labels):\n # Applies image cutout augmentation https://arxiv.org/abs/1708.04552\n h, w = image.shape[:2]\n\n def bbox_ioa(box1, box2):\n # Returns the intersection over box2 area given box1, box2. box1 is 4, box2 is nx4. boxes are x1y1x2y2\n box2 = box2.transpose()\n\n # Get the coordinates of bounding boxes\n b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]\n b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]\n\n # Intersection area\n inter_area = (np.minimum(b1_x2, b2_x2) - np.maximum(b1_x1, b2_x1)).clip(0) * \\\n (np.minimum(b1_y2, b2_y2) - np.maximum(b1_y1, b2_y1)).clip(0)\n\n # box2 area\n box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + 1e-16\n\n # Intersection over box2 area\n return inter_area / box2_area\n\n # create random masks\n scales = [0.5] * 1 + [0.25] * 2 + [0.125] * 4 + [0.0625] * 8 + [0.03125] * 16 # image size fraction\n for s in scales:\n mask_h = random.randint(1, int(h * s))\n mask_w = random.randint(1, int(w * s))\n\n # box\n xmin = max(0, random.randint(0, w) - mask_w // 2)\n ymin = max(0, random.randint(0, h) - mask_h // 2)\n xmax = min(w, xmin + mask_w)\n ymax = min(h, ymin + mask_h)\n\n # apply random color mask\n image[ymin:ymax, xmin:xmax] = [random.randint(64, 191) for _ in range(3)]\n\n # return unobscured labels\n if len(labels) and s > 0.03:\n box = np.array([xmin, ymin, xmax, ymax], dtype=np.float32)\n ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area\n labels = labels[ioa < 0.60] # remove >60% obscured labels\n\n return labels\n\n\ndef create_folder(path='./new'):\n # Create folder\n if os.path.exists(path):\n shutil.rmtree(path) # delete output folder\n os.makedirs(path) # make new output folder\n\n\ndef flatten_recursive(path='../coco128'):\n # Flatten a recursive directory by bringing all files to top level\n new_path = Path(path + '_flat')\n create_folder(new_path)\n for file in tqdm(glob.glob(str(Path(path)) + '/**/*.*', recursive=True)):\n shutil.copyfile(file, new_path / Path(file).name)\n\n\n" }, { "path": "asone/detectors/yolor/utils/export.py", "content": "import argparse\n\nimport torch\nfrom asone.detectors.yolor.models.models import *\nfrom asone.detectors.yolor.utils.google_utils import attempt_download\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument('--weights', type=str, default='./yolov4.pt', help='weights path')\n parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='image size')\n parser.add_argument('--batch-size', type=int, default=1, help='batch size')\n parser.add_argument('--cfg', type=str, default='cfg/yolor_p6.cfg', help='*.cfg path')\n opt = parser.parse_args()\n opt.img_size *= 2 if len(opt.img_size) == 1 else 1 # expand\n # print(opt)\n\n # Input\n img = torch.zeros((opt.batch_size, 3, *opt.img_size)) # image size(1,3,320,192) iDetection\n\n # Load PyTorch model\n attempt_download(opt.weights)\n # print(ad)\n # model = Darknet(cfg, ).cuda()\n model.load_state_dict(torch.load(opt.weights, map_location=device)['model'])\n print(type(model))\n print(\"*\"*50)\n exit()\n model.eval()\n model.model[-1].export = True # set Detect() layer export=True\n y = model(img) # dry run\n\n # print(\"-------------------\")\n # model = Darknet(cfg, imgsz).cuda()\n # model.load_state_dict(torch.load(weights[0], map_location=device)['model'])\n #model = attempt_load(weights, map_location=device) # load FP32 model\n #imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size\n # model.to(device).eval()\n # TorchScript export\n try:\n print('\\nStarting TorchScript export with torch %s...' % torch.__version__)\n f = opt.weights.replace('.pt', '.torchscript.pt') # filename\n ts = torch.jit.trace(model, img)\n ts.save(f)\n print('TorchScript export success, saved as %s' % f)\n except Exception as e:\n print('TorchScript export failure: %s' % e)\n\n # ONNX export\n try:\n import onnx\n\n print('\\nStarting ONNX export with onnx %s...' % onnx.__version__)\n f = opt.weights.replace('.pt', '.onnx') # filename\n model.fuse() # only for ONNX\n torch.onnx.export(model, img, f, verbose=False, opset_version=12, input_names=['images'],\n output_names=['classes', 'boxes'] if y is None else ['output'])\n\n # Checks\n onnx_model = onnx.load(f) # load onnx model\n onnx.checker.check_model(onnx_model) # check onnx model\n print(onnx.helper.printable_graph(onnx_model.graph)) # print a human readable model\n print('ONNX export success, saved as %s' % f)\n except Exception as e:\n print('ONNX export failure: %s' % e)\n\n # CoreML export\n try:\n import coremltools as ct\n\n print('\\nStarting CoreML export with coremltools %s...' % ct.__version__)\n # convert model from torchscript and apply pixel scaling as per detect.py\n model = ct.convert(ts, inputs=[ct.ImageType(name='images', shape=img.shape, scale=1 / 255.0, bias=[0, 0, 0])])\n f = opt.weights.replace('.pt', '.mlmodel') # filename\n model.save(f)\n print('CoreML export success, saved as %s' % f)\n except Exception as e:\n print('CoreML export failure: %s' % e)\n\n # Finish\n print('\\nExport complete. Visualize with https://github.com/lutzroeder/netron.')\n" }, { "path": "asone/detectors/yolor/utils/general.py", "content": "# General utils\n\nimport glob\nimport logging\nimport math\nimport os\nimport platform\nimport random\nimport re\nimport subprocess\nimport time\nfrom pathlib import Path\n\nimport cv2\nimport matplotlib\nimport numpy as np\nimport torch\nimport yaml\n\nfrom asone.detectors.yolor.utils.google_utils import gsutil_getsize\nfrom asone.detectors.yolor.utils.metrics import fitness \nfrom asone.detectors.yolor.utils.torch_utils import init_torch_seeds\n\n# Set printoptions\ntorch.set_printoptions(linewidth=320, precision=5, profile='long')\nnp.set_printoptions(linewidth=320, formatter={'float_kind': '{:11.5g}'.format}) # format short g, %precision=5\nmatplotlib.rc('font', **{'size': 11})\n\n# Prevent OpenCV from multithreading (to use PyTorch DataLoader)\ncv2.setNumThreads(0)\n\n\ndef set_logging(rank=-1):\n logging.basicConfig(\n format=\"%(message)s\",\n level=logging.INFO if rank in [-1, 0] else logging.WARN)\n\n\ndef init_seeds(seed=0):\n random.seed(seed)\n np.random.seed(seed)\n init_torch_seeds(seed)\n\n\ndef get_latest_run(search_dir='.'):\n # Return path to most recent 'last.pt' in /runs (i.e. to --resume from)\n last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True)\n return max(last_list, key=os.path.getctime) if last_list else ''\n\n\ndef check_git_status():\n # Suggest 'git pull' if repo is out of date\n if platform.system() in ['Linux', 'Darwin'] and not os.path.isfile('/.dockerenv'):\n s = subprocess.check_output('if [ -d .git ]; then git fetch && git status -uno; fi', shell=True).decode('utf-8')\n if 'Your branch is behind' in s:\n print(s[s.find('Your branch is behind'):s.find('\\n\\n')] + '\\n')\n\n\ndef check_img_size(img_size, s=32):\n # Verify img_size is a multiple of stride s\n new_size = make_divisible(img_size, int(s)) # ceil gs-multiple\n if new_size != img_size:\n print('WARNING: --img-size %g must be multiple of max stride %g, updating to %g' % (img_size, s, new_size))\n return new_size\n\n\ndef check_file(file):\n # Search for file if not found\n if os.path.isfile(file) or file == '':\n return file\n else:\n files = glob.glob('./**/' + file, recursive=True) # find file\n assert len(files), 'File Not Found: %s' % file # assert file was found\n assert len(files) == 1, \"Multiple files match '%s', specify exact path: %s\" % (file, files) # assert unique\n return files[0] # return file\n\n\ndef check_dataset(dict):\n # Download dataset if not found locally\n val, s = dict.get('val'), dict.get('download')\n if val and len(val):\n val = [Path(x).resolve() for x in (val if isinstance(val, list) else [val])] # val path\n if not all(x.exists() for x in val):\n print('\\nWARNING: Dataset not found, nonexistent paths: %s' % [str(x) for x in val if not x.exists()])\n if s and len(s): # download script\n print('Downloading %s ...' % s)\n if s.startswith('http') and s.endswith('.zip'): # URL\n f = Path(s).name # filename\n torch.hub.download_url_to_file(s, f)\n r = os.system('unzip -q %s -d ../ && rm %s' % (f, f)) # unzip\n else: # bash script\n r = os.system(s)\n print('Dataset autodownload %s\\n' % ('success' if r == 0 else 'failure')) # analyze return value\n else:\n raise Exception('Dataset not found.')\n\n\ndef make_divisible(x, divisor):\n # Returns x evenly divisible by divisor\n return math.ceil(x / divisor) * divisor\n\n\ndef labels_to_class_weights(labels, nc=80):\n # Get class weights (inverse frequency) from training labels\n if labels[0] is None: # no labels loaded\n return torch.Tensor()\n\n labels = np.concatenate(labels, 0) # labels.shape = (866643, 5) for COCO\n classes = labels[:, 0].astype(np.int) # labels = [class xywh]\n weights = np.bincount(classes, minlength=nc) # occurrences per class\n\n # Prepend gridpoint count (for uCE training)\n # gpi = ((320 / 32 * np.array([1, 2, 4])) ** 2 * 3).sum() # gridpoints per image\n # weights = np.hstack([gpi * len(labels) - weights.sum() * 9, weights * 9]) ** 0.5 # prepend gridpoints to start\n\n weights[weights == 0] = 1 # replace empty bins with 1\n weights = 1 / weights # number of targets per class\n weights /= weights.sum() # normalize\n return torch.from_numpy(weights)\n\n\ndef labels_to_image_weights(labels, nc=80, class_weights=np.ones(80)):\n # Produces image weights based on class mAPs\n n = len(labels)\n class_counts = np.array([np.bincount(labels[i][:, 0].astype(np.int), minlength=nc) for i in range(n)])\n image_weights = (class_weights.reshape(1, nc) * class_counts).sum(1)\n # index = random.choices(range(n), weights=image_weights, k=1) # weight image sample\n return image_weights\n\n\ndef coco80_to_coco91_class(): # converts 80-index (val2014) to 91-index (paper)\n # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/\n # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\\n')\n # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\\n')\n # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco\n # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet\n x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34,\n 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,\n 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90]\n return x\n\n\ndef xyxy2xywh(x):\n # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = (x[:, 0] + x[:, 2]) / 2 # x center\n y[:, 1] = (x[:, 1] + x[:, 3]) / 2 # y center\n y[:, 2] = x[:, 2] - x[:, 0] # width\n y[:, 3] = x[:, 3] - x[:, 1] # height\n return y\n\n\ndef xywh2xyxy(x):\n # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x\n y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y\n y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x\n y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y\n return y\n\n\ndef scale_coords(img1_shape, coords, img0_shape, ratio_pad=None):\n # Rescale coords (xyxy) from img1_shape to img0_shape\n if ratio_pad is None: # calculate from img0_shape\n gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new\n pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding\n else:\n gain = ratio_pad[0][0]\n pad = ratio_pad[1]\n\n coords[:, [0, 2]] -= pad[0] # x padding\n coords[:, [1, 3]] -= pad[1] # y padding\n coords[:, :4] /= gain\n clip_coords(coords, img0_shape)\n return coords\n\n\ndef clip_coords(boxes, img_shape):\n # Clip bounding xyxy bounding boxes to image shape (height, width)\n boxes[:, 0].clamp_(0, img_shape[1]) # x1\n boxes[:, 1].clamp_(0, img_shape[0]) # y1\n boxes[:, 2].clamp_(0, img_shape[1]) # x2\n boxes[:, 3].clamp_(0, img_shape[0]) # y2\n\n\ndef bbox_iou(box1, box2, x1y1x2y2=True, GIoU=False, DIoU=False, CIoU=False, EIoU=False, ECIoU=False, eps=1e-9):\n # Returns the IoU of box1 to box2. box1 is 4, box2 is nx4\n box2 = box2.T\n\n # Get the coordinates of bounding boxes\n if x1y1x2y2: # x1, y1, x2, y2 = box1\n b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]\n b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]\n else: # transform from xywh to xyxy\n b1_x1, b1_x2 = box1[0] - box1[2] / 2, box1[0] + box1[2] / 2\n b1_y1, b1_y2 = box1[1] - box1[3] / 2, box1[1] + box1[3] / 2\n b2_x1, b2_x2 = box2[0] - box2[2] / 2, box2[0] + box2[2] / 2\n b2_y1, b2_y2 = box2[1] - box2[3] / 2, box2[1] + box2[3] / 2\n\n # Intersection area\n inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \\\n (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0)\n\n # Union Area\n w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps\n w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps\n union = w1 * h1 + w2 * h2 - inter + eps\n\n iou = inter / union\n if GIoU or DIoU or CIoU or EIoU or ECIoU:\n cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1) # convex (smallest enclosing box) width\n ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height\n if CIoU or DIoU or EIoU or ECIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1\n c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared\n rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 +\n (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center distance squared\n if DIoU:\n return iou - rho2 / c2 # DIoU\n elif CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47\n v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2)\n with torch.no_grad():\n alpha = v / ((1 + eps) - iou + v)\n return iou - (rho2 / c2 + v * alpha) # CIoU\n elif EIoU: # Efficient IoU https://arxiv.org/abs/2101.08158\n rho3 = (w1-w2) **2\n c3 = cw ** 2 + eps\n rho4 = (h1-h2) **2\n c4 = ch ** 2 + eps\n return iou - rho2 / c2 - rho3 / c3 - rho4 / c4 # EIoU\n elif ECIoU:\n v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2)\n with torch.no_grad():\n alpha = v / ((1 + eps) - iou + v)\n rho3 = (w1-w2) **2\n c3 = cw ** 2 + eps\n rho4 = (h1-h2) **2\n c4 = ch ** 2 + eps\n return iou - v * alpha - rho2 / c2 - rho3 / c3 - rho4 / c4 # ECIoU\n else: # GIoU https://arxiv.org/pdf/1902.09630.pdf\n c_area = cw * ch + eps # convex area\n return iou - (c_area - union) / c_area # GIoU\n else:\n return iou # IoU\n\n\ndef box_iou(box1, box2):\n # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py\n \"\"\"\n Return intersection-over-union (Jaccard index) of boxes.\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\n Arguments:\n box1 (Tensor[N, 4])\n box2 (Tensor[M, 4])\n Returns:\n iou (Tensor[N, M]): the NxM matrix containing the pairwise\n IoU values for every element in boxes1 and boxes2\n \"\"\"\n\n def box_area(box):\n # box = 4xn\n return (box[2] - box[0]) * (box[3] - box[1])\n\n area1 = box_area(box1.T)\n area2 = box_area(box2.T)\n\n # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)\n inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2)\n return inter / (area1[:, None] + area2 - inter) # iou = inter / (area1 + area2 - inter)\n\n\ndef wh_iou(wh1, wh2):\n # Returns the nxm IoU matrix. wh1 is nx2, wh2 is mx2\n wh1 = wh1[:, None] # [N,1,2]\n wh2 = wh2[None] # [1,M,2]\n inter = torch.min(wh1, wh2).prod(2) # [N,M]\n return inter / (wh1.prod(2) + wh2.prod(2) - inter) # iou = inter / (area1 + area2 - inter)\n\n\ndef non_max_suppression(prediction, conf_thres=0.1, iou_thres=0.6, merge=False, classes=None, agnostic=False):\n \"\"\"Performs Non-Maximum Suppression (NMS) on inference results\n\n Returns:\n detections with shape: nx6 (x1, y1, x2, y2, conf, cls)\n \"\"\"\n\n nc = prediction[0].shape[1] - 5 # number of classes\n xc = prediction[..., 4] > conf_thres # candidates\n\n # Settings\n min_wh, max_wh = 2, 4096 # (pixels) minimum and maximum box width and height\n max_det = 300 # maximum number of detections per image\n time_limit = 10.0 # seconds to quit after\n redundant = True # require redundant detections\n multi_label = nc > 1 # multiple labels per box (adds 0.5ms/img)\n\n t = time.time()\n output = [torch.zeros(0, 6)] * prediction.shape[0]\n for xi, x in enumerate(prediction): # image index, image inference\n # Apply constraints\n # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height\n x = x[xc[xi]] # confidence\n\n # If none remain process next image\n if not x.shape[0]:\n continue\n\n # Compute conf\n x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf\n\n # Box (center x, center y, width, height) to (x1, y1, x2, y2)\n box = xywh2xyxy(x[:, :4])\n\n # Detections matrix nx6 (xyxy, conf, cls)\n if multi_label:\n i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T\n x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1)\n else: # best class only\n conf, j = x[:, 5:].max(1, keepdim=True)\n x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres]\n\n # Filter by class\n if classes:\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\n\n # Apply finite constraint\n # if not torch.isfinite(x).all():\n # x = x[torch.isfinite(x).all(1)]\n\n # If none remain process next image\n n = x.shape[0] # number of boxes\n if not n:\n continue\n\n # Sort by confidence\n # x = x[x[:, 4].argsort(descending=True)]\n\n # Batched NMS\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\n i = torch.ops.torchvision.nms(boxes, scores, iou_thres)\n if i.shape[0] > max_det: # limit detections\n i = i[:max_det]\n if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)\n # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\n weights = iou * scores[None] # box weights\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\n if redundant:\n i = i[iou.sum(1) > 1] # require redundancy\n\n output[xi] = x[i]\n if (time.time() - t) > time_limit:\n break # time limit exceeded\n\n return output\n\n\ndef strip_optimizer(f='weights/best.pt', s=''): # from utils.general import *; strip_optimizer()\n # Strip optimizer from 'f' to finalize training, optionally save as 's'\n x = torch.load(f, map_location=torch.device('cpu'))\n x['optimizer'] = None\n x['training_results'] = None\n x['epoch'] = -1\n #x['model'].half() # to FP16\n #for p in x['model'].parameters():\n # p.requires_grad = False\n torch.save(x, s or f)\n mb = os.path.getsize(s or f) / 1E6 # filesize\n print('Optimizer stripped from %s,%s %.1fMB' % (f, (' saved as %s,' % s) if s else '', mb))\n\n\ndef print_mutation(hyp, results, yaml_file='hyp_evolved.yaml', bucket=''):\n # Print mutation results to evolve.txt (for use with train.py --evolve)\n a = '%10s' * len(hyp) % tuple(hyp.keys()) # hyperparam keys\n b = '%10.3g' * len(hyp) % tuple(hyp.values()) # hyperparam values\n c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3)\n print('\\n%s\\n%s\\nEvolved fitness: %s\\n' % (a, b, c))\n\n if bucket:\n url = 'gs://%s/evolve.txt' % bucket\n if gsutil_getsize(url) > (os.path.getsize('evolve.txt') if os.path.exists('evolve.txt') else 0):\n os.system('gsutil cp %s .' % url) # download evolve.txt if larger than local\n\n with open('evolve.txt', 'a') as f: # append result\n f.write(c + b + '\\n')\n x = np.unique(np.loadtxt('evolve.txt', ndmin=2), axis=0) # load unique rows\n x = x[np.argsort(-fitness(x))] # sort\n np.savetxt('evolve.txt', x, '%10.3g') # save sort by fitness\n\n # Save yaml\n for i, k in enumerate(hyp.keys()):\n hyp[k] = float(x[0, i + 7])\n with open(yaml_file, 'w') as f:\n results = tuple(x[0, :7])\n c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3)\n f.write('# Hyperparameter Evolution Results\\n# Generations: %g\\n# Metrics: ' % len(x) + c + '\\n\\n')\n yaml.dump(hyp, f, sort_keys=False)\n\n if bucket:\n os.system('gsutil cp evolve.txt %s gs://%s' % (yaml_file, bucket)) # upload\n\n\ndef apply_classifier(x, model, img, im0):\n # applies a second stage classifier to yolo outputs\n im0 = [im0] if isinstance(im0, np.ndarray) else im0\n for i, d in enumerate(x): # per image\n if d is not None and len(d):\n d = d.clone()\n\n # Reshape and pad cutouts\n b = xyxy2xywh(d[:, :4]) # boxes\n b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # rectangle to square\n b[:, 2:] = b[:, 2:] * 1.3 + 30 # pad\n d[:, :4] = xywh2xyxy(b).long()\n\n # Rescale boxes from img_size to im0 size\n scale_coords(img.shape[2:], d[:, :4], im0[i].shape)\n\n # Classes\n pred_cls1 = d[:, 5].long()\n ims = []\n for j, a in enumerate(d): # per item\n cutout = im0[i][int(a[1]):int(a[3]), int(a[0]):int(a[2])]\n im = cv2.resize(cutout, (224, 224)) # BGR\n # cv2.imwrite('test%i.jpg' % j, cutout)\n\n im = im[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n im = np.ascontiguousarray(im, dtype=np.float32) # uint8 to float32\n im /= 255.0 # 0 - 255 to 0.0 - 1.0\n ims.append(im)\n\n pred_cls2 = model(torch.Tensor(ims).to(d.device)).argmax(1) # classifier prediction\n x[i] = x[i][pred_cls1 == pred_cls2] # retain matching class detections\n\n return x\n\n\ndef increment_path(path, exist_ok=True, sep=''):\n # Increment path, i.e. runs/exp --> runs/exp{sep}0, runs/exp{sep}1 etc.\n path = Path(path) # os-agnostic\n if (path.exists() and exist_ok) or (not path.exists()):\n return str(path)\n else:\n dirs = glob.glob(f\"{path}{sep}*\") # similar paths\n matches = [re.search(rf\"%s{sep}(\\d+)\" % path.stem, d) for d in dirs]\n i = [int(m.groups()[0]) for m in matches if m] # indices\n n = max(i) + 1 if i else 2 # increment number\n return f\"{path}{sep}{n}\" # update path\n" }, { "path": "asone/detectors/yolor/utils/google_utils.py", "content": "# Google utils: https://cloud.google.com/storage/docs/reference/libraries\n\nimport os\nimport platform\nimport subprocess\nimport time\nfrom pathlib import Path\n\nimport torch\nimport torch.nn as nn\n\n\ndef gsutil_getsize(url=''):\n # gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du\n s = subprocess.check_output('gsutil du %s' % url, shell=True).decode('utf-8')\n return eval(s.split(' ')[0]) if len(s) else 0 # bytes\n\n\ndef attempt_download(weights):\n # Attempt to download pretrained weights if not found locally\n weights = weights.strip().replace(\"'\", '')\n file = Path(weights).name\n\n msg = weights + ' missing, try downloading from https://github.com/WongKinYiu/yolor/releases/'\n models = ['yolor_p6.pt', 'yolor_w6.pt'] # available models\n\n if file in models and not os.path.isfile(weights):\n\n try: # GitHub\n url = 'https://github.com/WongKinYiu/yolor/releases/download/v1.0/' + file\n print('Downloading %s to %s...' % (url, weights))\n torch.hub.download_url_to_file(url, weights)\n assert os.path.exists(weights) and os.path.getsize(weights) > 1E6 # check\n except Exception as e: # GCP\n print('ERROR: Download failure.')\n print('')\n \n \ndef attempt_load(weights, map_location=None):\n # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a\n model = Ensemble()\n for w in weights if isinstance(weights, list) else [weights]:\n attempt_download(w)\n model.append(torch.load(w, map_location=map_location)['model'].float().fuse().eval()) # load FP32 model\n\n if len(model) == 1:\n return model[-1] # return model\n else:\n print('Ensemble created with %s\\n' % weights)\n for k in ['names', 'stride']:\n setattr(model, k, getattr(model[-1], k))\n return model # return ensemble\n\n\ndef gdrive_download(id='1n_oKgR81BJtqk75b00eAjdv03qVCQn2f', name='coco128.zip'):\n # Downloads a file from Google Drive. from utils.google_utils import *; gdrive_download()\n t = time.time()\n\n print('Downloading https://drive.google.com/uc?export=download&id=%s as %s... ' % (id, name), end='')\n os.remove(name) if os.path.exists(name) else None # remove existing\n os.remove('cookie') if os.path.exists('cookie') else None\n\n # Attempt file download\n out = \"NUL\" if platform.system() == \"Windows\" else \"/dev/null\"\n os.system('curl -c ./cookie -s -L \"drive.google.com/uc?export=download&id=%s\" > %s ' % (id, out))\n if os.path.exists('cookie'): # large file\n s = 'curl -Lb ./cookie \"drive.google.com/uc?export=download&confirm=%s&id=%s\" -o %s' % (get_token(), id, name)\n else: # small file\n s = 'curl -s -L -o %s \"drive.google.com/uc?export=download&id=%s\"' % (name, id)\n r = os.system(s) # execute, capture return\n os.remove('cookie') if os.path.exists('cookie') else None\n\n # Error check\n if r != 0:\n os.remove(name) if os.path.exists(name) else None # remove partial\n print('Download error ') # raise Exception('Download error')\n return r\n\n # Unzip if archive\n if name.endswith('.zip'):\n print('unzipping... ', end='')\n os.system('unzip -q %s' % name) # unzip\n os.remove(name) # remove zip to free space\n\n print('Done (%.1fs)' % (time.time() - t))\n return r\n\n\ndef get_token(cookie=\"./cookie\"):\n with open(cookie) as f:\n for line in f:\n if \"download\" in line:\n return line.split()[-1]\n return \"\"\n\nclass Ensemble(nn.ModuleList):\n # Ensemble of models\n def __init__(self):\n super().__init__()\n\n def forward(self, x, augment=False, profile=False, visualize=False):\n y = [module(x, augment, profile, visualize)[0] for module in self]\n # y = torch.stack(y).max(0)[0] # max ensemble\n # y = torch.stack(y).mean(0) # mean ensemble\n y = torch.cat(y, 1) # nms ensemble\n return y, None # inference, train output\n# def upload_blob(bucket_name, source_file_name, destination_blob_name):\n# # Uploads a file to a bucket\n# # https://cloud.google.com/storage/docs/uploading-objects#storage-upload-object-python\n#\n# storage_client = storage.Client()\n# bucket = storage_client.get_bucket(bucket_name)\n# blob = bucket.blob(destination_blob_name)\n#\n# blob.upload_from_filename(source_file_name)\n#\n# print('File {} uploaded to {}.'.format(\n# source_file_name,\n# destination_blob_name))\n#\n#\n# def download_blob(bucket_name, source_blob_name, destination_file_name):\n# # Uploads a blob from a bucket\n# storage_client = storage.Client()\n# bucket = storage_client.get_bucket(bucket_name)\n# blob = bucket.blob(source_blob_name)\n#\n# blob.download_to_filename(destination_file_name)\n#\n# print('Blob {} downloaded to {}.'.format(\n# source_blob_name,\n# destination_file_name))\n" }, { "path": "asone/detectors/yolor/utils/layers.py", "content": "from asone.detectors.yolor.utils.general import *\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\ntry:\n from mish_cuda import MishCuda as Mish\n \nexcept:\n class Mish(nn.Module): # https://github.com/digantamisra98/Mish\n def forward(self, x):\n return x * F.softplus(x).tanh()\n \ntry:\n from pytorch_wavelets import DWTForward, DWTInverse\n\n class DWT(nn.Module):\n def __init__(self):\n super(DWT, self).__init__()\n self.xfm = DWTForward(J=1, wave='db1', mode='zero')\n\n def forward(self, x):\n b,c,w,h = x.shape\n yl, yh = self.xfm(x)\n return torch.cat([yl/2., yh[0].view(b,-1,w//2,h//2)/2.+.5], 1)\n \nexcept: # using Reorg instead\n class DWT(nn.Module):\n def forward(self, x):\n return torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1)\n\n\nclass Reorg(nn.Module):\n def forward(self, x):\n return torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1)\n\n\ndef make_divisible(v, divisor):\n # Function ensures all layers have a channel number that is divisible by 8\n # https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py\n return math.ceil(v / divisor) * divisor\n\n\nclass Flatten(nn.Module):\n # Use after nn.AdaptiveAvgPool2d(1) to remove last 2 dimensions\n def forward(self, x):\n return x.view(x.size(0), -1)\n\n\nclass Concat(nn.Module):\n # Concatenate a list of tensors along dimension\n def __init__(self, dimension=1):\n super(Concat, self).__init__()\n self.d = dimension\n\n def forward(self, x):\n return torch.cat(x, self.d)\n\n\nclass FeatureConcat(nn.Module):\n def __init__(self, layers):\n super(FeatureConcat, self).__init__()\n self.layers = layers # layer indices\n self.multiple = len(layers) > 1 # multiple layers flag\n\n def forward(self, x, outputs):\n return torch.cat([outputs[i] for i in self.layers], 1) if self.multiple else outputs[self.layers[0]]\n\n\nclass FeatureConcat2(nn.Module):\n def __init__(self, layers):\n super(FeatureConcat2, self).__init__()\n self.layers = layers # layer indices\n self.multiple = len(layers) > 1 # multiple layers flag\n\n def forward(self, x, outputs):\n return torch.cat([outputs[self.layers[0]], outputs[self.layers[1]].detach()], 1)\n\n\nclass FeatureConcat3(nn.Module):\n def __init__(self, layers):\n super(FeatureConcat3, self).__init__()\n self.layers = layers # layer indices\n self.multiple = len(layers) > 1 # multiple layers flag\n\n def forward(self, x, outputs):\n return torch.cat([outputs[self.layers[0]], outputs[self.layers[1]].detach(), outputs[self.layers[2]].detach()], 1)\n\n\nclass FeatureConcat_l(nn.Module):\n def __init__(self, layers):\n super(FeatureConcat_l, self).__init__()\n self.layers = layers # layer indices\n self.multiple = len(layers) > 1 # multiple layers flag\n\n def forward(self, x, outputs):\n return torch.cat([outputs[i][:,:outputs[i].shape[1]//2,:,:] for i in self.layers], 1) if self.multiple else outputs[self.layers[0]][:,:outputs[self.layers[0]].shape[1]//2,:,:]\n\n\nclass WeightedFeatureFusion(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers, weight=False):\n super(WeightedFeatureFusion, self).__init__()\n self.layers = layers # layer indices\n self.weight = weight # apply weights boolean\n self.n = len(layers) + 1 # number of layers\n if weight:\n self.w = nn.Parameter(torch.zeros(self.n), requires_grad=True) # layer weights\n\n def forward(self, x, outputs):\n # Weights\n if self.weight:\n w = torch.sigmoid(self.w) * (2 / self.n) # sigmoid weights (0-1)\n x = x * w[0]\n\n # Fusion\n nx = x.shape[1] # input channels\n for i in range(self.n - 1):\n a = outputs[self.layers[i]] * w[i + 1] if self.weight else outputs[self.layers[i]] # feature to add\n na = a.shape[1] # feature channels\n\n # Adjust channels\n if nx == na: # same shape\n x = x + a\n elif nx > na: # slice input\n x[:, :na] = x[:, :na] + a # or a = nn.ZeroPad2d((0, 0, 0, 0, 0, dc))(a); x = x + a\n else: # slice feature\n x = x + a[:, :nx]\n\n return x\n\n\nclass MixConv2d(nn.Module): # MixConv: Mixed Depthwise Convolutional Kernels https://arxiv.org/abs/1907.09595\n def __init__(self, in_ch, out_ch, k=(3, 5, 7), stride=1, dilation=1, bias=True, method='equal_params'):\n super(MixConv2d, self).__init__()\n\n groups = len(k)\n if method == 'equal_ch': # equal channels per group\n i = torch.linspace(0, groups - 1E-6, out_ch).floor() # out_ch indices\n ch = [(i == g).sum() for g in range(groups)]\n else: # 'equal_params': equal parameter count per group\n b = [out_ch] + [0] * groups\n a = np.eye(groups + 1, groups, k=-1)\n a -= np.roll(a, 1, axis=1)\n a *= np.array(k) ** 2\n a[0] = 1\n ch = np.linalg.lstsq(a, b, rcond=None)[0].round().astype(int) # solve for equal weight indices, ax = b\n\n self.m = nn.ModuleList([nn.Conv2d(in_channels=in_ch,\n out_channels=ch[g],\n kernel_size=k[g],\n stride=stride,\n padding=k[g] // 2, # 'same' pad\n dilation=dilation,\n bias=bias) for g in range(groups)])\n\n def forward(self, x):\n return torch.cat([m(x) for m in self.m], 1)\n\n\n# Activation functions below -------------------------------------------------------------------------------------------\nclass SwishImplementation(torch.autograd.Function):\n @staticmethod\n def forward(ctx, x):\n ctx.save_for_backward(x)\n return x * torch.sigmoid(x)\n\n @staticmethod\n def backward(ctx, grad_output):\n x = ctx.saved_tensors[0]\n sx = torch.sigmoid(x) # sigmoid(ctx)\n return grad_output * (sx * (1 + x * (1 - sx)))\n\n\nclass MishImplementation(torch.autograd.Function):\n @staticmethod\n def forward(ctx, x):\n ctx.save_for_backward(x)\n return x.mul(torch.tanh(F.softplus(x))) # x * tanh(ln(1 + exp(x)))\n\n @staticmethod\n def backward(ctx, grad_output):\n x = ctx.saved_tensors[0]\n sx = torch.sigmoid(x)\n fx = F.softplus(x).tanh()\n return grad_output * (fx + x * sx * (1 - fx * fx))\n\n\nclass MemoryEfficientSwish(nn.Module):\n def forward(self, x):\n return SwishImplementation.apply(x)\n\n\nclass MemoryEfficientMish(nn.Module):\n def forward(self, x):\n return MishImplementation.apply(x)\n\n\nclass Swish(nn.Module):\n def forward(self, x):\n return x * torch.sigmoid(x)\n\n\nclass HardSwish(nn.Module): # https://arxiv.org/pdf/1905.02244.pdf\n def forward(self, x):\n return x * F.hardtanh(x + 3, 0., 6., True) / 6.\n\n\nclass DeformConv2d(nn.Module):\n def __init__(self, inc, outc, kernel_size=3, padding=1, stride=1, bias=None, modulation=False):\n \"\"\"\n Args:\n modulation (bool, optional): If True, Modulated Defomable Convolution (Deformable ConvNets v2).\n \"\"\"\n super(DeformConv2d, self).__init__()\n self.kernel_size = kernel_size\n self.padding = padding\n self.stride = stride\n self.zero_padding = nn.ZeroPad2d(padding)\n self.conv = nn.Conv2d(inc, outc, kernel_size=kernel_size, stride=kernel_size, bias=bias)\n\n self.p_conv = nn.Conv2d(inc, 2*kernel_size*kernel_size, kernel_size=3, padding=1, stride=stride)\n nn.init.constant_(self.p_conv.weight, 0)\n self.p_conv.register_backward_hook(self._set_lr)\n\n self.modulation = modulation\n if modulation:\n self.m_conv = nn.Conv2d(inc, kernel_size*kernel_size, kernel_size=3, padding=1, stride=stride)\n nn.init.constant_(self.m_conv.weight, 0)\n self.m_conv.register_backward_hook(self._set_lr)\n\n @staticmethod\n def _set_lr(module, grad_input, grad_output):\n grad_input = (grad_input[i] * 0.1 for i in range(len(grad_input)))\n grad_output = (grad_output[i] * 0.1 for i in range(len(grad_output)))\n\n def forward(self, x):\n offset = self.p_conv(x)\n if self.modulation:\n m = torch.sigmoid(self.m_conv(x))\n\n dtype = offset.data.type()\n ks = self.kernel_size\n N = offset.size(1) // 2\n\n if self.padding:\n x = self.zero_padding(x)\n\n # (b, 2N, h, w)\n p = self._get_p(offset, dtype)\n\n # (b, h, w, 2N)\n p = p.contiguous().permute(0, 2, 3, 1)\n q_lt = p.detach().floor()\n q_rb = q_lt + 1\n\n q_lt = torch.cat([torch.clamp(q_lt[..., :N], 0, x.size(2)-1), torch.clamp(q_lt[..., N:], 0, x.size(3)-1)], dim=-1).long()\n q_rb = torch.cat([torch.clamp(q_rb[..., :N], 0, x.size(2)-1), torch.clamp(q_rb[..., N:], 0, x.size(3)-1)], dim=-1).long()\n q_lb = torch.cat([q_lt[..., :N], q_rb[..., N:]], dim=-1)\n q_rt = torch.cat([q_rb[..., :N], q_lt[..., N:]], dim=-1)\n\n # clip p\n p = torch.cat([torch.clamp(p[..., :N], 0, x.size(2)-1), torch.clamp(p[..., N:], 0, x.size(3)-1)], dim=-1)\n\n # bilinear kernel (b, h, w, N)\n g_lt = (1 + (q_lt[..., :N].type_as(p) - p[..., :N])) * (1 + (q_lt[..., N:].type_as(p) - p[..., N:]))\n g_rb = (1 - (q_rb[..., :N].type_as(p) - p[..., :N])) * (1 - (q_rb[..., N:].type_as(p) - p[..., N:]))\n g_lb = (1 + (q_lb[..., :N].type_as(p) - p[..., :N])) * (1 - (q_lb[..., N:].type_as(p) - p[..., N:]))\n g_rt = (1 - (q_rt[..., :N].type_as(p) - p[..., :N])) * (1 + (q_rt[..., N:].type_as(p) - p[..., N:]))\n\n # (b, c, h, w, N)\n x_q_lt = self._get_x_q(x, q_lt, N)\n x_q_rb = self._get_x_q(x, q_rb, N)\n x_q_lb = self._get_x_q(x, q_lb, N)\n x_q_rt = self._get_x_q(x, q_rt, N)\n\n # (b, c, h, w, N)\n x_offset = g_lt.unsqueeze(dim=1) * x_q_lt + \\\n g_rb.unsqueeze(dim=1) * x_q_rb + \\\n g_lb.unsqueeze(dim=1) * x_q_lb + \\\n g_rt.unsqueeze(dim=1) * x_q_rt\n\n # modulation\n if self.modulation:\n m = m.contiguous().permute(0, 2, 3, 1)\n m = m.unsqueeze(dim=1)\n m = torch.cat([m for _ in range(x_offset.size(1))], dim=1)\n x_offset *= m\n\n x_offset = self._reshape_x_offset(x_offset, ks)\n out = self.conv(x_offset)\n\n return out\n\n def _get_p_n(self, N, dtype):\n p_n_x, p_n_y = torch.meshgrid(\n torch.arange(-(self.kernel_size-1)//2, (self.kernel_size-1)//2+1),\n torch.arange(-(self.kernel_size-1)//2, (self.kernel_size-1)//2+1))\n # (2N, 1)\n p_n = torch.cat([torch.flatten(p_n_x), torch.flatten(p_n_y)], 0)\n p_n = p_n.view(1, 2*N, 1, 1).type(dtype)\n\n return p_n\n\n def _get_p_0(self, h, w, N, dtype):\n p_0_x, p_0_y = torch.meshgrid(\n torch.arange(1, h*self.stride+1, self.stride),\n torch.arange(1, w*self.stride+1, self.stride))\n p_0_x = torch.flatten(p_0_x).view(1, 1, h, w).repeat(1, N, 1, 1)\n p_0_y = torch.flatten(p_0_y).view(1, 1, h, w).repeat(1, N, 1, 1)\n p_0 = torch.cat([p_0_x, p_0_y], 1).type(dtype)\n\n return p_0\n\n def _get_p(self, offset, dtype):\n N, h, w = offset.size(1)//2, offset.size(2), offset.size(3)\n\n # (1, 2N, 1, 1)\n p_n = self._get_p_n(N, dtype)\n # (1, 2N, h, w)\n p_0 = self._get_p_0(h, w, N, dtype)\n p = p_0 + p_n + offset\n return p\n\n def _get_x_q(self, x, q, N):\n b, h, w, _ = q.size()\n padded_w = x.size(3)\n c = x.size(1)\n # (b, c, h*w)\n x = x.contiguous().view(b, c, -1)\n\n # (b, h, w, N)\n index = q[..., :N]*padded_w + q[..., N:] # offset_x*w + offset_y\n # (b, c, h*w*N)\n index = index.contiguous().unsqueeze(dim=1).expand(-1, c, -1, -1, -1).contiguous().view(b, c, -1)\n\n x_offset = x.gather(dim=-1, index=index).contiguous().view(b, c, h, w, N)\n\n return x_offset\n\n @staticmethod\n def _reshape_x_offset(x_offset, ks):\n b, c, h, w, N = x_offset.size()\n x_offset = torch.cat([x_offset[..., s:s+ks].contiguous().view(b, c, h, w*ks) for s in range(0, N, ks)], dim=-1)\n x_offset = x_offset.contiguous().view(b, c, h*ks, w*ks)\n\n return x_offset\n \n \nclass GAP(nn.Module):\n def __init__(self):\n super(GAP, self).__init__()\n self.avg_pool = nn.AdaptiveAvgPool2d(1)\n def forward(self, x):\n #b, c, _, _ = x.size() \n return self.avg_pool(x)#.view(b, c)\n \n \nclass Silence(nn.Module):\n def __init__(self):\n super(Silence, self).__init__()\n def forward(self, x): \n return x\n\n\nclass ScaleChannel(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers):\n super(ScaleChannel, self).__init__()\n self.layers = layers # layer indices\n\n def forward(self, x, outputs):\n a = outputs[self.layers[0]]\n return x.expand_as(a) * a\n\n\nclass ShiftChannel(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers):\n super(ShiftChannel, self).__init__()\n self.layers = layers # layer indices\n\n def forward(self, x, outputs):\n a = outputs[self.layers[0]]\n return a.expand_as(x) + x\n\n\nclass ShiftChannel2D(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers):\n super(ShiftChannel2D, self).__init__()\n self.layers = layers # layer indices\n\n def forward(self, x, outputs):\n a = outputs[self.layers[0]].view(1,-1,1,1)\n return a.expand_as(x) + x\n\n\nclass ControlChannel(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers):\n super(ControlChannel, self).__init__()\n self.layers = layers # layer indices\n\n def forward(self, x, outputs):\n a = outputs[self.layers[0]]\n return a.expand_as(x) * x\n\n\nclass ControlChannel2D(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers):\n super(ControlChannel2D, self).__init__()\n self.layers = layers # layer indices\n\n def forward(self, x, outputs):\n a = outputs[self.layers[0]].view(1,-1,1,1)\n return a.expand_as(x) * x\n\n\nclass AlternateChannel(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers):\n super(AlternateChannel, self).__init__()\n self.layers = layers # layer indices\n\n def forward(self, x, outputs):\n a = outputs[self.layers[0]]\n return torch.cat([a.expand_as(x), x], dim=1)\n\n\nclass AlternateChannel2D(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers):\n super(AlternateChannel2D, self).__init__()\n self.layers = layers # layer indices\n\n def forward(self, x, outputs):\n a = outputs[self.layers[0]].view(1,-1,1,1)\n return torch.cat([a.expand_as(x), x], dim=1)\n\n\nclass SelectChannel(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers):\n super(SelectChannel, self).__init__()\n self.layers = layers # layer indices\n\n def forward(self, x, outputs):\n a = outputs[self.layers[0]]\n return a.sigmoid().expand_as(x) * x\n\n\nclass SelectChannel2D(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers):\n super(SelectChannel2D, self).__init__()\n self.layers = layers # layer indices\n\n def forward(self, x, outputs):\n a = outputs[self.layers[0]].view(1,-1,1,1)\n return a.sigmoid().expand_as(x) * x\n\n\nclass ScaleSpatial(nn.Module): # weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, layers):\n super(ScaleSpatial, self).__init__()\n self.layers = layers # layer indices\n\n def forward(self, x, outputs):\n a = outputs[self.layers[0]]\n return x * a\n \n\nclass ImplicitA(nn.Module):\n def __init__(self, channel):\n super(ImplicitA, self).__init__()\n self.channel = channel\n self.implicit = nn.Parameter(torch.zeros(1, channel, 1, 1))\n nn.init.normal_(self.implicit, std=.02)\n\n def forward(self):\n return self.implicit\n\n\nclass ImplicitC(nn.Module):\n def __init__(self, channel):\n super(ImplicitC, self).__init__()\n self.channel = channel\n self.implicit = nn.Parameter(torch.zeros(1, channel, 1, 1))\n nn.init.normal_(self.implicit, std=.02)\n\n def forward(self):\n return self.implicit\n\n\nclass ImplicitM(nn.Module):\n def __init__(self, channel):\n super(ImplicitM, self).__init__()\n self.channel = channel\n self.implicit = nn.Parameter(torch.ones(1, channel, 1, 1))\n nn.init.normal_(self.implicit, mean=1., std=.02)\n\n def forward(self):\n return self.implicit\n \n\n\nclass Implicit2DA(nn.Module):\n def __init__(self, atom, channel):\n super(Implicit2DA, self).__init__()\n self.channel = channel\n self.implicit = nn.Parameter(torch.zeros(1, atom, channel, 1))\n nn.init.normal_(self.implicit, std=.02)\n\n def forward(self):\n return self.implicit\n\n\nclass Implicit2DC(nn.Module):\n def __init__(self, atom, channel):\n super(Implicit2DC, self).__init__()\n self.channel = channel\n self.implicit = nn.Parameter(torch.zeros(1, atom, channel, 1))\n nn.init.normal_(self.implicit, std=.02)\n\n def forward(self):\n return self.implicit\n\n\nclass Implicit2DM(nn.Module):\n def __init__(self, atom, channel):\n super(Implicit2DM, self).__init__()\n self.channel = channel\n self.implicit = nn.Parameter(torch.ones(1, atom, channel, 1))\n nn.init.normal_(self.implicit, mean=1., std=.02)\n\n def forward(self):\n return self.implicit\n \n \n " }, { "path": "asone/detectors/yolor/utils/loss.py", "content": "# Loss functions\n\nimport torch\nimport torch.nn as nn\n\nfrom asone.detectors.yolor.utils.general import bbox_iou\nfrom asone.detectors.yolor.utils.torch_utils import is_parallel\n\n\ndef smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441\n # return positive, negative label smoothing BCE targets\n return 1.0 - 0.5 * eps, 0.5 * eps\n\n\nclass BCEBlurWithLogitsLoss(nn.Module):\n # BCEwithLogitLoss() with reduced missing label effects.\n def __init__(self, alpha=0.05):\n super(BCEBlurWithLogitsLoss, self).__init__()\n self.loss_fcn = nn.BCEWithLogitsLoss(reduction='none') # must be nn.BCEWithLogitsLoss()\n self.alpha = alpha\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n pred = torch.sigmoid(pred) # prob from logits\n dx = pred - true # reduce only missing label effects\n # dx = (pred - true).abs() # reduce missing label and false label effects\n alpha_factor = 1 - torch.exp((dx - 1) / (self.alpha + 1e-4))\n loss *= alpha_factor\n return loss.mean()\n\n\nclass FocalLoss(nn.Module):\n # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\n super(FocalLoss, self).__init__()\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\n self.gamma = gamma\n self.alpha = alpha\n self.reduction = loss_fcn.reduction\n self.loss_fcn.reduction = 'none' # required to apply FL to each element\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n # p_t = torch.exp(-loss)\n # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability\n\n # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py\n pred_prob = torch.sigmoid(pred) # prob from logits\n p_t = true * pred_prob + (1 - true) * (1 - pred_prob)\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\n modulating_factor = (1.0 - p_t) ** self.gamma\n loss *= alpha_factor * modulating_factor\n\n if self.reduction == 'mean':\n return loss.mean()\n elif self.reduction == 'sum':\n return loss.sum()\n else: # 'none'\n return loss\n\n\ndef compute_loss(p, targets, model): # predictions, targets, model\n device = targets.device\n #print(device)\n lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)\n tcls, tbox, indices, anchors = build_targets(p, targets, model) # targets\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['cls_pw']])).to(device)\n BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([h['obj_pw']])).to(device)\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n cp, cn = smooth_BCE(eps=0.0)\n\n # Focal loss\n g = h['fl_gamma'] # focal loss gamma\n if g > 0:\n BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)\n\n # Losses\n nt = 0 # number of targets\n no = len(p) # number of outputs\n balance = [4.0, 1.0, 0.4] if no == 3 else [4.0, 1.0, 0.4, 0.1] # P3-5 or P3-6\n balance = [4.0, 1.0, 0.5, 0.4, 0.1] if no == 5 else balance\n for i, pi in enumerate(p): # layer index, layer predictions\n b, a, gj, gi = indices[i] # image, anchor, gridy, gridx\n tobj = torch.zeros_like(pi[..., 0], device=device) # target obj\n\n n = b.shape[0] # number of targets\n if n:\n nt += n # cumulative targets\n ps = pi[b, a, gj, gi] # prediction subset corresponding to targets\n\n # Regression\n pxy = ps[:, :2].sigmoid() * 2. - 0.5\n pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]\n pbox = torch.cat((pxy, pwh), 1).to(device) # predicted box\n iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, CIoU=True) # iou(prediction, target)\n lbox += (1.0 - iou).mean() # iou loss\n\n # Objectness\n tobj[b, a, gj, gi] = (1.0 - model.gr) + model.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio\n\n # Classification\n if model.nc > 1: # cls loss (only if multiple classes)\n t = torch.full_like(ps[:, 5:], cn, device=device) # targets\n t[range(n), tcls[i]] = cp\n lcls += BCEcls(ps[:, 5:], t) # BCE\n\n # Append targets to text file\n # with open('targets.txt', 'a') as file:\n # [file.write('%11.5g ' * 4 % tuple(x) + '\\n') for x in torch.cat((txy[i], twh[i]), 1)]\n\n lobj += BCEobj(pi[..., 4], tobj) * balance[i] # obj loss\n\n s = 3 / no # output count scaling\n lbox *= h['box'] * s\n lobj *= h['obj'] * s * (1.4 if no >= 4 else 1.)\n lcls *= h['cls'] * s\n bs = tobj.shape[0] # batch size\n\n loss = lbox + lobj + lcls\n return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()\n\n\ndef build_targets(p, targets, model):\n nt = targets.shape[0] # number of anchors, targets\n tcls, tbox, indices, anch = [], [], [], []\n gain = torch.ones(6, device=targets.device) # normalized to gridspace gain\n off = torch.tensor([[1, 0], [0, 1], [-1, 0], [0, -1]], device=targets.device).float() # overlap offsets\n\n g = 0.5 # offset\n multi_gpu = is_parallel(model)\n for i, jj in enumerate(model.module.yolo_layers if multi_gpu else model.yolo_layers):\n # get number of grid points and anchor vec for this yolo layer\n anchors = model.module.module_list[jj].anchor_vec if multi_gpu else model.module_list[jj].anchor_vec\n gain[2:] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain\n\n # Match targets to anchors\n a, t, offsets = [], targets * gain, 0\n if nt:\n na = anchors.shape[0] # number of anchors\n at = torch.arange(na).view(na, 1).repeat(1, nt) # anchor tensor, same as .repeat_interleave(nt)\n r = t[None, :, 4:6] / anchors[:, None] # wh ratio\n j = torch.max(r, 1. / r).max(2)[0] < model.hyp['anchor_t'] # compare\n # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n) = wh_iou(anchors(3,2), gwh(n,2))\n a, t = at[j], t.repeat(na, 1, 1)[j] # filter\n\n # overlaps\n gxy = t[:, 2:4] # grid xy\n z = torch.zeros_like(gxy)\n j, k = ((gxy % 1. < g) & (gxy > 1.)).T\n l, m = ((gxy % 1. > (1 - g)) & (gxy < (gain[[2, 3]] - 1.))).T\n a, t = torch.cat((a, a[j], a[k], a[l], a[m]), 0), torch.cat((t, t[j], t[k], t[l], t[m]), 0)\n offsets = torch.cat((z, z[j] + off[0], z[k] + off[1], z[l] + off[2], z[m] + off[3]), 0) * g\n\n # Define\n b, c = t[:, :2].long().T # image, class\n gxy = t[:, 2:4] # grid xy\n gwh = t[:, 4:6] # grid wh\n gij = (gxy - offsets).long()\n gi, gj = gij.T # grid xy indices\n\n # Append\n #indices.append((b, a, gj, gi)) # image, anchor, grid indices\n indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices\n tbox.append(torch.cat((gxy - gij, gwh), 1)) # box\n anch.append(anchors[a]) # anchors\n tcls.append(c) # class\n\n return tcls, tbox, indices, anch\n\n" }, { "path": "asone/detectors/yolor/utils/metrics.py", "content": "# Model validation metrics\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n\ndef fitness(x):\n # Model fitness as a weighted combination of metrics\n w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]\n return (x[:, :4] * w).sum(1)\n\n\ndef fitness_p(x):\n # Model fitness as a weighted combination of metrics\n w = [1.0, 0.0, 0.0, 0.0] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]\n return (x[:, :4] * w).sum(1)\n\n\ndef fitness_r(x):\n # Model fitness as a weighted combination of metrics\n w = [0.0, 1.0, 0.0, 0.0] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]\n return (x[:, :4] * w).sum(1)\n\n\ndef fitness_ap50(x):\n # Model fitness as a weighted combination of metrics\n w = [0.0, 0.0, 1.0, 0.0] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]\n return (x[:, :4] * w).sum(1)\n\n\ndef fitness_ap(x):\n # Model fitness as a weighted combination of metrics\n w = [0.0, 0.0, 0.0, 1.0] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]\n return (x[:, :4] * w).sum(1)\n\n\ndef fitness_f(x):\n # Model fitness as a weighted combination of metrics\n #w = [0.0, 0.0, 0.0, 1.0] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]\n return ((x[:, 0]*x[:, 1])/(x[:, 0]+x[:, 1]))\n\n\ndef ap_per_class(tp, conf, pred_cls, target_cls, plot=False, fname='precision-recall_curve.png'):\n \"\"\" Compute the average precision, given the recall and precision curves.\n Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.\n # Arguments\n tp: True positives (nparray, nx1 or nx10).\n conf: Objectness value from 0-1 (nparray).\n pred_cls: Predicted object classes (nparray).\n target_cls: True object classes (nparray).\n plot: Plot precision-recall curve at mAP@0.5\n fname: Plot filename\n # Returns\n The average precision as computed in py-faster-rcnn.\n \"\"\"\n\n # Sort by objectness\n i = np.argsort(-conf)\n tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]\n\n # Find unique classes\n unique_classes = np.unique(target_cls)\n\n # Create Precision-Recall curve and compute AP for each class\n px, py = np.linspace(0, 1, 1000), [] # for plotting\n pr_score = 0.1 # score to evaluate P and R https://github.com/ultralytics/yolov3/issues/898\n s = [unique_classes.shape[0], tp.shape[1]] # number class, number iou thresholds (i.e. 10 for mAP0.5...0.95)\n ap, p, r = np.zeros(s), np.zeros(s), np.zeros(s)\n for ci, c in enumerate(unique_classes):\n i = pred_cls == c\n n_l = (target_cls == c).sum() # number of labels\n n_p = i.sum() # number of predictions\n\n if n_p == 0 or n_l == 0:\n continue\n else:\n # Accumulate FPs and TPs\n fpc = (1 - tp[i]).cumsum(0)\n tpc = tp[i].cumsum(0)\n\n # Recall\n recall = tpc / (n_l + 1e-16) # recall curve\n r[ci] = np.interp(-pr_score, -conf[i], recall[:, 0]) # r at pr_score, negative x, xp because xp decreases\n\n # Precision\n precision = tpc / (tpc + fpc) # precision curve\n p[ci] = np.interp(-pr_score, -conf[i], precision[:, 0]) # p at pr_score\n\n # AP from recall-precision curve\n for j in range(tp.shape[1]):\n ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j])\n if j == 0:\n py.append(np.interp(px, mrec, mpre)) # precision at mAP@0.5\n\n # Compute F1 score (harmonic mean of precision and recall)\n f1 = 2 * p * r / (p + r + 1e-16)\n\n if plot:\n py = np.stack(py, axis=1)\n fig, ax = plt.subplots(1, 1, figsize=(5, 5))\n ax.plot(px, py, linewidth=0.5, color='grey') # plot(recall, precision)\n ax.plot(px, py.mean(1), linewidth=2, color='blue', label='all classes %.3f mAP@0.5' % ap[:, 0].mean())\n ax.set_xlabel('Recall')\n ax.set_ylabel('Precision')\n ax.set_xlim(0, 1)\n ax.set_ylim(0, 1)\n plt.legend()\n fig.tight_layout()\n fig.savefig(fname, dpi=200)\n\n return p, r, ap, f1, unique_classes.astype('int32')\n\n\ndef compute_ap(recall, precision):\n \"\"\" Compute the average precision, given the recall and precision curves.\n Source: https://github.com/rbgirshick/py-faster-rcnn.\n # Arguments\n recall: The recall curve (list).\n precision: The precision curve (list).\n # Returns\n The average precision as computed in py-faster-rcnn.\n \"\"\"\n\n # Append sentinel values to beginning and end\n mrec = np.concatenate(([0.0], recall, [1.0]))\n mpre = np.concatenate(([1.0], precision, [0.0]))\n\n # Compute the precision envelope\n mpre = np.flip(np.maximum.accumulate(np.flip(mpre)))\n\n # Integrate area under curve\n method = 'interp' # methods: 'continuous', 'interp'\n if method == 'interp':\n x = np.linspace(0, 1, 101) # 101-point interp (COCO)\n ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate\n else: # 'continuous'\n i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes\n ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve\n\n return ap, mpre, mrec\n" }, { "path": "asone/detectors/yolor/utils/parse_config.py", "content": "import os\n\nimport numpy as np\n\n\ndef parse_model_cfg(path):\n # Parse the yolo *.cfg file and return module definitions path may be 'cfg/yolov3.cfg', 'yolov3.cfg', or 'yolov3'\n if not path.endswith('.cfg'): # add .cfg suffix if omitted\n path += '.cfg'\n if not os.path.exists(path) and os.path.exists('cfg' + os.sep + path): # add cfg/ prefix if omitted\n path = 'cfg' + os.sep + path\n\n with open(path, 'r') as f:\n lines = f.read().split('\\n')\n lines = [x for x in lines if x and not x.startswith('#')]\n lines = [x.rstrip().lstrip() for x in lines] # get rid of fringe whitespaces\n mdefs = [] # module definitions\n for line in lines:\n if line.startswith('['): # This marks the start of a new block\n mdefs.append({})\n mdefs[-1]['type'] = line[1:-1].rstrip()\n if mdefs[-1]['type'] == 'convolutional':\n mdefs[-1]['batch_normalize'] = 0 # pre-populate with zeros (may be overwritten later)\n \n else:\n key, val = line.split(\"=\")\n key = key.rstrip()\n\n if key == 'anchors': # return nparray\n mdefs[-1][key] = np.array([float(x) for x in val.split(',')]).reshape((-1, 2)) # np anchors\n elif (key in ['from', 'layers', 'mask']) or (key == 'size' and ',' in val): # return array\n mdefs[-1][key] = [int(x) for x in val.split(',')]\n else:\n val = val.strip()\n if val.isnumeric(): # return int or float\n mdefs[-1][key] = int(val) if (int(val) - float(val)) == 0 else float(val)\n else:\n mdefs[-1][key] = val # return string\n\n # Check all fields are supported\n supported = ['type', 'batch_normalize', 'filters', 'size', 'stride', 'pad', 'activation', 'layers', 'groups',\n 'from', 'mask', 'anchors', 'classes', 'num', 'jitter', 'ignore_thresh', 'truth_thresh', 'random',\n 'stride_x', 'stride_y', 'weights_type', 'weights_normalization', 'scale_x_y', 'beta_nms', 'nms_kind',\n 'iou_loss', 'iou_normalizer', 'cls_normalizer', 'iou_thresh', 'atoms', 'na', 'nc']\n\n f = [] # fields\n for x in mdefs[1:]:\n [f.append(k) for k in x if k not in f]\n u = [x for x in f if x not in supported] # unsupported fields\n assert not any(u), \"Unsupported fields %s in %s. See https://github.com/ultralytics/yolov3/issues/631\" % (u, path)\n\n return mdefs\n\n\ndef parse_data_cfg(path):\n # Parses the data configuration file\n if not os.path.exists(path) and os.path.exists('data' + os.sep + path): # add data/ prefix if omitted\n path = 'data' + os.sep + path\n\n with open(path, 'r') as f:\n lines = f.readlines()\n\n options = dict()\n for line in lines:\n line = line.strip()\n if line == '' or line.startswith('#'):\n continue\n key, val = line.split('=')\n options[key.strip()] = val.strip()\n\n return options\n" }, { "path": "asone/detectors/yolor/utils/plots.py", "content": "# Plotting utils\n\nimport glob\nimport math\nimport os\nimport random\nfrom copy import copy\nfrom pathlib import Path\n\nimport cv2\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\nimport yaml\nfrom PIL import Image\nfrom scipy.signal import butter, filtfilt\n\nfrom asone.detectors.yolor.utils.general import xywh2xyxy, xyxy2xywh\nfrom asone.detectors.yolor.utils.metrics import fitness\n\n# Settings\nmatplotlib.use('Agg') # for writing to files only\n\n\ndef color_list():\n # Return first 10 plt colors as (r,g,b) https://stackoverflow.com/questions/51350872/python-from-color-name-to-rgb\n def hex2rgb(h):\n return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4))\n\n return [hex2rgb(h) for h in plt.rcParams['axes.prop_cycle'].by_key()['color']]\n\n\ndef hist2d(x, y, n=100):\n # 2d histogram used in labels.png and evolve.png\n xedges, yedges = np.linspace(x.min(), x.max(), n), np.linspace(y.min(), y.max(), n)\n hist, xedges, yedges = np.histogram2d(x, y, (xedges, yedges))\n xidx = np.clip(np.digitize(x, xedges) - 1, 0, hist.shape[0] - 1)\n yidx = np.clip(np.digitize(y, yedges) - 1, 0, hist.shape[1] - 1)\n return np.log(hist[xidx, yidx])\n\n\ndef butter_lowpass_filtfilt(data, cutoff=1500, fs=50000, order=5):\n # https://stackoverflow.com/questions/28536191/how-to-filter-smooth-with-scipy-numpy\n def butter_lowpass(cutoff, fs, order):\n nyq = 0.5 * fs\n normal_cutoff = cutoff / nyq\n return butter(order, normal_cutoff, btype='low', analog=False)\n\n b, a = butter_lowpass(cutoff, fs, order=order)\n return filtfilt(b, a, data) # forward-backward filter\n\n\ndef plot_one_box(x, img, color=None, label=None, line_thickness=None):\n # Plots one bounding box on image img\n tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1 # line/font thickness\n color = color or [random.randint(0, 255) for _ in range(3)]\n c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n if label:\n tf = max(tl - 1, 1) # font thickness\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled\n cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\n\n\ndef plot_wh_methods(): # from utils.general import *; plot_wh_methods()\n # Compares the two methods for width-height anchor multiplication\n # https://github.com/ultralytics/yolov3/issues/168\n x = np.arange(-4.0, 4.0, .1)\n ya = np.exp(x)\n yb = torch.sigmoid(torch.from_numpy(x)).numpy() * 2\n\n fig = plt.figure(figsize=(6, 3), dpi=150)\n plt.plot(x, ya, '.-', label='YOLO')\n plt.plot(x, yb ** 2, '.-', label='YOLO ^2')\n plt.plot(x, yb ** 1.6, '.-', label='YOLO ^1.6')\n plt.xlim(left=-4, right=4)\n plt.ylim(bottom=0, top=6)\n plt.xlabel('input')\n plt.ylabel('output')\n plt.grid()\n plt.legend()\n fig.tight_layout()\n fig.savefig('comparison.png', dpi=200)\n\n\ndef output_to_target(output, width, height):\n # Convert model output to target format [batch_id, class_id, x, y, w, h, conf]\n if isinstance(output, torch.Tensor):\n output = output.cpu().numpy()\n\n targets = []\n for i, o in enumerate(output):\n if o is not None:\n for pred in o:\n box = pred[:4]\n w = (box[2] - box[0]) / width\n h = (box[3] - box[1]) / height\n x = box[0] / width + w / 2\n y = box[1] / height + h / 2\n conf = pred[4]\n cls = int(pred[5])\n\n targets.append([i, cls, x, y, w, h, conf])\n\n return np.array(targets)\n\n\ndef plot_images(images, targets, paths=None, fname='images.jpg', names=None, max_size=640, max_subplots=16):\n # Plot image grid with labels\n\n if isinstance(images, torch.Tensor):\n images = images.cpu().float().numpy()\n if isinstance(targets, torch.Tensor):\n targets = targets.cpu().numpy()\n\n # un-normalise\n if np.max(images[0]) <= 1:\n images *= 255\n\n tl = 3 # line thickness\n tf = max(tl - 1, 1) # font thickness\n bs, _, h, w = images.shape # batch size, _, height, width\n bs = min(bs, max_subplots) # limit plot images\n ns = np.ceil(bs ** 0.5) # number of subplots (square)\n\n # Check if we should resize\n scale_factor = max_size / max(h, w)\n if scale_factor < 1:\n h = math.ceil(scale_factor * h)\n w = math.ceil(scale_factor * w)\n\n colors = color_list() # list of colors\n mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init\n for i, img in enumerate(images):\n if i == max_subplots: # if last batch has fewer images than we expect\n break\n\n block_x = int(w * (i // ns))\n block_y = int(h * (i % ns))\n\n img = img.transpose(1, 2, 0)\n if scale_factor < 1:\n img = cv2.resize(img, (w, h))\n\n mosaic[block_y:block_y + h, block_x:block_x + w, :] = img\n if len(targets) > 0:\n image_targets = targets[targets[:, 0] == i]\n boxes = xywh2xyxy(image_targets[:, 2:6]).T\n classes = image_targets[:, 1].astype('int')\n labels = image_targets.shape[1] == 6 # labels if no conf column\n conf = None if labels else image_targets[:, 6] # check for confidence presence (label vs pred)\n\n boxes[[0, 2]] *= w\n boxes[[0, 2]] += block_x\n boxes[[1, 3]] *= h\n boxes[[1, 3]] += block_y\n for j, box in enumerate(boxes.T):\n cls = int(classes[j])\n color = colors[cls % len(colors)]\n cls = names[cls] if names else cls\n if labels or conf[j] > 0.25: # 0.25 conf thresh\n label = '%s' % cls if labels else '%s %.1f' % (cls, conf[j])\n plot_one_box(box, mosaic, label=label, color=color, line_thickness=tl)\n\n # Draw image filename labels\n if paths:\n label = Path(paths[i]).name[:40] # trim to 40 char\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n cv2.putText(mosaic, label, (block_x + 5, block_y + t_size[1] + 5), 0, tl / 3, [220, 220, 220], thickness=tf,\n lineType=cv2.LINE_AA)\n\n # Image border\n cv2.rectangle(mosaic, (block_x, block_y), (block_x + w, block_y + h), (255, 255, 255), thickness=3)\n\n if fname:\n r = min(1280. / max(h, w) / ns, 1.0) # ratio to limit image size\n mosaic = cv2.resize(mosaic, (int(ns * w * r), int(ns * h * r)), interpolation=cv2.INTER_AREA)\n # cv2.imwrite(fname, cv2.cvtColor(mosaic, cv2.COLOR_BGR2RGB)) # cv2 save\n Image.fromarray(mosaic).save(fname) # PIL save\n return mosaic\n\n\ndef plot_lr_scheduler(optimizer, scheduler, epochs=300, save_dir=''):\n # Plot LR simulating training for full epochs\n optimizer, scheduler = copy(optimizer), copy(scheduler) # do not modify originals\n y = []\n for _ in range(epochs):\n scheduler.step()\n y.append(optimizer.param_groups[0]['lr'])\n plt.plot(y, '.-', label='LR')\n plt.xlabel('epoch')\n plt.ylabel('LR')\n plt.grid()\n plt.xlim(0, epochs)\n plt.ylim(0)\n plt.tight_layout()\n plt.savefig(Path(save_dir) / 'LR.png', dpi=200)\n\n\ndef plot_test_txt(): # from utils.general import *; plot_test()\n # Plot test.txt histograms\n x = np.loadtxt('test.txt', dtype=np.float32)\n box = xyxy2xywh(x[:, :4])\n cx, cy = box[:, 0], box[:, 1]\n\n fig, ax = plt.subplots(1, 1, figsize=(6, 6), tight_layout=True)\n ax.hist2d(cx, cy, bins=600, cmax=10, cmin=0)\n ax.set_aspect('equal')\n plt.savefig('hist2d.png', dpi=300)\n\n fig, ax = plt.subplots(1, 2, figsize=(12, 6), tight_layout=True)\n ax[0].hist(cx, bins=600)\n ax[1].hist(cy, bins=600)\n plt.savefig('hist1d.png', dpi=200)\n\n\ndef plot_targets_txt(): # from utils.general import *; plot_targets_txt()\n # Plot targets.txt histograms\n x = np.loadtxt('targets.txt', dtype=np.float32).T\n s = ['x targets', 'y targets', 'width targets', 'height targets']\n fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)\n ax = ax.ravel()\n for i in range(4):\n ax[i].hist(x[i], bins=100, label='%.3g +/- %.3g' % (x[i].mean(), x[i].std()))\n ax[i].legend()\n ax[i].set_title(s[i])\n plt.savefig('targets.jpg', dpi=200)\n\n\ndef plot_study_txt(f='study.txt', x=None): # from utils.general import *; plot_study_txt()\n # Plot study.txt generated by test.py\n fig, ax = plt.subplots(2, 4, figsize=(10, 6), tight_layout=True)\n ax = ax.ravel()\n\n fig2, ax2 = plt.subplots(1, 1, figsize=(8, 4), tight_layout=True)\n for f in ['study/study_coco_yolo%s.txt' % x for x in ['s', 'm', 'l', 'x']]:\n y = np.loadtxt(f, dtype=np.float32, usecols=[0, 1, 2, 3, 7, 8, 9], ndmin=2).T\n x = np.arange(y.shape[1]) if x is None else np.array(x)\n s = ['P', 'R', 'mAP@.5', 'mAP@.5:.95', 't_inference (ms/img)', 't_NMS (ms/img)', 't_total (ms/img)']\n for i in range(7):\n ax[i].plot(x, y[i], '.-', linewidth=2, markersize=8)\n ax[i].set_title(s[i])\n\n j = y[3].argmax() + 1\n ax2.plot(y[6, :j], y[3, :j] * 1E2, '.-', linewidth=2, markersize=8,\n label=Path(f).stem.replace('study_coco_', '').replace('yolo', 'YOLO'))\n\n ax2.plot(1E3 / np.array([209, 140, 97, 58, 35, 18]), [34.6, 40.5, 43.0, 47.5, 49.7, 51.5],\n 'k.-', linewidth=2, markersize=8, alpha=.25, label='EfficientDet')\n\n ax2.grid()\n ax2.set_xlim(0, 30)\n ax2.set_ylim(28, 50)\n ax2.set_yticks(np.arange(30, 55, 5))\n ax2.set_xlabel('GPU Speed (ms/img)')\n ax2.set_ylabel('COCO AP val')\n ax2.legend(loc='lower right')\n plt.savefig('study_mAP_latency.png', dpi=300)\n plt.savefig(f.replace('.txt', '.png'), dpi=300)\n\n\ndef plot_labels(labels, save_dir=''):\n # plot dataset labels\n c, b = labels[:, 0], labels[:, 1:].transpose() # classes, boxes\n nc = int(c.max() + 1) # number of classes\n\n fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)\n ax = ax.ravel()\n ax[0].hist(c, bins=np.linspace(0, nc, nc + 1) - 0.5, rwidth=0.8)\n ax[0].set_xlabel('classes')\n ax[1].scatter(b[0], b[1], c=hist2d(b[0], b[1], 90), cmap='jet')\n ax[1].set_xlabel('x')\n ax[1].set_ylabel('y')\n ax[2].scatter(b[2], b[3], c=hist2d(b[2], b[3], 90), cmap='jet')\n ax[2].set_xlabel('width')\n ax[2].set_ylabel('height')\n plt.savefig(Path(save_dir) / 'labels.png', dpi=200)\n plt.close()\n\n # seaborn correlogram\n try:\n import seaborn as sns\n import pandas as pd\n x = pd.DataFrame(b.transpose(), columns=['x', 'y', 'width', 'height'])\n sns.pairplot(x, corner=True, diag_kind='hist', kind='scatter', markers='o',\n plot_kws=dict(s=3, edgecolor=None, linewidth=1, alpha=0.02),\n diag_kws=dict(bins=50))\n plt.savefig(Path(save_dir) / 'labels_correlogram.png', dpi=200)\n plt.close()\n except Exception as e:\n pass\n\n\ndef plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.general import *; plot_evolution()\n # Plot hyperparameter evolution results in evolve.txt\n with open(yaml_file) as f:\n hyp = yaml.load(f, Loader=yaml.FullLoader)\n x = np.loadtxt('evolve.txt', ndmin=2)\n f = fitness(x)\n # weights = (f - f.min()) ** 2 # for weighted results\n plt.figure(figsize=(10, 12), tight_layout=True)\n matplotlib.rc('font', **{'size': 8})\n for i, (k, v) in enumerate(hyp.items()):\n y = x[:, i + 7]\n # mu = (y * weights).sum() / weights.sum() # best weighted result\n mu = y[f.argmax()] # best single result\n plt.subplot(6, 5, i + 1)\n plt.scatter(y, f, c=hist2d(y, f, 20), cmap='viridis', alpha=.8, edgecolors='none')\n plt.plot(mu, f.max(), 'k+', markersize=15)\n plt.title('%s = %.3g' % (k, mu), fontdict={'size': 9}) # limit to 40 characters\n if i % 5 != 0:\n plt.yticks([])\n print('%15s: %.3g' % (k, mu))\n plt.savefig('evolve.png', dpi=200)\n print('\\nPlot saved as evolve.png')\n\n\ndef plot_results_overlay(start=0, stop=0): # from utils.general import *; plot_results_overlay()\n # Plot training 'results*.txt', overlaying train and val losses\n s = ['train', 'train', 'train', 'Precision', 'mAP@0.5', 'val', 'val', 'val', 'Recall', 'mAP@0.5:0.95'] # legends\n t = ['Box', 'Objectness', 'Classification', 'P-R', 'mAP-F1'] # titles\n for f in sorted(glob.glob('results*.txt') + glob.glob('../../Downloads/results*.txt')):\n results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T\n n = results.shape[1] # number of rows\n x = range(start, min(stop, n) if stop else n)\n fig, ax = plt.subplots(1, 5, figsize=(14, 3.5), tight_layout=True)\n ax = ax.ravel()\n for i in range(5):\n for j in [i, i + 5]:\n y = results[j, x]\n ax[i].plot(x, y, marker='.', label=s[j])\n # y_smooth = butter_lowpass_filtfilt(y)\n # ax[i].plot(x, np.gradient(y_smooth), marker='.', label=s[j])\n\n ax[i].set_title(t[i])\n ax[i].legend()\n ax[i].set_ylabel(f) if i == 0 else None # add filename\n fig.savefig(f.replace('.txt', '.png'), dpi=200)\n\n\ndef plot_results(start=0, stop=0, bucket='', id=(), labels=(), save_dir=''):\n # from utils.general import *; plot_results(save_dir='runs/train/exp0')\n # Plot training 'results*.txt'\n fig, ax = plt.subplots(2, 5, figsize=(12, 6))\n ax = ax.ravel()\n s = ['Box', 'Objectness', 'Classification', 'Precision', 'Recall',\n 'val Box', 'val Objectness', 'val Classification', 'mAP@0.5', 'mAP@0.5:0.95']\n if bucket:\n # os.system('rm -rf storage.googleapis.com')\n # files = ['https://storage.googleapis.com/%s/results%g.txt' % (bucket, x) for x in id]\n files = ['%g.txt' % x for x in id]\n c = ('gsutil cp ' + '%s ' * len(files) + '.') % tuple('gs://%s/%g.txt' % (bucket, x) for x in id)\n os.system(c)\n else:\n files = glob.glob(str(Path(save_dir) / '*.txt')) + glob.glob('../../Downloads/results*.txt')\n assert len(files), 'No results.txt files found in %s, nothing to plot.' % os.path.abspath(save_dir)\n for fi, f in enumerate(files):\n try:\n results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T\n n = results.shape[1] # number of rows\n x = range(start, min(stop, n) if stop else n)\n for i in range(10):\n y = results[i, x]\n if i in [0, 1, 2, 5, 6, 7]:\n y[y == 0] = np.nan # don't show zero loss values\n # y /= y[0] # normalize\n label = labels[fi] if len(labels) else Path(f).stem\n ax[i].plot(x, y, marker='.', label=label, linewidth=1, markersize=6)\n ax[i].set_title(s[i])\n # if i in [5, 6, 7]: # share train and val loss y axes\n # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5])\n except Exception as e:\n print('Warning: Plotting error for %s; %s' % (f, e))\n\n fig.tight_layout()\n ax[1].legend()\n fig.savefig(Path(save_dir) / 'results.png', dpi=200)\n" }, { "path": "asone/detectors/yolor/utils/torch_utils.py", "content": "# PyTorch utils\n\nimport logging\nimport math\nimport os\nimport time\nfrom contextlib import contextmanager\nfrom copy import deepcopy\n\nimport torch\nimport torch.backends.cudnn as cudnn\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\nlogger = logging.getLogger(__name__)\n\n\n@contextmanager\ndef torch_distributed_zero_first(local_rank: int):\n \"\"\"\n Decorator to make all processes in distributed training wait for each local_master to do something.\n \"\"\"\n if local_rank not in [-1, 0]:\n torch.distributed.barrier()\n yield\n if local_rank == 0:\n torch.distributed.barrier()\n\n\ndef init_torch_seeds(seed=0):\n # Speed-reproducibility tradeoff https://pytorch.org/docs/stable/notes/randomness.html\n torch.manual_seed(seed)\n if seed == 0: # slower, more reproducible\n cudnn.deterministic = True\n cudnn.benchmark = False\n else: # faster, less reproducible\n cudnn.deterministic = False\n cudnn.benchmark = True\n\n\ndef select_device(device='', batch_size=None):\n # device = 'cpu' or '0' or '0,1,2,3'\n cpu_request = device.lower() == 'cpu'\n if device and not cpu_request: # if device requested other than 'cpu'\n os.environ['CUDA_VISIBLE_DEVICES'] = device # set environment variable\n assert torch.cuda.is_available(), 'CUDA unavailable, invalid device %s requested' % device # check availablity\n\n cuda = False if cpu_request else torch.cuda.is_available()\n if cuda:\n c = 1024 ** 2 # bytes to MB\n ng = torch.cuda.device_count()\n if ng > 1 and batch_size: # check that batch_size is compatible with device_count\n assert batch_size % ng == 0, 'batch-size %g not multiple of GPU count %g' % (batch_size, ng)\n x = [torch.cuda.get_device_properties(i) for i in range(ng)]\n s = f'Using torch {torch.__version__} '\n for i in range(0, ng):\n if i == 1:\n s = ' ' * len(s)\n logger.info(\"%sCUDA:%g (%s, %dMB)\" % (s, i, x[i].name, x[i].total_memory / c))\n else:\n logger.info(f'Using torch {torch.__version__} CPU')\n\n logger.info('') # skip a line\n return torch.device('cuda:0' if cuda else 'cpu')\n\n\ndef time_synchronized():\n torch.cuda.synchronize() if torch.cuda.is_available() else None\n return time.time()\n\n\ndef is_parallel(model):\n return type(model) in (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)\n\n\ndef intersect_dicts(da, db, exclude=()):\n # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values\n return {k: v for k, v in da.items() if k in db and not any(x in k for x in exclude) and v.shape == db[k].shape}\n\n\ndef initialize_weights(model):\n for m in model.modules():\n t = type(m)\n if t is nn.Conv2d:\n pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n elif t is nn.BatchNorm2d:\n m.eps = 1e-3\n m.momentum = 0.03\n elif t in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6]:\n m.inplace = True\n\n\ndef find_modules(model, mclass=nn.Conv2d):\n # Finds layer indices matching module class 'mclass'\n return [i for i, m in enumerate(model.module_list) if isinstance(m, mclass)]\n\n\ndef sparsity(model):\n # Return global model sparsity\n a, b = 0., 0.\n for p in model.parameters():\n a += p.numel()\n b += (p == 0).sum()\n return b / a\n\n\ndef prune(model, amount=0.3):\n # Prune model to requested global sparsity\n import torch.nn.utils.prune as prune\n print('Pruning model... ', end='')\n for name, m in model.named_modules():\n if isinstance(m, nn.Conv2d):\n prune.l1_unstructured(m, name='weight', amount=amount) # prune\n prune.remove(m, 'weight') # make permanent\n print(' %.3g global sparsity' % sparsity(model))\n\n\ndef fuse_conv_and_bn(conv, bn):\n # Fuse convolution and batchnorm layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/\n fusedconv = nn.Conv2d(conv.in_channels,\n conv.out_channels,\n kernel_size=conv.kernel_size,\n stride=conv.stride,\n padding=conv.padding,\n groups=conv.groups,\n bias=True).requires_grad_(False).to(conv.weight.device)\n\n # prepare filters\n w_conv = conv.weight.clone().view(conv.out_channels, -1)\n w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))\n fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.size()))\n\n # prepare spatial bias\n b_conv = torch.zeros(conv.weight.size(0), device=conv.weight.device) if conv.bias is None else conv.bias\n b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps))\n fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)\n\n return fusedconv\n\n\ndef model_info(model, verbose=False, img_size=640):\n # Model information. img_size may be int or list, i.e. img_size=640 or img_size=[640, 320]\n n_p = sum(x.numel() for x in model.parameters()) # number parameters\n n_g = sum(x.numel() for x in model.parameters() if x.requires_grad) # number gradients\n if verbose:\n print('%5s %40s %9s %12s %20s %10s %10s' % ('layer', 'name', 'gradient', 'parameters', 'shape', 'mu', 'sigma'))\n for i, (name, p) in enumerate(model.named_parameters()):\n name = name.replace('module_list.', '')\n print('%5g %40s %9s %12g %20s %10.3g %10.3g' %\n (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std()))\n\n try: # FLOPS\n from thop import profile\n flops = profile(deepcopy(model), inputs=(torch.zeros(1, 3, img_size, img_size),), verbose=False)[0] / 1E9 * 2\n img_size = img_size if isinstance(img_size, list) else [img_size, img_size] # expand if int/float\n fs = ', %.9f GFLOPS' % (flops) # 640x640 FLOPS\n except (ImportError, Exception):\n fs = ''\n\n logger.info(f\"Model Summary: {len(list(model.modules()))} layers, {n_p} parameters, {n_g} gradients{fs}\")\n\n\ndef load_classifier(name='resnet101', n=2):\n # Loads a pretrained model reshaped to n-class output\n model = torchvision.models.__dict__[name](pretrained=True)\n\n # ResNet model properties\n # input_size = [3, 224, 224]\n # input_space = 'RGB'\n # input_range = [0, 1]\n # mean = [0.485, 0.456, 0.406]\n # std = [0.229, 0.224, 0.225]\n\n # Reshape output to n classes\n filters = model.fc.weight.shape[1]\n model.fc.bias = nn.Parameter(torch.zeros(n), requires_grad=True)\n model.fc.weight = nn.Parameter(torch.zeros(n, filters), requires_grad=True)\n model.fc.out_features = n\n return model\n\n\ndef scale_img(img, ratio=1.0, same_shape=False): # img(16,3,256,416), r=ratio\n # scales img(bs,3,y,x) by ratio\n if ratio == 1.0:\n return img\n else:\n h, w = img.shape[2:]\n s = (int(h * ratio), int(w * ratio)) # new size\n img = F.interpolate(img, size=s, mode='bilinear', align_corners=False) # resize\n if not same_shape: # pad/crop img\n gs = 32 # (pixels) grid size\n h, w = [math.ceil(x * ratio / gs) * gs for x in (h, w)]\n return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean\n\n\ndef copy_attr(a, b, include=(), exclude=()):\n # Copy attributes from b to a, options to only include [...] and to exclude [...]\n for k, v in b.__dict__.items():\n if (len(include) and k not in include) or k.startswith('_') or k in exclude:\n continue\n else:\n setattr(a, k, v)\n\n\nclass ModelEMA:\n \"\"\" Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models\n Keep a moving average of everything in the model state_dict (parameters and buffers).\n This is intended to allow functionality like\n https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage\n A smoothed version of the weights is necessary for some training schemes to perform well.\n This class is sensitive where it is initialized in the sequence of model init,\n GPU assignment and distributed training wrappers.\n \"\"\"\n\n def __init__(self, model, decay=0.9999, updates=0):\n # Create EMA\n self.ema = deepcopy(model.module if is_parallel(model) else model).eval() # FP32 EMA\n # if next(model.parameters()).device.type != 'cpu':\n # self.ema.half() # FP16 EMA\n self.updates = updates # number of EMA updates\n self.decay = lambda x: decay * (1 - math.exp(-x / 2000)) # decay exponential ramp (to help early epochs)\n for p in self.ema.parameters():\n p.requires_grad_(False)\n\n def update(self, model):\n # Update EMA parameters\n with torch.no_grad():\n self.updates += 1\n d = self.decay(self.updates)\n\n msd = model.module.state_dict() if is_parallel(model) else model.state_dict() # model state_dict\n for k, v in self.ema.state_dict().items():\n if v.dtype.is_floating_point:\n v *= d\n v += (1. - d) * msd[k].detach()\n\n def update_attr(self, model, include=(), exclude=('process_group', 'reducer')):\n # Update EMA attributes\n copy_attr(self.ema, model, include, exclude)\n" }, { "path": "asone/detectors/yolor/utils/yolor_utils.py", "content": "import torch\nimport torchvision\nimport time\nimport numpy as np\nimport cv2\n\n\nclass_names = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']\n\n# Create a list of colors for each class where each color is a tuple of 3 integer values\nrng = np.random.default_rng(3)\ncolors = rng.uniform(0, 255, size=(len(class_names), 3))\n\ndef box_area(box):\n # box = xyxy(4,n)\n return (box[2] - box[0]) * (box[3] - box[1])\n\n\ndef box_iou(box1, box2, eps=1e-7):\n # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py\n \"\"\"\n Return intersection-over-union (Jaccard index) of boxes.\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\n Arguments:\n box1 (Tensor[N, 4])\n box2 (Tensor[M, 4])\n Returns:\n iou (Tensor[N, M]): the NxM matrix containing the pairwise\n IoU values for every element in boxes1 and boxes2\n \"\"\"\n\n # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)\n (a1, a2), (b1, b2) = box1[:, None].chunk(2, 2), box2.chunk(2, 1)\n inter = (torch.min(a2, b2) - torch.max(a1, b1)).clamp(0).prod(2)\n\n # IoU = inter / (area1 + area2 - inter)\n return inter / (box_area(box1.T)[:, None] + box_area(box2.T) - inter + eps)\n\ndef xywh2xyxy(x):\n # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x\n y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y\n y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x\n y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y\n return y\n\ndef non_max_suppression(prediction,\n conf_thres=0.25,\n iou_thres=0.45,\n classes=None,\n agnostic=False,\n multi_label=False,\n labels=(),\n max_det=300):\n \"\"\"Non-Maximum Suppression (NMS) on inference results to reject overlapping bounding boxes\n Returns:\n list of detections, on (n,6) tensor per image [xyxy, conf, cls]\n \"\"\"\n # prediction = torch.Tensor(prediction)\n bs = prediction.shape[0] # batch size\n nc = prediction.shape[2] - 5 # number of classes\n xc = prediction[..., 4] > conf_thres # candidates\n # Checks\n assert 0 <= conf_thres <= 1, f'Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0'\n assert 0 <= iou_thres <= 1, f'Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0'\n\n # Settings\n # min_wh = 2 # (pixels) minimum box width and height\n max_wh = 7680 # (pixels) maximum box width and height\n max_nms = 30000 # maximum number of boxes into torchvision.ops.nms()\n time_limit = 0.3 + 0.03 * bs # seconds to quit after\n redundant = True # require redundant detections\n multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img)\n merge = False # use merge-NMS\n\n t = time.time()\n output = [torch.zeros((0, 6), device=prediction.device)] * bs\n for xi, x in enumerate(prediction): # image index, image inference\n # Apply constraints\n # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height\n x = x[xc[xi]] # confidence\n\n # Cat apriori labels if autolabelling\n if labels and len(labels[xi]):\n lb = labels[xi]\n v = torch.zeros((len(lb), nc + 5), device=x.device)\n v[:, :4] = lb[:, 1:5] # box\n v[:, 4] = 1.0 # conf\n v[range(len(lb)), lb[:, 0].long() + 5] = 1.0 # cls\n x = torch.cat((x, v), 0)\n\n # If none remain process next image\n if not x.shape[0]:\n continue\n\n # Compute conf\n x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf\n\n # Box (center x, center y, width, height) to (x1, y1, x2, y2)\n # print(type(x))\n box = xywh2xyxy(x[:, :4])\n\n # Detections matrix nx6 (xyxy, conf, cls)\n if multi_label:\n i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T\n x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1)\n else: # best class only\n conf, j = x[:, 5:].max(1, keepdim=True)\n x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres]\n\n # Filter by class\n if classes is not None:\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\n\n # Apply finite constraint\n # if not torch.isfinite(x).all():\n # x = x[torch.isfinite(x).all(1)]\n\n # Check shape\n n = x.shape[0] # number of boxes\n if not n: # no boxes\n continue\n elif n > max_nms: # excess boxes\n x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence\n\n # Batched NMS\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\n i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS\n if i.shape[0] > max_det: # limit detections\n i = i[:max_det]\n if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)\n # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\n weights = iou * scores[None] # box weights\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\n if redundant:\n i = i[iou.sum(1) > 1] # require redundancy\n\n output[xi] = x[i]\n if (time.time() - t) > time_limit:\n # LOGGER.warning(f'WARNING: NMS time limit {time_limit:.3f}s exceeded')\n break # time limit exceeded\n\n return output\n\ndef letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32):\n # Resize and pad image while meeting stride-multiple constraints\n shape = im.shape[:2] # current shape [height, width]\n if isinstance(new_shape, int):\n new_shape = (new_shape, new_shape)\n\n # Scale ratio (new / old)\n r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])\n if not scaleup: # only scale down, do not scale up (for better val mAP)\n r = min(r, 1.0)\n\n # Compute padding\n ratio = r, r # width, height ratios\n new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))\n dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding\n if auto: # minimum rectangle\n dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding\n elif scaleFill: # stretch\n dw, dh = 0.0, 0.0\n new_unpad = (new_shape[1], new_shape[0])\n ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios\n\n dw /= 2 # divide padding into 2 sides\n dh /= 2\n\n if shape[::-1] != new_unpad: # resize\n im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)\n top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))\n left, right = int(round(dw - 0.1)), int(round(dw + 0.1))\n im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border\n return im, ratio, (dw, dh)\n\ndef scale_coords(img1_shape, coords, img0_shape, ratio_pad=None):\n # Rescale coords (xyxy) from img1_shape to img0_shape\n if ratio_pad is None: # calculate from img0_shape\n gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new\n pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding\n else:\n gain = ratio_pad[0][0]\n pad = ratio_pad[1]\n\n coords[:, [0, 2]] -= pad[0] # x padding\n coords[:, [1, 3]] -= pad[1] # y padding\n coords[:, :4] /= gain\n clip_coords(coords, img0_shape)\n return coords\n\n\ndef clip_coords(boxes, shape):\n # Clip bounding xyxy bounding boxes to image shape (height, width)\n if isinstance(boxes, torch.Tensor): # faster individually\n boxes[:, 0].clamp_(0, shape[1]) # x1\n boxes[:, 1].clamp_(0, shape[0]) # y1\n boxes[:, 2].clamp_(0, shape[1]) # x2\n boxes[:, 3].clamp_(0, shape[0]) # y2\n else: # np.array (faster grouped)\n boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, shape[1]) # x1, x2\n boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, shape[0]) # y1, y2\n\n" }, { "path": "asone/detectors/yolor/yolor_detector.py", "content": "\nimport os\nfrom asone.utils import get_names\nimport numpy as np\nimport warnings\nimport torch\nimport onnxruntime\n\nfrom asone.detectors.yolor.models.models import *\nfrom asone import utils\nfrom asone.detectors.yolor.utils.yolor_utils import (non_max_suppression,\n scale_coords,\n letterbox)\n\nfrom asone.utils.utils import PathResolver\n\n\nclass YOLOrDetector:\n def __init__(self,\n weights=None,\n cfg=None,\n use_onnx=True,\n use_cuda=True,\n ):\n\n self.use_onnx = use_onnx\n self.device = 'cuda' if use_cuda else 'cpu'\n\n if not os.path.exists(weights):\n utils.download_weights(weights)\n\n if cfg == None:\n cfg = os.path.join(\"cfg\", \"yolor_p6.cfg\")\n # If incase weighst is a list of paths then select path at first index\n weights = str(weights[0] if isinstance(weights, list) else weights)\n with PathResolver():\n # Load Model\n self.model = self.load_model(use_cuda, weights, cfg=cfg, img_size=640)\n\n def load_model(self, use_cuda, weights, cfg, img_size, fp16=False):\n # Device: CUDA and if fp16=True only then half precision floating point works\n self.fp16 = fp16 & (\n (not self.use_onnx or self.use_onnx) and self.device != 'cpu')\n # Load onnx\n if self.use_onnx:\n if use_cuda:\n providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']\n else:\n providers = ['CPUExecutionProvider']\n model = onnxruntime.InferenceSession(weights, providers=providers)\n # Load Pytorch\n else:\n model = Darknet(cfg, img_size).to(self.device)\n model.load_state_dict(torch.load(\n weights, map_location=self.device)['model'])\n model.to(self.device).eval()\n model.half() if self.fp16 else model.float()\n return model\n\n def image_preprocessing(self,\n image: list,\n input_shape=(640, 640)) -> list:\n\n original_image = image.copy()\n image = letterbox(image, input_shape, stride=32, auto=False)[0]\n image = image.transpose((2, 0, 1))[::-1]\n image = np.ascontiguousarray(image, dtype=np.float32)\n image /= 255 # 0 - 255 to 0.0 - 1.0\n if len(image.shape) == 3:\n image = image[None] # expand for batch dim\n return original_image, image\n\n def detect(self, image: list,\n input_shape: tuple = (640, 640),\n conf_thres: float = 0.25,\n iou_thres: float = 0.45,\n max_det: int = 1000,\n filter_classes: bool = None,\n agnostic_nms: bool = True,\n with_p6: bool = False,\n return_image=False) -> list:\n\n # Image Preprocessing\n original_image, processed_image = self.image_preprocessing(\n image, input_shape)\n\n # Inference\n if self.use_onnx:\n # Input names of ONNX model on which it is exported\n input_name = self.model.get_inputs()[0].name\n # Run onnx model\n pred = self.model.run([self.model.get_outputs()[0].name], {\n input_name: processed_image})[0]\n # Run Pytorch model\n else:\n processed_image = torch.from_numpy(processed_image).to(self.device)\n # Change image floating point precision if fp16 set to true\n processed_image = processed_image.half() if self.fp16 else processed_image.float()\n pred = self.model(processed_image, augment=False)[0]\n pred = pred.detach().cpu().numpy()\n\n if isinstance(pred, np.ndarray):\n pred = torch.tensor(pred, device=self.device)\n predictions = non_max_suppression(\n pred, conf_thres,\n iou_thres,\n agnostic=agnostic_nms,\n max_det=max_det)\n\n for i, prediction in enumerate(predictions): # per image\n if len(prediction):\n prediction[:, :4] = scale_coords(\n processed_image.shape[2:], prediction[:, :4], original_image.shape).round()\n predictions[i] = prediction\n\n predictions = predictions[0].cpu().numpy()\n image_info = {\n 'width': original_image.shape[1],\n 'height': original_image.shape[0],\n }\n\n self.boxes = predictions[:, :4]\n self.scores = predictions[:, 4:5]\n self.class_ids = predictions[:, 5:6]\n\n if filter_classes:\n class_names = get_names()\n\n filter_class_idx = []\n if filter_classes:\n for _class in filter_classes:\n if _class.lower() in class_names:\n filter_class_idx.append(\n class_names.index(_class.lower()))\n else:\n warnings.warn(\n f\"class {_class} not found in model classes list.\")\n\n # detection = detection[np.in1d(\n # detection[:, 5].astype(int), filter_class_idx)]\n\n if return_image:\n return predictions, original_image\n else: \n return predictions, image_info\n \n" }, { "path": "asone/detectors/yolov5/__init__.py", "content": "from .yolov5_detector import YOLOv5Detector\n__all__ = ['YOLOv5Detector']" }, { "path": "asone/detectors/yolov5/yolov5/__init__.py", "content": "" }, { "path": "asone/detectors/yolov5/yolov5/export.py", "content": "# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license\n\"\"\"\nExport a YOLOv5 PyTorch model to other formats. TensorFlow exports authored by https://github.com/zldrobit\n\nFormat | `export.py --include` | Model\n--- | --- | ---\nPyTorch | - | yolov5s.pt\nTorchScript | `torchscript` | yolov5s.torchscript\nONNX | `onnx` | yolov5s.onnx\nOpenVINO | `openvino` | yolov5s_openvino_model/\nTensorRT | `engine` | yolov5s.engine\nCoreML | `coreml` | yolov5s.mlmodel\nTensorFlow SavedModel | `saved_model` | yolov5s_saved_model/\nTensorFlow GraphDef | `pb` | yolov5s.pb\nTensorFlow Lite | `tflite` | yolov5s.tflite\nTensorFlow Edge TPU | `edgetpu` | yolov5s_edgetpu.tflite\nTensorFlow.js | `tfjs` | yolov5s_web_model/\nPaddlePaddle | `paddle` | yolov5s_paddle_model/\n\nRequirements:\n $ pip install -r requirements.txt coremltools onnx onnx-simplifier onnxruntime openvino-dev tensorflow-cpu # CPU\n $ pip install -r requirements.txt coremltools onnx onnx-simplifier onnxruntime-gpu openvino-dev tensorflow # GPU\n\nUsage:\n $ python export.py --weights yolov5s.pt --include torchscript onnx openvino engine coreml tflite ...\n\nInference:\n $ python detect.py --weights yolov5s.pt # PyTorch\n yolov5s.torchscript # TorchScript\n yolov5s.onnx # ONNX Runtime or OpenCV DNN with --dnn\n yolov5s_openvino_model # OpenVINO\n yolov5s.engine # TensorRT\n yolov5s.mlmodel # CoreML (macOS-only)\n yolov5s_saved_model # TensorFlow SavedModel\n yolov5s.pb # TensorFlow GraphDef\n yolov5s.tflite # TensorFlow Lite\n yolov5s_edgetpu.tflite # TensorFlow Edge TPU\n yolov5s_paddle_model # PaddlePaddle\n\nTensorFlow.js:\n $ cd .. && git clone https://github.com/zldrobit/tfjs-yolov5-example.git && cd tfjs-yolov5-example\n $ npm install\n $ ln -s ../../yolov5/yolov5s_web_model public/yolov5s_web_model\n $ npm start\n\"\"\"\n\nimport argparse\nimport contextlib\nimport json\nimport os\nimport platform\nimport re\nimport subprocess\nimport sys\nimport time\nimport warnings\nfrom pathlib import Path\n\nimport pandas as pd\nimport torch\nfrom torch.utils.mobile_optimizer import optimize_for_mobile\n\nFILE = Path(__file__).resolve()\nROOT = FILE.parents[0] # YOLOv5 root directory\nif str(ROOT) not in sys.path:\n sys.path.append(str(ROOT)) # add ROOT to PATH\nif platform.system() != \"Windows\":\n ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative\n\nfrom asone.detectors.yolov5.yolov5.models.experimental import attempt_load\nfrom asone.detectors.yolov5.yolov5.models.yolo import ClassificationModel, Detect, DetectionModel, SegmentationModel\nfrom asone.detectors.yolov5.yolov5.utils.dataloaders import LoadImages\nfrom asone.detectors.yolov5.yolov5.utils.general import (\n LOGGER,\n Profile,\n check_dataset,\n check_img_size,\n check_requirements,\n check_version,\n check_yaml,\n colorstr,\n file_size,\n get_default_args,\n print_args,\n url2file,\n yaml_save,\n)\nfrom asone.detectors.yolov5.yolov5.utils.torch_utils import select_device, smart_inference_mode\n\nMACOS = platform.system() == \"Darwin\" # macOS environment\n\n\nclass iOSModel(torch.nn.Module):\n def __init__(self, model, im):\n \"\"\"Initializes an iOS compatible model with normalization based on image dimensions.\"\"\"\n super().__init__()\n b, c, h, w = im.shape # batch, channel, height, width\n self.model = model\n self.nc = model.nc # number of classes\n if w == h:\n self.normalize = 1.0 / w\n else:\n self.normalize = torch.tensor([1.0 / w, 1.0 / h, 1.0 / w, 1.0 / h]) # broadcast (slower, smaller)\n # np = model(im)[0].shape[1] # number of points\n # self.normalize = torch.tensor([1. / w, 1. / h, 1. / w, 1. / h]).expand(np, 4) # explicit (faster, larger)\n\n def forward(self, x):\n \"\"\"Runs forward pass on the input tensor, returning class confidences and normalized coordinates.\"\"\"\n xywh, conf, cls = self.model(x)[0].squeeze().split((4, 1, self.nc), 1)\n return cls * conf, xywh * self.normalize # confidence (3780, 80), coordinates (3780, 4)\n\n\ndef export_formats():\n \"\"\"Returns a DataFrame of supported YOLOv5 model export formats and their properties.\"\"\"\n x = [\n [\"PyTorch\", \"-\", \".pt\", True, True],\n [\"TorchScript\", \"torchscript\", \".torchscript\", True, True],\n [\"ONNX\", \"onnx\", \".onnx\", True, True],\n [\"OpenVINO\", \"openvino\", \"_openvino_model\", True, False],\n [\"TensorRT\", \"engine\", \".engine\", False, True],\n [\"CoreML\", \"coreml\", \".mlmodel\", True, False],\n [\"TensorFlow SavedModel\", \"saved_model\", \"_saved_model\", True, True],\n [\"TensorFlow GraphDef\", \"pb\", \".pb\", True, True],\n [\"TensorFlow Lite\", \"tflite\", \".tflite\", True, False],\n [\"TensorFlow Edge TPU\", \"edgetpu\", \"_edgetpu.tflite\", False, False],\n [\"TensorFlow.js\", \"tfjs\", \"_web_model\", False, False],\n [\"PaddlePaddle\", \"paddle\", \"_paddle_model\", True, True],\n ]\n return pd.DataFrame(x, columns=[\"Format\", \"Argument\", \"Suffix\", \"CPU\", \"GPU\"])\n\n\ndef try_export(inner_func):\n \"\"\"Decorator @try_export for YOLOv5 model export functions that logs success/failure, time taken, and file size.\"\"\"\n inner_args = get_default_args(inner_func)\n\n def outer_func(*args, **kwargs):\n prefix = inner_args[\"prefix\"]\n try:\n with Profile() as dt:\n f, model = inner_func(*args, **kwargs)\n LOGGER.info(f\"{prefix} export success ✅ {dt.t:.1f}s, saved as {f} ({file_size(f):.1f} MB)\")\n return f, model\n except Exception as e:\n LOGGER.info(f\"{prefix} export failure ❌ {dt.t:.1f}s: {e}\")\n return None, None\n\n return outer_func\n\n\n@try_export\ndef export_torchscript(model, im, file, optimize, prefix=colorstr(\"TorchScript:\")):\n \"\"\"Exports YOLOv5 model to TorchScript format, optionally optimized for mobile, with image shape and stride\n metadata.\n \"\"\"\n LOGGER.info(f\"\\n{prefix} starting export with torch {torch.__version__}...\")\n f = file.with_suffix(\".torchscript\")\n\n ts = torch.jit.trace(model, im, strict=False)\n d = {\"shape\": im.shape, \"stride\": int(max(model.stride)), \"names\": model.names}\n extra_files = {\"config.txt\": json.dumps(d)} # torch._C.ExtraFilesMap()\n if optimize: # https://pytorch.org/tutorials/recipes/mobile_interpreter.html\n optimize_for_mobile(ts)._save_for_lite_interpreter(str(f), _extra_files=extra_files)\n else:\n ts.save(str(f), _extra_files=extra_files)\n return f, None\n\n\n@try_export\ndef export_onnx(model, im, file, opset, dynamic, simplify, prefix=colorstr(\"ONNX:\")):\n \"\"\"Exports a YOLOv5 model to ONNX format with dynamic axes and optional simplification.\"\"\"\n check_requirements(\"onnx>=1.12.0\")\n import onnx\n\n LOGGER.info(f\"\\n{prefix} starting export with onnx {onnx.__version__}...\")\n f = str(file.with_suffix(\".onnx\"))\n\n output_names = [\"output0\", \"output1\"] if isinstance(model, SegmentationModel) else [\"output0\"]\n if dynamic:\n dynamic = {\"images\": {0: \"batch\", 2: \"height\", 3: \"width\"}} # shape(1,3,640,640)\n if isinstance(model, SegmentationModel):\n dynamic[\"output0\"] = {0: \"batch\", 1: \"anchors\"} # shape(1,25200,85)\n dynamic[\"output1\"] = {0: \"batch\", 2: \"mask_height\", 3: \"mask_width\"} # shape(1,32,160,160)\n elif isinstance(model, DetectionModel):\n dynamic[\"output0\"] = {0: \"batch\", 1: \"anchors\"} # shape(1,25200,85)\n\n torch.onnx.export(\n model.cpu() if dynamic else model, # --dynamic only compatible with cpu\n im.cpu() if dynamic else im,\n f,\n verbose=False,\n opset_version=opset,\n do_constant_folding=True, # WARNING: DNN inference with torch>=1.12 may require do_constant_folding=False\n input_names=[\"images\"],\n output_names=output_names,\n dynamic_axes=dynamic or None,\n )\n\n # Checks\n model_onnx = onnx.load(f) # load onnx model\n onnx.checker.check_model(model_onnx) # check onnx model\n\n # Metadata\n d = {\"stride\": int(max(model.stride)), \"names\": model.names}\n for k, v in d.items():\n meta = model_onnx.metadata_props.add()\n meta.key, meta.value = k, str(v)\n onnx.save(model_onnx, f)\n\n # Simplify\n if simplify:\n try:\n cuda = torch.cuda.is_available()\n check_requirements((\"onnxruntime-gpu\" if cuda else \"onnxruntime\", \"onnx-simplifier>=0.4.1\"))\n import onnxsim\n\n LOGGER.info(f\"{prefix} simplifying with onnx-simplifier {onnxsim.__version__}...\")\n model_onnx, check = onnxsim.simplify(model_onnx)\n assert check, \"assert check failed\"\n onnx.save(model_onnx, f)\n except Exception as e:\n LOGGER.info(f\"{prefix} simplifier failure: {e}\")\n return f, model_onnx\n\n\n@try_export\ndef export_openvino(file, metadata, half, int8, data, prefix=colorstr(\"OpenVINO:\")):\n # YOLOv5 OpenVINO export\n check_requirements(\"openvino-dev>=2023.0\") # requires openvino-dev: https://pypi.org/project/openvino-dev/\n import openvino.runtime as ov # noqa\n from openvino.tools import mo # noqa\n\n LOGGER.info(f\"\\n{prefix} starting export with openvino {ov.__version__}...\")\n f = str(file).replace(file.suffix, f\"_{'int8_' if int8 else ''}openvino_model{os.sep}\")\n f_onnx = file.with_suffix(\".onnx\")\n f_ov = str(Path(f) / file.with_suffix(\".xml\").name)\n\n ov_model = mo.convert_model(f_onnx, model_name=file.stem, framework=\"onnx\", compress_to_fp16=half) # export\n\n if int8:\n check_requirements(\"nncf>=2.5.0\") # requires at least version 2.5.0 to use the post-training quantization\n import nncf\n import numpy as np\n\n from asone.detectors.yolov5.yolov5.utils.dataloaders import create_dataloader\n\n def gen_dataloader(yaml_path, task=\"train\", imgsz=640, workers=4):\n data_yaml = check_yaml(yaml_path)\n data = check_dataset(data_yaml)\n dataloader = create_dataloader(\n data[task], imgsz=imgsz, batch_size=1, stride=32, pad=0.5, single_cls=False, rect=False, workers=workers\n )[0]\n return dataloader\n\n # noqa: F811\n\n def transform_fn(data_item):\n \"\"\"\n Quantization transform function.\n\n Extracts and preprocess input data from dataloader item for quantization.\n Parameters:\n data_item: Tuple with data item produced by DataLoader during iteration\n Returns:\n input_tensor: Input data for quantization\n \"\"\"\n assert data_item[0].dtype == torch.uint8, \"input image must be uint8 for the quantization preprocessing\"\n\n img = data_item[0].numpy().astype(np.float32) # uint8 to fp16/32\n img /= 255.0 # 0 - 255 to 0.0 - 1.0\n return np.expand_dims(img, 0) if img.ndim == 3 else img\n\n ds = gen_dataloader(data)\n quantization_dataset = nncf.Dataset(ds, transform_fn)\n ov_model = nncf.quantize(ov_model, quantization_dataset, preset=nncf.QuantizationPreset.MIXED)\n\n ov.serialize(ov_model, f_ov) # save\n yaml_save(Path(f) / file.with_suffix(\".yaml\").name, metadata) # add metadata.yaml\n return f, None\n\n\n@try_export\ndef export_paddle(model, im, file, metadata, prefix=colorstr(\"PaddlePaddle:\")):\n \"\"\"Exports a YOLOv5 model to PaddlePaddle format using X2Paddle, saving to `save_dir` and adding a metadata.yaml\n file.\n \"\"\"\n check_requirements((\"paddlepaddle\", \"x2paddle\"))\n import x2paddle\n from x2paddle.convert import pytorch2paddle\n\n LOGGER.info(f\"\\n{prefix} starting export with X2Paddle {x2paddle.__version__}...\")\n f = str(file).replace(\".pt\", f\"_paddle_model{os.sep}\")\n\n pytorch2paddle(module=model, save_dir=f, jit_type=\"trace\", input_examples=[im]) # export\n yaml_save(Path(f) / file.with_suffix(\".yaml\").name, metadata) # add metadata.yaml\n return f, None\n\n\n@try_export\ndef export_coreml(model, im, file, int8, half, nms, prefix=colorstr(\"CoreML:\")):\n \"\"\"Exports YOLOv5 model to CoreML format with optional NMS, INT8, and FP16 support; requires coremltools.\"\"\"\n check_requirements(\"coremltools\")\n import coremltools as ct\n\n LOGGER.info(f\"\\n{prefix} starting export with coremltools {ct.__version__}...\")\n f = file.with_suffix(\".mlmodel\")\n\n if nms:\n model = iOSModel(model, im)\n ts = torch.jit.trace(model, im, strict=False) # TorchScript model\n ct_model = ct.convert(ts, inputs=[ct.ImageType(\"image\", shape=im.shape, scale=1 / 255, bias=[0, 0, 0])])\n bits, mode = (8, \"kmeans_lut\") if int8 else (16, \"linear\") if half else (32, None)\n if bits < 32:\n if MACOS: # quantization only supported on macOS\n with warnings.catch_warnings():\n warnings.filterwarnings(\"ignore\", category=DeprecationWarning) # suppress numpy==1.20 float warning\n ct_model = ct.models.neural_network.quantization_utils.quantize_weights(ct_model, bits, mode)\n else:\n print(f\"{prefix} quantization only supported on macOS, skipping...\")\n ct_model.save(f)\n return f, ct_model\n\n\n@try_export\ndef export_engine(model, im, file, half, dynamic, simplify, workspace=4, verbose=False, prefix=colorstr(\"TensorRT:\")):\n \"\"\"\n Exports a YOLOv5 model to TensorRT engine format, requiring GPU and TensorRT>=7.0.0.\n\n https://developer.nvidia.com/tensorrt\n \"\"\"\n assert im.device.type != \"cpu\", \"export running on CPU but must be on GPU, i.e. `python export.py --device 0`\"\n try:\n import tensorrt as trt\n except Exception:\n if platform.system() == \"Linux\":\n check_requirements(\"nvidia-tensorrt\", cmds=\"-U --index-url https://pypi.ngc.nvidia.com\")\n import tensorrt as trt\n\n if trt.__version__[0] == \"7\": # TensorRT 7 handling https://github.com/ultralytics/yolov5/issues/6012\n grid = model.model[-1].anchor_grid\n model.model[-1].anchor_grid = [a[..., :1, :1, :] for a in grid]\n export_onnx(model, im, file, 12, dynamic, simplify) # opset 12\n model.model[-1].anchor_grid = grid\n else: # TensorRT >= 8\n check_version(trt.__version__, \"8.0.0\", hard=True) # require tensorrt>=8.0.0\n export_onnx(model, im, file, 12, dynamic, simplify) # opset 12\n onnx = file.with_suffix(\".onnx\")\n\n LOGGER.info(f\"\\n{prefix} starting export with TensorRT {trt.__version__}...\")\n assert onnx.exists(), f\"failed to export ONNX file: {onnx}\"\n f = file.with_suffix(\".engine\") # TensorRT engine file\n logger = trt.Logger(trt.Logger.INFO)\n if verbose:\n logger.min_severity = trt.Logger.Severity.VERBOSE\n\n builder = trt.Builder(logger)\n config = builder.create_builder_config()\n config.max_workspace_size = workspace * 1 << 30\n # config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, workspace << 30) # fix TRT 8.4 deprecation notice\n\n flag = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)\n network = builder.create_network(flag)\n parser = trt.OnnxParser(network, logger)\n if not parser.parse_from_file(str(onnx)):\n raise RuntimeError(f\"failed to load ONNX file: {onnx}\")\n\n inputs = [network.get_input(i) for i in range(network.num_inputs)]\n outputs = [network.get_output(i) for i in range(network.num_outputs)]\n for inp in inputs:\n LOGGER.info(f'{prefix} input \"{inp.name}\" with shape{inp.shape} {inp.dtype}')\n for out in outputs:\n LOGGER.info(f'{prefix} output \"{out.name}\" with shape{out.shape} {out.dtype}')\n\n if dynamic:\n if im.shape[0] <= 1:\n LOGGER.warning(f\"{prefix} WARNING ⚠️ --dynamic model requires maximum --batch-size argument\")\n profile = builder.create_optimization_profile()\n for inp in inputs:\n profile.set_shape(inp.name, (1, *im.shape[1:]), (max(1, im.shape[0] // 2), *im.shape[1:]), im.shape)\n config.add_optimization_profile(profile)\n\n LOGGER.info(f\"{prefix} building FP{16 if builder.platform_has_fast_fp16 and half else 32} engine as {f}\")\n if builder.platform_has_fast_fp16 and half:\n config.set_flag(trt.BuilderFlag.FP16)\n with builder.build_engine(network, config) as engine, open(f, \"wb\") as t:\n t.write(engine.serialize())\n return f, None\n\n\n@try_export\ndef export_saved_model(\n model,\n im,\n file,\n dynamic,\n tf_nms=False,\n agnostic_nms=False,\n topk_per_class=100,\n topk_all=100,\n iou_thres=0.45,\n conf_thres=0.25,\n keras=False,\n prefix=colorstr(\"TensorFlow SavedModel:\"),\n):\n # YOLOv5 TensorFlow SavedModel export\n try:\n import tensorflow as tf\n except Exception:\n check_requirements(f\"tensorflow{'' if torch.cuda.is_available() else '-macos' if MACOS else '-cpu'}<=2.15.1\")\n\n import tensorflow as tf\n from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2\n\n from asone.detectors.yolov5.yolov5.models.tf import TFModel\n\n LOGGER.info(f\"\\n{prefix} starting export with tensorflow {tf.__version__}...\")\n if tf.__version__ > \"2.13.1\":\n helper_url = \"https://github.com/ultralytics/yolov5/issues/12489\"\n LOGGER.info(\n f\"WARNING ⚠️ using Tensorflow {tf.__version__} > 2.13.1 might cause issue when exporting the model to tflite {helper_url}\"\n ) # handling issue https://github.com/ultralytics/yolov5/issues/12489\n f = str(file).replace(\".pt\", \"_saved_model\")\n batch_size, ch, *imgsz = list(im.shape) # BCHW\n\n tf_model = TFModel(cfg=model.yaml, model=model, nc=model.nc, imgsz=imgsz)\n im = tf.zeros((batch_size, *imgsz, ch)) # BHWC order for TensorFlow\n _ = tf_model.predict(im, tf_nms, agnostic_nms, topk_per_class, topk_all, iou_thres, conf_thres)\n inputs = tf.keras.Input(shape=(*imgsz, ch), batch_size=None if dynamic else batch_size)\n outputs = tf_model.predict(inputs, tf_nms, agnostic_nms, topk_per_class, topk_all, iou_thres, conf_thres)\n keras_model = tf.keras.Model(inputs=inputs, outputs=outputs)\n keras_model.trainable = False\n keras_model.summary()\n if keras:\n keras_model.save(f, save_format=\"tf\")\n else:\n spec = tf.TensorSpec(keras_model.inputs[0].shape, keras_model.inputs[0].dtype)\n m = tf.function(lambda x: keras_model(x)) # full model\n m = m.get_concrete_function(spec)\n frozen_func = convert_variables_to_constants_v2(m)\n tfm = tf.Module()\n tfm.__call__ = tf.function(lambda x: frozen_func(x)[:4] if tf_nms else frozen_func(x), [spec])\n tfm.__call__(im)\n tf.saved_model.save(\n tfm,\n f,\n options=tf.saved_model.SaveOptions(experimental_custom_gradients=False)\n if check_version(tf.__version__, \"2.6\")\n else tf.saved_model.SaveOptions(),\n )\n return f, keras_model\n\n\n@try_export\ndef export_pb(keras_model, file, prefix=colorstr(\"TensorFlow GraphDef:\")):\n \"\"\"Exports YOLOv5 model to TensorFlow GraphDef *.pb format; see https://github.com/leimao/Frozen_Graph_TensorFlow for details.\"\"\"\n import tensorflow as tf\n from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2\n\n LOGGER.info(f\"\\n{prefix} starting export with tensorflow {tf.__version__}...\")\n f = file.with_suffix(\".pb\")\n\n m = tf.function(lambda x: keras_model(x)) # full model\n m = m.get_concrete_function(tf.TensorSpec(keras_model.inputs[0].shape, keras_model.inputs[0].dtype))\n frozen_func = convert_variables_to_constants_v2(m)\n frozen_func.graph.as_graph_def()\n tf.io.write_graph(graph_or_graph_def=frozen_func.graph, logdir=str(f.parent), name=f.name, as_text=False)\n return f, None\n\n\n@try_export\ndef export_tflite(\n keras_model, im, file, int8, per_tensor, data, nms, agnostic_nms, prefix=colorstr(\"TensorFlow Lite:\")\n):\n # YOLOv5 TensorFlow Lite export\n import tensorflow as tf\n\n LOGGER.info(f\"\\n{prefix} starting export with tensorflow {tf.__version__}...\")\n batch_size, ch, *imgsz = list(im.shape) # BCHW\n f = str(file).replace(\".pt\", \"-fp16.tflite\")\n\n converter = tf.lite.TFLiteConverter.from_keras_model(keras_model)\n converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]\n converter.target_spec.supported_types = [tf.float16]\n converter.optimizations = [tf.lite.Optimize.DEFAULT]\n if int8:\n from models.tf import representative_dataset_gen\n\n dataset = LoadImages(check_dataset(check_yaml(data))[\"train\"], img_size=imgsz, auto=False)\n converter.representative_dataset = lambda: representative_dataset_gen(dataset, ncalib=100)\n converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]\n converter.target_spec.supported_types = []\n converter.inference_input_type = tf.uint8 # or tf.int8\n converter.inference_output_type = tf.uint8 # or tf.int8\n converter.experimental_new_quantizer = True\n if per_tensor:\n converter._experimental_disable_per_channel = True\n f = str(file).replace(\".pt\", \"-int8.tflite\")\n if nms or agnostic_nms:\n converter.target_spec.supported_ops.append(tf.lite.OpsSet.SELECT_TF_OPS)\n\n tflite_model = converter.convert()\n open(f, \"wb\").write(tflite_model)\n return f, None\n\n\n@try_export\ndef export_edgetpu(file, prefix=colorstr(\"Edge TPU:\")):\n \"\"\"\n Exports a YOLOv5 model to Edge TPU compatible TFLite format; requires Linux and Edge TPU compiler.\n\n https://coral.ai/docs/edgetpu/models-intro/\n \"\"\"\n cmd = \"edgetpu_compiler --version\"\n help_url = \"https://coral.ai/docs/edgetpu/compiler/\"\n assert platform.system() == \"Linux\", f\"export only supported on Linux. See {help_url}\"\n if subprocess.run(f\"{cmd} > /dev/null 2>&1\", shell=True).returncode != 0:\n LOGGER.info(f\"\\n{prefix} export requires Edge TPU compiler. Attempting install from {help_url}\")\n sudo = subprocess.run(\"sudo --version >/dev/null\", shell=True).returncode == 0 # sudo installed on system\n for c in (\n \"curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -\",\n 'echo \"deb https://packages.cloud.google.com/apt coral-edgetpu-stable main\" | sudo tee /etc/apt/sources.list.d/coral-edgetpu.list',\n \"sudo apt-get update\",\n \"sudo apt-get install edgetpu-compiler\",\n ):\n subprocess.run(c if sudo else c.replace(\"sudo \", \"\"), shell=True, check=True)\n ver = subprocess.run(cmd, shell=True, capture_output=True, check=True).stdout.decode().split()[-1]\n\n LOGGER.info(f\"\\n{prefix} starting export with Edge TPU compiler {ver}...\")\n f = str(file).replace(\".pt\", \"-int8_edgetpu.tflite\") # Edge TPU model\n f_tfl = str(file).replace(\".pt\", \"-int8.tflite\") # TFLite model\n\n subprocess.run(\n [\n \"edgetpu_compiler\",\n \"-s\",\n \"-d\",\n \"-k\",\n \"10\",\n \"--out_dir\",\n str(file.parent),\n f_tfl,\n ],\n check=True,\n )\n return f, None\n\n\n@try_export\ndef export_tfjs(file, int8, prefix=colorstr(\"TensorFlow.js:\")):\n \"\"\"Exports a YOLOv5 model to TensorFlow.js format, optionally with uint8 quantization.\"\"\"\n check_requirements(\"tensorflowjs\")\n import tensorflowjs as tfjs\n\n LOGGER.info(f\"\\n{prefix} starting export with tensorflowjs {tfjs.__version__}...\")\n f = str(file).replace(\".pt\", \"_web_model\") # js dir\n f_pb = file.with_suffix(\".pb\") # *.pb path\n f_json = f\"{f}/model.json\" # *.json path\n\n args = [\n \"tensorflowjs_converter\",\n \"--input_format=tf_frozen_model\",\n \"--quantize_uint8\" if int8 else \"\",\n \"--output_node_names=Identity,Identity_1,Identity_2,Identity_3\",\n str(f_pb),\n f,\n ]\n subprocess.run([arg for arg in args if arg], check=True)\n\n json = Path(f_json).read_text()\n with open(f_json, \"w\") as j: # sort JSON Identity_* in ascending order\n subst = re.sub(\n r'{\"outputs\": {\"Identity.?.?\": {\"name\": \"Identity.?.?\"}, '\n r'\"Identity.?.?\": {\"name\": \"Identity.?.?\"}, '\n r'\"Identity.?.?\": {\"name\": \"Identity.?.?\"}, '\n r'\"Identity.?.?\": {\"name\": \"Identity.?.?\"}}}',\n r'{\"outputs\": {\"Identity\": {\"name\": \"Identity\"}, '\n r'\"Identity_1\": {\"name\": \"Identity_1\"}, '\n r'\"Identity_2\": {\"name\": \"Identity_2\"}, '\n r'\"Identity_3\": {\"name\": \"Identity_3\"}}}',\n json,\n )\n j.write(subst)\n return f, None\n\n\ndef add_tflite_metadata(file, metadata, num_outputs):\n \"\"\"\n Adds TFLite metadata to a model file, supporting multiple outputs, as specified by TensorFlow guidelines.\n\n https://www.tensorflow.org/lite/models/convert/metadata\n \"\"\"\n with contextlib.suppress(ImportError):\n # check_requirements('tflite_support')\n from tflite_support import flatbuffers\n from tflite_support import metadata as _metadata\n from tflite_support import metadata_schema_py_generated as _metadata_fb\n\n tmp_file = Path(\"/tmp/meta.txt\")\n with open(tmp_file, \"w\") as meta_f:\n meta_f.write(str(metadata))\n\n model_meta = _metadata_fb.ModelMetadataT()\n label_file = _metadata_fb.AssociatedFileT()\n label_file.name = tmp_file.name\n model_meta.associatedFiles = [label_file]\n\n subgraph = _metadata_fb.SubGraphMetadataT()\n subgraph.inputTensorMetadata = [_metadata_fb.TensorMetadataT()]\n subgraph.outputTensorMetadata = [_metadata_fb.TensorMetadataT()] * num_outputs\n model_meta.subgraphMetadata = [subgraph]\n\n b = flatbuffers.Builder(0)\n b.Finish(model_meta.Pack(b), _metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER)\n metadata_buf = b.Output()\n\n populator = _metadata.MetadataPopulator.with_model_file(file)\n populator.load_metadata_buffer(metadata_buf)\n populator.load_associated_files([str(tmp_file)])\n populator.populate()\n tmp_file.unlink()\n\n\ndef pipeline_coreml(model, im, file, names, y, prefix=colorstr(\"CoreML Pipeline:\")):\n \"\"\"Converts a PyTorch YOLOv5 model to CoreML format with NMS, handling different input/output shapes and saving the\n model.\n \"\"\"\n import coremltools as ct\n from PIL import Image\n\n print(f\"{prefix} starting pipeline with coremltools {ct.__version__}...\")\n batch_size, ch, h, w = list(im.shape) # BCHW\n t = time.time()\n\n # YOLOv5 Output shapes\n spec = model.get_spec()\n out0, out1 = iter(spec.description.output)\n if platform.system() == \"Darwin\":\n img = Image.new(\"RGB\", (w, h)) # img(192 width, 320 height)\n # img = torch.zeros((*opt.img_size, 3)).numpy() # img size(320,192,3) iDetection\n out = model.predict({\"image\": img})\n out0_shape, out1_shape = out[out0.name].shape, out[out1.name].shape\n else: # linux and windows can not run model.predict(), get sizes from pytorch output y\n s = tuple(y[0].shape)\n out0_shape, out1_shape = (s[1], s[2] - 5), (s[1], 4) # (3780, 80), (3780, 4)\n\n # Checks\n nx, ny = spec.description.input[0].type.imageType.width, spec.description.input[0].type.imageType.height\n na, nc = out0_shape\n # na, nc = out0.type.multiArrayType.shape # number anchors, classes\n assert len(names) == nc, f\"{len(names)} names found for nc={nc}\" # check\n\n # Define output shapes (missing)\n out0.type.multiArrayType.shape[:] = out0_shape # (3780, 80)\n out1.type.multiArrayType.shape[:] = out1_shape # (3780, 4)\n # spec.neuralNetwork.preprocessing[0].featureName = '0'\n\n # Flexible input shapes\n # from coremltools.models.neural_network import flexible_shape_utils\n # s = [] # shapes\n # s.append(flexible_shape_utils.NeuralNetworkImageSize(320, 192))\n # s.append(flexible_shape_utils.NeuralNetworkImageSize(640, 384)) # (height, width)\n # flexible_shape_utils.add_enumerated_image_sizes(spec, feature_name='image', sizes=s)\n # r = flexible_shape_utils.NeuralNetworkImageSizeRange() # shape ranges\n # r.add_height_range((192, 640))\n # r.add_width_range((192, 640))\n # flexible_shape_utils.update_image_size_range(spec, feature_name='image', size_range=r)\n\n # Print\n print(spec.description)\n\n # Model from spec\n model = ct.models.MLModel(spec)\n\n # 3. Create NMS protobuf\n nms_spec = ct.proto.Model_pb2.Model()\n nms_spec.specificationVersion = 5\n for i in range(2):\n decoder_output = model._spec.description.output[i].SerializeToString()\n nms_spec.description.input.add()\n nms_spec.description.input[i].ParseFromString(decoder_output)\n nms_spec.description.output.add()\n nms_spec.description.output[i].ParseFromString(decoder_output)\n\n nms_spec.description.output[0].name = \"confidence\"\n nms_spec.description.output[1].name = \"coordinates\"\n\n output_sizes = [nc, 4]\n for i in range(2):\n ma_type = nms_spec.description.output[i].type.multiArrayType\n ma_type.shapeRange.sizeRanges.add()\n ma_type.shapeRange.sizeRanges[0].lowerBound = 0\n ma_type.shapeRange.sizeRanges[0].upperBound = -1\n ma_type.shapeRange.sizeRanges.add()\n ma_type.shapeRange.sizeRanges[1].lowerBound = output_sizes[i]\n ma_type.shapeRange.sizeRanges[1].upperBound = output_sizes[i]\n del ma_type.shape[:]\n\n nms = nms_spec.nonMaximumSuppression\n nms.confidenceInputFeatureName = out0.name # 1x507x80\n nms.coordinatesInputFeatureName = out1.name # 1x507x4\n nms.confidenceOutputFeatureName = \"confidence\"\n nms.coordinatesOutputFeatureName = \"coordinates\"\n nms.iouThresholdInputFeatureName = \"iouThreshold\"\n nms.confidenceThresholdInputFeatureName = \"confidenceThreshold\"\n nms.iouThreshold = 0.45\n nms.confidenceThreshold = 0.25\n nms.pickTop.perClass = True\n nms.stringClassLabels.vector.extend(names.values())\n nms_model = ct.models.MLModel(nms_spec)\n\n # 4. Pipeline models together\n pipeline = ct.models.pipeline.Pipeline(\n input_features=[\n (\"image\", ct.models.datatypes.Array(3, ny, nx)),\n (\"iouThreshold\", ct.models.datatypes.Double()),\n (\"confidenceThreshold\", ct.models.datatypes.Double()),\n ],\n output_features=[\"confidence\", \"coordinates\"],\n )\n pipeline.add_model(model)\n pipeline.add_model(nms_model)\n\n # Correct datatypes\n pipeline.spec.description.input[0].ParseFromString(model._spec.description.input[0].SerializeToString())\n pipeline.spec.description.output[0].ParseFromString(nms_model._spec.description.output[0].SerializeToString())\n pipeline.spec.description.output[1].ParseFromString(nms_model._spec.description.output[1].SerializeToString())\n\n # Update metadata\n pipeline.spec.specificationVersion = 5\n pipeline.spec.description.metadata.versionString = \"https://github.com/ultralytics/yolov5\"\n pipeline.spec.description.metadata.shortDescription = \"https://github.com/ultralytics/yolov5\"\n pipeline.spec.description.metadata.author = \"glenn.jocher@ultralytics.com\"\n pipeline.spec.description.metadata.license = \"https://github.com/ultralytics/yolov5/blob/master/LICENSE\"\n pipeline.spec.description.metadata.userDefined.update(\n {\n \"classes\": \",\".join(names.values()),\n \"iou_threshold\": str(nms.iouThreshold),\n \"confidence_threshold\": str(nms.confidenceThreshold),\n }\n )\n\n # Save the model\n f = file.with_suffix(\".mlmodel\") # filename\n model = ct.models.MLModel(pipeline.spec)\n model.input_description[\"image\"] = \"Input image\"\n model.input_description[\"iouThreshold\"] = f\"(optional) IOU Threshold override (default: {nms.iouThreshold})\"\n model.input_description[\"confidenceThreshold\"] = (\n f\"(optional) Confidence Threshold override (default: {nms.confidenceThreshold})\"\n )\n model.output_description[\"confidence\"] = 'Boxes × Class confidence (see user-defined metadata \"classes\")'\n model.output_description[\"coordinates\"] = \"Boxes × [x, y, width, height] (relative to image size)\"\n model.save(f) # pipelined\n print(f\"{prefix} pipeline success ({time.time() - t:.2f}s), saved as {f} ({file_size(f):.1f} MB)\")\n\n\n@smart_inference_mode()\ndef run(\n data=ROOT / \"data/coco128.yaml\", # 'dataset.yaml path'\n weights=ROOT / \"yolov5s.pt\", # weights path\n imgsz=(640, 640), # image (height, width)\n batch_size=1, # batch size\n device=\"cpu\", # cuda device, i.e. 0 or 0,1,2,3 or cpu\n include=(\"torchscript\", \"onnx\"), # include formats\n half=False, # FP16 half-precision export\n inplace=False, # set YOLOv5 Detect() inplace=True\n keras=False, # use Keras\n optimize=False, # TorchScript: optimize for mobile\n int8=False, # CoreML/TF INT8 quantization\n per_tensor=False, # TF per tensor quantization\n dynamic=False, # ONNX/TF/TensorRT: dynamic axes\n simplify=False, # ONNX: simplify model\n opset=12, # ONNX: opset version\n verbose=False, # TensorRT: verbose log\n workspace=4, # TensorRT: workspace size (GB)\n nms=False, # TF: add NMS to model\n agnostic_nms=False, # TF: add agnostic NMS to model\n topk_per_class=100, # TF.js NMS: topk per class to keep\n topk_all=100, # TF.js NMS: topk for all classes to keep\n iou_thres=0.45, # TF.js NMS: IoU threshold\n conf_thres=0.25, # TF.js NMS: confidence threshold\n):\n t = time.time()\n include = [x.lower() for x in include] # to lowercase\n fmts = tuple(export_formats()[\"Argument\"][1:]) # --include arguments\n flags = [x in include for x in fmts]\n assert sum(flags) == len(include), f\"ERROR: Invalid --include {include}, valid --include arguments are {fmts}\"\n jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle = flags # export booleans\n file = Path(url2file(weights) if str(weights).startswith((\"http:/\", \"https:/\")) else weights) # PyTorch weights\n\n # Load PyTorch model\n device = select_device(device)\n if half:\n assert device.type != \"cpu\" or coreml, \"--half only compatible with GPU export, i.e. use --device 0\"\n assert not dynamic, \"--half not compatible with --dynamic, i.e. use either --half or --dynamic but not both\"\n model = attempt_load(weights, device=device, inplace=True, fuse=True) # load FP32 model\n\n # Checks\n imgsz *= 2 if len(imgsz) == 1 else 1 # expand\n if optimize:\n assert device.type == \"cpu\", \"--optimize not compatible with cuda devices, i.e. use --device cpu\"\n\n # Input\n gs = int(max(model.stride)) # grid size (max stride)\n imgsz = [check_img_size(x, gs) for x in imgsz] # verify img_size are gs-multiples\n im = torch.zeros(batch_size, 3, *imgsz).to(device) # image size(1,3,320,192) BCHW iDetection\n\n # Update model\n model.eval()\n for k, m in model.named_modules():\n if isinstance(m, Detect):\n m.inplace = inplace\n m.dynamic = dynamic\n m.export = True\n\n for _ in range(2):\n y = model(im) # dry runs\n if half and not coreml:\n im, model = im.half(), model.half() # to FP16\n shape = tuple((y[0] if isinstance(y, tuple) else y).shape) # model output shape\n metadata = {\"stride\": int(max(model.stride)), \"names\": model.names} # model metadata\n LOGGER.info(f\"\\n{colorstr('PyTorch:')} starting from {file} with output shape {shape} ({file_size(file):.1f} MB)\")\n\n # Exports\n f = [\"\"] * len(fmts) # exported filenames\n warnings.filterwarnings(action=\"ignore\", category=torch.jit.TracerWarning) # suppress TracerWarning\n if jit: # TorchScript\n f[0], _ = export_torchscript(model, im, file, optimize)\n if engine: # TensorRT required before ONNX\n f[1], _ = export_engine(model, im, file, half, dynamic, simplify, workspace, verbose)\n if onnx or xml: # OpenVINO requires ONNX\n f[2], _ = export_onnx(model, im, file, opset, dynamic, simplify)\n if xml: # OpenVINO\n f[3], _ = export_openvino(file, metadata, half, int8, data)\n if coreml: # CoreML\n f[4], ct_model = export_coreml(model, im, file, int8, half, nms)\n if nms:\n pipeline_coreml(ct_model, im, file, model.names, y)\n if any((saved_model, pb, tflite, edgetpu, tfjs)): # TensorFlow formats\n assert not tflite or not tfjs, \"TFLite and TF.js models must be exported separately, please pass only one type.\"\n assert not isinstance(model, ClassificationModel), \"ClassificationModel export to TF formats not yet supported.\"\n f[5], s_model = export_saved_model(\n model.cpu(),\n im,\n file,\n dynamic,\n tf_nms=nms or agnostic_nms or tfjs,\n agnostic_nms=agnostic_nms or tfjs,\n topk_per_class=topk_per_class,\n topk_all=topk_all,\n iou_thres=iou_thres,\n conf_thres=conf_thres,\n keras=keras,\n )\n if pb or tfjs: # pb prerequisite to tfjs\n f[6], _ = export_pb(s_model, file)\n if tflite or edgetpu:\n f[7], _ = export_tflite(\n s_model, im, file, int8 or edgetpu, per_tensor, data=data, nms=nms, agnostic_nms=agnostic_nms\n )\n if edgetpu:\n f[8], _ = export_edgetpu(file)\n add_tflite_metadata(f[8] or f[7], metadata, num_outputs=len(s_model.outputs))\n if tfjs:\n f[9], _ = export_tfjs(file, int8)\n if paddle: # PaddlePaddle\n f[10], _ = export_paddle(model, im, file, metadata)\n\n # Finish\n f = [str(x) for x in f if x] # filter out '' and None\n if any(f):\n cls, det, seg = (isinstance(model, x) for x in (ClassificationModel, DetectionModel, SegmentationModel)) # type\n det &= not seg # segmentation models inherit from SegmentationModel(DetectionModel)\n dir = Path(\"segment\" if seg else \"classify\" if cls else \"\")\n h = \"--half\" if half else \"\" # --half FP16 inference arg\n s = (\n \"# WARNING ⚠️ ClassificationModel not yet supported for PyTorch Hub AutoShape inference\"\n if cls\n else \"# WARNING ⚠️ SegmentationModel not yet supported for PyTorch Hub AutoShape inference\"\n if seg\n else \"\"\n )\n LOGGER.info(\n f'\\nExport complete ({time.time() - t:.1f}s)'\n f\"\\nResults saved to {colorstr('bold', file.parent.resolve())}\"\n f\"\\nDetect: python {dir / ('detect.py' if det else 'predict.py')} --weights {f[-1]} {h}\"\n f\"\\nValidate: python {dir / 'val.py'} --weights {f[-1]} {h}\"\n f\"\\nPyTorch Hub: model = torch.hub.load('ultralytics/yolov5', 'custom', '{f[-1]}') {s}\"\n f'\\nVisualize: https://netron.app'\n )\n return f # return list of exported files/dirs\n\n\ndef parse_opt(known=False):\n \"\"\"Parses command-line arguments for YOLOv5 model export configurations, returning the parsed options.\"\"\"\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--data\", type=str, default=ROOT / \"data/coco128.yaml\", help=\"dataset.yaml path\")\n parser.add_argument(\"--weights\", nargs=\"+\", type=str, default=ROOT / \"yolov5s.pt\", help=\"model.pt path(s)\")\n parser.add_argument(\"--imgsz\", \"--img\", \"--img-size\", nargs=\"+\", type=int, default=[640, 640], help=\"image (h, w)\")\n parser.add_argument(\"--batch-size\", type=int, default=1, help=\"batch size\")\n parser.add_argument(\"--device\", default=\"cpu\", help=\"cuda device, i.e. 0 or 0,1,2,3 or cpu\")\n parser.add_argument(\"--half\", action=\"store_true\", help=\"FP16 half-precision export\")\n parser.add_argument(\"--inplace\", action=\"store_true\", help=\"set YOLOv5 Detect() inplace=True\")\n parser.add_argument(\"--keras\", action=\"store_true\", help=\"TF: use Keras\")\n parser.add_argument(\"--optimize\", action=\"store_true\", help=\"TorchScript: optimize for mobile\")\n parser.add_argument(\"--int8\", action=\"store_true\", help=\"CoreML/TF/OpenVINO INT8 quantization\")\n parser.add_argument(\"--per-tensor\", action=\"store_true\", help=\"TF per-tensor quantization\")\n parser.add_argument(\"--dynamic\", action=\"store_true\", help=\"ONNX/TF/TensorRT: dynamic axes\")\n parser.add_argument(\"--simplify\", action=\"store_true\", help=\"ONNX: simplify model\")\n parser.add_argument(\"--opset\", type=int, default=17, help=\"ONNX: opset version\")\n parser.add_argument(\"--verbose\", action=\"store_true\", help=\"TensorRT: verbose log\")\n parser.add_argument(\"--workspace\", type=int, default=4, help=\"TensorRT: workspace size (GB)\")\n parser.add_argument(\"--nms\", action=\"store_true\", help=\"TF: add NMS to model\")\n parser.add_argument(\"--agnostic-nms\", action=\"store_true\", help=\"TF: add agnostic NMS to model\")\n parser.add_argument(\"--topk-per-class\", type=int, default=100, help=\"TF.js NMS: topk per class to keep\")\n parser.add_argument(\"--topk-all\", type=int, default=100, help=\"TF.js NMS: topk for all classes to keep\")\n parser.add_argument(\"--iou-thres\", type=float, default=0.45, help=\"TF.js NMS: IoU threshold\")\n parser.add_argument(\"--conf-thres\", type=float, default=0.25, help=\"TF.js NMS: confidence threshold\")\n parser.add_argument(\n \"--include\",\n nargs=\"+\",\n default=[\"torchscript\"],\n help=\"torchscript, onnx, openvino, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle\",\n )\n opt = parser.parse_known_args()[0] if known else parser.parse_args()\n print_args(vars(opt))\n return opt\n\n\ndef main(opt):\n \"\"\"Executes the YOLOv5 model inference or export with specified weights and options.\"\"\"\n for opt.weights in opt.weights if isinstance(opt.weights, list) else [opt.weights]:\n run(**vars(opt))\n\n\nif __name__ == \"__main__\":\n opt = parse_opt()\n main(opt)" }, { "path": "asone/detectors/yolov5/yolov5/models/__init__.py", "content": "# import os\n# import sys\n# sys.path.append(os.path.dirname(__file__))\n" }, { "path": "asone/detectors/yolov5/yolov5/models/common.py", "content": "# YOLOv5 🚀 by Ultralytics, GPL-3.0 license\n\"\"\"\nCommon modules\n\"\"\"\n\nimport json\nimport math\nimport platform\nimport warnings\nfrom collections import OrderedDict, namedtuple\nfrom copy import copy\nfrom pathlib import Path\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport requests\nimport torch\nimport torch.nn as nn\nimport yaml\nfrom PIL import Image\nfrom torch.cuda import amp\n\nfrom asone.detectors.yolov5.yolov5.models.general import (LOGGER, check_requirements,\n check_suffix, check_version,\n colorstr, increment_path)\n \ndef autopad(k, p=None): # kernel, padding\n # Pad to 'same'\n if p is None:\n p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad\n return p\n\n\nclass Conv(nn.Module):\n # Standard convolution\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super().__init__()\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False)\n self.bn = nn.BatchNorm2d(c2)\n self.act = nn.SiLU() if act is True else (act if isinstance(act, nn.Module) else nn.Identity())\n\n def forward(self, x):\n return self.act(self.bn(self.conv(x)))\n\n def forward_fuse(self, x):\n return self.act(self.conv(x))\n\n\nclass DWConv(Conv):\n # Depth-wise convolution class\n def __init__(self, c1, c2, k=1, s=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super().__init__(c1, c2, k, s, g=math.gcd(c1, c2), act=act)\n\n\nclass DWConvTranspose2d(nn.ConvTranspose2d):\n # Depth-wise transpose convolution class\n def __init__(self, c1, c2, k=1, s=1, p1=0, p2=0): # ch_in, ch_out, kernel, stride, padding, padding_out\n super().__init__(c1, c2, k, s, p1, p2, groups=math.gcd(c1, c2))\n\n\nclass TransformerLayer(nn.Module):\n # Transformer layer https://arxiv.org/abs/2010.11929 (LayerNorm layers removed for better performance)\n def __init__(self, c, num_heads):\n super().__init__()\n self.q = nn.Linear(c, c, bias=False)\n self.k = nn.Linear(c, c, bias=False)\n self.v = nn.Linear(c, c, bias=False)\n self.ma = nn.MultiheadAttention(embed_dim=c, num_heads=num_heads)\n self.fc1 = nn.Linear(c, c, bias=False)\n self.fc2 = nn.Linear(c, c, bias=False)\n\n def forward(self, x):\n x = self.ma(self.q(x), self.k(x), self.v(x))[0] + x\n x = self.fc2(self.fc1(x)) + x\n return x\n\n\nclass TransformerBlock(nn.Module):\n # Vision Transformer https://arxiv.org/abs/2010.11929\n def __init__(self, c1, c2, num_heads, num_layers):\n super().__init__()\n self.conv = None\n if c1 != c2:\n self.conv = Conv(c1, c2)\n self.linear = nn.Linear(c2, c2) # learnable position embedding\n self.tr = nn.Sequential(*(TransformerLayer(c2, num_heads) for _ in range(num_layers)))\n self.c2 = c2\n\n def forward(self, x):\n if self.conv is not None:\n x = self.conv(x)\n b, _, w, h = x.shape\n p = x.flatten(2).permute(2, 0, 1)\n return self.tr(p + self.linear(p)).permute(1, 2, 0).reshape(b, self.c2, w, h)\n\n\nclass Bottleneck(nn.Module):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c2, 3, 1, g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass BottleneckCSP(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)))\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), 1))))\n\n\nclass CrossConv(nn.Module):\n # Cross Convolution Downsample\n def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False):\n # ch_in, ch_out, kernel, stride, groups, expansion, shortcut\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, (1, k), (1, s))\n self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass C3(nn.Module):\n # CSP Bottleneck with 3 convolutions\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2)\n self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)))\n\n def forward(self, x):\n return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1))\n\n\nclass C3x(C3):\n # C3 module with cross-convolutions\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5):\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e)\n self.m = nn.Sequential(*(CrossConv(c_, c_, 3, 1, g, 1.0, shortcut) for _ in range(n)))\n\n\nclass C3TR(C3):\n # C3 module with TransformerBlock()\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5):\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e)\n self.m = TransformerBlock(c_, c_, 4, n)\n\n\nclass C3SPP(C3):\n # C3 module with SPP()\n def __init__(self, c1, c2, k=(5, 9, 13), n=1, shortcut=True, g=1, e=0.5):\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e)\n self.m = SPP(c_, c_, k)\n\n\nclass C3Ghost(C3):\n # C3 module with GhostBottleneck()\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5):\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*(GhostBottleneck(c_, c_) for _ in range(n)))\n\n\nclass SPP(nn.Module):\n # Spatial Pyramid Pooling (SPP) layer https://arxiv.org/abs/1406.4729\n def __init__(self, c1, c2, k=(5, 9, 13)):\n super().__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1)\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\n\n def forward(self, x):\n x = self.cv1(x)\n with warnings.catch_warnings():\n warnings.simplefilter('ignore') # suppress torch 1.9.0 max_pool2d() warning\n return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1))\n\n\nclass SPPF(nn.Module):\n # Spatial Pyramid Pooling - Fast (SPPF) layer for YOLOv5 by Glenn Jocher\n def __init__(self, c1, c2, k=5): # equivalent to SPP(k=(5, 9, 13))\n super().__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_ * 4, c2, 1, 1)\n self.m = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2)\n\n def forward(self, x):\n x = self.cv1(x)\n with warnings.catch_warnings():\n warnings.simplefilter('ignore') # suppress torch 1.9.0 max_pool2d() warning\n y1 = self.m(x)\n y2 = self.m(y1)\n return self.cv2(torch.cat((x, y1, y2, self.m(y2)), 1))\n\n\nclass Focus(nn.Module):\n # Focus wh information into c-space\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super().__init__()\n self.conv = Conv(c1 * 4, c2, k, s, p, g, act)\n # self.contract = Contract(gain=2)\n\n def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2)\n return self.conv(torch.cat((x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]), 1))\n # return self.conv(self.contract(x))\n\n\nclass GhostConv(nn.Module):\n # Ghost Convolution https://github.com/huawei-noah/ghostnet\n def __init__(self, c1, c2, k=1, s=1, g=1, act=True): # ch_in, ch_out, kernel, stride, groups\n super().__init__()\n c_ = c2 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, k, s, None, g, act)\n self.cv2 = Conv(c_, c_, 5, 1, None, c_, act)\n\n def forward(self, x):\n y = self.cv1(x)\n return torch.cat((y, self.cv2(y)), 1)\n\n\nclass GhostBottleneck(nn.Module):\n # Ghost Bottleneck https://github.com/huawei-noah/ghostnet\n def __init__(self, c1, c2, k=3, s=1): # ch_in, ch_out, kernel, stride\n super().__init__()\n c_ = c2 // 2\n self.conv = nn.Sequential(\n GhostConv(c1, c_, 1, 1), # pw\n DWConv(c_, c_, k, s, act=False) if s == 2 else nn.Identity(), # dw\n GhostConv(c_, c2, 1, 1, act=False)) # pw-linear\n self.shortcut = nn.Sequential(DWConv(c1, c1, k, s, act=False), Conv(c1, c2, 1, 1,\n act=False)) if s == 2 else nn.Identity()\n\n def forward(self, x):\n return self.conv(x) + self.shortcut(x)\n\n\nclass Contract(nn.Module):\n # Contract width-height into channels, i.e. x(1,64,80,80) to x(1,256,40,40)\n def __init__(self, gain=2):\n super().__init__()\n self.gain = gain\n\n def forward(self, x):\n b, c, h, w = x.size() # assert (h / s == 0) and (W / s == 0), 'Indivisible gain'\n s = self.gain\n x = x.view(b, c, h // s, s, w // s, s) # x(1,64,40,2,40,2)\n x = x.permute(0, 3, 5, 1, 2, 4).contiguous() # x(1,2,2,64,40,40)\n return x.view(b, c * s * s, h // s, w // s) # x(1,256,40,40)\n\n\nclass Expand(nn.Module):\n # Expand channels into width-height, i.e. x(1,64,80,80) to x(1,16,160,160)\n def __init__(self, gain=2):\n super().__init__()\n self.gain = gain\n\n def forward(self, x):\n b, c, h, w = x.size() # assert C / s ** 2 == 0, 'Indivisible gain'\n s = self.gain\n x = x.view(b, s, s, c // s ** 2, h, w) # x(1,2,2,16,80,80)\n x = x.permute(0, 3, 4, 1, 5, 2).contiguous() # x(1,16,80,2,80,2)\n return x.view(b, c // s ** 2, h * s, w * s) # x(1,16,160,160)\n\n\nclass Concat(nn.Module):\n # Concatenate a list of tensors along dimension\n def __init__(self, dimension=1):\n super().__init__()\n self.d = dimension\n\n def forward(self, x):\n return torch.cat(x, self.d)\n\n\nclass DetectMultiBackend(nn.Module):\n # YOLOv5 MultiBackend class for python inference on various backends\n def __init__(self, weights='yolov5s.pt', device=torch.device('cpu'), dnn=False, data=None, fp16=False, fuse=True):\n # Usage:\n # PyTorch: weights = *.pt\n # TorchScript: *.torchscript\n # ONNX Runtime: *.onnx\n # ONNX OpenCV DNN: *.onnx with --dnn\n # OpenVINO: *.xml\n # CoreML: *.mlmodel\n # TensorRT: *.engine\n # TensorFlow SavedModel: *_saved_model\n # TensorFlow GraphDef: *.pb\n # TensorFlow Lite: *.tflite\n # TensorFlow Edge TPU: *_edgetpu.tflite\n from asone.detectors.yolov5.yolov5.models.experimental import attempt_download, attempt_load # scoped to avoid circular import\n\n super().__init__()\n w = str(weights[0] if isinstance(weights, list) else weights)\n pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs = self.model_type(w) # get backend\n w = attempt_download(w) # download if not local\n fp16 &= (pt or jit or onnx or engine) and device.type != 'cpu' # FP16\n stride, names = 32, [f'class{i}' for i in range(1000)] # assign defaults\n if data: # assign class names (optional)\n with open(data, errors='ignore') as f:\n names = yaml.safe_load(f)['names']\n\n if pt: # PyTorch\n model = attempt_load(weights if isinstance(weights, list) else w, device=device, inplace=True, fuse=fuse)\n stride = max(int(model.stride.max()), 32) # model stride\n names = model.module.names if hasattr(model, 'module') else model.names # get class names\n model.half() if fp16 else model.float()\n self.model = model # explicitly assign for to(), cpu(), cuda(), half()\n elif jit: # TorchScript\n LOGGER.info(f'Loading {w} for TorchScript inference...')\n extra_files = {'config.txt': ''} # model metadata\n model = torch.jit.load(w, _extra_files=extra_files)\n model.half() if fp16 else model.float()\n if extra_files['config.txt']:\n d = json.loads(extra_files['config.txt']) # extra_files dict\n stride, names = int(d['stride']), d['names']\n elif dnn: # ONNX OpenCV DNN\n LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...')\n check_requirements(('opencv-python>=4.5.4',))\n net = cv2.dnn.readNetFromONNX(w)\n elif onnx: # ONNX Runtime\n LOGGER.info(f'Loading {w} for ONNX Runtime inference...')\n cuda = torch.cuda.is_available()\n check_requirements(('onnx', 'onnxruntime-gpu' if cuda else 'onnxruntime'))\n import onnxruntime\n providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if cuda else ['CPUExecutionProvider']\n session = onnxruntime.InferenceSession(w, providers=providers)\n meta = session.get_modelmeta().custom_metadata_map # metadata\n if 'stride' in meta:\n stride, names = int(meta['stride']), eval(meta['names'])\n elif xml: # OpenVINO\n LOGGER.info(f'Loading {w} for OpenVINO inference...')\n check_requirements(('openvino',)) # requires openvino-dev: https://pypi.org/project/openvino-dev/\n import openvino\n from openvino.runtime import Core, Layout, get_batch\n ie = Core()\n if not Path(w).is_file(): # if not *.xml\n w = next(Path(w).glob('*.xml')) # get *.xml file from *_openvino_model dir\n network = ie.read_model(model=w, weights=Path(w).with_suffix('.bin'))\n if network.get_parameters()[0].get_layout().empty:\n network.get_parameters()[0].set_layout(Layout(\"NCHW\"))\n batch_dim = get_batch(network)\n if batch_dim.is_static:\n batch_size = batch_dim.get_length()\n executable_network = ie.compile_model(network, device_name=\"CPU\") # device_name=\"MYRIAD\" for Intel NCS2\n output_layer = next(iter(executable_network.outputs))\n meta = Path(w).with_suffix('.yaml')\n if meta.exists():\n stride, names = self._load_metadata(meta) # load metadata\n elif engine: # TensorRT\n LOGGER.info(f'Loading {w} for TensorRT inference...')\n import tensorrt as trt # https://developer.nvidia.com/nvidia-tensorrt-download\n check_version(trt.__version__, '7.0.0', hard=True) # require tensorrt>=7.0.0\n Binding = namedtuple('Binding', ('name', 'dtype', 'shape', 'data', 'ptr'))\n logger = trt.Logger(trt.Logger.INFO)\n with open(w, 'rb') as f, trt.Runtime(logger) as runtime:\n model = runtime.deserialize_cuda_engine(f.read())\n context = model.create_execution_context()\n bindings = OrderedDict()\n fp16 = False # default updated below\n dynamic_input = False\n for index in range(model.num_bindings):\n name = model.get_binding_name(index)\n dtype = trt.nptype(model.get_binding_dtype(index))\n if model.binding_is_input(index):\n if -1 in tuple(model.get_binding_shape(index)): # dynamic\n dynamic_input = True\n context.set_binding_shape(index, tuple(model.get_profile_shape(0, index)[2]))\n if dtype == np.float16:\n fp16 = True\n shape = tuple(context.get_binding_shape(index))\n data = torch.from_numpy(np.empty(shape, dtype=np.dtype(dtype))).to(device)\n bindings[name] = Binding(name, dtype, shape, data, int(data.data_ptr()))\n binding_addrs = OrderedDict((n, d.ptr) for n, d in bindings.items())\n batch_size = bindings['images'].shape[0] # if dynamic, this is instead max batch size\n elif coreml: # CoreML\n LOGGER.info(f'Loading {w} for CoreML inference...')\n import coremltools as ct\n model = ct.models.MLModel(w)\n else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU)\n if saved_model: # SavedModel\n LOGGER.info(f'Loading {w} for TensorFlow SavedModel inference...')\n import tensorflow as tf\n keras = False # assume TF1 saved_model\n model = tf.keras.models.load_model(w) if keras else tf.saved_model.load(w)\n elif pb: # GraphDef https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt\n LOGGER.info(f'Loading {w} for TensorFlow GraphDef inference...')\n import tensorflow as tf\n\n def wrap_frozen_graph(gd, inputs, outputs):\n x = tf.compat.v1.wrap_function(lambda: tf.compat.v1.import_graph_def(gd, name=\"\"), []) # wrapped\n ge = x.graph.as_graph_element\n return x.prune(tf.nest.map_structure(ge, inputs), tf.nest.map_structure(ge, outputs))\n\n gd = tf.Graph().as_graph_def() # graph_def\n with open(w, 'rb') as f:\n gd.ParseFromString(f.read())\n frozen_func = wrap_frozen_graph(gd, inputs=\"x:0\", outputs=\"Identity:0\")\n elif tflite or edgetpu: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python\n try: # https://coral.ai/docs/edgetpu/tflite-python/#update-existing-tf-lite-code-for-the-edge-tpu\n from tflite_runtime.interpreter import Interpreter, load_delegate\n except ImportError:\n import tensorflow as tf\n Interpreter, load_delegate = tf.lite.Interpreter, tf.lite.experimental.load_delegate,\n if edgetpu: # Edge TPU https://coral.ai/software/#edgetpu-runtime\n LOGGER.info(f'Loading {w} for TensorFlow Lite Edge TPU inference...')\n delegate = {\n 'Linux': 'libedgetpu.so.1',\n 'Darwin': 'libedgetpu.1.dylib',\n 'Windows': 'edgetpu.dll'}[platform.system()]\n interpreter = Interpreter(model_path=w, experimental_delegates=[load_delegate(delegate)])\n else: # Lite\n LOGGER.info(f'Loading {w} for TensorFlow Lite inference...')\n interpreter = Interpreter(model_path=w) # load TFLite model\n interpreter.allocate_tensors() # allocate\n input_details = interpreter.get_input_details() # inputs\n output_details = interpreter.get_output_details() # outputs\n elif tfjs:\n raise Exception('ERROR: YOLOv5 TF.js inference is not supported')\n else:\n raise Exception(f'ERROR: {w} is not a supported format')\n self.__dict__.update(locals()) # assign all variables to self\n\n def forward(self, im, augment=False, visualize=False, val=False):\n # YOLOv5 MultiBackend inference\n b, ch, h, w = im.shape # batch, channel, height, width\n if self.fp16 and im.dtype != torch.float16:\n im = im.half() # to FP16\n\n if self.pt: # PyTorch\n y = self.model(im, augment=augment, visualize=visualize)[0]\n elif self.jit: # TorchScript\n y = self.model(im)[0]\n elif self.dnn: # ONNX OpenCV DNN\n im = im.cpu().numpy() # torch to numpy\n self.net.setInput(im)\n y = self.net.forward()\n elif self.onnx: # ONNX Runtime\n im = im.cpu().numpy() # torch to numpy\n y = self.session.run([self.session.get_outputs()[0].name], {self.session.get_inputs()[0].name: im})[0]\n elif self.xml: # OpenVINO\n im = im.cpu().numpy() # FP32\n y = self.executable_network([im])[self.output_layer]\n elif self.engine: # TensorRT\n if im.shape != self.bindings['images'].shape and self.dynamic_input:\n self.context.set_binding_shape(self.model.get_binding_index('images'), im.shape) # reshape if dynamic\n self.bindings['images'] = self.bindings['images']._replace(shape=im.shape)\n assert im.shape == self.bindings['images'].shape, (\n f\"image shape {im.shape} exceeds model max shape {self.bindings['images'].shape}\" if self.dynamic_input\n else f\"image shape {im.shape} does not match model shape {self.bindings['images'].shape}\")\n self.binding_addrs['images'] = int(im.data_ptr())\n self.context.execute_v2(list(self.binding_addrs.values()))\n y = self.bindings['output'].data\n elif self.coreml: # CoreML\n im = im.permute(0, 2, 3, 1).cpu().numpy() # torch BCHW to numpy BHWC shape(1,320,192,3)\n im = Image.fromarray((im[0] * 255).astype('uint8'))\n # im = im.resize((192, 320), Image.ANTIALIAS)\n y = self.model.predict({'image': im}) # coordinates are xywh normalized\n if 'confidence' in y:\n box = xywh2xyxy(y['coordinates'] * [[w, h, w, h]]) # xyxy pixels\n conf, cls = y['confidence'].max(1), y['confidence'].argmax(1).astype(np.float)\n y = np.concatenate((box, conf.reshape(-1, 1), cls.reshape(-1, 1)), 1)\n else:\n k = 'var_' + str(sorted(int(k.replace('var_', '')) for k in y)[-1]) # output key\n y = y[k] # output\n else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU)\n im = im.permute(0, 2, 3, 1).cpu().numpy() # torch BCHW to numpy BHWC shape(1,320,192,3)\n if self.saved_model: # SavedModel\n y = (self.model(im, training=False) if self.keras else self.model(im)).numpy()\n elif self.pb: # GraphDef\n y = self.frozen_func(x=self.tf.constant(im)).numpy()\n else: # Lite or Edge TPU\n input, output = self.input_details[0], self.output_details[0]\n int8 = input['dtype'] == np.uint8 # is TFLite quantized uint8 model\n if int8:\n scale, zero_point = input['quantization']\n im = (im / scale + zero_point).astype(np.uint8) # de-scale\n self.interpreter.set_tensor(input['index'], im)\n self.interpreter.invoke()\n y = self.interpreter.get_tensor(output['index'])\n if int8:\n scale, zero_point = output['quantization']\n y = (y.astype(np.float32) - zero_point) * scale # re-scale\n y[..., :4] *= [w, h, w, h] # xywh normalized to pixels\n\n if isinstance(y, np.ndarray):\n y = torch.tensor(y, device=self.device)\n return (y, []) if val else y\n\n def warmup(self, imgsz=(1, 3, 640, 640)):\n # Warmup model by running inference once\n warmup_types = self.pt, self.jit, self.onnx, self.engine, self.saved_model, self.pb\n if any(warmup_types) and self.device.type != 'cpu':\n im = torch.zeros(*imgsz, dtype=torch.half if self.fp16 else torch.float, device=self.device) # input\n for _ in range(2 if self.jit else 1): #\n self.forward(im) # warmup\n\n @staticmethod\n def model_type(p='path/to/model.pt'):\n # Return model type from model path, i.e. path='path/to/model.onnx' -> type=onnx\n from asone.detectors.yolov5.yolov5.export import export_formats\n suffixes = list(export_formats().Suffix) + ['.xml'] # export suffixes\n check_suffix(p, suffixes) # checks\n p = Path(p).name # eliminate trailing separators\n pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, xml2 = (s in p for s in suffixes)\n xml |= xml2 # *_openvino_model or *.xml\n tflite &= not edgetpu # *.tflite\n return pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs\n\n @staticmethod\n def _load_metadata(f='path/to/meta.yaml'):\n # Load metadata from meta.yaml if it exists\n with open(f, errors='ignore') as f:\n d = yaml.safe_load(f)\n return d['stride'], d['names'] # assign stride, names\n\n\nclass AutoShape(nn.Module):\n # YOLOv5 input-robust model wrapper for passing cv2/np/PIL/torch inputs. Includes preprocessing, inference and NMS\n conf = 0.25 # NMS confidence threshold\n iou = 0.45 # NMS IoU threshold\n agnostic = False # NMS class-agnostic\n multi_label = False # NMS multiple labels per box\n classes = None # (optional list) filter by class, i.e. = [0, 15, 16] for COCO persons, cats and dogs\n max_det = 1000 # maximum number of detections per image\n amp = False # Automatic Mixed Precision (AMP) inference\n\n def __init__(self, model, verbose=True):\n super().__init__()\n if verbose:\n LOGGER.info('Adding AutoShape... ')\n copy_attr(self, model, include=('yaml', 'nc', 'hyp', 'names', 'stride', 'abc'), exclude=()) # copy attributes\n self.dmb = isinstance(model, DetectMultiBackend) # DetectMultiBackend() instance\n self.pt = not self.dmb or model.pt # PyTorch model\n self.model = model.eval()\n\n def _apply(self, fn):\n # Apply to(), cpu(), cuda(), half() to model tensors that are not parameters or registered buffers\n self = super()._apply(fn)\n if self.pt:\n m = self.model.model.model[-1] if self.dmb else self.model.model[-1] # Detect()\n m.stride = fn(m.stride)\n m.grid = list(map(fn, m.grid))\n if isinstance(m.anchor_grid, list):\n m.anchor_grid = list(map(fn, m.anchor_grid))\n return self\n\n @torch.no_grad()\n def forward(self, imgs, size=640, augment=False, profile=False):\n # Inference from various sources. For height=640, width=1280, RGB images example inputs are:\n # file: imgs = 'data/images/zidane.jpg' # str or PosixPath\n # URI: = 'https://ultralytics.com/images/zidane.jpg'\n # OpenCV: = cv2.imread('image.jpg')[:,:,::-1] # HWC BGR to RGB x(640,1280,3)\n # PIL: = Image.open('image.jpg') or ImageGrab.grab() # HWC x(640,1280,3)\n # numpy: = np.zeros((640,1280,3)) # HWC\n # torch: = torch.zeros(16,3,320,640) # BCHW (scaled to size=640, 0-1 values)\n # multiple: = [Image.open('image1.jpg'), Image.open('image2.jpg'), ...] # list of images\n\n t = [time_sync()]\n p = next(self.model.parameters()) if self.pt else torch.zeros(1, device=self.model.device) # for device, type\n autocast = self.amp and (p.device.type != 'cpu') # Automatic Mixed Precision (AMP) inference\n if isinstance(imgs, torch.Tensor): # torch\n with amp.autocast(autocast):\n return self.model(imgs.to(p.device).type_as(p), augment, profile) # inference\n\n # Pre-process\n n, imgs = (len(imgs), list(imgs)) if isinstance(imgs, (list, tuple)) else (1, [imgs]) # number, list of images\n shape0, shape1, files = [], [], [] # image and inference shapes, filenames\n for i, im in enumerate(imgs):\n f = f'image{i}' # filename\n if isinstance(im, (str, Path)): # filename or uri\n im, f = Image.open(requests.get(im, stream=True).raw if str(im).startswith('http') else im), im\n im = np.asarray(exif_transpose(im))\n elif isinstance(im, Image.Image): # PIL Image\n im, f = np.asarray(exif_transpose(im)), getattr(im, 'filename', f) or f\n files.append(Path(f).with_suffix('.jpg').name)\n if im.shape[0] < 5: # image in CHW\n im = im.transpose((1, 2, 0)) # reverse dataloader .transpose(2, 0, 1)\n im = im[..., :3] if im.ndim == 3 else np.tile(im[..., None], 3) # enforce 3ch input\n s = im.shape[:2] # HWC\n shape0.append(s) # image shape\n g = (size / max(s)) # gain\n shape1.append([y * g for y in s])\n imgs[i] = im if im.data.contiguous else np.ascontiguousarray(im) # update\n shape1 = [make_divisible(x, self.stride) if self.pt else size for x in np.array(shape1).max(0)] # inf shape\n x = [letterbox(im, shape1, auto=False)[0] for im in imgs] # pad\n x = np.ascontiguousarray(np.array(x).transpose((0, 3, 1, 2))) # stack and BHWC to BCHW\n x = torch.from_numpy(x).to(p.device).type_as(p) / 255 # uint8 to fp16/32\n t.append(time_sync())\n\n with amp.autocast(autocast):\n # Inference\n y = self.model(x, augment, profile) # forward\n t.append(time_sync())\n\n # Post-process\n y = non_max_suppression(y if self.dmb else y[0],\n self.conf,\n self.iou,\n self.classes,\n self.agnostic,\n self.multi_label,\n max_det=self.max_det) # NMS\n for i in range(n):\n scale_coords(shape1, y[i][:, :4], shape0[i])\n\n t.append(time_sync())\n return Detections(imgs, y, files, t, self.names, x.shape)\n\n\nclass Detections:\n # YOLOv5 detections class for inference results\n def __init__(self, imgs, pred, files, times=(0, 0, 0, 0), names=None, shape=None):\n super().__init__()\n d = pred[0].device # device\n gn = [torch.tensor([*(im.shape[i] for i in [1, 0, 1, 0]), 1, 1], device=d) for im in imgs] # normalizations\n self.imgs = imgs # list of images as numpy arrays\n self.pred = pred # list of tensors pred[0] = (xyxy, conf, cls)\n self.names = names # class names\n self.files = files # image filenames\n self.times = times # profiling times\n self.xyxy = pred # xyxy pixels\n self.xywh = [xyxy2xywh(x) for x in pred] # xywh pixels\n self.xyxyn = [x / g for x, g in zip(self.xyxy, gn)] # xyxy normalized\n self.xywhn = [x / g for x, g in zip(self.xywh, gn)] # xywh normalized\n self.n = len(self.pred) # number of images (batch size)\n self.t = tuple((times[i + 1] - times[i]) * 1000 / self.n for i in range(3)) # timestamps (ms)\n self.s = shape # inference BCHW shape\n\n def display(self, pprint=False, show=False, save=False, crop=False, render=False, labels=True, save_dir=Path('')):\n crops = []\n for i, (im, pred) in enumerate(zip(self.imgs, self.pred)):\n s = f'image {i + 1}/{len(self.pred)}: {im.shape[0]}x{im.shape[1]} ' # string\n if pred.shape[0]:\n for c in pred[:, -1].unique():\n n = (pred[:, -1] == c).sum() # detections per class\n s += f\"{n} {self.names[int(c)]}{'s' * (n > 1)}, \" # add to string\n if show or save or render or crop:\n annotator = Annotator(im, example=str(self.names))\n for *box, conf, cls in reversed(pred): # xyxy, confidence, class\n label = f'{self.names[int(cls)]} {conf:.2f}'\n if crop:\n file = save_dir / 'crops' / self.names[int(cls)] / self.files[i] if save else None\n crops.append({\n 'box': box,\n 'conf': conf,\n 'cls': cls,\n 'label': label,\n 'im': save_one_box(box, im, file=file, save=save)})\n else: # all others\n annotator.box_label(box, label if labels else '', color=colors(cls))\n im = annotator.im\n else:\n s += '(no detections)'\n\n im = Image.fromarray(im.astype(np.uint8)) if isinstance(im, np.ndarray) else im # from np\n if pprint:\n print(s.rstrip(', '))\n if show:\n im.show(self.files[i]) # show\n if save:\n f = self.files[i]\n im.save(save_dir / f) # save\n if i == self.n - 1:\n LOGGER.info(f\"Saved {self.n} image{'s' * (self.n > 1)} to {colorstr('bold', save_dir)}\")\n if render:\n self.imgs[i] = np.asarray(im)\n if crop:\n if save:\n LOGGER.info(f'Saved results to {save_dir}\\n')\n return crops\n\n def print(self):\n self.display(pprint=True) # print results\n print(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {tuple(self.s)}' % self.t)\n\n def show(self, labels=True):\n self.display(show=True, labels=labels) # show results\n\n def save(self, labels=True, save_dir='runs/detect/exp'):\n save_dir = increment_path(save_dir, exist_ok=save_dir != 'runs/detect/exp', mkdir=True) # increment save_dir\n self.display(save=True, labels=labels, save_dir=save_dir) # save results\n\n def crop(self, save=True, save_dir='runs/detect/exp'):\n save_dir = increment_path(save_dir, exist_ok=save_dir != 'runs/detect/exp', mkdir=True) if save else None\n return self.display(crop=True, save=save, save_dir=save_dir) # crop results\n\n def render(self, labels=True):\n self.display(render=True, labels=labels) # render results\n return self.imgs\n\n def pandas(self):\n # return detections as pandas DataFrames, i.e. print(results.pandas().xyxy[0])\n new = copy(self) # return copy\n ca = 'xmin', 'ymin', 'xmax', 'ymax', 'confidence', 'class', 'name' # xyxy columns\n cb = 'xcenter', 'ycenter', 'width', 'height', 'confidence', 'class', 'name' # xywh columns\n for k, c in zip(['xyxy', 'xyxyn', 'xywh', 'xywhn'], [ca, ca, cb, cb]):\n a = [[x[:5] + [int(x[5]), self.names[int(x[5])]] for x in x.tolist()] for x in getattr(self, k)] # update\n setattr(new, k, [pd.DataFrame(x, columns=c) for x in a])\n return new\n\n def tolist(self):\n # return a list of Detections objects, i.e. 'for result in results.tolist():'\n r = range(self.n) # iterable\n x = [Detections([self.imgs[i]], [self.pred[i]], [self.files[i]], self.times, self.names, self.s) for i in r]\n # for d in x:\n # for k in ['imgs', 'pred', 'xyxy', 'xyxyn', 'xywh', 'xywhn']:\n # setattr(d, k, getattr(d, k)[0]) # pop out of list\n return x\n\n def __len__(self):\n return self.n # override len(results)\n\n def __str__(self):\n self.print() # override print(results)\n return ''\n\n\nclass Classify(nn.Module):\n # Classification head, i.e. x(b,c1,20,20) to x(b,c2)\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1): # ch_in, ch_out, kernel, stride, padding, groups\n super().__init__()\n self.aap = nn.AdaptiveAvgPool2d(1) # to x(b,c1,1,1)\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g) # to x(b,c2,1,1)\n self.flat = nn.Flatten()\n\n def forward(self, x):\n z = torch.cat([self.aap(y) for y in (x if isinstance(x, list) else [x])], 1) # cat if list\n return self.flat(self.conv(z)) # flatten to x(b,c2)\n" }, { "path": "asone/detectors/yolov5/yolov5/models/experimental.py", "content": "# YOLOv5 🚀 by Ultralytics, GPL-3.0 license\n\"\"\"\nExperimental modules\n\"\"\"\nimport math\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom asone.detectors.yolov5.yolov5.utils.yolov5_utils import yolov5_in_syspath\n\nclass Ensemble(nn.ModuleList):\n # Ensemble of models\n def __init__(self):\n super().__init__()\n\n def forward(self, x, augment=False, profile=False, visualize=False):\n y = [module(x, augment, profile, visualize)[0] for module in self]\n # y = torch.stack(y).max(0)[0] # max ensemble\n # y = torch.stack(y).mean(0) # mean ensemble\n y = torch.cat(y, 1) # nms ensemble\n return y, None # inference, train output\n\n\ndef attempt_load(weights, device=None, inplace=True, fuse=True):\n # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a\n with yolov5_in_syspath():\n from asone.detectors.yolov5.yolov5.models.yolo import Detect, Model\n\n model = Ensemble()\n for w in weights if isinstance(weights, list) else [weights]:\n with yolov5_in_syspath():\n ckpt = torch.load(w, map_location='cpu') # load\n ckpt = (ckpt.get('ema') or ckpt['model']).to(device).float() # FP32 model\n model.append(ckpt.fuse().eval() if fuse else ckpt.eval()) # fused or un-fused model in eval mode\n\n # Compatibility updates\n for m in model.modules():\n t = type(m)\n if t in (nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU, Detect, Model):\n m.inplace = inplace # torch 1.7.0 compatibility\n if t is Detect and not isinstance(m.anchor_grid, list):\n delattr(m, 'anchor_grid')\n setattr(m, 'anchor_grid', [torch.zeros(1)] * m.nl)\n elif t is nn.Upsample and not hasattr(m, 'recompute_scale_factor'):\n m.recompute_scale_factor = None # torch 1.11.0 compatibility\n\n if len(model) == 1:\n return model[-1] # return model\n print(f'Ensemble created with {weights}\\n')\n for k in 'names', 'nc', 'yaml':\n setattr(model, k, getattr(model[0], k))\n model.stride = model[torch.argmax(torch.tensor([m.stride.max() for m in model])).int()].stride # max stride\n assert all(model[0].nc == m.nc for m in model), f'Models have different class counts: {[m.nc for m in model]}'\n return model # return ensemble\n\n" }, { "path": "asone/detectors/yolov5/yolov5/models/general.py", "content": "# YOLOv5 🚀 by Ultralytics, GPL-3.0 license\n\"\"\"\nGeneral utils\n\"\"\"\n\nimport contextlib\nimport glob\nimport inspect\nimport logging\nimport math\nimport os\nimport platform\nimport random\nimport re\nimport shutil\nimport signal\nimport threading\nimport time\nimport urllib\nfrom datetime import datetime\nfrom itertools import repeat\nfrom multiprocessing.pool import ThreadPool\nfrom pathlib import Path\nfrom subprocess import check_output\nfrom typing import Optional\nfrom zipfile import ZipFile\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport pkg_resources as pkg\nimport torch\nimport torchvision\nimport yaml\n\nFILE = Path(__file__).resolve()\nROOT = FILE.parents[1] # YOLOv5 root directory\nRANK = int(os.getenv('RANK', -1))\n\n# Settings\nDATASETS_DIR = ROOT.parent / 'datasets' # YOLOv5 datasets directory\nNUM_THREADS = min(8, max(1, os.cpu_count() - 1)) # number of YOLOv5 multiprocessing threads\nAUTOINSTALL = str(os.getenv('YOLOv5_AUTOINSTALL', True)).lower() == 'true' # global auto-install mode\nVERBOSE = str(os.getenv('YOLOv5_VERBOSE', True)).lower() == 'true' # global verbose mode\nFONT = 'Arial.ttf' # https://ultralytics.com/assets/Arial.ttf\n\ntorch.set_printoptions(linewidth=320, precision=5, profile='long')\nnp.set_printoptions(linewidth=320, formatter={'float_kind': '{:11.5g}'.format}) # format short g, %precision=5\npd.options.display.max_columns = 10\ncv2.setNumThreads(0) # prevent OpenCV from multithreading (incompatible with PyTorch DataLoader)\nos.environ['NUMEXPR_MAX_THREADS'] = str(NUM_THREADS) # NumExpr max threads\nos.environ['OMP_NUM_THREADS'] = '1' if platform.system() == 'darwin' else str(NUM_THREADS) # OpenMP (PyTorch and SciPy)\n\n\ndef is_kaggle():\n # Is environment a Kaggle Notebook?\n try:\n assert os.environ.get('PWD') == '/kaggle/working'\n assert os.environ.get('KAGGLE_URL_BASE') == 'https://www.kaggle.com'\n return True\n except AssertionError:\n return False\n\n\ndef is_writeable(dir, test=False):\n # Return True if directory has write permissions, test opening a file with write permissions if test=True\n if not test:\n return os.access(dir, os.R_OK) # possible issues on Windows\n file = Path(dir) / 'tmp.txt'\n try:\n with open(file, 'w'): # open file with write permissions\n pass\n file.unlink() # remove file\n return True\n except OSError:\n return False\n\n\ndef set_logging(name=None, verbose=VERBOSE):\n # Sets level and returns logger\n if is_kaggle():\n for h in logging.root.handlers:\n logging.root.removeHandler(h) # remove all handlers associated with the root logger object\n rank = int(os.getenv('RANK', -1)) # rank in world for Multi-GPU trainings\n level = logging.INFO if verbose and rank in {-1, 0} else logging.ERROR\n log = logging.getLogger(name)\n log.setLevel(level)\n handler = logging.StreamHandler()\n handler.setFormatter(logging.Formatter(\"%(message)s\"))\n handler.setLevel(level)\n log.addHandler(handler)\n\n\nset_logging() # run before defining LOGGER\nLOGGER = logging.getLogger(\"yolov5\") # define globally (used in train.py, val.py, detect.py, etc.)\n\n\ndef user_config_dir(dir='Ultralytics', env_var='YOLOV5_CONFIG_DIR'):\n # Return path of user configuration directory. Prefer environment variable if exists. Make dir if required.\n env = os.getenv(env_var)\n if env:\n path = Path(env) # use environment variable\n else:\n cfg = {'Windows': 'AppData/Roaming', 'Linux': '.config', 'Darwin': 'Library/Application Support'} # 3 OS dirs\n path = Path.home() / cfg.get(platform.system(), '') # OS-specific config dir\n path = (path if is_writeable(path) else Path('/tmp')) / dir # GCP and AWS lambda fix, only /tmp is writeable\n path.mkdir(exist_ok=True) # make if required\n return path\n\n\nCONFIG_DIR = user_config_dir() # Ultralytics settings dir\n\n\nclass Profile(contextlib.ContextDecorator):\n # Usage: @Profile() decorator or 'with Profile():' context manager\n def __enter__(self):\n self.start = time.time()\n\n def __exit__(self, type, value, traceback):\n print(f'Profile results: {time.time() - self.start:.5f}s')\n\n\nclass Timeout(contextlib.ContextDecorator):\n # Usage: @Timeout(seconds) decorator or 'with Timeout(seconds):' context manager\n def __init__(self, seconds, *, timeout_msg='', suppress_timeout_errors=True):\n self.seconds = int(seconds)\n self.timeout_message = timeout_msg\n self.suppress = bool(suppress_timeout_errors)\n\n def _timeout_handler(self, signum, frame):\n raise TimeoutError(self.timeout_message)\n\n def __enter__(self):\n if platform.system() != 'Windows': # not supported on Windows\n signal.signal(signal.SIGALRM, self._timeout_handler) # Set handler for SIGALRM\n signal.alarm(self.seconds) # start countdown for SIGALRM to be raised\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n if platform.system() != 'Windows':\n signal.alarm(0) # Cancel SIGALRM if it's scheduled\n if self.suppress and exc_type is TimeoutError: # Suppress TimeoutError\n return True\n\n\nclass WorkingDirectory(contextlib.ContextDecorator):\n # Usage: @WorkingDirectory(dir) decorator or 'with WorkingDirectory(dir):' context manager\n def __init__(self, new_dir):\n self.dir = new_dir # new dir\n self.cwd = Path.cwd().resolve() # current dir\n\n def __enter__(self):\n os.chdir(self.dir)\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n os.chdir(self.cwd)\n\n\ndef try_except(func):\n # try-except function. Usage: @try_except decorator\n def handler(*args, **kwargs):\n try:\n func(*args, **kwargs)\n except Exception as e:\n print(e)\n\n return handler\n\n\ndef threaded(func):\n # Multi-threads a target function and returns thread. Usage: @threaded decorator\n def wrapper(*args, **kwargs):\n thread = threading.Thread(target=func, args=args, kwargs=kwargs, daemon=True)\n thread.start()\n return thread\n\n return wrapper\n\n\ndef methods(instance):\n # Get class/instance methods\n return [f for f in dir(instance) if callable(getattr(instance, f)) and not f.startswith(\"__\")]\n\n\ndef print_args(args: Optional[dict] = None, show_file=True, show_fcn=False):\n # Print function arguments (optional args dict)\n x = inspect.currentframe().f_back # previous frame\n file, _, fcn, _, _ = inspect.getframeinfo(x)\n if args is None: # get args automatically\n args, _, _, frm = inspect.getargvalues(x)\n args = {k: v for k, v in frm.items() if k in args}\n s = (f'{Path(file).stem}: ' if show_file else '') + (f'{fcn}: ' if show_fcn else '')\n LOGGER.info(colorstr(s) + ', '.join(f'{k}={v}' for k, v in args.items()))\n\n\ndef init_seeds(seed=0, deterministic=False):\n # Initialize random number generator (RNG) seeds https://pytorch.org/docs/stable/notes/randomness.html\n # cudnn seed 0 settings are slower and more reproducible, else faster and less reproducible\n import torch.backends.cudnn as cudnn\n\n if deterministic and check_version(torch.__version__, '1.12.0'): # https://github.com/ultralytics/yolov5/pull/8213\n torch.use_deterministic_algorithms(True)\n os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8'\n os.environ['PYTHONHASHSEED'] = str(seed)\n\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n cudnn.benchmark, cudnn.deterministic = (False, True) if seed == 0 else (True, False)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) # for Multi-GPU, exception safe\n\n\ndef intersect_dicts(da, db, exclude=()):\n # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values\n return {k: v for k, v in da.items() if k in db and not any(x in k for x in exclude) and v.shape == db[k].shape}\n\n\ndef get_latest_run(search_dir='.'):\n # Return path to most recent 'last.pt' in /runs (i.e. to --resume from)\n last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True)\n return max(last_list, key=os.path.getctime) if last_list else ''\n\n\ndef is_docker() -> bool:\n \"\"\"Check if the process runs inside a docker container.\"\"\"\n if Path(\"/.dockerenv\").exists():\n return True\n try: # check if docker is in control groups\n with open(\"/proc/self/cgroup\") as file:\n return any(\"docker\" in line for line in file)\n except OSError:\n return False\n\n\ndef is_colab():\n # Is environment a Google Colab instance?\n try:\n import google.colab\n return True\n except ImportError:\n return False\n\n\ndef is_pip():\n # Is file in a pip package?\n return 'site-packages' in Path(__file__).resolve().parts\n\n\ndef is_ascii(s=''):\n # Is string composed of all ASCII (no UTF) characters? (note str().isascii() introduced in python 3.7)\n s = str(s) # convert list, tuple, None, etc. to str\n return len(s.encode().decode('ascii', 'ignore')) == len(s)\n\n\ndef is_chinese(s='人工智能'):\n # Is string composed of any Chinese characters?\n return bool(re.search('[\\u4e00-\\u9fff]', str(s)))\n\n\ndef emojis(str=''):\n # Return platform-dependent emoji-safe version of string\n return str.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else str\n\n\ndef file_age(path=__file__):\n # Return days since last file update\n dt = (datetime.now() - datetime.fromtimestamp(Path(path).stat().st_mtime)) # delta\n return dt.days # + dt.seconds / 86400 # fractional days\n\n\ndef file_date(path=__file__):\n # Return human-readable file modification date, i.e. '2021-3-26'\n t = datetime.fromtimestamp(Path(path).stat().st_mtime)\n return f'{t.year}-{t.month}-{t.day}'\n\n\ndef file_size(path):\n # Return file/dir size (MB)\n mb = 1 << 20 # bytes to MiB (1024 ** 2)\n path = Path(path)\n if path.is_file():\n return path.stat().st_size / mb\n elif path.is_dir():\n return sum(f.stat().st_size for f in path.glob('**/*') if f.is_file()) / mb\n else:\n return 0.0\n\n\ndef check_online():\n # Check internet connectivity\n import socket\n try:\n socket.create_connection((\"1.1.1.1\", 443), 5) # check host accessibility\n return True\n except OSError:\n return False\n\n\ndef git_describe(path=ROOT): # path must be a directory\n # Return human-readable git description, i.e. v5.0-5-g3e25f1e https://git-scm.com/docs/git-describe\n try:\n assert (Path(path) / '.git').is_dir()\n return check_output(f'git -C {path} describe --tags --long --always', shell=True).decode()[:-1]\n except Exception:\n return ''\n\n\n@try_except\n@WorkingDirectory(ROOT)\ndef check_git_status(repo='ultralytics/yolov5'):\n # YOLOv5 status check, recommend 'git pull' if code is out of date\n url = f'https://github.com/{repo}'\n msg = f', for updates see {url}'\n s = colorstr('github: ') # string\n assert Path('.git').exists(), s + 'skipping check (not a git repository)' + msg\n assert check_online(), s + 'skipping check (offline)' + msg\n\n splits = re.split(pattern=r'\\s', string=check_output('git remote -v', shell=True).decode())\n matches = [repo in s for s in splits]\n if any(matches):\n remote = splits[matches.index(True) - 1]\n else:\n remote = 'ultralytics'\n check_output(f'git remote add {remote} {url}', shell=True)\n check_output(f'git fetch {remote}', shell=True, timeout=5) # git fetch\n branch = check_output('git rev-parse --abbrev-ref HEAD', shell=True).decode().strip() # checked out\n n = int(check_output(f'git rev-list {branch}..{remote}/master --count', shell=True)) # commits behind\n if n > 0:\n pull = 'git pull' if remote == 'origin' else f'git pull {remote} master'\n s += f\"⚠️ YOLOv5 is out of date by {n} commit{'s' * (n > 1)}. Use `{pull}` or `git clone {url}` to update.\"\n else:\n s += f'up to date with {url} ✅'\n LOGGER.info(emojis(s)) # emoji-safe\n\n\ndef check_python(minimum='3.7.0'):\n # Check current python version vs. required python version\n check_version(platform.python_version(), minimum, name='Python ', hard=True)\n\n\ndef check_version(current='0.0.0', minimum='0.0.0', name='version ', pinned=False, hard=False, verbose=False):\n # Check version vs. required version\n current, minimum = (pkg.parse_version(x) for x in (current, minimum))\n result = (current == minimum) if pinned else (current >= minimum) # bool\n s = f'{name}{minimum} required by YOLOv5, but {name}{current} is currently installed' # string\n if hard:\n assert result, s # assert min requirements met\n if verbose and not result:\n LOGGER.warning(s)\n return result\n\n\n@try_except\ndef check_requirements(requirements=ROOT / 'requirements.txt', exclude=(), install=True, cmds=()):\n # Check installed dependencies meet requirements (pass *.txt file or list of packages)\n prefix = colorstr('red', 'bold', 'requirements:')\n check_python() # check python version\n if isinstance(requirements, (str, Path)): # requirements.txt file\n file = Path(requirements)\n assert file.exists(), f\"{prefix} {file.resolve()} not found, check failed.\"\n with file.open() as f:\n requirements = [f'{x.name}{x.specifier}' for x in pkg.parse_requirements(f) if x.name not in exclude]\n else: # list or tuple of packages\n requirements = [x for x in requirements if x not in exclude]\n\n n = 0 # number of packages updates\n for i, r in enumerate(requirements):\n try:\n pkg.require(r)\n except Exception: # DistributionNotFound or VersionConflict if requirements not met\n s = f\"{prefix} {r} not found and is required by YOLOv5\"\n if install and AUTOINSTALL: # check environment variable\n LOGGER.info(f\"{s}, attempting auto-update...\")\n try:\n assert check_online(), f\"'pip install {r}' skipped (offline)\"\n LOGGER.info(check_output(f'pip install \"{r}\" {cmds[i] if cmds else \"\"}', shell=True).decode())\n n += 1\n except Exception as e:\n LOGGER.warning(f'{prefix} {e}')\n else:\n LOGGER.info(f'{s}. Please install and rerun your command.')\n\n if n: # if packages updated\n source = file.resolve() if 'file' in locals() else requirements\n s = f\"{prefix} {n} package{'s' * (n > 1)} updated per {source}\\n\" \\\n f\"{prefix} ⚠️ {colorstr('bold', 'Restart runtime or rerun command for updates to take effect')}\\n\"\n LOGGER.info(emojis(s))\n\n\ndef check_img_size(imgsz, s=32, floor=0):\n # Verify image size is a multiple of stride s in each dimension\n if isinstance(imgsz, int): # integer i.e. img_size=640\n new_size = max(make_divisible(imgsz, int(s)), floor)\n else: # list i.e. img_size=[640, 480]\n imgsz = list(imgsz) # convert to list if tuple\n new_size = [max(make_divisible(x, int(s)), floor) for x in imgsz]\n if new_size != imgsz:\n LOGGER.warning(f'WARNING: --img-size {imgsz} must be multiple of max stride {s}, updating to {new_size}')\n return new_size\n\n\ndef check_imshow():\n # Check if environment supports image displays\n try:\n assert not is_docker(), 'cv2.imshow() is disabled in Docker environments'\n assert not is_colab(), 'cv2.imshow() is disabled in Google Colab environments'\n cv2.imshow('test', np.zeros((1, 1, 3)))\n cv2.waitKey(1)\n cv2.destroyAllWindows()\n cv2.waitKey(1)\n return True\n except Exception as e:\n LOGGER.warning(f'WARNING: Environment does not support cv2.imshow() or PIL Image.show() image displays\\n{e}')\n return False\n\n\ndef check_suffix(file='yolov5s.pt', suffix=('.pt',), msg=''):\n # Check file(s) for acceptable suffix\n if file and suffix:\n if isinstance(suffix, str):\n suffix = [suffix]\n for f in file if isinstance(file, (list, tuple)) else [file]:\n s = Path(f).suffix.lower() # file suffix\n if len(s):\n assert s in suffix, f\"{msg}{f} acceptable suffix is {suffix}\"\n\n\ndef check_yaml(file, suffix=('.yaml', '.yml')):\n # Search/download YAML file (if necessary) and return path, checking suffix\n return check_file(file, suffix)\n\n\ndef check_file(file, suffix=''):\n # Search/download file (if necessary) and return path\n check_suffix(file, suffix) # optional\n file = str(file) # convert to str()\n if Path(file).is_file() or not file: # exists\n return file\n elif file.startswith(('http:/', 'https:/')): # download\n url = file # warning: Pathlib turns :// -> :/\n file = Path(urllib.parse.unquote(file).split('?')[0]).name # '%2F' to '/', split https://url.com/file.txt?auth\n if Path(file).is_file():\n LOGGER.info(f'Found {url} locally at {file}') # file already exists\n else:\n LOGGER.info(f'Downloading {url} to {file}...')\n torch.hub.download_url_to_file(url, file)\n assert Path(file).exists() and Path(file).stat().st_size > 0, f'File download failed: {url}' # check\n return file\n else: # search\n files = []\n for d in 'data', 'models', 'utils': # search directories\n files.extend(glob.glob(str(ROOT / d / '**' / file), recursive=True)) # find file\n assert len(files), f'File not found: {file}' # assert file was found\n assert len(files) == 1, f\"Multiple files match '{file}', specify exact path: {files}\" # assert unique\n return files[0] # return file\n\n\ndef check_font(font=FONT, progress=False):\n # Download font to CONFIG_DIR if necessary\n font = Path(font)\n file = CONFIG_DIR / font.name\n if not font.exists() and not file.exists():\n url = \"https://ultralytics.com/assets/\" + font.name\n LOGGER.info(f'Downloading {url} to {file}...')\n torch.hub.download_url_to_file(url, str(file), progress=progress)\n\n\ndef check_dataset(data, autodownload=True):\n # Download, check and/or unzip dataset if not found locally\n\n # Download (optional)\n extract_dir = ''\n if isinstance(data, (str, Path)) and str(data).endswith('.zip'): # i.e. gs://bucket/dir/coco128.zip\n download(data, dir=DATASETS_DIR, unzip=True, delete=False, curl=False, threads=1)\n data = next((DATASETS_DIR / Path(data).stem).rglob('*.yaml'))\n extract_dir, autodownload = data.parent, False\n\n # Read yaml (optional)\n if isinstance(data, (str, Path)):\n with open(data, errors='ignore') as f:\n data = yaml.safe_load(f) # dictionary\n\n # Checks\n for k in 'train', 'val', 'nc':\n assert k in data, emojis(f\"data.yaml '{k}:' field missing ❌\")\n if 'names' not in data:\n LOGGER.warning(emojis(\"data.yaml 'names:' field missing ⚠️, assigning default names 'class0', 'class1', etc.\"))\n data['names'] = [f'class{i}' for i in range(data['nc'])] # default names\n\n # Resolve paths\n path = Path(extract_dir or data.get('path') or '') # optional 'path' default to '.'\n if not path.is_absolute():\n path = (ROOT / path).resolve()\n for k in 'train', 'val', 'test':\n if data.get(k): # prepend path\n data[k] = str(path / data[k]) if isinstance(data[k], str) else [str(path / x) for x in data[k]]\n\n # Parse yaml\n train, val, test, s = (data.get(x) for x in ('train', 'val', 'test', 'download'))\n if val:\n val = [Path(x).resolve() for x in (val if isinstance(val, list) else [val])] # val path\n if not all(x.exists() for x in val):\n LOGGER.info(emojis('\\nDataset not found ⚠️, missing paths %s' % [str(x) for x in val if not x.exists()]))\n if not s or not autodownload:\n raise Exception(emojis('Dataset not found ❌'))\n t = time.time()\n root = path.parent if 'path' in data else '..' # unzip directory i.e. '../'\n if s.startswith('http') and s.endswith('.zip'): # URL\n f = Path(s).name # filename\n LOGGER.info(f'Downloading {s} to {f}...')\n torch.hub.download_url_to_file(s, f)\n Path(root).mkdir(parents=True, exist_ok=True) # create root\n ZipFile(f).extractall(path=root) # unzip\n Path(f).unlink() # remove zip\n r = None # success\n elif s.startswith('bash '): # bash script\n LOGGER.info(f'Running {s} ...')\n r = os.system(s)\n else: # python script\n r = exec(s, {'yaml': data}) # return None\n dt = f'({round(time.time() - t, 1)}s)'\n s = f\"success ✅ {dt}, saved to {colorstr('bold', root)}\" if r in (0, None) else f\"failure {dt} ❌\"\n LOGGER.info(emojis(f\"Dataset download {s}\"))\n check_font('Arial.ttf' if is_ascii(data['names']) else 'Arial.Unicode.ttf', progress=True) # download fonts\n return data # dictionary\n\n\ndef check_amp(model):\n # Check PyTorch Automatic Mixed Precision (AMP) functionality. Return True on correct operation\n from asone.detectors.yolov5.yolov5.models.common import AutoShape, DetectMultiBackend\n\n def amp_allclose(model, im):\n # All close FP32 vs AMP results\n m = AutoShape(model, verbose=False) # model\n a = m(im).xywhn[0] # FP32 inference\n m.amp = True\n b = m(im).xywhn[0] # AMP inference\n return a.shape == b.shape and torch.allclose(a, b, atol=0.1) # close to 10% absolute tolerance\n\n prefix = colorstr('AMP: ')\n device = next(model.parameters()).device # get model device\n if device.type == 'cpu':\n return False # AMP disabled on CPU\n f = ROOT / 'data' / 'images' / 'bus.jpg' # image to check\n im = f if f.exists() else 'https://ultralytics.com/images/bus.jpg' if check_online() else np.ones((640, 640, 3))\n try:\n assert amp_allclose(model, im) or amp_allclose(DetectMultiBackend('yolov5n.pt', device), im)\n LOGGER.info(emojis(f'{prefix}checks passed ✅'))\n return True\n except Exception:\n help_url = 'https://github.com/ultralytics/yolov5/issues/7908'\n LOGGER.warning(emojis(f'{prefix}checks failed ❌, disabling Automatic Mixed Precision. See {help_url}'))\n return False\n\n\ndef url2file(url):\n # Convert URL to filename, i.e. https://url.com/file.txt?auth -> file.txt\n url = str(Path(url)).replace(':/', '://') # Pathlib turns :// -> :/\n return Path(urllib.parse.unquote(url)).name.split('?')[0] # '%2F' to '/', split https://url.com/file.txt?auth\n\n\ndef download(url, dir='.', unzip=True, delete=True, curl=False, threads=1, retry=3):\n # Multi-threaded file download and unzip function, used in data.yaml for autodownload\n def download_one(url, dir):\n # Download 1 file\n success = True\n f = dir / Path(url).name # filename\n if Path(url).is_file(): # exists in current path\n Path(url).rename(f) # move to dir\n elif not f.exists():\n LOGGER.info(f'Downloading {url} to {f}...')\n for i in range(retry + 1):\n if curl:\n s = 'sS' if threads > 1 else '' # silent\n r = os.system(f'curl -{s}L \"{url}\" -o \"{f}\" --retry 9 -C -') # curl download with retry, continue\n success = r == 0\n else:\n torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download\n success = f.is_file()\n if success:\n break\n elif i < retry:\n LOGGER.warning(f'Download failure, retrying {i + 1}/{retry} {url}...')\n else:\n LOGGER.warning(f'Failed to download {url}...')\n\n if unzip and success and f.suffix in ('.zip', '.gz'):\n LOGGER.info(f'Unzipping {f}...')\n if f.suffix == '.zip':\n ZipFile(f).extractall(path=dir) # unzip\n elif f.suffix == '.gz':\n os.system(f'tar xfz {f} --directory {f.parent}') # unzip\n if delete:\n f.unlink() # remove zip\n\n dir = Path(dir)\n dir.mkdir(parents=True, exist_ok=True) # make directory\n if threads > 1:\n pool = ThreadPool(threads)\n pool.imap(lambda x: download_one(*x), zip(url, repeat(dir))) # multi-threaded\n pool.close()\n pool.join()\n else:\n for u in [url] if isinstance(url, (str, Path)) else url:\n download_one(u, dir)\n\n\ndef make_divisible(x, divisor):\n # Returns nearest x divisible by divisor\n if isinstance(divisor, torch.Tensor):\n divisor = int(divisor.max()) # to int\n return math.ceil(x / divisor) * divisor\n\n\ndef clean_str(s):\n # Cleans a string by replacing special characters with underscore _\n return re.sub(pattern=\"[|@#!¡·$€%&()=?¿^*;:,¨´><+]\", repl=\"_\", string=s)\n\n\ndef one_cycle(y1=0.0, y2=1.0, steps=100):\n # lambda function for sinusoidal ramp from y1 to y2 https://arxiv.org/pdf/1812.01187.pdf\n return lambda x: ((1 - math.cos(x * math.pi / steps)) / 2) * (y2 - y1) + y1\n\n\ndef colorstr(*input):\n # Colors a string https://en.wikipedia.org/wiki/ANSI_escape_code, i.e. colorstr('blue', 'hello world')\n *args, string = input if len(input) > 1 else ('blue', 'bold', input[0]) # color arguments, string\n colors = {\n 'black': '\\033[30m', # basic colors\n 'red': '\\033[31m',\n 'green': '\\033[32m',\n 'yellow': '\\033[33m',\n 'blue': '\\033[34m',\n 'magenta': '\\033[35m',\n 'cyan': '\\033[36m',\n 'white': '\\033[37m',\n 'bright_black': '\\033[90m', # bright colors\n 'bright_red': '\\033[91m',\n 'bright_green': '\\033[92m',\n 'bright_yellow': '\\033[93m',\n 'bright_blue': '\\033[94m',\n 'bright_magenta': '\\033[95m',\n 'bright_cyan': '\\033[96m',\n 'bright_white': '\\033[97m',\n 'end': '\\033[0m', # misc\n 'bold': '\\033[1m',\n 'underline': '\\033[4m'}\n return ''.join(colors[x] for x in args) + f'{string}' + colors['end']\n\n\ndef labels_to_class_weights(labels, nc=80):\n # Get class weights (inverse frequency) from training labels\n if labels[0] is None: # no labels loaded\n return torch.Tensor()\n\n labels = np.concatenate(labels, 0) # labels.shape = (866643, 5) for COCO\n classes = labels[:, 0].astype(int) # labels = [class xywh]\n weights = np.bincount(classes, minlength=nc) # occurrences per class\n\n # Prepend gridpoint count (for uCE training)\n # gpi = ((320 / 32 * np.array([1, 2, 4])) ** 2 * 3).sum() # gridpoints per image\n # weights = np.hstack([gpi * len(labels) - weights.sum() * 9, weights * 9]) ** 0.5 # prepend gridpoints to start\n\n weights[weights == 0] = 1 # replace empty bins with 1\n weights = 1 / weights # number of targets per class\n weights /= weights.sum() # normalize\n return torch.from_numpy(weights).float()\n\n\ndef labels_to_image_weights(labels, nc=80, class_weights=np.ones(80)):\n # Produces image weights based on class_weights and image contents\n # Usage: index = random.choices(range(n), weights=image_weights, k=1) # weighted image sample\n class_counts = np.array([np.bincount(x[:, 0].astype(int), minlength=nc) for x in labels])\n return (class_weights.reshape(1, nc) * class_counts).sum(1)\n\n\ndef coco80_to_coco91_class(): # converts 80-index (val2014) to 91-index (paper)\n # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/\n # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\\n')\n # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\\n')\n # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco\n # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet\n return [\n 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34,\n 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,\n 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90]\n\n\ndef xyxy2xywh(x):\n # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = (x[:, 0] + x[:, 2]) / 2 # x center\n y[:, 1] = (x[:, 1] + x[:, 3]) / 2 # y center\n y[:, 2] = x[:, 2] - x[:, 0] # width\n y[:, 3] = x[:, 3] - x[:, 1] # height\n return y\n\n\ndef xywh2xyxy(x):\n # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x\n y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y\n y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x\n y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y\n return y\n\n\ndef xywhn2xyxy(x, w=640, h=640, padw=0, padh=0):\n # Convert nx4 boxes from [x, y, w, h] normalized to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = w * (x[:, 0] - x[:, 2] / 2) + padw # top left x\n y[:, 1] = h * (x[:, 1] - x[:, 3] / 2) + padh # top left y\n y[:, 2] = w * (x[:, 0] + x[:, 2] / 2) + padw # bottom right x\n y[:, 3] = h * (x[:, 1] + x[:, 3] / 2) + padh # bottom right y\n return y\n\n\ndef xyxy2xywhn(x, w=640, h=640, clip=False, eps=0.0):\n # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] normalized where xy1=top-left, xy2=bottom-right\n if clip:\n clip_coords(x, (h - eps, w - eps)) # warning: inplace clip\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = ((x[:, 0] + x[:, 2]) / 2) / w # x center\n y[:, 1] = ((x[:, 1] + x[:, 3]) / 2) / h # y center\n y[:, 2] = (x[:, 2] - x[:, 0]) / w # width\n y[:, 3] = (x[:, 3] - x[:, 1]) / h # height\n return y\n\n\ndef xyn2xy(x, w=640, h=640, padw=0, padh=0):\n # Convert normalized segments into pixel segments, shape (n,2)\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = w * x[:, 0] + padw # top left x\n y[:, 1] = h * x[:, 1] + padh # top left y\n return y\n\n\ndef segment2box(segment, width=640, height=640):\n # Convert 1 segment label to 1 box label, applying inside-image constraint, i.e. (xy1, xy2, ...) to (xyxy)\n x, y = segment.T # segment xy\n inside = (x >= 0) & (y >= 0) & (x <= width) & (y <= height)\n x, y, = x[inside], y[inside]\n return np.array([x.min(), y.min(), x.max(), y.max()]) if any(x) else np.zeros((1, 4)) # xyxy\n\n\ndef segments2boxes(segments):\n # Convert segment labels to box labels, i.e. (cls, xy1, xy2, ...) to (cls, xywh)\n boxes = []\n for s in segments:\n x, y = s.T # segment xy\n boxes.append([x.min(), y.min(), x.max(), y.max()]) # cls, xyxy\n return xyxy2xywh(np.array(boxes)) # cls, xywh\n\n\ndef resample_segments(segments, n=1000):\n # Up-sample an (n,2) segment\n for i, s in enumerate(segments):\n s = np.concatenate((s, s[0:1, :]), axis=0)\n x = np.linspace(0, len(s) - 1, n)\n xp = np.arange(len(s))\n segments[i] = np.concatenate([np.interp(x, xp, s[:, i]) for i in range(2)]).reshape(2, -1).T # segment xy\n return segments\n\n\ndef scale_coords(img1_shape, coords, img0_shape, ratio_pad=None):\n # Rescale coords (xyxy) from img1_shape to img0_shape\n if ratio_pad is None: # calculate from img0_shape\n gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new\n pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding\n else:\n gain = ratio_pad[0][0]\n pad = ratio_pad[1]\n\n coords[:, [0, 2]] -= pad[0] # x padding\n coords[:, [1, 3]] -= pad[1] # y padding\n coords[:, :4] /= gain\n clip_coords(coords, img0_shape)\n return coords\n\n\ndef clip_coords(boxes, shape):\n # Clip bounding xyxy bounding boxes to image shape (height, width)\n if isinstance(boxes, torch.Tensor): # faster individually\n boxes[:, 0].clamp_(0, shape[1]) # x1\n boxes[:, 1].clamp_(0, shape[0]) # y1\n boxes[:, 2].clamp_(0, shape[1]) # x2\n boxes[:, 3].clamp_(0, shape[0]) # y2\n else: # np.array (faster grouped)\n boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, shape[1]) # x1, x2\n boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, shape[0]) # y1, y2\n\n\ndef non_max_suppression(prediction,\n conf_thres=0.25,\n iou_thres=0.45,\n classes=None,\n agnostic=False,\n multi_label=False,\n labels=(),\n max_det=300):\n \"\"\"Non-Maximum Suppression (NMS) on inference results to reject overlapping bounding boxes\n\n Returns:\n list of detections, on (n,6) tensor per image [xyxy, conf, cls]\n \"\"\"\n\n bs = prediction.shape[0] # batch size\n nc = prediction.shape[2] - 5 # number of classes\n xc = prediction[..., 4] > conf_thres # candidates\n\n # Checks\n assert 0 <= conf_thres <= 1, f'Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0'\n assert 0 <= iou_thres <= 1, f'Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0'\n\n # Settings\n # min_wh = 2 # (pixels) minimum box width and height\n max_wh = 7680 # (pixels) maximum box width and height\n max_nms = 30000 # maximum number of boxes into torchvision.ops.nms()\n time_limit = 0.3 + 0.03 * bs # seconds to quit after\n redundant = True # require redundant detections\n multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img)\n merge = False # use merge-NMS\n\n t = time.time()\n output = [torch.zeros((0, 6), device=prediction.device)] * bs\n for xi, x in enumerate(prediction): # image index, image inference\n # Apply constraints\n # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height\n x = x[xc[xi]] # confidence\n\n # Cat apriori labels if autolabelling\n if labels and len(labels[xi]):\n lb = labels[xi]\n v = torch.zeros((len(lb), nc + 5), device=x.device)\n v[:, :4] = lb[:, 1:5] # box\n v[:, 4] = 1.0 # conf\n v[range(len(lb)), lb[:, 0].long() + 5] = 1.0 # cls\n x = torch.cat((x, v), 0)\n\n # If none remain process next image\n if not x.shape[0]:\n continue\n\n # Compute conf\n x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf\n\n # Box (center x, center y, width, height) to (x1, y1, x2, y2)\n box = xywh2xyxy(x[:, :4])\n\n # Detections matrix nx6 (xyxy, conf, cls)\n if multi_label:\n i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T\n x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1)\n else: # best class only\n conf, j = x[:, 5:].max(1, keepdim=True)\n x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres]\n\n # Filter by class\n if classes is not None:\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\n\n # Apply finite constraint\n # if not torch.isfinite(x).all():\n # x = x[torch.isfinite(x).all(1)]\n\n # Check shape\n n = x.shape[0] # number of boxes\n if not n: # no boxes\n continue\n elif n > max_nms: # excess boxes\n x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence\n\n # Batched NMS\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\n i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS\n if i.shape[0] > max_det: # limit detections\n i = i[:max_det]\n if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)\n # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\n weights = iou * scores[None] # box weights\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\n if redundant:\n i = i[iou.sum(1) > 1] # require redundancy\n\n output[xi] = x[i]\n if (time.time() - t) > time_limit:\n LOGGER.warning(f'WARNING: NMS time limit {time_limit:.3f}s exceeded')\n break # time limit exceeded\n\n return output\n\n\ndef strip_optimizer(f='best.pt', s=''): # from utils.general import *; strip_optimizer()\n # Strip optimizer from 'f' to finalize training, optionally save as 's'\n x = torch.load(f, map_location=torch.device('cpu'))\n if x.get('ema'):\n x['model'] = x['ema'] # replace model with ema\n for k in 'optimizer', 'best_fitness', 'wandb_id', 'ema', 'updates': # keys\n x[k] = None\n x['epoch'] = -1\n x['model'].half() # to FP16\n for p in x['model'].parameters():\n p.requires_grad = False\n torch.save(x, s or f)\n mb = os.path.getsize(s or f) / 1E6 # filesize\n LOGGER.info(f\"Optimizer stripped from {f},{f' saved as {s},' if s else ''} {mb:.1f}MB\")\n\n\ndef print_mutation(results, hyp, save_dir, bucket, prefix=colorstr('evolve: ')):\n evolve_csv = save_dir / 'evolve.csv'\n evolve_yaml = save_dir / 'hyp_evolve.yaml'\n keys = ('metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/mAP_0.5:0.95', 'val/box_loss',\n 'val/obj_loss', 'val/cls_loss') + tuple(hyp.keys()) # [results + hyps]\n keys = tuple(x.strip() for x in keys)\n vals = results + tuple(hyp.values())\n n = len(keys)\n\n # Download (optional)\n if bucket:\n url = f'gs://{bucket}/evolve.csv'\n if gsutil_getsize(url) > (evolve_csv.stat().st_size if evolve_csv.exists() else 0):\n os.system(f'gsutil cp {url} {save_dir}') # download evolve.csv if larger than local\n\n # Log to evolve.csv\n s = '' if evolve_csv.exists() else (('%20s,' * n % keys).rstrip(',') + '\\n') # add header\n with open(evolve_csv, 'a') as f:\n f.write(s + ('%20.5g,' * n % vals).rstrip(',') + '\\n')\n\n # Save yaml\n with open(evolve_yaml, 'w') as f:\n data = pd.read_csv(evolve_csv)\n data = data.rename(columns=lambda x: x.strip()) # strip keys\n i = np.argmax(fitness(data.values[:, :4])) #\n generations = len(data)\n f.write('# YOLOv5 Hyperparameter Evolution Results\\n' + f'# Best generation: {i}\\n' +\n f'# Last generation: {generations - 1}\\n' + '# ' + ', '.join(f'{x.strip():>20s}' for x in keys[:7]) +\n '\\n' + '# ' + ', '.join(f'{x:>20.5g}' for x in data.values[i, :7]) + '\\n\\n')\n yaml.safe_dump(data.loc[i][7:].to_dict(), f, sort_keys=False)\n\n # Print to screen\n LOGGER.info(prefix + f'{generations} generations finished, current result:\\n' + prefix +\n ', '.join(f'{x.strip():>20s}' for x in keys) + '\\n' + prefix + ', '.join(f'{x:20.5g}'\n for x in vals) + '\\n\\n')\n\n if bucket:\n os.system(f'gsutil cp {evolve_csv} {evolve_yaml} gs://{bucket}') # upload\n\n\ndef apply_classifier(x, model, img, im0):\n # Apply a second stage classifier to YOLO outputs\n # Example model = torchvision.models.__dict__['efficientnet_b0'](pretrained=True).to(device).eval()\n im0 = [im0] if isinstance(im0, np.ndarray) else im0\n for i, d in enumerate(x): # per image\n if d is not None and len(d):\n d = d.clone()\n\n # Reshape and pad cutouts\n b = xyxy2xywh(d[:, :4]) # boxes\n b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # rectangle to square\n b[:, 2:] = b[:, 2:] * 1.3 + 30 # pad\n d[:, :4] = xywh2xyxy(b).long()\n\n # Rescale boxes from img_size to im0 size\n scale_coords(img.shape[2:], d[:, :4], im0[i].shape)\n\n # Classes\n pred_cls1 = d[:, 5].long()\n ims = []\n for a in d:\n cutout = im0[i][int(a[1]):int(a[3]), int(a[0]):int(a[2])]\n im = cv2.resize(cutout, (224, 224)) # BGR\n\n im = im[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n im = np.ascontiguousarray(im, dtype=np.float32) # uint8 to float32\n im /= 255 # 0 - 255 to 0.0 - 1.0\n ims.append(im)\n\n pred_cls2 = model(torch.Tensor(ims).to(d.device)).argmax(1) # classifier prediction\n x[i] = x[i][pred_cls1 == pred_cls2] # retain matching class detections\n\n return x\n\n\ndef increment_path(path, exist_ok=False, sep='', mkdir=False):\n # Increment file or directory path, i.e. runs/exp --> runs/exp{sep}2, runs/exp{sep}3, ... etc.\n path = Path(path) # os-agnostic\n if path.exists() and not exist_ok:\n path, suffix = (path.with_suffix(''), path.suffix) if path.is_file() else (path, '')\n\n # Method 1\n for n in range(2, 9999):\n p = f'{path}{sep}{n}{suffix}' # increment path\n if not os.path.exists(p): #\n break\n path = Path(p)\n\n # Method 2 (deprecated)\n # dirs = glob.glob(f\"{path}{sep}*\") # similar paths\n # matches = [re.search(rf\"{path.stem}{sep}(\\d+)\", d) for d in dirs]\n # i = [int(m.groups()[0]) for m in matches if m] # indices\n # n = max(i) + 1 if i else 2 # increment number\n # path = Path(f\"{path}{sep}{n}{suffix}\") # increment path\n\n if mkdir:\n path.mkdir(parents=True, exist_ok=True) # make directory\n\n return path\n\n\n# OpenCV Chinese-friendly functions ------------------------------------------------------------------------------------\nimshow_ = cv2.imshow # copy to avoid recursion errors\n\n\ndef imread(path, flags=cv2.IMREAD_COLOR):\n return cv2.imdecode(np.fromfile(path, np.uint8), flags)\n\n\ndef imwrite(path, im):\n try:\n cv2.imencode(Path(path).suffix, im)[1].tofile(path)\n return True\n except Exception:\n return False\n\n\ndef imshow(path, im):\n imshow_(path.encode('unicode_escape').decode(), im)\n\n\ncv2.imread, cv2.imwrite, cv2.imshow = imread, imwrite, imshow # redefine\n\n# Variables ------------------------------------------------------------------------------------------------------------\nNCOLS = 0 if is_docker() else shutil.get_terminal_size().columns # terminal window size for tqdm\n" }, { "path": "asone/detectors/yolov5/yolov5/models/tf.py", "content": "# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license\n\"\"\"\nTensorFlow, Keras and TFLite versions of YOLOv5\nAuthored by https://github.com/zldrobit in PR https://github.com/ultralytics/yolov5/pull/1127\n\nUsage:\n $ python models/tf.py --weights yolov5s.pt\n\nExport:\n $ python export.py --weights yolov5s.pt --include saved_model pb tflite tfjs\n\"\"\"\n\nimport argparse\nimport sys\nfrom copy import deepcopy\nfrom pathlib import Path\n\nFILE = Path(__file__).resolve()\nROOT = FILE.parents[1] # YOLOv5 root directory\nif str(ROOT) not in sys.path:\n sys.path.append(str(ROOT)) # add ROOT to PATH\n# ROOT = ROOT.relative_to(Path.cwd()) # relative\n\nimport numpy as np\nimport tensorflow as tf\nimport torch\nimport torch.nn as nn\nfrom tensorflow import keras\n\nfrom asone.detectors.yolov5.yolov5.models.common import (\n C3,\n SPP,\n SPPF,\n Bottleneck,\n BottleneckCSP,\n C3x,\n Concat,\n Conv,\n CrossConv,\n DWConv,\n DWConvTranspose2d,\n Focus,\n autopad,\n)\nfrom asone.detectors.yolov5.yolov5.models.experimental import MixConv2d, attempt_load\nfrom asone.detectors.yolov5.yolov5.models.yolo import Detect, Segment\nfrom asone.detectors.yolov5.yolov5.utils.activations import SiLU\nfrom asone.detectors.yolov5.yolov5.utils.general import LOGGER, make_divisible, print_args\n\n\nclass TFBN(keras.layers.Layer):\n # TensorFlow BatchNormalization wrapper\n def __init__(self, w=None):\n \"\"\"Initializes a TensorFlow BatchNormalization layer with optional pretrained weights.\"\"\"\n super().__init__()\n self.bn = keras.layers.BatchNormalization(\n beta_initializer=keras.initializers.Constant(w.bias.numpy()),\n gamma_initializer=keras.initializers.Constant(w.weight.numpy()),\n moving_mean_initializer=keras.initializers.Constant(w.running_mean.numpy()),\n moving_variance_initializer=keras.initializers.Constant(w.running_var.numpy()),\n epsilon=w.eps,\n )\n\n def call(self, inputs):\n \"\"\"Applies batch normalization to the inputs.\"\"\"\n return self.bn(inputs)\n\n\nclass TFPad(keras.layers.Layer):\n # Pad inputs in spatial dimensions 1 and 2\n def __init__(self, pad):\n \"\"\"\n Initializes a padding layer for spatial dimensions 1 and 2 with specified padding, supporting both int and tuple\n inputs.\n\n Inputs are\n \"\"\"\n super().__init__()\n if isinstance(pad, int):\n self.pad = tf.constant([[0, 0], [pad, pad], [pad, pad], [0, 0]])\n else: # tuple/list\n self.pad = tf.constant([[0, 0], [pad[0], pad[0]], [pad[1], pad[1]], [0, 0]])\n\n def call(self, inputs):\n \"\"\"Pads input tensor with zeros using specified padding, suitable for int and tuple pad dimensions.\"\"\"\n return tf.pad(inputs, self.pad, mode=\"constant\", constant_values=0)\n\n\nclass TFConv(keras.layers.Layer):\n # Standard convolution\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None):\n \"\"\"\n Initializes a standard convolution layer with optional batch normalization and activation; supports only\n group=1.\n\n Inputs are ch_in, ch_out, weights, kernel, stride, padding, groups.\n \"\"\"\n super().__init__()\n assert g == 1, \"TF v2.2 Conv2D does not support 'groups' argument\"\n # TensorFlow convolution padding is inconsistent with PyTorch (e.g. k=3 s=2 'SAME' padding)\n # see https://stackoverflow.com/questions/52975843/comparing-conv2d-with-padding-between-tensorflow-and-pytorch\n conv = keras.layers.Conv2D(\n filters=c2,\n kernel_size=k,\n strides=s,\n padding=\"SAME\" if s == 1 else \"VALID\",\n use_bias=not hasattr(w, \"bn\"),\n kernel_initializer=keras.initializers.Constant(w.conv.weight.permute(2, 3, 1, 0).numpy()),\n bias_initializer=\"zeros\" if hasattr(w, \"bn\") else keras.initializers.Constant(w.conv.bias.numpy()),\n )\n self.conv = conv if s == 1 else keras.Sequential([TFPad(autopad(k, p)), conv])\n self.bn = TFBN(w.bn) if hasattr(w, \"bn\") else tf.identity\n self.act = activations(w.act) if act else tf.identity\n\n def call(self, inputs):\n \"\"\"Applies convolution, batch normalization, and activation function to input tensors.\"\"\"\n return self.act(self.bn(self.conv(inputs)))\n\n\nclass TFDWConv(keras.layers.Layer):\n # Depthwise convolution\n def __init__(self, c1, c2, k=1, s=1, p=None, act=True, w=None):\n \"\"\"\n Initializes a depthwise convolution layer with optional batch normalization and activation for TensorFlow\n models.\n\n Input are ch_in, ch_out, weights, kernel, stride, padding, groups.\n \"\"\"\n super().__init__()\n assert c2 % c1 == 0, f\"TFDWConv() output={c2} must be a multiple of input={c1} channels\"\n conv = keras.layers.DepthwiseConv2D(\n kernel_size=k,\n depth_multiplier=c2 // c1,\n strides=s,\n padding=\"SAME\" if s == 1 else \"VALID\",\n use_bias=not hasattr(w, \"bn\"),\n depthwise_initializer=keras.initializers.Constant(w.conv.weight.permute(2, 3, 1, 0).numpy()),\n bias_initializer=\"zeros\" if hasattr(w, \"bn\") else keras.initializers.Constant(w.conv.bias.numpy()),\n )\n self.conv = conv if s == 1 else keras.Sequential([TFPad(autopad(k, p)), conv])\n self.bn = TFBN(w.bn) if hasattr(w, \"bn\") else tf.identity\n self.act = activations(w.act) if act else tf.identity\n\n def call(self, inputs):\n \"\"\"Applies convolution, batch normalization, and activation function to input tensors.\"\"\"\n return self.act(self.bn(self.conv(inputs)))\n\n\nclass TFDWConvTranspose2d(keras.layers.Layer):\n # Depthwise ConvTranspose2d\n def __init__(self, c1, c2, k=1, s=1, p1=0, p2=0, w=None):\n \"\"\"\n Initializes depthwise ConvTranspose2D layer with specific channel, kernel, stride, and padding settings.\n\n Inputs are ch_in, ch_out, weights, kernel, stride, padding, groups.\n \"\"\"\n super().__init__()\n assert c1 == c2, f\"TFDWConv() output={c2} must be equal to input={c1} channels\"\n assert k == 4 and p1 == 1, \"TFDWConv() only valid for k=4 and p1=1\"\n weight, bias = w.weight.permute(2, 3, 1, 0).numpy(), w.bias.numpy()\n self.c1 = c1\n self.conv = [\n keras.layers.Conv2DTranspose(\n filters=1,\n kernel_size=k,\n strides=s,\n padding=\"VALID\",\n output_padding=p2,\n use_bias=True,\n kernel_initializer=keras.initializers.Constant(weight[..., i : i + 1]),\n bias_initializer=keras.initializers.Constant(bias[i]),\n )\n for i in range(c1)\n ]\n\n def call(self, inputs):\n \"\"\"Processes input through parallel convolutions and concatenates results, trimming border pixels.\"\"\"\n return tf.concat([m(x) for m, x in zip(self.conv, tf.split(inputs, self.c1, 3))], 3)[:, 1:-1, 1:-1]\n\n\nclass TFFocus(keras.layers.Layer):\n # Focus wh information into c-space\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None):\n \"\"\"\n Initializes TFFocus layer to focus width and height information into channel space with custom convolution\n parameters.\n\n Inputs are ch_in, ch_out, kernel, stride, padding, groups.\n \"\"\"\n super().__init__()\n self.conv = TFConv(c1 * 4, c2, k, s, p, g, act, w.conv)\n\n def call(self, inputs):\n \"\"\"\n Performs pixel shuffling and convolution on input tensor, downsampling by 2 and expanding channels by 4.\n\n Example x(b,w,h,c) -> y(b,w/2,h/2,4c).\n \"\"\"\n inputs = [inputs[:, ::2, ::2, :], inputs[:, 1::2, ::2, :], inputs[:, ::2, 1::2, :], inputs[:, 1::2, 1::2, :]]\n return self.conv(tf.concat(inputs, 3))\n\n\nclass TFBottleneck(keras.layers.Layer):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5, w=None):\n \"\"\"\n Initializes a standard bottleneck layer for TensorFlow models, expanding and contracting channels with optional\n shortcut.\n\n Arguments are ch_in, ch_out, shortcut, groups, expansion.\n \"\"\"\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv(c_, c2, 3, 1, g=g, w=w.cv2)\n self.add = shortcut and c1 == c2\n\n def call(self, inputs):\n \"\"\"Performs forward pass; if shortcut is True & input/output channels match, adds input to the convolution\n result.\n \"\"\"\n return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs))\n\n\nclass TFCrossConv(keras.layers.Layer):\n # Cross Convolution\n def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False, w=None):\n \"\"\"Initializes cross convolution layer with optional expansion, grouping, and shortcut addition capabilities.\"\"\"\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = TFConv(c1, c_, (1, k), (1, s), w=w.cv1)\n self.cv2 = TFConv(c_, c2, (k, 1), (s, 1), g=g, w=w.cv2)\n self.add = shortcut and c1 == c2\n\n def call(self, inputs):\n \"\"\"Passes input through two convolutions optionally adding the input if channel dimensions match.\"\"\"\n return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs))\n\n\nclass TFConv2d(keras.layers.Layer):\n # Substitution for PyTorch nn.Conv2D\n def __init__(self, c1, c2, k, s=1, g=1, bias=True, w=None):\n \"\"\"Initializes a TensorFlow 2D convolution layer, mimicking PyTorch's nn.Conv2D functionality for given filter\n sizes and stride.\n \"\"\"\n super().__init__()\n assert g == 1, \"TF v2.2 Conv2D does not support 'groups' argument\"\n self.conv = keras.layers.Conv2D(\n filters=c2,\n kernel_size=k,\n strides=s,\n padding=\"VALID\",\n use_bias=bias,\n kernel_initializer=keras.initializers.Constant(w.weight.permute(2, 3, 1, 0).numpy()),\n bias_initializer=keras.initializers.Constant(w.bias.numpy()) if bias else None,\n )\n\n def call(self, inputs):\n \"\"\"Applies a convolution operation to the inputs and returns the result.\"\"\"\n return self.conv(inputs)\n\n\nclass TFBottleneckCSP(keras.layers.Layer):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):\n \"\"\"\n Initializes CSP bottleneck layer with specified channel sizes, count, shortcut option, groups, and expansion\n ratio.\n\n Inputs are ch_in, ch_out, number, shortcut, groups, expansion.\n \"\"\"\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv2d(c1, c_, 1, 1, bias=False, w=w.cv2)\n self.cv3 = TFConv2d(c_, c_, 1, 1, bias=False, w=w.cv3)\n self.cv4 = TFConv(2 * c_, c2, 1, 1, w=w.cv4)\n self.bn = TFBN(w.bn)\n self.act = lambda x: keras.activations.swish(x)\n self.m = keras.Sequential([TFBottleneck(c_, c_, shortcut, g, e=1.0, w=w.m[j]) for j in range(n)])\n\n def call(self, inputs):\n \"\"\"Processes input through the model layers, concatenates, normalizes, activates, and reduces the output\n dimensions.\n \"\"\"\n y1 = self.cv3(self.m(self.cv1(inputs)))\n y2 = self.cv2(inputs)\n return self.cv4(self.act(self.bn(tf.concat((y1, y2), axis=3))))\n\n\nclass TFC3(keras.layers.Layer):\n # CSP Bottleneck with 3 convolutions\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):\n \"\"\"\n Initializes CSP Bottleneck with 3 convolutions, supporting optional shortcuts and group convolutions.\n\n Inputs are ch_in, ch_out, number, shortcut, groups, expansion.\n \"\"\"\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2)\n self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3)\n self.m = keras.Sequential([TFBottleneck(c_, c_, shortcut, g, e=1.0, w=w.m[j]) for j in range(n)])\n\n def call(self, inputs):\n \"\"\"\n Processes input through a sequence of transformations for object detection (YOLOv5).\n\n See https://github.com/ultralytics/yolov5.\n \"\"\"\n return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3))\n\n\nclass TFC3x(keras.layers.Layer):\n # 3 module with cross-convolutions\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):\n \"\"\"\n Initializes layer with cross-convolutions for enhanced feature extraction in object detection models.\n\n Inputs are ch_in, ch_out, number, shortcut, groups, expansion.\n \"\"\"\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2)\n self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3)\n self.m = keras.Sequential(\n [TFCrossConv(c_, c_, k=3, s=1, g=g, e=1.0, shortcut=shortcut, w=w.m[j]) for j in range(n)]\n )\n\n def call(self, inputs):\n \"\"\"Processes input through cascaded convolutions and merges features, returning the final tensor output.\"\"\"\n return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3))\n\n\nclass TFSPP(keras.layers.Layer):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, k=(5, 9, 13), w=None):\n \"\"\"Initializes a YOLOv3-SPP layer with specific input/output channels and kernel sizes for pooling.\"\"\"\n super().__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv(c_ * (len(k) + 1), c2, 1, 1, w=w.cv2)\n self.m = [keras.layers.MaxPool2D(pool_size=x, strides=1, padding=\"SAME\") for x in k]\n\n def call(self, inputs):\n \"\"\"Processes input through two TFConv layers and concatenates with max-pooled outputs at intermediate stage.\"\"\"\n x = self.cv1(inputs)\n return self.cv2(tf.concat([x] + [m(x) for m in self.m], 3))\n\n\nclass TFSPPF(keras.layers.Layer):\n # Spatial pyramid pooling-Fast layer\n def __init__(self, c1, c2, k=5, w=None):\n \"\"\"Initializes a fast spatial pyramid pooling layer with customizable in/out channels, kernel size, and\n weights.\n \"\"\"\n super().__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv(c_ * 4, c2, 1, 1, w=w.cv2)\n self.m = keras.layers.MaxPool2D(pool_size=k, strides=1, padding=\"SAME\")\n\n def call(self, inputs):\n \"\"\"Executes the model's forward pass, concatenating input features with three max-pooled versions before final\n convolution.\n \"\"\"\n x = self.cv1(inputs)\n y1 = self.m(x)\n y2 = self.m(y1)\n return self.cv2(tf.concat([x, y1, y2, self.m(y2)], 3))\n\n\nclass TFDetect(keras.layers.Layer):\n # TF YOLOv5 Detect layer\n def __init__(self, nc=80, anchors=(), ch=(), imgsz=(640, 640), w=None):\n \"\"\"Initializes YOLOv5 detection layer for TensorFlow with configurable classes, anchors, channels, and image\n size.\n \"\"\"\n super().__init__()\n self.stride = tf.convert_to_tensor(w.stride.numpy(), dtype=tf.float32)\n self.nc = nc # number of classes\n self.no = nc + 5 # number of outputs per anchor\n self.nl = len(anchors) # number of detection layers\n self.na = len(anchors[0]) // 2 # number of anchors\n self.grid = [tf.zeros(1)] * self.nl # init grid\n self.anchors = tf.convert_to_tensor(w.anchors.numpy(), dtype=tf.float32)\n self.anchor_grid = tf.reshape(self.anchors * tf.reshape(self.stride, [self.nl, 1, 1]), [self.nl, 1, -1, 1, 2])\n self.m = [TFConv2d(x, self.no * self.na, 1, w=w.m[i]) for i, x in enumerate(ch)]\n self.training = False # set to False after building model\n self.imgsz = imgsz\n for i in range(self.nl):\n ny, nx = self.imgsz[0] // self.stride[i], self.imgsz[1] // self.stride[i]\n self.grid[i] = self._make_grid(nx, ny)\n\n def call(self, inputs):\n \"\"\"Performs forward pass through the model layers to predict object bounding boxes and classifications.\"\"\"\n z = [] # inference output\n x = []\n for i in range(self.nl):\n x.append(self.m[i](inputs[i]))\n # x(bs,20,20,255) to x(bs,3,20,20,85)\n ny, nx = self.imgsz[0] // self.stride[i], self.imgsz[1] // self.stride[i]\n x[i] = tf.reshape(x[i], [-1, ny * nx, self.na, self.no])\n\n if not self.training: # inference\n y = x[i]\n grid = tf.transpose(self.grid[i], [0, 2, 1, 3]) - 0.5\n anchor_grid = tf.transpose(self.anchor_grid[i], [0, 2, 1, 3]) * 4\n xy = (tf.sigmoid(y[..., 0:2]) * 2 + grid) * self.stride[i] # xy\n wh = tf.sigmoid(y[..., 2:4]) ** 2 * anchor_grid\n # Normalize xywh to 0-1 to reduce calibration error\n xy /= tf.constant([[self.imgsz[1], self.imgsz[0]]], dtype=tf.float32)\n wh /= tf.constant([[self.imgsz[1], self.imgsz[0]]], dtype=tf.float32)\n y = tf.concat([xy, wh, tf.sigmoid(y[..., 4 : 5 + self.nc]), y[..., 5 + self.nc :]], -1)\n z.append(tf.reshape(y, [-1, self.na * ny * nx, self.no]))\n\n return tf.transpose(x, [0, 2, 1, 3]) if self.training else (tf.concat(z, 1),)\n\n @staticmethod\n def _make_grid(nx=20, ny=20):\n \"\"\"Generates a 2D grid of coordinates in (x, y) format with shape [1, 1, ny*nx, 2].\"\"\"\n # return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\n xv, yv = tf.meshgrid(tf.range(nx), tf.range(ny))\n return tf.cast(tf.reshape(tf.stack([xv, yv], 2), [1, 1, ny * nx, 2]), dtype=tf.float32)\n\n\nclass TFSegment(TFDetect):\n # YOLOv5 Segment head for segmentation models\n def __init__(self, nc=80, anchors=(), nm=32, npr=256, ch=(), imgsz=(640, 640), w=None):\n \"\"\"Initializes YOLOv5 Segment head with specified channel depths, anchors, and input size for segmentation\n models.\n \"\"\"\n super().__init__(nc, anchors, ch, imgsz, w)\n self.nm = nm # number of masks\n self.npr = npr # number of protos\n self.no = 5 + nc + self.nm # number of outputs per anchor\n self.m = [TFConv2d(x, self.no * self.na, 1, w=w.m[i]) for i, x in enumerate(ch)] # output conv\n self.proto = TFProto(ch[0], self.npr, self.nm, w=w.proto) # protos\n self.detect = TFDetect.call\n\n def call(self, x):\n \"\"\"Applies detection and proto layers on input, returning detections and optionally protos if training.\"\"\"\n p = self.proto(x[0])\n # p = TFUpsample(None, scale_factor=4, mode='nearest')(self.proto(x[0])) # (optional) full-size protos\n p = tf.transpose(p, [0, 3, 1, 2]) # from shape(1,160,160,32) to shape(1,32,160,160)\n x = self.detect(self, x)\n return (x, p) if self.training else (x[0], p)\n\n\nclass TFProto(keras.layers.Layer):\n def __init__(self, c1, c_=256, c2=32, w=None):\n \"\"\"Initializes TFProto layer with convolutional and upsampling layers for feature extraction and\n transformation.\n \"\"\"\n super().__init__()\n self.cv1 = TFConv(c1, c_, k=3, w=w.cv1)\n self.upsample = TFUpsample(None, scale_factor=2, mode=\"nearest\")\n self.cv2 = TFConv(c_, c_, k=3, w=w.cv2)\n self.cv3 = TFConv(c_, c2, w=w.cv3)\n\n def call(self, inputs):\n \"\"\"Performs forward pass through the model, applying convolutions and upscaling on input tensor.\"\"\"\n return self.cv3(self.cv2(self.upsample(self.cv1(inputs))))\n\n\nclass TFUpsample(keras.layers.Layer):\n # TF version of torch.nn.Upsample()\n def __init__(self, size, scale_factor, mode, w=None):\n \"\"\"\n Initializes a TensorFlow upsampling layer with specified size, scale_factor, and mode, ensuring scale_factor is\n even.\n\n Warning: all arguments needed including 'w'\n \"\"\"\n super().__init__()\n assert scale_factor % 2 == 0, \"scale_factor must be multiple of 2\"\n self.upsample = lambda x: tf.image.resize(x, (x.shape[1] * scale_factor, x.shape[2] * scale_factor), mode)\n # self.upsample = keras.layers.UpSampling2D(size=scale_factor, interpolation=mode)\n # with default arguments: align_corners=False, half_pixel_centers=False\n # self.upsample = lambda x: tf.raw_ops.ResizeNearestNeighbor(images=x,\n # size=(x.shape[1] * 2, x.shape[2] * 2))\n\n def call(self, inputs):\n \"\"\"Applies upsample operation to inputs using nearest neighbor interpolation.\"\"\"\n return self.upsample(inputs)\n\n\nclass TFConcat(keras.layers.Layer):\n # TF version of torch.concat()\n def __init__(self, dimension=1, w=None):\n \"\"\"Initializes a TensorFlow layer for NCHW to NHWC concatenation, requiring dimension=1.\"\"\"\n super().__init__()\n assert dimension == 1, \"convert only NCHW to NHWC concat\"\n self.d = 3\n\n def call(self, inputs):\n \"\"\"Concatenates a list of tensors along the last dimension, used for NCHW to NHWC conversion.\"\"\"\n return tf.concat(inputs, self.d)\n\n\ndef parse_model(d, ch, model, imgsz):\n \"\"\"Parses a model definition dict `d` to create YOLOv5 model layers, including dynamic channel adjustments.\"\"\"\n LOGGER.info(f\"\\n{'':>3}{'from':>18}{'n':>3}{'params':>10} {'module':<40}{'arguments':<30}\")\n anchors, nc, gd, gw, ch_mul = (\n d[\"anchors\"],\n d[\"nc\"],\n d[\"depth_multiple\"],\n d[\"width_multiple\"],\n d.get(\"channel_multiple\"),\n )\n na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors\n no = na * (nc + 5) # number of outputs = anchors * (classes + 5)\n if not ch_mul:\n ch_mul = 8\n\n layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out\n for i, (f, n, m, args) in enumerate(d[\"backbone\"] + d[\"head\"]): # from, number, module, args\n m_str = m\n m = eval(m) if isinstance(m, str) else m # eval strings\n for j, a in enumerate(args):\n try:\n args[j] = eval(a) if isinstance(a, str) else a # eval strings\n except NameError:\n pass\n\n n = max(round(n * gd), 1) if n > 1 else n # depth gain\n if m in [\n nn.Conv2d,\n Conv,\n DWConv,\n DWConvTranspose2d,\n Bottleneck,\n SPP,\n SPPF,\n MixConv2d,\n Focus,\n CrossConv,\n BottleneckCSP,\n C3,\n C3x,\n ]:\n c1, c2 = ch[f], args[0]\n c2 = make_divisible(c2 * gw, ch_mul) if c2 != no else c2\n\n args = [c1, c2, *args[1:]]\n if m in [BottleneckCSP, C3, C3x]:\n args.insert(2, n)\n n = 1\n elif m is nn.BatchNorm2d:\n args = [ch[f]]\n elif m is Concat:\n c2 = sum(ch[-1 if x == -1 else x + 1] for x in f)\n elif m in [Detect, Segment]:\n args.append([ch[x + 1] for x in f])\n if isinstance(args[1], int): # number of anchors\n args[1] = [list(range(args[1] * 2))] * len(f)\n if m is Segment:\n args[3] = make_divisible(args[3] * gw, ch_mul)\n args.append(imgsz)\n else:\n c2 = ch[f]\n\n tf_m = eval(\"TF\" + m_str.replace(\"nn.\", \"\"))\n m_ = (\n keras.Sequential([tf_m(*args, w=model.model[i][j]) for j in range(n)])\n if n > 1\n else tf_m(*args, w=model.model[i])\n ) # module\n\n torch_m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args) # module\n t = str(m)[8:-2].replace(\"__main__.\", \"\") # module type\n np = sum(x.numel() for x in torch_m_.parameters()) # number params\n m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params\n LOGGER.info(f\"{i:>3}{str(f):>18}{str(n):>3}{np:>10} {t:<40}{str(args):<30}\") # print\n save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist\n layers.append(m_)\n ch.append(c2)\n return keras.Sequential(layers), sorted(save)\n\n\nclass TFModel:\n # TF YOLOv5 model\n def __init__(self, cfg=\"yolov5s.yaml\", ch=3, nc=None, model=None, imgsz=(640, 640)):\n \"\"\"Initializes TF YOLOv5 model with specified configuration, channels, classes, model instance, and input\n size.\n \"\"\"\n super().__init__()\n if isinstance(cfg, dict):\n self.yaml = cfg # model dict\n else: # is *.yaml\n import yaml # for torch hub\n\n self.yaml_file = Path(cfg).name\n with open(cfg) as f:\n self.yaml = yaml.load(f, Loader=yaml.FullLoader) # model dict\n\n # Define model\n if nc and nc != self.yaml[\"nc\"]:\n LOGGER.info(f\"Overriding {cfg} nc={self.yaml['nc']} with nc={nc}\")\n self.yaml[\"nc\"] = nc # override yaml value\n self.model, self.savelist = parse_model(deepcopy(self.yaml), ch=[ch], model=model, imgsz=imgsz)\n\n def predict(\n self,\n inputs,\n tf_nms=False,\n agnostic_nms=False,\n topk_per_class=100,\n topk_all=100,\n iou_thres=0.45,\n conf_thres=0.25,\n ):\n y = [] # outputs\n x = inputs\n for m in self.model.layers:\n if m.f != -1: # if not from previous layer\n x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers\n\n x = m(x) # run\n y.append(x if m.i in self.savelist else None) # save output\n\n # Add TensorFlow NMS\n if tf_nms:\n boxes = self._xywh2xyxy(x[0][..., :4])\n probs = x[0][:, :, 4:5]\n classes = x[0][:, :, 5:]\n scores = probs * classes\n if agnostic_nms:\n nms = AgnosticNMS()((boxes, classes, scores), topk_all, iou_thres, conf_thres)\n else:\n boxes = tf.expand_dims(boxes, 2)\n nms = tf.image.combined_non_max_suppression(\n boxes, scores, topk_per_class, topk_all, iou_thres, conf_thres, clip_boxes=False\n )\n return (nms,)\n return x # output [1,6300,85] = [xywh, conf, class0, class1, ...]\n # x = x[0] # [x(1,6300,85), ...] to x(6300,85)\n # xywh = x[..., :4] # x(6300,4) boxes\n # conf = x[..., 4:5] # x(6300,1) confidences\n # cls = tf.reshape(tf.cast(tf.argmax(x[..., 5:], axis=1), tf.float32), (-1, 1)) # x(6300,1) classes\n # return tf.concat([conf, cls, xywh], 1)\n\n @staticmethod\n def _xywh2xyxy(xywh):\n \"\"\"Converts bounding box format from [x, y, w, h] to [x1, y1, x2, y2], where xy1=top-left and xy2=bottom-\n right.\n \"\"\"\n x, y, w, h = tf.split(xywh, num_or_size_splits=4, axis=-1)\n return tf.concat([x - w / 2, y - h / 2, x + w / 2, y + h / 2], axis=-1)\n\n\nclass AgnosticNMS(keras.layers.Layer):\n # TF Agnostic NMS\n def call(self, input, topk_all, iou_thres, conf_thres):\n \"\"\"Performs agnostic NMS on input tensors using given thresholds and top-K selection.\"\"\"\n return tf.map_fn(\n lambda x: self._nms(x, topk_all, iou_thres, conf_thres),\n input,\n fn_output_signature=(tf.float32, tf.float32, tf.float32, tf.int32),\n name=\"agnostic_nms\",\n )\n\n @staticmethod\n def _nms(x, topk_all=100, iou_thres=0.45, conf_thres=0.25):\n \"\"\"Performs agnostic non-maximum suppression (NMS) on detected objects, filtering based on IoU and confidence\n thresholds.\n \"\"\"\n boxes, classes, scores = x\n class_inds = tf.cast(tf.argmax(classes, axis=-1), tf.float32)\n scores_inp = tf.reduce_max(scores, -1)\n selected_inds = tf.image.non_max_suppression(\n boxes, scores_inp, max_output_size=topk_all, iou_threshold=iou_thres, score_threshold=conf_thres\n )\n selected_boxes = tf.gather(boxes, selected_inds)\n padded_boxes = tf.pad(\n selected_boxes,\n paddings=[[0, topk_all - tf.shape(selected_boxes)[0]], [0, 0]],\n mode=\"CONSTANT\",\n constant_values=0.0,\n )\n selected_scores = tf.gather(scores_inp, selected_inds)\n padded_scores = tf.pad(\n selected_scores,\n paddings=[[0, topk_all - tf.shape(selected_boxes)[0]]],\n mode=\"CONSTANT\",\n constant_values=-1.0,\n )\n selected_classes = tf.gather(class_inds, selected_inds)\n padded_classes = tf.pad(\n selected_classes,\n paddings=[[0, topk_all - tf.shape(selected_boxes)[0]]],\n mode=\"CONSTANT\",\n constant_values=-1.0,\n )\n valid_detections = tf.shape(selected_inds)[0]\n return padded_boxes, padded_scores, padded_classes, valid_detections\n\n\ndef activations(act=nn.SiLU):\n \"\"\"Converts PyTorch activations to TensorFlow equivalents, supporting LeakyReLU, Hardswish, and SiLU/Swish.\"\"\"\n if isinstance(act, nn.LeakyReLU):\n return lambda x: keras.activations.relu(x, alpha=0.1)\n elif isinstance(act, nn.Hardswish):\n return lambda x: x * tf.nn.relu6(x + 3) * 0.166666667\n elif isinstance(act, (nn.SiLU, SiLU)):\n return lambda x: keras.activations.swish(x)\n else:\n raise Exception(f\"no matching TensorFlow activation found for PyTorch activation {act}\")\n\n\ndef representative_dataset_gen(dataset, ncalib=100):\n \"\"\"Generates a representative dataset for calibration by yielding transformed numpy arrays from the input\n dataset.\n \"\"\"\n for n, (path, img, im0s, vid_cap, string) in enumerate(dataset):\n im = np.transpose(img, [1, 2, 0])\n im = np.expand_dims(im, axis=0).astype(np.float32)\n im /= 255\n yield [im]\n if n >= ncalib:\n break\n\n\ndef run(\n weights=ROOT / \"yolov5s.pt\", # weights path\n imgsz=(640, 640), # inference size h,w\n batch_size=1, # batch size\n dynamic=False, # dynamic batch size\n):\n # PyTorch model\n im = torch.zeros((batch_size, 3, *imgsz)) # BCHW image\n model = attempt_load(weights, device=torch.device(\"cpu\"), inplace=True, fuse=False)\n _ = model(im) # inference\n model.info()\n\n # TensorFlow model\n im = tf.zeros((batch_size, *imgsz, 3)) # BHWC image\n tf_model = TFModel(cfg=model.yaml, model=model, nc=model.nc, imgsz=imgsz)\n _ = tf_model.predict(im) # inference\n\n # Keras model\n im = keras.Input(shape=(*imgsz, 3), batch_size=None if dynamic else batch_size)\n keras_model = keras.Model(inputs=im, outputs=tf_model.predict(im))\n keras_model.summary()\n\n LOGGER.info(\"PyTorch, TensorFlow and Keras models successfully verified.\\nUse export.py for TF model export.\")\n\n\ndef parse_opt():\n \"\"\"Parses and returns command-line options for model inference, including weights path, image size, batch size, and\n dynamic batching.\n \"\"\"\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--weights\", type=str, default=ROOT / \"yolov5s.pt\", help=\"weights path\")\n parser.add_argument(\"--imgsz\", \"--img\", \"--img-size\", nargs=\"+\", type=int, default=[640], help=\"inference size h,w\")\n parser.add_argument(\"--batch-size\", type=int, default=1, help=\"batch size\")\n parser.add_argument(\"--dynamic\", action=\"store_true\", help=\"dynamic batch size\")\n opt = parser.parse_args()\n opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand\n print_args(vars(opt))\n return opt\n\n\ndef main(opt):\n \"\"\"Executes the YOLOv5 model run function with parsed command line options.\"\"\"\n run(**vars(opt))\n\n\nif __name__ == \"__main__\":\n opt = parse_opt()\n main(opt)" }, { "path": "asone/detectors/yolov5/yolov5/models/yolo.py", "content": "# YOLOv5 🚀 by Ultralytics, GPL-3.0 license\n\"\"\"\nYOLO-specific modules\n\nUsage:\n $ python path/to/models/yolo.py --cfg yolov5s.yaml\n\"\"\"\n\nimport argparse\nimport contextlib\nimport os\nimport platform\nimport sys\nfrom copy import deepcopy\nfrom pathlib import Path\n\nFILE = Path(__file__).resolve()\nROOT = FILE.parents[1] # YOLOv5 root directory\n# if str(ROOT) not in sys.path:\n# sys.path.append(str(ROOT)) # add ROOT to PATH\nif platform.system() != 'Windows':\n ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative\n\nfrom asone.detectors.yolov5.yolov5.models.common import *\nfrom asone.detectors.yolov5.yolov5.models.experimental import *\nfrom asone.detectors.yolov5.yolov5.models.general import (LOGGER, check_version,\n check_yaml, make_divisible,\n print_args)\nfrom asone.detectors.yolov5.yolov5.utils.torch_utils import (\n fuse_conv_and_bn,\n initialize_weights,\n model_info,\n profile,\n scale_img,\n select_device,\n time_sync)\n\ntry:\n import thop # for FLOPs computation\nexcept ImportError:\n thop = None\n\n\nclass Detect(nn.Module):\n stride = None # strides computed during build\n onnx_dynamic = False # ONNX export parameter\n export = False # export mode\n\n def __init__(self, nc=80, anchors=(), ch=(), inplace=True): # detection layer\n super().__init__()\n self.nc = nc # number of classes\n self.no = nc + 5 # number of outputs per anchor\n self.nl = len(anchors) # number of detection layers\n self.na = len(anchors[0]) // 2 # number of anchors\n self.grid = [torch.zeros(1)] * self.nl # init grid\n self.anchor_grid = [torch.zeros(1)] * self.nl # init anchor grid\n self.register_buffer('anchors', torch.tensor(anchors).float().view(self.nl, -1, 2)) # shape(nl,na,2)\n self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv\n self.inplace = inplace # use inplace ops (e.g. slice assignment)\n\n def forward(self, x):\n z = [] # inference output\n for i in range(self.nl):\n x[i] = self.m[i](x[i]) # conv\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()\n\n if not self.training: # inference\n if self.onnx_dynamic or self.grid[i].shape[2:4] != x[i].shape[2:4]:\n self.grid[i], self.anchor_grid[i] = self._make_grid(nx, ny, i)\n\n y = x[i].sigmoid()\n if self.inplace:\n y[..., 0:2] = (y[..., 0:2] * 2 + self.grid[i]) * self.stride[i] # xy\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\n else: # for YOLOv5 on AWS Inferentia https://github.com/ultralytics/yolov5/pull/2953\n xy, wh, conf = y.split((2, 2, self.nc + 1), 4) # y.tensor_split((2, 4, 5), 4) # torch 1.8.0\n xy = (xy * 2 + self.grid[i]) * self.stride[i] # xy\n wh = (wh * 2) ** 2 * self.anchor_grid[i] # wh\n y = torch.cat((xy, wh, conf), 4)\n z.append(y.view(bs, -1, self.no))\n\n return x if self.training else (torch.cat(z, 1),) if self.export else (torch.cat(z, 1), x)\n\n def _make_grid(self, nx=20, ny=20, i=0):\n d = self.anchors[i].device\n t = self.anchors[i].dtype\n shape = 1, self.na, ny, nx, 2 # grid shape\n y, x = torch.arange(ny, device=d, dtype=t), torch.arange(nx, device=d, dtype=t)\n if check_version(torch.__version__, '1.10.0'): # torch>=1.10.0 meshgrid workaround for torch>=0.7 compatibility\n yv, xv = torch.meshgrid(y, x, indexing='ij')\n else:\n yv, xv = torch.meshgrid(y, x)\n grid = torch.stack((xv, yv), 2).expand(shape) - 0.5 # add grid offset, i.e. y = 2.0 * x - 0.5\n anchor_grid = (self.anchors[i] * self.stride[i]).view((1, self.na, 1, 1, 2)).expand(shape)\n return grid, anchor_grid\n\n\nclass DetectionModel(nn.Module):\n # YOLOv5 model\n def __init__(self, cfg='yolov5s.yaml', ch=3, nc=None, anchors=None): # model, input channels, number of classes\n super().__init__()\n if isinstance(cfg, dict):\n self.yaml = cfg # model dict\n else: # is *.yaml\n import yaml # for torch hub\n self.yaml_file = Path(cfg).name\n with open(cfg, encoding='ascii', errors='ignore') as f:\n self.yaml = yaml.safe_load(f) # model dict\n\n # Define model\n ch = self.yaml['ch'] = self.yaml.get('ch', ch) # input channels\n if nc and nc != self.yaml['nc']:\n LOGGER.info(f\"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}\")\n self.yaml['nc'] = nc # override yaml value\n if anchors:\n LOGGER.info(f'Overriding model.yaml anchors with anchors={anchors}')\n self.yaml['anchors'] = round(anchors) # override yaml value\n self.model, self.save = parse_model(deepcopy(self.yaml), ch=[ch]) # model, savelist\n self.names = [str(i) for i in range(self.yaml['nc'])] # default names\n self.inplace = self.yaml.get('inplace', True)\n\n # Build strides, anchors\n m = self.model[-1] # Detect()\n if isinstance(m, Detect):\n s = 256 # 2x min stride\n m.inplace = self.inplace\n m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward\n check_anchor_order(m) # must be in pixel-space (not grid-space)\n m.anchors /= m.stride.view(-1, 1, 1)\n self.stride = m.stride\n self._initialize_biases() # only run once\n\n # Init weights, biases\n initialize_weights(self)\n self.info()\n LOGGER.info('')\n\n def forward(self, x, augment=False, profile=False, visualize=False):\n if augment:\n return self._forward_augment(x) # augmented inference, None\n return self._forward_once(x, profile, visualize) # single-scale inference, train\n\n def _forward_augment(self, x):\n img_size = x.shape[-2:] # height, width\n s = [1, 0.83, 0.67] # scales\n f = [None, 3, None] # flips (2-ud, 3-lr)\n y = [] # outputs\n for si, fi in zip(s, f):\n xi = scale_img(x.flip(fi) if fi else x, si, gs=int(self.stride.max()))\n yi = self._forward_once(xi)[0] # forward\n # cv2.imwrite(f'img_{si}.jpg', 255 * xi[0].cpu().numpy().transpose((1, 2, 0))[:, :, ::-1]) # save\n yi = self._descale_pred(yi, fi, si, img_size)\n y.append(yi)\n y = self._clip_augmented(y) # clip augmented tails\n return torch.cat(y, 1), None # augmented inference, train\n\n def _forward_once(self, x, profile=False, visualize=False):\n y, dt = [], [] # outputs\n for m in self.model:\n if m.f != -1: # if not from previous layer\n x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers\n if profile:\n self._profile_one_layer(m, x, dt)\n x = m(x) # run\n y.append(x if m.i in self.save else None) # save output\n if visualize:\n feature_visualization(x, m.type, m.i, save_dir=visualize)\n return x\n\n def _descale_pred(self, p, flips, scale, img_size):\n # de-scale predictions following augmented inference (inverse operation)\n if self.inplace:\n p[..., :4] /= scale # de-scale\n if flips == 2:\n p[..., 1] = img_size[0] - p[..., 1] # de-flip ud\n elif flips == 3:\n p[..., 0] = img_size[1] - p[..., 0] # de-flip lr\n else:\n x, y, wh = p[..., 0:1] / scale, p[..., 1:2] / scale, p[..., 2:4] / scale # de-scale\n if flips == 2:\n y = img_size[0] - y # de-flip ud\n elif flips == 3:\n x = img_size[1] - x # de-flip lr\n p = torch.cat((x, y, wh, p[..., 4:]), -1)\n return p\n\n def _clip_augmented(self, y):\n # Clip YOLOv5 augmented inference tails\n nl = self.model[-1].nl # number of detection layers (P3-P5)\n g = sum(4 ** x for x in range(nl)) # grid points\n e = 1 # exclude layer count\n i = (y[0].shape[1] // g) * sum(4 ** x for x in range(e)) # indices\n y[0] = y[0][:, :-i] # large\n i = (y[-1].shape[1] // g) * sum(4 ** (nl - 1 - x) for x in range(e)) # indices\n y[-1] = y[-1][:, i:] # small\n return y\n\n def _profile_one_layer(self, m, x, dt):\n c = isinstance(m, Detect) # is final layer, copy input as inplace fix\n o = thop.profile(m, inputs=(x.copy() if c else x,), verbose=False)[0] / 1E9 * 2 if thop else 0 # FLOPs\n t = time_sync()\n for _ in range(10):\n m(x.copy() if c else x)\n dt.append((time_sync() - t) * 100)\n if m == self.model[0]:\n LOGGER.info(f\"{'time (ms)':>10s} {'GFLOPs':>10s} {'params':>10s} module\")\n LOGGER.info(f'{dt[-1]:10.2f} {o:10.2f} {m.np:10.0f} {m.type}')\n if c:\n LOGGER.info(f\"{sum(dt):10.2f} {'-':>10s} {'-':>10s} Total\")\n\n def _initialize_biases(self, cf=None): # initialize biases into Detect(), cf is class frequency\n # https://arxiv.org/abs/1708.02002 section 3.3\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\n m = self.model[-1] # Detect() module\n for mi, s in zip(m.m, m.stride): # from\n b = mi.bias.view(m.na, -1).detach() # conv.bias(255) to (3,85)\n b[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image)\n b[:, 5:] += math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # cls\n mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n\n def _print_biases(self):\n m = self.model[-1] # Detect() module\n for mi in m.m: # from\n b = mi.bias.detach().view(m.na, -1).T # conv.bias(255) to (3,85)\n LOGGER.info(\n ('%6g Conv2d.bias:' + '%10.3g' * 6) % (mi.weight.shape[1], *b[:5].mean(1).tolist(), b[5:].mean()))\n\n def _print_weights(self):\n for m in self.model.modules():\n if type(m) is Bottleneck:\n LOGGER.info('%10.3g' % (m.w.detach().sigmoid() * 2)) # shortcut weights\n\n def fuse(self): # fuse model Conv2d() + BatchNorm2d() layers\n # LOGGER.info('Fusing layers... ')\n for m in self.model.modules():\n if isinstance(m, (Conv, DWConv)) and hasattr(m, 'bn'):\n m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv\n delattr(m, 'bn') # remove batchnorm\n m.forward = m.forward_fuse # update forward\n # self.info()\n return self\n\n def info(self, verbose=False, img_size=640): # print model information\n model_info(self, verbose, img_size)\n\n def _apply(self, fn):\n # Apply to(), cpu(), cuda(), half() to model tensors that are not parameters or registered buffers\n self = super()._apply(fn)\n m = self.model[-1] # Detect()\n if isinstance(m, Detect):\n m.stride = fn(m.stride)\n m.grid = list(map(fn, m.grid))\n if isinstance(m.anchor_grid, list):\n m.anchor_grid = list(map(fn, m.anchor_grid))\n return self\n\nModel = DetectionModel # retain YOLOv5 'Model' class for backwards compatibility\n\n\ndef parse_model(d, ch): # model_dict, input_channels(3)\n LOGGER.info(f\"\\n{'':>3}{'from':>18}{'n':>3}{'params':>10} {'module':<40}{'arguments':<30}\")\n anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple']\n na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors\n no = na * (nc + 5) # number of outputs = anchors * (classes + 5)\n\n layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out\n for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args\n m = eval(m) if isinstance(m, str) else m # eval strings\n for j, a in enumerate(args):\n with contextlib.suppress(NameError):\n args[j] = eval(a) if isinstance(a, str) else a # eval strings\n\n n = n_ = max(round(n * gd), 1) if n > 1 else n # depth gain\n if m in (Conv, GhostConv, Bottleneck, GhostBottleneck, SPP, SPPF, DWConv, MixConv2d, Focus, CrossConv,\n BottleneckCSP, C3, C3TR, C3SPP, C3Ghost, nn.ConvTranspose2d, DWConvTranspose2d, C3x):\n c1, c2 = ch[f], args[0]\n if c2 != no: # if not output\n c2 = make_divisible(c2 * gw, 8)\n\n args = [c1, c2, *args[1:]]\n if m in [BottleneckCSP, C3, C3TR, C3Ghost, C3x]:\n args.insert(2, n) # number of repeats\n n = 1\n elif m is nn.BatchNorm2d:\n args = [ch[f]]\n elif m is Concat:\n c2 = sum(ch[x] for x in f)\n elif m is Detect:\n args.append([ch[x] for x in f])\n if isinstance(args[1], int): # number of anchors\n args[1] = [list(range(args[1] * 2))] * len(f)\n elif m is Contract:\n c2 = ch[f] * args[0] ** 2\n elif m is Expand:\n c2 = ch[f] // args[0] ** 2\n else:\n c2 = ch[f]\n\n m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args) # module\n t = str(m)[8:-2].replace('__main__.', '') # module type\n np = sum(x.numel() for x in m_.parameters()) # number params\n m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params\n LOGGER.info(f'{i:>3}{str(f):>18}{n_:>3}{np:10.0f} {t:<40}{str(args):<30}') # print\n save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist\n layers.append(m_)\n if i == 0:\n ch = []\n ch.append(c2)\n return nn.Sequential(*layers), sorted(save)\n\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument('--cfg', type=str, default='yolov5s.yaml', help='model.yaml')\n parser.add_argument('--batch-size', type=int, default=1, help='total batch size for all GPUs')\n parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')\n parser.add_argument('--profile', action='store_true', help='profile model speed')\n parser.add_argument('--line-profile', action='store_true', help='profile model speed layer by layer')\n parser.add_argument('--test', action='store_true', help='test all yolo*.yaml')\n opt = parser.parse_args()\n opt.cfg = check_yaml(opt.cfg) # check YAML\n print_args(vars(opt))\n device = select_device(opt.device)\n\n # Create model\n im = torch.rand(opt.batch_size, 3, 640, 640).to(device)\n model = Model(opt.cfg).to(device)\n\n # Options\n if opt.line_profile: # profile layer by layer\n _ = model(im, profile=True)\n\n elif opt.profile: # profile forward-backward\n results = profile(input=im, ops=[model], n=3)\n\n elif opt.test: # test all models\n for cfg in Path(ROOT / 'models').rglob('yolo*.yaml'):\n try:\n _ = Model(cfg)\n except Exception as e:\n print(f'Error in {cfg}: {e}')\n\n else: # report fused model summary\n model.fuse()\n" }, { "path": "asone/detectors/yolov5/yolov5/utils/__init__.py", "content": "" }, { "path": "asone/detectors/yolov5/yolov5/utils/activations.py", "content": "# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license\n\"\"\"Activation functions.\"\"\"\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass SiLU(nn.Module):\n @staticmethod\n def forward(x):\n \"\"\"\n Applies the Sigmoid-weighted Linear Unit (SiLU) activation function.\n\n https://arxiv.org/pdf/1606.08415.pdf.\n \"\"\"\n return x * torch.sigmoid(x)\n\n\nclass Hardswish(nn.Module):\n @staticmethod\n def forward(x):\n \"\"\"\n Applies the Hardswish activation function, compatible with TorchScript, CoreML, and ONNX.\n\n Equivalent to x * F.hardsigmoid(x)\n \"\"\"\n return x * F.hardtanh(x + 3, 0.0, 6.0) / 6.0 # for TorchScript, CoreML and ONNX\n\n\nclass Mish(nn.Module):\n \"\"\"Mish activation https://github.com/digantamisra98/Mish.\"\"\"\n\n @staticmethod\n def forward(x):\n \"\"\"Applies the Mish activation function, a smooth alternative to ReLU.\"\"\"\n return x * F.softplus(x).tanh()\n\n\nclass MemoryEfficientMish(nn.Module):\n class F(torch.autograd.Function):\n @staticmethod\n def forward(ctx, x):\n \"\"\"Applies the Mish activation function, a smooth ReLU alternative, to the input tensor `x`.\"\"\"\n ctx.save_for_backward(x)\n return x.mul(torch.tanh(F.softplus(x))) # x * tanh(ln(1 + exp(x)))\n\n @staticmethod\n def backward(ctx, grad_output):\n \"\"\"Computes the gradient of the Mish activation function with respect to input `x`.\"\"\"\n x = ctx.saved_tensors[0]\n sx = torch.sigmoid(x)\n fx = F.softplus(x).tanh()\n return grad_output * (fx + x * sx * (1 - fx * fx))\n\n def forward(self, x):\n \"\"\"Applies the Mish activation function to the input tensor `x`.\"\"\"\n return self.F.apply(x)\n\n\nclass FReLU(nn.Module):\n \"\"\"FReLU activation https://arxiv.org/abs/2007.11824.\"\"\"\n\n def __init__(self, c1, k=3): # ch_in, kernel\n \"\"\"Initializes FReLU activation with channel `c1` and kernel size `k`.\"\"\"\n super().__init__()\n self.conv = nn.Conv2d(c1, c1, k, 1, 1, groups=c1, bias=False)\n self.bn = nn.BatchNorm2d(c1)\n\n def forward(self, x):\n \"\"\"\n Applies FReLU activation with max operation between input and BN-convolved input.\n\n https://arxiv.org/abs/2007.11824\n \"\"\"\n return torch.max(x, self.bn(self.conv(x)))\n\n\nclass AconC(nn.Module):\n \"\"\"\n ACON activation (activate or not) function.\n\n AconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is a learnable parameter\n See \"Activate or Not: Learning Customized Activation\" https://arxiv.org/pdf/2009.04759.pdf.\n \"\"\"\n\n def __init__(self, c1):\n \"\"\"Initializes AconC with learnable parameters p1, p2, and beta for channel-wise activation control.\"\"\"\n super().__init__()\n self.p1 = nn.Parameter(torch.randn(1, c1, 1, 1))\n self.p2 = nn.Parameter(torch.randn(1, c1, 1, 1))\n self.beta = nn.Parameter(torch.ones(1, c1, 1, 1))\n\n def forward(self, x):\n \"\"\"Applies AconC activation function with learnable parameters for channel-wise control on input tensor x.\"\"\"\n dpx = (self.p1 - self.p2) * x\n return dpx * torch.sigmoid(self.beta * dpx) + self.p2 * x\n\n\nclass MetaAconC(nn.Module):\n \"\"\"\n ACON activation (activate or not) function.\n\n AconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is a learnable parameter\n See \"Activate or Not: Learning Customized Activation\" https://arxiv.org/pdf/2009.04759.pdf.\n \"\"\"\n\n def __init__(self, c1, k=1, s=1, r=16):\n \"\"\"Initializes MetaAconC with params: channel_in (c1), kernel size (k=1), stride (s=1), reduction (r=16).\"\"\"\n super().__init__()\n c2 = max(r, c1 // r)\n self.p1 = nn.Parameter(torch.randn(1, c1, 1, 1))\n self.p2 = nn.Parameter(torch.randn(1, c1, 1, 1))\n self.fc1 = nn.Conv2d(c1, c2, k, s, bias=True)\n self.fc2 = nn.Conv2d(c2, c1, k, s, bias=True)\n # self.bn1 = nn.BatchNorm2d(c2)\n # self.bn2 = nn.BatchNorm2d(c1)\n\n def forward(self, x):\n \"\"\"Applies a forward pass transforming input `x` using learnable parameters and sigmoid activation.\"\"\"\n y = x.mean(dim=2, keepdims=True).mean(dim=3, keepdims=True)\n # batch-size 1 bug/instabilities https://github.com/ultralytics/yolov5/issues/2891\n # beta = torch.sigmoid(self.bn2(self.fc2(self.bn1(self.fc1(y))))) # bug/unstable\n beta = torch.sigmoid(self.fc2(self.fc1(y))) # bug patch BN layers removed\n dpx = (self.p1 - self.p2) * x\n return dpx * torch.sigmoid(beta * dpx) + self.p2 * x" }, { "path": "asone/detectors/yolov5/yolov5/utils/augmentations.py", "content": "# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license\n\"\"\"Image augmentation functions.\"\"\"\n\nimport math\nimport random\n\nimport cv2\nimport numpy as np\nimport torch\nimport torchvision.transforms as T\nimport torchvision.transforms.functional as TF\n\nfrom asone.detectors.yolov5.yolov5.utils.general import LOGGER, check_version, colorstr, resample_segments, segment2box, xywhn2xyxy\nfrom asone.detectors.yolov5.yolov5.utils.metrics import bbox_ioa\n\nIMAGENET_MEAN = 0.485, 0.456, 0.406 # RGB mean\nIMAGENET_STD = 0.229, 0.224, 0.225 # RGB standard deviation\n\n\nclass Albumentations:\n # YOLOv5 Albumentations class (optional, only used if package is installed)\n def __init__(self, size=640):\n \"\"\"Initializes Albumentations class for optional data augmentation in YOLOv5 with specified input size.\"\"\"\n self.transform = None\n prefix = colorstr(\"albumentations: \")\n try:\n import albumentations as A\n\n check_version(A.__version__, \"1.0.3\", hard=True) # version requirement\n\n T = [\n A.RandomResizedCrop(height=size, width=size, scale=(0.8, 1.0), ratio=(0.9, 1.11), p=0.0),\n A.Blur(p=0.01),\n A.MedianBlur(p=0.01),\n A.ToGray(p=0.01),\n A.CLAHE(p=0.01),\n A.RandomBrightnessContrast(p=0.0),\n A.RandomGamma(p=0.0),\n A.ImageCompression(quality_lower=75, p=0.0),\n ] # transforms\n self.transform = A.Compose(T, bbox_params=A.BboxParams(format=\"yolo\", label_fields=[\"class_labels\"]))\n\n LOGGER.info(prefix + \", \".join(f\"{x}\".replace(\"always_apply=False, \", \"\") for x in T if x.p))\n except ImportError: # package not installed, skip\n pass\n except Exception as e:\n LOGGER.info(f\"{prefix}{e}\")\n\n def __call__(self, im, labels, p=1.0):\n \"\"\"Applies transformations to an image and labels with probability `p`, returning updated image and labels.\"\"\"\n if self.transform and random.random() < p:\n new = self.transform(image=im, bboxes=labels[:, 1:], class_labels=labels[:, 0]) # transformed\n im, labels = new[\"image\"], np.array([[c, *b] for c, b in zip(new[\"class_labels\"], new[\"bboxes\"])])\n return im, labels\n\n\ndef normalize(x, mean=IMAGENET_MEAN, std=IMAGENET_STD, inplace=False):\n \"\"\"\n Applies ImageNet normalization to RGB images in BCHW format, modifying them in-place if specified.\n\n Example: y = (x - mean) / std\n \"\"\"\n return TF.normalize(x, mean, std, inplace=inplace)\n\n\ndef denormalize(x, mean=IMAGENET_MEAN, std=IMAGENET_STD):\n \"\"\"Reverses ImageNet normalization for BCHW format RGB images by applying `x = x * std + mean`.\"\"\"\n for i in range(3):\n x[:, i] = x[:, i] * std[i] + mean[i]\n return x\n\n\ndef augment_hsv(im, hgain=0.5, sgain=0.5, vgain=0.5):\n \"\"\"Applies HSV color-space augmentation to an image with random gains for hue, saturation, and value.\"\"\"\n if hgain or sgain or vgain:\n r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains\n hue, sat, val = cv2.split(cv2.cvtColor(im, cv2.COLOR_BGR2HSV))\n dtype = im.dtype # uint8\n\n x = np.arange(0, 256, dtype=r.dtype)\n lut_hue = ((x * r[0]) % 180).astype(dtype)\n lut_sat = np.clip(x * r[1], 0, 255).astype(dtype)\n lut_val = np.clip(x * r[2], 0, 255).astype(dtype)\n\n im_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val)))\n cv2.cvtColor(im_hsv, cv2.COLOR_HSV2BGR, dst=im) # no return needed\n\n\ndef hist_equalize(im, clahe=True, bgr=False):\n \"\"\"Equalizes image histogram, with optional CLAHE, for BGR or RGB image with shape (n,m,3) and range 0-255.\"\"\"\n yuv = cv2.cvtColor(im, cv2.COLOR_BGR2YUV if bgr else cv2.COLOR_RGB2YUV)\n if clahe:\n c = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n yuv[:, :, 0] = c.apply(yuv[:, :, 0])\n else:\n yuv[:, :, 0] = cv2.equalizeHist(yuv[:, :, 0]) # equalize Y channel histogram\n return cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR if bgr else cv2.COLOR_YUV2RGB) # convert YUV image to RGB\n\n\ndef replicate(im, labels):\n \"\"\"\n Replicates half of the smallest object labels in an image for data augmentation.\n\n Returns augmented image and labels.\n \"\"\"\n h, w = im.shape[:2]\n boxes = labels[:, 1:].astype(int)\n x1, y1, x2, y2 = boxes.T\n s = ((x2 - x1) + (y2 - y1)) / 2 # side length (pixels)\n for i in s.argsort()[: round(s.size * 0.5)]: # smallest indices\n x1b, y1b, x2b, y2b = boxes[i]\n bh, bw = y2b - y1b, x2b - x1b\n yc, xc = int(random.uniform(0, h - bh)), int(random.uniform(0, w - bw)) # offset x, y\n x1a, y1a, x2a, y2a = [xc, yc, xc + bw, yc + bh]\n im[y1a:y2a, x1a:x2a] = im[y1b:y2b, x1b:x2b] # im4[ymin:ymax, xmin:xmax]\n labels = np.append(labels, [[labels[i, 0], x1a, y1a, x2a, y2a]], axis=0)\n\n return im, labels\n\n\ndef letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32):\n \"\"\"Resizes and pads image to new_shape with stride-multiple constraints, returns resized image, ratio, padding.\"\"\"\n shape = im.shape[:2] # current shape [height, width]\n if isinstance(new_shape, int):\n new_shape = (new_shape, new_shape)\n\n # Scale ratio (new / old)\n r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])\n if not scaleup: # only scale down, do not scale up (for better val mAP)\n r = min(r, 1.0)\n\n # Compute padding\n ratio = r, r # width, height ratios\n new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))\n dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding\n if auto: # minimum rectangle\n dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding\n elif scaleFill: # stretch\n dw, dh = 0.0, 0.0\n new_unpad = (new_shape[1], new_shape[0])\n ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios\n\n dw /= 2 # divide padding into 2 sides\n dh /= 2\n\n if shape[::-1] != new_unpad: # resize\n im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)\n top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))\n left, right = int(round(dw - 0.1)), int(round(dw + 0.1))\n im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border\n return im, ratio, (dw, dh)\n\n\ndef random_perspective(\n im, targets=(), segments=(), degrees=10, translate=0.1, scale=0.1, shear=10, perspective=0.0, border=(0, 0)\n):\n # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(0.1, 0.1), scale=(0.9, 1.1), shear=(-10, 10))\n # targets = [cls, xyxy]\n\n height = im.shape[0] + border[0] * 2 # shape(h,w,c)\n width = im.shape[1] + border[1] * 2\n\n # Center\n C = np.eye(3)\n C[0, 2] = -im.shape[1] / 2 # x translation (pixels)\n C[1, 2] = -im.shape[0] / 2 # y translation (pixels)\n\n # Perspective\n P = np.eye(3)\n P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y)\n P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x)\n\n # Rotation and Scale\n R = np.eye(3)\n a = random.uniform(-degrees, degrees)\n # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations\n s = random.uniform(1 - scale, 1 + scale)\n # s = 2 ** random.uniform(-scale, scale)\n R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s)\n\n # Shear\n S = np.eye(3)\n S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg)\n S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg)\n\n # Translation\n T = np.eye(3)\n T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width # x translation (pixels)\n T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height # y translation (pixels)\n\n # Combined rotation matrix\n M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT\n if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed\n if perspective:\n im = cv2.warpPerspective(im, M, dsize=(width, height), borderValue=(114, 114, 114))\n else: # affine\n im = cv2.warpAffine(im, M[:2], dsize=(width, height), borderValue=(114, 114, 114))\n\n # Visualize\n # import matplotlib.pyplot as plt\n # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel()\n # ax[0].imshow(im[:, :, ::-1]) # base\n # ax[1].imshow(im2[:, :, ::-1]) # warped\n\n # Transform label coordinates\n n = len(targets)\n if n:\n use_segments = any(x.any() for x in segments) and len(segments) == n\n new = np.zeros((n, 4))\n if use_segments: # warp segments\n segments = resample_segments(segments) # upsample\n for i, segment in enumerate(segments):\n xy = np.ones((len(segment), 3))\n xy[:, :2] = segment\n xy = xy @ M.T # transform\n xy = xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2] # perspective rescale or affine\n\n # clip\n new[i] = segment2box(xy, width, height)\n\n else: # warp boxes\n xy = np.ones((n * 4, 3))\n xy[:, :2] = targets[:, [1, 2, 3, 4, 1, 4, 3, 2]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1\n xy = xy @ M.T # transform\n xy = (xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2]).reshape(n, 8) # perspective rescale or affine\n\n # create new boxes\n x = xy[:, [0, 2, 4, 6]]\n y = xy[:, [1, 3, 5, 7]]\n new = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T\n\n # clip\n new[:, [0, 2]] = new[:, [0, 2]].clip(0, width)\n new[:, [1, 3]] = new[:, [1, 3]].clip(0, height)\n\n # filter candidates\n i = box_candidates(box1=targets[:, 1:5].T * s, box2=new.T, area_thr=0.01 if use_segments else 0.10)\n targets = targets[i]\n targets[:, 1:5] = new[i]\n\n return im, targets\n\n\ndef copy_paste(im, labels, segments, p=0.5):\n \"\"\"\n Applies Copy-Paste augmentation by flipping and merging segments and labels on an image.\n\n Details at https://arxiv.org/abs/2012.07177.\n \"\"\"\n n = len(segments)\n if p and n:\n h, w, c = im.shape # height, width, channels\n im_new = np.zeros(im.shape, np.uint8)\n for j in random.sample(range(n), k=round(p * n)):\n l, s = labels[j], segments[j]\n box = w - l[3], l[2], w - l[1], l[4]\n ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area\n if (ioa < 0.30).all(): # allow 30% obscuration of existing labels\n labels = np.concatenate((labels, [[l[0], *box]]), 0)\n segments.append(np.concatenate((w - s[:, 0:1], s[:, 1:2]), 1))\n cv2.drawContours(im_new, [segments[j].astype(np.int32)], -1, (1, 1, 1), cv2.FILLED)\n\n result = cv2.flip(im, 1) # augment segments (flip left-right)\n i = cv2.flip(im_new, 1).astype(bool)\n im[i] = result[i] # cv2.imwrite('debug.jpg', im) # debug\n\n return im, labels, segments\n\n\ndef cutout(im, labels, p=0.5):\n \"\"\"\n Applies cutout augmentation to an image with optional label adjustment, using random masks of varying sizes.\n\n Details at https://arxiv.org/abs/1708.04552.\n \"\"\"\n if random.random() < p:\n h, w = im.shape[:2]\n scales = [0.5] * 1 + [0.25] * 2 + [0.125] * 4 + [0.0625] * 8 + [0.03125] * 16 # image size fraction\n for s in scales:\n mask_h = random.randint(1, int(h * s)) # create random masks\n mask_w = random.randint(1, int(w * s))\n\n # box\n xmin = max(0, random.randint(0, w) - mask_w // 2)\n ymin = max(0, random.randint(0, h) - mask_h // 2)\n xmax = min(w, xmin + mask_w)\n ymax = min(h, ymin + mask_h)\n\n # apply random color mask\n im[ymin:ymax, xmin:xmax] = [random.randint(64, 191) for _ in range(3)]\n\n # return unobscured labels\n if len(labels) and s > 0.03:\n box = np.array([xmin, ymin, xmax, ymax], dtype=np.float32)\n ioa = bbox_ioa(box, xywhn2xyxy(labels[:, 1:5], w, h)) # intersection over area\n labels = labels[ioa < 0.60] # remove >60% obscured labels\n\n return labels\n\n\ndef mixup(im, labels, im2, labels2):\n \"\"\"\n Applies MixUp augmentation by blending images and labels.\n\n See https://arxiv.org/pdf/1710.09412.pdf for details.\n \"\"\"\n r = np.random.beta(32.0, 32.0) # mixup ratio, alpha=beta=32.0\n im = (im * r + im2 * (1 - r)).astype(np.uint8)\n labels = np.concatenate((labels, labels2), 0)\n return im, labels\n\n\ndef box_candidates(box1, box2, wh_thr=2, ar_thr=100, area_thr=0.1, eps=1e-16):\n \"\"\"\n Filters bounding box candidates by minimum width-height threshold `wh_thr` (pixels), aspect ratio threshold\n `ar_thr`, and area ratio threshold `area_thr`.\n\n box1(4,n) is before augmentation, box2(4,n) is after augmentation.\n \"\"\"\n w1, h1 = box1[2] - box1[0], box1[3] - box1[1]\n w2, h2 = box2[2] - box2[0], box2[3] - box2[1]\n ar = np.maximum(w2 / (h2 + eps), h2 / (w2 + eps)) # aspect ratio\n return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + eps) > area_thr) & (ar < ar_thr) # candidates\n\n\ndef classify_albumentations(\n augment=True,\n size=224,\n scale=(0.08, 1.0),\n ratio=(0.75, 1.0 / 0.75), # 0.75, 1.33\n hflip=0.5,\n vflip=0.0,\n jitter=0.4,\n mean=IMAGENET_MEAN,\n std=IMAGENET_STD,\n auto_aug=False,\n):\n # YOLOv5 classification Albumentations (optional, only used if package is installed)\n prefix = colorstr(\"albumentations: \")\n try:\n import albumentations as A\n from albumentations.pytorch import ToTensorV2\n\n check_version(A.__version__, \"1.0.3\", hard=True) # version requirement\n if augment: # Resize and crop\n T = [A.RandomResizedCrop(height=size, width=size, scale=scale, ratio=ratio)]\n if auto_aug:\n # TODO: implement AugMix, AutoAug & RandAug in albumentation\n LOGGER.info(f\"{prefix}auto augmentations are currently not supported\")\n else:\n if hflip > 0:\n T += [A.HorizontalFlip(p=hflip)]\n if vflip > 0:\n T += [A.VerticalFlip(p=vflip)]\n if jitter > 0:\n color_jitter = (float(jitter),) * 3 # repeat value for brightness, contrast, satuaration, 0 hue\n T += [A.ColorJitter(*color_jitter, 0)]\n else: # Use fixed crop for eval set (reproducibility)\n T = [A.SmallestMaxSize(max_size=size), A.CenterCrop(height=size, width=size)]\n T += [A.Normalize(mean=mean, std=std), ToTensorV2()] # Normalize and convert to Tensor\n LOGGER.info(prefix + \", \".join(f\"{x}\".replace(\"always_apply=False, \", \"\") for x in T if x.p))\n return A.Compose(T)\n\n except ImportError: # package not installed, skip\n LOGGER.warning(f\"{prefix}⚠️ not found, install with `pip install albumentations` (recommended)\")\n except Exception as e:\n LOGGER.info(f\"{prefix}{e}\")\n\n\ndef classify_transforms(size=224):\n \"\"\"Applies a series of transformations including center crop, ToTensor, and normalization for classification.\"\"\"\n assert isinstance(size, int), f\"ERROR: classify_transforms size {size} must be integer, not (list, tuple)\"\n # T.Compose([T.ToTensor(), T.Resize(size), T.CenterCrop(size), T.Normalize(IMAGENET_MEAN, IMAGENET_STD)])\n return T.Compose([CenterCrop(size), ToTensor(), T.Normalize(IMAGENET_MEAN, IMAGENET_STD)])\n\n\nclass LetterBox:\n # YOLOv5 LetterBox class for image preprocessing, i.e. T.Compose([LetterBox(size), ToTensor()])\n def __init__(self, size=(640, 640), auto=False, stride=32):\n \"\"\"Initializes a LetterBox object for YOLOv5 image preprocessing with optional auto sizing and stride\n adjustment.\n \"\"\"\n super().__init__()\n self.h, self.w = (size, size) if isinstance(size, int) else size\n self.auto = auto # pass max size integer, automatically solve for short side using stride\n self.stride = stride # used with auto\n\n def __call__(self, im):\n \"\"\"\n Resizes and pads input image `im` (HWC format) to specified dimensions, maintaining aspect ratio.\n\n im = np.array HWC\n \"\"\"\n imh, imw = im.shape[:2]\n r = min(self.h / imh, self.w / imw) # ratio of new/old\n h, w = round(imh * r), round(imw * r) # resized image\n hs, ws = (math.ceil(x / self.stride) * self.stride for x in (h, w)) if self.auto else self.h, self.w\n top, left = round((hs - h) / 2 - 0.1), round((ws - w) / 2 - 0.1)\n im_out = np.full((self.h, self.w, 3), 114, dtype=im.dtype)\n im_out[top : top + h, left : left + w] = cv2.resize(im, (w, h), interpolation=cv2.INTER_LINEAR)\n return im_out\n\n\nclass CenterCrop:\n # YOLOv5 CenterCrop class for image preprocessing, i.e. T.Compose([CenterCrop(size), ToTensor()])\n def __init__(self, size=640):\n \"\"\"Initializes CenterCrop for image preprocessing, accepting single int or tuple for size, defaults to 640.\"\"\"\n super().__init__()\n self.h, self.w = (size, size) if isinstance(size, int) else size\n\n def __call__(self, im):\n \"\"\"\n Applies center crop to the input image and resizes it to a specified size, maintaining aspect ratio.\n\n im = np.array HWC\n \"\"\"\n imh, imw = im.shape[:2]\n m = min(imh, imw) # min dimension\n top, left = (imh - m) // 2, (imw - m) // 2\n return cv2.resize(im[top : top + m, left : left + m], (self.w, self.h), interpolation=cv2.INTER_LINEAR)\n\n\nclass ToTensor:\n # YOLOv5 ToTensor class for image preprocessing, i.e. T.Compose([LetterBox(size), ToTensor()])\n def __init__(self, half=False):\n \"\"\"Initializes ToTensor for YOLOv5 image preprocessing, with optional half precision (half=True for FP16).\"\"\"\n super().__init__()\n self.half = half\n\n def __call__(self, im):\n \"\"\"\n Converts BGR np.array image from HWC to RGB CHW format, and normalizes to [0, 1], with support for FP16 if\n `half=True`.\n\n im = np.array HWC in BGR order\n \"\"\"\n im = np.ascontiguousarray(im.transpose((2, 0, 1))[::-1]) # HWC to CHW -> BGR to RGB -> contiguous\n im = torch.from_numpy(im) # to torch\n im = im.half() if self.half else im.float() # uint8 to fp16/32\n im /= 255.0 # 0-255 to 0.0-1.0\n return im" }, { "path": "asone/detectors/yolov5/yolov5/utils/dataloaders.py", "content": "# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license\n\"\"\"Dataloaders and dataset utils.\"\"\"\n\nimport contextlib\nimport glob\nimport hashlib\nimport json\nimport math\nimport os\nimport random\nimport shutil\nimport time\nfrom itertools import repeat\nfrom multiprocessing.pool import Pool, ThreadPool\nfrom pathlib import Path\nfrom threading import Thread\nfrom urllib.parse import urlparse\n\nimport numpy as np\nimport psutil\nimport torch\nimport torch.nn.functional as F\nimport torchvision\nimport yaml\nfrom PIL import ExifTags, Image, ImageOps\nfrom torch.utils.data import DataLoader, Dataset, dataloader, distributed\nfrom tqdm import tqdm\n\nfrom asone.detectors.yolov5.yolov5.utils.augmentations import (\n Albumentations,\n augment_hsv,\n classify_albumentations,\n classify_transforms,\n copy_paste,\n letterbox,\n mixup,\n random_perspective,\n)\nfrom asone.detectors.yolov5.yolov5.utils.general import (\n DATASETS_DIR,\n LOGGER,\n NUM_THREADS,\n TQDM_BAR_FORMAT,\n check_dataset,\n check_requirements,\n check_yaml,\n clean_str,\n cv2,\n is_colab,\n is_kaggle,\n segments2boxes,\n unzip_file,\n xyn2xy,\n xywh2xyxy,\n xywhn2xyxy,\n xyxy2xywhn,\n)\nfrom asone.detectors.yolov5.yolov5.utils.torch_utils import torch_distributed_zero_first\n\n# Parameters\nHELP_URL = \"See https://docs.ultralytics.com/yolov5/tutorials/train_custom_data\"\nIMG_FORMATS = \"bmp\", \"dng\", \"jpeg\", \"jpg\", \"mpo\", \"png\", \"tif\", \"tiff\", \"webp\", \"pfm\" # include image suffixes\nVID_FORMATS = \"asf\", \"avi\", \"gif\", \"m4v\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"ts\", \"wmv\" # include video suffixes\nLOCAL_RANK = int(os.getenv(\"LOCAL_RANK\", -1)) # https://pytorch.org/docs/stable/elastic/run.html\nRANK = int(os.getenv(\"RANK\", -1))\nWORLD_SIZE = int(os.getenv(\"WORLD_SIZE\", 1))\nPIN_MEMORY = str(os.getenv(\"PIN_MEMORY\", True)).lower() == \"true\" # global pin_memory for dataloaders\n\n# Get orientation exif tag\nfor orientation in ExifTags.TAGS.keys():\n if ExifTags.TAGS[orientation] == \"Orientation\":\n break\n\n\ndef get_hash(paths):\n \"\"\"Generates a single SHA256 hash for a list of file or directory paths by combining their sizes and paths.\"\"\"\n size = sum(os.path.getsize(p) for p in paths if os.path.exists(p)) # sizes\n h = hashlib.sha256(str(size).encode()) # hash sizes\n h.update(\"\".join(paths).encode()) # hash paths\n return h.hexdigest() # return hash\n\n\ndef exif_size(img):\n \"\"\"Returns corrected PIL image size (width, height) considering EXIF orientation.\"\"\"\n s = img.size # (width, height)\n with contextlib.suppress(Exception):\n rotation = dict(img._getexif().items())[orientation]\n if rotation in [6, 8]: # rotation 270 or 90\n s = (s[1], s[0])\n return s\n\n\ndef exif_transpose(image):\n \"\"\"\n Transpose a PIL image accordingly if it has an EXIF Orientation tag.\n Inplace version of https://github.com/python-pillow/Pillow/blob/master/src/PIL/ImageOps.py exif_transpose()\n\n :param image: The image to transpose.\n :return: An image.\n \"\"\"\n exif = image.getexif()\n orientation = exif.get(0x0112, 1) # default 1\n if orientation > 1:\n method = {\n 2: Image.FLIP_LEFT_RIGHT,\n 3: Image.ROTATE_180,\n 4: Image.FLIP_TOP_BOTTOM,\n 5: Image.TRANSPOSE,\n 6: Image.ROTATE_270,\n 7: Image.TRANSVERSE,\n 8: Image.ROTATE_90,\n }.get(orientation)\n if method is not None:\n image = image.transpose(method)\n del exif[0x0112]\n image.info[\"exif\"] = exif.tobytes()\n return image\n\n\ndef seed_worker(worker_id):\n \"\"\"\n Sets the seed for a dataloader worker to ensure reproducibility, based on PyTorch's randomness notes.\n\n See https://pytorch.org/docs/stable/notes/randomness.html#dataloader.\n \"\"\"\n worker_seed = torch.initial_seed() % 2**32\n np.random.seed(worker_seed)\n random.seed(worker_seed)\n\n\n# Inherit from DistributedSampler and override iterator\n# https://github.com/pytorch/pytorch/blob/master/torch/utils/data/distributed.py\nclass SmartDistributedSampler(distributed.DistributedSampler):\n def __iter__(self):\n \"\"\"Yields indices for distributed data sampling, shuffled deterministically based on epoch and seed.\"\"\"\n g = torch.Generator()\n g.manual_seed(self.seed + self.epoch)\n\n # determine the the eventual size (n) of self.indices (DDP indices)\n n = int((len(self.dataset) - self.rank - 1) / self.num_replicas) + 1 # num_replicas == WORLD_SIZE\n idx = torch.randperm(n, generator=g)\n if not self.shuffle:\n idx = idx.sort()[0]\n\n idx = idx.tolist()\n if self.drop_last:\n idx = idx[: self.num_samples]\n else:\n padding_size = self.num_samples - len(idx)\n if padding_size <= len(idx):\n idx += idx[:padding_size]\n else:\n idx += (idx * math.ceil(padding_size / len(idx)))[:padding_size]\n\n return iter(idx)\n\n\ndef create_dataloader(\n path,\n imgsz,\n batch_size,\n stride,\n single_cls=False,\n hyp=None,\n augment=False,\n cache=False,\n pad=0.0,\n rect=False,\n rank=-1,\n workers=8,\n image_weights=False,\n quad=False,\n prefix=\"\",\n shuffle=False,\n seed=0,\n):\n if rect and shuffle:\n LOGGER.warning(\"WARNING ⚠️ --rect is incompatible with DataLoader shuffle, setting shuffle=False\")\n shuffle = False\n with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP\n dataset = LoadImagesAndLabels(\n path,\n imgsz,\n batch_size,\n augment=augment, # augmentation\n hyp=hyp, # hyperparameters\n rect=rect, # rectangular batches\n cache_images=cache,\n single_cls=single_cls,\n stride=int(stride),\n pad=pad,\n image_weights=image_weights,\n prefix=prefix,\n rank=rank,\n )\n\n batch_size = min(batch_size, len(dataset))\n nd = torch.cuda.device_count() # number of CUDA devices\n nw = min([os.cpu_count() // max(nd, 1), batch_size if batch_size > 1 else 0, workers]) # number of workers\n sampler = None if rank == -1 else SmartDistributedSampler(dataset, shuffle=shuffle)\n loader = DataLoader if image_weights else InfiniteDataLoader # only DataLoader allows for attribute updates\n generator = torch.Generator()\n generator.manual_seed(6148914691236517205 + seed + RANK)\n return loader(\n dataset,\n batch_size=batch_size,\n shuffle=shuffle and sampler is None,\n num_workers=nw,\n sampler=sampler,\n pin_memory=PIN_MEMORY,\n collate_fn=LoadImagesAndLabels.collate_fn4 if quad else LoadImagesAndLabels.collate_fn,\n worker_init_fn=seed_worker,\n generator=generator,\n ), dataset\n\n\nclass InfiniteDataLoader(dataloader.DataLoader):\n \"\"\"\n Dataloader that reuses workers.\n\n Uses same syntax as vanilla DataLoader\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n \"\"\"Initializes an InfiniteDataLoader that reuses workers with standard DataLoader syntax, augmenting with a\n repeating sampler.\n \"\"\"\n super().__init__(*args, **kwargs)\n object.__setattr__(self, \"batch_sampler\", _RepeatSampler(self.batch_sampler))\n self.iterator = super().__iter__()\n\n def __len__(self):\n \"\"\"Returns the length of the batch sampler's sampler in the InfiniteDataLoader.\"\"\"\n return len(self.batch_sampler.sampler)\n\n def __iter__(self):\n \"\"\"Yields batches of data indefinitely in a loop by resetting the sampler when exhausted.\"\"\"\n for _ in range(len(self)):\n yield next(self.iterator)\n\n\nclass _RepeatSampler:\n \"\"\"\n Sampler that repeats forever.\n\n Args:\n sampler (Sampler)\n \"\"\"\n\n def __init__(self, sampler):\n \"\"\"Initializes a perpetual sampler wrapping a provided `Sampler` instance for endless data iteration.\"\"\"\n self.sampler = sampler\n\n def __iter__(self):\n \"\"\"Returns an infinite iterator over the dataset by repeatedly yielding from the given sampler.\"\"\"\n while True:\n yield from iter(self.sampler)\n\n\nclass LoadScreenshots:\n # YOLOv5 screenshot dataloader, i.e. `python detect.py --source \"screen 0 100 100 512 256\"`\n def __init__(self, source, img_size=640, stride=32, auto=True, transforms=None):\n \"\"\"\n Initializes a screenshot dataloader for YOLOv5 with specified source region, image size, stride, auto, and\n transforms.\n\n Source = [screen_number left top width height] (pixels)\n \"\"\"\n check_requirements(\"mss\")\n import mss\n\n source, *params = source.split()\n self.screen, left, top, width, height = 0, None, None, None, None # default to full screen 0\n if len(params) == 1:\n self.screen = int(params[0])\n elif len(params) == 4:\n left, top, width, height = (int(x) for x in params)\n elif len(params) == 5:\n self.screen, left, top, width, height = (int(x) for x in params)\n self.img_size = img_size\n self.stride = stride\n self.transforms = transforms\n self.auto = auto\n self.mode = \"stream\"\n self.frame = 0\n self.sct = mss.mss()\n\n # Parse monitor shape\n monitor = self.sct.monitors[self.screen]\n self.top = monitor[\"top\"] if top is None else (monitor[\"top\"] + top)\n self.left = monitor[\"left\"] if left is None else (monitor[\"left\"] + left)\n self.width = width or monitor[\"width\"]\n self.height = height or monitor[\"height\"]\n self.monitor = {\"left\": self.left, \"top\": self.top, \"width\": self.width, \"height\": self.height}\n\n def __iter__(self):\n \"\"\"Iterates over itself, enabling use in loops and iterable contexts.\"\"\"\n return self\n\n def __next__(self):\n \"\"\"Captures and returns the next screen frame as a BGR numpy array, cropping to only the first three channels\n from BGRA.\n \"\"\"\n im0 = np.array(self.sct.grab(self.monitor))[:, :, :3] # [:, :, :3] BGRA to BGR\n s = f\"screen {self.screen} (LTWH): {self.left},{self.top},{self.width},{self.height}: \"\n\n if self.transforms:\n im = self.transforms(im0) # transforms\n else:\n im = letterbox(im0, self.img_size, stride=self.stride, auto=self.auto)[0] # padded resize\n im = im.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n im = np.ascontiguousarray(im) # contiguous\n self.frame += 1\n return str(self.screen), im, im0, None, s # screen, img, original img, im0s, s\n\n\nclass LoadImages:\n \"\"\"YOLOv5 image/video dataloader, i.e. `python detect.py --source image.jpg/vid.mp4`\"\"\"\n\n def __init__(self, path, img_size=640, stride=32, auto=True, transforms=None, vid_stride=1):\n \"\"\"Initializes YOLOv5 loader for images/videos, supporting glob patterns, directories, and lists of paths.\"\"\"\n if isinstance(path, str) and Path(path).suffix == \".txt\": # *.txt file with img/vid/dir on each line\n path = Path(path).read_text().rsplit()\n files = []\n for p in sorted(path) if isinstance(path, (list, tuple)) else [path]:\n p = str(Path(p).resolve())\n if \"*\" in p:\n files.extend(sorted(glob.glob(p, recursive=True))) # glob\n elif os.path.isdir(p):\n files.extend(sorted(glob.glob(os.path.join(p, \"*.*\")))) # dir\n elif os.path.isfile(p):\n files.append(p) # files\n else:\n raise FileNotFoundError(f\"{p} does not exist\")\n\n images = [x for x in files if x.split(\".\")[-1].lower() in IMG_FORMATS]\n videos = [x for x in files if x.split(\".\")[-1].lower() in VID_FORMATS]\n ni, nv = len(images), len(videos)\n\n self.img_size = img_size\n self.stride = stride\n self.files = images + videos\n self.nf = ni + nv # number of files\n self.video_flag = [False] * ni + [True] * nv\n self.mode = \"image\"\n self.auto = auto\n self.transforms = transforms # optional\n self.vid_stride = vid_stride # video frame-rate stride\n if any(videos):\n self._new_video(videos[0]) # new video\n else:\n self.cap = None\n assert self.nf > 0, (\n f\"No images or videos found in {p}. \"\n f\"Supported formats are:\\nimages: {IMG_FORMATS}\\nvideos: {VID_FORMATS}\"\n )\n\n def __iter__(self):\n \"\"\"Initializes iterator by resetting count and returns the iterator object itself.\"\"\"\n self.count = 0\n return self\n\n def __next__(self):\n \"\"\"Advances to the next file in the dataset, raising StopIteration if at the end.\"\"\"\n if self.count == self.nf:\n raise StopIteration\n path = self.files[self.count]\n\n if self.video_flag[self.count]:\n # Read video\n self.mode = \"video\"\n for _ in range(self.vid_stride):\n self.cap.grab()\n ret_val, im0 = self.cap.retrieve()\n while not ret_val:\n self.count += 1\n self.cap.release()\n if self.count == self.nf: # last video\n raise StopIteration\n path = self.files[self.count]\n self._new_video(path)\n ret_val, im0 = self.cap.read()\n\n self.frame += 1\n # im0 = self._cv2_rotate(im0) # for use if cv2 autorotation is False\n s = f\"video {self.count + 1}/{self.nf} ({self.frame}/{self.frames}) {path}: \"\n\n else:\n # Read image\n self.count += 1\n im0 = cv2.imread(path) # BGR\n assert im0 is not None, f\"Image Not Found {path}\"\n s = f\"image {self.count}/{self.nf} {path}: \"\n\n if self.transforms:\n im = self.transforms(im0) # transforms\n else:\n im = letterbox(im0, self.img_size, stride=self.stride, auto=self.auto)[0] # padded resize\n im = im.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n im = np.ascontiguousarray(im) # contiguous\n\n return path, im, im0, self.cap, s\n\n def _new_video(self, path):\n \"\"\"Initializes a new video capture object with path, frame count adjusted by stride, and orientation\n metadata.\n \"\"\"\n self.frame = 0\n self.cap = cv2.VideoCapture(path)\n self.frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT) / self.vid_stride)\n self.orientation = int(self.cap.get(cv2.CAP_PROP_ORIENTATION_META)) # rotation degrees\n # self.cap.set(cv2.CAP_PROP_ORIENTATION_AUTO, 0) # disable https://github.com/ultralytics/yolov5/issues/8493\n\n def _cv2_rotate(self, im):\n \"\"\"Rotates a cv2 image based on its orientation; supports 0, 90, and 180 degrees rotations.\"\"\"\n if self.orientation == 0:\n return cv2.rotate(im, cv2.ROTATE_90_CLOCKWISE)\n elif self.orientation == 180:\n return cv2.rotate(im, cv2.ROTATE_90_COUNTERCLOCKWISE)\n elif self.orientation == 90:\n return cv2.rotate(im, cv2.ROTATE_180)\n return im\n\n def __len__(self):\n \"\"\"Returns the number of files in the dataset.\"\"\"\n return self.nf # number of files\n\n\nclass LoadStreams:\n # YOLOv5 streamloader, i.e. `python detect.py --source 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP streams`\n def __init__(self, sources=\"file.streams\", img_size=640, stride=32, auto=True, transforms=None, vid_stride=1):\n \"\"\"Initializes a stream loader for processing video streams with YOLOv5, supporting various sources including\n YouTube.\n \"\"\"\n torch.backends.cudnn.benchmark = True # faster for fixed-size inference\n self.mode = \"stream\"\n self.img_size = img_size\n self.stride = stride\n self.vid_stride = vid_stride # video frame-rate stride\n sources = Path(sources).read_text().rsplit() if os.path.isfile(sources) else [sources]\n n = len(sources)\n self.sources = [clean_str(x) for x in sources] # clean source names for later\n self.imgs, self.fps, self.frames, self.threads = [None] * n, [0] * n, [0] * n, [None] * n\n for i, s in enumerate(sources): # index, source\n # Start thread to read frames from video stream\n st = f\"{i + 1}/{n}: {s}... \"\n if urlparse(s).hostname in (\"www.youtube.com\", \"youtube.com\", \"youtu.be\"): # if source is YouTube video\n # YouTube format i.e. 'https://www.youtube.com/watch?v=Zgi9g1ksQHc' or 'https://youtu.be/LNwODJXcvt4'\n check_requirements((\"pafy\", \"youtube_dl==2020.12.2\"))\n import pafy\n\n s = pafy.new(s).getbest(preftype=\"mp4\").url # YouTube URL\n s = eval(s) if s.isnumeric() else s # i.e. s = '0' local webcam\n if s == 0:\n assert not is_colab(), \"--source 0 webcam unsupported on Colab. Rerun command in a local environment.\"\n assert not is_kaggle(), \"--source 0 webcam unsupported on Kaggle. Rerun command in a local environment.\"\n cap = cv2.VideoCapture(s)\n assert cap.isOpened(), f\"{st}Failed to open {s}\"\n w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n fps = cap.get(cv2.CAP_PROP_FPS) # warning: may return 0 or nan\n self.frames[i] = max(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)), 0) or float(\"inf\") # infinite stream fallback\n self.fps[i] = max((fps if math.isfinite(fps) else 0) % 100, 0) or 30 # 30 FPS fallback\n\n _, self.imgs[i] = cap.read() # guarantee first frame\n self.threads[i] = Thread(target=self.update, args=([i, cap, s]), daemon=True)\n LOGGER.info(f\"{st} Success ({self.frames[i]} frames {w}x{h} at {self.fps[i]:.2f} FPS)\")\n self.threads[i].start()\n LOGGER.info(\"\") # newline\n\n # check for common shapes\n s = np.stack([letterbox(x, img_size, stride=stride, auto=auto)[0].shape for x in self.imgs])\n self.rect = np.unique(s, axis=0).shape[0] == 1 # rect inference if all shapes equal\n self.auto = auto and self.rect\n self.transforms = transforms # optional\n if not self.rect:\n LOGGER.warning(\"WARNING ⚠️ Stream shapes differ. For optimal performance supply similarly-shaped streams.\")\n\n def update(self, i, cap, stream):\n \"\"\"Reads frames from stream `i`, updating imgs array; handles stream reopening on signal loss.\"\"\"\n n, f = 0, self.frames[i] # frame number, frame array\n while cap.isOpened() and n < f:\n n += 1\n cap.grab() # .read() = .grab() followed by .retrieve()\n if n % self.vid_stride == 0:\n success, im = cap.retrieve()\n if success:\n self.imgs[i] = im\n else:\n LOGGER.warning(\"WARNING ⚠️ Video stream unresponsive, please check your IP camera connection.\")\n self.imgs[i] = np.zeros_like(self.imgs[i])\n cap.open(stream) # re-open stream if signal was lost\n time.sleep(0.0) # wait time\n\n def __iter__(self):\n \"\"\"Resets and returns the iterator for iterating over video frames or images in a dataset.\"\"\"\n self.count = -1\n return self\n\n def __next__(self):\n \"\"\"Iterates over video frames or images, halting on thread stop or 'q' key press, raising `StopIteration` when\n done.\n \"\"\"\n self.count += 1\n if not all(x.is_alive() for x in self.threads) or cv2.waitKey(1) == ord(\"q\"): # q to quit\n cv2.destroyAllWindows()\n raise StopIteration\n\n im0 = self.imgs.copy()\n if self.transforms:\n im = np.stack([self.transforms(x) for x in im0]) # transforms\n else:\n im = np.stack([letterbox(x, self.img_size, stride=self.stride, auto=self.auto)[0] for x in im0]) # resize\n im = im[..., ::-1].transpose((0, 3, 1, 2)) # BGR to RGB, BHWC to BCHW\n im = np.ascontiguousarray(im) # contiguous\n\n return self.sources, im, im0, None, \"\"\n\n def __len__(self):\n \"\"\"Returns the number of sources in the dataset, supporting up to 32 streams at 30 FPS over 30 years.\"\"\"\n return len(self.sources) # 1E12 frames = 32 streams at 30 FPS for 30 years\n\n\ndef img2label_paths(img_paths):\n \"\"\"Generates label file paths from corresponding image file paths by replacing `/images/` with `/labels/` and\n extension with `.txt`.\n \"\"\"\n sa, sb = f\"{os.sep}images{os.sep}\", f\"{os.sep}labels{os.sep}\" # /images/, /labels/ substrings\n return [sb.join(x.rsplit(sa, 1)).rsplit(\".\", 1)[0] + \".txt\" for x in img_paths]\n\n\nclass LoadImagesAndLabels(Dataset):\n # YOLOv5 train_loader/val_loader, loads images and labels for training and validation\n cache_version = 0.6 # dataset labels *.cache version\n rand_interp_methods = [cv2.INTER_NEAREST, cv2.INTER_LINEAR, cv2.INTER_CUBIC, cv2.INTER_AREA, cv2.INTER_LANCZOS4]\n\n def __init__(\n self,\n path,\n img_size=640,\n batch_size=16,\n augment=False,\n hyp=None,\n rect=False,\n image_weights=False,\n cache_images=False,\n single_cls=False,\n stride=32,\n pad=0.0,\n min_items=0,\n prefix=\"\",\n rank=-1,\n seed=0,\n ):\n self.img_size = img_size\n self.augment = augment\n self.hyp = hyp\n self.image_weights = image_weights\n self.rect = False if image_weights else rect\n self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training)\n self.mosaic_border = [-img_size // 2, -img_size // 2]\n self.stride = stride\n self.path = path\n self.albumentations = Albumentations(size=img_size) if augment else None\n\n try:\n f = [] # image files\n for p in path if isinstance(path, list) else [path]:\n p = Path(p) # os-agnostic\n if p.is_dir(): # dir\n f += glob.glob(str(p / \"**\" / \"*.*\"), recursive=True)\n # f = list(p.rglob('*.*')) # pathlib\n elif p.is_file(): # file\n with open(p) as t:\n t = t.read().strip().splitlines()\n parent = str(p.parent) + os.sep\n f += [x.replace(\"./\", parent, 1) if x.startswith(\"./\") else x for x in t] # to global path\n # f += [p.parent / x.lstrip(os.sep) for x in t] # to global path (pathlib)\n else:\n raise FileNotFoundError(f\"{prefix}{p} does not exist\")\n self.im_files = sorted(x.replace(\"/\", os.sep) for x in f if x.split(\".\")[-1].lower() in IMG_FORMATS)\n # self.img_files = sorted([x for x in f if x.suffix[1:].lower() in IMG_FORMATS]) # pathlib\n assert self.im_files, f\"{prefix}No images found\"\n except Exception as e:\n raise Exception(f\"{prefix}Error loading data from {path}: {e}\\n{HELP_URL}\") from e\n\n # Check cache\n self.label_files = img2label_paths(self.im_files) # labels\n cache_path = (p if p.is_file() else Path(self.label_files[0]).parent).with_suffix(\".cache\")\n try:\n cache, exists = np.load(cache_path, allow_pickle=True).item(), True # load dict\n assert cache[\"version\"] == self.cache_version # matches current version\n assert cache[\"hash\"] == get_hash(self.label_files + self.im_files) # identical hash\n except Exception:\n cache, exists = self.cache_labels(cache_path, prefix), False # run cache ops\n\n # Display cache\n nf, nm, ne, nc, n = cache.pop(\"results\") # found, missing, empty, corrupt, total\n if exists and LOCAL_RANK in {-1, 0}:\n d = f\"Scanning {cache_path}... {nf} images, {nm + ne} backgrounds, {nc} corrupt\"\n tqdm(None, desc=prefix + d, total=n, initial=n, bar_format=TQDM_BAR_FORMAT) # display cache results\n if cache[\"msgs\"]:\n LOGGER.info(\"\\n\".join(cache[\"msgs\"])) # display warnings\n assert nf > 0 or not augment, f\"{prefix}No labels found in {cache_path}, can not start training. {HELP_URL}\"\n\n # Read cache\n [cache.pop(k) for k in (\"hash\", \"version\", \"msgs\")] # remove items\n labels, shapes, self.segments = zip(*cache.values())\n nl = len(np.concatenate(labels, 0)) # number of labels\n assert nl > 0 or not augment, f\"{prefix}All labels empty in {cache_path}, can not start training. {HELP_URL}\"\n self.labels = list(labels)\n self.shapes = np.array(shapes)\n self.im_files = list(cache.keys()) # update\n self.label_files = img2label_paths(cache.keys()) # update\n\n # Filter images\n if min_items:\n include = np.array([len(x) >= min_items for x in self.labels]).nonzero()[0].astype(int)\n LOGGER.info(f\"{prefix}{n - len(include)}/{n} images filtered from dataset\")\n self.im_files = [self.im_files[i] for i in include]\n self.label_files = [self.label_files[i] for i in include]\n self.labels = [self.labels[i] for i in include]\n self.segments = [self.segments[i] for i in include]\n self.shapes = self.shapes[include] # wh\n\n # Create indices\n n = len(self.shapes) # number of images\n bi = np.floor(np.arange(n) / batch_size).astype(int) # batch index\n nb = bi[-1] + 1 # number of batches\n self.batch = bi # batch index of image\n self.n = n\n self.indices = np.arange(n)\n if rank > -1: # DDP indices (see: SmartDistributedSampler)\n # force each rank (i.e. GPU process) to sample the same subset of data on every epoch\n self.indices = self.indices[np.random.RandomState(seed=seed).permutation(n) % WORLD_SIZE == RANK]\n\n # Update labels\n include_class = [] # filter labels to include only these classes (optional)\n self.segments = list(self.segments)\n include_class_array = np.array(include_class).reshape(1, -1)\n for i, (label, segment) in enumerate(zip(self.labels, self.segments)):\n if include_class:\n j = (label[:, 0:1] == include_class_array).any(1)\n self.labels[i] = label[j]\n if segment:\n self.segments[i] = [segment[idx] for idx, elem in enumerate(j) if elem]\n if single_cls: # single-class training, merge all classes into 0\n self.labels[i][:, 0] = 0\n\n # Rectangular Training\n if self.rect:\n # Sort by aspect ratio\n s = self.shapes # wh\n ar = s[:, 1] / s[:, 0] # aspect ratio\n irect = ar.argsort()\n self.im_files = [self.im_files[i] for i in irect]\n self.label_files = [self.label_files[i] for i in irect]\n self.labels = [self.labels[i] for i in irect]\n self.segments = [self.segments[i] for i in irect]\n self.shapes = s[irect] # wh\n ar = ar[irect]\n\n # Set training image shapes\n shapes = [[1, 1]] * nb\n for i in range(nb):\n ari = ar[bi == i]\n mini, maxi = ari.min(), ari.max()\n if maxi < 1:\n shapes[i] = [maxi, 1]\n elif mini > 1:\n shapes[i] = [1, 1 / mini]\n\n self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(int) * stride\n\n # Cache images into RAM/disk for faster training\n if cache_images == \"ram\" and not self.check_cache_ram(prefix=prefix):\n cache_images = False\n self.ims = [None] * n\n self.npy_files = [Path(f).with_suffix(\".npy\") for f in self.im_files]\n if cache_images:\n b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes\n self.im_hw0, self.im_hw = [None] * n, [None] * n\n fcn = self.cache_images_to_disk if cache_images == \"disk\" else self.load_image\n results = ThreadPool(NUM_THREADS).imap(lambda i: (i, fcn(i)), self.indices)\n pbar = tqdm(results, total=len(self.indices), bar_format=TQDM_BAR_FORMAT, disable=LOCAL_RANK > 0)\n for i, x in pbar:\n if cache_images == \"disk\":\n b += self.npy_files[i].stat().st_size\n else: # 'ram'\n self.ims[i], self.im_hw0[i], self.im_hw[i] = x # im, hw_orig, hw_resized = load_image(self, i)\n b += self.ims[i].nbytes * WORLD_SIZE\n pbar.desc = f\"{prefix}Caching images ({b / gb:.1f}GB {cache_images})\"\n pbar.close()\n\n def check_cache_ram(self, safety_margin=0.1, prefix=\"\"):\n \"\"\"Checks if available RAM is sufficient for caching images, adjusting for a safety margin.\"\"\"\n b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes\n n = min(self.n, 30) # extrapolate from 30 random images\n for _ in range(n):\n im = cv2.imread(random.choice(self.im_files)) # sample image\n ratio = self.img_size / max(im.shape[0], im.shape[1]) # max(h, w) # ratio\n b += im.nbytes * ratio**2\n mem_required = b * self.n / n # GB required to cache dataset into RAM\n mem = psutil.virtual_memory()\n cache = mem_required * (1 + safety_margin) < mem.available # to cache or not to cache, that is the question\n if not cache:\n LOGGER.info(\n f'{prefix}{mem_required / gb:.1f}GB RAM required, '\n f'{mem.available / gb:.1f}/{mem.total / gb:.1f}GB available, '\n f\"{'caching images ✅' if cache else 'not caching images ⚠️'}\"\n )\n return cache\n\n def cache_labels(self, path=Path(\"./labels.cache\"), prefix=\"\"):\n \"\"\"Caches dataset labels, verifies images, reads shapes, and tracks dataset integrity.\"\"\"\n x = {} # dict\n nm, nf, ne, nc, msgs = 0, 0, 0, 0, [] # number missing, found, empty, corrupt, messages\n desc = f\"{prefix}Scanning {path.parent / path.stem}...\"\n with Pool(NUM_THREADS) as pool:\n pbar = tqdm(\n pool.imap(verify_image_label, zip(self.im_files, self.label_files, repeat(prefix))),\n desc=desc,\n total=len(self.im_files),\n bar_format=TQDM_BAR_FORMAT,\n )\n for im_file, lb, shape, segments, nm_f, nf_f, ne_f, nc_f, msg in pbar:\n nm += nm_f\n nf += nf_f\n ne += ne_f\n nc += nc_f\n if im_file:\n x[im_file] = [lb, shape, segments]\n if msg:\n msgs.append(msg)\n pbar.desc = f\"{desc} {nf} images, {nm + ne} backgrounds, {nc} corrupt\"\n\n pbar.close()\n if msgs:\n LOGGER.info(\"\\n\".join(msgs))\n if nf == 0:\n LOGGER.warning(f\"{prefix}WARNING ⚠️ No labels found in {path}. {HELP_URL}\")\n x[\"hash\"] = get_hash(self.label_files + self.im_files)\n x[\"results\"] = nf, nm, ne, nc, len(self.im_files)\n x[\"msgs\"] = msgs # warnings\n x[\"version\"] = self.cache_version # cache version\n try:\n np.save(path, x) # save cache for next time\n path.with_suffix(\".cache.npy\").rename(path) # remove .npy suffix\n LOGGER.info(f\"{prefix}New cache created: {path}\")\n except Exception as e:\n LOGGER.warning(f\"{prefix}WARNING ⚠️ Cache directory {path.parent} is not writeable: {e}\") # not writeable\n return x\n\n def __len__(self):\n \"\"\"Returns the number of images in the dataset.\"\"\"\n return len(self.im_files)\n\n # def __iter__(self):\n # self.count = -1\n # print('ran dataset iter')\n # #self.shuffled_vector = np.random.permutation(self.nF) if self.augment else np.arange(self.nF)\n # return self\n\n def __getitem__(self, index):\n \"\"\"Fetches the dataset item at the given index, considering linear, shuffled, or weighted sampling.\"\"\"\n index = self.indices[index] # linear, shuffled, or image_weights\n\n hyp = self.hyp\n mosaic = self.mosaic and random.random() < hyp[\"mosaic\"]\n if mosaic:\n # Load mosaic\n img, labels = self.load_mosaic(index)\n shapes = None\n\n # MixUp augmentation\n if random.random() < hyp[\"mixup\"]:\n img, labels = mixup(img, labels, *self.load_mosaic(random.choice(self.indices)))\n\n else:\n # Load image\n img, (h0, w0), (h, w) = self.load_image(index)\n\n # Letterbox\n shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape\n img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment)\n shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling\n\n labels = self.labels[index].copy()\n if labels.size: # normalized xywh to pixel xyxy format\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], ratio[0] * w, ratio[1] * h, padw=pad[0], padh=pad[1])\n\n if self.augment:\n img, labels = random_perspective(\n img,\n labels,\n degrees=hyp[\"degrees\"],\n translate=hyp[\"translate\"],\n scale=hyp[\"scale\"],\n shear=hyp[\"shear\"],\n perspective=hyp[\"perspective\"],\n )\n\n nl = len(labels) # number of labels\n if nl:\n labels[:, 1:5] = xyxy2xywhn(labels[:, 1:5], w=img.shape[1], h=img.shape[0], clip=True, eps=1e-3)\n\n if self.augment:\n # Albumentations\n img, labels = self.albumentations(img, labels)\n nl = len(labels) # update after albumentations\n\n # HSV color-space\n augment_hsv(img, hgain=hyp[\"hsv_h\"], sgain=hyp[\"hsv_s\"], vgain=hyp[\"hsv_v\"])\n\n # Flip up-down\n if random.random() < hyp[\"flipud\"]:\n img = np.flipud(img)\n if nl:\n labels[:, 2] = 1 - labels[:, 2]\n\n # Flip left-right\n if random.random() < hyp[\"fliplr\"]:\n img = np.fliplr(img)\n if nl:\n labels[:, 1] = 1 - labels[:, 1]\n\n # Cutouts\n # labels = cutout(img, labels, p=0.5)\n # nl = len(labels) # update after cutout\n\n labels_out = torch.zeros((nl, 6))\n if nl:\n labels_out[:, 1:] = torch.from_numpy(labels)\n\n # Convert\n img = img.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n img = np.ascontiguousarray(img)\n\n return torch.from_numpy(img), labels_out, self.im_files[index], shapes\n\n def load_image(self, i):\n \"\"\"\n Loads an image by index, returning the image, its original dimensions, and resized dimensions.\n\n Returns (im, original hw, resized hw)\n \"\"\"\n im, f, fn = (\n self.ims[i],\n self.im_files[i],\n self.npy_files[i],\n )\n if im is None: # not cached in RAM\n if fn.exists(): # load npy\n im = np.load(fn)\n else: # read image\n im = cv2.imread(f) # BGR\n assert im is not None, f\"Image Not Found {f}\"\n h0, w0 = im.shape[:2] # orig hw\n r = self.img_size / max(h0, w0) # ratio\n if r != 1: # if sizes are not equal\n interp = cv2.INTER_LINEAR if (self.augment or r > 1) else cv2.INTER_AREA\n im = cv2.resize(im, (math.ceil(w0 * r), math.ceil(h0 * r)), interpolation=interp)\n return im, (h0, w0), im.shape[:2] # im, hw_original, hw_resized\n return self.ims[i], self.im_hw0[i], self.im_hw[i] # im, hw_original, hw_resized\n\n def cache_images_to_disk(self, i):\n \"\"\"Saves an image to disk as an *.npy file for quicker loading, identified by index `i`.\"\"\"\n f = self.npy_files[i]\n if not f.exists():\n np.save(f.as_posix(), cv2.imread(self.im_files[i]))\n\n def load_mosaic(self, index):\n \"\"\"Loads a 4-image mosaic for YOLOv5, combining 1 selected and 3 random images, with labels and segments.\"\"\"\n labels4, segments4 = [], []\n s = self.img_size\n yc, xc = (int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border) # mosaic center x, y\n indices = [index] + random.choices(self.indices, k=3) # 3 additional image indices\n random.shuffle(indices)\n for i, index in enumerate(indices):\n # Load image\n img, _, (h, w) = self.load_image(index)\n\n # place img in img4\n if i == 0: # top left\n img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\n x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image)\n x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image)\n elif i == 1: # top right\n x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n elif i == 2: # bottom left\n x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h)\n elif i == 3: # bottom right\n x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n\n img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]\n padw = x1a - x1b\n padh = y1a - y1b\n\n # Labels\n labels, segments = self.labels[index].copy(), self.segments[index].copy()\n if labels.size:\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh) # normalized xywh to pixel xyxy format\n segments = [xyn2xy(x, w, h, padw, padh) for x in segments]\n labels4.append(labels)\n segments4.extend(segments)\n\n # Concat/clip labels\n labels4 = np.concatenate(labels4, 0)\n for x in (labels4[:, 1:], *segments4):\n np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective()\n # img4, labels4 = replicate(img4, labels4) # replicate\n\n # Augment\n img4, labels4, segments4 = copy_paste(img4, labels4, segments4, p=self.hyp[\"copy_paste\"])\n img4, labels4 = random_perspective(\n img4,\n labels4,\n segments4,\n degrees=self.hyp[\"degrees\"],\n translate=self.hyp[\"translate\"],\n scale=self.hyp[\"scale\"],\n shear=self.hyp[\"shear\"],\n perspective=self.hyp[\"perspective\"],\n border=self.mosaic_border,\n ) # border to remove\n\n return img4, labels4\n\n def load_mosaic9(self, index):\n \"\"\"Loads 1 image + 8 random images into a 9-image mosaic for augmented YOLOv5 training, returning labels and\n segments.\n \"\"\"\n labels9, segments9 = [], []\n s = self.img_size\n indices = [index] + random.choices(self.indices, k=8) # 8 additional image indices\n random.shuffle(indices)\n hp, wp = -1, -1 # height, width previous\n for i, index in enumerate(indices):\n # Load image\n img, _, (h, w) = self.load_image(index)\n\n # place img in img9\n if i == 0: # center\n img9 = np.full((s * 3, s * 3, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\n h0, w0 = h, w\n c = s, s, s + w, s + h # xmin, ymin, xmax, ymax (base) coordinates\n elif i == 1: # top\n c = s, s - h, s + w, s\n elif i == 2: # top right\n c = s + wp, s - h, s + wp + w, s\n elif i == 3: # right\n c = s + w0, s, s + w0 + w, s + h\n elif i == 4: # bottom right\n c = s + w0, s + hp, s + w0 + w, s + hp + h\n elif i == 5: # bottom\n c = s + w0 - w, s + h0, s + w0, s + h0 + h\n elif i == 6: # bottom left\n c = s + w0 - wp - w, s + h0, s + w0 - wp, s + h0 + h\n elif i == 7: # left\n c = s - w, s + h0 - h, s, s + h0\n elif i == 8: # top left\n c = s - w, s + h0 - hp - h, s, s + h0 - hp\n\n padx, pady = c[:2]\n x1, y1, x2, y2 = (max(x, 0) for x in c) # allocate coords\n\n # Labels\n labels, segments = self.labels[index].copy(), self.segments[index].copy()\n if labels.size:\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padx, pady) # normalized xywh to pixel xyxy format\n segments = [xyn2xy(x, w, h, padx, pady) for x in segments]\n labels9.append(labels)\n segments9.extend(segments)\n\n # Image\n img9[y1:y2, x1:x2] = img[y1 - pady :, x1 - padx :] # img9[ymin:ymax, xmin:xmax]\n hp, wp = h, w # height, width previous\n\n # Offset\n yc, xc = (int(random.uniform(0, s)) for _ in self.mosaic_border) # mosaic center x, y\n img9 = img9[yc : yc + 2 * s, xc : xc + 2 * s]\n\n # Concat/clip labels\n labels9 = np.concatenate(labels9, 0)\n labels9[:, [1, 3]] -= xc\n labels9[:, [2, 4]] -= yc\n c = np.array([xc, yc]) # centers\n segments9 = [x - c for x in segments9]\n\n for x in (labels9[:, 1:], *segments9):\n np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective()\n # img9, labels9 = replicate(img9, labels9) # replicate\n\n # Augment\n img9, labels9, segments9 = copy_paste(img9, labels9, segments9, p=self.hyp[\"copy_paste\"])\n img9, labels9 = random_perspective(\n img9,\n labels9,\n segments9,\n degrees=self.hyp[\"degrees\"],\n translate=self.hyp[\"translate\"],\n scale=self.hyp[\"scale\"],\n shear=self.hyp[\"shear\"],\n perspective=self.hyp[\"perspective\"],\n border=self.mosaic_border,\n ) # border to remove\n\n return img9, labels9\n\n @staticmethod\n def collate_fn(batch):\n \"\"\"Batches images, labels, paths, and shapes, assigning unique indices to targets in merged label tensor.\"\"\"\n im, label, path, shapes = zip(*batch) # transposed\n for i, lb in enumerate(label):\n lb[:, 0] = i # add target image index for build_targets()\n return torch.stack(im, 0), torch.cat(label, 0), path, shapes\n\n @staticmethod\n def collate_fn4(batch):\n \"\"\"Bundles a batch's data by quartering the number of shapes and paths, preparing it for model input.\"\"\"\n im, label, path, shapes = zip(*batch) # transposed\n n = len(shapes) // 4\n im4, label4, path4, shapes4 = [], [], path[:n], shapes[:n]\n\n ho = torch.tensor([[0.0, 0, 0, 1, 0, 0]])\n wo = torch.tensor([[0.0, 0, 1, 0, 0, 0]])\n s = torch.tensor([[1, 1, 0.5, 0.5, 0.5, 0.5]]) # scale\n for i in range(n): # zidane torch.zeros(16,3,720,1280) # BCHW\n i *= 4\n if random.random() < 0.5:\n im1 = F.interpolate(im[i].unsqueeze(0).float(), scale_factor=2.0, mode=\"bilinear\", align_corners=False)[\n 0\n ].type(im[i].type())\n lb = label[i]\n else:\n im1 = torch.cat((torch.cat((im[i], im[i + 1]), 1), torch.cat((im[i + 2], im[i + 3]), 1)), 2)\n lb = torch.cat((label[i], label[i + 1] + ho, label[i + 2] + wo, label[i + 3] + ho + wo), 0) * s\n im4.append(im1)\n label4.append(lb)\n\n for i, lb in enumerate(label4):\n lb[:, 0] = i # add target image index for build_targets()\n\n return torch.stack(im4, 0), torch.cat(label4, 0), path4, shapes4\n\n\n# Ancillary functions --------------------------------------------------------------------------------------------------\ndef flatten_recursive(path=DATASETS_DIR / \"coco128\"):\n \"\"\"Flattens a directory by copying all files from subdirectories to a new top-level directory, preserving\n filenames.\n \"\"\"\n new_path = Path(f\"{str(path)}_flat\")\n if os.path.exists(new_path):\n shutil.rmtree(new_path) # delete output folder\n os.makedirs(new_path) # make new output folder\n for file in tqdm(glob.glob(f\"{str(Path(path))}/**/*.*\", recursive=True)):\n shutil.copyfile(file, new_path / Path(file).name)\n\n\ndef extract_boxes(path=DATASETS_DIR / \"coco128\"):\n \"\"\"\n Converts a detection dataset to a classification dataset, creating a directory for each class and extracting\n bounding boxes.\n\n Example: from utils.dataloaders import *; extract_boxes()\n \"\"\"\n path = Path(path) # images dir\n shutil.rmtree(path / \"classification\") if (path / \"classification\").is_dir() else None # remove existing\n files = list(path.rglob(\"*.*\"))\n n = len(files) # number of files\n for im_file in tqdm(files, total=n):\n if im_file.suffix[1:] in IMG_FORMATS:\n # image\n im = cv2.imread(str(im_file))[..., ::-1] # BGR to RGB\n h, w = im.shape[:2]\n\n # labels\n lb_file = Path(img2label_paths([str(im_file)])[0])\n if Path(lb_file).exists():\n with open(lb_file) as f:\n lb = np.array([x.split() for x in f.read().strip().splitlines()], dtype=np.float32) # labels\n\n for j, x in enumerate(lb):\n c = int(x[0]) # class\n f = (path / \"classification\") / f\"{c}\" / f\"{path.stem}_{im_file.stem}_{j}.jpg\" # new filename\n if not f.parent.is_dir():\n f.parent.mkdir(parents=True)\n\n b = x[1:] * [w, h, w, h] # box\n # b[2:] = b[2:].max() # rectangle to square\n b[2:] = b[2:] * 1.2 + 3 # pad\n b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(int)\n\n b[[0, 2]] = np.clip(b[[0, 2]], 0, w) # clip boxes outside of image\n b[[1, 3]] = np.clip(b[[1, 3]], 0, h)\n assert cv2.imwrite(str(f), im[b[1] : b[3], b[0] : b[2]]), f\"box failure in {f}\"\n\n\ndef autosplit(path=DATASETS_DIR / \"coco128/images\", weights=(0.9, 0.1, 0.0), annotated_only=False):\n \"\"\"Autosplit a dataset into train/val/test splits and save path/autosplit_*.txt files\n Usage: from utils.dataloaders import *; autosplit()\n Arguments\n path: Path to images directory\n weights: Train, val, test weights (list, tuple)\n annotated_only: Only use images with an annotated txt file\n \"\"\"\n path = Path(path) # images dir\n files = sorted(x for x in path.rglob(\"*.*\") if x.suffix[1:].lower() in IMG_FORMATS) # image files only\n n = len(files) # number of files\n random.seed(0) # for reproducibility\n indices = random.choices([0, 1, 2], weights=weights, k=n) # assign each image to a split\n\n txt = [\"autosplit_train.txt\", \"autosplit_val.txt\", \"autosplit_test.txt\"] # 3 txt files\n for x in txt:\n if (path.parent / x).exists():\n (path.parent / x).unlink() # remove existing\n\n print(f\"Autosplitting images from {path}\" + \", using *.txt labeled images only\" * annotated_only)\n for i, img in tqdm(zip(indices, files), total=n):\n if not annotated_only or Path(img2label_paths([str(img)])[0]).exists(): # check label\n with open(path.parent / txt[i], \"a\") as f:\n f.write(f\"./{img.relative_to(path.parent).as_posix()}\" + \"\\n\") # add image to txt file\n\n\ndef verify_image_label(args):\n \"\"\"Verifies a single image-label pair, ensuring image format, size, and legal label values.\"\"\"\n im_file, lb_file, prefix = args\n nm, nf, ne, nc, msg, segments = 0, 0, 0, 0, \"\", [] # number (missing, found, empty, corrupt), message, segments\n try:\n # verify images\n im = Image.open(im_file)\n im.verify() # PIL verify\n shape = exif_size(im) # image size\n assert (shape[0] > 9) & (shape[1] > 9), f\"image size {shape} <10 pixels\"\n assert im.format.lower() in IMG_FORMATS, f\"invalid image format {im.format}\"\n if im.format.lower() in (\"jpg\", \"jpeg\"):\n with open(im_file, \"rb\") as f:\n f.seek(-2, 2)\n if f.read() != b\"\\xff\\xd9\": # corrupt JPEG\n ImageOps.exif_transpose(Image.open(im_file)).save(im_file, \"JPEG\", subsampling=0, quality=100)\n msg = f\"{prefix}WARNING ⚠️ {im_file}: corrupt JPEG restored and saved\"\n\n # verify labels\n if os.path.isfile(lb_file):\n nf = 1 # label found\n with open(lb_file) as f:\n lb = [x.split() for x in f.read().strip().splitlines() if len(x)]\n if any(len(x) > 6 for x in lb): # is segment\n classes = np.array([x[0] for x in lb], dtype=np.float32)\n segments = [np.array(x[1:], dtype=np.float32).reshape(-1, 2) for x in lb] # (cls, xy1...)\n lb = np.concatenate((classes.reshape(-1, 1), segments2boxes(segments)), 1) # (cls, xywh)\n lb = np.array(lb, dtype=np.float32)\n nl = len(lb)\n if nl:\n assert lb.shape[1] == 5, f\"labels require 5 columns, {lb.shape[1]} columns detected\"\n assert (lb >= 0).all(), f\"negative label values {lb[lb < 0]}\"\n assert (lb[:, 1:] <= 1).all(), f\"non-normalized or out of bounds coordinates {lb[:, 1:][lb[:, 1:] > 1]}\"\n _, i = np.unique(lb, axis=0, return_index=True)\n if len(i) < nl: # duplicate row check\n lb = lb[i] # remove duplicates\n if segments:\n segments = [segments[x] for x in i]\n msg = f\"{prefix}WARNING ⚠️ {im_file}: {nl - len(i)} duplicate labels removed\"\n else:\n ne = 1 # label empty\n lb = np.zeros((0, 5), dtype=np.float32)\n else:\n nm = 1 # label missing\n lb = np.zeros((0, 5), dtype=np.float32)\n return im_file, lb, shape, segments, nm, nf, ne, nc, msg\n except Exception as e:\n nc = 1\n msg = f\"{prefix}WARNING ⚠️ {im_file}: ignoring corrupt image/label: {e}\"\n return [None, None, None, None, nm, nf, ne, nc, msg]\n\n\nclass HUBDatasetStats:\n \"\"\"\n Class for generating HUB dataset JSON and `-hub` dataset directory.\n\n Arguments\n path: Path to data.yaml or data.zip (with data.yaml inside data.zip)\n autodownload: Attempt to download dataset if not found locally\n\n Usage\n from utils.dataloaders import HUBDatasetStats\n stats = HUBDatasetStats('coco128.yaml', autodownload=True) # usage 1\n stats = HUBDatasetStats('path/to/coco128.zip') # usage 2\n stats.get_json(save=False)\n stats.process_images()\n \"\"\"\n\n def __init__(self, path=\"coco128.yaml\", autodownload=False):\n \"\"\"Initializes HUBDatasetStats with optional auto-download for datasets, given a path to dataset YAML or ZIP\n file.\n \"\"\"\n zipped, data_dir, yaml_path = self._unzip(Path(path))\n try:\n with open(check_yaml(yaml_path), errors=\"ignore\") as f:\n data = yaml.safe_load(f) # data dict\n if zipped:\n data[\"path\"] = data_dir\n except Exception as e:\n raise Exception(\"error/HUB/dataset_stats/yaml_load\") from e\n\n check_dataset(data, autodownload) # download dataset if missing\n self.hub_dir = Path(data[\"path\"] + \"-hub\")\n self.im_dir = self.hub_dir / \"images\"\n self.im_dir.mkdir(parents=True, exist_ok=True) # makes /images\n self.stats = {\"nc\": data[\"nc\"], \"names\": list(data[\"names\"].values())} # statistics dictionary\n self.data = data\n\n @staticmethod\n def _find_yaml(dir):\n \"\"\"Finds and returns the path to a single '.yaml' file in the specified directory, preferring files that match\n the directory name.\n \"\"\"\n files = list(dir.glob(\"*.yaml\")) or list(dir.rglob(\"*.yaml\")) # try root level first and then recursive\n assert files, f\"No *.yaml file found in {dir}\"\n if len(files) > 1:\n files = [f for f in files if f.stem == dir.stem] # prefer *.yaml files that match dir name\n assert files, f\"Multiple *.yaml files found in {dir}, only 1 *.yaml file allowed\"\n assert len(files) == 1, f\"Multiple *.yaml files found: {files}, only 1 *.yaml file allowed in {dir}\"\n return files[0]\n\n def _unzip(self, path):\n \"\"\"Unzips a .zip file at 'path', returning success status, unzipped directory, and path to YAML file within.\"\"\"\n if not str(path).endswith(\".zip\"): # path is data.yaml\n return False, None, path\n assert Path(path).is_file(), f\"Error unzipping {path}, file not found\"\n unzip_file(path, path=path.parent)\n dir = path.with_suffix(\"\") # dataset directory == zip name\n assert dir.is_dir(), f\"Error unzipping {path}, {dir} not found. path/to/abc.zip MUST unzip to path/to/abc/\"\n return True, str(dir), self._find_yaml(dir) # zipped, data_dir, yaml_path\n\n def _hub_ops(self, f, max_dim=1920):\n \"\"\"Resizes and saves an image at reduced quality for web/app viewing, supporting both PIL and OpenCV.\"\"\"\n f_new = self.im_dir / Path(f).name # dataset-hub image filename\n try: # use PIL\n im = Image.open(f)\n r = max_dim / max(im.height, im.width) # ratio\n if r < 1.0: # image too large\n im = im.resize((int(im.width * r), int(im.height * r)))\n im.save(f_new, \"JPEG\", quality=50, optimize=True) # save\n except Exception as e: # use OpenCV\n LOGGER.info(f\"WARNING ⚠️ HUB ops PIL failure {f}: {e}\")\n im = cv2.imread(f)\n im_height, im_width = im.shape[:2]\n r = max_dim / max(im_height, im_width) # ratio\n if r < 1.0: # image too large\n im = cv2.resize(im, (int(im_width * r), int(im_height * r)), interpolation=cv2.INTER_AREA)\n cv2.imwrite(str(f_new), im)\n\n def get_json(self, save=False, verbose=False):\n \"\"\"Generates dataset JSON for Ultralytics HUB, optionally saves or prints it; save=bool, verbose=bool.\"\"\"\n\n def _round(labels):\n \"\"\"Rounds class labels to integers and coordinates to 4 decimal places for improved label accuracy.\"\"\"\n return [[int(c), *(round(x, 4) for x in points)] for c, *points in labels]\n\n for split in \"train\", \"val\", \"test\":\n if self.data.get(split) is None:\n self.stats[split] = None # i.e. no test set\n continue\n dataset = LoadImagesAndLabels(self.data[split]) # load dataset\n x = np.array(\n [\n np.bincount(label[:, 0].astype(int), minlength=self.data[\"nc\"])\n for label in tqdm(dataset.labels, total=dataset.n, desc=\"Statistics\")\n ]\n ) # shape(128x80)\n self.stats[split] = {\n \"instance_stats\": {\"total\": int(x.sum()), \"per_class\": x.sum(0).tolist()},\n \"image_stats\": {\n \"total\": dataset.n,\n \"unlabelled\": int(np.all(x == 0, 1).sum()),\n \"per_class\": (x > 0).sum(0).tolist(),\n },\n \"labels\": [{str(Path(k).name): _round(v.tolist())} for k, v in zip(dataset.im_files, dataset.labels)],\n }\n\n # Save, print and return\n if save:\n stats_path = self.hub_dir / \"stats.json\"\n print(f\"Saving {stats_path.resolve()}...\")\n with open(stats_path, \"w\") as f:\n json.dump(self.stats, f) # save stats.json\n if verbose:\n print(json.dumps(self.stats, indent=2, sort_keys=False))\n return self.stats\n\n def process_images(self):\n \"\"\"Compresses images for Ultralytics HUB across 'train', 'val', 'test' splits and saves to specified\n directory.\n \"\"\"\n for split in \"train\", \"val\", \"test\":\n if self.data.get(split) is None:\n continue\n dataset = LoadImagesAndLabels(self.data[split]) # load dataset\n desc = f\"{split} images\"\n for _ in tqdm(ThreadPool(NUM_THREADS).imap(self._hub_ops, dataset.im_files), total=dataset.n, desc=desc):\n pass\n print(f\"Done. All images saved to {self.im_dir}\")\n return self.im_dir\n\n\n# Classification dataloaders -------------------------------------------------------------------------------------------\nclass ClassificationDataset(torchvision.datasets.ImageFolder):\n \"\"\"\n YOLOv5 Classification Dataset.\n\n Arguments\n root: Dataset path\n transform: torchvision transforms, used by default\n album_transform: Albumentations transforms, used if installed\n \"\"\"\n\n def __init__(self, root, augment, imgsz, cache=False):\n \"\"\"Initializes YOLOv5 Classification Dataset with optional caching, augmentations, and transforms for image\n classification.\n \"\"\"\n super().__init__(root=root)\n self.torch_transforms = classify_transforms(imgsz)\n self.album_transforms = classify_albumentations(augment, imgsz) if augment else None\n self.cache_ram = cache is True or cache == \"ram\"\n self.cache_disk = cache == \"disk\"\n self.samples = [list(x) + [Path(x[0]).with_suffix(\".npy\"), None] for x in self.samples] # file, index, npy, im\n\n def __getitem__(self, i):\n \"\"\"Fetches and transforms an image sample by index, supporting RAM/disk caching and Augmentations.\"\"\"\n f, j, fn, im = self.samples[i] # filename, index, filename.with_suffix('.npy'), image\n if self.cache_ram and im is None:\n im = self.samples[i][3] = cv2.imread(f)\n elif self.cache_disk:\n if not fn.exists(): # load npy\n np.save(fn.as_posix(), cv2.imread(f))\n im = np.load(fn)\n else: # read image\n im = cv2.imread(f) # BGR\n if self.album_transforms:\n sample = self.album_transforms(image=cv2.cvtColor(im, cv2.COLOR_BGR2RGB))[\"image\"]\n else:\n sample = self.torch_transforms(im)\n return sample, j\n\n\ndef create_classification_dataloader(\n path, imgsz=224, batch_size=16, augment=True, cache=False, rank=-1, workers=8, shuffle=True\n):\n # Returns Dataloader object to be used with YOLOv5 Classifier\n with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP\n dataset = ClassificationDataset(root=path, imgsz=imgsz, augment=augment, cache=cache)\n batch_size = min(batch_size, len(dataset))\n nd = torch.cuda.device_count()\n nw = min([os.cpu_count() // max(nd, 1), batch_size if batch_size > 1 else 0, workers])\n sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle)\n generator = torch.Generator()\n generator.manual_seed(6148914691236517205 + RANK)\n return InfiniteDataLoader(\n dataset,\n batch_size=batch_size,\n shuffle=shuffle and sampler is None,\n num_workers=nw,\n sampler=sampler,\n pin_memory=PIN_MEMORY,\n worker_init_fn=seed_worker,\n generator=generator,\n ) # or DataLoader(persistent_workers=True)" }, { "path": "asone/detectors/yolov5/yolov5/utils/downloads.py.py", "content": "# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license\n\"\"\"Download utils.\"\"\"\n\nimport logging\nimport subprocess\nimport urllib\nfrom pathlib import Path\n\nimport requests\nimport torch\n\n\ndef is_url(url, check=True):\n \"\"\"Determines if a string is a URL and optionally checks its existence online, returning a boolean.\"\"\"\n try:\n url = str(url)\n result = urllib.parse.urlparse(url)\n assert all([result.scheme, result.netloc]) # check if is url\n return (urllib.request.urlopen(url).getcode() == 200) if check else True # check if exists online\n except (AssertionError, urllib.request.HTTPError):\n return False\n\n\ndef gsutil_getsize(url=\"\"):\n \"\"\"\n Returns the size in bytes of a file at a Google Cloud Storage URL using `gsutil du`.\n\n Returns 0 if the command fails or output is empty.\n \"\"\"\n output = subprocess.check_output([\"gsutil\", \"du\", url], shell=True, encoding=\"utf-8\")\n return int(output.split()[0]) if output else 0\n\n\ndef url_getsize(url=\"https://ultralytics.com/images/bus.jpg\"):\n \"\"\"Returns the size in bytes of a downloadable file at a given URL; defaults to -1 if not found.\"\"\"\n response = requests.head(url, allow_redirects=True)\n return int(response.headers.get(\"content-length\", -1))\n\n\ndef curl_download(url, filename, *, silent: bool = False) -> bool:\n \"\"\"Download a file from a url to a filename using curl.\"\"\"\n silent_option = \"sS\" if silent else \"\" # silent\n proc = subprocess.run(\n [\n \"curl\",\n \"-#\",\n f\"-{silent_option}L\",\n url,\n \"--output\",\n filename,\n \"--retry\",\n \"9\",\n \"-C\",\n \"-\",\n ]\n )\n return proc.returncode == 0\n\n\ndef safe_download(file, url, url2=None, min_bytes=1e0, error_msg=\"\"):\n \"\"\"\n Downloads a file from a URL (or alternate URL) to a specified path if file is above a minimum size.\n\n Removes incomplete downloads.\n \"\"\"\n from asone.detectors.yolov5.yolov5.utils.general import LOGGER\n\n file = Path(file)\n assert_msg = f\"Downloaded file '{file}' does not exist or size is < min_bytes={min_bytes}\"\n try: # url1\n LOGGER.info(f\"Downloading {url} to {file}...\")\n torch.hub.download_url_to_file(url, str(file), progress=LOGGER.level <= logging.INFO)\n assert file.exists() and file.stat().st_size > min_bytes, assert_msg # check\n except Exception as e: # url2\n if file.exists():\n file.unlink() # remove partial downloads\n LOGGER.info(f\"ERROR: {e}\\nRe-attempting {url2 or url} to {file}...\")\n # curl download, retry and resume on fail\n curl_download(url2 or url, file)\n finally:\n if not file.exists() or file.stat().st_size < min_bytes: # check\n if file.exists():\n file.unlink() # remove partial downloads\n LOGGER.info(f\"ERROR: {assert_msg}\\n{error_msg}\")\n LOGGER.info(\"\")\n\n\ndef attempt_download(file, repo=\"ultralytics/yolov5\", release=\"v7.0\"):\n \"\"\"Downloads a file from GitHub release assets or via direct URL if not found locally, supporting backup\n versions.\n \"\"\"\n from asone.detectors.yolov5.yolov5.utils.general import LOGGER\n\n def github_assets(repository, version=\"latest\"):\n # Return GitHub repo tag (i.e. 'v7.0') and assets (i.e. ['yolov5s.pt', 'yolov5m.pt', ...])\n if version != \"latest\":\n version = f\"tags/{version}\" # i.e. tags/v7.0\n response = requests.get(f\"https://api.github.com/repos/{repository}/releases/{version}\").json() # github api\n return response[\"tag_name\"], [x[\"name\"] for x in response[\"assets\"]] # tag, assets\n\n file = Path(str(file).strip().replace(\"'\", \"\"))\n if not file.exists():\n # URL specified\n name = Path(urllib.parse.unquote(str(file))).name # decode '%2F' to '/' etc.\n if str(file).startswith((\"http:/\", \"https:/\")): # download\n url = str(file).replace(\":/\", \"://\") # Pathlib turns :// -> :/\n file = name.split(\"?\")[0] # parse authentication https://url.com/file.txt?auth...\n if Path(file).is_file():\n LOGGER.info(f\"Found {url} locally at {file}\") # file already exists\n else:\n safe_download(file=file, url=url, min_bytes=1e5)\n return file\n\n # GitHub assets\n assets = [f\"yolov5{size}{suffix}.pt\" for size in \"nsmlx\" for suffix in (\"\", \"6\", \"-cls\", \"-seg\")] # default\n try:\n tag, assets = github_assets(repo, release)\n except Exception:\n try:\n tag, assets = github_assets(repo) # latest release\n except Exception:\n try:\n tag = subprocess.check_output(\"git tag\", shell=True, stderr=subprocess.STDOUT).decode().split()[-1]\n except Exception:\n tag = release\n\n if name in assets:\n file.parent.mkdir(parents=True, exist_ok=True) # make parent dir (if required)\n safe_download(\n file,\n url=f\"https://github.com/{repo}/releases/download/{tag}/{name}\",\n min_bytes=1e5,\n error_msg=f\"{file} missing, try downloading from https://github.com/{repo}/releases/{tag}\",\n )\n\n return str(file)" }, { "path": "asone/detectors/yolov5/yolov5/utils/general.py", "content": "# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license\n\"\"\"General utils.\"\"\"\n\nimport contextlib\nimport glob\nimport inspect\nimport logging\nimport logging.config\nimport math\nimport os\nimport platform\nimport random\nimport re\nimport signal\nimport subprocess\nimport sys\nimport time\nimport urllib\nfrom copy import deepcopy\nfrom datetime import datetime\nfrom itertools import repeat\nfrom multiprocessing.pool import ThreadPool\nfrom pathlib import Path\nfrom subprocess import check_output\nfrom tarfile import is_tarfile\nfrom typing import Optional\nfrom zipfile import ZipFile, is_zipfile\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport pkg_resources as pkg\nimport torch\nimport torchvision\nimport yaml\n\n# Import 'ultralytics' package or install if missing\ntry:\n import ultralytics\n\n assert hasattr(ultralytics, \"__version__\") # verify package is not directory\nexcept (ImportError, AssertionError):\n os.system(\"pip install -U ultralytics\")\n import ultralytics\n\nfrom ultralytics.utils.checks import check_requirements\n\nfrom asone.detectors.yolov5.yolov5.utils import TryExcept, emojis\nfrom asone.detectors.yolov5.yolov5.utils.downloads import curl_download, gsutil_getsize\nfrom asone.detectors.yolov5.yolov5.utils.metrics import box_iou, fitness\n\nFILE = Path(__file__).resolve()\nROOT = FILE.parents[1] # YOLOv5 root directory\nRANK = int(os.getenv(\"RANK\", -1))\n\n# Settings\nNUM_THREADS = min(8, max(1, os.cpu_count() - 1)) # number of YOLOv5 multiprocessing threads\nDATASETS_DIR = Path(os.getenv(\"YOLOv5_DATASETS_DIR\", ROOT.parent / \"datasets\")) # global datasets directory\nAUTOINSTALL = str(os.getenv(\"YOLOv5_AUTOINSTALL\", True)).lower() == \"true\" # global auto-install mode\nVERBOSE = str(os.getenv(\"YOLOv5_VERBOSE\", True)).lower() == \"true\" # global verbose mode\nTQDM_BAR_FORMAT = \"{l_bar}{bar:10}{r_bar}\" # tqdm bar format\nFONT = \"Arial.ttf\" # https://ultralytics.com/assets/Arial.ttf\n\ntorch.set_printoptions(linewidth=320, precision=5, profile=\"long\")\nnp.set_printoptions(linewidth=320, formatter={\"float_kind\": \"{:11.5g}\".format}) # format short g, %precision=5\npd.options.display.max_columns = 10\ncv2.setNumThreads(0) # prevent OpenCV from multithreading (incompatible with PyTorch DataLoader)\nos.environ[\"NUMEXPR_MAX_THREADS\"] = str(NUM_THREADS) # NumExpr max threads\nos.environ[\"OMP_NUM_THREADS\"] = \"1\" if platform.system() == \"darwin\" else str(NUM_THREADS) # OpenMP (PyTorch and SciPy)\nos.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"2\" # suppress verbose TF compiler warnings in Colab\n\n\ndef is_ascii(s=\"\"):\n \"\"\"Checks if input string `s` contains only ASCII characters; returns `True` if so, otherwise `False`.\"\"\"\n s = str(s) # convert list, tuple, None, etc. to str\n return len(s.encode().decode(\"ascii\", \"ignore\")) == len(s)\n\n\ndef is_chinese(s=\"人工智能\"):\n \"\"\"Determines if a string `s` contains any Chinese characters; returns `True` if so, otherwise `False`.\"\"\"\n return bool(re.search(\"[\\u4e00-\\u9fff]\", str(s)))\n\n\ndef is_colab():\n \"\"\"Checks if the current environment is a Google Colab instance; returns `True` for Colab, otherwise `False`.\"\"\"\n return \"google.colab\" in sys.modules\n\n\ndef is_jupyter():\n \"\"\"\n Check if the current script is running inside a Jupyter Notebook. Verified on Colab, Jupyterlab, Kaggle, Paperspace.\n\n Returns:\n bool: True if running inside a Jupyter Notebook, False otherwise.\n \"\"\"\n with contextlib.suppress(Exception):\n from IPython import get_ipython\n\n return get_ipython() is not None\n return False\n\n\ndef is_kaggle():\n \"\"\"Checks if the current environment is a Kaggle Notebook by validating environment variables.\"\"\"\n return os.environ.get(\"PWD\") == \"/kaggle/working\" and os.environ.get(\"KAGGLE_URL_BASE\") == \"https://www.kaggle.com\"\n\n\ndef is_docker() -> bool:\n \"\"\"Check if the process runs inside a docker container.\"\"\"\n if Path(\"/.dockerenv\").exists():\n return True\n try: # check if docker is in control groups\n with open(\"/proc/self/cgroup\") as file:\n return any(\"docker\" in line for line in file)\n except OSError:\n return False\n\n\ndef is_writeable(dir, test=False):\n \"\"\"Checks if a directory is writable, optionally testing by creating a temporary file if `test=True`.\"\"\"\n if not test:\n return os.access(dir, os.W_OK) # possible issues on Windows\n file = Path(dir) / \"tmp.txt\"\n try:\n with open(file, \"w\"): # open file with write permissions\n pass\n file.unlink() # remove file\n return True\n except OSError:\n return False\n\n\nLOGGING_NAME = \"yolov5\"\n\n\ndef set_logging(name=LOGGING_NAME, verbose=True):\n \"\"\"Configures logging with specified verbosity; `name` sets the logger's name, `verbose` controls logging level.\"\"\"\n rank = int(os.getenv(\"RANK\", -1)) # rank in world for Multi-GPU trainings\n level = logging.INFO if verbose and rank in {-1, 0} else logging.ERROR\n logging.config.dictConfig(\n {\n \"version\": 1,\n \"disable_existing_loggers\": False,\n \"formatters\": {name: {\"format\": \"%(message)s\"}},\n \"handlers\": {\n name: {\n \"class\": \"logging.StreamHandler\",\n \"formatter\": name,\n \"level\": level,\n }\n },\n \"loggers\": {\n name: {\n \"level\": level,\n \"handlers\": [name],\n \"propagate\": False,\n }\n },\n }\n )\n\n\nset_logging(LOGGING_NAME) # run before defining LOGGER\nLOGGER = logging.getLogger(LOGGING_NAME) # define globally (used in train.py, val.py, detect.py, etc.)\nif platform.system() == \"Windows\":\n for fn in LOGGER.info, LOGGER.warning:\n setattr(LOGGER, fn.__name__, lambda x: fn(emojis(x))) # emoji safe logging\n\n\ndef user_config_dir(dir=\"Ultralytics\", env_var=\"YOLOV5_CONFIG_DIR\"):\n \"\"\"Returns user configuration directory path, preferring environment variable `YOLOV5_CONFIG_DIR` if set, else OS-\n specific.\n \"\"\"\n env = os.getenv(env_var)\n if env:\n path = Path(env) # use environment variable\n else:\n cfg = {\"Windows\": \"AppData/Roaming\", \"Linux\": \".config\", \"Darwin\": \"Library/Application Support\"} # 3 OS dirs\n path = Path.home() / cfg.get(platform.system(), \"\") # OS-specific config dir\n path = (path if is_writeable(path) else Path(\"/tmp\")) / dir # GCP and AWS lambda fix, only /tmp is writeable\n path.mkdir(exist_ok=True) # make if required\n return path\n\n\nCONFIG_DIR = user_config_dir() # Ultralytics settings dir\n\n\nclass Profile(contextlib.ContextDecorator):\n # YOLOv5 Profile class. Usage: @Profile() decorator or 'with Profile():' context manager\n def __init__(self, t=0.0, device: torch.device = None):\n \"\"\"Initializes a profiling context for YOLOv5 with optional timing threshold and device specification.\"\"\"\n self.t = t\n self.device = device\n self.cuda = bool(device and str(device).startswith(\"cuda\"))\n\n def __enter__(self):\n \"\"\"Initializes timing at the start of a profiling context block for performance measurement.\"\"\"\n self.start = self.time()\n return self\n\n def __exit__(self, type, value, traceback):\n \"\"\"Concludes timing, updating duration for profiling upon exiting a context block.\"\"\"\n self.dt = self.time() - self.start # delta-time\n self.t += self.dt # accumulate dt\n\n def time(self):\n \"\"\"Measures and returns the current time, synchronizing CUDA operations if `cuda` is True.\"\"\"\n if self.cuda:\n torch.cuda.synchronize(self.device)\n return time.time()\n\n\nclass Timeout(contextlib.ContextDecorator):\n # YOLOv5 Timeout class. Usage: @Timeout(seconds) decorator or 'with Timeout(seconds):' context manager\n def __init__(self, seconds, *, timeout_msg=\"\", suppress_timeout_errors=True):\n \"\"\"Initializes a timeout context/decorator with defined seconds, optional message, and error suppression.\"\"\"\n self.seconds = int(seconds)\n self.timeout_message = timeout_msg\n self.suppress = bool(suppress_timeout_errors)\n\n def _timeout_handler(self, signum, frame):\n \"\"\"Raises a TimeoutError with a custom message when a timeout event occurs.\"\"\"\n raise TimeoutError(self.timeout_message)\n\n def __enter__(self):\n \"\"\"Initializes timeout mechanism on non-Windows platforms, starting a countdown to raise TimeoutError.\"\"\"\n if platform.system() != \"Windows\": # not supported on Windows\n signal.signal(signal.SIGALRM, self._timeout_handler) # Set handler for SIGALRM\n signal.alarm(self.seconds) # start countdown for SIGALRM to be raised\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n \"\"\"Disables active alarm on non-Windows systems and optionally suppresses TimeoutError if set.\"\"\"\n if platform.system() != \"Windows\":\n signal.alarm(0) # Cancel SIGALRM if it's scheduled\n if self.suppress and exc_type is TimeoutError: # Suppress TimeoutError\n return True\n\n\nclass WorkingDirectory(contextlib.ContextDecorator):\n # Usage: @WorkingDirectory(dir) decorator or 'with WorkingDirectory(dir):' context manager\n def __init__(self, new_dir):\n \"\"\"Initializes a context manager/decorator to temporarily change the working directory.\"\"\"\n self.dir = new_dir # new dir\n self.cwd = Path.cwd().resolve() # current dir\n\n def __enter__(self):\n \"\"\"Temporarily changes the working directory within a 'with' statement context.\"\"\"\n os.chdir(self.dir)\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n \"\"\"Restores the original working directory upon exiting a 'with' statement context.\"\"\"\n os.chdir(self.cwd)\n\n\ndef methods(instance):\n \"\"\"Returns list of method names for a class/instance excluding dunder methods.\"\"\"\n return [f for f in dir(instance) if callable(getattr(instance, f)) and not f.startswith(\"__\")]\n\n\ndef print_args(args: Optional[dict] = None, show_file=True, show_func=False):\n \"\"\"Logs the arguments of the calling function, with options to include the filename and function name.\"\"\"\n x = inspect.currentframe().f_back # previous frame\n file, _, func, _, _ = inspect.getframeinfo(x)\n if args is None: # get args automatically\n args, _, _, frm = inspect.getargvalues(x)\n args = {k: v for k, v in frm.items() if k in args}\n try:\n file = Path(file).resolve().relative_to(ROOT).with_suffix(\"\")\n except ValueError:\n file = Path(file).stem\n s = (f\"{file}: \" if show_file else \"\") + (f\"{func}: \" if show_func else \"\")\n LOGGER.info(colorstr(s) + \", \".join(f\"{k}={v}\" for k, v in args.items()))\n\n\ndef init_seeds(seed=0, deterministic=False):\n \"\"\"\n Initializes RNG seeds and sets deterministic options if specified.\n\n See https://pytorch.org/docs/stable/notes/randomness.html\n \"\"\"\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) # for Multi-GPU, exception safe\n # torch.backends.cudnn.benchmark = True # AutoBatch problem https://github.com/ultralytics/yolov5/issues/9287\n if deterministic and check_version(torch.__version__, \"1.12.0\"): # https://github.com/ultralytics/yolov5/pull/8213\n torch.use_deterministic_algorithms(True)\n torch.backends.cudnn.deterministic = True\n os.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n\n\ndef intersect_dicts(da, db, exclude=()):\n \"\"\"Returns intersection of `da` and `db` dicts with matching keys and shapes, excluding `exclude` keys; uses `da`\n values.\n \"\"\"\n return {k: v for k, v in da.items() if k in db and all(x not in k for x in exclude) and v.shape == db[k].shape}\n\n\ndef get_default_args(func):\n \"\"\"Returns a dict of `func` default arguments by inspecting its signature.\"\"\"\n signature = inspect.signature(func)\n return {k: v.default for k, v in signature.parameters.items() if v.default is not inspect.Parameter.empty}\n\n\ndef get_latest_run(search_dir=\".\"):\n \"\"\"Returns the path to the most recent 'last.pt' file in /runs to resume from, searches in `search_dir`.\"\"\"\n last_list = glob.glob(f\"{search_dir}/**/last*.pt\", recursive=True)\n return max(last_list, key=os.path.getctime) if last_list else \"\"\n\n\ndef file_age(path=__file__):\n \"\"\"Calculates and returns the age of a file in days based on its last modification time.\"\"\"\n dt = datetime.now() - datetime.fromtimestamp(Path(path).stat().st_mtime) # delta\n return dt.days # + dt.seconds / 86400 # fractional days\n\n\ndef file_date(path=__file__):\n \"\"\"Returns a human-readable file modification date in 'YYYY-M-D' format, given a file path.\"\"\"\n t = datetime.fromtimestamp(Path(path).stat().st_mtime)\n return f\"{t.year}-{t.month}-{t.day}\"\n\n\ndef file_size(path):\n \"\"\"Returns file or directory size in megabytes (MB) for a given path, where directories are recursively summed.\"\"\"\n mb = 1 << 20 # bytes to MiB (1024 ** 2)\n path = Path(path)\n if path.is_file():\n return path.stat().st_size / mb\n elif path.is_dir():\n return sum(f.stat().st_size for f in path.glob(\"**/*\") if f.is_file()) / mb\n else:\n return 0.0\n\n\ndef check_online():\n \"\"\"Checks internet connectivity by attempting to create a connection to \"1.1.1.1\" on port 443, retries once if the\n first attempt fails.\n \"\"\"\n import socket\n\n def run_once():\n # Check once\n try:\n socket.create_connection((\"1.1.1.1\", 443), 5) # check host accessibility\n return True\n except OSError:\n return False\n\n return run_once() or run_once() # check twice to increase robustness to intermittent connectivity issues\n\n\ndef git_describe(path=ROOT):\n \"\"\"\n Returns a human-readable git description of the repository at `path`, or an empty string on failure.\n\n Example output is 'fv5.0-5-g3e25f1e'. See https://git-scm.com/docs/git-describe.\n \"\"\"\n try:\n assert (Path(path) / \".git\").is_dir()\n return check_output(f\"git -C {path} describe --tags --long --always\", shell=True).decode()[:-1]\n except Exception:\n return \"\"\n\n\n@TryExcept()\n@WorkingDirectory(ROOT)\ndef check_git_status(repo=\"ultralytics/yolov5\", branch=\"master\"):\n \"\"\"Checks if YOLOv5 code is up-to-date with the repository, advising 'git pull' if behind; errors return informative\n messages.\n \"\"\"\n url = f\"https://github.com/{repo}\"\n msg = f\", for updates see {url}\"\n s = colorstr(\"github: \") # string\n assert Path(\".git\").exists(), s + \"skipping check (not a git repository)\" + msg\n assert check_online(), s + \"skipping check (offline)\" + msg\n\n splits = re.split(pattern=r\"\\s\", string=check_output(\"git remote -v\", shell=True).decode())\n matches = [repo in s for s in splits]\n if any(matches):\n remote = splits[matches.index(True) - 1]\n else:\n remote = \"ultralytics\"\n check_output(f\"git remote add {remote} {url}\", shell=True)\n check_output(f\"git fetch {remote}\", shell=True, timeout=5) # git fetch\n local_branch = check_output(\"git rev-parse --abbrev-ref HEAD\", shell=True).decode().strip() # checked out\n n = int(check_output(f\"git rev-list {local_branch}..{remote}/{branch} --count\", shell=True)) # commits behind\n if n > 0:\n pull = \"git pull\" if remote == \"origin\" else f\"git pull {remote} {branch}\"\n s += f\"⚠️ YOLOv5 is out of date by {n} commit{'s' * (n > 1)}. Use '{pull}' or 'git clone {url}' to update.\"\n else:\n s += f\"up to date with {url} ✅\"\n LOGGER.info(s)\n\n\n@WorkingDirectory(ROOT)\ndef check_git_info(path=\".\"):\n \"\"\"Checks YOLOv5 git info, returning a dict with remote URL, branch name, and commit hash.\"\"\"\n check_requirements(\"gitpython\")\n import git\n\n try:\n repo = git.Repo(path)\n remote = repo.remotes.origin.url.replace(\".git\", \"\") # i.e. 'https://github.com/ultralytics/yolov5'\n commit = repo.head.commit.hexsha # i.e. '3134699c73af83aac2a481435550b968d5792c0d'\n try:\n branch = repo.active_branch.name # i.e. 'main'\n except TypeError: # not on any branch\n branch = None # i.e. 'detached HEAD' state\n return {\"remote\": remote, \"branch\": branch, \"commit\": commit}\n except git.exc.InvalidGitRepositoryError: # path is not a git dir\n return {\"remote\": None, \"branch\": None, \"commit\": None}\n\n\ndef check_python(minimum=\"3.8.0\"):\n \"\"\"Checks if current Python version meets the minimum required version, exits if not.\"\"\"\n check_version(platform.python_version(), minimum, name=\"Python \", hard=True)\n\n\ndef check_version(current=\"0.0.0\", minimum=\"0.0.0\", name=\"version \", pinned=False, hard=False, verbose=False):\n \"\"\"Checks if the current version meets the minimum required version, exits or warns based on parameters.\"\"\"\n current, minimum = (pkg.parse_version(x) for x in (current, minimum))\n result = (current == minimum) if pinned else (current >= minimum) # bool\n s = f\"WARNING ⚠️ {name}{minimum} is required by YOLOv5, but {name}{current} is currently installed\" # string\n if hard:\n assert result, emojis(s) # assert min requirements met\n if verbose and not result:\n LOGGER.warning(s)\n return result\n\n\ndef check_img_size(imgsz, s=32, floor=0):\n \"\"\"Adjusts image size to be divisible by stride `s`, supports int or list/tuple input, returns adjusted size.\"\"\"\n if isinstance(imgsz, int): # integer i.e. img_size=640\n new_size = max(make_divisible(imgsz, int(s)), floor)\n else: # list i.e. img_size=[640, 480]\n imgsz = list(imgsz) # convert to list if tuple\n new_size = [max(make_divisible(x, int(s)), floor) for x in imgsz]\n if new_size != imgsz:\n LOGGER.warning(f\"WARNING ⚠️ --img-size {imgsz} must be multiple of max stride {s}, updating to {new_size}\")\n return new_size\n\n\ndef check_imshow(warn=False):\n \"\"\"Checks environment support for image display; warns on failure if `warn=True`.\"\"\"\n try:\n assert not is_jupyter()\n assert not is_docker()\n cv2.imshow(\"test\", np.zeros((1, 1, 3)))\n cv2.waitKey(1)\n cv2.destroyAllWindows()\n cv2.waitKey(1)\n return True\n except Exception as e:\n if warn:\n LOGGER.warning(f\"WARNING ⚠️ Environment does not support cv2.imshow() or PIL Image.show()\\n{e}\")\n return False\n\n\ndef check_suffix(file=\"yolov5s.pt\", suffix=(\".pt\",), msg=\"\"):\n \"\"\"Validates if a file or files have an acceptable suffix, raising an error if not.\"\"\"\n if file and suffix:\n if isinstance(suffix, str):\n suffix = [suffix]\n for f in file if isinstance(file, (list, tuple)) else [file]:\n s = Path(f).suffix.lower() # file suffix\n if len(s):\n assert s in suffix, f\"{msg}{f} acceptable suffix is {suffix}\"\n\n\ndef check_yaml(file, suffix=(\".yaml\", \".yml\")):\n \"\"\"Searches/downloads a YAML file, verifies its suffix (.yaml or .yml), and returns the file path.\"\"\"\n return check_file(file, suffix)\n\n\ndef check_file(file, suffix=\"\"):\n \"\"\"Searches/downloads a file, checks its suffix (if provided), and returns the file path.\"\"\"\n check_suffix(file, suffix) # optional\n file = str(file) # convert to str()\n if os.path.isfile(file) or not file: # exists\n return file\n elif file.startswith((\"http:/\", \"https:/\")): # download\n url = file # warning: Pathlib turns :// -> :/\n file = Path(urllib.parse.unquote(file).split(\"?\")[0]).name # '%2F' to '/', split https://url.com/file.txt?auth\n if os.path.isfile(file):\n LOGGER.info(f\"Found {url} locally at {file}\") # file already exists\n else:\n LOGGER.info(f\"Downloading {url} to {file}...\")\n torch.hub.download_url_to_file(url, file)\n assert Path(file).exists() and Path(file).stat().st_size > 0, f\"File download failed: {url}\" # check\n return file\n elif file.startswith(\"clearml://\"): # ClearML Dataset ID\n assert (\n \"clearml\" in sys.modules\n ), \"ClearML is not installed, so cannot use ClearML dataset. Try running 'pip install clearml'.\"\n return file\n else: # search\n files = []\n for d in \"data\", \"models\", \"utils\": # search directories\n files.extend(glob.glob(str(ROOT / d / \"**\" / file), recursive=True)) # find file\n assert len(files), f\"File not found: {file}\" # assert file was found\n assert len(files) == 1, f\"Multiple files match '{file}', specify exact path: {files}\" # assert unique\n return files[0] # return file\n\n\ndef check_font(font=FONT, progress=False):\n \"\"\"Ensures specified font exists or downloads it from Ultralytics assets, optionally displaying progress.\"\"\"\n font = Path(font)\n file = CONFIG_DIR / font.name\n if not font.exists() and not file.exists():\n url = f\"https://ultralytics.com/assets/{font.name}\"\n LOGGER.info(f\"Downloading {url} to {file}...\")\n torch.hub.download_url_to_file(url, str(file), progress=progress)\n\n\ndef check_dataset(data, autodownload=True):\n \"\"\"Validates and/or auto-downloads a dataset, returning its configuration as a dictionary.\"\"\"\n\n # Download (optional)\n extract_dir = \"\"\n if isinstance(data, (str, Path)) and (is_zipfile(data) or is_tarfile(data)):\n download(data, dir=f\"{DATASETS_DIR}/{Path(data).stem}\", unzip=True, delete=False, curl=False, threads=1)\n data = next((DATASETS_DIR / Path(data).stem).rglob(\"*.yaml\"))\n extract_dir, autodownload = data.parent, False\n\n # Read yaml (optional)\n if isinstance(data, (str, Path)):\n data = yaml_load(data) # dictionary\n\n # Checks\n for k in \"train\", \"val\", \"names\":\n assert k in data, emojis(f\"data.yaml '{k}:' field missing ❌\")\n if isinstance(data[\"names\"], (list, tuple)): # old array format\n data[\"names\"] = dict(enumerate(data[\"names\"])) # convert to dict\n assert all(isinstance(k, int) for k in data[\"names\"].keys()), \"data.yaml names keys must be integers, i.e. 2: car\"\n data[\"nc\"] = len(data[\"names\"])\n\n # Resolve paths\n path = Path(extract_dir or data.get(\"path\") or \"\") # optional 'path' default to '.'\n if not path.is_absolute():\n path = (ROOT / path).resolve()\n data[\"path\"] = path # download scripts\n for k in \"train\", \"val\", \"test\":\n if data.get(k): # prepend path\n if isinstance(data[k], str):\n x = (path / data[k]).resolve()\n if not x.exists() and data[k].startswith(\"../\"):\n x = (path / data[k][3:]).resolve()\n data[k] = str(x)\n else:\n data[k] = [str((path / x).resolve()) for x in data[k]]\n\n # Parse yaml\n train, val, test, s = (data.get(x) for x in (\"train\", \"val\", \"test\", \"download\"))\n if val:\n val = [Path(x).resolve() for x in (val if isinstance(val, list) else [val])] # val path\n if not all(x.exists() for x in val):\n LOGGER.info(\"\\nDataset not found ⚠️, missing paths %s\" % [str(x) for x in val if not x.exists()])\n if not s or not autodownload:\n raise Exception(\"Dataset not found ❌\")\n t = time.time()\n if s.startswith(\"http\") and s.endswith(\".zip\"): # URL\n f = Path(s).name # filename\n LOGGER.info(f\"Downloading {s} to {f}...\")\n torch.hub.download_url_to_file(s, f)\n Path(DATASETS_DIR).mkdir(parents=True, exist_ok=True) # create root\n unzip_file(f, path=DATASETS_DIR) # unzip\n Path(f).unlink() # remove zip\n r = None # success\n elif s.startswith(\"bash \"): # bash script\n LOGGER.info(f\"Running {s} ...\")\n r = subprocess.run(s, shell=True)\n else: # python script\n r = exec(s, {\"yaml\": data}) # return None\n dt = f\"({round(time.time() - t, 1)}s)\"\n s = f\"success ✅ {dt}, saved to {colorstr('bold', DATASETS_DIR)}\" if r in (0, None) else f\"failure {dt} ❌\"\n LOGGER.info(f\"Dataset download {s}\")\n check_font(\"Arial.ttf\" if is_ascii(data[\"names\"]) else \"Arial.Unicode.ttf\", progress=True) # download fonts\n return data # dictionary\n\n\ndef check_amp(model):\n \"\"\"Checks PyTorch AMP functionality for a model, returns True if AMP operates correctly, otherwise False.\"\"\"\n from asone.detectors.yolov5.yolov5.models.common import AutoShape, DetectMultiBackend\n\n def amp_allclose(model, im):\n # All close FP32 vs AMP results\n m = AutoShape(model, verbose=False) # model\n a = m(im).xywhn[0] # FP32 inference\n m.amp = True\n b = m(im).xywhn[0] # AMP inference\n return a.shape == b.shape and torch.allclose(a, b, atol=0.1) # close to 10% absolute tolerance\n\n prefix = colorstr(\"AMP: \")\n device = next(model.parameters()).device # get model device\n if device.type in (\"cpu\", \"mps\"):\n return False # AMP only used on CUDA devices\n f = ROOT / \"data\" / \"images\" / \"bus.jpg\" # image to check\n im = f if f.exists() else \"https://ultralytics.com/images/bus.jpg\" if check_online() else np.ones((640, 640, 3))\n try:\n assert amp_allclose(deepcopy(model), im) or amp_allclose(DetectMultiBackend(\"yolov5n.pt\", device), im)\n LOGGER.info(f\"{prefix}checks passed ✅\")\n return True\n except Exception:\n help_url = \"https://github.com/ultralytics/yolov5/issues/7908\"\n LOGGER.warning(f\"{prefix}checks failed ❌, disabling Automatic Mixed Precision. See {help_url}\")\n return False\n\n\ndef yaml_load(file=\"data.yaml\"):\n \"\"\"Safely loads and returns the contents of a YAML file specified by `file` argument.\"\"\"\n with open(file, errors=\"ignore\") as f:\n return yaml.safe_load(f)\n\n\ndef yaml_save(file=\"data.yaml\", data={}):\n \"\"\"Safely saves `data` to a YAML file specified by `file`, converting `Path` objects to strings; `data` is a\n dictionary.\n \"\"\"\n with open(file, \"w\") as f:\n yaml.safe_dump({k: str(v) if isinstance(v, Path) else v for k, v in data.items()}, f, sort_keys=False)\n\n\ndef unzip_file(file, path=None, exclude=(\".DS_Store\", \"__MACOSX\")):\n \"\"\"Unzips `file` to `path` (default: file's parent), excluding filenames containing any in `exclude` (`.DS_Store`,\n `__MACOSX`).\n \"\"\"\n if path is None:\n path = Path(file).parent # default path\n with ZipFile(file) as zipObj:\n for f in zipObj.namelist(): # list all archived filenames in the zip\n if all(x not in f for x in exclude):\n zipObj.extract(f, path=path)\n\n\ndef url2file(url):\n \"\"\"\n Converts a URL string to a valid filename by stripping protocol, domain, and any query parameters.\n\n Example https://url.com/file.txt?auth -> file.txt\n \"\"\"\n url = str(Path(url)).replace(\":/\", \"://\") # Pathlib turns :// -> :/\n return Path(urllib.parse.unquote(url)).name.split(\"?\")[0] # '%2F' to '/', split https://url.com/file.txt?auth\n\n\ndef download(url, dir=\".\", unzip=True, delete=True, curl=False, threads=1, retry=3):\n \"\"\"Downloads and optionally unzips files concurrently, supporting retries and curl fallback.\"\"\"\n\n def download_one(url, dir):\n # Download 1 file\n success = True\n if os.path.isfile(url):\n f = Path(url) # filename\n else: # does not exist\n f = dir / Path(url).name\n LOGGER.info(f\"Downloading {url} to {f}...\")\n for i in range(retry + 1):\n if curl:\n success = curl_download(url, f, silent=(threads > 1))\n else:\n torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download\n success = f.is_file()\n if success:\n break\n elif i < retry:\n LOGGER.warning(f\"⚠️ Download failure, retrying {i + 1}/{retry} {url}...\")\n else:\n LOGGER.warning(f\"❌ Failed to download {url}...\")\n\n if unzip and success and (f.suffix == \".gz\" or is_zipfile(f) or is_tarfile(f)):\n LOGGER.info(f\"Unzipping {f}...\")\n if is_zipfile(f):\n unzip_file(f, dir) # unzip\n elif is_tarfile(f):\n subprocess.run([\"tar\", \"xf\", f, \"--directory\", f.parent], check=True) # unzip\n elif f.suffix == \".gz\":\n subprocess.run([\"tar\", \"xfz\", f, \"--directory\", f.parent], check=True) # unzip\n if delete:\n f.unlink() # remove zip\n\n dir = Path(dir)\n dir.mkdir(parents=True, exist_ok=True) # make directory\n if threads > 1:\n pool = ThreadPool(threads)\n pool.imap(lambda x: download_one(*x), zip(url, repeat(dir))) # multithreaded\n pool.close()\n pool.join()\n else:\n for u in [url] if isinstance(url, (str, Path)) else url:\n download_one(u, dir)\n\n\ndef make_divisible(x, divisor):\n \"\"\"Adjusts `x` to be divisible by `divisor`, returning the nearest greater or equal value.\"\"\"\n if isinstance(divisor, torch.Tensor):\n divisor = int(divisor.max()) # to int\n return math.ceil(x / divisor) * divisor\n\n\ndef clean_str(s):\n \"\"\"Cleans a string by replacing special characters with underscore, e.g., `clean_str('#example!')` returns\n '_example_'.\n \"\"\"\n return re.sub(pattern=\"[|@#!¡·$€%&()=?¿^*;:,¨´><+]\", repl=\"_\", string=s)\n\n\ndef one_cycle(y1=0.0, y2=1.0, steps=100):\n \"\"\"\n Generates a lambda for a sinusoidal ramp from y1 to y2 over 'steps'.\n\n See https://arxiv.org/pdf/1812.01187.pdf for details.\n \"\"\"\n return lambda x: ((1 - math.cos(x * math.pi / steps)) / 2) * (y2 - y1) + y1\n\n\ndef colorstr(*input):\n \"\"\"\n Colors a string using ANSI escape codes, e.g., colorstr('blue', 'hello world').\n\n See https://en.wikipedia.org/wiki/ANSI_escape_code.\n \"\"\"\n *args, string = input if len(input) > 1 else (\"blue\", \"bold\", input[0]) # color arguments, string\n colors = {\n \"black\": \"\\033[30m\", # basic colors\n \"red\": \"\\033[31m\",\n \"green\": \"\\033[32m\",\n \"yellow\": \"\\033[33m\",\n \"blue\": \"\\033[34m\",\n \"magenta\": \"\\033[35m\",\n \"cyan\": \"\\033[36m\",\n \"white\": \"\\033[37m\",\n \"bright_black\": \"\\033[90m\", # bright colors\n \"bright_red\": \"\\033[91m\",\n \"bright_green\": \"\\033[92m\",\n \"bright_yellow\": \"\\033[93m\",\n \"bright_blue\": \"\\033[94m\",\n \"bright_magenta\": \"\\033[95m\",\n \"bright_cyan\": \"\\033[96m\",\n \"bright_white\": \"\\033[97m\",\n \"end\": \"\\033[0m\", # misc\n \"bold\": \"\\033[1m\",\n \"underline\": \"\\033[4m\",\n }\n return \"\".join(colors[x] for x in args) + f\"{string}\" + colors[\"end\"]\n\n\ndef labels_to_class_weights(labels, nc=80):\n \"\"\"Calculates class weights from labels to handle class imbalance in training; input shape: (n, 5).\"\"\"\n if labels[0] is None: # no labels loaded\n return torch.Tensor()\n\n labels = np.concatenate(labels, 0) # labels.shape = (866643, 5) for COCO\n classes = labels[:, 0].astype(int) # labels = [class xywh]\n weights = np.bincount(classes, minlength=nc) # occurrences per class\n\n # Prepend gridpoint count (for uCE training)\n # gpi = ((320 / 32 * np.array([1, 2, 4])) ** 2 * 3).sum() # gridpoints per image\n # weights = np.hstack([gpi * len(labels) - weights.sum() * 9, weights * 9]) ** 0.5 # prepend gridpoints to start\n\n weights[weights == 0] = 1 # replace empty bins with 1\n weights = 1 / weights # number of targets per class\n weights /= weights.sum() # normalize\n return torch.from_numpy(weights).float()\n\n\ndef labels_to_image_weights(labels, nc=80, class_weights=np.ones(80)):\n \"\"\"Calculates image weights from labels using class weights for weighted sampling.\"\"\"\n # Usage: index = random.choices(range(n), weights=image_weights, k=1) # weighted image sample\n class_counts = np.array([np.bincount(x[:, 0].astype(int), minlength=nc) for x in labels])\n return (class_weights.reshape(1, nc) * class_counts).sum(1)\n\n\ndef coco80_to_coco91_class():\n \"\"\"\n Converts COCO 80-class index to COCO 91-class index used in the paper.\n\n Reference: https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/\n \"\"\"\n # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\\n')\n # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\\n')\n # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco\n # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet\n return [\n 1,\n 2,\n 3,\n 4,\n 5,\n 6,\n 7,\n 8,\n 9,\n 10,\n 11,\n 13,\n 14,\n 15,\n 16,\n 17,\n 18,\n 19,\n 20,\n 21,\n 22,\n 23,\n 24,\n 25,\n 27,\n 28,\n 31,\n 32,\n 33,\n 34,\n 35,\n 36,\n 37,\n 38,\n 39,\n 40,\n 41,\n 42,\n 43,\n 44,\n 46,\n 47,\n 48,\n 49,\n 50,\n 51,\n 52,\n 53,\n 54,\n 55,\n 56,\n 57,\n 58,\n 59,\n 60,\n 61,\n 62,\n 63,\n 64,\n 65,\n 67,\n 70,\n 72,\n 73,\n 74,\n 75,\n 76,\n 77,\n 78,\n 79,\n 80,\n 81,\n 82,\n 84,\n 85,\n 86,\n 87,\n 88,\n 89,\n 90,\n ]\n\n\ndef xyxy2xywh(x):\n \"\"\"Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right.\"\"\"\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = (x[..., 0] + x[..., 2]) / 2 # x center\n y[..., 1] = (x[..., 1] + x[..., 3]) / 2 # y center\n y[..., 2] = x[..., 2] - x[..., 0] # width\n y[..., 3] = x[..., 3] - x[..., 1] # height\n return y\n\n\ndef xywh2xyxy(x):\n \"\"\"Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right.\"\"\"\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = x[..., 0] - x[..., 2] / 2 # top left x\n y[..., 1] = x[..., 1] - x[..., 3] / 2 # top left y\n y[..., 2] = x[..., 0] + x[..., 2] / 2 # bottom right x\n y[..., 3] = x[..., 1] + x[..., 3] / 2 # bottom right y\n return y\n\n\ndef xywhn2xyxy(x, w=640, h=640, padw=0, padh=0):\n \"\"\"Convert nx4 boxes from [x, y, w, h] normalized to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right.\"\"\"\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = w * (x[..., 0] - x[..., 2] / 2) + padw # top left x\n y[..., 1] = h * (x[..., 1] - x[..., 3] / 2) + padh # top left y\n y[..., 2] = w * (x[..., 0] + x[..., 2] / 2) + padw # bottom right x\n y[..., 3] = h * (x[..., 1] + x[..., 3] / 2) + padh # bottom right y\n return y\n\n\ndef xyxy2xywhn(x, w=640, h=640, clip=False, eps=0.0):\n \"\"\"Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] normalized where xy1=top-left, xy2=bottom-right.\"\"\"\n if clip:\n clip_boxes(x, (h - eps, w - eps)) # warning: inplace clip\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = ((x[..., 0] + x[..., 2]) / 2) / w # x center\n y[..., 1] = ((x[..., 1] + x[..., 3]) / 2) / h # y center\n y[..., 2] = (x[..., 2] - x[..., 0]) / w # width\n y[..., 3] = (x[..., 3] - x[..., 1]) / h # height\n return y\n\n\ndef xyn2xy(x, w=640, h=640, padw=0, padh=0):\n \"\"\"Convert normalized segments into pixel segments, shape (n,2).\"\"\"\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = w * x[..., 0] + padw # top left x\n y[..., 1] = h * x[..., 1] + padh # top left y\n return y\n\n\ndef segment2box(segment, width=640, height=640):\n \"\"\"Convert 1 segment label to 1 box label, applying inside-image constraint, i.e. (xy1, xy2, ...) to (xyxy).\"\"\"\n x, y = segment.T # segment xy\n inside = (x >= 0) & (y >= 0) & (x <= width) & (y <= height)\n (\n x,\n y,\n ) = x[inside], y[inside]\n return np.array([x.min(), y.min(), x.max(), y.max()]) if any(x) else np.zeros((1, 4)) # xyxy\n\n\ndef segments2boxes(segments):\n \"\"\"Convert segment labels to box labels, i.e. (cls, xy1, xy2, ...) to (cls, xywh).\"\"\"\n boxes = []\n for s in segments:\n x, y = s.T # segment xy\n boxes.append([x.min(), y.min(), x.max(), y.max()]) # cls, xyxy\n return xyxy2xywh(np.array(boxes)) # cls, xywh\n\n\ndef resample_segments(segments, n=1000):\n \"\"\"Resamples an (n,2) segment to a fixed number of points for consistent representation.\"\"\"\n for i, s in enumerate(segments):\n s = np.concatenate((s, s[0:1, :]), axis=0)\n x = np.linspace(0, len(s) - 1, n)\n xp = np.arange(len(s))\n segments[i] = np.concatenate([np.interp(x, xp, s[:, i]) for i in range(2)]).reshape(2, -1).T # segment xy\n return segments\n\n\ndef scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None):\n \"\"\"Rescales (xyxy) bounding boxes from img1_shape to img0_shape, optionally using provided `ratio_pad`.\"\"\"\n if ratio_pad is None: # calculate from img0_shape\n gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new\n pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding\n else:\n gain = ratio_pad[0][0]\n pad = ratio_pad[1]\n\n boxes[..., [0, 2]] -= pad[0] # x padding\n boxes[..., [1, 3]] -= pad[1] # y padding\n boxes[..., :4] /= gain\n clip_boxes(boxes, img0_shape)\n return boxes\n\n\ndef scale_segments(img1_shape, segments, img0_shape, ratio_pad=None, normalize=False):\n \"\"\"Rescales segment coordinates from img1_shape to img0_shape, optionally normalizing them with custom padding.\"\"\"\n if ratio_pad is None: # calculate from img0_shape\n gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new\n pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding\n else:\n gain = ratio_pad[0][0]\n pad = ratio_pad[1]\n\n segments[:, 0] -= pad[0] # x padding\n segments[:, 1] -= pad[1] # y padding\n segments /= gain\n clip_segments(segments, img0_shape)\n if normalize:\n segments[:, 0] /= img0_shape[1] # width\n segments[:, 1] /= img0_shape[0] # height\n return segments\n\n\ndef clip_boxes(boxes, shape):\n \"\"\"Clips bounding box coordinates (xyxy) to fit within the specified image shape (height, width).\"\"\"\n if isinstance(boxes, torch.Tensor): # faster individually\n boxes[..., 0].clamp_(0, shape[1]) # x1\n boxes[..., 1].clamp_(0, shape[0]) # y1\n boxes[..., 2].clamp_(0, shape[1]) # x2\n boxes[..., 3].clamp_(0, shape[0]) # y2\n else: # np.array (faster grouped)\n boxes[..., [0, 2]] = boxes[..., [0, 2]].clip(0, shape[1]) # x1, x2\n boxes[..., [1, 3]] = boxes[..., [1, 3]].clip(0, shape[0]) # y1, y2\n\n\ndef clip_segments(segments, shape):\n \"\"\"Clips segment coordinates (xy1, xy2, ...) to an image's boundaries given its shape (height, width).\"\"\"\n if isinstance(segments, torch.Tensor): # faster individually\n segments[:, 0].clamp_(0, shape[1]) # x\n segments[:, 1].clamp_(0, shape[0]) # y\n else: # np.array (faster grouped)\n segments[:, 0] = segments[:, 0].clip(0, shape[1]) # x\n segments[:, 1] = segments[:, 1].clip(0, shape[0]) # y\n\n\ndef non_max_suppression(\n prediction,\n conf_thres=0.25,\n iou_thres=0.45,\n classes=None,\n agnostic=False,\n multi_label=False,\n labels=(),\n max_det=300,\n nm=0, # number of masks\n):\n \"\"\"\n Non-Maximum Suppression (NMS) on inference results to reject overlapping detections.\n\n Returns:\n list of detections, on (n,6) tensor per image [xyxy, conf, cls]\n \"\"\"\n\n # Checks\n assert 0 <= conf_thres <= 1, f\"Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0\"\n assert 0 <= iou_thres <= 1, f\"Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0\"\n if isinstance(prediction, (list, tuple)): # YOLOv5 model in validation model, output = (inference_out, loss_out)\n prediction = prediction[0] # select only inference output\n\n device = prediction.device\n mps = \"mps\" in device.type # Apple MPS\n if mps: # MPS not fully supported yet, convert tensors to CPU before NMS\n prediction = prediction.cpu()\n bs = prediction.shape[0] # batch size\n nc = prediction.shape[2] - nm - 5 # number of classes\n xc = prediction[..., 4] > conf_thres # candidates\n\n # Settings\n # min_wh = 2 # (pixels) minimum box width and height\n max_wh = 7680 # (pixels) maximum box width and height\n max_nms = 30000 # maximum number of boxes into torchvision.ops.nms()\n time_limit = 0.5 + 0.05 * bs # seconds to quit after\n redundant = True # require redundant detections\n multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img)\n merge = False # use merge-NMS\n\n t = time.time()\n mi = 5 + nc # mask start index\n output = [torch.zeros((0, 6 + nm), device=prediction.device)] * bs\n for xi, x in enumerate(prediction): # image index, image inference\n # Apply constraints\n # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height\n x = x[xc[xi]] # confidence\n\n # Cat apriori labels if autolabelling\n if labels and len(labels[xi]):\n lb = labels[xi]\n v = torch.zeros((len(lb), nc + nm + 5), device=x.device)\n v[:, :4] = lb[:, 1:5] # box\n v[:, 4] = 1.0 # conf\n v[range(len(lb)), lb[:, 0].long() + 5] = 1.0 # cls\n x = torch.cat((x, v), 0)\n\n # If none remain process next image\n if not x.shape[0]:\n continue\n\n # Compute conf\n x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf\n\n # Box/Mask\n box = xywh2xyxy(x[:, :4]) # center_x, center_y, width, height) to (x1, y1, x2, y2)\n mask = x[:, mi:] # zero columns if no masks\n\n # Detections matrix nx6 (xyxy, conf, cls)\n if multi_label:\n i, j = (x[:, 5:mi] > conf_thres).nonzero(as_tuple=False).T\n x = torch.cat((box[i], x[i, 5 + j, None], j[:, None].float(), mask[i]), 1)\n else: # best class only\n conf, j = x[:, 5:mi].max(1, keepdim=True)\n x = torch.cat((box, conf, j.float(), mask), 1)[conf.view(-1) > conf_thres]\n\n # Filter by class\n if classes is not None:\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\n\n # Apply finite constraint\n # if not torch.isfinite(x).all():\n # x = x[torch.isfinite(x).all(1)]\n\n # Check shape\n n = x.shape[0] # number of boxes\n if not n: # no boxes\n continue\n x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence and remove excess boxes\n\n # Batched NMS\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\n i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS\n i = i[:max_det] # limit detections\n if merge and (1 < n < 3e3): # Merge NMS (boxes merged using weighted mean)\n # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\n weights = iou * scores[None] # box weights\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\n if redundant:\n i = i[iou.sum(1) > 1] # require redundancy\n\n output[xi] = x[i]\n if mps:\n output[xi] = output[xi].to(device)\n if (time.time() - t) > time_limit:\n LOGGER.warning(f\"WARNING ⚠️ NMS time limit {time_limit:.3f}s exceeded\")\n break # time limit exceeded\n\n return output\n\n\ndef strip_optimizer(f=\"best.pt\", s=\"\"):\n \"\"\"\n Strips optimizer and optionally saves checkpoint to finalize training; arguments are file path 'f' and save path\n 's'.\n\n Example: from utils.general import *; strip_optimizer()\n \"\"\"\n x = torch.load(f, map_location=torch.device(\"cpu\"))\n if x.get(\"ema\"):\n x[\"model\"] = x[\"ema\"] # replace model with ema\n for k in \"optimizer\", \"best_fitness\", \"ema\", \"updates\": # keys\n x[k] = None\n x[\"epoch\"] = -1\n x[\"model\"].half() # to FP16\n for p in x[\"model\"].parameters():\n p.requires_grad = False\n torch.save(x, s or f)\n mb = os.path.getsize(s or f) / 1e6 # filesize\n LOGGER.info(f\"Optimizer stripped from {f},{f' saved as {s},' if s else ''} {mb:.1f}MB\")\n\n\ndef print_mutation(keys, results, hyp, save_dir, bucket, prefix=colorstr(\"evolve: \")):\n \"\"\"Logs evolution results and saves to CSV and YAML in `save_dir`, optionally syncs with `bucket`.\"\"\"\n evolve_csv = save_dir / \"evolve.csv\"\n evolve_yaml = save_dir / \"hyp_evolve.yaml\"\n keys = tuple(keys) + tuple(hyp.keys()) # [results + hyps]\n keys = tuple(x.strip() for x in keys)\n vals = results + tuple(hyp.values())\n n = len(keys)\n\n # Download (optional)\n if bucket:\n url = f\"gs://{bucket}/evolve.csv\"\n if gsutil_getsize(url) > (evolve_csv.stat().st_size if evolve_csv.exists() else 0):\n subprocess.run([\"gsutil\", \"cp\", f\"{url}\", f\"{save_dir}\"]) # download evolve.csv if larger than local\n\n # Log to evolve.csv\n s = \"\" if evolve_csv.exists() else ((\"%20s,\" * n % keys).rstrip(\",\") + \"\\n\") # add header\n with open(evolve_csv, \"a\") as f:\n f.write(s + (\"%20.5g,\" * n % vals).rstrip(\",\") + \"\\n\")\n\n # Save yaml\n with open(evolve_yaml, \"w\") as f:\n data = pd.read_csv(evolve_csv, skipinitialspace=True)\n data = data.rename(columns=lambda x: x.strip()) # strip keys\n i = np.argmax(fitness(data.values[:, :4])) #\n generations = len(data)\n f.write(\n \"# YOLOv5 Hyperparameter Evolution Results\\n\"\n + f\"# Best generation: {i}\\n\"\n + f\"# Last generation: {generations - 1}\\n\"\n + \"# \"\n + \", \".join(f\"{x.strip():>20s}\" for x in keys[:7])\n + \"\\n\"\n + \"# \"\n + \", \".join(f\"{x:>20.5g}\" for x in data.values[i, :7])\n + \"\\n\\n\"\n )\n yaml.safe_dump(data.loc[i][7:].to_dict(), f, sort_keys=False)\n\n # Print to screen\n LOGGER.info(\n prefix\n + f\"{generations} generations finished, current result:\\n\"\n + prefix\n + \", \".join(f\"{x.strip():>20s}\" for x in keys)\n + \"\\n\"\n + prefix\n + \", \".join(f\"{x:20.5g}\" for x in vals)\n + \"\\n\\n\"\n )\n\n if bucket:\n subprocess.run([\"gsutil\", \"cp\", f\"{evolve_csv}\", f\"{evolve_yaml}\", f\"gs://{bucket}\"]) # upload\n\n\ndef apply_classifier(x, model, img, im0):\n \"\"\"Applies second-stage classifier to YOLO outputs, filtering detections by class match.\"\"\"\n # Example model = torchvision.models.__dict__['efficientnet_b0'](pretrained=True).to(device).eval()\n im0 = [im0] if isinstance(im0, np.ndarray) else im0\n for i, d in enumerate(x): # per image\n if d is not None and len(d):\n d = d.clone()\n\n # Reshape and pad cutouts\n b = xyxy2xywh(d[:, :4]) # boxes\n b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # rectangle to square\n b[:, 2:] = b[:, 2:] * 1.3 + 30 # pad\n d[:, :4] = xywh2xyxy(b).long()\n\n # Rescale boxes from img_size to im0 size\n scale_boxes(img.shape[2:], d[:, :4], im0[i].shape)\n\n # Classes\n pred_cls1 = d[:, 5].long()\n ims = []\n for a in d:\n cutout = im0[i][int(a[1]) : int(a[3]), int(a[0]) : int(a[2])]\n im = cv2.resize(cutout, (224, 224)) # BGR\n\n im = im[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n im = np.ascontiguousarray(im, dtype=np.float32) # uint8 to float32\n im /= 255 # 0 - 255 to 0.0 - 1.0\n ims.append(im)\n\n pred_cls2 = model(torch.Tensor(ims).to(d.device)).argmax(1) # classifier prediction\n x[i] = x[i][pred_cls1 == pred_cls2] # retain matching class detections\n\n return x\n\n\ndef increment_path(path, exist_ok=False, sep=\"\", mkdir=False):\n \"\"\"\n Generates an incremented file or directory path if it exists, with optional mkdir; args: path, exist_ok=False,\n sep=\"\", mkdir=False.\n\n Example: runs/exp --> runs/exp{sep}2, runs/exp{sep}3, ... etc\n \"\"\"\n path = Path(path) # os-agnostic\n if path.exists() and not exist_ok:\n path, suffix = (path.with_suffix(\"\"), path.suffix) if path.is_file() else (path, \"\")\n\n # Method 1\n for n in range(2, 9999):\n p = f\"{path}{sep}{n}{suffix}\" # increment path\n if not os.path.exists(p): #\n break\n path = Path(p)\n\n # Method 2 (deprecated)\n # dirs = glob.glob(f\"{path}{sep}*\") # similar paths\n # matches = [re.search(rf\"{path.stem}{sep}(\\d+)\", d) for d in dirs]\n # i = [int(m.groups()[0]) for m in matches if m] # indices\n # n = max(i) + 1 if i else 2 # increment number\n # path = Path(f\"{path}{sep}{n}{suffix}\") # increment path\n\n if mkdir:\n path.mkdir(parents=True, exist_ok=True) # make directory\n\n return path\n\n\n# OpenCV Multilanguage-friendly functions ------------------------------------------------------------------------------------\nimshow_ = cv2.imshow # copy to avoid recursion errors\n\n\ndef imread(filename, flags=cv2.IMREAD_COLOR):\n \"\"\"Reads an image from a file and returns it as a numpy array, using OpenCV's imdecode to support multilanguage\n paths.\n \"\"\"\n return cv2.imdecode(np.fromfile(filename, np.uint8), flags)\n\n\ndef imwrite(filename, img):\n \"\"\"Writes an image to a file, returns True on success and False on failure, supports multilanguage paths.\"\"\"\n try:\n cv2.imencode(Path(filename).suffix, img)[1].tofile(filename)\n return True\n except Exception:\n return False\n\n\ndef imshow(path, im):\n \"\"\"Displays an image using Unicode path, requires encoded path and image matrix as input.\"\"\"\n imshow_(path.encode(\"unicode_escape\").decode(), im)\n\n\nif Path(inspect.stack()[0].filename).parent.parent.as_posix() in inspect.stack()[-1].filename:\n cv2.imread, cv2.imwrite, cv2.imshow = imread, imwrite, imshow # redefine\n\n# Variables ------------------------------------------------------------------------------------------------------------" }, { "path": "asone/detectors/yolov5/yolov5/utils/metrics.py", "content": "# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license\n\"\"\"Model validation metrics.\"\"\"\n\nimport math\nimport warnings\nfrom pathlib import Path\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\n\nfrom asone.detectors.yolov5.yolov5.utils import TryExcept, threaded\n\n\ndef fitness(x):\n \"\"\"Calculates fitness of a model using weighted sum of metrics P, R, mAP@0.5, mAP@0.5:0.95.\"\"\"\n w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]\n return (x[:, :4] * w).sum(1)\n\n\ndef smooth(y, f=0.05):\n \"\"\"Applies box filter smoothing to array `y` with fraction `f`, yielding a smoothed array.\"\"\"\n nf = round(len(y) * f * 2) // 2 + 1 # number of filter elements (must be odd)\n p = np.ones(nf // 2) # ones padding\n yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded\n return np.convolve(yp, np.ones(nf) / nf, mode=\"valid\") # y-smoothed\n\n\ndef ap_per_class(tp, conf, pred_cls, target_cls, plot=False, save_dir=\".\", names=(), eps=1e-16, prefix=\"\"):\n \"\"\"\n Compute the average precision, given the recall and precision curves.\n\n Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.\n # Arguments\n tp: True positives (nparray, nx1 or nx10).\n conf: Objectness value from 0-1 (nparray).\n pred_cls: Predicted object classes (nparray).\n target_cls: True object classes (nparray).\n plot: Plot precision-recall curve at mAP@0.5\n save_dir: Plot save directory\n # Returns\n The average precision as computed in py-faster-rcnn.\n \"\"\"\n\n # Sort by objectness\n i = np.argsort(-conf)\n tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]\n\n # Find unique classes\n unique_classes, nt = np.unique(target_cls, return_counts=True)\n nc = unique_classes.shape[0] # number of classes, number of detections\n\n # Create Precision-Recall curve and compute AP for each class\n px, py = np.linspace(0, 1, 1000), [] # for plotting\n ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000))\n for ci, c in enumerate(unique_classes):\n i = pred_cls == c\n n_l = nt[ci] # number of labels\n n_p = i.sum() # number of predictions\n if n_p == 0 or n_l == 0:\n continue\n\n # Accumulate FPs and TPs\n fpc = (1 - tp[i]).cumsum(0)\n tpc = tp[i].cumsum(0)\n\n # Recall\n recall = tpc / (n_l + eps) # recall curve\n r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases\n\n # Precision\n precision = tpc / (tpc + fpc) # precision curve\n p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score\n\n # AP from recall-precision curve\n for j in range(tp.shape[1]):\n ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j])\n if plot and j == 0:\n py.append(np.interp(px, mrec, mpre)) # precision at mAP@0.5\n\n # Compute F1 (harmonic mean of precision and recall)\n f1 = 2 * p * r / (p + r + eps)\n names = [v for k, v in names.items() if k in unique_classes] # list: only classes that have data\n names = dict(enumerate(names)) # to dict\n if plot:\n plot_pr_curve(px, py, ap, Path(save_dir) / f\"{prefix}PR_curve.png\", names)\n plot_mc_curve(px, f1, Path(save_dir) / f\"{prefix}F1_curve.png\", names, ylabel=\"F1\")\n plot_mc_curve(px, p, Path(save_dir) / f\"{prefix}P_curve.png\", names, ylabel=\"Precision\")\n plot_mc_curve(px, r, Path(save_dir) / f\"{prefix}R_curve.png\", names, ylabel=\"Recall\")\n\n i = smooth(f1.mean(0), 0.1).argmax() # max F1 index\n p, r, f1 = p[:, i], r[:, i], f1[:, i]\n tp = (r * nt).round() # true positives\n fp = (tp / (p + eps) - tp).round() # false positives\n return tp, fp, p, r, f1, ap, unique_classes.astype(int)\n\n\ndef compute_ap(recall, precision):\n \"\"\"Compute the average precision, given the recall and precision curves\n # Arguments\n recall: The recall curve (list)\n precision: The precision curve (list)\n # Returns\n Average precision, precision curve, recall curve\n \"\"\"\n\n # Append sentinel values to beginning and end\n mrec = np.concatenate(([0.0], recall, [1.0]))\n mpre = np.concatenate(([1.0], precision, [0.0]))\n\n # Compute the precision envelope\n mpre = np.flip(np.maximum.accumulate(np.flip(mpre)))\n\n # Integrate area under curve\n method = \"interp\" # methods: 'continuous', 'interp'\n if method == \"interp\":\n x = np.linspace(0, 1, 101) # 101-point interp (COCO)\n ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate\n else: # 'continuous'\n i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes\n ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve\n\n return ap, mpre, mrec\n\n\nclass ConfusionMatrix:\n # Updated version of https://github.com/kaanakan/object_detection_confusion_matrix\n def __init__(self, nc, conf=0.25, iou_thres=0.45):\n \"\"\"Initializes ConfusionMatrix with given number of classes, confidence, and IoU threshold.\"\"\"\n self.matrix = np.zeros((nc + 1, nc + 1))\n self.nc = nc # number of classes\n self.conf = conf\n self.iou_thres = iou_thres\n\n def process_batch(self, detections, labels):\n \"\"\"\n Return intersection-over-union (Jaccard index) of boxes.\n\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\n Arguments:\n detections (Array[N, 6]), x1, y1, x2, y2, conf, class\n labels (Array[M, 5]), class, x1, y1, x2, y2\n Returns:\n None, updates confusion matrix accordingly\n \"\"\"\n if detections is None:\n gt_classes = labels.int()\n for gc in gt_classes:\n self.matrix[self.nc, gc] += 1 # background FN\n return\n\n detections = detections[detections[:, 4] > self.conf]\n gt_classes = labels[:, 0].int()\n detection_classes = detections[:, 5].int()\n iou = box_iou(labels[:, 1:], detections[:, :4])\n\n x = torch.where(iou > self.iou_thres)\n if x[0].shape[0]:\n matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy()\n if x[0].shape[0] > 1:\n matches = matches[matches[:, 2].argsort()[::-1]]\n matches = matches[np.unique(matches[:, 1], return_index=True)[1]]\n matches = matches[matches[:, 2].argsort()[::-1]]\n matches = matches[np.unique(matches[:, 0], return_index=True)[1]]\n else:\n matches = np.zeros((0, 3))\n\n n = matches.shape[0] > 0\n m0, m1, _ = matches.transpose().astype(int)\n for i, gc in enumerate(gt_classes):\n j = m0 == i\n if n and sum(j) == 1:\n self.matrix[detection_classes[m1[j]], gc] += 1 # correct\n else:\n self.matrix[self.nc, gc] += 1 # true background\n\n if n:\n for i, dc in enumerate(detection_classes):\n if not any(m1 == i):\n self.matrix[dc, self.nc] += 1 # predicted background\n\n def tp_fp(self):\n \"\"\"Calculates true positives (tp) and false positives (fp) excluding the background class from the confusion\n matrix.\n \"\"\"\n tp = self.matrix.diagonal() # true positives\n fp = self.matrix.sum(1) - tp # false positives\n # fn = self.matrix.sum(0) - tp # false negatives (missed detections)\n return tp[:-1], fp[:-1] # remove background class\n\n @TryExcept(\"WARNING ⚠️ ConfusionMatrix plot failure\")\n def plot(self, normalize=True, save_dir=\"\", names=()):\n \"\"\"Plots confusion matrix using seaborn, optional normalization; can save plot to specified directory.\"\"\"\n import seaborn as sn\n\n array = self.matrix / ((self.matrix.sum(0).reshape(1, -1) + 1e-9) if normalize else 1) # normalize columns\n array[array < 0.005] = np.nan # don't annotate (would appear as 0.00)\n\n fig, ax = plt.subplots(1, 1, figsize=(12, 9), tight_layout=True)\n nc, nn = self.nc, len(names) # number of classes, names\n sn.set(font_scale=1.0 if nc < 50 else 0.8) # for label size\n labels = (0 < nn < 99) and (nn == nc) # apply names to ticklabels\n ticklabels = (names + [\"background\"]) if labels else \"auto\"\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\") # suppress empty matrix RuntimeWarning: All-NaN slice encountered\n sn.heatmap(\n array,\n ax=ax,\n annot=nc < 30,\n annot_kws={\"size\": 8},\n cmap=\"Blues\",\n fmt=\".2f\",\n square=True,\n vmin=0.0,\n xticklabels=ticklabels,\n yticklabels=ticklabels,\n ).set_facecolor((1, 1, 1))\n ax.set_xlabel(\"True\")\n ax.set_ylabel(\"Predicted\")\n ax.set_title(\"Confusion Matrix\")\n fig.savefig(Path(save_dir) / \"confusion_matrix.png\", dpi=250)\n plt.close(fig)\n\n def print(self):\n \"\"\"Prints the confusion matrix row-wise, with each class and its predictions separated by spaces.\"\"\"\n for i in range(self.nc + 1):\n print(\" \".join(map(str, self.matrix[i])))\n\n\ndef bbox_iou(box1, box2, xywh=True, GIoU=False, DIoU=False, CIoU=False, eps=1e-7):\n \"\"\"\n Calculates IoU, GIoU, DIoU, or CIoU between two boxes, supporting xywh/xyxy formats.\n\n Input shapes are box1(1,4) to box2(n,4).\n \"\"\"\n\n # Get the coordinates of bounding boxes\n if xywh: # transform from xywh to xyxy\n (x1, y1, w1, h1), (x2, y2, w2, h2) = box1.chunk(4, -1), box2.chunk(4, -1)\n w1_, h1_, w2_, h2_ = w1 / 2, h1 / 2, w2 / 2, h2 / 2\n b1_x1, b1_x2, b1_y1, b1_y2 = x1 - w1_, x1 + w1_, y1 - h1_, y1 + h1_\n b2_x1, b2_x2, b2_y1, b2_y2 = x2 - w2_, x2 + w2_, y2 - h2_, y2 + h2_\n else: # x1, y1, x2, y2 = box1\n b1_x1, b1_y1, b1_x2, b1_y2 = box1.chunk(4, -1)\n b2_x1, b2_y1, b2_x2, b2_y2 = box2.chunk(4, -1)\n w1, h1 = b1_x2 - b1_x1, (b1_y2 - b1_y1).clamp(eps)\n w2, h2 = b2_x2 - b2_x1, (b2_y2 - b2_y1).clamp(eps)\n\n # Intersection area\n inter = (b1_x2.minimum(b2_x2) - b1_x1.maximum(b2_x1)).clamp(0) * (\n b1_y2.minimum(b2_y2) - b1_y1.maximum(b2_y1)\n ).clamp(0)\n\n # Union Area\n union = w1 * h1 + w2 * h2 - inter + eps\n\n # IoU\n iou = inter / union\n if CIoU or DIoU or GIoU:\n cw = b1_x2.maximum(b2_x2) - b1_x1.minimum(b2_x1) # convex (smallest enclosing box) width\n ch = b1_y2.maximum(b2_y2) - b1_y1.minimum(b2_y1) # convex height\n if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1\n c2 = cw**2 + ch**2 + eps # convex diagonal squared\n rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 + (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center dist ** 2\n if CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47\n v = (4 / math.pi**2) * (torch.atan(w2 / h2) - torch.atan(w1 / h1)).pow(2)\n with torch.no_grad():\n alpha = v / (v - iou + (1 + eps))\n return iou - (rho2 / c2 + v * alpha) # CIoU\n return iou - rho2 / c2 # DIoU\n c_area = cw * ch + eps # convex area\n return iou - (c_area - union) / c_area # GIoU https://arxiv.org/pdf/1902.09630.pdf\n return iou # IoU\n\n\ndef box_iou(box1, box2, eps=1e-7):\n # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py\n \"\"\"\n Return intersection-over-union (Jaccard index) of boxes.\n\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\n Arguments:\n box1 (Tensor[N, 4])\n box2 (Tensor[M, 4])\n Returns:\n iou (Tensor[N, M]): the NxM matrix containing the pairwise\n IoU values for every element in boxes1 and boxes2\n \"\"\"\n\n # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)\n (a1, a2), (b1, b2) = box1.unsqueeze(1).chunk(2, 2), box2.unsqueeze(0).chunk(2, 2)\n inter = (torch.min(a2, b2) - torch.max(a1, b1)).clamp(0).prod(2)\n\n # IoU = inter / (area1 + area2 - inter)\n return inter / ((a2 - a1).prod(2) + (b2 - b1).prod(2) - inter + eps)\n\n\ndef bbox_ioa(box1, box2, eps=1e-7):\n \"\"\"\n Returns the intersection over box2 area given box1, box2.\n\n Boxes are x1y1x2y2\n box1: np.array of shape(4)\n box2: np.array of shape(nx4)\n returns: np.array of shape(n)\n \"\"\"\n\n # Get the coordinates of bounding boxes\n b1_x1, b1_y1, b1_x2, b1_y2 = box1\n b2_x1, b2_y1, b2_x2, b2_y2 = box2.T\n\n # Intersection area\n inter_area = (np.minimum(b1_x2, b2_x2) - np.maximum(b1_x1, b2_x1)).clip(0) * (\n np.minimum(b1_y2, b2_y2) - np.maximum(b1_y1, b2_y1)\n ).clip(0)\n\n # box2 area\n box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + eps\n\n # Intersection over box2 area\n return inter_area / box2_area\n\n\ndef wh_iou(wh1, wh2, eps=1e-7):\n \"\"\"Calculates the Intersection over Union (IoU) for two sets of widths and heights; `wh1` and `wh2` should be nx2\n and mx2 tensors.\n \"\"\"\n wh1 = wh1[:, None] # [N,1,2]\n wh2 = wh2[None] # [1,M,2]\n inter = torch.min(wh1, wh2).prod(2) # [N,M]\n return inter / (wh1.prod(2) + wh2.prod(2) - inter + eps) # iou = inter / (area1 + area2 - inter)\n\n\n# Plots ----------------------------------------------------------------------------------------------------------------\n\n\n@threaded\ndef plot_pr_curve(px, py, ap, save_dir=Path(\"pr_curve.png\"), names=()):\n \"\"\"Plots precision-recall curve, optionally per class, saving to `save_dir`; `px`, `py` are lists, `ap` is Nx2\n array, `names` optional.\n \"\"\"\n fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)\n py = np.stack(py, axis=1)\n\n if 0 < len(names) < 21: # display per-class legend if < 21 classes\n for i, y in enumerate(py.T):\n ax.plot(px, y, linewidth=1, label=f\"{names[i]} {ap[i, 0]:.3f}\") # plot(recall, precision)\n else:\n ax.plot(px, py, linewidth=1, color=\"grey\") # plot(recall, precision)\n\n ax.plot(px, py.mean(1), linewidth=3, color=\"blue\", label=\"all classes %.3f mAP@0.5\" % ap[:, 0].mean())\n ax.set_xlabel(\"Recall\")\n ax.set_ylabel(\"Precision\")\n ax.set_xlim(0, 1)\n ax.set_ylim(0, 1)\n ax.legend(bbox_to_anchor=(1.04, 1), loc=\"upper left\")\n ax.set_title(\"Precision-Recall Curve\")\n fig.savefig(save_dir, dpi=250)\n plt.close(fig)\n\n\n@threaded\ndef plot_mc_curve(px, py, save_dir=Path(\"mc_curve.png\"), names=(), xlabel=\"Confidence\", ylabel=\"Metric\"):\n \"\"\"Plots a metric-confidence curve for model predictions, supporting per-class visualization and smoothing.\"\"\"\n fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)\n\n if 0 < len(names) < 21: # display per-class legend if < 21 classes\n for i, y in enumerate(py):\n ax.plot(px, y, linewidth=1, label=f\"{names[i]}\") # plot(confidence, metric)\n else:\n ax.plot(px, py.T, linewidth=1, color=\"grey\") # plot(confidence, metric)\n\n y = smooth(py.mean(0), 0.05)\n ax.plot(px, y, linewidth=3, color=\"blue\", label=f\"all classes {y.max():.2f} at {px[y.argmax()]:.3f}\")\n ax.set_xlabel(xlabel)\n ax.set_ylabel(ylabel)\n ax.set_xlim(0, 1)\n ax.set_ylim(0, 1)\n ax.legend(bbox_to_anchor=(1.04, 1), loc=\"upper left\")\n ax.set_title(f\"{ylabel}-Confidence Curve\")\n fig.savefig(save_dir, dpi=250)\n plt.close(fig)" }, { "path": "asone/detectors/yolov5/yolov5/utils/torch_utils.py", "content": "# YOLOv5 🚀 by Ultralytics, GPL-3.0 license\n\"\"\"\nPyTorch utils\n\"\"\"\n\nimport math\nimport os\nimport platform\nimport subprocess\nimport time\nimport warnings\nfrom contextlib import contextmanager\nfrom copy import deepcopy\nfrom pathlib import Path\n\nimport torch\nimport torch.distributed as dist\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn.parallel import DistributedDataParallel as DDP\n\n# from utils.general import LOGGER, check_version, colorstr, file_date, git_describe\n\nLOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html\nRANK = int(os.getenv('RANK', -1))\nWORLD_SIZE = int(os.getenv('WORLD_SIZE', 1))\n\ntry:\n import thop # for FLOPs computation\nexcept ImportError:\n thop = None\n\n# Suppress PyTorch warnings\nwarnings.filterwarnings('ignore', message='User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n\n\ndef smart_DDP(model):\n # Model DDP creation with checks\n assert not check_version(torch.__version__, '1.12.0', pinned=True), \\\n 'torch==1.12.0 torchvision==0.13.0 DDP training is not supported due to a known issue. ' \\\n 'Please upgrade or downgrade torch to use DDP. See https://github.com/ultralytics/yolov5/issues/8395'\n if check_version(torch.__version__, '1.11.0'):\n return DDP(model, device_ids=[LOCAL_RANK], output_device=LOCAL_RANK, static_graph=True)\n else:\n return DDP(model, device_ids=[LOCAL_RANK], output_device=LOCAL_RANK)\n\n\n@contextmanager\ndef torch_distributed_zero_first(local_rank: int):\n # Decorator to make all processes in distributed training wait for each local_master to do something\n if local_rank not in [-1, 0]:\n dist.barrier(device_ids=[local_rank])\n yield\n if local_rank == 0:\n dist.barrier(device_ids=[0])\n\n\ndef device_count():\n # Returns number of CUDA devices available. Safe version of torch.cuda.device_count(). Supports Linux and Windows\n assert platform.system() in ('Linux', 'Windows'), 'device_count() only supported on Linux or Windows'\n try:\n cmd = 'nvidia-smi -L | wc -l' if platform.system() == 'Linux' else 'nvidia-smi -L | find /c /v \"\"' # Windows\n return int(subprocess.run(cmd, shell=True, capture_output=True, check=True).stdout.decode().split()[-1])\n except Exception:\n return 0\n\n\ndef select_device(device='', batch_size=0, newline=True):\n # device = None or 'cpu' or 0 or '0' or '0,1,2,3'\n s = f'YOLOv5 🚀 {git_describe() or file_date()} Python-{platform.python_version()} torch-{torch.__version__} '\n device = str(device).strip().lower().replace('cuda:', '').replace('none', '') # to string, 'cuda:0' to '0'\n cpu = device == 'cpu'\n mps = device == 'mps' # Apple Metal Performance Shaders (MPS)\n if cpu or mps:\n os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # force torch.cuda.is_available() = False\n elif device: # non-cpu device requested\n os.environ['CUDA_VISIBLE_DEVICES'] = device # set environment variable - must be before assert is_available()\n assert torch.cuda.is_available() and torch.cuda.device_count() >= len(device.replace(',', '')), \\\n f\"Invalid CUDA '--device {device}' requested, use '--device cpu' or pass valid CUDA device(s)\"\n\n if not (cpu or mps) and torch.cuda.is_available(): # prefer GPU if available\n devices = device.split(',') if device else '0' # range(torch.cuda.device_count()) # i.e. 0,1,6,7\n n = len(devices) # device count\n if n > 1 and batch_size > 0: # check batch_size is divisible by device_count\n assert batch_size % n == 0, f'batch-size {batch_size} not multiple of GPU count {n}'\n space = ' ' * (len(s) + 1)\n for i, d in enumerate(devices):\n p = torch.cuda.get_device_properties(i)\n s += f\"{'' if i == 0 else space}CUDA:{d} ({p.name}, {p.total_memory / (1 << 20):.0f}MiB)\\n\" # bytes to MB\n arg = 'cuda:0'\n elif mps and getattr(torch, 'has_mps', False) and torch.backends.mps.is_available(): # prefer MPS if available\n s += 'MPS\\n'\n arg = 'mps'\n else: # revert to CPU\n s += 'CPU\\n'\n arg = 'cpu'\n\n if not newline:\n s = s.rstrip()\n LOGGER.info(s.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else s) # emoji-safe\n return torch.device(arg)\n\n\ndef time_sync():\n # PyTorch-accurate time\n if torch.cuda.is_available():\n torch.cuda.synchronize()\n return time.time()\n\n\ndef profile(input, ops, n=10, device=None):\n results = []\n if not isinstance(device, torch.device):\n device = select_device(device)\n print(f\"{'Params':>12s}{'GFLOPs':>12s}{'GPU_mem (GB)':>14s}{'forward (ms)':>14s}{'backward (ms)':>14s}\"\n f\"{'input':>24s}{'output':>24s}\")\n\n for x in input if isinstance(input, list) else [input]:\n x = x.to(device)\n x.requires_grad = True\n for m in ops if isinstance(ops, list) else [ops]:\n m = m.to(device) if hasattr(m, 'to') else m # device\n m = m.half() if hasattr(m, 'half') and isinstance(x, torch.Tensor) and x.dtype is torch.float16 else m\n tf, tb, t = 0, 0, [0, 0, 0] # dt forward, backward\n try:\n flops = thop.profile(m, inputs=(x,), verbose=False)[0] / 1E9 * 2 # GFLOPs\n except Exception:\n flops = 0\n\n try:\n for _ in range(n):\n t[0] = time_sync()\n y = m(x)\n t[1] = time_sync()\n try:\n _ = (sum(yi.sum() for yi in y) if isinstance(y, list) else y).sum().backward()\n t[2] = time_sync()\n except Exception: # no backward method\n # print(e) # for debug\n t[2] = float('nan')\n tf += (t[1] - t[0]) * 1000 / n # ms per op forward\n tb += (t[2] - t[1]) * 1000 / n # ms per op backward\n mem = torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0 # (GB)\n s_in, s_out = (tuple(x.shape) if isinstance(x, torch.Tensor) else 'list' for x in (x, y)) # shapes\n p = sum(x.numel() for x in m.parameters()) if isinstance(m, nn.Module) else 0 # parameters\n print(f'{p:12}{flops:12.4g}{mem:>14.3f}{tf:14.4g}{tb:14.4g}{str(s_in):>24s}{str(s_out):>24s}')\n results.append([p, flops, mem, tf, tb, s_in, s_out])\n except Exception as e:\n print(e)\n results.append(None)\n torch.cuda.empty_cache()\n return results\n\n\ndef is_parallel(model):\n # Returns True if model is of type DP or DDP\n return type(model) in (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)\n\n\ndef de_parallel(model):\n # De-parallelize a model: returns single-GPU model if model is of type DP or DDP\n return model.module if is_parallel(model) else model\n\n\ndef initialize_weights(model):\n for m in model.modules():\n t = type(m)\n if t is nn.Conv2d:\n pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n elif t is nn.BatchNorm2d:\n m.eps = 1e-3\n m.momentum = 0.03\n elif t in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]:\n m.inplace = True\n\n\ndef find_modules(model, mclass=nn.Conv2d):\n # Finds layer indices matching module class 'mclass'\n return [i for i, m in enumerate(model.module_list) if isinstance(m, mclass)]\n\n\ndef sparsity(model):\n # Return global model sparsity\n a, b = 0, 0\n for p in model.parameters():\n a += p.numel()\n b += (p == 0).sum()\n return b / a\n\n\ndef prune(model, amount=0.3):\n # Prune model to requested global sparsity\n import torch.nn.utils.prune as prune\n print('Pruning model... ', end='')\n for name, m in model.named_modules():\n if isinstance(m, nn.Conv2d):\n prune.l1_unstructured(m, name='weight', amount=amount) # prune\n prune.remove(m, 'weight') # make permanent\n print(' %.3g global sparsity' % sparsity(model))\n\n\ndef fuse_conv_and_bn(conv, bn):\n # Fuse Conv2d() and BatchNorm2d() layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/\n fusedconv = nn.Conv2d(conv.in_channels,\n conv.out_channels,\n kernel_size=conv.kernel_size,\n stride=conv.stride,\n padding=conv.padding,\n groups=conv.groups,\n bias=True).requires_grad_(False).to(conv.weight.device)\n\n # Prepare filters\n w_conv = conv.weight.clone().view(conv.out_channels, -1)\n w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))\n fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape))\n\n # Prepare spatial bias\n b_conv = torch.zeros(conv.weight.size(0), device=conv.weight.device) if conv.bias is None else conv.bias\n b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps))\n fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)\n\n return fusedconv\n\n\ndef model_info(model, verbose=False, img_size=640):\n # Model information. img_size may be int or list, i.e. img_size=640 or img_size=[640, 320]\n n_p = sum(x.numel() for x in model.parameters()) # number parameters\n n_g = sum(x.numel() for x in model.parameters() if x.requires_grad) # number gradients\n if verbose:\n print(f\"{'layer':>5} {'name':>40} {'gradient':>9} {'parameters':>12} {'shape':>20} {'mu':>10} {'sigma':>10}\")\n for i, (name, p) in enumerate(model.named_parameters()):\n name = name.replace('module_list.', '')\n print('%5g %40s %9s %12g %20s %10.3g %10.3g' %\n (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std()))\n\n try: # FLOPs\n from thop import profile\n stride = max(int(model.stride.max()), 32) if hasattr(model, 'stride') else 32\n img = torch.zeros((1, model.yaml.get('ch', 3), stride, stride), device=next(model.parameters()).device) # input\n flops = profile(deepcopy(model), inputs=(img,), verbose=False)[0] / 1E9 * 2 # stride GFLOPs\n img_size = img_size if isinstance(img_size, list) else [img_size, img_size] # expand if int/float\n fs = ', %.1f GFLOPs' % (flops * img_size[0] / stride * img_size[1] / stride) # 640x640 GFLOPs\n except Exception:\n fs = ''\n\n name = Path(model.yaml_file).stem.replace('yolov5', 'YOLOv5') if hasattr(model, 'yaml_file') else 'Model'\n # LOGGER.info(f\"{name} summary: {len(list(model.modules()))} layers, {n_p} parameters, {n_g} gradients{fs}\")\n\n\ndef scale_img(img, ratio=1.0, same_shape=False, gs=32): # img(16,3,256,416)\n # Scales img(bs,3,y,x) by ratio constrained to gs-multiple\n if ratio == 1.0:\n return img\n h, w = img.shape[2:]\n s = (int(h * ratio), int(w * ratio)) # new size\n img = F.interpolate(img, size=s, mode='bilinear', align_corners=False) # resize\n if not same_shape: # pad/crop img\n h, w = (math.ceil(x * ratio / gs) * gs for x in (h, w))\n return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean\n\n\ndef copy_attr(a, b, include=(), exclude=()):\n # Copy attributes from b to a, options to only include [...] and to exclude [...]\n for k, v in b.__dict__.items():\n if (len(include) and k not in include) or k.startswith('_') or k in exclude:\n continue\n else:\n setattr(a, k, v)\n\n\ndef smart_optimizer(model, name='Adam', lr=0.001, momentum=0.9, weight_decay=1e-5):\n # YOLOv5 3-param group optimizer: 0) weights with decay, 1) weights no decay, 2) biases no decay\n g = [], [], [] # optimizer parameter groups\n bn = tuple(v for k, v in nn.__dict__.items() if 'Norm' in k) # normalization layers, i.e. BatchNorm2d()\n for v in model.modules():\n if hasattr(v, 'bias') and isinstance(v.bias, nn.Parameter): # bias (no decay)\n g[2].append(v.bias)\n if isinstance(v, bn): # weight (no decay)\n g[1].append(v.weight)\n elif hasattr(v, 'weight') and isinstance(v.weight, nn.Parameter): # weight (with decay)\n g[0].append(v.weight)\n\n if name == 'Adam':\n optimizer = torch.optim.Adam(g[2], lr=lr, betas=(momentum, 0.999)) # adjust beta1 to momentum\n elif name == 'AdamW':\n optimizer = torch.optim.AdamW(g[2], lr=lr, betas=(momentum, 0.999), weight_decay=0.0)\n elif name == 'RMSProp':\n optimizer = torch.optim.RMSprop(g[2], lr=lr, momentum=momentum)\n elif name == 'SGD':\n optimizer = torch.optim.SGD(g[2], lr=lr, momentum=momentum, nesterov=True)\n else:\n raise NotImplementedError(f'Optimizer {name} not implemented.')\n\n optimizer.add_param_group({'params': g[0], 'weight_decay': weight_decay}) # add g0 with weight_decay\n optimizer.add_param_group({'params': g[1], 'weight_decay': 0.0}) # add g1 (BatchNorm2d weights)\n LOGGER.info(f\"{colorstr('optimizer:')} {type(optimizer).__name__} with parameter groups \"\n f\"{len(g[1])} weight (no decay), {len(g[0])} weight, {len(g[2])} bias\")\n return optimizer\n\n\nclass EarlyStopping:\n # YOLOv5 simple early stopper\n def __init__(self, patience=30):\n self.best_fitness = 0.0 # i.e. mAP\n self.best_epoch = 0\n self.patience = patience or float('inf') # epochs to wait after fitness stops improving to stop\n self.possible_stop = False # possible stop may occur next epoch\n\n def __call__(self, epoch, fitness):\n if fitness >= self.best_fitness: # >= 0 to allow for early zero-fitness stage of training\n self.best_epoch = epoch\n self.best_fitness = fitness\n delta = epoch - self.best_epoch # epochs without improvement\n self.possible_stop = delta >= (self.patience - 1) # possible stop may occur next epoch\n stop = delta >= self.patience # stop training if patience exceeded\n if stop:\n LOGGER.info(f'Stopping training early as no improvement observed in last {self.patience} epochs. '\n f'Best results observed at epoch {self.best_epoch}, best model saved as best.pt.\\n'\n f'To update EarlyStopping(patience={self.patience}) pass a new patience value, '\n f'i.e. `python train.py --patience 300` or use `--patience 0` to disable EarlyStopping.')\n return stop\n\n\nclass ModelEMA:\n \"\"\" Updated Exponential Moving Average (EMA) from https://github.com/rwightman/pytorch-image-models\n Keeps a moving average of everything in the model state_dict (parameters and buffers)\n For EMA details see https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage\n \"\"\"\n\n def __init__(self, model, decay=0.9999, tau=2000, updates=0):\n # Create EMA\n self.ema = deepcopy(de_parallel(model)).eval() # FP32 EMA\n # if next(model.parameters()).device.type != 'cpu':\n # self.ema.half() # FP16 EMA\n self.updates = updates # number of EMA updates\n self.decay = lambda x: decay * (1 - math.exp(-x / tau)) # decay exponential ramp (to help early epochs)\n for p in self.ema.parameters():\n p.requires_grad_(False)\n\n def update(self, model):\n # Update EMA parameters\n with torch.no_grad():\n self.updates += 1\n d = self.decay(self.updates)\n\n msd = de_parallel(model).state_dict() # model state_dict\n for k, v in self.ema.state_dict().items():\n if v.dtype.is_floating_point:\n v *= d\n v += (1 - d) * msd[k].detach()\n\n def update_attr(self, model, include=(), exclude=('process_group', 'reducer')):\n # Update EMA attributes\n copy_attr(self.ema, model, include, exclude)\n" }, { "path": "asone/detectors/yolov5/yolov5/utils/yolov5_utils.py", "content": "import contextlib\nimport time\nimport numpy as np\nimport torch\nimport torchvision\nimport cv2\nimport sys\nfrom pathlib import Path\n\n\n\ndef box_area(box):\n # box = xyxy(4,n)\n return (box[2] - box[0]) * (box[3] - box[1])\n\n\ndef box_iou(box1, box2, eps=1e-7):\n # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py\n \"\"\"\n Return intersection-over-union (Jaccard index) of boxes.\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\n Arguments:\n box1 (Tensor[N, 4])\n box2 (Tensor[M, 4])\n Returns:\n iou (Tensor[N, M]): the NxM matrix containing the pairwise\n IoU values for every element in boxes1 and boxes2\n \"\"\"\n\n # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)\n (a1, a2), (b1, b2) = box1[:, None].chunk(2, 2), box2.chunk(2, 1)\n inter = (torch.min(a2, b2) - torch.max(a1, b1)).clamp(0).prod(2)\n\n # IoU = inter / (area1 + area2 - inter)\n return inter / (box_area(box1.T)[:, None] + box_area(box2.T) - inter + eps)\n\ndef xywh2xyxy(x):\n # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x\n y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y\n y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x\n y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y\n return y\n\ndef non_max_suppression(prediction,\n conf_thres=0.25,\n iou_thres=0.45,\n classes=None,\n agnostic=False,\n multi_label=False,\n labels=(),\n max_det=300):\n \"\"\"Non-Maximum Suppression (NMS) on inference results to reject overlapping bounding boxes\n Returns:\n list of detections, on (n,6) tensor per image [xyxy, conf, cls]\n \"\"\"\n # prediction = torch.Tensor(prediction)\n bs = prediction.shape[0] # batch size\n nc = prediction.shape[2] - 5 # number of classes\n xc = prediction[..., 4] > conf_thres # candidates\n # Checks\n assert 0 <= conf_thres <= 1, f'Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0'\n assert 0 <= iou_thres <= 1, f'Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0'\n\n # Settings\n # min_wh = 2 # (pixels) minimum box width and height\n max_wh = 7680 # (pixels) maximum box width and height\n max_nms = 30000 # maximum number of boxes into torchvision.ops.nms()\n time_limit = 0.3 + 0.03 * bs # seconds to quit after\n redundant = True # require redundant detections\n multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img)\n merge = False # use merge-NMS\n\n t = time.time()\n output = [torch.zeros((0, 6), device=prediction.device)] * bs\n for xi, x in enumerate(prediction): # image index, image inference\n # Apply constraints\n # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height\n x = x[xc[xi]] # confidence\n\n # Cat apriori labels if autolabelling\n if labels and len(labels[xi]):\n lb = labels[xi]\n v = torch.zeros((len(lb), nc + 5), device=x.device)\n v[:, :4] = lb[:, 1:5] # box\n v[:, 4] = 1.0 # conf\n v[range(len(lb)), lb[:, 0].long() + 5] = 1.0 # cls\n x = torch.cat((x, v), 0)\n\n # If none remain process next image\n if not x.shape[0]:\n continue\n\n # Compute conf\n x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf\n\n # Box (center x, center y, width, height) to (x1, y1, x2, y2)\n # print(type(x))\n box = xywh2xyxy(x[:, :4])\n\n # Detections matrix nx6 (xyxy, conf, cls)\n if multi_label:\n i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T\n x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1)\n else: # best class only\n conf, j = x[:, 5:].max(1, keepdim=True)\n x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres]\n\n # Filter by class\n if classes is not None:\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\n\n # Apply finite constraint\n # if not torch.isfinite(x).all():\n # x = x[torch.isfinite(x).all(1)]\n\n # Check shape\n n = x.shape[0] # number of boxes\n if not n: # no boxes\n continue\n elif n > max_nms: # excess boxes\n x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence\n\n # Batched NMS\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\n i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS\n if i.shape[0] > max_det: # limit detections\n i = i[:max_det]\n if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)\n # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\n weights = iou * scores[None] # box weights\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\n if redundant:\n i = i[iou.sum(1) > 1] # require redundancy\n\n output[xi] = x[i]\n if (time.time() - t) > time_limit:\n # LOGGER.warning(f'WARNING: NMS time limit {time_limit:.3f}s exceeded')\n break # time limit exceeded\n\n return output\n\ndef letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32):\n # Resize and pad image while meeting stride-multiple constraints\n shape = im.shape[:2] # current shape [height, width]\n if isinstance(new_shape, int):\n new_shape = (new_shape, new_shape)\n\n # Scale ratio (new / old)\n r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])\n if not scaleup: # only scale down, do not scale up (for better val mAP)\n r = min(r, 1.0)\n\n # Compute padding\n ratio = r, r # width, height ratios\n new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))\n dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding\n if auto: # minimum rectangle\n dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding\n elif scaleFill: # stretch\n dw, dh = 0.0, 0.0\n new_unpad = (new_shape[1], new_shape[0])\n ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios\n\n dw /= 2 # divide padding into 2 sides\n dh /= 2\n\n if shape[::-1] != new_unpad: # resize\n im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)\n top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))\n left, right = int(round(dw - 0.1)), int(round(dw + 0.1))\n im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border\n return im, ratio, (dw, dh)\n\n\n\ndef scale_coords(img1_shape, coords, img0_shape, ratio_pad=None):\n # Rescale coords (xyxy) from img1_shape to img0_shape\n if ratio_pad is None: # calculate from img0_shape\n gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new\n pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding\n else:\n gain = ratio_pad[0][0]\n pad = ratio_pad[1]\n\n coords[:, [0, 2]] -= pad[0] # x padding\n coords[:, [1, 3]] -= pad[1] # y padding\n coords[:, :4] /= gain\n clip_coords(coords, img0_shape)\n return coords\n\n\ndef clip_coords(boxes, shape):\n # Clip bounding xyxy bounding boxes to image shape (height, width)\n if isinstance(boxes, torch.Tensor): # faster individually\n boxes[:, 0].clamp_(0, shape[1]) # x1\n boxes[:, 1].clamp_(0, shape[0]) # y1\n boxes[:, 2].clamp_(0, shape[1]) # x2\n boxes[:, 3].clamp_(0, shape[0]) # y2\n else: # np.array (faster grouped)\n boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, shape[1]) # x1, x2\n boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, shape[0]) # y1, y2\n\n@contextlib.contextmanager\ndef yolov5_in_syspath():\n \"\"\"\n Temporarily add yolov5 folder to `sys.path`.\n \n torch.hub handles it in the same way: https://github.com/pytorch/pytorch/blob/75024e228ca441290b6a1c2e564300ad507d7af6/torch/hub.py#L387\n \n Proper fix for: #22, #134, #353, #1155, #1389, #1680, #2531, #3071 \n No need for such workarounds: #869, #1052, #2949\n \"\"\"\n yolov5_folder_dir = str(Path(__file__).parents[1].absolute())\n try:\n sys.path.insert(0, yolov5_folder_dir)\n yield\n finally:\n sys.path.remove(yolov5_folder_dir)" }, { "path": "asone/detectors/yolov5/yolov5_detector.py", "content": "import os\nfrom asone.utils import get_names\nimport numpy as np\nimport warnings\nimport torch\nfrom PIL import Image\nimport onnxruntime\nimport coremltools as ct\nfrom asone.detectors.yolov5.yolov5.utils.yolov5_utils import (non_max_suppression,\n scale_coords,\n letterbox)\nfrom asone.detectors.yolov5.yolov5.models.experimental import attempt_load\nfrom asone import utils\nfrom asone.detectors.utils.coreml_utils import yolo_to_xyxy, generalize_output_format, scale_bboxes\n\nfrom asone.utils.utils import PathResolver\n\n\nclass YOLOv5Detector:\n def __init__(self,\n weights=None,\n use_onnx=False,\n mlmodel=False,\n use_cuda=True):\n\n self.use_onnx = use_onnx\n self.mlmodel = mlmodel\n self.device = 'cuda' if use_cuda else 'cpu'\n\n if not os.path.exists(weights):\n utils.download_weights(weights)\n \n with PathResolver():\n # Load Model\n self.model = self.load_model(use_cuda, weights)\n \n def load_model(self, use_cuda, weights, fp16=False):\n # Device: CUDA and if fp16=True only then half precision floating point works \n self.fp16 = fp16 & ((not self.use_onnx or self.use_onnx) and self.device != 'cpu')\n # Load onnx \n if self.use_onnx:\n if use_cuda:\n providers = ['CUDAExecutionProvider','CPUExecutionProvider']\n else:\n providers = ['CPUExecutionProvider']\n model = onnxruntime.InferenceSession(weights, providers=providers)\n #Load coreml \n elif self.mlmodel:\n model = ct.models.MLModel(weights)\n #Load Pytorch\n else: \n model = attempt_load(weights, device=self.device, inplace=True, fuse=True)\n model.half() if self.fp16 else model.float()\n return model\n\n def image_preprocessing(self,\n image: list,\n input_shape=(640, 640))-> list:\n\n original_image = image.copy()\n image = letterbox(image, input_shape, stride=32, auto=False)[0]\n image = image.transpose((2, 0, 1))[::-1]\n image = np.ascontiguousarray(image, dtype=np.float32)\n image /= 255 # 0 - 255 to 0.0 - 1.0\n if len(image.shape) == 3:\n image = image[None] # expand for batch dim \n return original_image, image\n \n def detect(self, image: list,\n input_shape: tuple = (640, 640),\n conf_thres: float = 0.25,\n iou_thres: float = 0.45,\n max_det: int = 1000,\n filter_classes: bool = None,\n agnostic_nms: bool = True,\n with_p6: bool = False, \n return_image=False) -> list:\n \n # Image Preprocessing\n original_image, processed_image = self.image_preprocessing(image, input_shape)\n \n # Inference\n if self.use_onnx:\n # Input names of ONNX model on which it is exported \n input_name = self.model.get_inputs()[0].name\n # Run onnx model \n pred = self.model.run([self.model.get_outputs()[0].name], {input_name: processed_image})[0]\n # Run mlmodel \n \n elif self.mlmodel:\n h ,w = image.shape[:2]\n pred = self.model.predict({\"image\":Image.fromarray(image).resize(input_shape)})\n xyxy = yolo_to_xyxy(pred['coordinates'], input_shape)\n out = generalize_output_format(xyxy, pred['confidence'], conf_thres)\n if out != []:\n detections = scale_bboxes(out, image.shape[:2], input_shape)\n else:\n detections = np.empty((0, 6))\n \n if filter_classes:\n class_names = get_names()\n\n filter_class_idx = []\n if filter_classes:\n for _class in filter_classes:\n if _class.lower() in class_names:\n filter_class_idx.append(class_names.index(_class.lower()))\n else:\n warnings.warn(f\"class {_class} not found in model classes list.\")\n\n detections = detections[np.in1d(detections[:,5].astype(int), filter_class_idx)]\n \n return detections, {'width':w, 'height':h}\n # Run Pytorch model \n else:\n processed_image = torch.from_numpy(processed_image).to(self.device)\n # Change image floating point precision if fp16 set to true\n processed_image = processed_image.half() if self.fp16 else processed_image.float() \n pred = self.model(processed_image, augment=False, visualize=False)[0]\n \n # Post Processing\n if isinstance(pred, np.ndarray):\n pred = torch.tensor(pred, device=self.device)\n # print(pred)\n predictions = non_max_suppression(pred, conf_thres, \n iou_thres, \n agnostic=agnostic_nms, \n max_det=max_det)\n \n for i, prediction in enumerate(predictions): # per image\n if len(prediction):\n prediction[:, :4] = scale_coords(\n processed_image.shape[2:], prediction[:, :4], original_image.shape).round()\n predictions[i] = prediction\n detections = predictions[0].cpu().numpy()\n image_info = {\n 'width': original_image.shape[1],\n 'height': original_image.shape[0],\n }\n\n self.boxes = detections[:, :4]\n self.scores = detections[:, 4:5]\n self.class_ids = detections[:, 5:6]\n\n if filter_classes:\n class_names = get_names()\n\n filter_class_idx = []\n if filter_classes:\n for _class in filter_classes:\n if _class.lower() in class_names:\n filter_class_idx.append(class_names.index(_class.lower()))\n else:\n warnings.warn(f\"class {_class} not found in model classes list.\")\n\n detections = detections[np.in1d(detections[:,5].astype(int), filter_class_idx)]\n\n if return_image:\n return detections, original_image\n else: \n return detections, image_info\n\n " }, { "path": "asone/detectors/yolov6/__init__.py", "content": "from .yolov6_detector import YOLOv6Detector\n__all__ = ['YOLOv6Detector']" }, { "path": "asone/detectors/yolov6/yolov6/__init__.py", "content": "" }, { "path": "asone/detectors/yolov6/yolov6/assigners/__init__.py", "content": "from .atss_assigner import ATSSAssigner\nfrom .tal_assigner import TaskAlignedAssigner" }, { "path": "asone/detectors/yolov6/yolov6/assigners/anchor_generator.py", "content": "import torch\n\n\ndef generate_anchors(feats, fpn_strides, grid_cell_size=5.0, grid_cell_offset=0.5, device='cpu', is_eval=False):\n '''Generate anchors from features.'''\n anchors = []\n anchor_points = []\n stride_tensor = []\n num_anchors_list = []\n assert feats is not None\n if is_eval:\n for i, stride in enumerate(fpn_strides):\n _, _, h, w = feats[i].shape\n shift_x = torch.arange(end=w, device=device) + grid_cell_offset\n shift_y = torch.arange(end=h, device=device) + grid_cell_offset\n shift_y, shift_x = torch.meshgrid(shift_y, shift_x)\n anchor_point = torch.stack(\n [shift_x, shift_y], axis=-1).to(torch.float)\n anchor_points.append(anchor_point.reshape([-1, 2]))\n stride_tensor.append(\n torch.full(\n (h * w, 1), stride, dtype=torch.float, device=device))\n anchor_points = torch.cat(anchor_points)\n stride_tensor = torch.cat(stride_tensor)\n return anchor_points, stride_tensor\n else:\n for i, stride in enumerate(fpn_strides):\n _, _, h, w = feats[i].shape\n cell_half_size = grid_cell_size * stride * 0.5\n shift_x = (torch.arange(end=w, device=device) + grid_cell_offset) * stride\n shift_y = (torch.arange(end=h, device=device) + grid_cell_offset) * stride\n shift_y, shift_x = torch.meshgrid(shift_y, shift_x)\n anchor = torch.stack(\n [\n shift_x - cell_half_size, shift_y - cell_half_size,\n shift_x + cell_half_size, shift_y + cell_half_size\n ],\n axis=-1).clone().to(feats[0].dtype)\n anchor_point = torch.stack(\n [shift_x, shift_y], axis=-1).clone().to(feats[0].dtype)\n\n anchors.append(anchor.reshape([-1, 4]))\n anchor_points.append(anchor_point.reshape([-1, 2]))\n num_anchors_list.append(len(anchors[-1]))\n stride_tensor.append(\n torch.full(\n [num_anchors_list[-1], 1], stride, dtype=feats[0].dtype))\n anchors = torch.cat(anchors)\n anchor_points = torch.cat(anchor_points).to(device)\n stride_tensor = torch.cat(stride_tensor).to(device)\n return anchors, anchor_points, num_anchors_list, stride_tensor\n\n" }, { "path": "asone/detectors/yolov6/yolov6/assigners/assigner_utils.py", "content": "import torch\nimport torch.nn.functional as F\n\ndef dist_calculator(gt_bboxes, anchor_bboxes):\n \"\"\"compute center distance between all bbox and gt\n\n Args:\n gt_bboxes (Tensor): shape(bs*n_max_boxes, 4)\n anchor_bboxes (Tensor): shape(num_total_anchors, 4)\n Return:\n distances (Tensor): shape(bs*n_max_boxes, num_total_anchors)\n ac_points (Tensor): shape(num_total_anchors, 2)\n \"\"\" \n gt_cx = (gt_bboxes[:, 0] + gt_bboxes[:, 2]) / 2.0\n gt_cy = (gt_bboxes[:, 1] + gt_bboxes[:, 3]) / 2.0\n gt_points = torch.stack([gt_cx, gt_cy], dim=1)\n ac_cx = (anchor_bboxes[:, 0] + anchor_bboxes[:, 2]) / 2.0\n ac_cy = (anchor_bboxes[:, 1] + anchor_bboxes[:, 3]) / 2.0\n ac_points = torch.stack([ac_cx, ac_cy], dim=1)\n\n distances = (gt_points[:, None, :] - ac_points[None, :, :]).pow(2).sum(-1).sqrt()\n \n return distances, ac_points\n\ndef select_candidates_in_gts(xy_centers, gt_bboxes, eps=1e-9):\n \"\"\"select the positive anchors's center in gt\n\n Args:\n xy_centers (Tensor): shape(bs*n_max_boxes, num_total_anchors, 4)\n gt_bboxes (Tensor): shape(bs, n_max_boxes, 4)\n Return:\n (Tensor): shape(bs, n_max_boxes, num_total_anchors)\n \"\"\" \n n_anchors = xy_centers.size(0)\n bs, n_max_boxes, _ = gt_bboxes.size()\n _gt_bboxes = gt_bboxes.reshape([-1, 4])\n xy_centers = xy_centers.unsqueeze(0).repeat(bs * n_max_boxes, 1, 1) \n gt_bboxes_lt = _gt_bboxes[:, 0:2].unsqueeze(1).repeat(1, n_anchors, 1) \n gt_bboxes_rb = _gt_bboxes[:, 2:4].unsqueeze(1).repeat(1, n_anchors, 1) \n b_lt = xy_centers - gt_bboxes_lt\n b_rb = gt_bboxes_rb - xy_centers\n bbox_deltas = torch.cat([b_lt, b_rb], dim=-1)\n bbox_deltas = bbox_deltas.reshape([bs, n_max_boxes, n_anchors, -1])\n return (bbox_deltas.min(axis=-1)[0] > eps).to(gt_bboxes.dtype)\n\ndef select_highest_overlaps(mask_pos, overlaps, n_max_boxes):\n \"\"\"if an anchor box is assigned to multiple gts,\n the one with the highest iou will be selected.\n\n Args:\n mask_pos (Tensor): shape(bs, n_max_boxes, num_total_anchors)\n overlaps (Tensor): shape(bs, n_max_boxes, num_total_anchors)\n Return:\n target_gt_idx (Tensor): shape(bs, num_total_anchors)\n fg_mask (Tensor): shape(bs, num_total_anchors)\n mask_pos (Tensor): shape(bs, n_max_boxes, num_total_anchors)\n \"\"\" \n fg_mask = mask_pos.sum(axis=-2)\n if fg_mask.max() > 1:\n mask_multi_gts = (fg_mask.unsqueeze(1) > 1).repeat([1, n_max_boxes, 1])\n max_overlaps_idx = overlaps.argmax(axis=1)\n is_max_overlaps = F.one_hot(max_overlaps_idx, n_max_boxes)\n is_max_overlaps = is_max_overlaps.permute(0, 2, 1).to(overlaps.dtype)\n mask_pos = torch.where(mask_multi_gts, is_max_overlaps, mask_pos)\n fg_mask = mask_pos.sum(axis=-2)\n target_gt_idx = mask_pos.argmax(axis=-2)\n return target_gt_idx, fg_mask , mask_pos\n\ndef iou_calculator(box1, box2, eps=1e-9):\n \"\"\"Calculate iou for batch\n\n Args:\n box1 (Tensor): shape(bs, n_max_boxes, 1, 4)\n box2 (Tensor): shape(bs, 1, num_total_anchors, 4)\n Return:\n (Tensor): shape(bs, n_max_boxes, num_total_anchors)\n \"\"\"\n box1 = box1.unsqueeze(2) # [N, M1, 4] -> [N, M1, 1, 4]\n box2 = box2.unsqueeze(1) # [N, M2, 4] -> [N, 1, M2, 4]\n px1y1, px2y2 = box1[:, :, :, 0:2], box1[:, :, :, 2:4]\n gx1y1, gx2y2 = box2[:, :, :, 0:2], box2[:, :, :, 2:4]\n x1y1 = torch.maximum(px1y1, gx1y1)\n x2y2 = torch.minimum(px2y2, gx2y2)\n overlap = (x2y2 - x1y1).clip(0).prod(-1)\n area1 = (px2y2 - px1y1).clip(0).prod(-1)\n area2 = (gx2y2 - gx1y1).clip(0).prod(-1)\n union = area1 + area2 - overlap + eps\n\n return overlap / union" }, { "path": "asone/detectors/yolov6/yolov6/assigners/atss_assigner.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom asone.detectors.yolov6.yolov6.assigners.iou2d_calculator import iou2d_calculator\nfrom asone.detectors.yolov6.yolov6.assigners.assigner_utils import dist_calculator, select_candidates_in_gts, select_highest_overlaps, iou_calculator\n\nclass ATSSAssigner(nn.Module):\n '''Adaptive Training Sample Selection Assigner'''\n def __init__(self,\n topk=9,\n num_classes=80):\n super(ATSSAssigner, self).__init__()\n self.topk = topk\n self.num_classes = num_classes\n self.bg_idx = num_classes\n\n @torch.no_grad()\n def forward(self,\n anc_bboxes,\n n_level_bboxes,\n gt_labels,\n gt_bboxes,\n mask_gt,\n pd_bboxes):\n r\"\"\"This code is based on\n https://github.com/fcjian/TOOD/blob/master/mmdet/core/bbox/assigners/atss_assigner.py\n\n Args:\n anc_bboxes (Tensor): shape(num_total_anchors, 4)\n n_level_bboxes (List):len(3)\n gt_labels (Tensor): shape(bs, n_max_boxes, 1)\n gt_bboxes (Tensor): shape(bs, n_max_boxes, 4)\n mask_gt (Tensor): shape(bs, n_max_boxes, 1)\n pd_bboxes (Tensor): shape(bs, n_max_boxes, 4)\n Returns:\n target_labels (Tensor): shape(bs, num_total_anchors)\n target_bboxes (Tensor): shape(bs, num_total_anchors, 4)\n target_scores (Tensor): shape(bs, num_total_anchors, num_classes)\n fg_mask (Tensor): shape(bs, num_total_anchors)\n \"\"\"\n self.n_anchors = anc_bboxes.size(0)\n self.bs = gt_bboxes.size(0)\n self.n_max_boxes = gt_bboxes.size(1)\n\n if self.n_max_boxes == 0:\n device = gt_bboxes.device\n return torch.full( [self.bs, self.n_anchors], self.bg_idx).to(device), \\\n torch.zeros([self.bs, self.n_anchors, 4]).to(device), \\\n torch.zeros([self.bs, self.n_anchors, self.num_classes]).to(device), \\\n torch.zeros([self.bs, self.n_anchors]).to(device)\n\n\n overlaps = iou2d_calculator(gt_bboxes.reshape([-1, 4]), anc_bboxes)\n overlaps = overlaps.reshape([self.bs, -1, self.n_anchors])\n\n distances, ac_points = dist_calculator(gt_bboxes.reshape([-1, 4]), anc_bboxes)\n distances = distances.reshape([self.bs, -1, self.n_anchors])\n\n is_in_candidate, candidate_idxs = self.select_topk_candidates(\n distances, n_level_bboxes, mask_gt)\n\n overlaps_thr_per_gt, iou_candidates = self.thres_calculator(\n is_in_candidate, candidate_idxs, overlaps)\n \n # select candidates iou >= threshold as positive\n is_pos = torch.where(\n iou_candidates > overlaps_thr_per_gt.repeat([1, 1, self.n_anchors]),\n is_in_candidate, torch.zeros_like(is_in_candidate))\n\n is_in_gts = select_candidates_in_gts(ac_points, gt_bboxes)\n mask_pos = is_pos * is_in_gts * mask_gt\n\n target_gt_idx, fg_mask, mask_pos = select_highest_overlaps(\n mask_pos, overlaps, self.n_max_boxes)\n \n # assigned target\n target_labels, target_bboxes, target_scores = self.get_targets(\n gt_labels, gt_bboxes, target_gt_idx, fg_mask)\n\n # soft label with iou\n if pd_bboxes is not None:\n ious = iou_calculator(gt_bboxes, pd_bboxes) * mask_pos\n ious = ious.max(axis=-2)[0].unsqueeze(-1)\n target_scores *= ious\n\n return target_labels.long(), target_bboxes, target_scores, fg_mask.bool()\n\n def select_topk_candidates(self,\n distances, \n n_level_bboxes, \n mask_gt):\n\n mask_gt = mask_gt.repeat(1, 1, self.topk).bool()\n level_distances = torch.split(distances, n_level_bboxes, dim=-1)\n is_in_candidate_list = []\n candidate_idxs = []\n start_idx = 0\n for per_level_distances, per_level_boxes in zip(level_distances, n_level_bboxes):\n\n end_idx = start_idx + per_level_boxes\n selected_k = min(self.topk, per_level_boxes)\n _, per_level_topk_idxs = per_level_distances.topk(selected_k, dim=-1, largest=False)\n candidate_idxs.append(per_level_topk_idxs + start_idx)\n per_level_topk_idxs = torch.where(mask_gt, \n per_level_topk_idxs, torch.zeros_like(per_level_topk_idxs))\n is_in_candidate = F.one_hot(per_level_topk_idxs, per_level_boxes).sum(dim=-2)\n is_in_candidate = torch.where(is_in_candidate > 1, \n torch.zeros_like(is_in_candidate), is_in_candidate)\n is_in_candidate_list.append(is_in_candidate.to(distances.dtype))\n start_idx = end_idx\n\n is_in_candidate_list = torch.cat(is_in_candidate_list, dim=-1)\n candidate_idxs = torch.cat(candidate_idxs, dim=-1)\n\n return is_in_candidate_list, candidate_idxs\n\n def thres_calculator(self,\n is_in_candidate, \n candidate_idxs, \n overlaps):\n\n n_bs_max_boxes = self.bs * self.n_max_boxes\n _candidate_overlaps = torch.where(is_in_candidate > 0, \n overlaps, torch.zeros_like(overlaps))\n candidate_idxs = candidate_idxs.reshape([n_bs_max_boxes, -1])\n assist_idxs = self.n_anchors * torch.arange(n_bs_max_boxes, device=candidate_idxs.device)\n assist_idxs = assist_idxs[:,None]\n faltten_idxs = candidate_idxs + assist_idxs\n candidate_overlaps = _candidate_overlaps.reshape(-1)[faltten_idxs]\n candidate_overlaps = candidate_overlaps.reshape([self.bs, self.n_max_boxes, -1])\n\n overlaps_mean_per_gt = candidate_overlaps.mean(axis=-1, keepdim=True)\n overlaps_std_per_gt = candidate_overlaps.std(axis=-1, keepdim=True)\n overlaps_thr_per_gt = overlaps_mean_per_gt + overlaps_std_per_gt\n\n return overlaps_thr_per_gt, _candidate_overlaps\n\n def get_targets(self,\n gt_labels, \n gt_bboxes, \n target_gt_idx, \n fg_mask):\n \n # assigned target labels\n batch_idx = torch.arange(self.bs, dtype=gt_labels.dtype, device=gt_labels.device)\n batch_idx = batch_idx[...,None]\n target_gt_idx = (target_gt_idx + batch_idx * self.n_max_boxes).long()\n target_labels = gt_labels.flatten()[target_gt_idx.flatten()]\n target_labels = target_labels.reshape([self.bs, self.n_anchors])\n target_labels = torch.where(fg_mask > 0, \n target_labels, torch.full_like(target_labels, self.bg_idx))\n\n # assigned target boxes\n target_bboxes = gt_bboxes.reshape([-1, 4])[target_gt_idx.flatten()]\n target_bboxes = target_bboxes.reshape([self.bs, self.n_anchors, 4])\n\n # assigned target scores\n target_scores = F.one_hot(target_labels.long(), self.num_classes + 1).float()\n target_scores = target_scores[:, :, :self.num_classes]\n\n return target_labels, target_bboxes, target_scores\n\n \n" }, { "path": "asone/detectors/yolov6/yolov6/assigners/iou2d_calculator.py", "content": "#This code is based on\n#https://github.com/fcjian/TOOD/blob/master/mmdet/core/bbox/iou_calculators/iou2d_calculator.py\n\nimport torch\n\n\ndef cast_tensor_type(x, scale=1., dtype=None):\n if dtype == 'fp16':\n # scale is for preventing overflows\n x = (x / scale).half()\n return x\n\n\ndef fp16_clamp(x, min=None, max=None):\n if not x.is_cuda and x.dtype == torch.float16:\n # clamp for cpu float16, tensor fp16 has no clamp implementation\n return x.float().clamp(min, max).half()\n\n return x.clamp(min, max)\n\n\ndef iou2d_calculator(bboxes1, bboxes2, mode='iou', is_aligned=False, scale=1., dtype=None):\n \"\"\"2D Overlaps (e.g. IoUs, GIoUs) Calculator.\"\"\"\n\n \"\"\"Calculate IoU between 2D bboxes.\n\n Args:\n bboxes1 (Tensor): bboxes have shape (m, 4) in \n format, or shape (m, 5) in format.\n bboxes2 (Tensor): bboxes have shape (m, 4) in \n format, shape (m, 5) in format, or be\n empty. If ``is_aligned `` is ``True``, then m and n must be\n equal.\n mode (str): \"iou\" (intersection over union), \"iof\" (intersection\n over foreground), or \"giou\" (generalized intersection over\n union).\n is_aligned (bool, optional): If True, then m and n must be equal.\n Default False.\n\n Returns:\n Tensor: shape (m, n) if ``is_aligned `` is False else shape (m,)\n \"\"\"\n assert bboxes1.size(-1) in [0, 4, 5]\n assert bboxes2.size(-1) in [0, 4, 5]\n if bboxes2.size(-1) == 5:\n bboxes2 = bboxes2[..., :4]\n if bboxes1.size(-1) == 5:\n bboxes1 = bboxes1[..., :4]\n\n if dtype == 'fp16':\n # change tensor type to save cpu and cuda memory and keep speed\n bboxes1 = cast_tensor_type(bboxes1, scale, dtype)\n bboxes2 = cast_tensor_type(bboxes2, scale, dtype)\n overlaps = bbox_overlaps(bboxes1, bboxes2, mode, is_aligned)\n if not overlaps.is_cuda and overlaps.dtype == torch.float16:\n # resume cpu float32\n overlaps = overlaps.float()\n return overlaps\n\n return bbox_overlaps(bboxes1, bboxes2, mode, is_aligned)\n\n\ndef bbox_overlaps(bboxes1, bboxes2, mode='iou', is_aligned=False, eps=1e-6):\n \"\"\"Calculate overlap between two set of bboxes.\n\n FP16 Contributed by https://github.com/open-mmlab/mmdetection/pull/4889\n Note:\n Assume bboxes1 is M x 4, bboxes2 is N x 4, when mode is 'iou',\n there are some new generated variable when calculating IOU\n using bbox_overlaps function:\n\n 1) is_aligned is False\n area1: M x 1\n area2: N x 1\n lt: M x N x 2\n rb: M x N x 2\n wh: M x N x 2\n overlap: M x N x 1\n union: M x N x 1\n ious: M x N x 1\n\n Total memory:\n S = (9 x N x M + N + M) * 4 Byte,\n\n When using FP16, we can reduce:\n R = (9 x N x M + N + M) * 4 / 2 Byte\n R large than (N + M) * 4 * 2 is always true when N and M >= 1.\n Obviously, N + M <= N * M < 3 * N * M, when N >=2 and M >=2,\n N + 1 < 3 * N, when N or M is 1.\n\n Given M = 40 (ground truth), N = 400000 (three anchor boxes\n in per grid, FPN, R-CNNs),\n R = 275 MB (one times)\n\n A special case (dense detection), M = 512 (ground truth),\n R = 3516 MB = 3.43 GB\n\n When the batch size is B, reduce:\n B x R\n\n Therefore, CUDA memory runs out frequently.\n\n Experiments on GeForce RTX 2080Ti (11019 MiB):\n\n | dtype | M | N | Use | Real | Ideal |\n |:----:|:----:|:----:|:----:|:----:|:----:|\n | FP32 | 512 | 400000 | 8020 MiB | -- | -- |\n | FP16 | 512 | 400000 | 4504 MiB | 3516 MiB | 3516 MiB |\n | FP32 | 40 | 400000 | 1540 MiB | -- | -- |\n | FP16 | 40 | 400000 | 1264 MiB | 276MiB | 275 MiB |\n\n 2) is_aligned is True\n area1: N x 1\n area2: N x 1\n lt: N x 2\n rb: N x 2\n wh: N x 2\n overlap: N x 1\n union: N x 1\n ious: N x 1\n\n Total memory:\n S = 11 x N * 4 Byte\n\n When using FP16, we can reduce:\n R = 11 x N * 4 / 2 Byte\n\n So do the 'giou' (large than 'iou').\n\n Time-wise, FP16 is generally faster than FP32.\n\n When gpu_assign_thr is not -1, it takes more time on cpu\n but not reduce memory.\n There, we can reduce half the memory and keep the speed.\n\n If ``is_aligned`` is ``False``, then calculate the overlaps between each\n bbox of bboxes1 and bboxes2, otherwise the overlaps between each aligned\n pair of bboxes1 and bboxes2.\n\n Args:\n bboxes1 (Tensor): shape (B, m, 4) in format or empty.\n bboxes2 (Tensor): shape (B, n, 4) in format or empty.\n B indicates the batch dim, in shape (B1, B2, ..., Bn).\n If ``is_aligned`` is ``True``, then m and n must be equal.\n mode (str): \"iou\" (intersection over union), \"iof\" (intersection over\n foreground) or \"giou\" (generalized intersection over union).\n Default \"iou\".\n is_aligned (bool, optional): If True, then m and n must be equal.\n Default False.\n eps (float, optional): A value added to the denominator for numerical\n stability. Default 1e-6.\n\n Returns:\n Tensor: shape (m, n) if ``is_aligned`` is False else shape (m,)\n\n Example:\n >>> bboxes1 = torch.FloatTensor([\n >>> [0, 0, 10, 10],\n >>> [10, 10, 20, 20],\n >>> [32, 32, 38, 42],\n >>> ])\n >>> bboxes2 = torch.FloatTensor([\n >>> [0, 0, 10, 20],\n >>> [0, 10, 10, 19],\n >>> [10, 10, 20, 20],\n >>> ])\n >>> overlaps = bbox_overlaps(bboxes1, bboxes2)\n >>> assert overlaps.shape == (3, 3)\n >>> overlaps = bbox_overlaps(bboxes1, bboxes2, is_aligned=True)\n >>> assert overlaps.shape == (3, )\n\n Example:\n >>> empty = torch.empty(0, 4)\n >>> nonempty = torch.FloatTensor([[0, 0, 10, 9]])\n >>> assert tuple(bbox_overlaps(empty, nonempty).shape) == (0, 1)\n >>> assert tuple(bbox_overlaps(nonempty, empty).shape) == (1, 0)\n >>> assert tuple(bbox_overlaps(empty, empty).shape) == (0, 0)\n \"\"\"\n\n assert mode in ['iou', 'iof', 'giou'], f'Unsupported mode {mode}'\n # Either the boxes are empty or the length of boxes' last dimension is 4\n assert (bboxes1.size(-1) == 4 or bboxes1.size(0) == 0)\n assert (bboxes2.size(-1) == 4 or bboxes2.size(0) == 0)\n\n # Batch dim must be the same\n # Batch dim: (B1, B2, ... Bn)\n assert bboxes1.shape[:-2] == bboxes2.shape[:-2]\n batch_shape = bboxes1.shape[:-2]\n\n rows = bboxes1.size(-2)\n cols = bboxes2.size(-2)\n if is_aligned:\n assert rows == cols\n\n if rows * cols == 0:\n if is_aligned:\n return bboxes1.new(batch_shape + (rows, ))\n else:\n return bboxes1.new(batch_shape + (rows, cols))\n\n area1 = (bboxes1[..., 2] - bboxes1[..., 0]) * (\n bboxes1[..., 3] - bboxes1[..., 1])\n area2 = (bboxes2[..., 2] - bboxes2[..., 0]) * (\n bboxes2[..., 3] - bboxes2[..., 1])\n\n if is_aligned:\n lt = torch.max(bboxes1[..., :2], bboxes2[..., :2]) # [B, rows, 2]\n rb = torch.min(bboxes1[..., 2:], bboxes2[..., 2:]) # [B, rows, 2]\n\n wh = fp16_clamp(rb - lt, min=0)\n overlap = wh[..., 0] * wh[..., 1]\n\n if mode in ['iou', 'giou']:\n union = area1 + area2 - overlap\n else:\n union = area1\n if mode == 'giou':\n enclosed_lt = torch.min(bboxes1[..., :2], bboxes2[..., :2])\n enclosed_rb = torch.max(bboxes1[..., 2:], bboxes2[..., 2:])\n else:\n lt = torch.max(bboxes1[..., :, None, :2],\n bboxes2[..., None, :, :2]) # [B, rows, cols, 2]\n rb = torch.min(bboxes1[..., :, None, 2:],\n bboxes2[..., None, :, 2:]) # [B, rows, cols, 2]\n\n wh = fp16_clamp(rb - lt, min=0)\n overlap = wh[..., 0] * wh[..., 1]\n\n if mode in ['iou', 'giou']:\n union = area1[..., None] + area2[..., None, :] - overlap\n else:\n union = area1[..., None]\n if mode == 'giou':\n enclosed_lt = torch.min(bboxes1[..., :, None, :2],\n bboxes2[..., None, :, :2])\n enclosed_rb = torch.max(bboxes1[..., :, None, 2:],\n bboxes2[..., None, :, 2:])\n\n eps = union.new_tensor([eps])\n union = torch.max(union, eps)\n ious = overlap / union\n if mode in ['iou', 'iof']:\n return ious\n # calculate gious\n enclose_wh = fp16_clamp(enclosed_rb - enclosed_lt, min=0)\n enclose_area = enclose_wh[..., 0] * enclose_wh[..., 1]\n enclose_area = torch.max(enclose_area, eps)\n gious = ious - (enclose_area - union) / enclose_area\n return gious\n" }, { "path": "asone/detectors/yolov6/yolov6/assigners/tal_assigner.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom asone.detectors.yolov6.yolov6.assigners.assigner_utils import select_candidates_in_gts, select_highest_overlaps, iou_calculator\n\nclass TaskAlignedAssigner(nn.Module):\n def __init__(self, \n topk=13,\n num_classes=80, \n alpha=1.0, \n beta=6.0, \n eps=1e-9):\n super(TaskAlignedAssigner, self).__init__()\n self.topk = topk\n self.num_classes = num_classes\n self.bg_idx = num_classes\n self.alpha = alpha\n self.beta = beta\n self.eps = eps\n\n @torch.no_grad()\n def forward(self,\n pd_scores,\n pd_bboxes,\n anc_points,\n gt_labels,\n gt_bboxes,\n mask_gt):\n \"\"\"This code referenced to\n https://github.com/Nioolek/PPYOLOE_pytorch/blob/master/ppyoloe/assigner/tal_assigner.py\n\n Args:\n pd_scores (Tensor): shape(bs, num_total_anchors, num_classes)\n pd_bboxes (Tensor): shape(bs, num_total_anchors, 4)\n anc_points (Tensor): shape(num_total_anchors, 2)\n gt_labels (Tensor): shape(bs, n_max_boxes, 1)\n gt_bboxes (Tensor): shape(bs, n_max_boxes, 4)\n mask_gt (Tensor): shape(bs, n_max_boxes, 1)\n Returns:\n target_labels (Tensor): shape(bs, num_total_anchors)\n target_bboxes (Tensor): shape(bs, num_total_anchors, 4)\n target_scores (Tensor): shape(bs, num_total_anchors, num_classes)\n fg_mask (Tensor): shape(bs, num_total_anchors)\n \"\"\"\n self.bs = pd_scores.size(0)\n self.n_max_boxes = gt_bboxes.size(1)\n\n if self.n_max_boxes == 0:\n device = gt_bboxes.device\n return torch.full_like(pd_scores[..., 0], self.bg_idx).to(device), \\\n torch.zeros_like(pd_bboxes).to(device), \\\n torch.zeros_like(pd_scores).to(device), \\\n torch.zeros_like(pd_scores[..., 0]).to(device)\n\n \n mask_pos, align_metric, overlaps = self.get_pos_mask(\n pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points, mask_gt)\n\n target_gt_idx, fg_mask, mask_pos = select_highest_overlaps(\n mask_pos, overlaps, self.n_max_boxes)\n \n # assigned target\n target_labels, target_bboxes, target_scores = self.get_targets(\n gt_labels, gt_bboxes, target_gt_idx, fg_mask)\n\n # normalize\n align_metric *= mask_pos\n pos_align_metrics = align_metric.max(axis=-1, keepdim=True)[0]\n pos_overlaps = (overlaps * mask_pos).max(axis=-1, keepdim=True)[0]\n norm_align_metric = (align_metric * pos_overlaps / (pos_align_metrics + self.eps)).max(-2)[0].unsqueeze(-1)\n target_scores = target_scores * norm_align_metric\n\n return target_labels, target_bboxes, target_scores, fg_mask.bool()\n\n def get_pos_mask(self, \n pd_scores, \n pd_bboxes, \n gt_labels, \n gt_bboxes, \n anc_points, \n mask_gt):\n\n # get anchor_align metric\n align_metric, overlaps = self.get_box_metrics(pd_scores, pd_bboxes, gt_labels, gt_bboxes)\n # get in_gts mask\n mask_in_gts = select_candidates_in_gts(anc_points, gt_bboxes)\n # get topk_metric mask\n mask_topk = self.select_topk_candidates(\n align_metric * mask_in_gts, topk_mask=mask_gt.repeat([1, 1, self.topk]).bool())\n # merge all mask to a final mask\n mask_pos = mask_topk * mask_in_gts * mask_gt\n\n return mask_pos, align_metric, overlaps\n \n def get_box_metrics(self, \n pd_scores, \n pd_bboxes, \n gt_labels, \n gt_bboxes):\n\n pd_scores = pd_scores.permute(0, 2, 1)\n gt_labels = gt_labels.to(torch.long)\n ind = torch.zeros([2, self.bs, self.n_max_boxes], dtype=torch.long)\n ind[0] = torch.arange(end=self.bs).view(-1, 1).repeat(1, self.n_max_boxes)\n ind[1] = gt_labels.squeeze(-1)\n bbox_scores = pd_scores[ind[0], ind[1]]\n\n overlaps = iou_calculator(gt_bboxes, pd_bboxes)\n align_metric = bbox_scores.pow(self.alpha) * overlaps.pow(self.beta)\n\n return align_metric, overlaps\n\n def select_topk_candidates(self,\n metrics, \n largest=True, \n topk_mask=None):\n\n num_anchors = metrics.shape[-1]\n topk_metrics, topk_idxs = torch.topk(\n metrics, self.topk, axis=-1, largest=largest)\n if topk_mask is None:\n topk_mask = (topk_metrics.max(axis=-1, keepdim=True) > self.eps).tile(\n [1, 1, self.topk])\n topk_idxs = torch.where(topk_mask, topk_idxs, torch.zeros_like(topk_idxs))\n is_in_topk = F.one_hot(topk_idxs, num_anchors).sum(axis=-2)\n is_in_topk = torch.where(is_in_topk > 1,\n torch.zeros_like(is_in_topk), is_in_topk)\n return is_in_topk.to(metrics.dtype)\n\n def get_targets(self, \n gt_labels, \n gt_bboxes, \n target_gt_idx, \n fg_mask):\n \n # assigned target labels\n batch_ind = torch.arange(end=self.bs, dtype=torch.int64, device=gt_labels.device)[...,None]\n target_gt_idx = target_gt_idx + batch_ind * self.n_max_boxes\n target_labels = gt_labels.long().flatten()[target_gt_idx]\n\n # assigned target boxes\n target_bboxes = gt_bboxes.reshape([-1, 4])[target_gt_idx]\n\n # assigned target scores \n target_labels[target_labels<0] = 0\n target_scores = F.one_hot(target_labels, self.num_classes)\n fg_scores_mask = fg_mask[:, :, None].repeat(1, 1, self.num_classes)\n target_scores = torch.where(fg_scores_mask > 0, target_scores,\n torch.full_like(target_scores, 0))\n \n return target_labels, target_bboxes, target_scores" }, { "path": "asone/detectors/yolov6/yolov6/layers/__init__.py", "content": "" }, { "path": "asone/detectors/yolov6/yolov6/layers/common.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n\nimport warnings\nfrom pathlib import Path\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.nn.parameter import Parameter\nimport torch.nn.init as init\n\n\nclass SiLU(nn.Module):\n '''Activation of SiLU'''\n @staticmethod\n def forward(x):\n return x * torch.sigmoid(x)\n\n\nclass Conv(nn.Module):\n '''Normal Conv with SiLU activation'''\n def __init__(self, in_channels, out_channels, kernel_size, stride, groups=1, bias=False):\n super().__init__()\n padding = kernel_size // 2\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n kernel_size=kernel_size,\n stride=stride,\n padding=padding,\n groups=groups,\n bias=bias,\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.act = nn.SiLU()\n\n def forward(self, x):\n return self.act(self.bn(self.conv(x)))\n\n def forward_fuse(self, x):\n return self.act(self.conv(x))\n\n\nclass SimConv(nn.Module):\n '''Normal Conv with ReLU activation'''\n def __init__(self, in_channels, out_channels, kernel_size, stride, groups=1, bias=False):\n super().__init__()\n padding = kernel_size // 2\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n kernel_size=kernel_size,\n stride=stride,\n padding=padding,\n groups=groups,\n bias=bias,\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.act = nn.ReLU()\n\n def forward(self, x):\n return self.act(self.bn(self.conv(x)))\n\n def forward_fuse(self, x):\n return self.act(self.conv(x))\n\nclass ConvWrapper(nn.Module):\n '''Wrapper for normal Conv with SiLU activation'''\n def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, groups=1, bias=True):\n super().__init__()\n self.block = Conv(in_channels, out_channels, kernel_size, stride, groups, bias)\n\n def forward(self, x):\n return self.block(x)\n\n\nclass SimConvWrapper(nn.Module):\n '''Wrapper for normal Conv with ReLU activation'''\n def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, groups=1, bias=True):\n super().__init__()\n self.block = SimConv(in_channels, out_channels, kernel_size, stride, groups, bias)\n\n def forward(self, x):\n return self.block(x)\n\n\nclass SimSPPF(nn.Module):\n '''Simplified SPPF with ReLU activation'''\n def __init__(self, in_channels, out_channels, kernel_size=5):\n super().__init__()\n c_ = in_channels // 2 # hidden channels\n self.cv1 = SimConv(in_channels, c_, 1, 1)\n self.cv2 = SimConv(c_ * 4, out_channels, 1, 1)\n self.m = nn.MaxPool2d(kernel_size=kernel_size, stride=1, padding=kernel_size // 2)\n\n def forward(self, x):\n x = self.cv1(x)\n with warnings.catch_warnings():\n warnings.simplefilter('ignore')\n y1 = self.m(x)\n y2 = self.m(y1)\n return self.cv2(torch.cat([x, y1, y2, self.m(y2)], 1))\n\n\nclass SPPF(nn.Module):\n '''Spatial Pyramid Pooling - Fast (SPPF) layer for YOLOv5 by Glenn Jocher'''\n def __init__(self, in_channels, out_channels, kernel_size=5): # equivalent to SPP(k=(5, 9, 13))\n super().__init__()\n c_ = in_channels // 2 # hidden channels\n self.cv1 = Conv(in_channels, c_, 1, 1)\n self.cv2 = Conv(c_ * 4, out_channels, 1, 1)\n self.m = nn.MaxPool2d(kernel_size=kernel_size, stride=1, padding=kernel_size // 2)\n\n def forward(self, x):\n x = self.cv1(x)\n with warnings.catch_warnings():\n warnings.simplefilter('ignore') # suppress torch 1.9.0 max_pool2d() warning\n y1 = self.m(x)\n y2 = self.m(y1)\n return self.cv2(torch.cat((x, y1, y2, self.m(y2)), 1))\n\n\nclass Transpose(nn.Module):\n '''Normal Transpose, default for upsampling'''\n def __init__(self, in_channels, out_channels, kernel_size=2, stride=2):\n super().__init__()\n self.upsample_transpose = torch.nn.ConvTranspose2d(\n in_channels=in_channels,\n out_channels=out_channels,\n kernel_size=kernel_size,\n stride=stride,\n bias=True\n )\n\n def forward(self, x):\n return self.upsample_transpose(x)\n\n\nclass Concat(nn.Module):\n def __init__(self, dimension=1):\n super().__init__()\n self.d = dimension\n\n def forward(self, x):\n return torch.cat(x, self.d)\n\n\ndef conv_bn(in_channels, out_channels, kernel_size, stride, padding, groups=1):\n '''Basic cell for rep-style block, including conv and bn'''\n result = nn.Sequential()\n result.add_module('conv', nn.Conv2d(in_channels=in_channels, out_channels=out_channels,\n kernel_size=kernel_size, stride=stride, padding=padding, groups=groups, bias=False))\n result.add_module('bn', nn.BatchNorm2d(num_features=out_channels))\n return result\n\n\nclass RepVGGBlock(nn.Module):\n '''RepVGGBlock is a basic rep-style block, including training and deploy status\n This code is based on https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py\n '''\n def __init__(self, in_channels, out_channels, kernel_size=3,\n stride=1, padding=1, dilation=1, groups=1, padding_mode='zeros', deploy=False, use_se=False):\n super(RepVGGBlock, self).__init__()\n \"\"\" Initialization of the class.\n Args:\n in_channels (int): Number of channels in the input image\n out_channels (int): Number of channels produced by the convolution\n kernel_size (int or tuple): Size of the convolving kernel\n stride (int or tuple, optional): Stride of the convolution. Default: 1\n padding (int or tuple, optional): Zero-padding added to both sides of\n the input. Default: 1\n dilation (int or tuple, optional): Spacing between kernel elements. Default: 1\n groups (int, optional): Number of blocked connections from input\n channels to output channels. Default: 1\n padding_mode (string, optional): Default: 'zeros'\n deploy: Whether to be deploy status or training status. Default: False\n use_se: Whether to use se. Default: False\n \"\"\"\n self.deploy = deploy\n self.groups = groups\n self.in_channels = in_channels\n self.out_channels = out_channels\n\n assert kernel_size == 3\n assert padding == 1\n\n padding_11 = padding - kernel_size // 2\n\n self.nonlinearity = nn.ReLU()\n\n if use_se:\n raise NotImplementedError(\"se block not supported yet\")\n else:\n self.se = nn.Identity()\n\n if deploy:\n self.rbr_reparam = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride,\n padding=padding, dilation=dilation, groups=groups, bias=True, padding_mode=padding_mode)\n\n else:\n self.rbr_identity = nn.BatchNorm2d(num_features=in_channels) if out_channels == in_channels and stride == 1 else None\n self.rbr_dense = conv_bn(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, groups=groups)\n self.rbr_1x1 = conv_bn(in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=stride, padding=padding_11, groups=groups)\n\n def forward(self, inputs):\n '''Forward process'''\n if hasattr(self, 'rbr_reparam'):\n return self.nonlinearity(self.se(self.rbr_reparam(inputs)))\n\n if self.rbr_identity is None:\n id_out = 0\n else:\n id_out = self.rbr_identity(inputs)\n\n return self.nonlinearity(self.se(self.rbr_dense(inputs) + self.rbr_1x1(inputs) + id_out))\n\n def get_equivalent_kernel_bias(self):\n kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense)\n kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1)\n kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity)\n return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid\n\n def _pad_1x1_to_3x3_tensor(self, kernel1x1):\n if kernel1x1 is None:\n return 0\n else:\n return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1])\n\n def _fuse_bn_tensor(self, branch):\n if branch is None:\n return 0, 0\n if isinstance(branch, nn.Sequential):\n kernel = branch.conv.weight\n running_mean = branch.bn.running_mean\n running_var = branch.bn.running_var\n gamma = branch.bn.weight\n beta = branch.bn.bias\n eps = branch.bn.eps\n else:\n assert isinstance(branch, nn.BatchNorm2d)\n if not hasattr(self, 'id_tensor'):\n input_dim = self.in_channels // self.groups\n kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), dtype=np.float32)\n for i in range(self.in_channels):\n kernel_value[i, i % input_dim, 1, 1] = 1\n self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device)\n kernel = self.id_tensor\n running_mean = branch.running_mean\n running_var = branch.running_var\n gamma = branch.weight\n beta = branch.bias\n eps = branch.eps\n std = (running_var + eps).sqrt()\n t = (gamma / std).reshape(-1, 1, 1, 1)\n return kernel * t, beta - running_mean * gamma / std\n\n def switch_to_deploy(self):\n if hasattr(self, 'rbr_reparam'):\n return\n kernel, bias = self.get_equivalent_kernel_bias()\n self.rbr_reparam = nn.Conv2d(in_channels=self.rbr_dense.conv.in_channels, out_channels=self.rbr_dense.conv.out_channels,\n kernel_size=self.rbr_dense.conv.kernel_size, stride=self.rbr_dense.conv.stride,\n padding=self.rbr_dense.conv.padding, dilation=self.rbr_dense.conv.dilation, groups=self.rbr_dense.conv.groups, bias=True)\n self.rbr_reparam.weight.data = kernel\n self.rbr_reparam.bias.data = bias\n for para in self.parameters():\n para.detach_()\n self.__delattr__('rbr_dense')\n self.__delattr__('rbr_1x1')\n if hasattr(self, 'rbr_identity'):\n self.__delattr__('rbr_identity')\n if hasattr(self, 'id_tensor'):\n self.__delattr__('id_tensor')\n self.deploy = True\n\n\nclass RealVGGBlock(nn.Module):\n\n def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1,\n dilation=1, groups=1, padding_mode='zeros', use_se=False,\n ):\n super(RealVGGBlock, self).__init__()\n self.relu = nn.ReLU()\n self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)\n self.bn = nn.BatchNorm2d(out_channels)\n\n if use_se:\n raise NotImplementedError(\"se block not supported yet\")\n else:\n self.se = nn.Identity()\n\n def forward(self, inputs):\n out = self.relu(self.se(self.bn(self.conv(inputs))))\n return out\n\n\nclass ScaleLayer(torch.nn.Module):\n\n def __init__(self, num_features, use_bias=True, scale_init=1.0):\n super(ScaleLayer, self).__init__()\n self.weight = Parameter(torch.Tensor(num_features))\n init.constant_(self.weight, scale_init)\n self.num_features = num_features\n if use_bias:\n self.bias = Parameter(torch.Tensor(num_features))\n init.zeros_(self.bias)\n else:\n self.bias = None\n\n def forward(self, inputs):\n if self.bias is None:\n return inputs * self.weight.view(1, self.num_features, 1, 1)\n else:\n return inputs * self.weight.view(1, self.num_features, 1, 1) + self.bias.view(1, self.num_features, 1, 1)\n\n\n# A CSLA block is a LinearAddBlock with is_csla=True\nclass LinearAddBlock(nn.Module):\n\n def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1,\n dilation=1, groups=1, padding_mode='zeros', use_se=False, is_csla=False, conv_scale_init=1.0):\n super(LinearAddBlock, self).__init__()\n self.in_channels = in_channels\n self.relu = nn.ReLU()\n self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=padding, bias=False)\n self.scale_conv = ScaleLayer(num_features=out_channels, use_bias=False, scale_init=conv_scale_init)\n self.conv_1x1 = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=stride, padding=0, bias=False)\n self.scale_1x1 = ScaleLayer(num_features=out_channels, use_bias=False, scale_init=conv_scale_init)\n if in_channels == out_channels and stride == 1:\n self.scale_identity = ScaleLayer(num_features=out_channels, use_bias=False, scale_init=1.0)\n self.bn = nn.BatchNorm2d(out_channels)\n if is_csla: # Make them constant\n self.scale_1x1.requires_grad_(False)\n self.scale_conv.requires_grad_(False)\n if use_se:\n raise NotImplementedError(\"se block not supported yet\")\n else:\n self.se = nn.Identity()\n\n def forward(self, inputs):\n out = self.scale_conv(self.conv(inputs)) + self.scale_1x1(self.conv_1x1(inputs))\n if hasattr(self, 'scale_identity'):\n out += self.scale_identity(inputs)\n out = self.relu(self.se(self.bn(out)))\n return out\n\n\nclass DetectBackend(nn.Module):\n def __init__(self, weights='yolov6s.pt', device=None, dnn=True):\n\n super().__init__()\n assert isinstance(weights, str) and Path(weights).suffix == '.pt', f'{Path(weights).suffix} format is not supported.'\n from asone.detectors.yolov6.yolov6.utils.checkpoint import load_checkpoint\n model = load_checkpoint(weights, map_location=device)\n stride = int(model.stride.max())\n self.__dict__.update(locals()) # assign all variables to self\n\n def forward(self, im, val=False):\n y, _ = self.model(im)\n if isinstance(y, np.ndarray):\n y = torch.tensor(y, device=self.device)\n return y\n\n\nclass RepBlock(nn.Module):\n '''\n RepBlock is a stage block with rep-style basic block\n '''\n def __init__(self, in_channels, out_channels, n=1, block=RepVGGBlock, basic_block=RepVGGBlock):\n super().__init__()\n \n self.conv1 = block(in_channels, out_channels)\n self.block = nn.Sequential(*(block(out_channels, out_channels) for _ in range(n - 1))) if n > 1 else None\n if block == BottleRep:\n self.conv1 = BottleRep(in_channels, out_channels, basic_block=basic_block, weight=True)\n n = n // 2\n self.block = nn.Sequential(*(BottleRep(out_channels, out_channels, basic_block=basic_block, weight=True) for _ in range(n - 1))) if n > 1 else None\n \n def forward(self, x):\n x = self.conv1(x)\n if self.block is not None:\n x = self.block(x)\n return x\n\n\nclass BottleRep(nn.Module):\n\n def __init__(self, in_channels, out_channels, basic_block=RepVGGBlock, weight=False):\n super().__init__()\n self.conv1 = basic_block(in_channels, out_channels)\n self.conv2 = basic_block(out_channels, out_channels)\n if in_channels != out_channels:\n self.shortcut = False\n else:\n self.shortcut = True\n if weight:\n self.alpha = Parameter(torch.ones(1))\n else:\n self.alpha = 1.0\n\n def forward(self, x):\n outputs = self.conv1(x)\n outputs = self.conv2(outputs)\n return outputs + self.alpha * x if self.shortcut else outputs\n \n \n \ndef autopad(k, p=None): # kernel, padding\n # Pad to 'same'\n if p is None:\n p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad\n return p\n\n\nclass Conv_C3(nn.Module):\n '''Standard convolution in BepC3-Block'''\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super().__init__()\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False)\n self.bn = nn.BatchNorm2d(c2)\n self.act = nn.ReLU() if act is True else (act if isinstance(act, nn.Module) else nn.Identity())\n def forward(self, x):\n return self.act(self.bn(self.conv(x)))\n def forward_fuse(self, x):\n return self.act(self.conv(x))\n\n\nclass BepC3(nn.Module):\n '''Beer-mug RepC3 Block'''\n def __init__(self, in_channels, out_channels, n=1, e=0.5, concat=True, block=RepVGGBlock): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n c_ = int(out_channels * e) # hidden channels\n self.cv1 = Conv_C3(in_channels, c_, 1, 1)\n self.cv2 = Conv_C3(in_channels, c_, 1, 1)\n self.cv3 = Conv_C3(2 * c_, out_channels, 1, 1)\n if block == ConvWrapper:\n self.cv1 = Conv_C3(in_channels, c_, 1, 1, act=nn.SiLU())\n self.cv2 = Conv_C3(in_channels, c_, 1, 1, act=nn.SiLU())\n self.cv3 = Conv_C3(2 * c_, out_channels, 1, 1, act=nn.SiLU())\n \n self.m = RepBlock(in_channels=c_, out_channels=c_, n=n, block=BottleRep, basic_block=block)\n self.concat = concat\n if not concat:\n self.cv3 = Conv_C3(c_, out_channels, 1, 1)\n \n def forward(self, x):\n if self.concat is True:\n return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), dim=1))\n else:\n return self.cv3(self.m(self.cv1(x)))\n\n\ndef get_block(mode):\n if mode == 'repvgg':\n return RepVGGBlock\n elif mode == 'hyper_search':\n return LinearAddBlock\n elif mode == 'repopt':\n return RealVGGBlock\n elif mode == 'conv_relu':\n return SimConvWrapper\n elif mode == 'conv_silu':\n return ConvWrapper\n else:\n raise NotImplementedError(\"Undefied Repblock choice for mode {}\".format(mode))\n" }, { "path": "asone/detectors/yolov6/yolov6/layers/dbb_transforms.py", "content": "import torch\nimport numpy as np\nimport torch.nn.functional as F\n\n\ndef transI_fusebn(kernel, bn):\n gamma = bn.weight\n std = (bn.running_var + bn.eps).sqrt()\n return kernel * ((gamma / std).reshape(-1, 1, 1, 1)), bn.bias - bn.running_mean * gamma / std\n\n\ndef transII_addbranch(kernels, biases):\n return sum(kernels), sum(biases)\n\n\ndef transIII_1x1_kxk(k1, b1, k2, b2, groups):\n if groups == 1:\n k = F.conv2d(k2, k1.permute(1, 0, 2, 3)) #\n b_hat = (k2 * b1.reshape(1, -1, 1, 1)).sum((1, 2, 3))\n else:\n k_slices = []\n b_slices = []\n k1_T = k1.permute(1, 0, 2, 3)\n k1_group_width = k1.size(0) // groups\n k2_group_width = k2.size(0) // groups\n for g in range(groups):\n k1_T_slice = k1_T[:, g*k1_group_width:(g+1)*k1_group_width, :, :]\n k2_slice = k2[g*k2_group_width:(g+1)*k2_group_width, :, :, :]\n k_slices.append(F.conv2d(k2_slice, k1_T_slice))\n b_slices.append((k2_slice * b1[g * k1_group_width:(g+1) * k1_group_width].reshape(1, -1, 1, 1)).sum((1, 2, 3)))\n k, b_hat = transIV_depthconcat(k_slices, b_slices)\n return k, b_hat + b2\n\n\ndef transIV_depthconcat(kernels, biases):\n return torch.cat(kernels, dim=0), torch.cat(biases)\n\n\ndef transV_avg(channels, kernel_size, groups):\n input_dim = channels // groups\n k = torch.zeros((channels, input_dim, kernel_size, kernel_size))\n k[np.arange(channels), np.tile(np.arange(input_dim), groups), :, :] = 1.0 / kernel_size ** 2\n return k\n\n\n# This has not been tested with non-square kernels (kernel.size(2) != kernel.size(3)) nor even-size kernels\ndef transVI_multiscale(kernel, target_kernel_size):\n H_pixels_to_pad = (target_kernel_size - kernel.size(2)) // 2\n W_pixels_to_pad = (target_kernel_size - kernel.size(3)) // 2\n return F.pad(kernel, [H_pixels_to_pad, H_pixels_to_pad, W_pixels_to_pad, W_pixels_to_pad])\n" }, { "path": "asone/detectors/yolov6/yolov6/models/__init__.py", "content": "" }, { "path": "asone/detectors/yolov6/yolov6/models/efficientrep.py", "content": "from torch import nn\nfrom asone.detectors.yolov6.yolov6.layers.common import BottleRep, RepVGGBlock, RepBlock, BepC3, SimSPPF, SPPF, ConvWrapper\n\n\nclass EfficientRep(nn.Module):\n '''EfficientRep Backbone\n EfficientRep is handcrafted by hardware-aware neural network design.\n With rep-style struct, EfficientRep is friendly to high-computation hardware(e.g. GPU).\n '''\n\n def __init__(\n self,\n in_channels=3,\n channels_list=None,\n num_repeats=None,\n block=RepVGGBlock\n ):\n super().__init__()\n\n assert channels_list is not None\n assert num_repeats is not None\n\n self.stem = block(\n in_channels=in_channels,\n out_channels=channels_list[0],\n kernel_size=3,\n stride=2\n )\n\n self.ERBlock_2 = nn.Sequential(\n block(\n in_channels=channels_list[0],\n out_channels=channels_list[1],\n kernel_size=3,\n stride=2\n ),\n RepBlock(\n in_channels=channels_list[1],\n out_channels=channels_list[1],\n n=num_repeats[1],\n block=block,\n )\n )\n\n self.ERBlock_3 = nn.Sequential(\n block(\n in_channels=channels_list[1],\n out_channels=channels_list[2],\n kernel_size=3,\n stride=2\n ),\n RepBlock(\n in_channels=channels_list[2],\n out_channels=channels_list[2],\n n=num_repeats[2],\n block=block,\n )\n )\n\n self.ERBlock_4 = nn.Sequential(\n block(\n in_channels=channels_list[2],\n out_channels=channels_list[3],\n kernel_size=3,\n stride=2\n ),\n RepBlock(\n in_channels=channels_list[3],\n out_channels=channels_list[3],\n n=num_repeats[3],\n block=block,\n )\n )\n\n self.ERBlock_5 = nn.Sequential(\n block(\n in_channels=channels_list[3],\n out_channels=channels_list[4],\n kernel_size=3,\n stride=2,\n ),\n RepBlock(\n in_channels=channels_list[4],\n out_channels=channels_list[4],\n n=num_repeats[4],\n block=block,\n ),\n SimSPPF(\n in_channels=channels_list[4],\n out_channels=channels_list[4],\n kernel_size=5\n )\n )\n\n def forward(self, x):\n\n outputs = []\n x = self.stem(x)\n x = self.ERBlock_2(x)\n x = self.ERBlock_3(x)\n outputs.append(x)\n x = self.ERBlock_4(x)\n outputs.append(x)\n x = self.ERBlock_5(x)\n outputs.append(x)\n\n return tuple(outputs)\n\n\nclass CSPBepBackbone(nn.Module):\n \"\"\"\n CSPBepBackbone module. \n \"\"\"\n\n def __init__(\n self,\n in_channels=3,\n channels_list=None,\n num_repeats=None,\n block=RepVGGBlock,\n csp_e=float(1)/2,\n ):\n super().__init__()\n\n assert channels_list is not None\n assert num_repeats is not None\n\n self.stem = block(\n in_channels=in_channels,\n out_channels=channels_list[0],\n kernel_size=3,\n stride=2\n )\n\n self.ERBlock_2 = nn.Sequential(\n block(\n in_channels=channels_list[0],\n out_channels=channels_list[1],\n kernel_size=3,\n stride=2\n ),\n BepC3(\n in_channels=channels_list[1],\n out_channels=channels_list[1],\n n=num_repeats[1],\n e=csp_e,\n block=block,\n )\n )\n\n self.ERBlock_3 = nn.Sequential(\n block(\n in_channels=channels_list[1],\n out_channels=channels_list[2],\n kernel_size=3,\n stride=2\n ),\n BepC3(\n in_channels=channels_list[2],\n out_channels=channels_list[2],\n n=num_repeats[2],\n e=csp_e,\n block=block,\n )\n )\n\n self.ERBlock_4 = nn.Sequential(\n block(\n in_channels=channels_list[2],\n out_channels=channels_list[3],\n kernel_size=3,\n stride=2\n ),\n BepC3(\n in_channels=channels_list[3],\n out_channels=channels_list[3],\n n=num_repeats[3],\n e=csp_e,\n block=block,\n )\n )\n\n channel_merge_layer = SimSPPF\n if block == ConvWrapper:\n channel_merge_layer = SPPF\n\n self.ERBlock_5 = nn.Sequential(\n block(\n in_channels=channels_list[3],\n out_channels=channels_list[4],\n kernel_size=3,\n stride=2,\n ),\n BepC3(\n in_channels=channels_list[4],\n out_channels=channels_list[4],\n n=num_repeats[4],\n e=csp_e,\n block=block,\n ),\n channel_merge_layer(\n in_channels=channels_list[4],\n out_channels=channels_list[4],\n kernel_size=5\n )\n )\n\n def forward(self, x):\n\n outputs = []\n x = self.stem(x)\n x = self.ERBlock_2(x)\n x = self.ERBlock_3(x)\n outputs.append(x)\n x = self.ERBlock_4(x)\n outputs.append(x)\n x = self.ERBlock_5(x)\n outputs.append(x)\n\n return tuple(outputs)\n" }, { "path": "asone/detectors/yolov6/yolov6/models/effidehead.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport math\nfrom asone.detectors.yolov6.yolov6.layers.common import *\nfrom asone.detectors.yolov6.yolov6.assigners.anchor_generator import generate_anchors\nfrom asone.detectors.yolov6.yolov6.utils.general import dist2bbox\n\n\nclass Detect(nn.Module):\n '''Efficient Decoupled Head\n With hardware-aware degisn, the decoupled head is optimized with\n hybridchannels methods.\n '''\n def __init__(self, num_classes=80, anchors=1, num_layers=3, inplace=True, head_layers=None, use_dfl=True, reg_max=16): # detection layer\n super().__init__()\n assert head_layers is not None\n self.nc = num_classes # number of classes\n self.no = num_classes + 5 # number of outputs per anchor\n self.nl = num_layers # number of detection layers\n if isinstance(anchors, (list, tuple)):\n self.na = len(anchors[0]) // 2\n else:\n self.na = anchors\n self.anchors = anchors\n self.grid = [torch.zeros(1)] * num_layers\n self.prior_prob = 1e-2\n self.inplace = inplace\n stride = [8, 16, 32] # strides computed during build\n self.stride = torch.tensor(stride)\n self.use_dfl = use_dfl\n self.reg_max = reg_max\n self.proj_conv = nn.Conv2d(self.reg_max + 1, 1, 1, bias=False)\n self.grid_cell_offset = 0.5\n self.grid_cell_size = 5.0\n\n # Init decouple head\n self.stems = nn.ModuleList()\n self.cls_convs = nn.ModuleList()\n self.reg_convs = nn.ModuleList()\n self.cls_preds = nn.ModuleList()\n self.reg_preds = nn.ModuleList()\n\n # Efficient decoupled head layers\n for i in range(num_layers):\n idx = i*5\n self.stems.append(head_layers[idx])\n self.cls_convs.append(head_layers[idx+1])\n self.reg_convs.append(head_layers[idx+2])\n self.cls_preds.append(head_layers[idx+3])\n self.reg_preds.append(head_layers[idx+4])\n\n def initialize_biases(self):\n\n for conv in self.cls_preds:\n b = conv.bias.view(-1, )\n b.data.fill_(-math.log((1 - self.prior_prob) / self.prior_prob))\n conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n w = conv.weight\n w.data.fill_(0.)\n conv.weight = torch.nn.Parameter(w, requires_grad=True)\n\n for conv in self.reg_preds:\n b = conv.bias.view(-1, )\n b.data.fill_(1.0)\n conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n w = conv.weight\n w.data.fill_(0.)\n conv.weight = torch.nn.Parameter(w, requires_grad=True)\n \n self.proj = nn.Parameter(torch.linspace(0, self.reg_max, self.reg_max + 1), requires_grad=False)\n self.proj_conv.weight = nn.Parameter(self.proj.view([1, self.reg_max + 1, 1, 1]).clone().detach(),\n requires_grad=False)\n\n def forward(self, x):\n if self.training:\n cls_score_list = []\n reg_distri_list = []\n\n for i in range(self.nl):\n x[i] = self.stems[i](x[i])\n cls_x = x[i]\n reg_x = x[i]\n cls_feat = self.cls_convs[i](cls_x)\n cls_output = self.cls_preds[i](cls_feat)\n reg_feat = self.reg_convs[i](reg_x)\n reg_output = self.reg_preds[i](reg_feat)\n\n cls_output = torch.sigmoid(cls_output)\n cls_score_list.append(cls_output.flatten(2).permute((0, 2, 1)))\n reg_distri_list.append(reg_output.flatten(2).permute((0, 2, 1)))\n \n cls_score_list = torch.cat(cls_score_list, axis=1)\n reg_distri_list = torch.cat(reg_distri_list, axis=1)\n\n return x, cls_score_list, reg_distri_list\n else:\n cls_score_list = []\n reg_dist_list = []\n anchor_points, stride_tensor = generate_anchors(\n x, self.stride, self.grid_cell_size, self.grid_cell_offset, device=x[0].device, is_eval=True)\n\n for i in range(self.nl):\n b, _, h, w = x[i].shape\n l = h * w\n x[i] = self.stems[i](x[i])\n cls_x = x[i]\n reg_x = x[i]\n cls_feat = self.cls_convs[i](cls_x)\n cls_output = self.cls_preds[i](cls_feat)\n reg_feat = self.reg_convs[i](reg_x)\n reg_output = self.reg_preds[i](reg_feat)\n \n if self.use_dfl:\n reg_output = reg_output.reshape([-1, 4, self.reg_max + 1, l]).permute(0, 2, 1, 3)\n reg_output = self.proj_conv(F.softmax(reg_output, dim=1))\n \n cls_output = torch.sigmoid(cls_output)\n cls_score_list.append(cls_output.reshape([b, self.nc, l]))\n reg_dist_list.append(reg_output.reshape([b, 4, l]))\n \n cls_score_list = torch.cat(cls_score_list, axis=-1).permute(0, 2, 1)\n reg_dist_list = torch.cat(reg_dist_list, axis=-1).permute(0, 2, 1)\n\n\n pred_bboxes = dist2bbox(reg_dist_list, anchor_points, box_format='xywh')\n pred_bboxes *= stride_tensor\n return torch.cat(\n [\n pred_bboxes,\n torch.ones((b, pred_bboxes.shape[1], 1), device=pred_bboxes.device, dtype=pred_bboxes.dtype),\n cls_score_list\n ],\n axis=-1)\n\n\ndef build_effidehead_layer(channels_list, num_anchors, num_classes, reg_max=16):\n head_layers = nn.Sequential(\n # stem0\n Conv(\n in_channels=channels_list[6],\n out_channels=channels_list[6],\n kernel_size=1,\n stride=1\n ),\n # cls_conv0\n Conv(\n in_channels=channels_list[6],\n out_channels=channels_list[6],\n kernel_size=3,\n stride=1\n ),\n # reg_conv0\n Conv(\n in_channels=channels_list[6],\n out_channels=channels_list[6],\n kernel_size=3,\n stride=1\n ),\n # cls_pred0\n nn.Conv2d(\n in_channels=channels_list[6],\n out_channels=num_classes * num_anchors,\n kernel_size=1\n ),\n # reg_pred0\n nn.Conv2d(\n in_channels=channels_list[6],\n out_channels=4 * (reg_max + num_anchors),\n kernel_size=1\n ),\n # stem1\n Conv(\n in_channels=channels_list[8],\n out_channels=channels_list[8],\n kernel_size=1,\n stride=1\n ),\n # cls_conv1\n Conv(\n in_channels=channels_list[8],\n out_channels=channels_list[8],\n kernel_size=3,\n stride=1\n ),\n # reg_conv1\n Conv(\n in_channels=channels_list[8],\n out_channels=channels_list[8],\n kernel_size=3,\n stride=1\n ),\n # cls_pred1\n nn.Conv2d(\n in_channels=channels_list[8],\n out_channels=num_classes * num_anchors,\n kernel_size=1\n ),\n # reg_pred1\n nn.Conv2d(\n in_channels=channels_list[8],\n out_channels=4 * (reg_max + num_anchors),\n kernel_size=1\n ),\n # stem2\n Conv(\n in_channels=channels_list[10],\n out_channels=channels_list[10],\n kernel_size=1,\n stride=1\n ),\n # cls_conv2\n Conv(\n in_channels=channels_list[10],\n out_channels=channels_list[10],\n kernel_size=3,\n stride=1\n ),\n # reg_conv2\n Conv(\n in_channels=channels_list[10],\n out_channels=channels_list[10],\n kernel_size=3,\n stride=1\n ),\n # cls_pred2\n nn.Conv2d(\n in_channels=channels_list[10],\n out_channels=num_classes * num_anchors,\n kernel_size=1\n ),\n # reg_pred2\n nn.Conv2d(\n in_channels=channels_list[10],\n out_channels=4 * (reg_max + num_anchors),\n kernel_size=1\n )\n )\n return head_layers\n" }, { "path": "asone/detectors/yolov6/yolov6/models/end2end.py", "content": "import torch\nimport torch.nn as nn\nimport random\n\n\nclass ORT_NMS(torch.autograd.Function):\n '''ONNX-Runtime NMS operation'''\n @staticmethod\n def forward(ctx,\n boxes,\n scores,\n max_output_boxes_per_class=torch.tensor([100]),\n iou_threshold=torch.tensor([0.45]),\n score_threshold=torch.tensor([0.25])):\n device = boxes.device\n batch = scores.shape[0]\n num_det = random.randint(0, 100)\n batches = torch.randint(0, batch, (num_det,)).sort()[0].to(device)\n idxs = torch.arange(100, 100 + num_det).to(device)\n zeros = torch.zeros((num_det,), dtype=torch.int64).to(device)\n selected_indices = torch.cat([batches[None], zeros[None], idxs[None]], 0).T.contiguous()\n selected_indices = selected_indices.to(torch.int64)\n return selected_indices\n\n @staticmethod\n def symbolic(g, boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold):\n return g.op(\"NonMaxSuppression\", boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold)\n\n\nclass TRT8_NMS(torch.autograd.Function):\n '''TensorRT NMS operation'''\n @staticmethod\n def forward(\n ctx,\n boxes,\n scores,\n background_class=-1,\n box_coding=1,\n iou_threshold=0.45,\n max_output_boxes=100,\n plugin_version=\"1\",\n score_activation=0,\n score_threshold=0.25,\n ):\n batch_size, num_boxes, num_classes = scores.shape\n num_det = torch.randint(0, max_output_boxes, (batch_size, 1), dtype=torch.int32)\n det_boxes = torch.randn(batch_size, max_output_boxes, 4)\n det_scores = torch.randn(batch_size, max_output_boxes)\n det_classes = torch.randint(0, num_classes, (batch_size, max_output_boxes), dtype=torch.int32)\n return num_det, det_boxes, det_scores, det_classes\n\n @staticmethod\n def symbolic(g,\n boxes,\n scores,\n background_class=-1,\n box_coding=1,\n iou_threshold=0.45,\n max_output_boxes=100,\n plugin_version=\"1\",\n score_activation=0,\n score_threshold=0.25):\n out = g.op(\"TRT::EfficientNMS_TRT\",\n boxes,\n scores,\n background_class_i=background_class,\n box_coding_i=box_coding,\n iou_threshold_f=iou_threshold,\n max_output_boxes_i=max_output_boxes,\n plugin_version_s=plugin_version,\n score_activation_i=score_activation,\n score_threshold_f=score_threshold,\n outputs=4)\n nums, boxes, scores, classes = out\n return nums, boxes, scores, classes\n\nclass TRT7_NMS(torch.autograd.Function):\n '''TensorRT NMS operation'''\n @staticmethod\n def forward(\n ctx,\n boxes,\n scores,\n plugin_version=\"1\",\n shareLocation=1,\n backgroundLabelId=-1,\n numClasses=80,\n topK=1000,\n keepTopK=100,\n scoreThreshold=0.25,\n iouThreshold=0.45,\n isNormalized=0,\n clipBoxes=0,\n scoreBits=16,\n caffeSemantics=1,\n ):\n batch_size, num_boxes, numClasses = scores.shape\n num_det = torch.randint(0, keepTopK, (batch_size, 1), dtype=torch.int32)\n det_boxes = torch.randn(batch_size, keepTopK, 4)\n det_scores = torch.randn(batch_size, keepTopK)\n det_classes = torch.randint(0, numClasses, (batch_size, keepTopK)).float()\n return num_det, det_boxes, det_scores, det_classes\n @staticmethod\n def symbolic(g,\n boxes,\n scores,\n plugin_version='1',\n shareLocation=1,\n backgroundLabelId=-1,\n numClasses=80,\n topK=1000,\n keepTopK=100,\n scoreThreshold=0.25,\n iouThreshold=0.45,\n isNormalized=0,\n clipBoxes=0,\n scoreBits=16,\n caffeSemantics=1,\n ):\n out = g.op(\"TRT::BatchedNMSDynamic_TRT\", # BatchedNMS_TRT BatchedNMSDynamic_TRT\n boxes,\n scores,\n shareLocation_i=shareLocation,\n plugin_version_s=plugin_version,\n backgroundLabelId_i=backgroundLabelId,\n numClasses_i=numClasses,\n topK_i=topK,\n keepTopK_i=keepTopK,\n scoreThreshold_f=scoreThreshold,\n iouThreshold_f=iouThreshold,\n isNormalized_i=isNormalized,\n clipBoxes_i=clipBoxes,\n scoreBits_i=scoreBits,\n caffeSemantics_i=caffeSemantics,\n outputs=4)\n nums, boxes, scores, classes = out\n return nums, boxes, scores, classes\n\n\nclass ONNX_ORT(nn.Module):\n '''onnx module with ONNX-Runtime NMS operation.'''\n def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=640, device=None):\n super().__init__()\n self.device = device if device else torch.device(\"cpu\")\n self.max_obj = torch.tensor([max_obj]).to(device)\n self.iou_threshold = torch.tensor([iou_thres]).to(device)\n self.score_threshold = torch.tensor([score_thres]).to(device)\n self.max_wh = max_wh\n self.convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],\n dtype=torch.float32,\n device=self.device)\n\n def forward(self, x):\n box = x[:, :, :4]\n conf = x[:, :, 4:5]\n score = x[:, :, 5:]\n score *= conf\n box @= self.convert_matrix\n objScore, objCls = score.max(2, keepdim=True)\n dis = objCls.float() * self.max_wh\n nmsbox = box + dis\n objScore1 = objScore.transpose(1, 2).contiguous()\n selected_indices = ORT_NMS.apply(nmsbox, objScore1, self.max_obj, self.iou_threshold, self.score_threshold)\n X, Y = selected_indices[:, 0], selected_indices[:, 2]\n resBoxes = box[X, Y, :]\n resClasses = objCls[X, Y, :].float()\n resScores = objScore[X, Y, :]\n X = X.unsqueeze(1).float()\n return torch.cat([X, resBoxes, resClasses, resScores], 1)\n\nclass ONNX_TRT7(nn.Module):\n '''onnx module with TensorRT NMS operation.'''\n def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None ,device=None):\n super().__init__()\n assert max_wh is None\n self.device = device if device else torch.device('cpu')\n self.shareLocation = 1\n self.backgroundLabelId = -1\n self.numClasses = 80\n self.topK = 1000\n self.keepTopK = max_obj\n self.scoreThreshold = score_thres\n self.iouThreshold = iou_thres\n self.isNormalized = 0\n self.clipBoxes = 0\n self.scoreBits = 16\n self.caffeSemantics = 1\n self.plugin_version = '1'\n self.convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],\n dtype=torch.float32,\n device=self.device)\n def forward(self, x):\n box = x[:, :, :4]\n conf = x[:, :, 4:5]\n score = x[:, :, 5:]\n score *= conf\n box @= self.convert_matrix\n box = box.unsqueeze(2)\n self.numClasses = int(score.shape[2])\n num_det, det_boxes, det_scores, det_classes = TRT7_NMS.apply(box, score, self.plugin_version,\n self.shareLocation,\n self.backgroundLabelId,\n self.numClasses,\n self.topK,\n self.keepTopK,\n self.scoreThreshold,\n self.iouThreshold,\n self.isNormalized,\n self.clipBoxes,\n self.scoreBits,\n self.caffeSemantics,\n )\n return num_det, det_boxes, det_scores, det_classes.int()\n\n\nclass ONNX_TRT8(nn.Module):\n '''onnx module with TensorRT NMS operation.'''\n def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None ,device=None):\n super().__init__()\n assert max_wh is None\n self.device = device if device else torch.device('cpu')\n self.background_class = -1,\n self.box_coding = 1,\n self.iou_threshold = iou_thres\n self.max_obj = max_obj\n self.plugin_version = '1'\n self.score_activation = 0\n self.score_threshold = score_thres\n\n def forward(self, x):\n box = x[:, :, :4]\n conf = x[:, :, 4:5]\n score = x[:, :, 5:]\n score *= conf\n num_det, det_boxes, det_scores, det_classes = TRT8_NMS.apply(box, score, self.background_class, self.box_coding,\n self.iou_threshold, self.max_obj,\n self.plugin_version, self.score_activation,\n self.score_threshold)\n return num_det, det_boxes, det_scores, det_classes\n\n\nclass End2End(nn.Module):\n '''export onnx or tensorrt model with NMS operation.'''\n def __init__(self, model, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None, device=None, trt_version=8, with_preprocess=False):\n super().__init__()\n device = device if device else torch.device('cpu')\n self.with_preprocess = with_preprocess\n self.model = model.to(device)\n TRT = ONNX_TRT8 if trt_version >= 8 else ONNX_TRT7\n self.patch_model = TRT if max_wh is None else ONNX_ORT\n self.end2end = self.patch_model(max_obj, iou_thres, score_thres, max_wh, device)\n self.end2end.eval()\n\n def forward(self, x):\n if self.with_preprocess:\n x = x[:,[2,1,0],...]\n x = x * (1/255)\n x = self.model(x)\n x = self.end2end(x)\n return x\n" }, { "path": "asone/detectors/yolov6/yolov6/models/loss.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport torch.nn.functional as F\nfrom asone.detectors.yolov6.yolov6.assigners.anchor_generator import generate_anchors\nfrom asone.detectors.yolov6.yolov6.utils.general import dist2bbox, bbox2dist, xywh2xyxy\nfrom asone.detectors.yolov6.yolov6.utils.figure_iou import IOUloss\nfrom asone.detectors.yolov6.yolov6.assigners.atss_assigner import ATSSAssigner\nfrom asone.detectors.yolov6.yolov6.assigners.tal_assigner import TaskAlignedAssigner\n\n\nclass ComputeLoss:\n '''Loss computation func.'''\n def __init__(self, \n fpn_strides=[8, 16, 32],\n grid_cell_size=5.0,\n grid_cell_offset=0.5,\n num_classes=80,\n ori_img_size=640,\n warmup_epoch=4,\n use_dfl=True,\n reg_max=16,\n iou_type='giou',\n loss_weight={\n 'class': 1.0,\n 'iou': 2.5,\n 'dfl': 0.5}\n ):\n \n self.fpn_strides = fpn_strides\n self.grid_cell_size = grid_cell_size\n self.grid_cell_offset = grid_cell_offset\n self.num_classes = num_classes\n self.ori_img_size = ori_img_size\n \n self.warmup_epoch = warmup_epoch\n self.warmup_assigner = ATSSAssigner(9, num_classes=self.num_classes)\n self.formal_assigner = TaskAlignedAssigner(topk=13, num_classes=self.num_classes, alpha=1.0, beta=6.0)\n\n self.use_dfl = use_dfl\n self.reg_max = reg_max\n self.proj = nn.Parameter(torch.linspace(0, self.reg_max, self.reg_max + 1), requires_grad=False)\n self.iou_type = iou_type\n self.varifocal_loss = VarifocalLoss().cuda()\n self.bbox_loss = BboxLoss(self.num_classes, self.reg_max, self.use_dfl, self.iou_type).cuda()\n self.loss_weight = loss_weight \n \n def __call__(\n self,\n outputs,\n targets,\n epoch_num\n ):\n \n feats, pred_scores, pred_distri = outputs\n anchors, anchor_points, n_anchors_list, stride_tensor = \\\n generate_anchors(feats, self.fpn_strides, self.grid_cell_size, self.grid_cell_offset, device=feats[0].device)\n \n assert pred_scores.type() == pred_distri.type()\n gt_bboxes_scale = torch.full((1,4), self.ori_img_size).type_as(pred_scores)\n batch_size = pred_scores.shape[0]\n\n # targets\n targets =self.preprocess(targets, batch_size, gt_bboxes_scale)\n gt_labels = targets[:, :, :1]\n gt_bboxes = targets[:, :, 1:] #xyxy\n mask_gt = (gt_bboxes.sum(-1, keepdim=True) > 0).float()\n \n # pboxes\n anchor_points_s = anchor_points / stride_tensor\n pred_bboxes = self.bbox_decode(anchor_points_s, pred_distri) #xyxy\n\n if epoch_num < self.warmup_epoch:\n target_labels, target_bboxes, target_scores, fg_mask = \\\n self.warmup_assigner(\n anchors,\n n_anchors_list,\n gt_labels,\n gt_bboxes,\n mask_gt,\n pred_bboxes.detach() * stride_tensor)\n else:\n target_labels, target_bboxes, target_scores, fg_mask = \\\n self.formal_assigner(\n pred_scores.detach(),\n pred_bboxes.detach() * stride_tensor,\n anchor_points,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n # rescale bbox\n target_bboxes /= stride_tensor\n\n # cls loss\n target_labels = torch.where(fg_mask > 0, target_labels, torch.full_like(target_labels, self.num_classes))\n one_hot_label = F.one_hot(target_labels.long(), self.num_classes + 1)[..., :-1]\n loss_cls = self.varifocal_loss(pred_scores, target_scores, one_hot_label)\n \n target_scores_sum = target_scores.sum()\n loss_cls /= target_scores_sum\n \n # bbox loss\n loss_iou, loss_dfl = self.bbox_loss(pred_distri, pred_bboxes, anchor_points_s, target_bboxes, \n target_scores, target_scores_sum, fg_mask)\n \n loss = self.loss_weight['class'] * loss_cls + \\\n self.loss_weight['iou'] * loss_iou + \\\n self.loss_weight['dfl'] * loss_dfl\n \n return loss, \\\n torch.cat(((self.loss_weight['iou'] * loss_iou).unsqueeze(0), \n (self.loss_weight['dfl'] * loss_dfl).unsqueeze(0),\n (self.loss_weight['class'] * loss_cls).unsqueeze(0))).detach()\n \n def preprocess(self, targets, batch_size, scale_tensor):\n targets_list = np.zeros((batch_size, 1, 5)).tolist() \n for i, item in enumerate(targets.cpu().numpy().tolist()):\n targets_list[int(item[0])].append(item[1:])\n max_len = max((len(l) for l in targets_list))\n targets = torch.from_numpy(np.array(list(map(lambda l:l + [[-1,0,0,0,0]]*(max_len - len(l)), targets_list)))[:,1:,:]).to(targets.device)\n batch_target = targets[:, :, 1:5].mul_(scale_tensor)\n targets[..., 1:] = xywh2xyxy(batch_target)\n return targets\n \n def bbox_decode(self, anchor_points, pred_dist):\n if self.use_dfl:\n batch_size, n_anchors, _ = pred_dist.shape\n pred_dist = F.softmax(pred_dist.view(batch_size, n_anchors, 4, self.reg_max + 1), dim=-1).matmul(self.proj.to(pred_dist.device))\n return dist2bbox(pred_dist, anchor_points)\n\n\nclass VarifocalLoss(nn.Module):\n def __init__(self):\n super(VarifocalLoss, self).__init__()\n\n def forward(self, pred_score,gt_score, label, alpha=0.75, gamma=2.0):\n\n weight = alpha * pred_score.pow(gamma) * (1 - label) + gt_score * label\n with torch.cuda.amp.autocast(enabled=False):\n loss = (F.binary_cross_entropy(pred_score.float(), gt_score.float(), reduction='none') * weight).sum()\n\n return loss\n\n\nclass BboxLoss(nn.Module):\n def __init__(self, num_classes, reg_max, use_dfl=False, iou_type='giou'):\n super(BboxLoss, self).__init__()\n self.num_classes = num_classes\n self.iou_loss = IOUloss(box_format='xyxy', iou_type=iou_type, eps=1e-10)\n self.reg_max = reg_max\n self.use_dfl = use_dfl\n\n def forward(self, pred_dist, pred_bboxes, anchor_points,\n target_bboxes, target_scores, target_scores_sum, fg_mask):\n \n # select positive samples mask\n num_pos = fg_mask.sum()\n if num_pos > 0:\n # iou loss\n bbox_mask = fg_mask.unsqueeze(-1).repeat([1, 1, 4])\n pred_bboxes_pos = torch.masked_select(pred_bboxes,\n bbox_mask).reshape([-1, 4])\n target_bboxes_pos = torch.masked_select(\n target_bboxes, bbox_mask).reshape([-1, 4])\n bbox_weight = torch.masked_select(\n target_scores.sum(-1), fg_mask).unsqueeze(-1)\n loss_iou = self.iou_loss(pred_bboxes_pos,\n target_bboxes_pos) * bbox_weight\n loss_iou = loss_iou.sum() / target_scores_sum\n \n # dfl loss\n if self.use_dfl:\n dist_mask = fg_mask.unsqueeze(-1).repeat(\n [1, 1, (self.reg_max + 1) * 4])\n pred_dist_pos = torch.masked_select(\n pred_dist, dist_mask).reshape([-1, 4, self.reg_max + 1])\n target_ltrb = bbox2dist(anchor_points, target_bboxes, self.reg_max)\n target_ltrb_pos = torch.masked_select(\n target_ltrb, bbox_mask).reshape([-1, 4])\n loss_dfl = self._df_loss(pred_dist_pos,\n target_ltrb_pos) * bbox_weight\n loss_dfl = loss_dfl.sum() / target_scores_sum\n else:\n loss_dfl = torch.tensor(0.).to(pred_dist.device)\n\n else:\n loss_iou = torch.tensor(0.).to(pred_dist.device)\n loss_dfl = torch.tensor(0.).to(pred_dist.device)\n\n return loss_iou, loss_dfl\n\n def _df_loss(self, pred_dist, target):\n target_left = target.to(torch.long)\n target_right = target_left + 1\n weight_left = target_right.to(torch.float) - target\n weight_right = 1 - weight_left\n loss_left = F.cross_entropy(\n pred_dist.view(-1, self.reg_max + 1), target_left.view(-1), reduction='none').view(\n target_left.shape) * weight_left\n loss_right = F.cross_entropy(\n pred_dist.view(-1, self.reg_max + 1), target_right.view(-1), reduction='none').view(\n target_left.shape) * weight_right\n return (loss_left + loss_right).mean(-1, keepdim=True)\n\n" }, { "path": "asone/detectors/yolov6/yolov6/models/loss_distill.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport torch.nn.functional as F\nfrom asone.detectors.yolov6.yolov6.assigners.anchor_generator import generate_anchors\nfrom asone.detectors.yolov6.yolov6.utils.general import dist2bbox, bbox2dist, xywh2xyxy\nfrom asone.detectors.yolov6.yolov6.utils.figure_iou import IOUloss\nfrom asone.detectors.yolov6.yolov6.assigners.atss_assigner import ATSSAssigner\nfrom asone.detectors.yolov6.yolov6.assigners.tal_assigner import TaskAlignedAssigner\n\n\nclass ComputeLoss:\n '''Loss computation func.'''\n def __init__(self, \n fpn_strides=[8, 16, 32],\n grid_cell_size=5.0,\n grid_cell_offset=0.5,\n num_classes=80,\n ori_img_size=640,\n warmup_epoch=4,\n use_dfl=True,\n reg_max=16,\n iou_type='giou',\n loss_weight={\n 'class': 1.0,\n 'iou': 2.5,\n 'dfl': 0.5,\n 'cwd': 10.0},\n distill_feat = False,\n distill_weight={\n 'class': 1.0,\n 'dfl': 1.0,\n }\n ):\n \n self.fpn_strides = fpn_strides\n self.grid_cell_size = grid_cell_size\n self.grid_cell_offset = grid_cell_offset\n self.num_classes = num_classes\n self.ori_img_size = ori_img_size\n \n self.warmup_epoch = warmup_epoch\n self.warmup_assigner = ATSSAssigner(9, num_classes=self.num_classes)\n self.formal_assigner = TaskAlignedAssigner(topk=13, num_classes=self.num_classes, alpha=1.0, beta=6.0)\n\n self.use_dfl = use_dfl\n self.reg_max = reg_max\n self.proj = nn.Parameter(torch.linspace(0, self.reg_max, self.reg_max + 1), requires_grad=False)\n self.iou_type = iou_type\n self.varifocal_loss = VarifocalLoss().cuda()\n self.bbox_loss = BboxLoss(self.num_classes, self.reg_max, self.use_dfl, self.iou_type).cuda()\n self.loss_weight = loss_weight\n self.distill_feat = distill_feat\n self.distill_weight = distill_weight\n \n def __call__(\n self,\n outputs,\n t_outputs,\n s_featmaps,\n t_featmaps,\n targets,\n epoch_num,\n max_epoch,\n temperature\n ):\n \n feats, pred_scores, pred_distri = outputs\n t_feats, t_pred_scores, t_pred_distri = t_outputs\n anchors, anchor_points, n_anchors_list, stride_tensor = \\\n generate_anchors(feats, self.fpn_strides, self.grid_cell_size, self.grid_cell_offset, device=feats[0].device)\n t_anchors, t_anchor_points, t_n_anchors_list, t_stride_tensor = \\\n generate_anchors(t_feats, self.fpn_strides, self.grid_cell_size, self.grid_cell_offset, device=feats[0].device)\n\n assert pred_scores.type() == pred_distri.type()\n gt_bboxes_scale = torch.full((1,4), self.ori_img_size).type_as(pred_scores)\n batch_size = pred_scores.shape[0]\n\n # targets\n targets =self.preprocess(targets, batch_size, gt_bboxes_scale)\n gt_labels = targets[:, :, :1]\n gt_bboxes = targets[:, :, 1:] #xyxy\n mask_gt = (gt_bboxes.sum(-1, keepdim=True) > 0).float()\n \n # pboxes\n anchor_points_s = anchor_points / stride_tensor\n pred_bboxes = self.bbox_decode(anchor_points_s, pred_distri) #xyxy\n t_anchor_points_s = t_anchor_points / t_stride_tensor\n t_pred_bboxes = self.bbox_decode(t_anchor_points_s, t_pred_distri) #xyxy\n\n if epoch_num < self.warmup_epoch:\n target_labels, target_bboxes, target_scores, fg_mask = \\\n self.warmup_assigner(\n anchors,\n n_anchors_list,\n gt_labels,\n gt_bboxes,\n mask_gt,\n pred_bboxes.detach() * stride_tensor)\n else:\n target_labels, target_bboxes, target_scores, fg_mask = \\\n self.formal_assigner(\n pred_scores.detach(),\n pred_bboxes.detach() * stride_tensor,\n anchor_points,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n # rescale bbox\n target_bboxes /= stride_tensor\n\n # cls loss\n target_labels = torch.where(fg_mask > 0, target_labels, torch.full_like(target_labels, self.num_classes))\n one_hot_label = F.one_hot(target_labels, self.num_classes + 1)[..., :-1]\n loss_cls = self.varifocal_loss(pred_scores, target_scores, one_hot_label)\n \n target_scores_sum = target_scores.sum()\n loss_cls /= target_scores_sum\n \n # bbox loss\n loss_iou, loss_dfl, d_loss_dfl = self.bbox_loss(pred_distri, pred_bboxes, t_pred_distri, t_pred_bboxes, temperature, anchor_points_s,\n target_bboxes, target_scores, target_scores_sum, fg_mask)\n \n logits_student = pred_scores\n logits_teacher = t_pred_scores\n distill_num_classes = self.num_classes\n d_loss_cls = self.distill_loss_cls(logits_student, logits_teacher, distill_num_classes, temperature)\n if self.distill_feat:\n d_loss_cw = self.distill_loss_cw(s_featmaps, t_featmaps)\n else:\n d_loss_cw = torch.tensor(0.).to(feats[0].device)\n import math\n distill_weightdecay = ((1 - math.cos(epoch_num * math.pi / max_epoch)) / 2) * (0.01- 1) + 1\n d_loss_dfl *= distill_weightdecay\n d_loss_cls *= distill_weightdecay\n d_loss_cw *= distill_weightdecay\n loss_cls_all = loss_cls + d_loss_cls * self.distill_weight['class']\n loss_dfl_all = loss_dfl + d_loss_dfl * self.distill_weight['dfl']\n loss = self.loss_weight['class'] * loss_cls_all + \\\n self.loss_weight['iou'] * loss_iou + \\\n self.loss_weight['dfl'] * loss_dfl_all + \\\n self.loss_weight['cwd'] * d_loss_cw\n \n return loss, \\\n torch.cat(((self.loss_weight['iou'] * loss_iou).unsqueeze(0), \n (self.loss_weight['dfl'] * loss_dfl_all).unsqueeze(0),\n (self.loss_weight['class'] * loss_cls_all).unsqueeze(0),\n (self.loss_weight['cwd'] * d_loss_cw).unsqueeze(0))).detach()\n\n def distill_loss_cls(self, logits_student, logits_teacher, num_classes, temperature=20):\n logits_student = logits_student.view(-1, num_classes)\n logits_teacher = logits_teacher.view(-1, num_classes)\n pred_student = F.softmax(logits_student / temperature, dim=1)\n pred_teacher = F.softmax(logits_teacher / temperature, dim=1)\n log_pred_student = torch.log(pred_student)\n\n d_loss_cls = F.kl_div(log_pred_student, pred_teacher, reduction=\"sum\")\n d_loss_cls *= temperature**2\n return d_loss_cls\n\n def distill_loss_cw(self, s_feats, t_feats, temperature=1):\n N,C,H,W = s_feats[0].shape\n # print(N,C,H,W)\n loss_cw = F.kl_div(F.log_softmax(s_feats[0].view(N,C,H*W)/temperature, dim=2),\n F.log_softmax(t_feats[0].view(N,C,H*W).detach()/temperature, dim=2),\n reduction='sum',\n log_target=True) * (temperature * temperature)/ (N*C)\n\n N,C,H,W = s_feats[1].shape\n # print(N,C,H,W)\n loss_cw += F.kl_div(F.log_softmax(s_feats[1].view(N,C,H*W)/temperature, dim=2),\n F.log_softmax(t_feats[1].view(N,C,H*W).detach()/temperature, dim=2),\n reduction='sum',\n log_target=True) * (temperature * temperature)/ (N*C)\n\n N,C,H,W = s_feats[2].shape\n # print(N,C,H,W)\n loss_cw += F.kl_div(F.log_softmax(s_feats[2].view(N,C,H*W)/temperature, dim=2),\n F.log_softmax(t_feats[2].view(N,C,H*W).detach()/temperature, dim=2),\n reduction='sum',\n log_target=True) * (temperature * temperature)/ (N*C)\n # print(loss_cw)\n return loss_cw\n \n def preprocess(self, targets, batch_size, scale_tensor):\n targets_list = np.zeros((batch_size, 1, 5)).tolist() \n for i, item in enumerate(targets.cpu().numpy().tolist()):\n targets_list[int(item[0])].append(item[1:])\n max_len = max((len(l) for l in targets_list))\n targets = torch.from_numpy(np.array(list(map(lambda l:l + [[-1,0,0,0,0]]*(max_len - len(l)), targets_list)))[:,1:,:]).to(targets.device)\n batch_target = targets[:, :, 1:5].mul_(scale_tensor)\n targets[..., 1:] = xywh2xyxy(batch_target)\n return targets\n \n def bbox_decode(self, anchor_points, pred_dist):\n if self.use_dfl:\n batch_size, n_anchors, _ = pred_dist.shape\n pred_dist = F.softmax(pred_dist.view(batch_size, n_anchors, 4, self.reg_max + 1), dim=-1).matmul(self.proj.to(pred_dist.device))\n return dist2bbox(pred_dist, anchor_points)\n\n\nclass VarifocalLoss(nn.Module):\n def __init__(self):\n super(VarifocalLoss, self).__init__()\n\n def forward(self, pred_score,gt_score, label, alpha=0.75, gamma=2.0):\n\n weight = alpha * pred_score.pow(gamma) * (1 - label) + gt_score * label\n with torch.cuda.amp.autocast(enabled=False):\n loss = (F.binary_cross_entropy(pred_score.float(), gt_score.float(), reduction='none') * weight).sum()\n\n return loss\n\n\nclass BboxLoss(nn.Module):\n def __init__(self, num_classes, reg_max, use_dfl=False, iou_type='giou'):\n super(BboxLoss, self).__init__()\n self.num_classes = num_classes\n self.iou_loss = IOUloss(box_format='xyxy', iou_type=iou_type, eps=1e-10)\n self.reg_max = reg_max\n self.use_dfl = use_dfl\n\n def forward(self, pred_dist, pred_bboxes, t_pred_dist, t_pred_bboxes, temperature, anchor_points, \n target_bboxes, target_scores, target_scores_sum, fg_mask):\n # select positive samples mask\n num_pos = fg_mask.sum()\n if num_pos > 0:\n # iou loss\n bbox_mask = fg_mask.unsqueeze(-1).repeat([1, 1, 4])\n pred_bboxes_pos = torch.masked_select(pred_bboxes,\n bbox_mask).reshape([-1, 4])\n t_pred_bboxes_pos = torch.masked_select(t_pred_bboxes,\n bbox_mask).reshape([-1, 4])\n target_bboxes_pos = torch.masked_select(\n target_bboxes, bbox_mask).reshape([-1, 4])\n bbox_weight = torch.masked_select(\n target_scores.sum(-1), fg_mask).unsqueeze(-1)\n loss_iou = self.iou_loss(pred_bboxes_pos,\n target_bboxes_pos) * bbox_weight\n loss_iou = loss_iou.sum() / target_scores_sum\n \n # dfl loss\n if self.use_dfl:\n dist_mask = fg_mask.unsqueeze(-1).repeat(\n [1, 1, (self.reg_max + 1) * 4])\n pred_dist_pos = torch.masked_select(\n pred_dist, dist_mask).reshape([-1, 4, self.reg_max + 1])\n t_pred_dist_pos = torch.masked_select(\n t_pred_dist, dist_mask).reshape([-1, 4, self.reg_max + 1])\n target_ltrb = bbox2dist(anchor_points, target_bboxes, self.reg_max)\n target_ltrb_pos = torch.masked_select(\n target_ltrb, bbox_mask).reshape([-1, 4])\n loss_dfl = self._df_loss(pred_dist_pos,\n target_ltrb_pos) * bbox_weight\n d_loss_dfl = self.distill_loss_dfl(pred_dist_pos, t_pred_dist_pos, temperature) * bbox_weight\n loss_dfl = loss_dfl.sum() / target_scores_sum\n d_loss_dfl = d_loss_dfl.sum() / target_scores_sum\n else:\n loss_dfl = torch.tensor(0.).to(pred_dist.device)\n d_loss_dfl = torch.tensor(0.).to(pred_dist.device)\n\n else:\n\n loss_iou = torch.tensor(0.).to(pred_dist.device)\n loss_dfl = torch.tensor(0.).to(pred_dist.device)\n d_loss_dfl = torch.tensor(0.).to(pred_dist.device)\n\n return loss_iou, loss_dfl, d_loss_dfl\n\n def _df_loss(self, pred_dist, target):\n target_left = target.to(torch.long)\n target_right = target_left + 1\n weight_left = target_right.to(torch.float) - target\n weight_right = 1 - weight_left\n loss_left = F.cross_entropy(\n pred_dist.view(-1, self.reg_max + 1), target_left.view(-1), reduction='none').view(\n target_left.shape) * weight_left\n loss_right = F.cross_entropy(\n pred_dist.view(-1, self.reg_max + 1), target_right.view(-1), reduction='none').view(\n target_left.shape) * weight_right\n return (loss_left + loss_right).mean(-1, keepdim=True)\n\n def distill_loss_dfl(self, logits_student, logits_teacher, temperature=20):\n\n logits_student = logits_student.view(-1,17)\n logits_teacher = logits_teacher.view(-1,17)\n pred_student = F.softmax(logits_student / temperature, dim=1)\n pred_teacher = F.softmax(logits_teacher / temperature, dim=1)\n log_pred_student = torch.log(pred_student)\n\n d_loss_dfl = F.kl_div(log_pred_student, pred_teacher, reduction=\"none\").sum(1).mean()\n d_loss_dfl *= temperature**2\n return d_loss_dfl\n" }, { "path": "asone/detectors/yolov6/yolov6/models/reppan.py", "content": "import torch\nfrom torch import nn\nfrom asone.detectors.yolov6.yolov6.layers.common import RepBlock, RepVGGBlock, BottleRep, BepC3, SimConv, Transpose\n\n_QUANT=False\nclass RepPANNeck(nn.Module):\n \"\"\"RepPANNeck Module\n EfficientRep is the default backbone of this model.\n RepPANNeck has the balance of feature fusion ability and hardware efficiency.\n \"\"\"\n\n def __init__(\n self,\n channels_list=None,\n num_repeats=None,\n block=RepVGGBlock\n ):\n super().__init__()\n\n assert channels_list is not None\n assert num_repeats is not None\n\n self.Rep_p4 = RepBlock(\n in_channels=channels_list[3] + channels_list[5],\n out_channels=channels_list[5],\n n=num_repeats[5],\n block=block\n )\n\n self.Rep_p3 = RepBlock(\n in_channels=channels_list[2] + channels_list[6],\n out_channels=channels_list[6],\n n=num_repeats[6],\n block=block\n )\n\n self.Rep_n3 = RepBlock(\n in_channels=channels_list[6] + channels_list[7],\n out_channels=channels_list[8],\n n=num_repeats[7],\n block=block\n )\n\n self.Rep_n4 = RepBlock(\n in_channels=channels_list[5] + channels_list[9],\n out_channels=channels_list[10],\n n=num_repeats[8],\n block=block\n )\n\n self.reduce_layer0 = SimConv(\n in_channels=channels_list[4],\n out_channels=channels_list[5],\n kernel_size=1,\n stride=1\n )\n\n self.upsample0 = Transpose(\n in_channels=channels_list[5],\n out_channels=channels_list[5],\n )\n\n self.reduce_layer1 = SimConv(\n in_channels=channels_list[5],\n out_channels=channels_list[6],\n kernel_size=1,\n stride=1\n )\n\n self.upsample1 = Transpose(\n in_channels=channels_list[6],\n out_channels=channels_list[6]\n )\n\n self.downsample2 = SimConv(\n in_channels=channels_list[6],\n out_channels=channels_list[7],\n kernel_size=3,\n stride=2\n )\n\n self.downsample1 = SimConv(\n in_channels=channels_list[8],\n out_channels=channels_list[9],\n kernel_size=3,\n stride=2\n )\n\n def upsample_enable_quant(self):\n print(\"Insert fakequant after upsample\")\n # Insert fakequant after upsample op to build TensorRT engine\n from pytorch_quantization import nn as quant_nn\n from pytorch_quantization.tensor_quant import QuantDescriptor\n conv2d_input_default_desc = QuantDescriptor(num_bits=8, calib_method='histogram')\n self.upsample_feat0_quant = quant_nn.TensorQuantizer(conv2d_input_default_desc)\n self.upsample_feat1_quant = quant_nn.TensorQuantizer(conv2d_input_default_desc)\n global _QUANT\n _QUANT = True\n\n def forward(self, input):\n\n (x2, x1, x0) = input\n\n fpn_out0 = self.reduce_layer0(x0)\n upsample_feat0 = self.upsample0(fpn_out0)\n if _QUANT:\n upsample_feat0 = self.upsample_feat0_quant(upsample_feat0)\n f_concat_layer0 = torch.cat([upsample_feat0, x1], 1)\n f_out0 = self.Rep_p4(f_concat_layer0)\n\n fpn_out1 = self.reduce_layer1(f_out0)\n upsample_feat1 = self.upsample1(fpn_out1)\n if _QUANT:\n upsample_feat1 = self.upsample_feat1_quant(upsample_feat1)\n f_concat_layer1 = torch.cat([upsample_feat1, x2], 1)\n pan_out2 = self.Rep_p3(f_concat_layer1)\n\n down_feat1 = self.downsample2(pan_out2)\n p_concat_layer1 = torch.cat([down_feat1, fpn_out1], 1)\n pan_out1 = self.Rep_n3(p_concat_layer1)\n\n down_feat0 = self.downsample1(pan_out1)\n p_concat_layer2 = torch.cat([down_feat0, fpn_out0], 1)\n pan_out0 = self.Rep_n4(p_concat_layer2)\n\n outputs = [pan_out2, pan_out1, pan_out0]\n\n return outputs\n\n\nclass CSPRepPANNeck(nn.Module):\n \"\"\"\n CSPRepPANNeck module. \n \"\"\"\n\n def __init__(\n self,\n channels_list=None,\n num_repeats=None,\n block=BottleRep,\n csp_e=float(1)/2\n ):\n super().__init__()\n\n assert channels_list is not None\n assert num_repeats is not None\n \n self.Rep_p4 = BepC3(\n in_channels=channels_list[3] + channels_list[5], # 512 + 256\n out_channels=channels_list[5], # 256\n n=num_repeats[5],\n e=csp_e,\n block=block\n )\n\n self.Rep_p3 = BepC3(\n in_channels=channels_list[2] + channels_list[6], # 256 + 128\n out_channels=channels_list[6], # 128 \n n=num_repeats[6],\n e=csp_e,\n block=block\n )\n\n self.Rep_n3 = BepC3(\n in_channels=channels_list[6] + channels_list[7], # 128 + 128 \n out_channels=channels_list[8], # 256\n n=num_repeats[7],\n e=csp_e,\n block=block\n )\n\n self.Rep_n4 = BepC3(\n in_channels=channels_list[5] + channels_list[9], # 256 + 256 \n out_channels=channels_list[10], # 512 \n n=num_repeats[8],\n e=csp_e,\n block=block\n )\n\n self.reduce_layer0 = SimConv(\n in_channels=channels_list[4], # 1024 \n out_channels=channels_list[5], # 256 \n kernel_size=1,\n stride=1\n )\n\n self.upsample0 = Transpose(\n in_channels=channels_list[5], # 256 \n out_channels=channels_list[5], # 256 \n )\n\n self.reduce_layer1 = SimConv(\n in_channels=channels_list[5], # 256\n out_channels=channels_list[6], # 128 \n kernel_size=1,\n stride=1\n )\n\n self.upsample1 = Transpose(\n in_channels=channels_list[6], # 128 \n out_channels=channels_list[6] # 128 \n )\n\n self.downsample2 = SimConv(\n in_channels=channels_list[6], # 128 \n out_channels=channels_list[7], # 128 \n kernel_size=3,\n stride=2\n )\n\n self.downsample1 = SimConv(\n in_channels=channels_list[8], # 256 \n out_channels=channels_list[9], # 256 \n kernel_size=3,\n stride=2\n )\n\n def forward(self, input):\n\n (x2, x1, x0) = input\n\n fpn_out0 = self.reduce_layer0(x0)\n upsample_feat0 = self.upsample0(fpn_out0)\n f_concat_layer0 = torch.cat([upsample_feat0, x1], 1)\n f_out0 = self.Rep_p4(f_concat_layer0)\n\n fpn_out1 = self.reduce_layer1(f_out0)\n upsample_feat1 = self.upsample1(fpn_out1)\n f_concat_layer1 = torch.cat([upsample_feat1, x2], 1)\n pan_out2 = self.Rep_p3(f_concat_layer1)\n\n down_feat1 = self.downsample2(pan_out2)\n p_concat_layer1 = torch.cat([down_feat1, fpn_out1], 1)\n pan_out1 = self.Rep_n3(p_concat_layer1)\n\n down_feat0 = self.downsample1(pan_out1)\n p_concat_layer2 = torch.cat([down_feat0, fpn_out0], 1)\n pan_out0 = self.Rep_n4(p_concat_layer2)\n\n outputs = [pan_out2, pan_out1, pan_out0]\n\n return outputs\n" }, { "path": "asone/detectors/yolov6/yolov6/models/yolo.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom asone.detectors.yolov6.yolov6.layers.common import *\nfrom asone.detectors.yolov6.yolov6.utils.torch_utils import initialize_weights\nfrom asone.detectors.yolov6.yolov6.models.efficientrep import *\nfrom asone.detectors.yolov6.yolov6.models.reppan import *\nfrom asone.detectors.yolov6.yolov6.models.effidehead import Detect, build_effidehead_layer\n\n\nclass Model(nn.Module):\n '''YOLOv6 model with backbone, neck and head.\n The default parts are EfficientRep Backbone, Rep-PAN and\n Efficient Decoupled Head.\n '''\n def __init__(self, config, channels=3, num_classes=None, anchors=None): # model, input channels, number of classes\n super().__init__()\n # Build network\n num_layers = config.model.head.num_layers\n #self.mode = config.training_mode\n self.backbone, self.neck, self.detect = build_network(config, channels, num_classes, anchors, num_layers)\n\n # Init Detect head\n begin_indices = config.model.head.begin_indices\n out_indices_head = config.model.head.out_indices\n self.stride = self.detect.stride\n self.detect.i = begin_indices\n self.detect.f = out_indices_head\n self.detect.initialize_biases()\n\n # Init weights\n initialize_weights(self)\n\n def forward(self, x):\n export_mode = torch.onnx.is_in_onnx_export()\n x = self.backbone(x)\n x = self.neck(x)\n if export_mode == False:\n featmaps = []\n featmaps.extend(x)\n x = self.detect(x)\n return x if export_mode is True else [x, featmaps]\n\n def _apply(self, fn):\n self = super()._apply(fn)\n self.detect.stride = fn(self.detect.stride)\n self.detect.grid = list(map(fn, self.detect.grid))\n return self\n\n\ndef make_divisible(x, divisor):\n # Upward revision the value x to make it evenly divisible by the divisor.\n return math.ceil(x / divisor) * divisor\n\n\ndef build_network(config, channels, num_classes, anchors, num_layers):\n depth_mul = config.model.depth_multiple\n width_mul = config.model.width_multiple\n num_repeat_backbone = config.model.backbone.num_repeats\n channels_list_backbone = config.model.backbone.out_channels\n num_repeat_neck = config.model.neck.num_repeats\n channels_list_neck = config.model.neck.out_channels\n num_anchors = config.model.head.anchors\n use_dfl = config.model.head.use_dfl\n reg_max = config.model.head.reg_max\n num_repeat = [(max(round(i * depth_mul), 1) if i > 1 else i) for i in (num_repeat_backbone + num_repeat_neck)]\n channels_list = [make_divisible(i * width_mul, 8) for i in (channels_list_backbone + channels_list_neck)]\n\n block = get_block(config.training_mode)\n BACKBONE = eval(config.model.backbone.type)\n NECK = eval(config.model.neck.type)\n \n if 'CSP' in config.model.backbone.type:\n backbone = BACKBONE(\n in_channels=channels,\n channels_list=channels_list,\n num_repeats=num_repeat,\n block=block,\n csp_e=config.model.backbone.csp_e\n )\n\n neck = NECK(\n channels_list=channels_list,\n num_repeats=num_repeat,\n block=block,\n csp_e=config.model.neck.csp_e\n )\n else:\n backbone = BACKBONE(\n in_channels=channels,\n channels_list=channels_list,\n num_repeats=num_repeat,\n block=block\n )\n\n neck = NECK(\n channels_list=channels_list,\n num_repeats=num_repeat,\n block=block\n )\n\n head_layers = build_effidehead_layer(channels_list, num_anchors, num_classes, reg_max)\n\n head = Detect(num_classes, anchors, num_layers, head_layers=head_layers, use_dfl=use_dfl)\n\n return backbone, neck, head\n\n\ndef build_model(cfg, num_classes, device):\n model = Model(cfg, channels=3, num_classes=num_classes, anchors=cfg.model.head.anchors).to(device)\n return model\n" }, { "path": "asone/detectors/yolov6/yolov6/utils/__init__.py", "content": "" }, { "path": "asone/detectors/yolov6/yolov6/utils/checkpoint.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\nimport os\nimport shutil\nimport torch\nimport os.path as osp\nfrom asone.detectors.yolov6.yolov6.utils.events import LOGGER\nfrom asone.detectors.yolov6.yolov6.utils.torch_utils import fuse_model\n\n\ndef load_state_dict(weights, model, map_location=None):\n \"\"\"Load weights from checkpoint file, only assign weights those layers' name and shape are match.\"\"\"\n ckpt = torch.load(weights, map_location=map_location)\n state_dict = ckpt['model'].float().state_dict()\n model_state_dict = model.state_dict()\n state_dict = {k: v for k, v in state_dict.items() if k in model_state_dict and v.shape == model_state_dict[k].shape}\n model.load_state_dict(state_dict, strict=False)\n del ckpt, state_dict, model_state_dict\n return model\n\n\ndef load_checkpoint(weights, map_location=None, inplace=True, fuse=True):\n \"\"\"Load model from checkpoint file.\"\"\"\n LOGGER.info(\"Loading checkpoint from {}\".format(weights))\n ckpt = torch.load(weights, map_location=map_location) # load\n model = ckpt['ema' if ckpt.get('ema') else 'model'].float()\n if fuse:\n LOGGER.info(\"\\nFusing model...\")\n model = fuse_model(model).eval()\n else:\n model = model.eval()\n return model\n\n\ndef save_checkpoint(ckpt, is_best, save_dir, model_name=\"\"):\n \"\"\" Save checkpoint to the disk.\"\"\"\n if not osp.exists(save_dir):\n os.makedirs(save_dir)\n filename = osp.join(save_dir, model_name + '.pt')\n torch.save(ckpt, filename)\n if is_best:\n best_filename = osp.join(save_dir, 'best_ckpt.pt')\n shutil.copyfile(filename, best_filename)\n\n\ndef strip_optimizer(ckpt_dir, epoch):\n \"\"\"Delete optimizer from saved checkpoint file\"\"\"\n for s in ['best', 'last']:\n ckpt_path = osp.join(ckpt_dir, '{}_ckpt.pt'.format(s))\n if not osp.exists(ckpt_path):\n continue\n ckpt = torch.load(ckpt_path, map_location=torch.device('cpu'))\n if ckpt.get('ema'):\n ckpt['model'] = ckpt['ema'] # replace model with ema\n for k in ['optimizer', 'ema', 'updates']: # keys\n ckpt[k] = None\n ckpt['epoch'] = epoch\n ckpt['model'].half() # to FP16\n for p in ckpt['model'].parameters():\n p.requires_grad = False\n torch.save(ckpt, ckpt_path)" }, { "path": "asone/detectors/yolov6/yolov6/utils/events.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\nimport os\nimport yaml\nimport logging\nimport shutil\n\n\ndef set_logging(name=None):\n rank = int(os.getenv('RANK', -1))\n logging.basicConfig(format=\"%(message)s\", level=logging.INFO if (rank in (-1, 0)) else logging.WARNING)\n return logging.getLogger(name)\n\n\nLOGGER = set_logging(__name__)\nNCOLS = shutil.get_terminal_size().columns\n\n\ndef load_yaml(file_path):\n \"\"\"Load data from yaml file.\"\"\"\n if isinstance(file_path, str):\n with open(file_path, errors='ignore') as f:\n data_dict = yaml.safe_load(f)\n return data_dict\n\n\ndef save_yaml(data_dict, save_path):\n \"\"\"Save data to yaml file\"\"\"\n with open(save_path, 'w') as f:\n yaml.safe_dump(data_dict, f, sort_keys=False)\n\n\ndef write_tblog(tblogger, epoch, results, losses):\n \"\"\"Display mAP and loss information to log.\"\"\"\n tblogger.add_scalar(\"val/mAP@0.5\", results[0], epoch + 1)\n tblogger.add_scalar(\"val/mAP@0.50:0.95\", results[1], epoch + 1)\n\n tblogger.add_scalar(\"train/iou_loss\", losses[0], epoch + 1)\n tblogger.add_scalar(\"train/l1_loss\", losses[1], epoch + 1)\n tblogger.add_scalar(\"train/obj_loss\", losses[2], epoch + 1)\n tblogger.add_scalar(\"train/cls_loss\", losses[3], epoch + 1)\n\n tblogger.add_scalar(\"x/lr0\", results[2], epoch + 1)\n tblogger.add_scalar(\"x/lr1\", results[3], epoch + 1)\n tblogger.add_scalar(\"x/lr2\", results[4], epoch + 1)\n\ndef write_tbimg(tblogger, imgs, step, type='train'):\n \"\"\"Display train_batch and validation predictions to tensorboard.\"\"\"\n if type == 'train':\n tblogger.add_image(f'train_batch', imgs, step + 1, dataformats='HWC')\n elif type == 'val':\n for idx, img in enumerate(imgs):\n tblogger.add_image(f'val_img_{idx + 1}', img, step + 1, dataformats='HWC')\n else:\n LOGGER.warning('WARNING: Unknown image type to visualize.\\n')\n" }, { "path": "asone/detectors/yolov6/yolov6/utils/figure_iou.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\nimport math\nimport torch\n\n\nclass IOUloss:\n \"\"\" Calculate IoU loss.\n \"\"\"\n def __init__(self, box_format='xywh', iou_type='ciou', reduction='none', eps=1e-7):\n \"\"\" Setting of the class.\n Args:\n box_format: (string), must be one of 'xywh' or 'xyxy'.\n iou_type: (string), can be one of 'ciou', 'diou', 'giou' or 'siou'\n reduction: (string), specifies the reduction to apply to the output, must be one of 'none', 'mean','sum'.\n eps: (float), a value to avoid divide by zero error.\n \"\"\"\n self.box_format = box_format\n self.iou_type = iou_type.lower()\n self.reduction = reduction\n self.eps = eps\n\n def __call__(self, box1, box2):\n \"\"\" calculate iou. box1 and box2 are torch tensor with shape [M, 4] and [Nm 4].\n \"\"\"\n if box1.shape[0] != box2.shape[0]:\n box2 = box2.T\n if self.box_format == 'xyxy':\n b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]\n b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]\n elif self.box_format == 'xywh':\n b1_x1, b1_x2 = box1[0] - box1[2] / 2, box1[0] + box1[2] / 2\n b1_y1, b1_y2 = box1[1] - box1[3] / 2, box1[1] + box1[3] / 2\n b2_x1, b2_x2 = box2[0] - box2[2] / 2, box2[0] + box2[2] / 2\n b2_y1, b2_y2 = box2[1] - box2[3] / 2, box2[1] + box2[3] / 2\n else:\n if self.box_format == 'xyxy':\n b1_x1, b1_y1, b1_x2, b1_y2 = torch.split(box1, 1, dim=-1)\n b2_x1, b2_y1, b2_x2, b2_y2 = torch.split(box2, 1, dim=-1)\n\n elif self.box_format == 'xywh':\n b1_x1, b1_y1, b1_w, b1_h = torch.split(box1, 1, dim=-1)\n b2_x1, b2_y1, b2_w, b2_h = torch.split(box2, 1, dim=-1)\n b1_x1, b1_x2 = b1_x1 - b1_w / 2, b1_x1 + b1_w / 2\n b1_y1, b1_y2 = b1_y1 - b1_h / 2, b1_y1 + b1_h / 2\n b2_x1, b2_x2 = b2_x1 - b2_w / 2, b2_x1 + b2_w / 2\n b2_y1, b2_y2 = b2_y1 - b2_h / 2, b2_y1 + b2_h / 2\n\n # Intersection area\n inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \\\n (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0)\n\n # Union Area\n w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + self.eps\n w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + self.eps\n union = w1 * h1 + w2 * h2 - inter + self.eps\n iou = inter / union\n\n cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1) # convex width\n ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height\n if self.iou_type == 'giou':\n c_area = cw * ch + self.eps # convex area\n iou = iou - (c_area - union) / c_area\n elif self.iou_type in ['diou', 'ciou']:\n c2 = cw ** 2 + ch ** 2 + self.eps # convex diagonal squared\n rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 +\n (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center distance squared\n if self.iou_type == 'diou':\n iou = iou - rho2 / c2\n elif self.iou_type == 'ciou':\n v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2)\n with torch.no_grad():\n alpha = v / (v - iou + (1 + self.eps))\n iou = iou - (rho2 / c2 + v * alpha)\n elif self.iou_type == 'siou':\n # SIoU Loss https://arxiv.org/pdf/2205.12740.pdf\n s_cw = (b2_x1 + b2_x2 - b1_x1 - b1_x2) * 0.5 + self.eps\n s_ch = (b2_y1 + b2_y2 - b1_y1 - b1_y2) * 0.5 + self.eps\n sigma = torch.pow(s_cw ** 2 + s_ch ** 2, 0.5)\n sin_alpha_1 = torch.abs(s_cw) / sigma\n sin_alpha_2 = torch.abs(s_ch) / sigma\n threshold = pow(2, 0.5) / 2\n sin_alpha = torch.where(sin_alpha_1 > threshold, sin_alpha_2, sin_alpha_1)\n angle_cost = torch.cos(torch.arcsin(sin_alpha) * 2 - math.pi / 2)\n rho_x = (s_cw / cw) ** 2\n rho_y = (s_ch / ch) ** 2\n gamma = angle_cost - 2\n distance_cost = 2 - torch.exp(gamma * rho_x) - torch.exp(gamma * rho_y)\n omiga_w = torch.abs(w1 - w2) / torch.max(w1, w2)\n omiga_h = torch.abs(h1 - h2) / torch.max(h1, h2)\n shape_cost = torch.pow(1 - torch.exp(-1 * omiga_w), 4) + torch.pow(1 - torch.exp(-1 * omiga_h), 4)\n iou = iou - 0.5 * (distance_cost + shape_cost)\n loss = 1.0 - iou\n\n if self.reduction == 'sum':\n loss = loss.sum()\n elif self.reduction == 'mean':\n loss = loss.mean()\n\n return loss\n\n\ndef pairwise_bbox_iou(box1, box2, box_format='xywh'):\n \"\"\"Calculate iou.\n This code is based on https://github.com/Megvii-BaseDetection/YOLOX/blob/main/yolox/utils/boxes.py\n \"\"\"\n if box_format == 'xyxy':\n lt = torch.max(box1[:, None, :2], box2[:, :2])\n rb = torch.min(box1[:, None, 2:], box2[:, 2:])\n area_1 = torch.prod(box1[:, 2:] - box1[:, :2], 1)\n area_2 = torch.prod(box2[:, 2:] - box2[:, :2], 1)\n\n elif box_format == 'xywh':\n lt = torch.max(\n (box1[:, None, :2] - box1[:, None, 2:] / 2),\n (box2[:, :2] - box2[:, 2:] / 2),\n )\n rb = torch.min(\n (box1[:, None, :2] + box1[:, None, 2:] / 2),\n (box2[:, :2] + box2[:, 2:] / 2),\n )\n\n area_1 = torch.prod(box1[:, 2:], 1)\n area_2 = torch.prod(box2[:, 2:], 1)\n valid = (lt < rb).type(lt.type()).prod(dim=2)\n inter = torch.prod(rb - lt, 2) * valid\n return inter / (area_1[:, None] + area_2 - inter)" }, { "path": "asone/detectors/yolov6/yolov6/utils/general.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\nimport os\nimport glob\nimport torch\nfrom pathlib import Path\n\n\ndef increment_name(path):\n '''increase save directory's id'''\n path = Path(path)\n sep = ''\n if path.exists():\n path, suffix = (path.with_suffix(''), path.suffix) if path.is_file() else (path, '')\n for n in range(1, 9999):\n p = f'{path}{sep}{n}{suffix}'\n if not os.path.exists(p):\n break\n path = Path(p)\n return path\n\n\ndef find_latest_checkpoint(search_dir='.'):\n '''Find the most recent saved checkpoint in search_dir.'''\n checkpoint_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True)\n return max(checkpoint_list, key=os.path.getctime) if checkpoint_list else ''\n\n\ndef dist2bbox(distance, anchor_points, box_format='xyxy'):\n '''Transform distance(ltrb) to box(xywh or xyxy).'''\n lt, rb = torch.split(distance, 2, -1)\n x1y1 = anchor_points - lt\n x2y2 = anchor_points + rb\n if box_format == 'xyxy':\n bbox = torch.cat([x1y1, x2y2], -1)\n elif box_format == 'xywh':\n c_xy = (x1y1 + x2y2) / 2\n wh = x2y2 - x1y1\n bbox = torch.cat([c_xy, wh], -1)\n return bbox\n\n\ndef bbox2dist(anchor_points, bbox, reg_max):\n '''Transform bbox(xyxy) to dist(ltrb).'''\n x1y1, x2y2 = torch.split(bbox, 2, -1)\n lt = anchor_points - x1y1\n rb = x2y2 - anchor_points\n dist = torch.cat([lt, rb], -1).clip(0, reg_max - 0.01)\n return dist\n\n\ndef xywh2xyxy(bboxes):\n '''Transform bbox(xywh) to box(xyxy).'''\n bboxes[..., 0] = bboxes[..., 0] - bboxes[..., 2] * 0.5 \n bboxes[..., 1] = bboxes[..., 1] - bboxes[..., 3] * 0.5\n bboxes[..., 2] = bboxes[..., 0] + bboxes[..., 2]\n bboxes[..., 3] = bboxes[..., 1] + bboxes[..., 3]\n return bboxes\n\ndef box_iou(box1, box2):\n # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py\n \"\"\"\n Return intersection-over-union (Jaccard index) of boxes.\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\n Arguments:\n box1 (Tensor[N, 4])\n box2 (Tensor[M, 4])\n Returns:\n iou (Tensor[N, M]): the NxM matrix containing the pairwise\n IoU values for every element in boxes1 and boxes2\n \"\"\"\n\n def box_area(box):\n # box = 4xn\n return (box[2] - box[0]) * (box[3] - box[1])\n\n area1 = box_area(box1.T)\n area2 = box_area(box2.T)\n\n # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)\n inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2)\n return inter / (area1[:, None] + area2 - inter) # iou = inter / (area1 + area2 - inter)" }, { "path": "asone/detectors/yolov6/yolov6/utils/torch_utils.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n\nimport time\nfrom contextlib import contextmanager\nfrom copy import deepcopy\nimport torch\nimport torch.distributed as dist\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom asone.detectors.yolov6.yolov6.utils.events import LOGGER\n\ntry:\n import thop # for FLOPs computation\nexcept ImportError:\n thop = None\n\n\n@contextmanager\ndef torch_distributed_zero_first(local_rank: int):\n \"\"\"\n Decorator to make all processes in distributed training wait for each local_master to do something.\n \"\"\"\n if local_rank not in [-1, 0]:\n dist.barrier(device_ids=[local_rank])\n yield\n if local_rank == 0:\n dist.barrier(device_ids=[0])\n\n\ndef time_sync():\n # Waits for all kernels in all streams on a CUDA device to complete if cuda is available.\n if torch.cuda.is_available():\n torch.cuda.synchronize()\n return time.time()\n\n\ndef initialize_weights(model):\n for m in model.modules():\n t = type(m)\n if t is nn.Conv2d:\n pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n elif t is nn.BatchNorm2d:\n m.eps = 1e-3\n m.momentum = 0.03\n elif t in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]:\n m.inplace = True\n\n\ndef fuse_conv_and_bn(conv, bn):\n # Fuse convolution and batchnorm layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/\n fusedconv = (\n nn.Conv2d(\n conv.in_channels,\n conv.out_channels,\n kernel_size=conv.kernel_size,\n stride=conv.stride,\n padding=conv.padding,\n groups=conv.groups,\n bias=True,\n )\n .requires_grad_(False)\n .to(conv.weight.device)\n )\n\n # prepare filters\n w_conv = conv.weight.clone().view(conv.out_channels, -1)\n w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))\n fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape))\n\n # prepare spatial bias\n b_conv = (\n torch.zeros(conv.weight.size(0), device=conv.weight.device)\n if conv.bias is None\n else conv.bias\n )\n b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(\n torch.sqrt(bn.running_var + bn.eps)\n )\n fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)\n\n return fusedconv\n\n\ndef fuse_model(model):\n from asone.detectors.yolov6.yolov6.layers.common import Conv\n\n for m in model.modules():\n if type(m) is Conv and hasattr(m, \"bn\"):\n m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv\n delattr(m, \"bn\") # remove batchnorm\n m.forward = m.forward_fuse # update forward\n return model\n\n\ndef get_model_info(model, img_size=640):\n \"\"\"Get model Params and GFlops.\n Code base on https://github.com/Megvii-BaseDetection/YOLOX/blob/main/yolox/utils/model_utils.py\n \"\"\"\n from thop import profile\n stride = 32\n img = torch.zeros((1, 3, stride, stride), device=next(model.parameters()).device)\n\n flops, params = profile(deepcopy(model), inputs=(img,), verbose=False)\n params /= 1e6\n flops /= 1e9\n img_size = img_size if isinstance(img_size, list) else [img_size, img_size]\n flops *= img_size[0] * img_size[1] / stride / stride * 2 # Gflops\n info = \"Params: {:.2f}M, Gflops: {:.2f}\".format(params, flops)\n return info\n" }, { "path": "asone/detectors/yolov6/yolov6/utils/yolov6_utils.py", "content": "import time\nimport sys\nimport os\nimport numpy as np\nimport cv2\nimport torch.nn as nn\nimport torch\nimport torchvision\n\nfrom asone.detectors.yolov6.yolov6.layers.common import Conv\n\ndef nms(boxes, scores, iou_threshold):\n # Sort by score\n sorted_indices = np.argsort(scores)[::-1]\n\n keep_boxes = []\n while sorted_indices.size > 0:\n # Pick the last box\n box_id = sorted_indices[0]\n keep_boxes.append(box_id)\n\n # Compute IoU of the picked box with the rest\n ious = compute_iou(boxes[box_id, :], boxes[sorted_indices[1:], :])\n\n # Remove boxes with IoU over the threshold\n keep_indices = np.where(ious < iou_threshold)[0]\n\n # print(keep_indices.shape, sorted_indices.shape)\n sorted_indices = sorted_indices[keep_indices + 1]\n\n return keep_boxes\n\n\ndef process_output(output, img_height, img_width, \n input_width, input_height,\n conf_thres, iou_thres):\n predictions = np.squeeze(output)\n\n # Filter out object confidence scores below threshold\n obj_conf = predictions[:, 4]\n predictions = predictions[obj_conf > conf_thres]\n obj_conf = obj_conf[obj_conf > conf_thres]\n\n # Multiply class confidence with bounding box confidence\n predictions[:, 5:] *= obj_conf[:, np.newaxis]\n\n # Get the scores\n scores = np.max(predictions[:, 5:], axis=1)\n\n # Filter out the objects with a low score\n predictions = predictions[obj_conf > conf_thres]\n scores = scores[scores > conf_thres]\n\n # Get the class with the highest confidence\n class_ids = np.argmax(predictions[:, 5:], axis=1)\n\n # Get bounding boxes for each object\n boxes = process_and_scale_boxes(predictions, img_height, img_width, \n input_width, input_height)\n\n # Apply non-maxima suppression to suppress weak, overlapping bounding boxes\n indices = nms(boxes, scores, iou_thres)\n\n return boxes[indices], scores[indices], class_ids[indices]\n\ndef compute_iou(box, boxes):\n # Compute xmin, ymin, xmax, ymax for both boxes\n xmin = np.maximum(box[0], boxes[:, 0])\n ymin = np.maximum(box[1], boxes[:, 1])\n xmax = np.minimum(box[2], boxes[:, 2])\n ymax = np.minimum(box[3], boxes[:, 3])\n\n # Compute intersection area\n intersection_area = np.maximum(0, xmax - xmin) * np.maximum(0, ymax - ymin)\n\n # Compute union area\n box_area = (box[2] - box[0]) * (box[3] - box[1])\n boxes_area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])\n union_area = box_area + boxes_area - intersection_area\n\n # Compute IoU\n iou = intersection_area / union_area\n\n return iou\n\n\ndef xywh2xyxy(x):\n # Convert bounding box (x, y, w, h) to bounding box (x1, y1, x2, y2)\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = x[..., 0] - x[..., 2] / 2\n y[..., 1] = x[..., 1] - x[..., 3] / 2\n y[..., 2] = x[..., 0] + x[..., 2] / 2\n y[..., 3] = x[..., 1] + x[..., 3] / 2\n return y\n\ndef prepare_input(image, input_width, input_height):\n \n input_img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n # Resize input image\n input_img = cv2.resize(input_img, (input_width, input_height))\n\n # Scale input pixel values to 0 to 1\n input_img = input_img / 255.0\n input_img = input_img.transpose(2, 0, 1)\n input_tensor = input_img[np.newaxis, :, :, :].astype(np.float32)\n\n return input_tensor\n\ndef process_and_scale_boxes(predictions, img_height, img_width, \n input_width, input_height):\n \n predictions = np.delete(predictions, 0, axis=1)\n # Extract boxes from predictions\n boxes = predictions[:, :4]\n # Scale boxes to original image dimensions\n boxes /= np.array([input_width, input_height, input_width, input_height])\n boxes *= np.array([img_width, img_height, img_width, img_height])\n # Convert boxes to xyxy format\n # boxes = xywh2xyxy(boxes)\n \n boxes = boxes[:,:4]\n class_ids = predictions[:,4:5]\n scores = predictions[:,5:]\n return boxes, scores, class_ids\n\ndef load_pytorch(weights, map_location=None, inplace=True, fuse=False):\n \"\"\"Load model from checkpoint file.\"\"\"\n ckpt = torch.load(weights, map_location=map_location) # load\n model = ckpt['ema' if ckpt.get('ema') else 'model'].float()\n if fuse:\n model = fuse_model(model).eval()\n else:\n model = model.eval()\n return model\n\ndef fuse_model(model):\n for m in model.modules():\n if type(m) is Conv and hasattr(m, \"bn\"):\n m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv\n delattr(m, \"bn\") # remove batchnorm\n m.forward = m.forward_fuse # update forward\n return model\n\ndef fuse_conv_and_bn(conv, bn):\n # Fuse convolution and batchnorm layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/\n fusedconv = (\n nn.Conv2d(\n conv.in_channels,\n conv.out_channels,\n kernel_size=conv.kernel_size,\n stride=conv.stride,\n padding=conv.padding,\n groups=conv.groups,\n bias=True,\n )\n .requires_grad_(False)\n .to(conv.weight.device)\n )\n\n # prepare filters\n w_conv = conv.weight.clone().view(conv.out_channels, -1)\n w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))\n fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape))\n\n # prepare spatial bias\n b_conv = (\n torch.zeros(conv.weight.size(0), device=conv.weight.device)\n if conv.bias is None\n else conv.bias\n )\n b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(\n torch.sqrt(bn.running_var + bn.eps)\n )\n fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)\n\n return fusedconv\n\n\ndef non_max_suppression(prediction, conf_thres=0.25, iou_thres=0.45, classes=None, agnostic=False, multi_label=False, max_det=300):\n \"\"\"Runs Non-Maximum Suppression (NMS) on inference results.\n This code is borrowed from: https://github.com/ultralytics/yolov5/blob/47233e1698b89fc437a4fb9463c815e9171be955/utils/general.py#L775\n Args:\n prediction: (tensor), with shape [N, 5 + num_classes], N is the number of bboxes.\n conf_thres: (float) confidence threshold.\n iou_thres: (float) iou threshold.\n classes: (None or list[int]), if a list is provided, nms only keep the classes you provide.\n agnostic: (bool), when it is set to True, we do class-independent nms, otherwise, different class would do nms respectively.\n multi_label: (bool), when it is set to True, one box can have multi labels, otherwise, one box only huave one label.\n max_det:(int), max number of output bboxes.\n\n Returns:\n list of detections, echo item is one tensor with shape (num_boxes, 6), 6 is for [xyxy, conf, cls].\n \"\"\"\n\n num_classes = prediction.shape[2] - 5 # number of classes\n pred_candidates = prediction[..., 4] > conf_thres # candidates\n\n # Check the parameters.\n assert 0 <= conf_thres <= 1, f'conf_thresh must be in 0.0 to 1.0, however {conf_thres} is provided.'\n assert 0 <= iou_thres <= 1, f'iou_thres must be in 0.0 to 1.0, however {iou_thres} is provided.'\n\n # Function settings.\n max_wh = 4096 # maximum box width and height\n max_nms = 30000 # maximum number of boxes put into torchvision.ops.nms()\n time_limit = 10.0 # quit the function when nms cost time exceed the limit time.\n multi_label &= num_classes > 1 # multiple labels per box\n\n tik = time.time()\n output = [torch.zeros((0, 6), device=prediction.device)] * prediction.shape[0]\n for img_idx, x in enumerate(prediction): # image index, image inference\n x = x[pred_candidates[img_idx]] # confidence\n\n # If no box remains, skip the next process.\n if not x.shape[0]:\n continue\n\n # confidence multiply the objectness\n x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf\n\n # (center x, center y, width, height) to (x1, y1, x2, y2)\n box = xywh2xyxy(x[:, :4])\n\n # Detections matrix's shape is (n,6), each row represents (xyxy, conf, cls)\n if multi_label:\n box_idx, class_idx = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T\n x = torch.cat((box[box_idx], x[box_idx, class_idx + 5, None], class_idx[:, None].float()), 1)\n else: # Only keep the class with highest scores.\n conf, class_idx = x[:, 5:].max(1, keepdim=True)\n x = torch.cat((box, conf, class_idx.float()), 1)[conf.view(-1) > conf_thres]\n\n # Filter by class, only keep boxes whose category is in classes.\n if classes is not None:\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\n\n # Check shape\n num_box = x.shape[0] # number of boxes\n if not num_box: # no boxes kept.\n continue\n elif num_box > max_nms: # excess max boxes' number.\n x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence\n\n # Batched NMS\n class_offset = x[:, 5:6] * (0 if agnostic else max_wh) # classes\n boxes, scores = x[:, :4] + class_offset, x[:, 4] # boxes (offset by class), scores\n keep_box_idx = torchvision.ops.nms(boxes, scores, iou_thres) # NMS\n if keep_box_idx.shape[0] > max_det: # limit detections\n keep_box_idx = keep_box_idx[:max_det]\n\n output[img_idx] = x[keep_box_idx]\n if (time.time() - tik) > time_limit:\n print(f'WARNING: NMS cost time exceed the limited {time_limit}s.')\n break # time limit exceeded\n\n return output\n\n\n\n\n" }, { "path": "asone/detectors/yolov6/yolov6_detector.py", "content": "import os\nimport sys\nfrom asone.utils import get_names\nimport numpy as np\nimport warnings\nimport torch\nimport onnxruntime\n\nfrom asone import utils\nfrom asone.detectors.yolov6.yolov6.utils.yolov6_utils import (prepare_input, load_pytorch,\n non_max_suppression, process_and_scale_boxes) \n# sys.path.append(os.path.dirname(__file__))\nfrom asone.utils.utils import PathResolver\n \n\nclass YOLOv6Detector:\n def __init__(self,\n weights=None,\n use_onnx=False,\n use_cuda=True):\n\n self.use_onnx = use_onnx\n self.device = 'cuda' if use_cuda else 'cpu'\n\n if not os.path.exists(weights):\n utils.download_weights(weights)\n #If incase weighst is a list of paths then select path at first index\n weights = str(weights[0] if isinstance(weights, list) else weights)\n \n with PathResolver():\n # Load Model\n self.model = self.load_model(use_cuda, weights)\n \n if use_onnx:\n # Get Some ONNX model details \n self.input_shape, self.input_height, self.input_width = self.ONNXModel_detail(self.model)\n self.input_names, self.output_names = self.ONNXModel_names(self.model)\n\n\n def load_model(self, use_cuda, weights, fp16=False):\n # Device: CUDA and if fp16=True only then half precision floating point works \n self.fp16 = fp16 & ((not self.use_onnx or self.use_onnx) and self.device != 'cpu')\n # Load onnx \n if self.use_onnx:\n if use_cuda:\n providers = ['CUDAExecutionProvider','CPUExecutionProvider']\n else:\n providers = ['CPUExecutionProvider']\n model = onnxruntime.InferenceSession(weights, providers=providers)\n #Load Pytorch\n else:\n model = load_pytorch(weights, map_location=self.device)\n model.half() if self.fp16 else model.float()\n return model\n\n def ONNXModel_detail(self, model):\n # Get Model Input\n model_inputs = model.get_inputs()\n # Input shape\n input_shape = model_inputs[0].shape\n input_height = input_shape[2]\n input_width = input_shape[3]\n \n return input_shape, input_height, input_width\n\n def ONNXModel_names(self, model):\n # Get Model Input\n model_inputs = model.get_inputs()\n input_names = [model_inputs[i].name for i in range(len(model_inputs))]\n # Get Model Output\n model_outputs = model.get_outputs()\n output_names = [model_outputs[i].name for i in range(len(model_outputs))]\n \n return input_names, output_names \n \n def detect(self, image: list,\n input_shape: tuple = (640, 640),\n conf_thres: float = 0.25,\n iou_thres: float = 0.45,\n max_det: int = 1000,\n filter_classes: bool = None,\n agnostic_nms: bool = True,\n with_p6: bool = False,\n return_image=False) -> list:\n \n # Prepare Input\n img_height, img_width = image.shape[:2]\n processed_image = prepare_input(image, input_shape[0], input_shape[1])\n \n # Perform Inference on the Image\n if self.use_onnx:\n # Run ONNX model \n prediction = self.model.run(self.output_names,\n {self.input_names[0]: processed_image})[0] \n # Run Pytorch model\n else:\n processed_image = torch.from_numpy(processed_image).to(self.device)\n # Change image floating point precision if fp16 set to true\n processed_image = processed_image.half() if self.fp16 else processed_image.float() \n prediction = self.model(processed_image)[0]\n\n # Post Procesing, non-max-suppression and rescaling\n if self.use_onnx:\n # Process ONNX Output\n \n boxes, scores, class_ids = process_and_scale_boxes(prediction, img_height, img_width,\n input_shape[1], input_shape[0])\n detection = []\n for box in range(len(boxes)):\n pred = np.append(boxes[box], scores[box])\n pred = np.append(pred, class_ids[box])\n detection.append(pred)\n detection = np.array(detection)\n else:\n detection = non_max_suppression(prediction,\n conf_thres,\n iou_thres,\n agnostic=agnostic_nms, \n max_det=max_det)[0]\n \n detection = detection.detach().cpu().numpy()\n detection[:, :4] /= np.array([input_shape[1], input_shape[0], input_shape[1], input_shape[0]])\n detection[:, :4] *= np.array([img_width, img_height, img_width, img_height])\n \n if filter_classes:\n class_names = get_names()\n\n filter_class_idx = []\n if filter_classes:\n for _class in filter_classes:\n if _class.lower() in class_names:\n filter_class_idx.append(class_names.index(_class.lower()))\n else:\n warnings.warn(f\"class {_class} not found in model classes list.\")\n\n detection = detection[np.in1d(detection[:,5].astype(int), filter_class_idx)]\n \n image_info = {\n 'width': image.shape[1],\n 'height': image.shape[0],\n }\n\n if return_image:\n return detection, image\n else: \n return detection, image_info" }, { "path": "asone/detectors/yolov7/__init__.py", "content": "from .yolov7_detector import YOLOv7Detector\n__all__ = ['YOLOv7Detector']" }, { "path": "asone/detectors/yolov7/yolov7/__init__.py", "content": "" }, { "path": "asone/detectors/yolov7/yolov7/models/__init__.py", "content": "" }, { "path": "asone/detectors/yolov7/yolov7/models/common.py", "content": "import math\nfrom copy import copy\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport requests\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom PIL import Image\nfrom torch.cuda import amp\n\nfrom asone.detectors.yolov7.yolov7.utils.torch_utils import time_synchronized\n\n\n##### basic ####\n\ndef autopad(k, p=None): # kernel, padding\n # Pad to 'same'\n if p is None:\n p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad\n return p\n\n\nclass MP(nn.Module):\n def __init__(self, k=2):\n super(MP, self).__init__()\n self.m = nn.MaxPool2d(kernel_size=k, stride=k)\n\n def forward(self, x):\n return self.m(x)\n\n\nclass SP(nn.Module):\n def __init__(self, k=3, s=1):\n super(SP, self).__init__()\n self.m = nn.MaxPool2d(kernel_size=k, stride=s, padding=k // 2)\n\n def forward(self, x):\n return self.m(x)\n \n \nclass ReOrg(nn.Module):\n def __init__(self):\n super(ReOrg, self).__init__()\n\n def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2)\n return torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1)\n\n\nclass Concat(nn.Module):\n def __init__(self, dimension=1):\n super(Concat, self).__init__()\n self.d = dimension\n\n def forward(self, x):\n return torch.cat(x, self.d)\n\n\nclass Chuncat(nn.Module):\n def __init__(self, dimension=1):\n super(Chuncat, self).__init__()\n self.d = dimension\n\n def forward(self, x):\n x1 = []\n x2 = []\n for xi in x:\n xi1, xi2 = xi.chunk(2, self.d)\n x1.append(xi1)\n x2.append(xi2)\n return torch.cat(x1+x2, self.d)\n\n\nclass Shortcut(nn.Module):\n def __init__(self, dimension=0):\n super(Shortcut, self).__init__()\n self.d = dimension\n\n def forward(self, x):\n return x[0]+x[1]\n\n\nclass Foldcut(nn.Module):\n def __init__(self, dimension=0):\n super(Foldcut, self).__init__()\n self.d = dimension\n\n def forward(self, x):\n x1, x2 = x.chunk(2, self.d)\n return x1+x2\n\n\nclass Conv(nn.Module):\n # Standard convolution\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super(Conv, self).__init__()\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False)\n self.bn = nn.BatchNorm2d(c2)\n self.act = nn.SiLU() if act is True else (act if isinstance(act, nn.Module) else nn.Identity())\n\n def forward(self, x):\n return self.act(self.bn(self.conv(x)))\n\n def fuseforward(self, x):\n return self.act(self.conv(x))\n \n\nclass RobustConv(nn.Module):\n # Robust convolution (use high kernel size 7-11 for: downsampling and other layers). Train for 300 - 450 epochs.\n def __init__(self, c1, c2, k=7, s=1, p=None, g=1, act=True, layer_scale_init_value=1e-6): # ch_in, ch_out, kernel, stride, padding, groups\n super(RobustConv, self).__init__()\n self.conv_dw = Conv(c1, c1, k=k, s=s, p=p, g=c1, act=act)\n self.conv1x1 = nn.Conv2d(c1, c2, 1, 1, 0, groups=1, bias=True)\n self.gamma = nn.Parameter(layer_scale_init_value * torch.ones(c2)) if layer_scale_init_value > 0 else None\n\n def forward(self, x):\n x = x.to(memory_format=torch.channels_last)\n x = self.conv1x1(self.conv_dw(x))\n if self.gamma is not None:\n x = x.mul(self.gamma.reshape(1, -1, 1, 1)) \n return x\n\n\nclass RobustConv2(nn.Module):\n # Robust convolution 2 (use [32, 5, 2] or [32, 7, 4] or [32, 11, 8] for one of the paths in CSP).\n def __init__(self, c1, c2, k=7, s=4, p=None, g=1, act=True, layer_scale_init_value=1e-6): # ch_in, ch_out, kernel, stride, padding, groups\n super(RobustConv2, self).__init__()\n self.conv_strided = Conv(c1, c1, k=k, s=s, p=p, g=c1, act=act)\n self.conv_deconv = nn.ConvTranspose2d(in_channels=c1, out_channels=c2, kernel_size=s, stride=s, \n padding=0, bias=True, dilation=1, groups=1\n )\n self.gamma = nn.Parameter(layer_scale_init_value * torch.ones(c2)) if layer_scale_init_value > 0 else None\n\n def forward(self, x):\n x = self.conv_deconv(self.conv_strided(x))\n if self.gamma is not None:\n x = x.mul(self.gamma.reshape(1, -1, 1, 1)) \n return x\n \n\ndef DWConv(c1, c2, k=1, s=1, act=True):\n # Depthwise convolution\n return Conv(c1, c2, k, s, g=math.gcd(c1, c2), act=act)\n\n\nclass GhostConv(nn.Module):\n # Ghost Convolution https://github.com/huawei-noah/ghostnet\n def __init__(self, c1, c2, k=1, s=1, g=1, act=True): # ch_in, ch_out, kernel, stride, groups\n super(GhostConv, self).__init__()\n c_ = c2 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, k, s, None, g, act)\n self.cv2 = Conv(c_, c_, 5, 1, None, c_, act)\n\n def forward(self, x):\n y = self.cv1(x)\n return torch.cat([y, self.cv2(y)], 1)\n\n\nclass Stem(nn.Module):\n # Stem\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super(Stem, self).__init__()\n c_ = int(c2/2) # hidden channels\n self.cv1 = Conv(c1, c_, 3, 2)\n self.cv2 = Conv(c_, c_, 1, 1)\n self.cv3 = Conv(c_, c_, 3, 2)\n self.pool = torch.nn.MaxPool2d(2, stride=2)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n\n def forward(self, x):\n x = self.cv1(x)\n return self.cv4(torch.cat((self.cv3(self.cv2(x)), self.pool(x)), dim=1))\n\n\nclass DownC(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, n=1, k=2):\n super(DownC, self).__init__()\n c_ = int(c1) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c2//2, 3, k)\n self.cv3 = Conv(c1, c2//2, 1, 1)\n self.mp = nn.MaxPool2d(kernel_size=k, stride=k)\n\n def forward(self, x):\n return torch.cat((self.cv2(self.cv1(x)), self.cv3(self.mp(x))), dim=1)\n\n\nclass SPP(nn.Module):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, k=(5, 9, 13)):\n super(SPP, self).__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1)\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\n\n def forward(self, x):\n x = self.cv1(x)\n return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1))\n \n\nclass Bottleneck(nn.Module):\n # Darknet bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super(Bottleneck, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c2, 3, 1, g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass Res(nn.Module):\n # ResNet bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super(Res, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c_, 3, 1, g=g)\n self.cv3 = Conv(c_, c2, 1, 1)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv3(self.cv2(self.cv1(x))) if self.add else self.cv3(self.cv2(self.cv1(x)))\n\n\nclass ResX(Res):\n # ResNet bottleneck\n def __init__(self, c1, c2, shortcut=True, g=32, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super().__init__(c1, c2, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n\n\nclass Ghost(nn.Module):\n # Ghost Bottleneck https://github.com/huawei-noah/ghostnet\n def __init__(self, c1, c2, k=3, s=1): # ch_in, ch_out, kernel, stride\n super(Ghost, self).__init__()\n c_ = c2 // 2\n self.conv = nn.Sequential(GhostConv(c1, c_, 1, 1), # pw\n DWConv(c_, c_, k, s, act=False) if s == 2 else nn.Identity(), # dw\n GhostConv(c_, c2, 1, 1, act=False)) # pw-linear\n self.shortcut = nn.Sequential(DWConv(c1, c1, k, s, act=False),\n Conv(c1, c2, 1, 1, act=False)) if s == 2 else nn.Identity()\n\n def forward(self, x):\n return self.conv(x) + self.shortcut(x)\n\n##### end of basic #####\n\n\n##### cspnet #####\n\nclass SPPCSPC(nn.Module):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, k=(5, 9, 13)):\n super(SPPCSPC, self).__init__()\n c_ = int(2 * c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(c_, c_, 3, 1)\n self.cv4 = Conv(c_, c_, 1, 1)\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\n self.cv5 = Conv(4 * c_, c_, 1, 1)\n self.cv6 = Conv(c_, c_, 3, 1)\n self.cv7 = Conv(2 * c_, c2, 1, 1)\n\n def forward(self, x):\n x1 = self.cv4(self.cv3(self.cv1(x)))\n y1 = self.cv6(self.cv5(torch.cat([x1] + [m(x1) for m in self.m], 1)))\n y2 = self.cv2(x)\n return self.cv7(torch.cat((y1, y2), dim=1))\n\nclass GhostSPPCSPC(SPPCSPC):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, k=(5, 9, 13)):\n super().__init__(c1, c2, n, shortcut, g, e, k)\n c_ = int(2 * c2 * e) # hidden channels\n self.cv1 = GhostConv(c1, c_, 1, 1)\n self.cv2 = GhostConv(c1, c_, 1, 1)\n self.cv3 = GhostConv(c_, c_, 3, 1)\n self.cv4 = GhostConv(c_, c_, 1, 1)\n self.cv5 = GhostConv(4 * c_, c_, 1, 1)\n self.cv6 = GhostConv(c_, c_, 3, 1)\n self.cv7 = GhostConv(2 * c_, c2, 1, 1)\n\n\nclass GhostStem(Stem):\n # Stem\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super().__init__(c1, c2, k, s, p, g, act)\n c_ = int(c2/2) # hidden channels\n self.cv1 = GhostConv(c1, c_, 3, 2)\n self.cv2 = GhostConv(c_, c_, 1, 1)\n self.cv3 = GhostConv(c_, c_, 3, 2)\n self.cv4 = GhostConv(2 * c_, c2, 1, 1)\n \n\nclass BottleneckCSPA(nn.Module):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPA, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(2 * c_, c2, 1, 1)\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.m(self.cv1(x))\n y2 = self.cv2(x)\n return self.cv3(torch.cat((y1, y2), dim=1))\n\n\nclass BottleneckCSPB(nn.Module):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPB, self).__init__()\n c_ = int(c2) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c_, 1, 1)\n self.cv3 = Conv(2 * c_, c2, 1, 1)\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n x1 = self.cv1(x)\n y1 = self.m(x1)\n y2 = self.cv2(x1)\n return self.cv3(torch.cat((y1, y2), dim=1))\n\n\nclass BottleneckCSPC(nn.Module):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(BottleneckCSPC, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(c_, c_, 1, 1)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(torch.cat((y1, y2), dim=1))\n\n\nclass ResCSPA(BottleneckCSPA):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\n\n\nclass ResCSPB(BottleneckCSPB):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2) # hidden channels\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\n\n\nclass ResCSPC(BottleneckCSPC):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\n\n\nclass ResXCSPA(ResCSPA):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n\nclass ResXCSPB(ResCSPB):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2) # hidden channels\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n\nclass ResXCSPC(ResCSPC):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n\nclass GhostCSPA(BottleneckCSPA):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[Ghost(c_, c_) for _ in range(n)])\n\n\nclass GhostCSPB(BottleneckCSPB):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2) # hidden channels\n self.m = nn.Sequential(*[Ghost(c_, c_) for _ in range(n)])\n\n\nclass GhostCSPC(BottleneckCSPC):\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[Ghost(c_, c_) for _ in range(n)])\n\n##### end of cspnet #####\n\n\n##### yolor #####\n\nclass ImplicitA(nn.Module):\n def __init__(self, channel, mean=0., std=.02):\n super(ImplicitA, self).__init__()\n self.channel = channel\n self.mean = mean\n self.std = std\n self.implicit = nn.Parameter(torch.zeros(1, channel, 1, 1))\n nn.init.normal_(self.implicit, mean=self.mean, std=self.std)\n\n def forward(self, x):\n return self.implicit + x\n \n\nclass ImplicitM(nn.Module):\n def __init__(self, channel, mean=0., std=.02):\n super(ImplicitM, self).__init__()\n self.channel = channel\n self.mean = mean\n self.std = std\n self.implicit = nn.Parameter(torch.ones(1, channel, 1, 1))\n nn.init.normal_(self.implicit, mean=self.mean, std=self.std)\n\n def forward(self, x):\n return self.implicit * x\n \n##### end of yolor #####\n\n\n##### repvgg #####\n\nclass RepConv(nn.Module):\n # Represented convolution\n # https://arxiv.org/abs/2101.03697\n\n def __init__(self, c1, c2, k=3, s=1, p=None, g=1, act=True, deploy=False):\n super(RepConv, self).__init__()\n\n self.deploy = deploy\n self.groups = g\n self.in_channels = c1\n self.out_channels = c2\n\n assert k == 3\n assert autopad(k, p) == 1\n\n padding_11 = autopad(k, p) - k // 2\n\n self.act = nn.SiLU() if act is True else (act if isinstance(act, nn.Module) else nn.Identity())\n\n if deploy:\n self.rbr_reparam = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=True)\n\n else:\n self.rbr_identity = (nn.BatchNorm2d(num_features=c1) if c2 == c1 and s == 1 else None)\n\n self.rbr_dense = nn.Sequential(\n nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False),\n nn.BatchNorm2d(num_features=c2),\n )\n\n self.rbr_1x1 = nn.Sequential(\n nn.Conv2d( c1, c2, 1, s, padding_11, groups=g, bias=False),\n nn.BatchNorm2d(num_features=c2),\n )\n\n def forward(self, inputs):\n if hasattr(self, \"rbr_reparam\"):\n return self.act(self.rbr_reparam(inputs))\n\n if self.rbr_identity is None:\n id_out = 0\n else:\n id_out = self.rbr_identity(inputs)\n\n return self.act(self.rbr_dense(inputs) + self.rbr_1x1(inputs) + id_out)\n \n def get_equivalent_kernel_bias(self):\n kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense)\n kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1)\n kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity)\n return (\n kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid,\n bias3x3 + bias1x1 + biasid,\n )\n\n def _pad_1x1_to_3x3_tensor(self, kernel1x1):\n if kernel1x1 is None:\n return 0\n else:\n return nn.functional.pad(kernel1x1, [1, 1, 1, 1])\n\n def _fuse_bn_tensor(self, branch):\n if branch is None:\n return 0, 0\n if isinstance(branch, nn.Sequential):\n kernel = branch[0].weight\n running_mean = branch[1].running_mean\n running_var = branch[1].running_var\n gamma = branch[1].weight\n beta = branch[1].bias\n eps = branch[1].eps\n else:\n assert isinstance(branch, nn.BatchNorm2d)\n if not hasattr(self, \"id_tensor\"):\n input_dim = self.in_channels // self.groups\n kernel_value = np.zeros(\n (self.in_channels, input_dim, 3, 3), dtype=np.float32\n )\n for i in range(self.in_channels):\n kernel_value[i, i % input_dim, 1, 1] = 1\n self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device)\n kernel = self.id_tensor\n running_mean = branch.running_mean\n running_var = branch.running_var\n gamma = branch.weight\n beta = branch.bias\n eps = branch.eps\n std = (running_var + eps).sqrt()\n t = (gamma / std).reshape(-1, 1, 1, 1)\n return kernel * t, beta - running_mean * gamma / std\n\n def repvgg_convert(self):\n kernel, bias = self.get_equivalent_kernel_bias()\n return (\n kernel.detach().cpu().numpy(),\n bias.detach().cpu().numpy(),\n )\n\n def fuse_conv_bn(self, conv, bn):\n\n std = (bn.running_var + bn.eps).sqrt()\n bias = bn.bias - bn.running_mean * bn.weight / std\n\n t = (bn.weight / std).reshape(-1, 1, 1, 1)\n weights = conv.weight * t\n\n bn = nn.Identity()\n conv = nn.Conv2d(in_channels = conv.in_channels,\n out_channels = conv.out_channels,\n kernel_size = conv.kernel_size,\n stride=conv.stride,\n padding = conv.padding,\n dilation = conv.dilation,\n groups = conv.groups,\n bias = True,\n padding_mode = conv.padding_mode)\n\n conv.weight = torch.nn.Parameter(weights)\n conv.bias = torch.nn.Parameter(bias)\n return conv\n\n def fuse_repvgg_block(self): \n if self.deploy:\n return\n print(f\"RepConv.fuse_repvgg_block\")\n \n self.rbr_dense = self.fuse_conv_bn(self.rbr_dense[0], self.rbr_dense[1])\n \n self.rbr_1x1 = self.fuse_conv_bn(self.rbr_1x1[0], self.rbr_1x1[1])\n rbr_1x1_bias = self.rbr_1x1.bias\n weight_1x1_expanded = torch.nn.functional.pad(self.rbr_1x1.weight, [1, 1, 1, 1])\n \n # Fuse self.rbr_identity\n if (isinstance(self.rbr_identity, nn.BatchNorm2d) or isinstance(self.rbr_identity, nn.modules.batchnorm.SyncBatchNorm)):\n # print(f\"fuse: rbr_identity == BatchNorm2d or SyncBatchNorm\")\n identity_conv_1x1 = nn.Conv2d(\n in_channels=self.in_channels,\n out_channels=self.out_channels,\n kernel_size=1,\n stride=1,\n padding=0,\n groups=self.groups, \n bias=False)\n identity_conv_1x1.weight.data = identity_conv_1x1.weight.data.to(self.rbr_1x1.weight.data.device)\n identity_conv_1x1.weight.data = identity_conv_1x1.weight.data.squeeze().squeeze()\n # print(f\" identity_conv_1x1.weight = {identity_conv_1x1.weight.shape}\")\n identity_conv_1x1.weight.data.fill_(0.0)\n identity_conv_1x1.weight.data.fill_diagonal_(1.0)\n identity_conv_1x1.weight.data = identity_conv_1x1.weight.data.unsqueeze(2).unsqueeze(3)\n # print(f\" identity_conv_1x1.weight = {identity_conv_1x1.weight.shape}\")\n\n identity_conv_1x1 = self.fuse_conv_bn(identity_conv_1x1, self.rbr_identity)\n bias_identity_expanded = identity_conv_1x1.bias\n weight_identity_expanded = torch.nn.functional.pad(identity_conv_1x1.weight, [1, 1, 1, 1]) \n else:\n # print(f\"fuse: rbr_identity != BatchNorm2d, rbr_identity = {self.rbr_identity}\")\n bias_identity_expanded = torch.nn.Parameter( torch.zeros_like(rbr_1x1_bias) )\n weight_identity_expanded = torch.nn.Parameter( torch.zeros_like(weight_1x1_expanded) ) \n \n\n #print(f\"self.rbr_1x1.weight = {self.rbr_1x1.weight.shape}, \")\n #print(f\"weight_1x1_expanded = {weight_1x1_expanded.shape}, \")\n #print(f\"self.rbr_dense.weight = {self.rbr_dense.weight.shape}, \")\n\n self.rbr_dense.weight = torch.nn.Parameter(self.rbr_dense.weight + weight_1x1_expanded + weight_identity_expanded)\n self.rbr_dense.bias = torch.nn.Parameter(self.rbr_dense.bias + rbr_1x1_bias + bias_identity_expanded)\n \n self.rbr_reparam = self.rbr_dense\n self.deploy = True\n\n if self.rbr_identity is not None:\n del self.rbr_identity\n self.rbr_identity = None\n\n if self.rbr_1x1 is not None:\n del self.rbr_1x1\n self.rbr_1x1 = None\n\n if self.rbr_dense is not None:\n del self.rbr_dense\n self.rbr_dense = None\n\n\nclass RepBottleneck(Bottleneck):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super().__init__(c1, c2, shortcut=True, g=1, e=0.5)\n c_ = int(c2 * e) # hidden channels\n self.cv2 = RepConv(c_, c2, 3, 1, g=g)\n\n\nclass RepBottleneckCSPA(BottleneckCSPA):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[RepBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n\nclass RepBottleneckCSPB(BottleneckCSPB):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2) # hidden channels\n self.m = nn.Sequential(*[RepBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n\nclass RepBottleneckCSPC(BottleneckCSPC):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[RepBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n\nclass RepRes(Res):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super().__init__(c1, c2, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.cv2 = RepConv(c_, c_, 3, 1, g=g)\n\n\nclass RepResCSPA(ResCSPA):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[RepRes(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\n\n\nclass RepResCSPB(ResCSPB):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2) # hidden channels\n self.m = nn.Sequential(*[RepRes(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\n\n\nclass RepResCSPC(ResCSPC):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[RepRes(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\n\n\nclass RepResX(ResX):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=32, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super().__init__(c1, c2, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.cv2 = RepConv(c_, c_, 3, 1, g=g)\n\n\nclass RepResXCSPA(ResXCSPA):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[RepResX(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\n\n\nclass RepResXCSPB(ResXCSPB):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2) # hidden channels\n self.m = nn.Sequential(*[RepResX(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\n\n\nclass RepResXCSPC(ResXCSPC):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__(c1, c2, n, shortcut, g, e)\n c_ = int(c2 * e) # hidden channels\n self.m = nn.Sequential(*[RepResX(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\n\n##### end of repvgg #####\n\n\n##### transformer #####\n\nclass TransformerLayer(nn.Module):\n # Transformer layer https://arxiv.org/abs/2010.11929 (LayerNorm layers removed for better performance)\n def __init__(self, c, num_heads):\n super().__init__()\n self.q = nn.Linear(c, c, bias=False)\n self.k = nn.Linear(c, c, bias=False)\n self.v = nn.Linear(c, c, bias=False)\n self.ma = nn.MultiheadAttention(embed_dim=c, num_heads=num_heads)\n self.fc1 = nn.Linear(c, c, bias=False)\n self.fc2 = nn.Linear(c, c, bias=False)\n\n def forward(self, x):\n x = self.ma(self.q(x), self.k(x), self.v(x))[0] + x\n x = self.fc2(self.fc1(x)) + x\n return x\n\n\nclass TransformerBlock(nn.Module):\n # Vision Transformer https://arxiv.org/abs/2010.11929\n def __init__(self, c1, c2, num_heads, num_layers):\n super().__init__()\n self.conv = None\n if c1 != c2:\n self.conv = Conv(c1, c2)\n self.linear = nn.Linear(c2, c2) # learnable position embedding\n self.tr = nn.Sequential(*[TransformerLayer(c2, num_heads) for _ in range(num_layers)])\n self.c2 = c2\n\n def forward(self, x):\n if self.conv is not None:\n x = self.conv(x)\n b, _, w, h = x.shape\n p = x.flatten(2)\n p = p.unsqueeze(0)\n p = p.transpose(0, 3)\n p = p.squeeze(3)\n e = self.linear(p)\n x = p + e\n\n x = self.tr(x)\n x = x.unsqueeze(3)\n x = x.transpose(0, 3)\n x = x.reshape(b, self.c2, w, h)\n return x\n\n##### end of transformer #####\n\n\n##### yolov5 #####\n\nclass Focus(nn.Module):\n # Focus wh information into c-space\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\n super(Focus, self).__init__()\n self.conv = Conv(c1 * 4, c2, k, s, p, g, act)\n # self.contract = Contract(gain=2)\n\n def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2)\n return self.conv(torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1))\n # return self.conv(self.contract(x))\n \n\nclass SPPF(nn.Module):\n # Spatial Pyramid Pooling - Fast (SPPF) layer for YOLOv5 by Glenn Jocher\n def __init__(self, c1, c2, k=5): # equivalent to SPP(k=(5, 9, 13))\n super().__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_ * 4, c2, 1, 1)\n self.m = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2)\n\n def forward(self, x):\n x = self.cv1(x)\n y1 = self.m(x)\n y2 = self.m(y1)\n return self.cv2(torch.cat([x, y1, y2, self.m(y2)], 1))\n \n \nclass Contract(nn.Module):\n # Contract width-height into channels, i.e. x(1,64,80,80) to x(1,256,40,40)\n def __init__(self, gain=2):\n super().__init__()\n self.gain = gain\n\n def forward(self, x):\n N, C, H, W = x.size() # assert (H / s == 0) and (W / s == 0), 'Indivisible gain'\n s = self.gain\n x = x.view(N, C, H // s, s, W // s, s) # x(1,64,40,2,40,2)\n x = x.permute(0, 3, 5, 1, 2, 4).contiguous() # x(1,2,2,64,40,40)\n return x.view(N, C * s * s, H // s, W // s) # x(1,256,40,40)\n\n\nclass Expand(nn.Module):\n # Expand channels into width-height, i.e. x(1,64,80,80) to x(1,16,160,160)\n def __init__(self, gain=2):\n super().__init__()\n self.gain = gain\n\n def forward(self, x):\n N, C, H, W = x.size() # assert C / s ** 2 == 0, 'Indivisible gain'\n s = self.gain\n x = x.view(N, s, s, C // s ** 2, H, W) # x(1,2,2,16,80,80)\n x = x.permute(0, 3, 4, 1, 5, 2).contiguous() # x(1,16,80,2,80,2)\n return x.view(N, C // s ** 2, H * s, W * s) # x(1,16,160,160)\n\n\nclass NMS(nn.Module):\n # Non-Maximum Suppression (NMS) module\n conf = 0.25 # confidence threshold\n iou = 0.45 # IoU threshold\n classes = None # (optional list) filter by class\n\n def __init__(self):\n super(NMS, self).__init__()\n\n def forward(self, x):\n return non_max_suppression(x[0], conf_thres=self.conf, iou_thres=self.iou, classes=self.classes)\n\n\nclass autoShape(nn.Module):\n # input-robust model wrapper for passing cv2/np/PIL/torch inputs. Includes preprocessing, inference and NMS\n conf = 0.25 # NMS confidence threshold\n iou = 0.45 # NMS IoU threshold\n classes = None # (optional list) filter by class\n\n def __init__(self, model):\n super(autoShape, self).__init__()\n self.model = model.eval()\n\n def autoshape(self):\n print('autoShape already enabled, skipping... ') # model already converted to model.autoshape()\n return self\n\n @torch.no_grad()\n def forward(self, imgs, size=640, augment=False, profile=False):\n # Inference from various sources. For height=640, width=1280, RGB images example inputs are:\n # filename: imgs = 'data/samples/zidane.jpg'\n # URI: = 'https://github.com/ultralytics/yolov5/releases/download/v1.0/zidane.jpg'\n # OpenCV: = cv2.imread('image.jpg')[:,:,::-1] # HWC BGR to RGB x(640,1280,3)\n # PIL: = Image.open('image.jpg') # HWC x(640,1280,3)\n # numpy: = np.zeros((640,1280,3)) # HWC\n # torch: = torch.zeros(16,3,320,640) # BCHW (scaled to size=640, 0-1 values)\n # multiple: = [Image.open('image1.jpg'), Image.open('image2.jpg'), ...] # list of images\n\n t = [time_synchronized()]\n p = next(self.model.parameters()) # for device and type\n if isinstance(imgs, torch.Tensor): # torch\n with amp.autocast(enabled=p.device.type != 'cpu'):\n return self.model(imgs.to(p.device).type_as(p), augment, profile) # inference\n\n # Pre-process\n n, imgs = (len(imgs), imgs) if isinstance(imgs, list) else (1, [imgs]) # number of images, list of images\n shape0, shape1, files = [], [], [] # image and inference shapes, filenames\n for i, im in enumerate(imgs):\n f = f'image{i}' # filename\n if isinstance(im, str): # filename or uri\n im, f = np.asarray(Image.open(requests.get(im, stream=True).raw if im.startswith('http') else im)), im\n elif isinstance(im, Image.Image): # PIL Image\n im, f = np.asarray(im), getattr(im, 'filename', f) or f\n files.append(Path(f).with_suffix('.jpg').name)\n if im.shape[0] < 5: # image in CHW\n im = im.transpose((1, 2, 0)) # reverse dataloader .transpose(2, 0, 1)\n im = im[:, :, :3] if im.ndim == 3 else np.tile(im[:, :, None], 3) # enforce 3ch input\n s = im.shape[:2] # HWC\n shape0.append(s) # image shape\n g = (size / max(s)) # gain\n shape1.append([y * g for y in s])\n imgs[i] = im # update\n shape1 = [make_divisible(x, int(self.stride.max())) for x in np.stack(shape1, 0).max(0)] # inference shape\n x = [letterbox(im, new_shape=shape1, auto=False)[0] for im in imgs] # pad\n x = np.stack(x, 0) if n > 1 else x[0][None] # stack\n x = np.ascontiguousarray(x.transpose((0, 3, 1, 2))) # BHWC to BCHW\n x = torch.from_numpy(x).to(p.device).type_as(p) / 255. # uint8 to fp16/32\n t.append(time_synchronized())\n\n with amp.autocast(enabled=p.device.type != 'cpu'):\n # Inference\n y = self.model(x, augment, profile)[0] # forward\n t.append(time_synchronized())\n\n # Post-process\n y = non_max_suppression(y, conf_thres=self.conf, iou_thres=self.iou, classes=self.classes) # NMS\n for i in range(n):\n scale_coords(shape1, y[i][:, :4], shape0[i])\n\n t.append(time_synchronized())\n return Detections(imgs, y, files, t, self.names, x.shape)\n\n\nclass Detections:\n # detections class for YOLOv5 inference results\n def __init__(self, imgs, pred, files, times=None, names=None, shape=None):\n super(Detections, self).__init__()\n d = pred[0].device # device\n gn = [torch.tensor([*[im.shape[i] for i in [1, 0, 1, 0]], 1., 1.], device=d) for im in imgs] # normalizations\n self.imgs = imgs # list of images as numpy arrays\n self.pred = pred # list of tensors pred[0] = (xyxy, conf, cls)\n self.names = names # class names\n self.files = files # image filenames\n self.xyxy = pred # xyxy pixels\n self.xywh = [xyxy2xywh(x) for x in pred] # xywh pixels\n self.xyxyn = [x / g for x, g in zip(self.xyxy, gn)] # xyxy normalized\n self.xywhn = [x / g for x, g in zip(self.xywh, gn)] # xywh normalized\n self.n = len(self.pred) # number of images (batch size)\n self.t = tuple((times[i + 1] - times[i]) * 1000 / self.n for i in range(3)) # timestamps (ms)\n self.s = shape # inference BCHW shape\n\n def display(self, pprint=False, show=False, save=False, render=False, save_dir=''):\n colors = color_list()\n for i, (img, pred) in enumerate(zip(self.imgs, self.pred)):\n str = f'image {i + 1}/{len(self.pred)}: {img.shape[0]}x{img.shape[1]} '\n if pred is not None:\n for c in pred[:, -1].unique():\n n = (pred[:, -1] == c).sum() # detections per class\n str += f\"{n} {self.names[int(c)]}{'s' * (n > 1)}, \" # add to string\n if show or save or render:\n for *box, conf, cls in pred: # xyxy, confidence, class\n label = f'{self.names[int(cls)]} {conf:.2f}'\n plot_one_box(box, img, label=label, color=colors[int(cls) % 10])\n img = Image.fromarray(img.astype(np.uint8)) if isinstance(img, np.ndarray) else img # from np\n if pprint:\n print(str.rstrip(', '))\n if show:\n img.show(self.files[i]) # show\n if save:\n f = self.files[i]\n img.save(Path(save_dir) / f) # save\n print(f\"{'Saved' * (i == 0)} {f}\", end=',' if i < self.n - 1 else f' to {save_dir}\\n')\n if render:\n self.imgs[i] = np.asarray(img)\n\n def print(self):\n self.display(pprint=True) # print results\n print(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {tuple(self.s)}' % self.t)\n\n def show(self):\n self.display(show=True) # show results\n\n def save(self, save_dir='runs/hub/exp'):\n save_dir = increment_path(save_dir, exist_ok=save_dir != 'runs/hub/exp') # increment save_dir\n Path(save_dir).mkdir(parents=True, exist_ok=True)\n self.display(save=True, save_dir=save_dir) # save results\n\n def render(self):\n self.display(render=True) # render results\n return self.imgs\n\n def pandas(self):\n # return detections as pandas DataFrames, i.e. print(results.pandas().xyxy[0])\n new = copy(self) # return copy\n ca = 'xmin', 'ymin', 'xmax', 'ymax', 'confidence', 'class', 'name' # xyxy columns\n cb = 'xcenter', 'ycenter', 'width', 'height', 'confidence', 'class', 'name' # xywh columns\n for k, c in zip(['xyxy', 'xyxyn', 'xywh', 'xywhn'], [ca, ca, cb, cb]):\n a = [[x[:5] + [int(x[5]), self.names[int(x[5])]] for x in x.tolist()] for x in getattr(self, k)] # update\n setattr(new, k, [pd.DataFrame(x, columns=c) for x in a])\n return new\n\n def tolist(self):\n # return a list of Detections objects, i.e. 'for result in results.tolist():'\n x = [Detections([self.imgs[i]], [self.pred[i]], self.names, self.s) for i in range(self.n)]\n for d in x:\n for k in ['imgs', 'pred', 'xyxy', 'xyxyn', 'xywh', 'xywhn']:\n setattr(d, k, getattr(d, k)[0]) # pop out of list\n return x\n\n def __len__(self):\n return self.n\n\n\nclass Classify(nn.Module):\n # Classification head, i.e. x(b,c1,20,20) to x(b,c2)\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1): # ch_in, ch_out, kernel, stride, padding, groups\n super(Classify, self).__init__()\n self.aap = nn.AdaptiveAvgPool2d(1) # to x(b,c1,1,1)\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g) # to x(b,c2,1,1)\n self.flat = nn.Flatten()\n\n def forward(self, x):\n z = torch.cat([self.aap(y) for y in (x if isinstance(x, list) else [x])], 1) # cat if list\n return self.flat(self.conv(z)) # flatten to x(b,c2)\n\n##### end of yolov5 ######\n\n\n##### orepa #####\n\ndef transI_fusebn(kernel, bn):\n gamma = bn.weight\n std = (bn.running_var + bn.eps).sqrt()\n return kernel * ((gamma / std).reshape(-1, 1, 1, 1)), bn.bias - bn.running_mean * gamma / std\n \n \nclass ConvBN(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size,\n stride=1, padding=0, dilation=1, groups=1, deploy=False, nonlinear=None):\n super().__init__()\n if nonlinear is None:\n self.nonlinear = nn.Identity()\n else:\n self.nonlinear = nonlinear\n if deploy:\n self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,\n stride=stride, padding=padding, dilation=dilation, groups=groups, bias=True)\n else:\n self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,\n stride=stride, padding=padding, dilation=dilation, groups=groups, bias=False)\n self.bn = nn.BatchNorm2d(num_features=out_channels)\n\n def forward(self, x):\n if hasattr(self, 'bn'):\n return self.nonlinear(self.bn(self.conv(x)))\n else:\n return self.nonlinear(self.conv(x))\n\n def switch_to_deploy(self):\n kernel, bias = transI_fusebn(self.conv.weight, self.bn)\n conv = nn.Conv2d(in_channels=self.conv.in_channels, out_channels=self.conv.out_channels, kernel_size=self.conv.kernel_size,\n stride=self.conv.stride, padding=self.conv.padding, dilation=self.conv.dilation, groups=self.conv.groups, bias=True)\n conv.weight.data = kernel\n conv.bias.data = bias\n for para in self.parameters():\n para.detach_()\n self.__delattr__('conv')\n self.__delattr__('bn')\n self.conv = conv \n\nclass OREPA_3x3_RepConv(nn.Module):\n\n def __init__(self, in_channels, out_channels, kernel_size,\n stride=1, padding=0, dilation=1, groups=1,\n internal_channels_1x1_3x3=None,\n deploy=False, nonlinear=None, single_init=False):\n super(OREPA_3x3_RepConv, self).__init__()\n self.deploy = deploy\n\n if nonlinear is None:\n self.nonlinear = nn.Identity()\n else:\n self.nonlinear = nonlinear\n\n self.kernel_size = kernel_size\n self.in_channels = in_channels\n self.out_channels = out_channels\n self.groups = groups\n assert padding == kernel_size // 2\n\n self.stride = stride\n self.padding = padding\n self.dilation = dilation\n\n self.branch_counter = 0\n\n self.weight_rbr_origin = nn.Parameter(torch.Tensor(out_channels, int(in_channels/self.groups), kernel_size, kernel_size))\n nn.init.kaiming_uniform_(self.weight_rbr_origin, a=math.sqrt(1.0))\n self.branch_counter += 1\n\n\n if groups < out_channels:\n self.weight_rbr_avg_conv = nn.Parameter(torch.Tensor(out_channels, int(in_channels/self.groups), 1, 1))\n self.weight_rbr_pfir_conv = nn.Parameter(torch.Tensor(out_channels, int(in_channels/self.groups), 1, 1))\n nn.init.kaiming_uniform_(self.weight_rbr_avg_conv, a=1.0)\n nn.init.kaiming_uniform_(self.weight_rbr_pfir_conv, a=1.0)\n self.weight_rbr_avg_conv.data\n self.weight_rbr_pfir_conv.data\n self.register_buffer('weight_rbr_avg_avg', torch.ones(kernel_size, kernel_size).mul(1.0/kernel_size/kernel_size))\n self.branch_counter += 1\n\n else:\n raise NotImplementedError\n self.branch_counter += 1\n\n if internal_channels_1x1_3x3 is None:\n internal_channels_1x1_3x3 = in_channels if groups < out_channels else 2 * in_channels # For mobilenet, it is better to have 2X internal channels\n\n if internal_channels_1x1_3x3 == in_channels:\n self.weight_rbr_1x1_kxk_idconv1 = nn.Parameter(torch.zeros(in_channels, int(in_channels/self.groups), 1, 1))\n id_value = np.zeros((in_channels, int(in_channels/self.groups), 1, 1))\n for i in range(in_channels):\n id_value[i, i % int(in_channels/self.groups), 0, 0] = 1\n id_tensor = torch.from_numpy(id_value).type_as(self.weight_rbr_1x1_kxk_idconv1)\n self.register_buffer('id_tensor', id_tensor)\n\n else:\n self.weight_rbr_1x1_kxk_conv1 = nn.Parameter(torch.Tensor(internal_channels_1x1_3x3, int(in_channels/self.groups), 1, 1))\n nn.init.kaiming_uniform_(self.weight_rbr_1x1_kxk_conv1, a=math.sqrt(1.0))\n self.weight_rbr_1x1_kxk_conv2 = nn.Parameter(torch.Tensor(out_channels, int(internal_channels_1x1_3x3/self.groups), kernel_size, kernel_size))\n nn.init.kaiming_uniform_(self.weight_rbr_1x1_kxk_conv2, a=math.sqrt(1.0))\n self.branch_counter += 1\n\n expand_ratio = 8\n self.weight_rbr_gconv_dw = nn.Parameter(torch.Tensor(in_channels*expand_ratio, 1, kernel_size, kernel_size))\n self.weight_rbr_gconv_pw = nn.Parameter(torch.Tensor(out_channels, in_channels*expand_ratio, 1, 1))\n nn.init.kaiming_uniform_(self.weight_rbr_gconv_dw, a=math.sqrt(1.0))\n nn.init.kaiming_uniform_(self.weight_rbr_gconv_pw, a=math.sqrt(1.0))\n self.branch_counter += 1\n\n if out_channels == in_channels and stride == 1:\n self.branch_counter += 1\n\n self.vector = nn.Parameter(torch.Tensor(self.branch_counter, self.out_channels))\n self.bn = nn.BatchNorm2d(out_channels)\n\n self.fre_init()\n\n nn.init.constant_(self.vector[0, :], 0.25) #origin\n nn.init.constant_(self.vector[1, :], 0.25) #avg\n nn.init.constant_(self.vector[2, :], 0.0) #prior\n nn.init.constant_(self.vector[3, :], 0.5) #1x1_kxk\n nn.init.constant_(self.vector[4, :], 0.5) #dws_conv\n\n\n def fre_init(self):\n prior_tensor = torch.Tensor(self.out_channels, self.kernel_size, self.kernel_size)\n half_fg = self.out_channels/2\n for i in range(self.out_channels):\n for h in range(3):\n for w in range(3):\n if i < half_fg:\n prior_tensor[i, h, w] = math.cos(math.pi*(h+0.5)*(i+1)/3)\n else:\n prior_tensor[i, h, w] = math.cos(math.pi*(w+0.5)*(i+1-half_fg)/3)\n\n self.register_buffer('weight_rbr_prior', prior_tensor)\n\n def weight_gen(self):\n\n weight_rbr_origin = torch.einsum('oihw,o->oihw', self.weight_rbr_origin, self.vector[0, :])\n\n weight_rbr_avg = torch.einsum('oihw,o->oihw', torch.einsum('oihw,hw->oihw', self.weight_rbr_avg_conv, self.weight_rbr_avg_avg), self.vector[1, :])\n \n weight_rbr_pfir = torch.einsum('oihw,o->oihw', torch.einsum('oihw,ohw->oihw', self.weight_rbr_pfir_conv, self.weight_rbr_prior), self.vector[2, :])\n\n weight_rbr_1x1_kxk_conv1 = None\n if hasattr(self, 'weight_rbr_1x1_kxk_idconv1'):\n weight_rbr_1x1_kxk_conv1 = (self.weight_rbr_1x1_kxk_idconv1 + self.id_tensor).squeeze()\n elif hasattr(self, 'weight_rbr_1x1_kxk_conv1'):\n weight_rbr_1x1_kxk_conv1 = self.weight_rbr_1x1_kxk_conv1.squeeze()\n else:\n raise NotImplementedError\n weight_rbr_1x1_kxk_conv2 = self.weight_rbr_1x1_kxk_conv2\n\n if self.groups > 1:\n g = self.groups\n t, ig = weight_rbr_1x1_kxk_conv1.size()\n o, tg, h, w = weight_rbr_1x1_kxk_conv2.size()\n weight_rbr_1x1_kxk_conv1 = weight_rbr_1x1_kxk_conv1.view(g, int(t/g), ig)\n weight_rbr_1x1_kxk_conv2 = weight_rbr_1x1_kxk_conv2.view(g, int(o/g), tg, h, w)\n weight_rbr_1x1_kxk = torch.einsum('gti,gothw->goihw', weight_rbr_1x1_kxk_conv1, weight_rbr_1x1_kxk_conv2).view(o, ig, h, w)\n else:\n weight_rbr_1x1_kxk = torch.einsum('ti,othw->oihw', weight_rbr_1x1_kxk_conv1, weight_rbr_1x1_kxk_conv2)\n\n weight_rbr_1x1_kxk = torch.einsum('oihw,o->oihw', weight_rbr_1x1_kxk, self.vector[3, :])\n\n weight_rbr_gconv = self.dwsc2full(self.weight_rbr_gconv_dw, self.weight_rbr_gconv_pw, self.in_channels)\n weight_rbr_gconv = torch.einsum('oihw,o->oihw', weight_rbr_gconv, self.vector[4, :]) \n\n weight = weight_rbr_origin + weight_rbr_avg + weight_rbr_1x1_kxk + weight_rbr_pfir + weight_rbr_gconv\n\n return weight\n\n def dwsc2full(self, weight_dw, weight_pw, groups):\n \n t, ig, h, w = weight_dw.size()\n o, _, _, _ = weight_pw.size()\n tg = int(t/groups)\n i = int(ig*groups)\n weight_dw = weight_dw.view(groups, tg, ig, h, w)\n weight_pw = weight_pw.squeeze().view(o, groups, tg)\n \n weight_dsc = torch.einsum('gtihw,ogt->ogihw', weight_dw, weight_pw)\n return weight_dsc.view(o, i, h, w)\n\n def forward(self, inputs):\n weight = self.weight_gen()\n out = F.conv2d(inputs, weight, bias=None, stride=self.stride, padding=self.padding, dilation=self.dilation, groups=self.groups)\n\n return self.nonlinear(self.bn(out))\n\nclass RepConv_OREPA(nn.Module):\n\n def __init__(self, c1, c2, k=3, s=1, padding=1, dilation=1, groups=1, padding_mode='zeros', deploy=False, use_se=False, nonlinear=nn.SiLU()):\n super(RepConv_OREPA, self).__init__()\n self.deploy = deploy\n self.groups = groups\n self.in_channels = c1\n self.out_channels = c2\n\n self.padding = padding\n self.dilation = dilation\n self.groups = groups\n\n assert k == 3\n assert padding == 1\n\n padding_11 = padding - k // 2\n\n if nonlinear is None:\n self.nonlinearity = nn.Identity()\n else:\n self.nonlinearity = nonlinear\n\n if use_se:\n self.se = SEBlock(self.out_channels, internal_neurons=self.out_channels // 16)\n else:\n self.se = nn.Identity()\n\n if deploy:\n self.rbr_reparam = nn.Conv2d(in_channels=self.in_channels, out_channels=self.out_channels, kernel_size=k, stride=s,\n padding=padding, dilation=dilation, groups=groups, bias=True, padding_mode=padding_mode)\n\n else:\n self.rbr_identity = nn.BatchNorm2d(num_features=self.in_channels) if self.out_channels == self.in_channels and s == 1 else None\n self.rbr_dense = OREPA_3x3_RepConv(in_channels=self.in_channels, out_channels=self.out_channels, kernel_size=k, stride=s, padding=padding, groups=groups, dilation=1)\n self.rbr_1x1 = ConvBN(in_channels=self.in_channels, out_channels=self.out_channels, kernel_size=1, stride=s, padding=padding_11, groups=groups, dilation=1)\n print('RepVGG Block, identity = ', self.rbr_identity)\n\n\n def forward(self, inputs):\n if hasattr(self, 'rbr_reparam'):\n return self.nonlinearity(self.se(self.rbr_reparam(inputs)))\n\n if self.rbr_identity is None:\n id_out = 0\n else:\n id_out = self.rbr_identity(inputs)\n\n out1 = self.rbr_dense(inputs)\n out2 = self.rbr_1x1(inputs)\n out3 = id_out\n out = out1 + out2 + out3\n\n return self.nonlinearity(self.se(out))\n\n\n # Optional. This improves the accuracy and facilitates quantization.\n # 1. Cancel the original weight decay on rbr_dense.conv.weight and rbr_1x1.conv.weight.\n # 2. Use like this.\n # loss = criterion(....)\n # for every RepVGGBlock blk:\n # loss += weight_decay_coefficient * 0.5 * blk.get_cust_L2()\n # optimizer.zero_grad()\n # loss.backward()\n\n # Not used for OREPA\n def get_custom_L2(self):\n K3 = self.rbr_dense.weight_gen()\n K1 = self.rbr_1x1.conv.weight\n t3 = (self.rbr_dense.bn.weight / ((self.rbr_dense.bn.running_var + self.rbr_dense.bn.eps).sqrt())).reshape(-1, 1, 1, 1).detach()\n t1 = (self.rbr_1x1.bn.weight / ((self.rbr_1x1.bn.running_var + self.rbr_1x1.bn.eps).sqrt())).reshape(-1, 1, 1, 1).detach()\n\n l2_loss_circle = (K3 ** 2).sum() - (K3[:, :, 1:2, 1:2] ** 2).sum() # The L2 loss of the \"circle\" of weights in 3x3 kernel. Use regular L2 on them.\n eq_kernel = K3[:, :, 1:2, 1:2] * t3 + K1 * t1 # The equivalent resultant central point of 3x3 kernel.\n l2_loss_eq_kernel = (eq_kernel ** 2 / (t3 ** 2 + t1 ** 2)).sum() # Normalize for an L2 coefficient comparable to regular L2.\n return l2_loss_eq_kernel + l2_loss_circle\n\n def get_equivalent_kernel_bias(self):\n kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense)\n kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1)\n kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity)\n return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid\n\n def _pad_1x1_to_3x3_tensor(self, kernel1x1):\n if kernel1x1 is None:\n return 0\n else:\n return torch.nn.functional.pad(kernel1x1, [1,1,1,1])\n\n def _fuse_bn_tensor(self, branch):\n if branch is None:\n return 0, 0\n if not isinstance(branch, nn.BatchNorm2d):\n if isinstance(branch, OREPA_3x3_RepConv):\n kernel = branch.weight_gen()\n elif isinstance(branch, ConvBN):\n kernel = branch.conv.weight\n else:\n raise NotImplementedError\n running_mean = branch.bn.running_mean\n running_var = branch.bn.running_var\n gamma = branch.bn.weight\n beta = branch.bn.bias\n eps = branch.bn.eps\n else:\n if not hasattr(self, 'id_tensor'):\n input_dim = self.in_channels // self.groups\n kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), dtype=np.float32)\n for i in range(self.in_channels):\n kernel_value[i, i % input_dim, 1, 1] = 1\n self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device)\n kernel = self.id_tensor\n running_mean = branch.running_mean\n running_var = branch.running_var\n gamma = branch.weight\n beta = branch.bias\n eps = branch.eps\n std = (running_var + eps).sqrt()\n t = (gamma / std).reshape(-1, 1, 1, 1)\n return kernel * t, beta - running_mean * gamma / std\n\n def switch_to_deploy(self):\n if hasattr(self, 'rbr_reparam'):\n return\n print(f\"RepConv_OREPA.switch_to_deploy\")\n kernel, bias = self.get_equivalent_kernel_bias()\n self.rbr_reparam = nn.Conv2d(in_channels=self.rbr_dense.in_channels, out_channels=self.rbr_dense.out_channels,\n kernel_size=self.rbr_dense.kernel_size, stride=self.rbr_dense.stride,\n padding=self.rbr_dense.padding, dilation=self.rbr_dense.dilation, groups=self.rbr_dense.groups, bias=True)\n self.rbr_reparam.weight.data = kernel\n self.rbr_reparam.bias.data = bias\n for para in self.parameters():\n para.detach_()\n self.__delattr__('rbr_dense')\n self.__delattr__('rbr_1x1')\n if hasattr(self, 'rbr_identity'):\n self.__delattr__('rbr_identity') \n\n##### end of orepa #####\n\n\n##### swin transformer ##### \n \nclass WindowAttention(nn.Module):\n\n def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.):\n\n super().__init__()\n self.dim = dim\n self.window_size = window_size # Wh, Ww\n self.num_heads = num_heads\n head_dim = dim // num_heads\n self.scale = qk_scale or head_dim ** -0.5\n\n # define a parameter table of relative position bias\n self.relative_position_bias_table = nn.Parameter(\n torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads)) # 2*Wh-1 * 2*Ww-1, nH\n\n # get pair-wise relative position index for each token inside the window\n coords_h = torch.arange(self.window_size[0])\n coords_w = torch.arange(self.window_size[1])\n coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww\n coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww\n relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww\n relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2\n relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0\n relative_coords[:, :, 1] += self.window_size[1] - 1\n relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1\n relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww\n self.register_buffer(\"relative_position_index\", relative_position_index)\n\n self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)\n self.attn_drop = nn.Dropout(attn_drop)\n self.proj = nn.Linear(dim, dim)\n self.proj_drop = nn.Dropout(proj_drop)\n\n nn.init.normal_(self.relative_position_bias_table, std=.02)\n self.softmax = nn.Softmax(dim=-1)\n\n def forward(self, x, mask=None):\n\n B_, N, C = x.shape\n qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\n q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple)\n\n q = q * self.scale\n attn = (q @ k.transpose(-2, -1))\n\n relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(\n self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH\n relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww\n attn = attn + relative_position_bias.unsqueeze(0)\n\n if mask is not None:\n nW = mask.shape[0]\n attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)\n attn = attn.view(-1, self.num_heads, N, N)\n attn = self.softmax(attn)\n else:\n attn = self.softmax(attn)\n\n attn = self.attn_drop(attn)\n\n # print(attn.dtype, v.dtype)\n try:\n x = (attn @ v).transpose(1, 2).reshape(B_, N, C)\n except:\n #print(attn.dtype, v.dtype)\n x = (attn.half() @ v).transpose(1, 2).reshape(B_, N, C)\n x = self.proj(x)\n x = self.proj_drop(x)\n return x\n\nclass Mlp(nn.Module):\n\n def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, drop=0.):\n super().__init__()\n out_features = out_features or in_features\n hidden_features = hidden_features or in_features\n self.fc1 = nn.Linear(in_features, hidden_features)\n self.act = act_layer()\n self.fc2 = nn.Linear(hidden_features, out_features)\n self.drop = nn.Dropout(drop)\n\n def forward(self, x):\n x = self.fc1(x)\n x = self.act(x)\n x = self.drop(x)\n x = self.fc2(x)\n x = self.drop(x)\n return x\n\ndef window_partition(x, window_size):\n\n B, H, W, C = x.shape\n assert H % window_size == 0, 'feature map h and w can not divide by window size'\n x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)\n windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)\n return windows\n\ndef window_reverse(windows, window_size, H, W):\n \n B = int(windows.shape[0] / (H * W / window_size / window_size))\n x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)\n x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)\n return x\n\n\nclass SwinTransformerLayer(nn.Module):\n\n def __init__(self, dim, num_heads, window_size=8, shift_size=0,\n mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0.,\n act_layer=nn.SiLU, norm_layer=nn.LayerNorm):\n super().__init__()\n self.dim = dim\n self.num_heads = num_heads\n self.window_size = window_size\n self.shift_size = shift_size\n self.mlp_ratio = mlp_ratio\n # if min(self.input_resolution) <= self.window_size:\n # # if window size is larger than input resolution, we don't partition windows\n # self.shift_size = 0\n # self.window_size = min(self.input_resolution)\n assert 0 <= self.shift_size < self.window_size, \"shift_size must in 0-window_size\"\n\n self.norm1 = norm_layer(dim)\n self.attn = WindowAttention(\n dim, window_size=(self.window_size, self.window_size), num_heads=num_heads,\n qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)\n\n self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n self.norm2 = norm_layer(dim)\n mlp_hidden_dim = int(dim * mlp_ratio)\n self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)\n\n def create_mask(self, H, W):\n # calculate attention mask for SW-MSA\n img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1\n h_slices = (slice(0, -self.window_size),\n slice(-self.window_size, -self.shift_size),\n slice(-self.shift_size, None))\n w_slices = (slice(0, -self.window_size),\n slice(-self.window_size, -self.shift_size),\n slice(-self.shift_size, None))\n cnt = 0\n for h in h_slices:\n for w in w_slices:\n img_mask[:, h, w, :] = cnt\n cnt += 1\n\n mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1\n mask_windows = mask_windows.view(-1, self.window_size * self.window_size)\n attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)\n attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))\n\n return attn_mask\n\n def forward(self, x):\n # reshape x[b c h w] to x[b l c]\n _, _, H_, W_ = x.shape\n\n Padding = False\n if min(H_, W_) < self.window_size or H_ % self.window_size!=0 or W_ % self.window_size!=0:\n Padding = True\n # print(f'img_size {min(H_, W_)} is less than (or not divided by) window_size {self.window_size}, Padding.')\n pad_r = (self.window_size - W_ % self.window_size) % self.window_size\n pad_b = (self.window_size - H_ % self.window_size) % self.window_size\n x = F.pad(x, (0, pad_r, 0, pad_b))\n\n # print('2', x.shape)\n B, C, H, W = x.shape\n L = H * W\n x = x.permute(0, 2, 3, 1).contiguous().view(B, L, C) # b, L, c\n\n # create mask from init to forward\n if self.shift_size > 0:\n attn_mask = self.create_mask(H, W).to(x.device)\n else:\n attn_mask = None\n\n shortcut = x\n x = self.norm1(x)\n x = x.view(B, H, W, C)\n\n # cyclic shift\n if self.shift_size > 0:\n shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))\n else:\n shifted_x = x\n\n # partition windows\n x_windows = window_partition(shifted_x, self.window_size) # nW*B, window_size, window_size, C\n x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C\n\n # W-MSA/SW-MSA\n attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C\n\n # merge windows\n attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)\n shifted_x = window_reverse(attn_windows, self.window_size, H, W) # B H' W' C\n\n # reverse cyclic shift\n if self.shift_size > 0:\n x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))\n else:\n x = shifted_x\n x = x.view(B, H * W, C)\n\n # FFN\n x = shortcut + self.drop_path(x)\n x = x + self.drop_path(self.mlp(self.norm2(x)))\n\n x = x.permute(0, 2, 1).contiguous().view(-1, C, H, W) # b c h w\n\n if Padding:\n x = x[:, :, :H_, :W_] # reverse padding\n\n return x\n\n\nclass SwinTransformerBlock(nn.Module):\n def __init__(self, c1, c2, num_heads, num_layers, window_size=8):\n super().__init__()\n self.conv = None\n if c1 != c2:\n self.conv = Conv(c1, c2)\n\n # remove input_resolution\n self.blocks = nn.Sequential(*[SwinTransformerLayer(dim=c2, num_heads=num_heads, window_size=window_size,\n shift_size=0 if (i % 2 == 0) else window_size // 2) for i in range(num_layers)])\n\n def forward(self, x):\n if self.conv is not None:\n x = self.conv(x)\n x = self.blocks(x)\n return x\n\n\nclass STCSPA(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(STCSPA, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(2 * c_, c2, 1, 1)\n num_heads = c_ // 32\n self.m = SwinTransformerBlock(c_, c_, num_heads, n)\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.m(self.cv1(x))\n y2 = self.cv2(x)\n return self.cv3(torch.cat((y1, y2), dim=1))\n\n\nclass STCSPB(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(STCSPB, self).__init__()\n c_ = int(c2) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c_, 1, 1)\n self.cv3 = Conv(2 * c_, c2, 1, 1)\n num_heads = c_ // 32\n self.m = SwinTransformerBlock(c_, c_, num_heads, n)\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n x1 = self.cv1(x)\n y1 = self.m(x1)\n y2 = self.cv2(x1)\n return self.cv3(torch.cat((y1, y2), dim=1))\n\n\nclass STCSPC(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(STCSPC, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(c_, c_, 1, 1)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n num_heads = c_ // 32\n self.m = SwinTransformerBlock(c_, c_, num_heads, n)\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(torch.cat((y1, y2), dim=1))\n\n##### end of swin transformer ##### \n\n\n##### swin transformer v2 ##### \n \nclass WindowAttention_v2(nn.Module):\n\n def __init__(self, dim, window_size, num_heads, qkv_bias=True, attn_drop=0., proj_drop=0.,\n pretrained_window_size=[0, 0]):\n\n super().__init__()\n self.dim = dim\n self.window_size = window_size # Wh, Ww\n self.pretrained_window_size = pretrained_window_size\n self.num_heads = num_heads\n\n self.logit_scale = nn.Parameter(torch.log(10 * torch.ones((num_heads, 1, 1))), requires_grad=True)\n\n # mlp to generate continuous relative position bias\n self.cpb_mlp = nn.Sequential(nn.Linear(2, 512, bias=True),\n nn.ReLU(inplace=True),\n nn.Linear(512, num_heads, bias=False))\n\n # get relative_coords_table\n relative_coords_h = torch.arange(-(self.window_size[0] - 1), self.window_size[0], dtype=torch.float32)\n relative_coords_w = torch.arange(-(self.window_size[1] - 1), self.window_size[1], dtype=torch.float32)\n relative_coords_table = torch.stack(\n torch.meshgrid([relative_coords_h,\n relative_coords_w])).permute(1, 2, 0).contiguous().unsqueeze(0) # 1, 2*Wh-1, 2*Ww-1, 2\n if pretrained_window_size[0] > 0:\n relative_coords_table[:, :, :, 0] /= (pretrained_window_size[0] - 1)\n relative_coords_table[:, :, :, 1] /= (pretrained_window_size[1] - 1)\n else:\n relative_coords_table[:, :, :, 0] /= (self.window_size[0] - 1)\n relative_coords_table[:, :, :, 1] /= (self.window_size[1] - 1)\n relative_coords_table *= 8 # normalize to -8, 8\n relative_coords_table = torch.sign(relative_coords_table) * torch.log2(\n torch.abs(relative_coords_table) + 1.0) / np.log2(8)\n\n self.register_buffer(\"relative_coords_table\", relative_coords_table)\n\n # get pair-wise relative position index for each token inside the window\n coords_h = torch.arange(self.window_size[0])\n coords_w = torch.arange(self.window_size[1])\n coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww\n coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww\n relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww\n relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2\n relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0\n relative_coords[:, :, 1] += self.window_size[1] - 1\n relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1\n relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww\n self.register_buffer(\"relative_position_index\", relative_position_index)\n\n self.qkv = nn.Linear(dim, dim * 3, bias=False)\n if qkv_bias:\n self.q_bias = nn.Parameter(torch.zeros(dim))\n self.v_bias = nn.Parameter(torch.zeros(dim))\n else:\n self.q_bias = None\n self.v_bias = None\n self.attn_drop = nn.Dropout(attn_drop)\n self.proj = nn.Linear(dim, dim)\n self.proj_drop = nn.Dropout(proj_drop)\n self.softmax = nn.Softmax(dim=-1)\n\n def forward(self, x, mask=None):\n \n B_, N, C = x.shape\n qkv_bias = None\n if self.q_bias is not None:\n qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias))\n qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias)\n qkv = qkv.reshape(B_, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)\n q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple)\n\n # cosine attention\n attn = (F.normalize(q, dim=-1) @ F.normalize(k, dim=-1).transpose(-2, -1))\n logit_scale = torch.clamp(self.logit_scale, max=torch.log(torch.tensor(1. / 0.01))).exp()\n attn = attn * logit_scale\n\n relative_position_bias_table = self.cpb_mlp(self.relative_coords_table).view(-1, self.num_heads)\n relative_position_bias = relative_position_bias_table[self.relative_position_index.view(-1)].view(\n self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH\n relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww\n relative_position_bias = 16 * torch.sigmoid(relative_position_bias)\n attn = attn + relative_position_bias.unsqueeze(0)\n\n if mask is not None:\n nW = mask.shape[0]\n attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)\n attn = attn.view(-1, self.num_heads, N, N)\n attn = self.softmax(attn)\n else:\n attn = self.softmax(attn)\n\n attn = self.attn_drop(attn)\n\n try:\n x = (attn @ v).transpose(1, 2).reshape(B_, N, C)\n except:\n x = (attn.half() @ v).transpose(1, 2).reshape(B_, N, C)\n \n x = self.proj(x)\n x = self.proj_drop(x)\n return x\n\n def extra_repr(self) -> str:\n return f'dim={self.dim}, window_size={self.window_size}, ' \\\n f'pretrained_window_size={self.pretrained_window_size}, num_heads={self.num_heads}'\n\n def flops(self, N):\n # calculate flops for 1 window with token length of N\n flops = 0\n # qkv = self.qkv(x)\n flops += N * self.dim * 3 * self.dim\n # attn = (q @ k.transpose(-2, -1))\n flops += self.num_heads * N * (self.dim // self.num_heads) * N\n # x = (attn @ v)\n flops += self.num_heads * N * N * (self.dim // self.num_heads)\n # x = self.proj(x)\n flops += N * self.dim * self.dim\n return flops\n \nclass Mlp_v2(nn.Module):\n def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, drop=0.):\n super().__init__()\n out_features = out_features or in_features\n hidden_features = hidden_features or in_features\n self.fc1 = nn.Linear(in_features, hidden_features)\n self.act = act_layer()\n self.fc2 = nn.Linear(hidden_features, out_features)\n self.drop = nn.Dropout(drop)\n\n def forward(self, x):\n x = self.fc1(x)\n x = self.act(x)\n x = self.drop(x)\n x = self.fc2(x)\n x = self.drop(x)\n return x\n\n\ndef window_partition_v2(x, window_size):\n \n B, H, W, C = x.shape\n x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)\n windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)\n return windows\n\n\ndef window_reverse_v2(windows, window_size, H, W):\n \n B = int(windows.shape[0] / (H * W / window_size / window_size))\n x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)\n x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)\n return x\n\n\nclass SwinTransformerLayer_v2(nn.Module):\n\n def __init__(self, dim, num_heads, window_size=7, shift_size=0,\n mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0., drop_path=0.,\n act_layer=nn.SiLU, norm_layer=nn.LayerNorm, pretrained_window_size=0):\n super().__init__()\n self.dim = dim\n #self.input_resolution = input_resolution\n self.num_heads = num_heads\n self.window_size = window_size\n self.shift_size = shift_size\n self.mlp_ratio = mlp_ratio\n #if min(self.input_resolution) <= self.window_size:\n # # if window size is larger than input resolution, we don't partition windows\n # self.shift_size = 0\n # self.window_size = min(self.input_resolution)\n assert 0 <= self.shift_size < self.window_size, \"shift_size must in 0-window_size\"\n\n self.norm1 = norm_layer(dim)\n self.attn = WindowAttention_v2(\n dim, window_size=(self.window_size, self.window_size), num_heads=num_heads,\n qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop,\n pretrained_window_size=(pretrained_window_size, pretrained_window_size))\n\n self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n self.norm2 = norm_layer(dim)\n mlp_hidden_dim = int(dim * mlp_ratio)\n self.mlp = Mlp_v2(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)\n\n def create_mask(self, H, W):\n # calculate attention mask for SW-MSA\n img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1\n h_slices = (slice(0, -self.window_size),\n slice(-self.window_size, -self.shift_size),\n slice(-self.shift_size, None))\n w_slices = (slice(0, -self.window_size),\n slice(-self.window_size, -self.shift_size),\n slice(-self.shift_size, None))\n cnt = 0\n for h in h_slices:\n for w in w_slices:\n img_mask[:, h, w, :] = cnt\n cnt += 1\n\n mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1\n mask_windows = mask_windows.view(-1, self.window_size * self.window_size)\n attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)\n attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))\n\n return attn_mask\n\n def forward(self, x):\n # reshape x[b c h w] to x[b l c]\n _, _, H_, W_ = x.shape\n\n Padding = False\n if min(H_, W_) < self.window_size or H_ % self.window_size!=0 or W_ % self.window_size!=0:\n Padding = True\n # print(f'img_size {min(H_, W_)} is less than (or not divided by) window_size {self.window_size}, Padding.')\n pad_r = (self.window_size - W_ % self.window_size) % self.window_size\n pad_b = (self.window_size - H_ % self.window_size) % self.window_size\n x = F.pad(x, (0, pad_r, 0, pad_b))\n\n # print('2', x.shape)\n B, C, H, W = x.shape\n L = H * W\n x = x.permute(0, 2, 3, 1).contiguous().view(B, L, C) # b, L, c\n\n # create mask from init to forward\n if self.shift_size > 0:\n attn_mask = self.create_mask(H, W).to(x.device)\n else:\n attn_mask = None\n\n shortcut = x\n x = x.view(B, H, W, C)\n\n # cyclic shift\n if self.shift_size > 0:\n shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))\n else:\n shifted_x = x\n\n # partition windows\n x_windows = window_partition_v2(shifted_x, self.window_size) # nW*B, window_size, window_size, C\n x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C\n\n # W-MSA/SW-MSA\n attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C\n\n # merge windows\n attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)\n shifted_x = window_reverse_v2(attn_windows, self.window_size, H, W) # B H' W' C\n\n # reverse cyclic shift\n if self.shift_size > 0:\n x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))\n else:\n x = shifted_x\n x = x.view(B, H * W, C)\n x = shortcut + self.drop_path(self.norm1(x))\n\n # FFN\n x = x + self.drop_path(self.norm2(self.mlp(x)))\n x = x.permute(0, 2, 1).contiguous().view(-1, C, H, W) # b c h w\n \n if Padding:\n x = x[:, :, :H_, :W_] # reverse padding\n\n return x\n\n def extra_repr(self) -> str:\n return f\"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, \" \\\n f\"window_size={self.window_size}, shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}\"\n\n def flops(self):\n flops = 0\n H, W = self.input_resolution\n # norm1\n flops += self.dim * H * W\n # W-MSA/SW-MSA\n nW = H * W / self.window_size / self.window_size\n flops += nW * self.attn.flops(self.window_size * self.window_size)\n # mlp\n flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio\n # norm2\n flops += self.dim * H * W\n return flops\n\n\nclass SwinTransformer2Block(nn.Module):\n def __init__(self, c1, c2, num_heads, num_layers, window_size=7):\n super().__init__()\n self.conv = None\n if c1 != c2:\n self.conv = Conv(c1, c2)\n\n # remove input_resolution\n self.blocks = nn.Sequential(*[SwinTransformerLayer_v2(dim=c2, num_heads=num_heads, window_size=window_size,\n shift_size=0 if (i % 2 == 0) else window_size // 2) for i in range(num_layers)])\n\n def forward(self, x):\n if self.conv is not None:\n x = self.conv(x)\n x = self.blocks(x)\n return x\n\n\nclass ST2CSPA(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(ST2CSPA, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(2 * c_, c2, 1, 1)\n num_heads = c_ // 32\n self.m = SwinTransformer2Block(c_, c_, num_heads, n)\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.m(self.cv1(x))\n y2 = self.cv2(x)\n return self.cv3(torch.cat((y1, y2), dim=1))\n\n\nclass ST2CSPB(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(ST2CSPB, self).__init__()\n c_ = int(c2) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c_, 1, 1)\n self.cv3 = Conv(2 * c_, c2, 1, 1)\n num_heads = c_ // 32\n self.m = SwinTransformer2Block(c_, c_, num_heads, n)\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n x1 = self.cv1(x)\n y1 = self.m(x1)\n y2 = self.cv2(x1)\n return self.cv3(torch.cat((y1, y2), dim=1))\n\n\nclass ST2CSPC(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super(ST2CSPC, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(c_, c_, 1, 1)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n num_heads = c_ // 32\n self.m = SwinTransformer2Block(c_, c_, num_heads, n)\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(torch.cat((y1, y2), dim=1))\n\n##### end of swin transformer v2 ##### \n" }, { "path": "asone/detectors/yolov7/yolov7/models/experimental.py", "content": "import torch\nimport torch.nn as nn\n\nfrom asone.detectors.yolov7.yolov7.models.common import Conv\n\nclass Ensemble(nn.ModuleList):\n # Ensemble of models\n def __init__(self):\n super(Ensemble, self).__init__()\n\n def forward(self, x, augment=False):\n y = []\n for module in self:\n y.append(module(x, augment)[0])\n # y = torch.stack(y).max(0)[0] # max ensemble\n # y = torch.stack(y).mean(0) # mean ensemble\n y = torch.cat(y, 1) # nms ensemble\n return y, None # inference, train output\n\ndef attempt_load(weights, map_location=None):\n # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a\n\n model = Ensemble()\n for w in weights if isinstance(weights, list) else [weights]:\n ckpt = torch.load(w, map_location=map_location) # load\n model.append(ckpt['ema' if ckpt.get('ema') else 'model'].float().fuse().eval()) # FP32 model\n \n # Compatibility updates\n for m in model.modules():\n if type(m) in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]:\n m.inplace = True # pytorch 1.7.0 compatibility\n elif type(m) is nn.Upsample:\n m.recompute_scale_factor = None # torch 1.11.0 compatibility\n elif type(m) is Conv:\n m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatibility\n \n if len(model) == 1:\n return model[-1] # return model\n else:\n print('Ensemble created with %s\\n' % weights)\n for k in ['names', 'stride']:\n setattr(model, k, getattr(model[-1], k))\n return model # return ensemble\n\n\n" }, { "path": "asone/detectors/yolov7/yolov7/models/yolo.py", "content": "from asone.detectors.yolov7.yolov7.utils.torch_utils import time_synchronized, fuse_conv_and_bn, model_info, scale_img, \\\n initialize_weights, select_device, copy_attr\nfrom asone.detectors.yolov7.yolov7.models.experimental import *\nfrom asone.detectors.yolov7.yolov7.models.common import *\nimport torch\nimport argparse\nimport logging\nimport sys\nfrom copy import deepcopy\n\n# sys.path.append('./') # to run '$ python *.py' files in subdirectories\nlogger = logging.getLogger(__name__)\n\n\ntry:\n import thop # for FLOPS computation\nexcept ImportError:\n thop = None\n\n\nclass Detect(nn.Module):\n stride = None # strides computed during build\n export = False # onnx export\n end2end = False\n include_nms = False\n concat = False\n\n def __init__(self, nc=80, anchors=(), ch=()): # detection layer\n super(Detect, self).__init__()\n self.nc = nc # number of classes\n self.no = nc + 5 # number of outputs per anchor\n self.nl = len(anchors) # number of detection layers\n self.na = len(anchors[0]) // 2 # number of anchors\n self.grid = [torch.zeros(1)] * self.nl # init grid\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\n self.register_buffer('anchors', a) # shape(nl,na,2)\n self.register_buffer('anchor_grid', a.clone().view(\n self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\n self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1)\n for x in ch) # output conv\n\n def forward(self, x):\n # x = x.copy() # for profiling\n z = [] # inference output\n self.training |= self.export\n for i in range(self.nl):\n x[i] = self.m[i](x[i]) # conv\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(\n 0, 1, 3, 4, 2).contiguous()\n\n if not self.training: # inference\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\n y = x[i].sigmoid()\n if not torch.onnx.is_in_onnx_export():\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 +\n self.grid[i]) * self.stride[i] # xy\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * \\\n self.anchor_grid[i] # wh\n else:\n # y.tensor_split((2, 4, 5), 4) # torch 1.8.0\n xy, wh, conf = y.split((2, 2, self.nc + 1), 4)\n # new xy\n xy = xy * (2. * self.stride[i]) + \\\n (self.stride[i] * (self.grid[i] - 0.5))\n wh = wh ** 2 * (4 * self.anchor_grid[i].data) # new wh\n y = torch.cat((xy, wh, conf), 4)\n z.append(y.view(bs, -1, self.no))\n\n if self.training:\n out = x\n elif self.end2end:\n out = torch.cat(z, 1)\n elif self.include_nms:\n z = self.convert(z)\n out = (z, )\n elif self.concat:\n out = torch.cat(z, 1)\n else:\n out = (torch.cat(z, 1), x)\n\n return out\n\n @staticmethod\n def _make_grid(nx=20, ny=20):\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\n\n def convert(self, z):\n z = torch.cat(z, 1)\n box = z[:, :, :4]\n conf = z[:, :, 4:5]\n score = z[:, :, 5:]\n score *= conf\n convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],\n dtype=torch.float32,\n device=z.device)\n box @= convert_matrix\n return (box, score)\n\n\nclass IDetect(nn.Module):\n stride = None # strides computed during build\n export = False # onnx export\n end2end = False\n include_nms = False\n concat = False\n\n def __init__(self, nc=80, anchors=(), ch=()): # detection layer\n super(IDetect, self).__init__()\n self.nc = nc # number of classes\n self.no = nc + 5 # number of outputs per anchor\n self.nl = len(anchors) # number of detection layers\n self.na = len(anchors[0]) // 2 # number of anchors\n self.grid = [torch.zeros(1)] * self.nl # init grid\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\n self.register_buffer('anchors', a) # shape(nl,na,2)\n self.register_buffer('anchor_grid', a.clone().view(\n self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\n self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1)\n for x in ch) # output conv\n\n self.ia = nn.ModuleList(ImplicitA(x) for x in ch)\n self.im = nn.ModuleList(ImplicitM(self.no * self.na) for _ in ch)\n\n def forward(self, x):\n # x = x.copy() # for profiling\n z = [] # inference output\n self.training |= self.export\n for i in range(self.nl):\n x[i] = self.m[i](self.ia[i](x[i])) # conv\n x[i] = self.im[i](x[i])\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(\n 0, 1, 3, 4, 2).contiguous()\n\n if not self.training: # inference\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\n\n y = x[i].sigmoid()\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 +\n self.grid[i]) * self.stride[i] # xy\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * \\\n self.anchor_grid[i] # wh\n z.append(y.view(bs, -1, self.no))\n\n return x if self.training else (torch.cat(z, 1), x)\n\n def fuseforward(self, x):\n # x = x.copy() # for profiling\n z = [] # inference output\n self.training |= self.export\n for i in range(self.nl):\n x[i] = self.m[i](x[i]) # conv\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(\n 0, 1, 3, 4, 2).contiguous()\n\n if not self.training: # inference\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\n\n y = x[i].sigmoid()\n if not torch.onnx.is_in_onnx_export():\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 +\n self.grid[i]) * self.stride[i] # xy\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * \\\n self.anchor_grid[i] # wh\n else:\n # y.tensor_split((2, 4, 5), 4) # torch 1.8.0\n xy, wh, conf = y.split((2, 2, self.nc + 1), 4)\n # new xy\n xy = xy * (2. * self.stride[i]) + \\\n (self.stride[i] * (self.grid[i] - 0.5))\n wh = wh ** 2 * (4 * self.anchor_grid[i].data) # new wh\n y = torch.cat((xy, wh, conf), 4)\n z.append(y.view(bs, -1, self.no))\n\n if self.training:\n out = x\n elif self.end2end:\n out = torch.cat(z, 1)\n elif self.include_nms:\n z = self.convert(z)\n out = (z, )\n elif self.concat:\n out = torch.cat(z, 1)\n else:\n out = (torch.cat(z, 1), x)\n\n return out\n\n def fuse(self):\n print(\"IDetect.fuse\")\n # fuse ImplicitA and Convolution\n for i in range(len(self.m)):\n c1, c2, _, _ = self.m[i].weight.shape\n c1_, c2_, _, _ = self.ia[i].implicit.shape\n self.m[i].bias += torch.matmul(self.m[i].weight.reshape(\n c1, c2), self.ia[i].implicit.reshape(c2_, c1_)).squeeze(1)\n\n # fuse ImplicitM and Convolution\n for i in range(len(self.m)):\n c1, c2, _, _ = self.im[i].implicit.shape\n self.m[i].bias *= self.im[i].implicit.reshape(c2)\n self.m[i].weight *= self.im[i].implicit.transpose(0, 1)\n\n @staticmethod\n def _make_grid(nx=20, ny=20):\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\n\n def convert(self, z):\n z = torch.cat(z, 1)\n box = z[:, :, :4]\n conf = z[:, :, 4:5]\n score = z[:, :, 5:]\n score *= conf\n convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],\n dtype=torch.float32,\n device=z.device)\n box @= convert_matrix\n return (box, score)\n\n\nclass IKeypoint(nn.Module):\n stride = None # strides computed during build\n export = False # onnx export\n\n def __init__(self, nc=80, anchors=(), nkpt=17, ch=(), inplace=True, dw_conv_kpt=False): # detection layer\n super(IKeypoint, self).__init__()\n self.nc = nc # number of classes\n self.nkpt = nkpt\n self.dw_conv_kpt = dw_conv_kpt\n # number of outputs per anchor for box and class\n self.no_det = (nc + 5)\n self.no_kpt = 3*self.nkpt # number of outputs per anchor for keypoints\n self.no = self.no_det+self.no_kpt\n self.nl = len(anchors) # number of detection layers\n self.na = len(anchors[0]) // 2 # number of anchors\n self.grid = [torch.zeros(1)] * self.nl # init grid\n self.flip_test = False\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\n self.register_buffer('anchors', a) # shape(nl,na,2)\n self.register_buffer('anchor_grid', a.clone().view(\n self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\n self.m = nn.ModuleList(nn.Conv2d(x, self.no_det * self.na, 1)\n for x in ch) # output conv\n\n self.ia = nn.ModuleList(ImplicitA(x) for x in ch)\n self.im = nn.ModuleList(ImplicitM(self.no_det * self.na) for _ in ch)\n\n if self.nkpt is not None:\n if self.dw_conv_kpt: # keypoint head is slightly more complex\n self.m_kpt = nn.ModuleList(\n nn.Sequential(DWConv(x, x, k=3), Conv(x, x),\n DWConv(x, x, k=3), Conv(x, x),\n DWConv(x, x, k=3), Conv(x, x),\n DWConv(x, x, k=3), Conv(x, x),\n DWConv(x, x, k=3), Conv(x, x),\n DWConv(x, x, k=3), nn.Conv2d(x, self.no_kpt * self.na, 1)) for x in ch)\n else: # keypoint head is a single convolution\n self.m_kpt = nn.ModuleList(\n nn.Conv2d(x, self.no_kpt * self.na, 1) for x in ch)\n\n self.inplace = inplace # use in-place ops (e.g. slice assignment)\n\n def forward(self, x):\n # x = x.copy() # for profiling\n z = [] # inference output\n self.training |= self.export\n for i in range(self.nl):\n if self.nkpt is None or self.nkpt == 0:\n x[i] = self.im[i](self.m[i](self.ia[i](x[i]))) # conv\n else:\n x[i] = torch.cat(\n (self.im[i](self.m[i](self.ia[i](x[i]))), self.m_kpt[i](x[i])), axis=1)\n\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(\n 0, 1, 3, 4, 2).contiguous()\n x_det = x[i][..., :6]\n x_kpt = x[i][..., 6:]\n\n if not self.training: # inference\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\n kpt_grid_x = self.grid[i][..., 0:1]\n kpt_grid_y = self.grid[i][..., 1:2]\n\n if self.nkpt == 0:\n y = x[i].sigmoid()\n else:\n y = x_det.sigmoid()\n\n if self.inplace:\n xy = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * \\\n self.stride[i] # xy\n wh = (y[..., 2:4] * 2) ** 2 * \\\n self.anchor_grid[i].view(1, self.na, 1, 1, 2) # wh\n if self.nkpt != 0:\n x_kpt[..., 0::3] = (\n x_kpt[..., ::3] * 2. - 0.5 + kpt_grid_x.repeat(1, 1, 1, 1, 17)) * self.stride[i] # xy\n x_kpt[..., 1::3] = (\n x_kpt[..., 1::3] * 2. - 0.5 + kpt_grid_y.repeat(1, 1, 1, 1, 17)) * self.stride[i] # xy\n # x_kpt[..., 0::3] = (x_kpt[..., ::3] + kpt_grid_x.repeat(1,1,1,1,17)) * self.stride[i] # xy\n # x_kpt[..., 1::3] = (x_kpt[..., 1::3] + kpt_grid_y.repeat(1,1,1,1,17)) * self.stride[i] # xy\n # print('=============')\n # print(self.anchor_grid[i].shape)\n # print(self.anchor_grid[i][...,0].unsqueeze(4).shape)\n #print(x_kpt[..., 0::3].shape)\n # x_kpt[..., 0::3] = ((x_kpt[..., 0::3].tanh() * 2.) ** 3 * self.anchor_grid[i][...,0].unsqueeze(4).repeat(1,1,1,1,self.nkpt)) + kpt_grid_x.repeat(1,1,1,1,17) * self.stride[i] # xy\n # x_kpt[..., 1::3] = ((x_kpt[..., 1::3].tanh() * 2.) ** 3 * self.anchor_grid[i][...,1].unsqueeze(4).repeat(1,1,1,1,self.nkpt)) + kpt_grid_y.repeat(1,1,1,1,17) * self.stride[i] # xy\n # x_kpt[..., 0::3] = (((x_kpt[..., 0::3].sigmoid() * 4.) ** 2 - 8.) * self.anchor_grid[i][...,0].unsqueeze(4).repeat(1,1,1,1,self.nkpt)) + kpt_grid_x.repeat(1,1,1,1,17) * self.stride[i] # xy\n # x_kpt[..., 1::3] = (((x_kpt[..., 1::3].sigmoid() * 4.) ** 2 - 8.) * self.anchor_grid[i][...,1].unsqueeze(4).repeat(1,1,1,1,self.nkpt)) + kpt_grid_y.repeat(1,1,1,1,17) * self.stride[i] # xy\n x_kpt[..., 2::3] = x_kpt[..., 2::3].sigmoid()\n\n y = torch.cat((xy, wh, y[..., 4:], x_kpt), dim=-1)\n\n else: # for YOLOv5 on AWS Inferentia https://github.com/ultralytics/yolov5/pull/2953\n xy = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * \\\n self.stride[i] # xy\n wh = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\n if self.nkpt != 0:\n y[..., 6:] = (y[..., 6:] * 2. - 0.5 + self.grid[i].repeat(\n (1, 1, 1, 1, self.nkpt))) * self.stride[i] # xy\n y = torch.cat((xy, wh, y[..., 4:]), -1)\n\n z.append(y.view(bs, -1, self.no))\n\n return x if self.training else (torch.cat(z, 1), x)\n\n @staticmethod\n def _make_grid(nx=20, ny=20):\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\n\n\nclass IAuxDetect(nn.Module):\n stride = None # strides computed during build\n export = False # onnx export\n end2end = False\n include_nms = False\n concat = False\n\n def __init__(self, nc=80, anchors=(), ch=()): # detection layer\n super(IAuxDetect, self).__init__()\n self.nc = nc # number of classes\n self.no = nc + 5 # number of outputs per anchor\n self.nl = len(anchors) # number of detection layers\n self.na = len(anchors[0]) // 2 # number of anchors\n self.grid = [torch.zeros(1)] * self.nl # init grid\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\n self.register_buffer('anchors', a) # shape(nl,na,2)\n self.register_buffer('anchor_grid', a.clone().view(\n self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\n self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1)\n for x in ch[:self.nl]) # output conv\n self.m2 = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1)\n for x in ch[self.nl:]) # output conv\n\n self.ia = nn.ModuleList(ImplicitA(x) for x in ch[:self.nl])\n self.im = nn.ModuleList(ImplicitM(self.no * self.na)\n for _ in ch[:self.nl])\n\n def forward(self, x):\n # x = x.copy() # for profiling\n z = [] # inference output\n self.training |= self.export\n for i in range(self.nl):\n x[i] = self.m[i](self.ia[i](x[i])) # conv\n x[i] = self.im[i](x[i])\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(\n 0, 1, 3, 4, 2).contiguous()\n\n x[i+self.nl] = self.m2[i](x[i+self.nl])\n x[i+self.nl] = x[i+self.nl].view(bs, self.na, self.no,\n ny, nx).permute(0, 1, 3, 4, 2).contiguous()\n\n if not self.training: # inference\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\n\n y = x[i].sigmoid()\n if not torch.onnx.is_in_onnx_export():\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 +\n self.grid[i]) * self.stride[i] # xy\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * \\\n self.anchor_grid[i] # wh\n else:\n # y.tensor_split((2, 4, 5), 4) # torch 1.8.0\n xy, wh, conf = y.split((2, 2, self.nc + 1), 4)\n # new xy\n xy = xy * (2. * self.stride[i]) + \\\n (self.stride[i] * (self.grid[i] - 0.5))\n wh = wh ** 2 * (4 * self.anchor_grid[i].data) # new wh\n y = torch.cat((xy, wh, conf), 4)\n z.append(y.view(bs, -1, self.no))\n\n return x if self.training else (torch.cat(z, 1), x[:self.nl])\n\n def fuseforward(self, x):\n # x = x.copy() # for profiling\n z = [] # inference output\n self.training |= self.export\n for i in range(self.nl):\n x[i] = self.m[i](x[i]) # conv\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(\n 0, 1, 3, 4, 2).contiguous()\n\n if not self.training: # inference\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\n\n y = x[i].sigmoid()\n if not torch.onnx.is_in_onnx_export():\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 +\n self.grid[i]) * self.stride[i] # xy\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * \\\n self.anchor_grid[i] # wh\n else:\n xy = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * \\\n self.stride[i] # xy\n wh = (y[..., 2:4] * 2) ** 2 * \\\n self.anchor_grid[i].data # wh\n y = torch.cat((xy, wh, y[..., 4:]), -1)\n z.append(y.view(bs, -1, self.no))\n\n if self.training:\n out = x\n elif self.end2end:\n out = torch.cat(z, 1)\n elif self.include_nms:\n z = self.convert(z)\n out = (z, )\n elif self.concat:\n out = torch.cat(z, 1)\n else:\n out = (torch.cat(z, 1), x)\n\n return out\n\n def fuse(self):\n print(\"IAuxDetect.fuse\")\n # fuse ImplicitA and Convolution\n for i in range(len(self.m)):\n c1, c2, _, _ = self.m[i].weight.shape\n c1_, c2_, _, _ = self.ia[i].implicit.shape\n self.m[i].bias += torch.matmul(self.m[i].weight.reshape(\n c1, c2), self.ia[i].implicit.reshape(c2_, c1_)).squeeze(1)\n\n # fuse ImplicitM and Convolution\n for i in range(len(self.m)):\n c1, c2, _, _ = self.im[i].implicit.shape\n self.m[i].bias *= self.im[i].implicit.reshape(c2)\n self.m[i].weight *= self.im[i].implicit.transpose(0, 1)\n\n @staticmethod\n def _make_grid(nx=20, ny=20):\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\n\n def convert(self, z):\n z = torch.cat(z, 1)\n box = z[:, :, :4]\n conf = z[:, :, 4:5]\n score = z[:, :, 5:]\n score *= conf\n convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],\n dtype=torch.float32,\n device=z.device)\n box @= convert_matrix\n return (box, score)\n\n\nclass IBin(nn.Module):\n stride = None # strides computed during build\n export = False # onnx export\n\n def __init__(self, nc=80, anchors=(), ch=(), bin_count=21): # detection layer\n super(IBin, self).__init__()\n self.nc = nc # number of classes\n self.bin_count = bin_count\n\n self.w_bin_sigmoid = SigmoidBin(\n bin_count=self.bin_count, min=0.0, max=4.0)\n self.h_bin_sigmoid = SigmoidBin(\n bin_count=self.bin_count, min=0.0, max=4.0)\n # classes, x,y,obj\n self.no = nc + 3 + \\\n self.w_bin_sigmoid.get_length() + self.h_bin_sigmoid.get_length() # w-bce, h-bce\n # + self.x_bin_sigmoid.get_length() + self.y_bin_sigmoid.get_length()\n\n self.nl = len(anchors) # number of detection layers\n self.na = len(anchors[0]) // 2 # number of anchors\n self.grid = [torch.zeros(1)] * self.nl # init grid\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\n self.register_buffer('anchors', a) # shape(nl,na,2)\n self.register_buffer('anchor_grid', a.clone().view(\n self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\n self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1)\n for x in ch) # output conv\n\n self.ia = nn.ModuleList(ImplicitA(x) for x in ch)\n self.im = nn.ModuleList(ImplicitM(self.no * self.na) for _ in ch)\n\n def forward(self, x):\n\n #self.x_bin_sigmoid.use_fw_regression = True\n #self.y_bin_sigmoid.use_fw_regression = True\n self.w_bin_sigmoid.use_fw_regression = True\n self.h_bin_sigmoid.use_fw_regression = True\n\n # x = x.copy() # for profiling\n z = [] # inference output\n self.training |= self.export\n for i in range(self.nl):\n x[i] = self.m[i](self.ia[i](x[i])) # conv\n x[i] = self.im[i](x[i])\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(\n 0, 1, 3, 4, 2).contiguous()\n\n if not self.training: # inference\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\n\n y = x[i].sigmoid()\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 +\n self.grid[i]) * self.stride[i] # xy\n # y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\n\n #px = (self.x_bin_sigmoid.forward(y[..., 0:12]) + self.grid[i][..., 0]) * self.stride[i]\n #py = (self.y_bin_sigmoid.forward(y[..., 12:24]) + self.grid[i][..., 1]) * self.stride[i]\n\n pw = self.w_bin_sigmoid.forward(\n y[..., 2:24]) * self.anchor_grid[i][..., 0]\n ph = self.h_bin_sigmoid.forward(\n y[..., 24:46]) * self.anchor_grid[i][..., 1]\n\n #y[..., 0] = px\n #y[..., 1] = py\n y[..., 2] = pw\n y[..., 3] = ph\n\n y = torch.cat((y[..., 0:4], y[..., 46:]), dim=-1)\n\n z.append(y.view(bs, -1, y.shape[-1]))\n\n return x if self.training else (torch.cat(z, 1), x)\n\n @staticmethod\n def _make_grid(nx=20, ny=20):\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\n\n\nclass Model(nn.Module):\n # model, input channels, number of classes\n def __init__(self, cfg='yolor-csp-c.yaml', ch=3, nc=None, anchors=None):\n super(Model, self).__init__()\n self.traced = False\n if isinstance(cfg, dict):\n self.yaml = cfg # model dict\n else: # is *.yaml\n import yaml # for torch hub\n self.yaml_file = Path(cfg).name\n with open(cfg) as f:\n self.yaml = yaml.load(f, Loader=yaml.SafeLoader) # model dict\n\n # Define model\n ch = self.yaml['ch'] = self.yaml.get('ch', ch) # input channels\n if nc and nc != self.yaml['nc']:\n logger.info(\n f\"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}\")\n self.yaml['nc'] = nc # override yaml value\n if anchors:\n logger.info(\n f'Overriding model.yaml anchors with anchors={anchors}')\n self.yaml['anchors'] = round(anchors) # override yaml value\n self.model, self.save = parse_model(\n deepcopy(self.yaml), ch=[ch]) # model, savelist\n self.names = [str(i) for i in range(self.yaml['nc'])] # default names\n # print([x.shape for x in self.forward(torch.zeros(1, ch, 64, 64))])\n\n # Build strides, anchors\n m = self.model[-1] # Detect()\n if isinstance(m, Detect):\n s = 256 # 2x min stride\n m.stride = torch.tensor(\n [s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward\n check_anchor_order(m)\n m.anchors /= m.stride.view(-1, 1, 1)\n self.stride = m.stride\n self._initialize_biases() # only run once\n # print('Strides: %s' % m.stride.tolist())\n if isinstance(m, IDetect):\n s = 256 # 2x min stride\n m.stride = torch.tensor(\n [s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward\n check_anchor_order(m)\n m.anchors /= m.stride.view(-1, 1, 1)\n self.stride = m.stride\n self._initialize_biases() # only run once\n # print('Strides: %s' % m.stride.tolist())\n if isinstance(m, IAuxDetect):\n s = 256 # 2x min stride\n m.stride = torch.tensor(\n [s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))[:4]]) # forward\n # print(m.stride)\n check_anchor_order(m)\n m.anchors /= m.stride.view(-1, 1, 1)\n self.stride = m.stride\n self._initialize_aux_biases() # only run once\n # print('Strides: %s' % m.stride.tolist())\n if isinstance(m, IBin):\n s = 256 # 2x min stride\n m.stride = torch.tensor(\n [s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward\n check_anchor_order(m)\n m.anchors /= m.stride.view(-1, 1, 1)\n self.stride = m.stride\n self._initialize_biases_bin() # only run once\n # print('Strides: %s' % m.stride.tolist())\n if isinstance(m, IKeypoint):\n s = 256 # 2x min stride\n m.stride = torch.tensor(\n [s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward\n check_anchor_order(m)\n m.anchors /= m.stride.view(-1, 1, 1)\n self.stride = m.stride\n self._initialize_biases_kpt() # only run once\n # print('Strides: %s' % m.stride.tolist())\n\n # Init weights, biases\n initialize_weights(self)\n self.info()\n logger.info('')\n\n def forward(self, x, augment=False, profile=False):\n if augment:\n img_size = x.shape[-2:] # height, width\n s = [1, 0.83, 0.67] # scales\n f = [None, 3, None] # flips (2-ud, 3-lr)\n y = [] # outputs\n for si, fi in zip(s, f):\n xi = scale_img(x.flip(fi) if fi else x, si,\n gs=int(self.stride.max()))\n yi = self.forward_once(xi)[0] # forward\n # cv2.imwrite(f'img_{si}.jpg', 255 * xi[0].cpu().numpy().transpose((1, 2, 0))[:, :, ::-1]) # save\n yi[..., :4] /= si # de-scale\n if fi == 2:\n yi[..., 1] = img_size[0] - yi[..., 1] # de-flip ud\n elif fi == 3:\n yi[..., 0] = img_size[1] - yi[..., 0] # de-flip lr\n y.append(yi)\n return torch.cat(y, 1), None # augmented inference, train\n else:\n # single-scale inference, train\n return self.forward_once(x, profile)\n\n def forward_once(self, x, profile=False):\n y, dt = [], [] # outputs\n for m in self.model:\n if m.f != -1: # if not from previous layer\n x = y[m.f] if isinstance(m.f, int) else [\n x if j == -1 else y[j] for j in m.f] # from earlier layers\n\n if not hasattr(self, 'traced'):\n self.traced = False\n\n if self.traced:\n if isinstance(m, Detect) or isinstance(m, IDetect) or isinstance(m, IAuxDetect) or isinstance(m, IKeypoint):\n break\n\n if profile:\n c = isinstance(m, (Detect, IDetect, IAuxDetect, IBin))\n o = thop.profile(m, inputs=(x.copy() if c else x,), verbose=False)[\n 0] / 1E9 * 2 if thop else 0 # FLOPS\n for _ in range(10):\n m(x.copy() if c else x)\n t = time_synchronized()\n for _ in range(10):\n m(x.copy() if c else x)\n dt.append((time_synchronized() - t) * 100)\n print('%10.1f%10.0f%10.1fms %-40s' % (o, m.np, dt[-1], m.type))\n\n x = m(x) # run\n\n y.append(x if m.i in self.save else None) # save output\n\n if profile:\n print('%.1fms total' % sum(dt))\n return x\n\n # initialize biases into Detect(), cf is class frequency\n def _initialize_biases(self, cf=None):\n # https://arxiv.org/abs/1708.02002 section 3.3\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\n m = self.model[-1] # Detect() module\n for mi, s in zip(m.m, m.stride): # from\n b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85)\n # obj (8 objects per 640 image)\n b.data[:, 4] += math.log(8 / (640 / s) ** 2)\n b.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)\n ) if cf is None else torch.log(cf / cf.sum()) # cls\n mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n\n # initialize biases into Detect(), cf is class frequency\n def _initialize_aux_biases(self, cf=None):\n # https://arxiv.org/abs/1708.02002 section 3.3\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\n m = self.model[-1] # Detect() module\n for mi, mi2, s in zip(m.m, m.m2, m.stride): # from\n b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85)\n # obj (8 objects per 640 image)\n b.data[:, 4] += math.log(8 / (640 / s) ** 2)\n b.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)\n ) if cf is None else torch.log(cf / cf.sum()) # cls\n mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n b2 = mi2.bias.view(m.na, -1) # conv.bias(255) to (3,85)\n # obj (8 objects per 640 image)\n b2.data[:, 4] += math.log(8 / (640 / s) ** 2)\n b2.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)\n ) if cf is None else torch.log(cf / cf.sum()) # cls\n mi2.bias = torch.nn.Parameter(b2.view(-1), requires_grad=True)\n\n # initialize biases into Detect(), cf is class frequency\n def _initialize_biases_bin(self, cf=None):\n # https://arxiv.org/abs/1708.02002 section 3.3\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\n m = self.model[-1] # Bin() module\n bc = m.bin_count\n for mi, s in zip(m.m, m.stride): # from\n b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85)\n old = b[:, (0, 1, 2, bc+3)].data\n obj_idx = 2*bc+4\n b[:, :obj_idx].data += math.log(0.6 / (bc + 1 - 0.99))\n # obj (8 objects per 640 image)\n b[:, obj_idx].data += math.log(8 / (640 / s) ** 2)\n b[:, (obj_idx+1):].data += math.log(0.6 / (m.nc - 0.99)\n ) if cf is None else torch.log(cf / cf.sum()) # cls\n b[:, (0, 1, 2, bc+3)].data = old\n mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n\n # initialize biases into Detect(), cf is class frequency\n def _initialize_biases_kpt(self, cf=None):\n # https://arxiv.org/abs/1708.02002 section 3.3\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\n m = self.model[-1] # Detect() module\n for mi, s in zip(m.m, m.stride): # from\n b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85)\n # obj (8 objects per 640 image)\n b.data[:, 4] += math.log(8 / (640 / s) ** 2)\n b.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)\n ) if cf is None else torch.log(cf / cf.sum()) # cls\n mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n\n def _print_biases(self):\n m = self.model[-1] # Detect() module\n for mi in m.m: # from\n b = mi.bias.detach().view(m.na, -1).T # conv.bias(255) to (3,85)\n print(('%6g Conv2d.bias:' + '%10.3g' * 6) %\n (mi.weight.shape[1], *b[:5].mean(1).tolist(), b[5:].mean()))\n\n # def _print_weights(self):\n # for m in self.model.modules():\n # if type(m) is Bottleneck:\n # print('%10.3g' % (m.w.detach().sigmoid() * 2)) # shortcut weights\n\n def fuse(self): # fuse model Conv2d() + BatchNorm2d() layers\n print('Fusing layers... ')\n for m in self.model.modules():\n if isinstance(m, RepConv):\n #print(f\" fuse_repvgg_block\")\n m.fuse_repvgg_block()\n elif isinstance(m, RepConv_OREPA):\n #print(f\" switch_to_deploy\")\n m.switch_to_deploy()\n elif type(m) is Conv and hasattr(m, 'bn'):\n m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv\n delattr(m, 'bn') # remove batchnorm\n m.forward = m.fuseforward # update forward\n elif isinstance(m, (IDetect, IAuxDetect)):\n m.fuse()\n m.forward = m.fuseforward\n self.info()\n return self\n\n def nms(self, mode=True): # add or remove NMS module\n present = type(self.model[-1]) is NMS # last layer is NMS\n if mode and not present:\n print('Adding NMS... ')\n m = NMS() # module\n m.f = -1 # from\n m.i = self.model[-1].i + 1 # index\n self.model.add_module(name='%s' % m.i, module=m) # add\n self.eval()\n elif not mode and present:\n print('Removing NMS... ')\n self.model = self.model[:-1] # remove\n return self\n\n def autoshape(self): # add autoShape module\n print('Adding autoShape... ')\n m = autoShape(self) # wrap model\n copy_attr(m, self, include=('yaml', 'nc', 'hyp', 'names',\n 'stride'), exclude=()) # copy attributes\n return m\n\n def info(self, verbose=False, img_size=640): # print model information\n model_info(self, verbose, img_size)\n\n\ndef parse_model(d, ch): # model_dict, input_channels(3)\n logger.info('\\n%3s%18s%3s%10s %-40s%-30s' %\n ('', 'from', 'n', 'params', 'module', 'arguments'))\n anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple']\n na = (len(anchors[0]) // 2) if isinstance(anchors,\n list) else anchors # number of anchors\n no = na * (nc + 5) # number of outputs = anchors * (classes + 5)\n\n layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out\n # from, number, module, args\n for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']):\n m = eval(m) if isinstance(m, str) else m # eval strings\n for j, a in enumerate(args):\n try:\n args[j] = eval(a) if isinstance(a, str) else a # eval strings\n except:\n pass\n\n n = max(round(n * gd), 1) if n > 1 else n # depth gain\n if m in [nn.Conv2d, Conv, RobustConv, RobustConv2, DWConv, GhostConv, RepConv, RepConv_OREPA, DownC,\n SPP, SPPF, SPPCSPC, GhostSPPCSPC, MixConv2d, Focus, Stem, GhostStem, CrossConv,\n Bottleneck, BottleneckCSPA, BottleneckCSPB, BottleneckCSPC,\n RepBottleneck, RepBottleneckCSPA, RepBottleneckCSPB, RepBottleneckCSPC,\n Res, ResCSPA, ResCSPB, ResCSPC,\n RepRes, RepResCSPA, RepResCSPB, RepResCSPC,\n ResX, ResXCSPA, ResXCSPB, ResXCSPC,\n RepResX, RepResXCSPA, RepResXCSPB, RepResXCSPC,\n Ghost, GhostCSPA, GhostCSPB, GhostCSPC,\n SwinTransformerBlock, STCSPA, STCSPB, STCSPC,\n SwinTransformer2Block, ST2CSPA, ST2CSPB, ST2CSPC]:\n c1, c2 = ch[f], args[0]\n if c2 != no: # if not output\n c2 = make_divisible(c2 * gw, 8)\n\n args = [c1, c2, *args[1:]]\n if m in [DownC, SPPCSPC, GhostSPPCSPC,\n BottleneckCSPA, BottleneckCSPB, BottleneckCSPC,\n RepBottleneckCSPA, RepBottleneckCSPB, RepBottleneckCSPC,\n ResCSPA, ResCSPB, ResCSPC,\n RepResCSPA, RepResCSPB, RepResCSPC,\n ResXCSPA, ResXCSPB, ResXCSPC,\n RepResXCSPA, RepResXCSPB, RepResXCSPC,\n GhostCSPA, GhostCSPB, GhostCSPC,\n STCSPA, STCSPB, STCSPC,\n ST2CSPA, ST2CSPB, ST2CSPC]:\n args.insert(2, n) # number of repeats\n n = 1\n elif m is nn.BatchNorm2d:\n args = [ch[f]]\n elif m is Concat:\n c2 = sum([ch[x] for x in f])\n elif m is Chuncat:\n c2 = sum([ch[x] for x in f])\n elif m is Shortcut:\n c2 = ch[f[0]]\n elif m is Foldcut:\n c2 = ch[f] // 2\n elif m in [Detect, IDetect, IAuxDetect, IBin, IKeypoint]:\n args.append([ch[x] for x in f])\n if isinstance(args[1], int): # number of anchors\n args[1] = [list(range(args[1] * 2))] * len(f)\n elif m is ReOrg:\n c2 = ch[f] * 4\n elif m is Contract:\n c2 = ch[f] * args[0] ** 2\n elif m is Expand:\n c2 = ch[f] // args[0] ** 2\n else:\n c2 = ch[f]\n\n m_ = nn.Sequential(*[m(*args) for _ in range(n)]\n ) if n > 1 else m(*args) # module\n t = str(m)[8:-2].replace('__main__.', '') # module type\n np = sum([x.numel() for x in m_.parameters()]) # number params\n # attach index, 'from' index, type, number params\n m_.i, m_.f, m_.type, m_.np = i, f, t, np\n logger.info('%3s%18s%3s%10.0f %-40s%-30s' %\n (i, f, n, np, t, args)) # print\n save.extend(x % i for x in ([f] if isinstance(\n f, int) else f) if x != -1) # append to savelist\n layers.append(m_)\n if i == 0:\n ch = []\n ch.append(c2)\n return nn.Sequential(*layers), sorted(save)\n\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument('--cfg', type=str,\n default='yolor-csp-c.yaml', help='model.yaml')\n parser.add_argument('--device', default='',\n help='cuda device, i.e. 0 or 0,1,2,3 or cpu')\n parser.add_argument('--profile', action='store_true',\n help='profile model speed')\n opt = parser.parse_args()\n opt.cfg = check_file(opt.cfg) # check file\n set_logging()\n device = select_device(opt.device)\n\n # Create model\n model = Model(opt.cfg).to(device)\n model.train()\n\n if opt.profile:\n img = torch.rand(1, 3, 640, 640).to(device)\n y = model(img, profile=True)\n\n # Profile\n # img = torch.rand(8 if torch.cuda.is_available() else 1, 3, 640, 640).to(device)\n # y = model(img, profile=True)\n\n # Tensorboard\n # from torch.utils.tensorboard import SummaryWriter\n # tb_writer = SummaryWriter()\n # print(\"Run 'tensorboard --logdir=models/runs' to view tensorboard at http://localhost:6006/\")\n # tb_writer.add_graph(model.model, img) # add model to tensorboard\n # tb_writer.add_image('test', img[0], dataformats='CWH') # add model to tensorboard\n" }, { "path": "asone/detectors/yolov7/yolov7/utils/__init__.py", "content": "" }, { "path": "asone/detectors/yolov7/yolov7/utils/torch_utils.py", "content": "# YOLOR PyTorch utils\n\nimport datetime\nimport logging\nimport math\nimport os\nimport platform\nimport subprocess\nimport time\nfrom contextlib import contextmanager\nfrom copy import deepcopy\nfrom pathlib import Path\n\nimport torch\nimport torch.backends.cudnn as cudnn\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\n\ntry:\n import thop # for FLOPS computation\nexcept ImportError:\n thop = None\nlogger = logging.getLogger(__name__)\n\n\n@contextmanager\ndef torch_distributed_zero_first(local_rank: int):\n \"\"\"\n Decorator to make all processes in distributed training wait for each local_master to do something.\n \"\"\"\n if local_rank not in [-1, 0]:\n torch.distributed.barrier()\n yield\n if local_rank == 0:\n torch.distributed.barrier()\n\n\ndef init_torch_seeds(seed=0):\n # Speed-reproducibility tradeoff https://pytorch.org/docs/stable/notes/randomness.html\n torch.manual_seed(seed)\n if seed == 0: # slower, more reproducible\n cudnn.benchmark, cudnn.deterministic = False, True\n else: # faster, less reproducible\n cudnn.benchmark, cudnn.deterministic = True, False\n\n\ndef date_modified(path=__file__):\n # return human-readable file modification date, i.e. '2021-3-26'\n t = datetime.datetime.fromtimestamp(Path(path).stat().st_mtime)\n return f'{t.year}-{t.month}-{t.day}'\n\n\ndef git_describe(path=Path(__file__).parent): # path must be a directory\n # return human-readable git description, i.e. v5.0-5-g3e25f1e https://git-scm.com/docs/git-describe\n s = f'git -C {path} describe --tags --long --always'\n try:\n return subprocess.check_output(s, shell=True, stderr=subprocess.STDOUT).decode()[:-1]\n except subprocess.CalledProcessError as e:\n return '' # not a git repository\n\n\ndef select_device(device='', batch_size=None):\n # device = 'cpu' or '0' or '0,1,2,3'\n s = f'YOLOR 🚀 {git_describe() or date_modified()} torch {torch.__version__} ' # string\n cpu = device.lower() == 'cpu'\n if cpu:\n os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # force torch.cuda.is_available() = False\n elif device: # non-cpu device requested\n os.environ['CUDA_VISIBLE_DEVICES'] = device # set environment variable\n assert torch.cuda.is_available(), f'CUDA unavailable, invalid device {device} requested' # check availability\n\n cuda = not cpu and torch.cuda.is_available()\n if cuda:\n n = torch.cuda.device_count()\n if n > 1 and batch_size: # check that batch_size is compatible with device_count\n assert batch_size % n == 0, f'batch-size {batch_size} not multiple of GPU count {n}'\n space = ' ' * len(s)\n for i, d in enumerate(device.split(',') if device else range(n)):\n p = torch.cuda.get_device_properties(i)\n s += f\"{'' if i == 0 else space}CUDA:{d} ({p.name}, {p.total_memory / 1024 ** 2}MB)\\n\" # bytes to MB\n else:\n s += 'CPU\\n'\n\n logger.info(s.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else s) # emoji-safe\n return torch.device('cuda:0' if cuda else 'cpu')\n\n\ndef time_synchronized():\n # pytorch-accurate time\n if torch.cuda.is_available():\n torch.cuda.synchronize()\n return time.time()\n\n\ndef profile(x, ops, n=100, device=None):\n # profile a pytorch module or list of modules. Example usage:\n # x = torch.randn(16, 3, 640, 640) # input\n # m1 = lambda x: x * torch.sigmoid(x)\n # m2 = nn.SiLU()\n # profile(x, [m1, m2], n=100) # profile speed over 100 iterations\n\n device = device or torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n x = x.to(device)\n x.requires_grad = True\n print(torch.__version__, device.type, torch.cuda.get_device_properties(0) if device.type == 'cuda' else '')\n print(f\"\\n{'Params':>12s}{'GFLOPS':>12s}{'forward (ms)':>16s}{'backward (ms)':>16s}{'input':>24s}{'output':>24s}\")\n for m in ops if isinstance(ops, list) else [ops]:\n m = m.to(device) if hasattr(m, 'to') else m # device\n m = m.half() if hasattr(m, 'half') and isinstance(x, torch.Tensor) and x.dtype is torch.float16 else m # type\n dtf, dtb, t = 0., 0., [0., 0., 0.] # dt forward, backward\n try:\n flops = thop.profile(m, inputs=(x,), verbose=False)[0] / 1E9 * 2 # GFLOPS\n except:\n flops = 0\n\n for _ in range(n):\n t[0] = time_synchronized()\n y = m(x)\n t[1] = time_synchronized()\n try:\n _ = y.sum().backward()\n t[2] = time_synchronized()\n except: # no backward method\n t[2] = float('nan')\n dtf += (t[1] - t[0]) * 1000 / n # ms per op forward\n dtb += (t[2] - t[1]) * 1000 / n # ms per op backward\n\n s_in = tuple(x.shape) if isinstance(x, torch.Tensor) else 'list'\n s_out = tuple(y.shape) if isinstance(y, torch.Tensor) else 'list'\n p = sum(list(x.numel() for x in m.parameters())) if isinstance(m, nn.Module) else 0 # parameters\n print(f'{p:12}{flops:12.4g}{dtf:16.4g}{dtb:16.4g}{str(s_in):>24s}{str(s_out):>24s}')\n\n\ndef is_parallel(model):\n return type(model) in (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)\n\n\ndef intersect_dicts(da, db, exclude=()):\n # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values\n return {k: v for k, v in da.items() if k in db and not any(x in k for x in exclude) and v.shape == db[k].shape}\n\n\ndef initialize_weights(model):\n for m in model.modules():\n t = type(m)\n if t is nn.Conv2d:\n pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n elif t is nn.BatchNorm2d:\n m.eps = 1e-3\n m.momentum = 0.03\n elif t in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6]:\n m.inplace = True\n\n\ndef find_modules(model, mclass=nn.Conv2d):\n # Finds layer indices matching module class 'mclass'\n return [i for i, m in enumerate(model.module_list) if isinstance(m, mclass)]\n\n\ndef sparsity(model):\n # Return global model sparsity\n a, b = 0., 0.\n for p in model.parameters():\n a += p.numel()\n b += (p == 0).sum()\n return b / a\n\n\ndef prune(model, amount=0.3):\n # Prune model to requested global sparsity\n import torch.nn.utils.prune as prune\n print('Pruning model... ', end='')\n for name, m in model.named_modules():\n if isinstance(m, nn.Conv2d):\n prune.l1_unstructured(m, name='weight', amount=amount) # prune\n prune.remove(m, 'weight') # make permanent\n print(' %.3g global sparsity' % sparsity(model))\n\n\ndef fuse_conv_and_bn(conv, bn):\n # Fuse convolution and batchnorm layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/\n fusedconv = nn.Conv2d(conv.in_channels,\n conv.out_channels,\n kernel_size=conv.kernel_size,\n stride=conv.stride,\n padding=conv.padding,\n groups=conv.groups,\n bias=True).requires_grad_(False).to(conv.weight.device)\n\n # prepare filters\n w_conv = conv.weight.clone().view(conv.out_channels, -1)\n w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))\n fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape))\n\n # prepare spatial bias\n b_conv = torch.zeros(conv.weight.size(0), device=conv.weight.device) if conv.bias is None else conv.bias\n b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps))\n fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)\n\n return fusedconv\n\n\ndef model_info(model, verbose=False, img_size=640):\n # Model information. img_size may be int or list, i.e. img_size=640 or img_size=[640, 320]\n n_p = sum(x.numel() for x in model.parameters()) # number parameters\n n_g = sum(x.numel() for x in model.parameters() if x.requires_grad) # number gradients\n if verbose:\n print('%5s %40s %9s %12s %20s %10s %10s' % ('layer', 'name', 'gradient', 'parameters', 'shape', 'mu', 'sigma'))\n for i, (name, p) in enumerate(model.named_parameters()):\n name = name.replace('module_list.', '')\n print('%5g %40s %9s %12g %20s %10.3g %10.3g' %\n (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std()))\n\n try: # FLOPS\n from thop import profile\n stride = max(int(model.stride.max()), 32) if hasattr(model, 'stride') else 32\n img = torch.zeros((1, model.yaml.get('ch', 3), stride, stride), device=next(model.parameters()).device) # input\n flops = profile(deepcopy(model), inputs=(img,), verbose=False)[0] / 1E9 * 2 # stride GFLOPS\n img_size = img_size if isinstance(img_size, list) else [img_size, img_size] # expand if int/float\n fs = ', %.1f GFLOPS' % (flops * img_size[0] / stride * img_size[1] / stride) # 640x640 GFLOPS\n except (ImportError, Exception):\n fs = ''\n\n logger.info(f\"Model Summary: {len(list(model.modules()))} layers, {n_p} parameters, {n_g} gradients{fs}\")\n\n\ndef load_classifier(name='resnet101', n=2):\n # Loads a pretrained model reshaped to n-class output\n model = torchvision.models.__dict__[name](pretrained=True)\n\n # ResNet model properties\n # input_size = [3, 224, 224]\n # input_space = 'RGB'\n # input_range = [0, 1]\n # mean = [0.485, 0.456, 0.406]\n # std = [0.229, 0.224, 0.225]\n\n # Reshape output to n classes\n filters = model.fc.weight.shape[1]\n model.fc.bias = nn.Parameter(torch.zeros(n), requires_grad=True)\n model.fc.weight = nn.Parameter(torch.zeros(n, filters), requires_grad=True)\n model.fc.out_features = n\n return model\n\n\ndef scale_img(img, ratio=1.0, same_shape=False, gs=32): # img(16,3,256,416)\n # scales img(bs,3,y,x) by ratio constrained to gs-multiple\n if ratio == 1.0:\n return img\n else:\n h, w = img.shape[2:]\n s = (int(h * ratio), int(w * ratio)) # new size\n img = F.interpolate(img, size=s, mode='bilinear', align_corners=False) # resize\n if not same_shape: # pad/crop img\n h, w = [math.ceil(x * ratio / gs) * gs for x in (h, w)]\n return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean\n\n\ndef copy_attr(a, b, include=(), exclude=()):\n # Copy attributes from b to a, options to only include [...] and to exclude [...]\n for k, v in b.__dict__.items():\n if (len(include) and k not in include) or k.startswith('_') or k in exclude:\n continue\n else:\n setattr(a, k, v)\n\n\nclass ModelEMA:\n \"\"\" Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models\n Keep a moving average of everything in the model state_dict (parameters and buffers).\n This is intended to allow functionality like\n https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage\n A smoothed version of the weights is necessary for some training schemes to perform well.\n This class is sensitive where it is initialized in the sequence of model init,\n GPU assignment and distributed training wrappers.\n \"\"\"\n\n def __init__(self, model, decay=0.9999, updates=0):\n # Create EMA\n self.ema = deepcopy(model.module if is_parallel(model) else model).eval() # FP32 EMA\n # if next(model.parameters()).device.type != 'cpu':\n # self.ema.half() # FP16 EMA\n self.updates = updates # number of EMA updates\n self.decay = lambda x: decay * (1 - math.exp(-x / 2000)) # decay exponential ramp (to help early epochs)\n for p in self.ema.parameters():\n p.requires_grad_(False)\n\n def update(self, model):\n # Update EMA parameters\n with torch.no_grad():\n self.updates += 1\n d = self.decay(self.updates)\n\n msd = model.module.state_dict() if is_parallel(model) else model.state_dict() # model state_dict\n for k, v in self.ema.state_dict().items():\n if v.dtype.is_floating_point:\n v *= d\n v += (1. - d) * msd[k].detach()\n\n def update_attr(self, model, include=(), exclude=('process_group', 'reducer')):\n # Update EMA attributes\n copy_attr(self.ema, model, include, exclude)\n\n\nclass BatchNormXd(torch.nn.modules.batchnorm._BatchNorm):\n def _check_input_dim(self, input):\n # The only difference between BatchNorm1d, BatchNorm2d, BatchNorm3d, etc\n # is this method that is overwritten by the sub-class\n # This original goal of this method was for tensor sanity checks\n # If you're ok bypassing those sanity checks (eg. if you trust your inference\n # to provide the right dimensional inputs), then you can just use this method\n # for easy conversion from SyncBatchNorm\n # (unfortunately, SyncBatchNorm does not store the original class - if it did\n # we could return the one that was originally created)\n return\n\ndef revert_sync_batchnorm(module):\n # this is very similar to the function that it is trying to revert:\n # https://github.com/pytorch/pytorch/blob/c8b3686a3e4ba63dc59e5dcfe5db3430df256833/torch/nn/modules/batchnorm.py#L679\n module_output = module\n if isinstance(module, torch.nn.modules.batchnorm.SyncBatchNorm):\n new_cls = BatchNormXd\n module_output = BatchNormXd(module.num_features,\n module.eps, module.momentum,\n module.affine,\n module.track_running_stats)\n if module.affine:\n with torch.no_grad():\n module_output.weight = module.weight\n module_output.bias = module.bias\n module_output.running_mean = module.running_mean\n module_output.running_var = module.running_var\n module_output.num_batches_tracked = module.num_batches_tracked\n if hasattr(module, \"qconfig\"):\n module_output.qconfig = module.qconfig\n for name, child in module.named_children():\n module_output.add_module(name, revert_sync_batchnorm(child))\n del module\n return module_output\n\n\nclass TracedModel(nn.Module):\n\n def __init__(self, model=None, device=None, img_size=(640,640)): \n super(TracedModel, self).__init__()\n \n print(\" Convert model to Traced-model... \") \n self.stride = model.stride\n self.names = model.names\n self.model = model\n\n self.model = revert_sync_batchnorm(self.model)\n self.model.to('cpu')\n self.model.eval()\n\n self.detect_layer = self.model.model[-1]\n self.model.traced = True\n \n rand_example = torch.rand(1, 3, img_size, img_size)\n \n traced_script_module = torch.jit.trace(self.model, rand_example, strict=False)\n #traced_script_module = torch.jit.script(self.model)\n traced_script_module.save(\"traced_model.pt\")\n print(\" traced_script_module saved! \")\n self.model = traced_script_module\n self.model.to(device)\n self.detect_layer.to(device)\n print(\" model is traced! \\n\") \n\n def forward(self, x, augment=False, profile=False):\n out = self.model(x)\n out = self.detect_layer(out)\n return out" }, { "path": "asone/detectors/yolov7/yolov7/utils/yolov7_utils.py", "content": "import cv2\nimport numpy as np\nimport torch\nimport torchvision\nimport time\n\ndef prepare_input(image, input_shape):\n input_height, input_width = input_shape\n input_img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n # Resize input image\n input_img = cv2.resize(input_img, (input_width, input_height))\n # Scale input pixel values to 0 to 1\n input_img = input_img / 255.0\n input_img = input_img.transpose(2, 0, 1)\n input_tensor = input_img[np.newaxis, :, :, :].astype(np.float32)\n\n return input_tensor\n\ndef process_output(output, ori_shape, input_shape, conf_threshold, iou_threshold):\n predictions = output[0]\n # predictions = np.squeeze(output[0])\n\n # Filter out object confidence scores below threshold\n # obj_conf = predictions[:, 4]\n obj_conf = predictions[:, 6]\n # predictions = predictions[obj_conf > conf_threshold]\n # obj_conf = obj_conf[obj_conf > conf_threshold]\n\n # print(obj_conf[0])\n\n # Multiply class confidence with bounding box confidence\n # predictions[:, 5] *= obj_conf[:, np.newaxis]\n # predictions[:, 6] *= obj_conf\n\n # Get the scores\n # scores = np.max(predictions[:, 5:], axis=1)\n scores = predictions[:, 6]\n\n # Filter out the objects with a low score\n predictions = predictions[obj_conf > conf_threshold]\n scores = scores[scores > conf_threshold]\n\n if len(scores) == 0:\n return []\n\n # Get the class with the highest confidence\n # class_ids = np.argmax(predictions[:, 5:], axis=1)\n class_ids = predictions[:, 5].astype(np.uint16)\n\n # Extract boxes from predictions\n boxes = predictions[:, 1:5]\n\n # Scale boxes to original image dimensions\n boxes = rescale_boxes(boxes, ori_shape, input_shape)\n\n # Convert boxes to xyxy format\n # boxes = xywh2xyxy(boxes)\n\n # Apply non-maxima suppression to suppress weak, overlapping bounding boxes\n indices = nms(boxes, scores, iou_threshold)\n\n dets = []\n for i in indices:\n dets.append([*boxes[i], scores[i], class_ids[i]])\n\n # return boxes[indices], scores[indices], class_ids[indices]\n return np.array(dets)\n\n\ndef rescale_boxes(boxes, ori_shape, input_shape):\n \n input_height, input_width = input_shape\n img_height, img_width = ori_shape\n # Rescale boxes to original image dimensions\n input_shape = np.array([input_width, input_height, input_width, input_height])\n boxes = np.divide(boxes, input_shape, dtype=np.float32)\n boxes *= np.array([img_width, img_height, img_width, img_height])\n return boxes\n\ndef nms(boxes, scores, iou_threshold):\n # Sort by score\n sorted_indices = np.argsort(scores)[::-1]\n\n keep_boxes = []\n while sorted_indices.size > 0:\n # Pick the last box\n box_id = sorted_indices[0]\n keep_boxes.append(box_id)\n\n # Compute IoU of the picked box with the rest\n ious = compute_iou(boxes[box_id, :], boxes[sorted_indices[1:], :])\n\n # Remove boxes with IoU over the threshold\n keep_indices = np.where(ious < iou_threshold)[0]\n\n # print(keep_indices.shape, sorted_indices.shape)\n sorted_indices = sorted_indices[keep_indices + 1]\n\n return keep_boxes\n\n\ndef compute_iou(box, boxes):\n # Compute xmin, ymin, xmax, ymax for both boxes\n xmin = np.maximum(box[0], boxes[:, 0])\n ymin = np.maximum(box[1], boxes[:, 1])\n xmax = np.minimum(box[2], boxes[:, 2])\n ymax = np.minimum(box[3], boxes[:, 3])\n\n # Compute intersection area\n intersection_area = np.maximum(0, xmax - xmin) * np.maximum(0, ymax - ymin)\n\n # Compute union area\n box_area = (box[2] - box[0]) * (box[3] - box[1])\n boxes_area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])\n union_area = box_area + boxes_area - intersection_area\n\n # Compute IoU\n iou = intersection_area / union_area\n\n return iou\n\n\ndef xywh2xyxy(x):\n # Convert bounding box (x, y, w, h) to bounding box (x1, y1, x2, y2)\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = x[..., 0] - x[..., 2] / 2\n y[..., 1] = x[..., 1] - x[..., 3] / 2\n y[..., 2] = x[..., 0] + x[..., 2] / 2\n y[..., 3] = x[..., 1] + x[..., 3] / 2\n return y\n\ndef non_max_suppression(prediction, conf_thres=0.25, iou_thres=0.45, classes=None, agnostic=False, multi_label=False,\n labels=()):\n \"\"\"Runs Non-Maximum Suppression (NMS) on inference results\n\n Returns:\n list of detections, on (n,6) tensor per image [xyxy, conf, cls]\n \"\"\"\n\n nc = prediction.shape[2] - 5 # number of classes\n xc = prediction[..., 4] > conf_thres # candidates\n\n # Settings\n min_wh, max_wh = 2, 4096 # (pixels) minimum and maximum box width and height\n max_det = 300 # maximum number of detections per image\n max_nms = 30000 # maximum number of boxes into torchvision.ops.nms()\n time_limit = 10.0 # seconds to quit after\n redundant = True # require redundant detections\n multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img)\n merge = False # use merge-NMS\n\n t = time.time()\n \n output = [torch.zeros((0, 6), device=prediction.device)] * prediction.shape[0]\n for xi, x in enumerate(prediction): # image index, image inference\n # Apply constraints\n # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height\n x = x[xc[xi]] # confidence\n\n # Cat apriori labels if autolabelling\n if labels and len(labels[xi]):\n l = labels[xi]\n v = torch.zeros((len(l), nc + 5), device=x.device)\n v[:, :4] = l[:, 1:5] # box\n v[:, 4] = 1.0 # conf\n v[range(len(l)), l[:, 0].long() + 5] = 1.0 # cls\n x = torch.cat((x, v), 0)\n\n # If none remain process next image\n if not x.shape[0]:\n continue\n\n # Compute conf\n if nc == 1:\n x[:, 5:] = x[:, 4:5] # for models with one class, cls_loss is 0 and cls_conf is always 0.5,\n # so there is no need to multiplicate.\n else:\n x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf\n\n # Box (center x, center y, width, height) to (x1, y1, x2, y2)\n box = xywh2xyxy(x[:, :4])\n\n # Detections matrix nx6 (xyxy, conf, cls)\n if multi_label:\n i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T\n x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1)\n else: # best class only\n conf, j = x[:, 5:].max(1, keepdim=True)\n x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres]\n\n # Filter by class\n if classes is not None:\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\n\n # Apply finite constraint\n # if not torch.isfinite(x).all():\n # x = x[torch.isfinite(x).all(1)]\n\n # Check shape\n n = x.shape[0] # number of boxes\n if not n: # no boxes\n continue\n elif n > max_nms: # excess boxes\n x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence\n\n # Batched NMS\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\n i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS\n if i.shape[0] > max_det: # limit detections\n i = i[:max_det]\n if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)\n # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\n weights = iou * scores[None] # box weights\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\n if redundant:\n i = i[iou.sum(1) > 1] # require redundancy\n\n output[xi] = x[i]\n if (time.time() - t) > time_limit:\n print(f'WARNING: NMS time limit {time_limit}s exceeded')\n break # time limit exceeded\n\n return output\n\n\n\n\n\n" }, { "path": "asone/detectors/yolov7/yolov7_detector.py", "content": "import os\nimport sys\nimport onnxruntime\nimport torch\nimport coremltools as ct\nfrom asone.utils import get_names\nimport numpy as np\nimport warnings\nfrom asone.detectors.yolov7.yolov7.utils.yolov7_utils import (prepare_input,\n process_output,\n non_max_suppression)\nfrom asone.detectors.yolov7.yolov7.models.experimental import attempt_load\nfrom asone import utils\nfrom PIL import Image\nfrom asone.detectors.utils.coreml_utils import yolo_to_xyxy, generalize_output_format, scale_bboxes\n\nfrom asone.utils.utils import PathResolver\n\ndef xywh2xyxy(x):\n # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x\n y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y\n y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x\n y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y\n return y\n\n\nclass YOLOv7Detector:\n def __init__(self,\n weights=None,\n use_onnx=False,\n mlmodel=False,\n use_cuda=True):\n self.use_onnx = use_onnx\n self.mlmodel = mlmodel\n self.device = 'cuda' if use_cuda else 'cpu'\n\n #If incase weighst is a list of paths then select path at first index\n\n weights = str(weights[0] if isinstance(weights, list) else weights)\n\n if not os.path.exists(weights):\n utils.download_weights(weights)\n \n with PathResolver():\n # Load Model\n self.model = self.load_model(use_cuda, weights)\n \n def load_model(self, use_cuda, weights, fp16=False):\n # Device: CUDA and if fp16=True only then half precision floating point works \n self.fp16 = fp16 & ((not self.use_onnx or self.use_onnx) and self.device != 'cpu')\n # Load onnx \n if self.use_onnx: \n if use_cuda:\n providers = ['CUDAExecutionProvider','CPUExecutionProvider']\n else:\n providers = ['CPUExecutionProvider']\n\n model = onnxruntime.InferenceSession(weights, providers=providers)\n # Load coreml\n elif self.mlmodel:\n model = ct.models.MLModel(weights)\n #Load Pytorch\n else: \n model = attempt_load(weights, map_location=self.device)\n model.half() if self.fp16 else model.float()\n return model\n\n\n def detect(self, image: list,\n input_shape: tuple = (640, 640),\n conf_thres: float = 0.25,\n iou_thres: float = 0.45,\n max_det: int = 1000,\n filter_classes: bool = None,\n agnostic_nms: bool = True,\n with_p6: bool = False,\n return_image=False) -> list:\n\n # Preprocess input image and also copying original image for later use\n original_image = image.copy()\n img_height, img_width = original_image.shape[:2]\n processed_image = prepare_input(image, input_shape)\n \n # Perform Inference on the Image\n if self.use_onnx:\n # Run ONNX model \n input_name = self.model.get_inputs()[0].name\n prediction = self.model.run([self.model.get_outputs()[0].name], {\n input_name: processed_image})\n # Run Coreml model \n elif self.mlmodel:\n h ,w = image.shape[:2]\n pred = self.model.predict({\"image\":Image.fromarray(image).resize(input_shape)})\n xyxy = yolo_to_xyxy(pred['coordinates'], input_shape)\n out = generalize_output_format(xyxy, pred['confidence'], conf_thres)\n if out != []:\n detections = scale_bboxes(out, image.shape[:2], input_shape)\n else:\n detections = np.empty((0, 6))\n \n if filter_classes:\n class_names = get_names()\n\n filter_class_idx = []\n if filter_classes:\n for _class in filter_classes:\n if _class.lower() in class_names:\n filter_class_idx.append(class_names.index(_class.lower()))\n else:\n warnings.warn(f\"class {_class} not found in model classes list.\")\n\n detections = detections[np.in1d(detections[:,5].astype(int), filter_class_idx)]\n \n return detections, {'width':w, 'height':h}\n # Run Pytorch model\n else:\n processed_image = torch.from_numpy(processed_image).to(self.device)\n # Change image floating point precision if fp16 set to true\n processed_image = processed_image.half() if self.fp16 else processed_image.float() \n\n with torch.no_grad():\n prediction = self.model(processed_image, augment=False)[0]\n\n \n detection = []\n # Postprocess prediction\n if self.use_onnx:\n \n detection = process_output(prediction,\n original_image.shape[:2],\n input_shape,\n conf_thres,\n iou_thres)\n else:\n detection = non_max_suppression(prediction,\n conf_thres,\n iou_thres,\n agnostic=agnostic_nms)[0]\n \n detection = detection.detach().cpu().numpy()\n # detection = yolo_to_xyxy(detection, input_shape)\n # print(detection)\n \n # Rescaling Bounding Boxes\n detection[:, :4] /= np.array([input_shape[1], input_shape[0], input_shape[1], input_shape[0]])\n detection[:, :4] *= np.array([img_width, img_height, img_width, img_height])\n\n image_info = {\n 'width': original_image.shape[1],\n 'height': original_image.shape[0],\n }\n\n if len(detection) > 0:\n self.boxes = detection[:, :4]\n self.scores = detection[:, 4:5]\n self.class_ids = detection[:, 5:6]\n\n if filter_classes:\n class_names = get_names()\n\n filter_class_idx = []\n if filter_classes:\n for _class in filter_classes:\n if _class.lower() in class_names:\n filter_class_idx.append(class_names.index(_class.lower()))\n else:\n warnings.warn(f\"class {_class} not found in model classes list.\")\n\n detection = detection[np.in1d(detection[:,5].astype(int), filter_class_idx)]\n\n if return_image:\n return detection, original_image\n else: \n return detection, image_info\n" }, { "path": "asone/detectors/yolov8/__init__.py", "content": "from .yolov8_detector import YOLOv8Detector\n__all__ = ['YOLOv8Detector']" }, { "path": "asone/detectors/yolov8/utils/__init__.py", "content": "" }, { "path": "asone/detectors/yolov8/utils/yolov8_utils.py", "content": "import cv2\nimport numpy as np\nfrom ultralytics.utils import ops\nimport torch\nfrom ultralytics.data.augment import LetterBox\n\ndef prepare_input(image, input_shape, stride, pt):\n input_tensor = LetterBox(input_shape, auto=pt, stride=stride)(image=image)\n input_tensor = input_tensor.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n input_tensor = np.ascontiguousarray(input_tensor).astype(np.float32) # contiguous\n input_tensor /= 255.0 # 0 - 255 to 0.0 - 1.0\n input_tensor = input_tensor[None].astype(np.float32)\n return input_tensor\n\n\ndef process_output(detections, \n ori_shape, \n input_shape, \n conf_threshold, \n iou_threshold,\n classes=None,\n mlmodel=False,\n agnostic=False,\n max_det=300,\n ):\n detections = ops.non_max_suppression(detections,\n conf_thres=conf_threshold,\n iou_thres=iou_threshold,\n classes=classes,\n agnostic=agnostic,\n max_det=max_det,\n )\n\n if mlmodel:\n detection = detections[0].cpu().numpy()\n return detection\n\n for i in range(len(detections)): \n # Extract boxes from predictions\n detections[i][:, :4] = ops.scale_boxes(input_shape, detections[i][:, :4], ori_shape).round()\n\n \n return detections[0].cpu().numpy()\n\n\ndef rescale_boxes(boxes, ori_shape, input_shape):\n\n input_height, input_width = input_shape\n img_height, img_width = ori_shape\n # Rescale boxes to original image dimensions\n input_shape = np.array(\n [input_width, input_height, input_width, input_height])\n boxes = np.divide(boxes, input_shape, dtype=np.float32)\n boxes *= np.array([img_width, img_height, img_width, img_height])\n return boxes\n" }, { "path": "asone/detectors/yolov8/yolov8_detector.py", "content": "import os\nfrom asone import utils\nfrom asone.utils import get_names\nimport onnxruntime\nimport torch\nfrom asone.detectors.yolov8.utils.yolov8_utils import prepare_input, process_output\nimport numpy as np\nimport warnings\nfrom ultralytics.nn.autobackend import AutoBackend\nfrom ultralytics.nn.tasks import DetectionModel, attempt_load_one_weight\nimport coremltools as ct\nfrom PIL import Image\nfrom asone.detectors.utils.coreml_utils import yolo_to_xyxy, generalize_output_format, scale_bboxes\n\nfrom asone.utils.utils import PathResolver\n\n\nclass YOLOv8Detector:\n def __init__(self,\n weights=None,\n use_onnx=False,\n mlmodel=False,\n use_cuda=True):\n\n self.mlmodel = mlmodel\n self.use_onnx = use_onnx\n self.device = 'cuda' if use_cuda else 'cpu'\n\n # If incase weighst is a list of paths then select path at first index\n weights = str(weights[0] if isinstance(weights, list) else weights)\n\n if not os.path.exists(weights):\n utils.download_weights(weights)\n\n with PathResolver():\n # Load Model\n self.model = self.load_model(use_cuda, weights)\n\n def load_model(self, use_cuda, weights, fp16=False):\n\n # Device: CUDA and if fp16=True only then half precision floating point works\n self.fp16 = fp16 & (\n (not self.use_onnx or self.use_onnx) and self.device != 'cpu')\n\n # Load onnx\n if self.use_onnx:\n if use_cuda:\n providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']\n else:\n providers = ['CPUExecutionProvider']\n\n model = onnxruntime.InferenceSession(weights, providers=providers)\n # Load coreml\n elif self.mlmodel:\n model = ct.models.MLModel(weights)\n \n # Load Pytorch\n else:\n model, ckpt = attempt_load_one_weight(weights)\n model = AutoBackend(model, fp16=False, dnn=False).to(self.device)\n model.half() if self.fp16 else model.float()\n return model\n\n def detect(self, image: list,\n input_shape: tuple = (640, 640),\n conf_thres: float = 0.25,\n iou_thres: float = 0.45,\n max_det: int = 1000,\n filter_classes: bool = None,\n agnostic_nms: bool = True,\n with_p6: bool = False,\n return_image=False\n ) -> list:\n\n # Preprocess input image and also copying original image for later use\n original_image = image.copy()\n processed_image = prepare_input(\n image, input_shape, 32, False if self.use_onnx else True)\n\n # Perform Inference on the Image\n if self.use_onnx:\n # Run ONNX model\n input_name = self.model.get_inputs()[0].name\n prediction = self.model.run([self.model.get_outputs()[0].name], {\n input_name: processed_image})[0]\n prediction = torch.from_numpy(prediction)\n # Run Coreml model\n elif self.mlmodel:\n im = Image.fromarray(image).resize(input_shape)\n y = self.model.predict({\"image\":im})\n \n if 'confidence' in y:\n box = xywh2xyxy(y['coordinates'] * [[w, h, w, h]]) # xyxy pixels\n conf, cls = y['confidence'].max(1), y['confidence'].argmax(1).astype(np.float)\n y = np.concatenate((box, conf.reshape(-1, 1), cls.reshape(-1, 1)), 1)\n else:\n k = 'var_' + str(sorted(int(k.replace('var_', '')) for k in y)[-1]) # output key\n y = y[k] # output\n width, height = im.size\n prediction = torch.from_numpy(y)\n # Run Pytorch model\n else:\n processed_image = torch.from_numpy(processed_image).to(self.device)\n # Change image floating point precision if fp16 set to true\n processed_image = processed_image.half() if self.fp16 else processed_image.float()\n\n with torch.no_grad():\n prediction = self.model(processed_image, augment=False)\n \n detection = []\n # Postprocess prediction\n \n if self.mlmodel:\n detection = process_output(prediction,\n original_image.shape[:2],\n [640, 640],\n conf_thres,\n iou_thres,\n mlmodel=True,\n agnostic=agnostic_nms,\n max_det=max_det)\n \n detection = scale_bboxes(detection, original_image.shape[:2], input_shape)\n else:\n detection = process_output(prediction,\n original_image.shape[:2],\n processed_image.shape[2:],\n conf_thres,\n iou_thres,\n agnostic=agnostic_nms,\n max_det=max_det)\n\n image_info = {\n 'width': original_image.shape[1],\n 'height': original_image.shape[0],\n }\n\n if filter_classes:\n class_names = get_names()\n\n filter_class_idx = []\n if filter_classes:\n for _class in filter_classes:\n if _class.lower() in class_names:\n filter_class_idx.append(\n class_names.index(_class.lower()))\n else:\n warnings.warn(\n f\"class {_class} not found in model classes list.\")\n\n detection = detection[np.in1d(\n detection[:, 5].astype(int), filter_class_idx)]\n\n if return_image:\n return detection, original_image\n else: \n return detection, image_info\n \n" }, { "path": "asone/detectors/yolov9/__init__.py", "content": "from .yolov9_detector import YOLOv9Detector\n__all__ = ['YOLOv9Detector']" }, { "path": "asone/detectors/yolov9/export.py", "content": "import argparse\nimport contextlib\nimport json\nimport os\nimport platform\nimport re\nimport subprocess\nimport sys\nimport time\nimport warnings\nfrom pathlib import Path\n\nimport pandas as pd\nimport torch\nfrom torch.utils.mobile_optimizer import optimize_for_mobile\n\nFILE = Path(__file__).resolve()\nROOT = FILE.parents[0] # YOLO root directory\n# if str(ROOT) not in sys.path:\n# sys.path.append(str(ROOT)) # add ROOT to PATH\nif platform.system() != 'Windows':\n ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative\n\nfrom asone.detectors.yolov9.yolov9.models.experimental import attempt_load, End2End\nfrom asone.detectors.yolov9.yolov9.models.yolo import ClassificationModel, Detect, DDetect, DualDetect, DualDDetect, DetectionModel, SegmentationModel\nfrom asone.detectors.yolov9.yolov9.utils.dataloaders import LoadImages\nfrom asone.detectors.yolov9.yolov9.utils.general import (LOGGER, Profile, check_dataset, check_img_size, check_requirements, check_version,\n check_yaml, colorstr, file_size, get_default_args, print_args, url2file, yaml_save)\nfrom asone.detectors.yolov9.yolov9.utils.torch_utils import select_device, smart_inference_mode\n\nMACOS = platform.system() == 'Darwin' # macOS environment\n\n\ndef export_formats():\n # YOLO export formats\n x = [\n ['PyTorch', '-', '.pt', True, True],\n ['TorchScript', 'torchscript', '.torchscript', True, True],\n ['ONNX', 'onnx', '.onnx', True, True],\n ['ONNX END2END', 'onnx_end2end', '_end2end.onnx', True, True],\n ['OpenVINO', 'openvino', '_openvino_model', True, False],\n ['TensorRT', 'engine', '.engine', False, True],\n ['CoreML', 'coreml', '.mlmodel', True, False],\n ['TensorFlow SavedModel', 'saved_model', '_saved_model', True, True],\n ['TensorFlow GraphDef', 'pb', '.pb', True, True],\n ['TensorFlow Lite', 'tflite', '.tflite', True, False],\n ['TensorFlow Edge TPU', 'edgetpu', '_edgetpu.tflite', False, False],\n ['TensorFlow.js', 'tfjs', '_web_model', False, False],\n ['PaddlePaddle', 'paddle', '_paddle_model', True, True],]\n return pd.DataFrame(x, columns=['Format', 'Argument', 'Suffix', 'CPU', 'GPU'])\n\n\ndef try_export(inner_func):\n # YOLO export decorator, i..e @try_export\n inner_args = get_default_args(inner_func)\n\n def outer_func(*args, **kwargs):\n prefix = inner_args['prefix']\n try:\n with Profile() as dt:\n f, model = inner_func(*args, **kwargs)\n LOGGER.info(f'{prefix} export success ✅ {dt.t:.1f}s, saved as {f} ({file_size(f):.1f} MB)')\n return f, model\n except Exception as e:\n LOGGER.info(f'{prefix} export failure ❌ {dt.t:.1f}s: {e}')\n return None, None\n\n return outer_func\n\n\n@try_export\ndef export_torchscript(model, im, file, optimize, prefix=colorstr('TorchScript:')):\n # YOLO TorchScript model export\n LOGGER.info(f'\\n{prefix} starting export with torch {torch.__version__}...')\n f = file.with_suffix('.torchscript')\n\n ts = torch.jit.trace(model, im, strict=False)\n d = {\"shape\": im.shape, \"stride\": int(max(model.stride)), \"names\": model.names}\n extra_files = {'config.txt': json.dumps(d)} # torch._C.ExtraFilesMap()\n if optimize: # https://pytorch.org/tutorials/recipes/mobile_interpreter.html\n optimize_for_mobile(ts)._save_for_lite_interpreter(str(f), _extra_files=extra_files)\n else:\n ts.save(str(f), _extra_files=extra_files)\n return f, None\n\n\n@try_export\ndef export_onnx(model, im, file, opset, dynamic, simplify, prefix=colorstr('ONNX:')):\n # YOLO ONNX export\n check_requirements('onnx')\n import onnx\n\n LOGGER.info(f'\\n{prefix} starting export with onnx {onnx.__version__}...')\n f = file.with_suffix('.onnx')\n\n output_names = ['output0', 'output1'] if isinstance(model, SegmentationModel) else ['output0']\n if dynamic:\n dynamic = {'images': {0: 'batch', 2: 'height', 3: 'width'}} # shape(1,3,640,640)\n if isinstance(model, SegmentationModel):\n dynamic['output0'] = {0: 'batch', 1: 'anchors'} # shape(1,25200,85)\n dynamic['output1'] = {0: 'batch', 2: 'mask_height', 3: 'mask_width'} # shape(1,32,160,160)\n elif isinstance(model, DetectionModel):\n dynamic['output0'] = {0: 'batch', 1: 'anchors'} # shape(1,25200,85)\n\n torch.onnx.export(\n model.cpu() if dynamic else model, # --dynamic only compatible with cpu\n im.cpu() if dynamic else im,\n f,\n verbose=False,\n opset_version=opset,\n do_constant_folding=True,\n input_names=['images'],\n output_names=output_names,\n dynamic_axes=dynamic or None)\n\n # Checks\n model_onnx = onnx.load(f) # load onnx model\n onnx.checker.check_model(model_onnx) # check onnx model\n\n # Metadata\n d = {'stride': int(max(model.stride)), 'names': model.names}\n for k, v in d.items():\n meta = model_onnx.metadata_props.add()\n meta.key, meta.value = k, str(v)\n onnx.save(model_onnx, f)\n\n # Simplify\n if simplify:\n try:\n cuda = torch.cuda.is_available()\n check_requirements(('onnxruntime-gpu' if cuda else 'onnxruntime', 'onnx-simplifier>=0.4.1'))\n import onnxsim\n\n LOGGER.info(f'{prefix} simplifying with onnx-simplifier {onnxsim.__version__}...')\n model_onnx, check = onnxsim.simplify(model_onnx)\n assert check, 'assert check failed'\n onnx.save(model_onnx, f)\n except Exception as e:\n LOGGER.info(f'{prefix} simplifier failure: {e}')\n return f, model_onnx\n \n\n@try_export\ndef export_onnx_end2end(model, im, file, simplify, topk_all, iou_thres, conf_thres, device, labels, prefix=colorstr('ONNX END2END:')):\n # YOLO ONNX export\n check_requirements('onnx')\n import onnx\n LOGGER.info(f'\\n{prefix} starting export with onnx {onnx.__version__}...')\n f = os.path.splitext(file)[0] + \"-end2end.onnx\"\n batch_size = 'batch'\n\n dynamic_axes = {'images': {0 : 'batch', 2: 'height', 3:'width'}, } # variable length axes\n\n output_axes = {\n 'num_dets': {0: 'batch'},\n 'det_boxes': {0: 'batch'},\n 'det_scores': {0: 'batch'},\n 'det_classes': {0: 'batch'},\n }\n dynamic_axes.update(output_axes)\n model = End2End(model, topk_all, iou_thres, conf_thres, None ,device, labels)\n\n output_names = ['num_dets', 'det_boxes', 'det_scores', 'det_classes']\n shapes = [ batch_size, 1, batch_size, topk_all, 4,\n batch_size, topk_all, batch_size, topk_all]\n\n torch.onnx.export(model, \n im, \n f, \n verbose=False, \n export_params=True, # store the trained parameter weights inside the model file\n opset_version=12, \n do_constant_folding=True, # whether to execute constant folding for optimization\n input_names=['images'],\n output_names=output_names,\n dynamic_axes=dynamic_axes)\n\n # Checks\n model_onnx = onnx.load(f) # load onnx model\n onnx.checker.check_model(model_onnx) # check onnx model\n for i in model_onnx.graph.output:\n for j in i.type.tensor_type.shape.dim:\n j.dim_param = str(shapes.pop(0))\n\n if simplify:\n try:\n import onnxsim\n\n print('\\nStarting to simplify ONNX...')\n model_onnx, check = onnxsim.simplify(model_onnx)\n assert check, 'assert check failed'\n except Exception as e:\n print(f'Simplifier failure: {e}')\n\n # print(onnx.helper.printable_graph(onnx_model.graph)) # print a human readable model\n onnx.save(model_onnx,f)\n print('ONNX export success, saved as %s' % f)\n return f, model_onnx\n\n\n@try_export\ndef export_openvino(file, metadata, half, prefix=colorstr('OpenVINO:')):\n # YOLO OpenVINO export\n check_requirements('openvino-dev') # requires openvino-dev: https://pypi.org/project/openvino-dev/\n import openvino.inference_engine as ie\n\n LOGGER.info(f'\\n{prefix} starting export with openvino {ie.__version__}...')\n f = str(file).replace('.pt', f'_openvino_model{os.sep}')\n\n #cmd = f\"mo --input_model {file.with_suffix('.onnx')} --output_dir {f} --data_type {'FP16' if half else 'FP32'}\"\n #cmd = f\"mo --input_model {file.with_suffix('.onnx')} --output_dir {f} {\"--compress_to_fp16\" if half else \"\"}\"\n half_arg = \"--compress_to_fp16\" if half else \"\"\n cmd = f\"mo --input_model {file.with_suffix('.onnx')} --output_dir {f} {half_arg}\"\n subprocess.run(cmd.split(), check=True, env=os.environ) # export\n yaml_save(Path(f) / file.with_suffix('.yaml').name, metadata) # add metadata.yaml\n return f, None\n\n\n@try_export\ndef export_paddle(model, im, file, metadata, prefix=colorstr('PaddlePaddle:')):\n # YOLO Paddle export\n check_requirements(('paddlepaddle', 'x2paddle'))\n import x2paddle\n from x2paddle.convert import pytorch2paddle\n\n LOGGER.info(f'\\n{prefix} starting export with X2Paddle {x2paddle.__version__}...')\n f = str(file).replace('.pt', f'_paddle_model{os.sep}')\n\n pytorch2paddle(module=model, save_dir=f, jit_type='trace', input_examples=[im]) # export\n yaml_save(Path(f) / file.with_suffix('.yaml').name, metadata) # add metadata.yaml\n return f, None\n\n\n@try_export\ndef export_coreml(model, im, file, int8, half, prefix=colorstr('CoreML:')):\n # YOLO CoreML export\n check_requirements('coremltools')\n import coremltools as ct\n\n LOGGER.info(f'\\n{prefix} starting export with coremltools {ct.__version__}...')\n f = file.with_suffix('.mlmodel')\n\n ts = torch.jit.trace(model, im, strict=False) # TorchScript model\n ct_model = ct.convert(ts, inputs=[ct.ImageType('image', shape=im.shape, scale=1 / 255, bias=[0, 0, 0])])\n bits, mode = (8, 'kmeans_lut') if int8 else (16, 'linear') if half else (32, None)\n if bits < 32:\n if MACOS: # quantization only supported on macOS\n with warnings.catch_warnings():\n warnings.filterwarnings(\"ignore\", category=DeprecationWarning) # suppress numpy==1.20 float warning\n ct_model = ct.models.neural_network.quantization_utils.quantize_weights(ct_model, bits, mode)\n else:\n print(f'{prefix} quantization only supported on macOS, skipping...')\n ct_model.save(f)\n return f, ct_model\n\n\n@try_export\ndef export_engine(model, im, file, half, dynamic, simplify, workspace=4, verbose=False, prefix=colorstr('TensorRT:')):\n # YOLO TensorRT export https://developer.nvidia.com/tensorrt\n assert im.device.type != 'cpu', 'export running on CPU but must be on GPU, i.e. `python export.py --device 0`'\n try:\n import tensorrt as trt\n except Exception:\n if platform.system() == 'Linux':\n check_requirements('nvidia-tensorrt', cmds='-U --index-url https://pypi.ngc.nvidia.com')\n import tensorrt as trt\n\n if trt.__version__[0] == '7': # TensorRT 7 handling https://github.com/ultralytics/yolov5/issues/6012\n grid = model.model[-1].anchor_grid\n model.model[-1].anchor_grid = [a[..., :1, :1, :] for a in grid]\n export_onnx(model, im, file, 12, dynamic, simplify) # opset 12\n model.model[-1].anchor_grid = grid\n else: # TensorRT >= 8\n check_version(trt.__version__, '8.0.0', hard=True) # require tensorrt>=8.0.0\n export_onnx(model, im, file, 12, dynamic, simplify) # opset 12\n onnx = file.with_suffix('.onnx')\n\n LOGGER.info(f'\\n{prefix} starting export with TensorRT {trt.__version__}...')\n assert onnx.exists(), f'failed to export ONNX file: {onnx}'\n f = file.with_suffix('.engine') # TensorRT engine file\n logger = trt.Logger(trt.Logger.INFO)\n if verbose:\n logger.min_severity = trt.Logger.Severity.VERBOSE\n\n builder = trt.Builder(logger)\n config = builder.create_builder_config()\n config.max_workspace_size = workspace * 1 << 30\n # config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, workspace << 30) # fix TRT 8.4 deprecation notice\n\n flag = (1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))\n network = builder.create_network(flag)\n parser = trt.OnnxParser(network, logger)\n if not parser.parse_from_file(str(onnx)):\n raise RuntimeError(f'failed to load ONNX file: {onnx}')\n\n inputs = [network.get_input(i) for i in range(network.num_inputs)]\n outputs = [network.get_output(i) for i in range(network.num_outputs)]\n for inp in inputs:\n LOGGER.info(f'{prefix} input \"{inp.name}\" with shape{inp.shape} {inp.dtype}')\n for out in outputs:\n LOGGER.info(f'{prefix} output \"{out.name}\" with shape{out.shape} {out.dtype}')\n\n if dynamic:\n if im.shape[0] <= 1:\n LOGGER.warning(f\"{prefix} WARNING ⚠️ --dynamic model requires maximum --batch-size argument\")\n profile = builder.create_optimization_profile()\n for inp in inputs:\n profile.set_shape(inp.name, (1, *im.shape[1:]), (max(1, im.shape[0] // 2), *im.shape[1:]), im.shape)\n config.add_optimization_profile(profile)\n\n LOGGER.info(f'{prefix} building FP{16 if builder.platform_has_fast_fp16 and half else 32} engine as {f}')\n if builder.platform_has_fast_fp16 and half:\n config.set_flag(trt.BuilderFlag.FP16)\n with builder.build_engine(network, config) as engine, open(f, 'wb') as t:\n t.write(engine.serialize())\n return f, None\n\n\n@try_export\ndef export_saved_model(model,\n im,\n file,\n dynamic,\n tf_nms=False,\n agnostic_nms=False,\n topk_per_class=100,\n topk_all=100,\n iou_thres=0.45,\n conf_thres=0.25,\n keras=False,\n prefix=colorstr('TensorFlow SavedModel:')):\n # YOLO TensorFlow SavedModel export\n try:\n import tensorflow as tf\n except Exception:\n check_requirements(f\"tensorflow{'' if torch.cuda.is_available() else '-macos' if MACOS else '-cpu'}\")\n import tensorflow as tf\n from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2\n\n from asone.detectors.yolov9.yolov9.models.tf import TFModel\n\n LOGGER.info(f'\\n{prefix} starting export with tensorflow {tf.__version__}...')\n f = str(file).replace('.pt', '_saved_model')\n batch_size, ch, *imgsz = list(im.shape) # BCHW\n\n tf_model = TFModel(cfg=model.yaml, model=model, nc=model.nc, imgsz=imgsz)\n im = tf.zeros((batch_size, *imgsz, ch)) # BHWC order for TensorFlow\n _ = tf_model.predict(im, tf_nms, agnostic_nms, topk_per_class, topk_all, iou_thres, conf_thres)\n inputs = tf.keras.Input(shape=(*imgsz, ch), batch_size=None if dynamic else batch_size)\n outputs = tf_model.predict(inputs, tf_nms, agnostic_nms, topk_per_class, topk_all, iou_thres, conf_thres)\n keras_model = tf.keras.Model(inputs=inputs, outputs=outputs)\n keras_model.trainable = False\n keras_model.summary()\n if keras:\n keras_model.save(f, save_format='tf')\n else:\n spec = tf.TensorSpec(keras_model.inputs[0].shape, keras_model.inputs[0].dtype)\n m = tf.function(lambda x: keras_model(x)) # full model\n m = m.get_concrete_function(spec)\n frozen_func = convert_variables_to_constants_v2(m)\n tfm = tf.Module()\n tfm.__call__ = tf.function(lambda x: frozen_func(x)[:4] if tf_nms else frozen_func(x), [spec])\n tfm.__call__(im)\n tf.saved_model.save(tfm,\n f,\n options=tf.saved_model.SaveOptions(experimental_custom_gradients=False) if check_version(\n tf.__version__, '2.6') else tf.saved_model.SaveOptions())\n return f, keras_model\n\n\n@try_export\ndef export_pb(keras_model, file, prefix=colorstr('TensorFlow GraphDef:')):\n # YOLO TensorFlow GraphDef *.pb export https://github.com/leimao/Frozen_Graph_TensorFlow\n import tensorflow as tf\n from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2\n\n LOGGER.info(f'\\n{prefix} starting export with tensorflow {tf.__version__}...')\n f = file.with_suffix('.pb')\n\n m = tf.function(lambda x: keras_model(x)) # full model\n m = m.get_concrete_function(tf.TensorSpec(keras_model.inputs[0].shape, keras_model.inputs[0].dtype))\n frozen_func = convert_variables_to_constants_v2(m)\n frozen_func.graph.as_graph_def()\n tf.io.write_graph(graph_or_graph_def=frozen_func.graph, logdir=str(f.parent), name=f.name, as_text=False)\n return f, None\n\n\n@try_export\ndef export_tflite(keras_model, im, file, int8, data, nms, agnostic_nms, prefix=colorstr('TensorFlow Lite:')):\n # YOLOv5 TensorFlow Lite export\n import tensorflow as tf\n\n LOGGER.info(f'\\n{prefix} starting export with tensorflow {tf.__version__}...')\n batch_size, ch, *imgsz = list(im.shape) # BCHW\n f = str(file).replace('.pt', '-fp16.tflite')\n\n converter = tf.lite.TFLiteConverter.from_keras_model(keras_model)\n converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]\n converter.target_spec.supported_types = [tf.float16]\n converter.optimizations = [tf.lite.Optimize.DEFAULT]\n if int8:\n from asone.detectors.yolov9.yolov9.models.tf import representative_dataset_gen\n dataset = LoadImages(check_dataset(check_yaml(data))['train'], img_size=imgsz, auto=False)\n converter.representative_dataset = lambda: representative_dataset_gen(dataset, ncalib=100)\n converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]\n converter.target_spec.supported_types = []\n converter.inference_input_type = tf.uint8 # or tf.int8\n converter.inference_output_type = tf.uint8 # or tf.int8\n converter.experimental_new_quantizer = True\n f = str(file).replace('.pt', '-int8.tflite')\n if nms or agnostic_nms:\n converter.target_spec.supported_ops.append(tf.lite.OpsSet.SELECT_TF_OPS)\n\n tflite_model = converter.convert()\n open(f, \"wb\").write(tflite_model)\n return f, None\n\n\n@try_export\ndef export_edgetpu(file, prefix=colorstr('Edge TPU:')):\n # YOLO Edge TPU export https://coral.ai/docs/edgetpu/models-intro/\n cmd = 'edgetpu_compiler --version'\n help_url = 'https://coral.ai/docs/edgetpu/compiler/'\n assert platform.system() == 'Linux', f'export only supported on Linux. See {help_url}'\n if subprocess.run(f'{cmd} >/dev/null', shell=True).returncode != 0:\n LOGGER.info(f'\\n{prefix} export requires Edge TPU compiler. Attempting install from {help_url}')\n sudo = subprocess.run('sudo --version >/dev/null', shell=True).returncode == 0 # sudo installed on system\n for c in (\n 'curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -',\n 'echo \"deb https://packages.cloud.google.com/apt coral-edgetpu-stable main\" | sudo tee /etc/apt/sources.list.d/coral-edgetpu.list',\n 'sudo apt-get update', 'sudo apt-get install edgetpu-compiler'):\n subprocess.run(c if sudo else c.replace('sudo ', ''), shell=True, check=True)\n ver = subprocess.run(cmd, shell=True, capture_output=True, check=True).stdout.decode().split()[-1]\n\n LOGGER.info(f'\\n{prefix} starting export with Edge TPU compiler {ver}...')\n f = str(file).replace('.pt', '-int8_edgetpu.tflite') # Edge TPU model\n f_tfl = str(file).replace('.pt', '-int8.tflite') # TFLite model\n\n cmd = f\"edgetpu_compiler -s -d -k 10 --out_dir {file.parent} {f_tfl}\"\n subprocess.run(cmd.split(), check=True)\n return f, None\n\n\n@try_export\ndef export_tfjs(file, prefix=colorstr('TensorFlow.js:')):\n # YOLO TensorFlow.js export\n check_requirements('tensorflowjs')\n import tensorflowjs as tfjs\n\n LOGGER.info(f'\\n{prefix} starting export with tensorflowjs {tfjs.__version__}...')\n f = str(file).replace('.pt', '_web_model') # js dir\n f_pb = file.with_suffix('.pb') # *.pb path\n f_json = f'{f}/model.json' # *.json path\n\n cmd = f'tensorflowjs_converter --input_format=tf_frozen_model ' \\\n f'--output_node_names=Identity,Identity_1,Identity_2,Identity_3 {f_pb} {f}'\n subprocess.run(cmd.split())\n\n json = Path(f_json).read_text()\n with open(f_json, 'w') as j: # sort JSON Identity_* in ascending order\n subst = re.sub(\n r'{\"outputs\": {\"Identity.?.?\": {\"name\": \"Identity.?.?\"}, '\n r'\"Identity.?.?\": {\"name\": \"Identity.?.?\"}, '\n r'\"Identity.?.?\": {\"name\": \"Identity.?.?\"}, '\n r'\"Identity.?.?\": {\"name\": \"Identity.?.?\"}}}', r'{\"outputs\": {\"Identity\": {\"name\": \"Identity\"}, '\n r'\"Identity_1\": {\"name\": \"Identity_1\"}, '\n r'\"Identity_2\": {\"name\": \"Identity_2\"}, '\n r'\"Identity_3\": {\"name\": \"Identity_3\"}}}', json)\n j.write(subst)\n return f, None\n\n\ndef add_tflite_metadata(file, metadata, num_outputs):\n # Add metadata to *.tflite models per https://www.tensorflow.org/lite/models/convert/metadata\n with contextlib.suppress(ImportError):\n # check_requirements('tflite_support')\n from tflite_support import flatbuffers\n from tflite_support import metadata as _metadata\n from tflite_support import metadata_schema_py_generated as _metadata_fb\n\n tmp_file = Path('/tmp/meta.txt')\n with open(tmp_file, 'w') as meta_f:\n meta_f.write(str(metadata))\n\n model_meta = _metadata_fb.ModelMetadataT()\n label_file = _metadata_fb.AssociatedFileT()\n label_file.name = tmp_file.name\n model_meta.associatedFiles = [label_file]\n\n subgraph = _metadata_fb.SubGraphMetadataT()\n subgraph.inputTensorMetadata = [_metadata_fb.TensorMetadataT()]\n subgraph.outputTensorMetadata = [_metadata_fb.TensorMetadataT()] * num_outputs\n model_meta.subgraphMetadata = [subgraph]\n\n b = flatbuffers.Builder(0)\n b.Finish(model_meta.Pack(b), _metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER)\n metadata_buf = b.Output()\n\n populator = _metadata.MetadataPopulator.with_model_file(file)\n populator.load_metadata_buffer(metadata_buf)\n populator.load_associated_files([str(tmp_file)])\n populator.populate()\n tmp_file.unlink()\n\n\n@smart_inference_mode()\ndef run(\n data=ROOT / 'data/coco.yaml', # 'dataset.yaml path'\n weights=ROOT / 'yolo.pt', # weights path\n imgsz=(640, 640), # image (height, width)\n batch_size=1, # batch size\n device='cpu', # cuda device, i.e. 0 or 0,1,2,3 or cpu\n include=('torchscript', 'onnx'), # include formats\n half=False, # FP16 half-precision export\n inplace=False, # set YOLO Detect() inplace=True\n keras=False, # use Keras\n optimize=False, # TorchScript: optimize for mobile\n int8=False, # CoreML/TF INT8 quantization\n dynamic=False, # ONNX/TF/TensorRT: dynamic axes\n simplify=False, # ONNX: simplify model\n opset=12, # ONNX: opset version\n verbose=False, # TensorRT: verbose log\n workspace=4, # TensorRT: workspace size (GB)\n nms=False, # TF: add NMS to model\n agnostic_nms=False, # TF: add agnostic NMS to model\n topk_per_class=100, # TF.js NMS: topk per class to keep\n topk_all=100, # TF.js NMS: topk for all classes to keep\n iou_thres=0.45, # TF.js NMS: IoU threshold\n conf_thres=0.25, # TF.js NMS: confidence threshold\n):\n t = time.time()\n include = [x.lower() for x in include] # to lowercase\n fmts = tuple(export_formats()['Argument'][1:]) # --include arguments\n flags = [x in include for x in fmts]\n assert sum(flags) == len(include), f'ERROR: Invalid --include {include}, valid --include arguments are {fmts}'\n jit, onnx, onnx_end2end, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle = flags # export booleans\n file = Path(url2file(weights) if str(weights).startswith(('http:/', 'https:/')) else weights) # PyTorch weights\n\n # Load PyTorch model\n device = select_device(device)\n if half:\n assert device.type != 'cpu' or coreml, '--half only compatible with GPU export, i.e. use --device 0'\n assert not dynamic, '--half not compatible with --dynamic, i.e. use either --half or --dynamic but not both'\n model = attempt_load(weights, device=device, inplace=True, fuse=True) # load FP32 model\n\n # Checks\n imgsz *= 2 if len(imgsz) == 1 else 1 # expand\n if optimize:\n assert device.type == 'cpu', '--optimize not compatible with cuda devices, i.e. use --device cpu'\n\n # Input\n gs = int(max(model.stride)) # grid size (max stride)\n imgsz = [check_img_size(x, gs) for x in imgsz] # verify img_size are gs-multiples\n im = torch.zeros(batch_size, 3, *imgsz).to(device) # image size(1,3,320,192) BCHW iDetection\n\n # Update model\n model.eval()\n for k, m in model.named_modules():\n if isinstance(m, (Detect, DDetect, DualDetect, DualDDetect)):\n m.inplace = inplace\n m.dynamic = dynamic\n m.export = True\n\n for _ in range(2):\n y = model(im) # dry runs\n if half and not coreml:\n im, model = im.half(), model.half() # to FP16\n shape = tuple((y[0] if isinstance(y, (tuple, list)) else y).shape) # model output shape\n metadata = {'stride': int(max(model.stride)), 'names': model.names} # model metadata\n LOGGER.info(f\"\\n{colorstr('PyTorch:')} starting from {file} with output shape {shape} ({file_size(file):.1f} MB)\")\n\n # Exports\n f = [''] * len(fmts) # exported filenames\n warnings.filterwarnings(action='ignore', category=torch.jit.TracerWarning) # suppress TracerWarning\n if jit: # TorchScript\n f[0], _ = export_torchscript(model, im, file, optimize)\n if engine: # TensorRT required before ONNX\n f[1], _ = export_engine(model, im, file, half, dynamic, simplify, workspace, verbose)\n if onnx or xml: # OpenVINO requires ONNX\n f[2], _ = export_onnx(model, im, file, opset, dynamic, simplify)\n if onnx_end2end:\n if isinstance(model, DetectionModel):\n labels = model.names\n f[2], _ = export_onnx_end2end(model, im, file, simplify, topk_all, iou_thres, conf_thres, device, len(labels))\n else:\n raise RuntimeError(\"The model is not a DetectionModel.\")\n if xml: # OpenVINO\n f[3], _ = export_openvino(file, metadata, half)\n if coreml: # CoreML\n f[4], _ = export_coreml(model, im, file, int8, half)\n if any((saved_model, pb, tflite, edgetpu, tfjs)): # TensorFlow formats\n assert not tflite or not tfjs, 'TFLite and TF.js models must be exported separately, please pass only one type.'\n assert not isinstance(model, ClassificationModel), 'ClassificationModel export to TF formats not yet supported.'\n f[5], s_model = export_saved_model(model.cpu(),\n im,\n file,\n dynamic,\n tf_nms=nms or agnostic_nms or tfjs,\n agnostic_nms=agnostic_nms or tfjs,\n topk_per_class=topk_per_class,\n topk_all=topk_all,\n iou_thres=iou_thres,\n conf_thres=conf_thres,\n keras=keras)\n if pb or tfjs: # pb prerequisite to tfjs\n f[6], _ = export_pb(s_model, file)\n if tflite or edgetpu:\n f[7], _ = export_tflite(s_model, im, file, int8 or edgetpu, data=data, nms=nms, agnostic_nms=agnostic_nms)\n if edgetpu:\n f[8], _ = export_edgetpu(file)\n add_tflite_metadata(f[8] or f[7], metadata, num_outputs=len(s_model.outputs))\n if tfjs:\n f[9], _ = export_tfjs(file)\n if paddle: # PaddlePaddle\n f[10], _ = export_paddle(model, im, file, metadata)\n\n # Finish\n f = [str(x) for x in f if x] # filter out '' and None\n if any(f):\n cls, det, seg = (isinstance(model, x) for x in (ClassificationModel, DetectionModel, SegmentationModel)) # type\n dir = Path('segment' if seg else 'classify' if cls else '')\n h = '--half' if half else '' # --half FP16 inference arg\n s = \"# WARNING ⚠️ ClassificationModel not yet supported for PyTorch Hub AutoShape inference\" if cls else \\\n \"# WARNING ⚠️ SegmentationModel not yet supported for PyTorch Hub AutoShape inference\" if seg else ''\n if onnx_end2end:\n LOGGER.info(f'\\nExport complete ({time.time() - t:.1f}s)'\n f\"\\nResults saved to {colorstr('bold', file.parent.resolve())}\"\n f\"\\nVisualize: https://netron.app\")\n else:\n LOGGER.info(f'\\nExport complete ({time.time() - t:.1f}s)'\n f\"\\nResults saved to {colorstr('bold', file.parent.resolve())}\"\n f\"\\nDetect: python {dir / ('detect.py' if det else 'predict.py')} --weights {f[-1]} {h}\"\n f\"\\nValidate: python {dir / 'val.py'} --weights {f[-1]} {h}\"\n f\"\\nPyTorch Hub: model = torch.hub.load('ultralytics/yolov5', 'custom', '{f[-1]}') {s}\"\n f\"\\nVisualize: https://netron.app\")\n return f # return list of exported files/dirs\n\n\ndef parse_opt():\n parser = argparse.ArgumentParser()\n parser.add_argument('--data', type=str, default=ROOT / 'data/coco.yaml', help='dataset.yaml path')\n parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolo.pt', help='model.pt path(s)')\n parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640, 640], help='image (h, w)')\n parser.add_argument('--batch-size', type=int, default=1, help='batch size')\n parser.add_argument('--device', default='cpu', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')\n parser.add_argument('--half', action='store_true', help='FP16 half-precision export')\n parser.add_argument('--inplace', action='store_true', help='set YOLO Detect() inplace=True')\n parser.add_argument('--keras', action='store_true', help='TF: use Keras')\n parser.add_argument('--optimize', action='store_true', help='TorchScript: optimize for mobile')\n parser.add_argument('--int8', action='store_true', help='CoreML/TF INT8 quantization')\n parser.add_argument('--dynamic', action='store_true', help='ONNX/TF/TensorRT: dynamic axes')\n parser.add_argument('--simplify', action='store_true', help='ONNX: simplify model')\n parser.add_argument('--opset', type=int, default=12, help='ONNX: opset version')\n parser.add_argument('--verbose', action='store_true', help='TensorRT: verbose log')\n parser.add_argument('--workspace', type=int, default=4, help='TensorRT: workspace size (GB)')\n parser.add_argument('--nms', action='store_true', help='TF: add NMS to model')\n parser.add_argument('--agnostic-nms', action='store_true', help='TF: add agnostic NMS to model')\n parser.add_argument('--topk-per-class', type=int, default=100, help='TF.js NMS: topk per class to keep')\n parser.add_argument('--topk-all', type=int, default=100, help='ONNX END2END/TF.js NMS: topk for all classes to keep')\n parser.add_argument('--iou-thres', type=float, default=0.45, help='ONNX END2END/TF.js NMS: IoU threshold')\n parser.add_argument('--conf-thres', type=float, default=0.25, help='ONNX END2END/TF.js NMS: confidence threshold')\n parser.add_argument(\n '--include',\n nargs='+',\n default=['torchscript'],\n help='torchscript, onnx, onnx_end2end, openvino, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle')\n opt = parser.parse_args()\n\n if 'onnx_end2end' in opt.include: \n opt.simplify = True\n opt.dynamic = True\n opt.inplace = True\n opt.half = False\n\n print_args(vars(opt))\n return opt\n\n\ndef main(opt):\n for opt.weights in (opt.weights if isinstance(opt.weights, list) else [opt.weights]):\n run(**vars(opt))\n\n\nif __name__ == \"__main__\":\n opt = parse_opt()\n main(opt)\n" }, { "path": "asone/detectors/yolov9/yolov9/__init__.py", "content": "" }, { "path": "asone/detectors/yolov9/yolov9/models/__init__.py", "content": "# init\n" }, { "path": "asone/detectors/yolov9/yolov9/models/common.py", "content": "import ast\nimport contextlib\nimport json\nimport math\nimport platform\nimport warnings\nimport zipfile\nfrom collections import OrderedDict, namedtuple\nfrom copy import copy\nfrom pathlib import Path\nfrom urllib.parse import urlparse\n\nfrom typing import Optional\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport requests\nimport torch\nimport torch.nn as nn\nfrom IPython.display import display\nfrom PIL import Image\nfrom torch.cuda import amp\n\nfrom asone.detectors.yolov9.yolov9.utils import TryExcept\nfrom asone.detectors.yolov9.yolov9.utils.dataloaders import exif_transpose, letterbox\nfrom asone.detectors.yolov9.yolov9.utils.general import (LOGGER, ROOT, Profile, check_requirements, check_suffix, check_version, colorstr,\n increment_path, is_notebook, make_divisible, non_max_suppression, scale_boxes,\n xywh2xyxy, xyxy2xywh, yaml_load)\nfrom asone.detectors.yolov9.yolov9.utils.plots import Annotator, colors, save_one_box\nfrom asone.detectors.yolov9.yolov9.utils.torch_utils import copy_attr, smart_inference_mode\n\n\ndef autopad(k, p=None, d=1): # kernel, padding, dilation\n # Pad to 'same' shape outputs\n if d > 1:\n k = d * (k - 1) + 1 if isinstance(k, int) else [d * (x - 1) + 1 for x in k] # actual kernel-size\n if p is None:\n p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad\n return p\n\n\nclass Conv(nn.Module):\n # Standard convolution with args(ch_in, ch_out, kernel, stride, padding, groups, dilation, activation)\n default_act = nn.SiLU() # default activation\n\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, d=1, act=True):\n super().__init__()\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p, d), groups=g, dilation=d, bias=False)\n self.bn = nn.BatchNorm2d(c2)\n self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity()\n\n def forward(self, x):\n return self.act(self.bn(self.conv(x)))\n\n def forward_fuse(self, x):\n return self.act(self.conv(x))\n\n\nclass AConv(nn.Module):\n def __init__(self, c1, c2): # ch_in, ch_out, shortcut, kernels, groups, expand\n super().__init__()\n self.cv1 = Conv(c1, c2, 3, 2, 1)\n\n def forward(self, x):\n x = torch.nn.functional.avg_pool2d(x, 2, 1, 0, False, True)\n return self.cv1(x)\n\n\nclass ADown(nn.Module):\n def __init__(self, c1, c2): # ch_in, ch_out, shortcut, kernels, groups, expand\n super().__init__()\n self.c = c2 // 2\n self.cv1 = Conv(c1 // 2, self.c, 3, 2, 1)\n self.cv2 = Conv(c1 // 2, self.c, 1, 1, 0)\n\n def forward(self, x):\n x = torch.nn.functional.avg_pool2d(x, 2, 1, 0, False, True)\n x1,x2 = x.chunk(2, 1)\n x1 = self.cv1(x1)\n x2 = torch.nn.functional.max_pool2d(x2, 3, 2, 1)\n x2 = self.cv2(x2)\n return torch.cat((x1, x2), 1)\n\n\nclass RepConvN(nn.Module):\n \"\"\"RepConv is a basic rep-style block, including training and deploy status\n This code is based on https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py\n \"\"\"\n default_act = nn.SiLU() # default activation\n\n def __init__(self, c1, c2, k=3, s=1, p=1, g=1, d=1, act=True, bn=False, deploy=False):\n super().__init__()\n assert k == 3 and p == 1\n self.g = g\n self.c1 = c1\n self.c2 = c2\n self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity()\n\n self.bn = None\n self.conv1 = Conv(c1, c2, k, s, p=p, g=g, act=False)\n self.conv2 = Conv(c1, c2, 1, s, p=(p - k // 2), g=g, act=False)\n\n def forward_fuse(self, x):\n \"\"\"Forward process\"\"\"\n return self.act(self.conv(x))\n\n def forward(self, x):\n \"\"\"Forward process\"\"\"\n id_out = 0 if self.bn is None else self.bn(x)\n return self.act(self.conv1(x) + self.conv2(x) + id_out)\n\n def get_equivalent_kernel_bias(self):\n kernel3x3, bias3x3 = self._fuse_bn_tensor(self.conv1)\n kernel1x1, bias1x1 = self._fuse_bn_tensor(self.conv2)\n kernelid, biasid = self._fuse_bn_tensor(self.bn)\n return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid\n\n def _avg_to_3x3_tensor(self, avgp):\n channels = self.c1\n groups = self.g\n kernel_size = avgp.kernel_size\n input_dim = channels // groups\n k = torch.zeros((channels, input_dim, kernel_size, kernel_size))\n k[np.arange(channels), np.tile(np.arange(input_dim), groups), :, :] = 1.0 / kernel_size ** 2\n return k\n\n def _pad_1x1_to_3x3_tensor(self, kernel1x1):\n if kernel1x1 is None:\n return 0\n else:\n return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1])\n\n def _fuse_bn_tensor(self, branch):\n if branch is None:\n return 0, 0\n if isinstance(branch, Conv):\n kernel = branch.conv.weight\n running_mean = branch.bn.running_mean\n running_var = branch.bn.running_var\n gamma = branch.bn.weight\n beta = branch.bn.bias\n eps = branch.bn.eps\n elif isinstance(branch, nn.BatchNorm2d):\n if not hasattr(self, 'id_tensor'):\n input_dim = self.c1 // self.g\n kernel_value = np.zeros((self.c1, input_dim, 3, 3), dtype=np.float32)\n for i in range(self.c1):\n kernel_value[i, i % input_dim, 1, 1] = 1\n self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device)\n kernel = self.id_tensor\n running_mean = branch.running_mean\n running_var = branch.running_var\n gamma = branch.weight\n beta = branch.bias\n eps = branch.eps\n std = (running_var + eps).sqrt()\n t = (gamma / std).reshape(-1, 1, 1, 1)\n return kernel * t, beta - running_mean * gamma / std\n\n def fuse_convs(self):\n if hasattr(self, 'conv'):\n return\n kernel, bias = self.get_equivalent_kernel_bias()\n self.conv = nn.Conv2d(in_channels=self.conv1.conv.in_channels,\n out_channels=self.conv1.conv.out_channels,\n kernel_size=self.conv1.conv.kernel_size,\n stride=self.conv1.conv.stride,\n padding=self.conv1.conv.padding,\n dilation=self.conv1.conv.dilation,\n groups=self.conv1.conv.groups,\n bias=True).requires_grad_(False)\n self.conv.weight.data = kernel\n self.conv.bias.data = bias\n for para in self.parameters():\n para.detach_()\n self.__delattr__('conv1')\n self.__delattr__('conv2')\n if hasattr(self, 'nm'):\n self.__delattr__('nm')\n if hasattr(self, 'bn'):\n self.__delattr__('bn')\n if hasattr(self, 'id_tensor'):\n self.__delattr__('id_tensor')\n\n\nclass SP(nn.Module):\n def __init__(self, k=3, s=1):\n super(SP, self).__init__()\n self.m = nn.MaxPool2d(kernel_size=k, stride=s, padding=k // 2)\n\n def forward(self, x):\n return self.m(x)\n\n\nclass MP(nn.Module):\n # Max pooling\n def __init__(self, k=2):\n super(MP, self).__init__()\n self.m = nn.MaxPool2d(kernel_size=k, stride=k)\n\n def forward(self, x):\n return self.m(x)\n\n\nclass ConvTranspose(nn.Module):\n # Convolution transpose 2d layer\n default_act = nn.SiLU() # default activation\n\n def __init__(self, c1, c2, k=2, s=2, p=0, bn=True, act=True):\n super().__init__()\n self.conv_transpose = nn.ConvTranspose2d(c1, c2, k, s, p, bias=not bn)\n self.bn = nn.BatchNorm2d(c2) if bn else nn.Identity()\n self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity()\n\n def forward(self, x):\n return self.act(self.bn(self.conv_transpose(x)))\n\n\nclass DWConv(Conv):\n # Depth-wise convolution\n def __init__(self, c1, c2, k=1, s=1, d=1, act=True): # ch_in, ch_out, kernel, stride, dilation, activation\n super().__init__(c1, c2, k, s, g=math.gcd(c1, c2), d=d, act=act)\n\n\nclass DWConvTranspose2d(nn.ConvTranspose2d):\n # Depth-wise transpose convolution\n def __init__(self, c1, c2, k=1, s=1, p1=0, p2=0): # ch_in, ch_out, kernel, stride, padding, padding_out\n super().__init__(c1, c2, k, s, p1, p2, groups=math.gcd(c1, c2))\n\n\nclass DFL(nn.Module):\n # DFL module\n def __init__(self, c1=17):\n super().__init__()\n self.conv = nn.Conv2d(c1, 1, 1, bias=False).requires_grad_(False)\n self.conv.weight.data[:] = nn.Parameter(torch.arange(c1, dtype=torch.float).view(1, c1, 1, 1)) # / 120.0\n self.c1 = c1\n # self.bn = nn.BatchNorm2d(4)\n\n def forward(self, x):\n b, c, a = x.shape # batch, channels, anchors\n return self.conv(x.view(b, 4, self.c1, a).transpose(2, 1).softmax(1)).view(b, 4, a)\n # return self.conv(x.view(b, self.c1, 4, a).softmax(1)).view(b, 4, a)\n\n\nclass BottleneckBase(nn.Module):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, k=(1, 3), e=0.5): # ch_in, ch_out, shortcut, kernels, groups, expand\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, k[0], 1)\n self.cv2 = Conv(c_, c2, k[1], 1, g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass RBottleneckBase(nn.Module):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 1), e=0.5): # ch_in, ch_out, shortcut, kernels, groups, expand\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, k[0], 1)\n self.cv2 = Conv(c_, c2, k[1], 1, g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass RepNRBottleneckBase(nn.Module):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 1), e=0.5): # ch_in, ch_out, shortcut, kernels, groups, expand\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = RepConvN(c1, c_, k[0], 1)\n self.cv2 = Conv(c_, c2, k[1], 1, g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass Bottleneck(nn.Module):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 3), e=0.5): # ch_in, ch_out, shortcut, kernels, groups, expand\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, k[0], 1)\n self.cv2 = Conv(c_, c2, k[1], 1, g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass RepNBottleneck(nn.Module):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 3), e=0.5): # ch_in, ch_out, shortcut, kernels, groups, expand\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = RepConvN(c1, c_, k[0], 1)\n self.cv2 = Conv(c_, c2, k[1], 1, g=g)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\n\n\nclass Res(nn.Module):\n # ResNet bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super(Res, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_, c_, 3, 1, g=g)\n self.cv3 = Conv(c_, c2, 1, 1)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv3(self.cv2(self.cv1(x))) if self.add else self.cv3(self.cv2(self.cv1(x)))\n\n\nclass RepNRes(nn.Module):\n # ResNet bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\n super(RepNRes, self).__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = RepConvN(c_, c_, 3, 1, g=g)\n self.cv3 = Conv(c_, c2, 1, 1)\n self.add = shortcut and c1 == c2\n\n def forward(self, x):\n return x + self.cv3(self.cv2(self.cv1(x))) if self.add else self.cv3(self.cv2(self.cv1(x)))\n\n\nclass BottleneckCSP(nn.Module):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False)\n self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False)\n self.cv4 = Conv(2 * c_, c2, 1, 1)\n self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3)\n self.act = nn.SiLU()\n self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)))\n\n def forward(self, x):\n y1 = self.cv3(self.m(self.cv1(x)))\n y2 = self.cv2(x)\n return self.cv4(self.act(self.bn(torch.cat((y1, y2), 1))))\n\n\nclass CSP(nn.Module):\n # CSP Bottleneck with 3 convolutions\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2)\n self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)))\n\n def forward(self, x):\n return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1))\n\n\nclass RepNCSP(nn.Module):\n # CSP Bottleneck with 3 convolutions\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2)\n self.m = nn.Sequential(*(RepNBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)))\n\n def forward(self, x):\n return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1))\n\n\nclass CSPBase(nn.Module):\n # CSP Bottleneck with 3 convolutions\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2)\n self.m = nn.Sequential(*(BottleneckBase(c_, c_, shortcut, g, e=1.0) for _ in range(n)))\n\n def forward(self, x):\n return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1))\n\n\nclass SPP(nn.Module):\n # Spatial Pyramid Pooling (SPP) layer https://arxiv.org/abs/1406.4729\n def __init__(self, c1, c2, k=(5, 9, 13)):\n super().__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1)\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\n\n def forward(self, x):\n x = self.cv1(x)\n with warnings.catch_warnings():\n warnings.simplefilter('ignore') # suppress torch 1.9.0 max_pool2d() warning\n return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1))\n\n \nclass ASPP(torch.nn.Module):\n\n def __init__(self, in_channels, out_channels):\n super().__init__()\n kernel_sizes = [1, 3, 3, 1]\n dilations = [1, 3, 6, 1]\n paddings = [0, 3, 6, 0]\n self.aspp = torch.nn.ModuleList()\n for aspp_idx in range(len(kernel_sizes)):\n conv = torch.nn.Conv2d(\n in_channels,\n out_channels,\n kernel_size=kernel_sizes[aspp_idx],\n stride=1,\n dilation=dilations[aspp_idx],\n padding=paddings[aspp_idx],\n bias=True)\n self.aspp.append(conv)\n self.gap = torch.nn.AdaptiveAvgPool2d(1)\n self.aspp_num = len(kernel_sizes)\n for m in self.modules():\n if isinstance(m, torch.nn.Conv2d):\n n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n m.weight.data.normal_(0, math.sqrt(2. / n))\n m.bias.data.fill_(0)\n\n def forward(self, x):\n avg_x = self.gap(x)\n out = []\n for aspp_idx in range(self.aspp_num):\n inp = avg_x if (aspp_idx == self.aspp_num - 1) else x\n out.append(F.relu_(self.aspp[aspp_idx](inp)))\n out[-1] = out[-1].expand_as(out[-2])\n out = torch.cat(out, dim=1)\n return out\n\n\nclass SPPCSPC(nn.Module):\n # CSP SPP https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, k=(5, 9, 13)):\n super(SPPCSPC, self).__init__()\n c_ = int(2 * c2 * e) # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c1, c_, 1, 1)\n self.cv3 = Conv(c_, c_, 3, 1)\n self.cv4 = Conv(c_, c_, 1, 1)\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\n self.cv5 = Conv(4 * c_, c_, 1, 1)\n self.cv6 = Conv(c_, c_, 3, 1)\n self.cv7 = Conv(2 * c_, c2, 1, 1)\n\n def forward(self, x):\n x1 = self.cv4(self.cv3(self.cv1(x)))\n y1 = self.cv6(self.cv5(torch.cat([x1] + [m(x1) for m in self.m], 1)))\n y2 = self.cv2(x)\n return self.cv7(torch.cat((y1, y2), dim=1))\n\n\nclass SPPF(nn.Module):\n # Spatial Pyramid Pooling - Fast (SPPF) layer by Glenn Jocher\n def __init__(self, c1, c2, k=5): # equivalent to SPP(k=(5, 9, 13))\n super().__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = Conv(c1, c_, 1, 1)\n self.cv2 = Conv(c_ * 4, c2, 1, 1)\n self.m = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2)\n # self.m = SoftPool2d(kernel_size=k, stride=1, padding=k // 2)\n\n def forward(self, x):\n x = self.cv1(x)\n with warnings.catch_warnings():\n warnings.simplefilter('ignore') # suppress torch 1.9.0 max_pool2d() warning\n y1 = self.m(x)\n y2 = self.m(y1)\n return self.cv2(torch.cat((x, y1, y2, self.m(y2)), 1))\n\n\nimport torch.nn.functional as F\nfrom torch.nn.modules.utils import _pair\n \n \nclass ReOrg(nn.Module):\n # yolo\n def __init__(self):\n super(ReOrg, self).__init__()\n\n def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2)\n return torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1)\n\n\nclass Contract(nn.Module):\n # Contract width-height into channels, i.e. x(1,64,80,80) to x(1,256,40,40)\n def __init__(self, gain=2):\n super().__init__()\n self.gain = gain\n\n def forward(self, x):\n b, c, h, w = x.size() # assert (h / s == 0) and (W / s == 0), 'Indivisible gain'\n s = self.gain\n x = x.view(b, c, h // s, s, w // s, s) # x(1,64,40,2,40,2)\n x = x.permute(0, 3, 5, 1, 2, 4).contiguous() # x(1,2,2,64,40,40)\n return x.view(b, c * s * s, h // s, w // s) # x(1,256,40,40)\n\n\nclass Expand(nn.Module):\n # Expand channels into width-height, i.e. x(1,64,80,80) to x(1,16,160,160)\n def __init__(self, gain=2):\n super().__init__()\n self.gain = gain\n\n def forward(self, x):\n b, c, h, w = x.size() # assert C / s ** 2 == 0, 'Indivisible gain'\n s = self.gain\n x = x.view(b, s, s, c // s ** 2, h, w) # x(1,2,2,16,80,80)\n x = x.permute(0, 3, 4, 1, 5, 2).contiguous() # x(1,16,80,2,80,2)\n return x.view(b, c // s ** 2, h * s, w * s) # x(1,16,160,160)\n\n\nclass Concat(nn.Module):\n # Concatenate a list of tensors along dimension\n def __init__(self, dimension=1):\n super().__init__()\n self.d = dimension\n\n def forward(self, x):\n return torch.cat(x, self.d)\n\n\nclass Shortcut(nn.Module):\n def __init__(self, dimension=0):\n super(Shortcut, self).__init__()\n self.d = dimension\n\n def forward(self, x):\n return x[0]+x[1]\n \n \nclass Silence(nn.Module):\n def __init__(self):\n super(Silence, self).__init__()\n def forward(self, x): \n return x\n\n\n##### GELAN ##### \n \nclass SPPELAN(nn.Module):\n # spp-elan\n def __init__(self, c1, c2, c3): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n self.c = c3\n self.cv1 = Conv(c1, c3, 1, 1)\n self.cv2 = SP(5)\n self.cv3 = SP(5)\n self.cv4 = SP(5)\n self.cv5 = Conv(4*c3, c2, 1, 1)\n\n def forward(self, x):\n y = [self.cv1(x)]\n y.extend(m(y[-1]) for m in [self.cv2, self.cv3, self.cv4])\n return self.cv5(torch.cat(y, 1))\n \n \nclass RepNCSPELAN4(nn.Module):\n # csp-elan\n def __init__(self, c1, c2, c3, c4, c5=1): # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n self.c = c3//2\n self.cv1 = Conv(c1, c3, 1, 1)\n self.cv2 = nn.Sequential(RepNCSP(c3//2, c4, c5), Conv(c4, c4, 3, 1))\n self.cv3 = nn.Sequential(RepNCSP(c4, c4, c5), Conv(c4, c4, 3, 1))\n self.cv4 = Conv(c3+(2*c4), c2, 1, 1)\n\n def forward(self, x):\n y = list(self.cv1(x).chunk(2, 1))\n y.extend((m(y[-1])) for m in [self.cv2, self.cv3])\n return self.cv4(torch.cat(y, 1))\n\n def forward_split(self, x):\n y = list(self.cv1(x).split((self.c, self.c), 1))\n y.extend(m(y[-1]) for m in [self.cv2, self.cv3])\n return self.cv4(torch.cat(y, 1))\n\n#################\n\n\n##### YOLOR #####\n\nclass ImplicitA(nn.Module):\n def __init__(self, channel):\n super(ImplicitA, self).__init__()\n self.channel = channel\n self.implicit = nn.Parameter(torch.zeros(1, channel, 1, 1))\n nn.init.normal_(self.implicit, std=.02) \n\n def forward(self, x):\n return self.implicit + x\n\n\nclass ImplicitM(nn.Module):\n def __init__(self, channel):\n super(ImplicitM, self).__init__()\n self.channel = channel\n self.implicit = nn.Parameter(torch.ones(1, channel, 1, 1))\n nn.init.normal_(self.implicit, mean=1., std=.02) \n\n def forward(self, x):\n return self.implicit * x\n\n#################\n\n\n##### CBNet #####\n\nclass CBLinear(nn.Module):\n def __init__(self, c1, c2s, k=1, s=1, p=None, g=1): # ch_in, ch_outs, kernel, stride, padding, groups\n super(CBLinear, self).__init__()\n self.c2s = c2s\n self.conv = nn.Conv2d(c1, sum(c2s), k, s, autopad(k, p), groups=g, bias=True)\n\n def forward(self, x):\n outs = self.conv(x).split(self.c2s, dim=1)\n return outs\n\nclass CBFuse(nn.Module):\n def __init__(self, idx):\n super(CBFuse, self).__init__()\n self.idx = idx\n\n def forward(self, xs):\n target_size = xs[-1].shape[2:]\n res = [F.interpolate(x[self.idx[i]], size=target_size, mode='nearest') for i, x in enumerate(xs[:-1])]\n out = torch.sum(torch.stack(res + xs[-1:]), dim=0)\n return out\n\n#################\n\n\nclass DetectMultiBackend(nn.Module):\n # YOLO MultiBackend class for python inference on various backends\n def __init__(self, weights='yolo.pt', device=torch.device('cpu'), dnn=False, data=None, fp16=False, fuse=True):\n # Usage:\n # PyTorch: weights = *.pt\n # TorchScript: *.torchscript\n # ONNX Runtime: *.onnx\n # ONNX OpenCV DNN: *.onnx --dnn\n # OpenVINO: *_openvino_model\n # CoreML: *.mlmodel\n # TensorRT: *.engine\n # TensorFlow SavedModel: *_saved_model\n # TensorFlow GraphDef: *.pb\n # TensorFlow Lite: *.tflite\n # TensorFlow Edge TPU: *_edgetpu.tflite\n # PaddlePaddle: *_paddle_model\n from asone.detectors.yolov9.yolov9.models.experimental import attempt_download, attempt_load # scoped to avoid circular import\n\n super().__init__()\n w = str(weights[0] if isinstance(weights, list) else weights)\n pt, jit, onnx, onnx_end2end, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle, triton = self._model_type(w)\n fp16 &= pt or jit or onnx or engine # FP16\n nhwc = coreml or saved_model or pb or tflite or edgetpu # BHWC formats (vs torch BCWH)\n stride = 32 # default stride\n cuda = torch.cuda.is_available() and device.type != 'cpu' # use CUDA\n if not (pt or triton):\n w = attempt_download(w) # download if not local\n\n if pt: # PyTorch\n model = attempt_load(weights if isinstance(weights, list) else w, device=device, inplace=True, fuse=fuse)\n stride = max(int(model.stride.max()), 32) # model stride\n names = model.module.names if hasattr(model, 'module') else model.names # get class names\n model.half() if fp16 else model.float()\n self.model = model # explicitly assign for to(), cpu(), cuda(), half()\n elif jit: # TorchScript\n LOGGER.info(f'Loading {w} for TorchScript inference...')\n extra_files = {'config.txt': ''} # model metadata\n model = torch.jit.load(w, _extra_files=extra_files, map_location=device)\n model.half() if fp16 else model.float()\n if extra_files['config.txt']: # load metadata dict\n d = json.loads(extra_files['config.txt'],\n object_hook=lambda d: {int(k) if k.isdigit() else k: v\n for k, v in d.items()})\n stride, names = int(d['stride']), d['names']\n elif dnn: # ONNX OpenCV DNN\n LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...')\n check_requirements('opencv-python>=4.5.4')\n net = cv2.dnn.readNetFromONNX(w)\n elif onnx: # ONNX Runtime\n LOGGER.info(f'Loading {w} for ONNX Runtime inference...')\n check_requirements(('onnx', 'onnxruntime-gpu' if cuda else 'onnxruntime'))\n import onnxruntime\n providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if cuda else ['CPUExecutionProvider']\n session = onnxruntime.InferenceSession(w, providers=providers)\n output_names = [x.name for x in session.get_outputs()]\n meta = session.get_modelmeta().custom_metadata_map # metadata\n if 'stride' in meta:\n stride, names = int(meta['stride']), eval(meta['names'])\n elif xml: # OpenVINO\n LOGGER.info(f'Loading {w} for OpenVINO inference...')\n check_requirements('openvino') # requires openvino-dev: https://pypi.org/project/openvino-dev/\n from openvino.runtime import Core, Layout, get_batch\n ie = Core()\n if not Path(w).is_file(): # if not *.xml\n w = next(Path(w).glob('*.xml')) # get *.xml file from *_openvino_model dir\n network = ie.read_model(model=w, weights=Path(w).with_suffix('.bin'))\n if network.get_parameters()[0].get_layout().empty:\n network.get_parameters()[0].set_layout(Layout(\"NCHW\"))\n batch_dim = get_batch(network)\n if batch_dim.is_static:\n batch_size = batch_dim.get_length()\n executable_network = ie.compile_model(network, device_name=\"CPU\") # device_name=\"MYRIAD\" for Intel NCS2\n stride, names = self._load_metadata(Path(w).with_suffix('.yaml')) # load metadata\n elif engine: # TensorRT\n LOGGER.info(f'Loading {w} for TensorRT inference...')\n import tensorrt as trt # https://developer.nvidia.com/nvidia-tensorrt-download\n check_version(trt.__version__, '7.0.0', hard=True) # require tensorrt>=7.0.0\n if device.type == 'cpu':\n device = torch.device('cuda:0')\n Binding = namedtuple('Binding', ('name', 'dtype', 'shape', 'data', 'ptr'))\n logger = trt.Logger(trt.Logger.INFO)\n with open(w, 'rb') as f, trt.Runtime(logger) as runtime:\n model = runtime.deserialize_cuda_engine(f.read())\n context = model.create_execution_context()\n bindings = OrderedDict()\n output_names = []\n fp16 = False # default updated below\n dynamic = False\n for i in range(model.num_bindings):\n name = model.get_binding_name(i)\n dtype = trt.nptype(model.get_binding_dtype(i))\n if model.binding_is_input(i):\n if -1 in tuple(model.get_binding_shape(i)): # dynamic\n dynamic = True\n context.set_binding_shape(i, tuple(model.get_profile_shape(0, i)[2]))\n if dtype == np.float16:\n fp16 = True\n else: # output\n output_names.append(name)\n shape = tuple(context.get_binding_shape(i))\n im = torch.from_numpy(np.empty(shape, dtype=dtype)).to(device)\n bindings[name] = Binding(name, dtype, shape, im, int(im.data_ptr()))\n binding_addrs = OrderedDict((n, d.ptr) for n, d in bindings.items())\n batch_size = bindings['images'].shape[0] # if dynamic, this is instead max batch size\n elif coreml: # CoreML\n LOGGER.info(f'Loading {w} for CoreML inference...')\n import coremltools as ct\n model = ct.models.MLModel(w)\n elif saved_model: # TF SavedModel\n LOGGER.info(f'Loading {w} for TensorFlow SavedModel inference...')\n import tensorflow as tf\n keras = False # assume TF1 saved_model\n model = tf.keras.models.load_model(w) if keras else tf.saved_model.load(w)\n elif pb: # GraphDef https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt\n LOGGER.info(f'Loading {w} for TensorFlow GraphDef inference...')\n import tensorflow as tf\n\n def wrap_frozen_graph(gd, inputs, outputs):\n x = tf.compat.v1.wrap_function(lambda: tf.compat.v1.import_graph_def(gd, name=\"\"), []) # wrapped\n ge = x.graph.as_graph_element\n return x.prune(tf.nest.map_structure(ge, inputs), tf.nest.map_structure(ge, outputs))\n\n def gd_outputs(gd):\n name_list, input_list = [], []\n for node in gd.node: # tensorflow.core.framework.node_def_pb2.NodeDef\n name_list.append(node.name)\n input_list.extend(node.input)\n return sorted(f'{x}:0' for x in list(set(name_list) - set(input_list)) if not x.startswith('NoOp'))\n\n gd = tf.Graph().as_graph_def() # TF GraphDef\n with open(w, 'rb') as f:\n gd.ParseFromString(f.read())\n frozen_func = wrap_frozen_graph(gd, inputs=\"x:0\", outputs=gd_outputs(gd))\n elif tflite or edgetpu: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python\n try: # https://coral.ai/docs/edgetpu/tflite-python/#update-existing-tf-lite-code-for-the-edge-tpu\n from tflite_runtime.interpreter import Interpreter, load_delegate\n except ImportError:\n import tensorflow as tf\n Interpreter, load_delegate = tf.lite.Interpreter, tf.lite.experimental.load_delegate,\n if edgetpu: # TF Edge TPU https://coral.ai/software/#edgetpu-runtime\n LOGGER.info(f'Loading {w} for TensorFlow Lite Edge TPU inference...')\n delegate = {\n 'Linux': 'libedgetpu.so.1',\n 'Darwin': 'libedgetpu.1.dylib',\n 'Windows': 'edgetpu.dll'}[platform.system()]\n interpreter = Interpreter(model_path=w, experimental_delegates=[load_delegate(delegate)])\n else: # TFLite\n LOGGER.info(f'Loading {w} for TensorFlow Lite inference...')\n interpreter = Interpreter(model_path=w) # load TFLite model\n interpreter.allocate_tensors() # allocate\n input_details = interpreter.get_input_details() # inputs\n output_details = interpreter.get_output_details() # outputs\n # load metadata\n with contextlib.suppress(zipfile.BadZipFile):\n with zipfile.ZipFile(w, \"r\") as model:\n meta_file = model.namelist()[0]\n meta = ast.literal_eval(model.read(meta_file).decode(\"utf-8\"))\n stride, names = int(meta['stride']), meta['names']\n elif tfjs: # TF.js\n raise NotImplementedError('ERROR: YOLO TF.js inference is not supported')\n elif paddle: # PaddlePaddle\n LOGGER.info(f'Loading {w} for PaddlePaddle inference...')\n check_requirements('paddlepaddle-gpu' if cuda else 'paddlepaddle')\n import paddle.inference as pdi\n if not Path(w).is_file(): # if not *.pdmodel\n w = next(Path(w).rglob('*.pdmodel')) # get *.pdmodel file from *_paddle_model dir\n weights = Path(w).with_suffix('.pdiparams')\n config = pdi.Config(str(w), str(weights))\n if cuda:\n config.enable_use_gpu(memory_pool_init_size_mb=2048, device_id=0)\n predictor = pdi.create_predictor(config)\n input_handle = predictor.get_input_handle(predictor.get_input_names()[0])\n output_names = predictor.get_output_names()\n elif triton: # NVIDIA Triton Inference Server\n LOGGER.info(f'Using {w} as Triton Inference Server...')\n check_requirements('tritonclient[all]')\n from asone.detectors.yolov9.yolov9.utils.triton import TritonRemoteModel\n model = TritonRemoteModel(url=w)\n nhwc = model.runtime.startswith(\"tensorflow\")\n else:\n raise NotImplementedError(f'ERROR: {w} is not a supported format')\n\n # class names\n if 'names' not in locals():\n names = yaml_load(data)['names'] if data else {i: f'class{i}' for i in range(999)}\n if names[0] == 'n01440764' and len(names) == 1000: # ImageNet\n names = yaml_load(ROOT / 'data/ImageNet.yaml')['names'] # human-readable names\n\n self.__dict__.update(locals()) # assign all variables to self\n\n def forward(self, im, augment=False, visualize=False):\n # YOLO MultiBackend inference\n b, ch, h, w = im.shape # batch, channel, height, width\n if self.fp16 and im.dtype != torch.float16:\n im = im.half() # to FP16\n if self.nhwc:\n im = im.permute(0, 2, 3, 1) # torch BCHW to numpy BHWC shape(1,320,192,3)\n\n if self.pt: # PyTorch\n y = self.model(im, augment=augment, visualize=visualize) if augment or visualize else self.model(im)\n elif self.jit: # TorchScript\n y = self.model(im)\n elif self.dnn: # ONNX OpenCV DNN\n im = im.cpu().numpy() # torch to numpy\n self.net.setInput(im)\n y = self.net.forward()\n elif self.onnx: # ONNX Runtime\n im = im.cpu().numpy() # torch to numpy\n y = self.session.run(self.output_names, {self.session.get_inputs()[0].name: im})\n elif self.xml: # OpenVINO\n im = im.cpu().numpy() # FP32\n y = list(self.executable_network([im]).values())\n elif self.engine: # TensorRT\n if self.dynamic and im.shape != self.bindings['images'].shape:\n i = self.model.get_binding_index('images')\n self.context.set_binding_shape(i, im.shape) # reshape if dynamic\n self.bindings['images'] = self.bindings['images']._replace(shape=im.shape)\n for name in self.output_names:\n i = self.model.get_binding_index(name)\n self.bindings[name].data.resize_(tuple(self.context.get_binding_shape(i)))\n s = self.bindings['images'].shape\n assert im.shape == s, f\"input size {im.shape} {'>' if self.dynamic else 'not equal to'} max model size {s}\"\n self.binding_addrs['images'] = int(im.data_ptr())\n self.context.execute_v2(list(self.binding_addrs.values()))\n y = [self.bindings[x].data for x in sorted(self.output_names)]\n elif self.coreml: # CoreML\n im = im.cpu().numpy()\n im = Image.fromarray((im[0] * 255).astype('uint8'))\n # im = im.resize((192, 320), Image.ANTIALIAS)\n y = self.model.predict({'image': im}) # coordinates are xywh normalized\n if 'confidence' in y:\n box = xywh2xyxy(y['coordinates'] * [[w, h, w, h]]) # xyxy pixels\n conf, cls = y['confidence'].max(1), y['confidence'].argmax(1).astype(np.float)\n y = np.concatenate((box, conf.reshape(-1, 1), cls.reshape(-1, 1)), 1)\n else:\n y = list(reversed(y.values())) # reversed for segmentation models (pred, proto)\n elif self.paddle: # PaddlePaddle\n im = im.cpu().numpy().astype(np.float32)\n self.input_handle.copy_from_cpu(im)\n self.predictor.run()\n y = [self.predictor.get_output_handle(x).copy_to_cpu() for x in self.output_names]\n elif self.triton: # NVIDIA Triton Inference Server\n y = self.model(im)\n else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU)\n im = im.cpu().numpy()\n if self.saved_model: # SavedModel\n y = self.model(im, training=False) if self.keras else self.model(im)\n elif self.pb: # GraphDef\n y = self.frozen_func(x=self.tf.constant(im))\n else: # Lite or Edge TPU\n input = self.input_details[0]\n int8 = input['dtype'] == np.uint8 # is TFLite quantized uint8 model\n if int8:\n scale, zero_point = input['quantization']\n im = (im / scale + zero_point).astype(np.uint8) # de-scale\n self.interpreter.set_tensor(input['index'], im)\n self.interpreter.invoke()\n y = []\n for output in self.output_details:\n x = self.interpreter.get_tensor(output['index'])\n if int8:\n scale, zero_point = output['quantization']\n x = (x.astype(np.float32) - zero_point) * scale # re-scale\n y.append(x)\n y = [x if isinstance(x, np.ndarray) else x.numpy() for x in y]\n y[0][..., :4] *= [w, h, w, h] # xywh normalized to pixels\n\n if isinstance(y, (list, tuple)):\n return self.from_numpy(y[0]) if len(y) == 1 else [self.from_numpy(x) for x in y]\n else:\n return self.from_numpy(y)\n\n def from_numpy(self, x):\n return torch.from_numpy(x).to(self.device) if isinstance(x, np.ndarray) else x\n\n def warmup(self, imgsz=(1, 3, 640, 640)):\n # Warmup model by running inference once\n warmup_types = self.pt, self.jit, self.onnx, self.engine, self.saved_model, self.pb, self.triton\n if any(warmup_types) and (self.device.type != 'cpu' or self.triton):\n im = torch.empty(*imgsz, dtype=torch.half if self.fp16 else torch.float, device=self.device) # input\n for _ in range(2 if self.jit else 1): #\n self.forward(im) # warmup\n\n @staticmethod\n def _model_type(p='path/to/model.pt'):\n # Return model type from model path, i.e. path='path/to/model.onnx' -> type=onnx\n # types = [pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle]\n from asone.detectors.yolov9.export import export_formats\n from asone.detectors.yolov9.yolov9.utils.downloads import is_url\n sf = list(export_formats().Suffix) # export suffixes\n if not is_url(p, check=False):\n check_suffix(p, sf) # checks\n url = urlparse(p) # if url may be Triton inference server\n types = [s in Path(p).name for s in sf]\n types[8] &= not types[9] # tflite &= not edgetpu\n triton = not any(types) and all([any(s in url.scheme for s in [\"http\", \"grpc\"]), url.netloc])\n return types + [triton]\n\n @staticmethod\n def _load_metadata(f=Path('path/to/meta.yaml')):\n # Load metadata from meta.yaml if it exists\n if f.exists():\n d = yaml_load(f)\n return d['stride'], d['names'] # assign stride, names\n return None, None\n\n\nclass AutoShape(nn.Module):\n # YOLO input-robust model wrapper for passing cv2/np/PIL/torch inputs. Includes preprocessing, inference and NMS\n conf = 0.25 # NMS confidence threshold\n iou = 0.45 # NMS IoU threshold\n agnostic = False # NMS class-agnostic\n multi_label = False # NMS multiple labels per box\n classes = None # (optional list) filter by class, i.e. = [0, 15, 16] for COCO persons, cats and dogs\n max_det = 1000 # maximum number of detections per image\n amp = False # Automatic Mixed Precision (AMP) inference\n\n def __init__(self, model, verbose=True):\n super().__init__()\n if verbose:\n LOGGER.info('Adding AutoShape... ')\n copy_attr(self, model, include=('yaml', 'nc', 'hyp', 'names', 'stride', 'abc'), exclude=()) # copy attributes\n self.dmb = isinstance(model, DetectMultiBackend) # DetectMultiBackend() instance\n self.pt = not self.dmb or model.pt # PyTorch model\n self.model = model.eval()\n if self.pt:\n m = self.model.model.model[-1] if self.dmb else self.model.model[-1] # Detect()\n m.inplace = False # Detect.inplace=False for safe multithread inference\n m.export = True # do not output loss values\n\n def _apply(self, fn):\n # Apply to(), cpu(), cuda(), half() to model tensors that are not parameters or registered buffers\n self = super()._apply(fn)\n from asone.detectors.yolov9.yolov9.models.yolo import Detect, Segment\n if self.pt:\n m = self.model.model.model[-1] if self.dmb else self.model.model[-1] # Detect()\n if isinstance(m, (Detect, Segment)):\n for k in 'stride', 'anchor_grid', 'stride_grid', 'grid':\n x = getattr(m, k)\n setattr(m, k, list(map(fn, x))) if isinstance(x, (list, tuple)) else setattr(m, k, fn(x))\n return self\n\n @smart_inference_mode()\n def forward(self, ims, size=640, augment=False, profile=False):\n # Inference from various sources. For size(height=640, width=1280), RGB images example inputs are:\n # file: ims = 'data/images/zidane.jpg' # str or PosixPath\n # URI: = 'https://ultralytics.com/images/zidane.jpg'\n # OpenCV: = cv2.imread('image.jpg')[:,:,::-1] # HWC BGR to RGB x(640,1280,3)\n # PIL: = Image.open('image.jpg') or ImageGrab.grab() # HWC x(640,1280,3)\n # numpy: = np.zeros((640,1280,3)) # HWC\n # torch: = torch.zeros(16,3,320,640) # BCHW (scaled to size=640, 0-1 values)\n # multiple: = [Image.open('image1.jpg'), Image.open('image2.jpg'), ...] # list of images\n\n dt = (Profile(), Profile(), Profile())\n with dt[0]:\n if isinstance(size, int): # expand\n size = (size, size)\n p = next(self.model.parameters()) if self.pt else torch.empty(1, device=self.model.device) # param\n autocast = self.amp and (p.device.type != 'cpu') # Automatic Mixed Precision (AMP) inference\n if isinstance(ims, torch.Tensor): # torch\n with amp.autocast(autocast):\n return self.model(ims.to(p.device).type_as(p), augment=augment) # inference\n\n # Pre-process\n n, ims = (len(ims), list(ims)) if isinstance(ims, (list, tuple)) else (1, [ims]) # number, list of images\n shape0, shape1, files = [], [], [] # image and inference shapes, filenames\n for i, im in enumerate(ims):\n f = f'image{i}' # filename\n if isinstance(im, (str, Path)): # filename or uri\n im, f = Image.open(requests.get(im, stream=True).raw if str(im).startswith('http') else im), im\n im = np.asarray(exif_transpose(im))\n elif isinstance(im, Image.Image): # PIL Image\n im, f = np.asarray(exif_transpose(im)), getattr(im, 'filename', f) or f\n files.append(Path(f).with_suffix('.jpg').name)\n if im.shape[0] < 5: # image in CHW\n im = im.transpose((1, 2, 0)) # reverse dataloader .transpose(2, 0, 1)\n im = im[..., :3] if im.ndim == 3 else cv2.cvtColor(im, cv2.COLOR_GRAY2BGR) # enforce 3ch input\n s = im.shape[:2] # HWC\n shape0.append(s) # image shape\n g = max(size) / max(s) # gain\n shape1.append([int(y * g) for y in s])\n ims[i] = im if im.data.contiguous else np.ascontiguousarray(im) # update\n shape1 = [make_divisible(x, self.stride) for x in np.array(shape1).max(0)] # inf shape\n x = [letterbox(im, shape1, auto=False)[0] for im in ims] # pad\n x = np.ascontiguousarray(np.array(x).transpose((0, 3, 1, 2))) # stack and BHWC to BCHW\n x = torch.from_numpy(x).to(p.device).type_as(p) / 255 # uint8 to fp16/32\n\n with amp.autocast(autocast):\n # Inference\n with dt[1]:\n y = self.model(x, augment=augment) # forward\n\n # Post-process\n with dt[2]:\n y = non_max_suppression(y if self.dmb else y[0],\n self.conf,\n self.iou,\n self.classes,\n self.agnostic,\n self.multi_label,\n max_det=self.max_det) # NMS\n for i in range(n):\n scale_boxes(shape1, y[i][:, :4], shape0[i])\n\n return Detections(ims, y, files, dt, self.names, x.shape)\n\n\nclass Detections:\n # YOLO detections class for inference results\n def __init__(self, ims, pred, files, times=(0, 0, 0), names=None, shape=None):\n super().__init__()\n d = pred[0].device # device\n gn = [torch.tensor([*(im.shape[i] for i in [1, 0, 1, 0]), 1, 1], device=d) for im in ims] # normalizations\n self.ims = ims # list of images as numpy arrays\n self.pred = pred # list of tensors pred[0] = (xyxy, conf, cls)\n self.names = names # class names\n self.files = files # image filenames\n self.times = times # profiling times\n self.xyxy = pred # xyxy pixels\n self.xywh = [xyxy2xywh(x) for x in pred] # xywh pixels\n self.xyxyn = [x / g for x, g in zip(self.xyxy, gn)] # xyxy normalized\n self.xywhn = [x / g for x, g in zip(self.xywh, gn)] # xywh normalized\n self.n = len(self.pred) # number of images (batch size)\n self.t = tuple(x.t / self.n * 1E3 for x in times) # timestamps (ms)\n self.s = tuple(shape) # inference BCHW shape\n\n def _run(self, pprint=False, show=False, save=False, crop=False, render=False, labels=True, save_dir=Path('')):\n s, crops = '', []\n for i, (im, pred) in enumerate(zip(self.ims, self.pred)):\n s += f'\\nimage {i + 1}/{len(self.pred)}: {im.shape[0]}x{im.shape[1]} ' # string\n if pred.shape[0]:\n for c in pred[:, -1].unique():\n n = (pred[:, -1] == c).sum() # detections per class\n s += f\"{n} {self.names[int(c)]}{'s' * (n > 1)}, \" # add to string\n s = s.rstrip(', ')\n if show or save or render or crop:\n annotator = Annotator(im, example=str(self.names))\n for *box, conf, cls in reversed(pred): # xyxy, confidence, class\n label = f'{self.names[int(cls)]} {conf:.2f}'\n if crop:\n file = save_dir / 'crops' / self.names[int(cls)] / self.files[i] if save else None\n crops.append({\n 'box': box,\n 'conf': conf,\n 'cls': cls,\n 'label': label,\n 'im': save_one_box(box, im, file=file, save=save)})\n else: # all others\n annotator.box_label(box, label if labels else '', color=colors(cls))\n im = annotator.im\n else:\n s += '(no detections)'\n\n im = Image.fromarray(im.astype(np.uint8)) if isinstance(im, np.ndarray) else im # from np\n if show:\n display(im) if is_notebook() else im.show(self.files[i])\n if save:\n f = self.files[i]\n im.save(save_dir / f) # save\n if i == self.n - 1:\n LOGGER.info(f\"Saved {self.n} image{'s' * (self.n > 1)} to {colorstr('bold', save_dir)}\")\n if render:\n self.ims[i] = np.asarray(im)\n if pprint:\n s = s.lstrip('\\n')\n return f'{s}\\nSpeed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {self.s}' % self.t\n if crop:\n if save:\n LOGGER.info(f'Saved results to {save_dir}\\n')\n return crops\n\n @TryExcept('Showing images is not supported in this environment')\n def show(self, labels=True):\n self._run(show=True, labels=labels) # show results\n\n def save(self, labels=True, save_dir='runs/detect/exp', exist_ok=False):\n save_dir = increment_path(save_dir, exist_ok, mkdir=True) # increment save_dir\n self._run(save=True, labels=labels, save_dir=save_dir) # save results\n\n def crop(self, save=True, save_dir='runs/detect/exp', exist_ok=False):\n save_dir = increment_path(save_dir, exist_ok, mkdir=True) if save else None\n return self._run(crop=True, save=save, save_dir=save_dir) # crop results\n\n def render(self, labels=True):\n self._run(render=True, labels=labels) # render results\n return self.ims\n\n def pandas(self):\n # return detections as pandas DataFrames, i.e. print(results.pandas().xyxy[0])\n new = copy(self) # return copy\n ca = 'xmin', 'ymin', 'xmax', 'ymax', 'confidence', 'class', 'name' # xyxy columns\n cb = 'xcenter', 'ycenter', 'width', 'height', 'confidence', 'class', 'name' # xywh columns\n for k, c in zip(['xyxy', 'xyxyn', 'xywh', 'xywhn'], [ca, ca, cb, cb]):\n a = [[x[:5] + [int(x[5]), self.names[int(x[5])]] for x in x.tolist()] for x in getattr(self, k)] # update\n setattr(new, k, [pd.DataFrame(x, columns=c) for x in a])\n return new\n\n def tolist(self):\n # return a list of Detections objects, i.e. 'for result in results.tolist():'\n r = range(self.n) # iterable\n x = [Detections([self.ims[i]], [self.pred[i]], [self.files[i]], self.times, self.names, self.s) for i in r]\n # for d in x:\n # for k in ['ims', 'pred', 'xyxy', 'xyxyn', 'xywh', 'xywhn']:\n # setattr(d, k, getattr(d, k)[0]) # pop out of list\n return x\n\n def print(self):\n LOGGER.info(self.__str__())\n\n def __len__(self): # override len(results)\n return self.n\n\n def __str__(self): # override print(results)\n return self._run(pprint=True) # print results\n\n def __repr__(self):\n return f'YOLO {self.__class__} instance\\n' + self.__str__()\n\n\nclass Proto(nn.Module):\n # YOLO mask Proto module for segmentation models\n def __init__(self, c1, c_=256, c2=32): # ch_in, number of protos, number of masks\n super().__init__()\n self.cv1 = Conv(c1, c_, k=3)\n self.upsample = nn.Upsample(scale_factor=2, mode='nearest')\n self.cv2 = Conv(c_, c_, k=3)\n self.cv3 = Conv(c_, c2)\n\n def forward(self, x):\n return self.cv3(self.cv2(self.upsample(self.cv1(x))))\n\n\nclass UConv(nn.Module):\n def __init__(self, c1, c_=256, c2=256): # ch_in, number of protos, number of masks\n super().__init__()\n \n self.cv1 = Conv(c1, c_, k=3)\n self.cv2 = nn.Conv2d(c_, c2, 1, 1)\n self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n\n def forward(self, x):\n return self.up(self.cv2(self.cv1(x)))\n\n\nclass Classify(nn.Module):\n # YOLO classification head, i.e. x(b,c1,20,20) to x(b,c2)\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1): # ch_in, ch_out, kernel, stride, padding, groups\n super().__init__()\n c_ = 1280 # efficientnet_b0 size\n self.conv = Conv(c1, c_, k, s, autopad(k, p), g)\n self.pool = nn.AdaptiveAvgPool2d(1) # to x(b,c_,1,1)\n self.drop = nn.Dropout(p=0.0, inplace=True)\n self.linear = nn.Linear(c_, c2) # to x(b,c2)\n\n def forward(self, x):\n if isinstance(x, list):\n x = torch.cat(x, 1)\n return self.linear(self.drop(self.pool(self.conv(x)).flatten(1)))\n" }, { "path": "asone/detectors/yolov9/yolov9/models/experimental.py", "content": "import math\nimport random\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\n\nfrom asone.detectors.yolov9.yolov9.utils.downloads import attempt_download\n\n\nclass Sum(nn.Module):\n # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\n def __init__(self, n, weight=False): # n: number of inputs\n super().__init__()\n self.weight = weight # apply weights boolean\n self.iter = range(n - 1) # iter object\n if weight:\n self.w = nn.Parameter(-torch.arange(1.0, n) / 2, requires_grad=True) # layer weights\n\n def forward(self, x):\n y = x[0] # no weight\n if self.weight:\n w = torch.sigmoid(self.w) * 2\n for i in self.iter:\n y = y + x[i + 1] * w[i]\n else:\n for i in self.iter:\n y = y + x[i + 1]\n return y\n\n\nclass MixConv2d(nn.Module):\n # Mixed Depth-wise Conv https://arxiv.org/abs/1907.09595\n def __init__(self, c1, c2, k=(1, 3), s=1, equal_ch=True): # ch_in, ch_out, kernel, stride, ch_strategy\n super().__init__()\n n = len(k) # number of convolutions\n if equal_ch: # equal c_ per group\n i = torch.linspace(0, n - 1E-6, c2).floor() # c2 indices\n c_ = [(i == g).sum() for g in range(n)] # intermediate channels\n else: # equal weight.numel() per group\n b = [c2] + [0] * n\n a = np.eye(n + 1, n, k=-1)\n a -= np.roll(a, 1, axis=1)\n a *= np.array(k) ** 2\n a[0] = 1\n c_ = np.linalg.lstsq(a, b, rcond=None)[0].round() # solve for equal weight indices, ax = b\n\n self.m = nn.ModuleList([\n nn.Conv2d(c1, int(c_), k, s, k // 2, groups=math.gcd(c1, int(c_)), bias=False) for k, c_ in zip(k, c_)])\n self.bn = nn.BatchNorm2d(c2)\n self.act = nn.SiLU()\n\n def forward(self, x):\n return self.act(self.bn(torch.cat([m(x) for m in self.m], 1)))\n\n\nclass Ensemble(nn.ModuleList):\n # Ensemble of models\n def __init__(self):\n super().__init__()\n\n def forward(self, x, augment=False, profile=False, visualize=False):\n y = [module(x, augment, profile, visualize)[0] for module in self]\n # y = torch.stack(y).max(0)[0] # max ensemble\n # y = torch.stack(y).mean(0) # mean ensemble\n y = torch.cat(y, 1) # nms ensemble\n return y, None # inference, train output\n\n\nclass ORT_NMS(torch.autograd.Function):\n '''ONNX-Runtime NMS operation'''\n @staticmethod\n def forward(ctx,\n boxes,\n scores,\n max_output_boxes_per_class=torch.tensor([100]),\n iou_threshold=torch.tensor([0.45]),\n score_threshold=torch.tensor([0.25])):\n device = boxes.device\n batch = scores.shape[0]\n num_det = random.randint(0, 100)\n batches = torch.randint(0, batch, (num_det,)).sort()[0].to(device)\n idxs = torch.arange(100, 100 + num_det).to(device)\n zeros = torch.zeros((num_det,), dtype=torch.int64).to(device)\n selected_indices = torch.cat([batches[None], zeros[None], idxs[None]], 0).T.contiguous()\n selected_indices = selected_indices.to(torch.int64)\n return selected_indices\n\n @staticmethod\n def symbolic(g, boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold):\n return g.op(\"NonMaxSuppression\", boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold)\n\n\nclass TRT_NMS(torch.autograd.Function):\n '''TensorRT NMS operation'''\n @staticmethod\n def forward(\n ctx,\n boxes,\n scores,\n background_class=-1,\n box_coding=1,\n iou_threshold=0.45,\n max_output_boxes=100,\n plugin_version=\"1\",\n score_activation=0,\n score_threshold=0.25,\n ):\n\n batch_size, num_boxes, num_classes = scores.shape\n num_det = torch.randint(0, max_output_boxes, (batch_size, 1), dtype=torch.int32)\n det_boxes = torch.randn(batch_size, max_output_boxes, 4)\n det_scores = torch.randn(batch_size, max_output_boxes)\n det_classes = torch.randint(0, num_classes, (batch_size, max_output_boxes), dtype=torch.int32)\n return num_det, det_boxes, det_scores, det_classes\n\n @staticmethod\n def symbolic(g,\n boxes,\n scores,\n background_class=-1,\n box_coding=1,\n iou_threshold=0.45,\n max_output_boxes=100,\n plugin_version=\"1\",\n score_activation=0,\n score_threshold=0.25):\n out = g.op(\"TRT::EfficientNMS_TRT\",\n boxes,\n scores,\n background_class_i=background_class,\n box_coding_i=box_coding,\n iou_threshold_f=iou_threshold,\n max_output_boxes_i=max_output_boxes,\n plugin_version_s=plugin_version,\n score_activation_i=score_activation,\n score_threshold_f=score_threshold,\n outputs=4)\n nums, boxes, scores, classes = out\n return nums, boxes, scores, classes\n\n\nclass ONNX_ORT(nn.Module):\n '''onnx module with ONNX-Runtime NMS operation.'''\n def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=640, device=None, n_classes=80):\n super().__init__()\n self.device = device if device else torch.device(\"cpu\")\n self.max_obj = torch.tensor([max_obj]).to(device)\n self.iou_threshold = torch.tensor([iou_thres]).to(device)\n self.score_threshold = torch.tensor([score_thres]).to(device)\n self.max_wh = max_wh # if max_wh != 0 : non-agnostic else : agnostic\n self.convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],\n dtype=torch.float32,\n device=self.device)\n self.n_classes=n_classes\n\n def forward(self, x):\n ## https://github.com/thaitc-hust/yolov9-tensorrt/blob/main/torch2onnx.py\n ## thanks https://github.com/thaitc-hust\n if isinstance(x, list): ## yolov9-c.pt and yolov9-e.pt return list\n x = x[1]\n x = x.permute(0, 2, 1)\n bboxes_x = x[..., 0:1]\n bboxes_y = x[..., 1:2]\n bboxes_w = x[..., 2:3]\n bboxes_h = x[..., 3:4]\n bboxes = torch.cat([bboxes_x, bboxes_y, bboxes_w, bboxes_h], dim = -1)\n bboxes = bboxes.unsqueeze(2) # [n_batch, n_bboxes, 4] -> [n_batch, n_bboxes, 1, 4]\n obj_conf = x[..., 4:]\n scores = obj_conf\n bboxes @= self.convert_matrix\n max_score, category_id = scores.max(2, keepdim=True)\n dis = category_id.float() * self.max_wh\n nmsbox = bboxes + dis\n max_score_tp = max_score.transpose(1, 2).contiguous()\n selected_indices = ORT_NMS.apply(nmsbox, max_score_tp, self.max_obj, self.iou_threshold, self.score_threshold)\n X, Y = selected_indices[:, 0], selected_indices[:, 2]\n selected_boxes = bboxes[X, Y, :]\n selected_categories = category_id[X, Y, :].float()\n selected_scores = max_score[X, Y, :]\n X = X.unsqueeze(1).float()\n return torch.cat([X, selected_boxes, selected_categories, selected_scores], 1)\n\n\nclass ONNX_TRT(nn.Module):\n '''onnx module with TensorRT NMS operation.'''\n def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None ,device=None, n_classes=80):\n super().__init__()\n assert max_wh is None\n self.device = device if device else torch.device('cpu')\n self.background_class = -1,\n self.box_coding = 1,\n self.iou_threshold = iou_thres\n self.max_obj = max_obj\n self.plugin_version = '1'\n self.score_activation = 0\n self.score_threshold = score_thres\n self.n_classes=n_classes\n\n def forward(self, x):\n ## https://github.com/thaitc-hust/yolov9-tensorrt/blob/main/torch2onnx.py\n ## thanks https://github.com/thaitc-hust\n if isinstance(x, list): ## yolov9-c.pt and yolov9-e.pt return list\n x = x[1]\n x = x.permute(0, 2, 1)\n bboxes_x = x[..., 0:1]\n bboxes_y = x[..., 1:2]\n bboxes_w = x[..., 2:3]\n bboxes_h = x[..., 3:4]\n bboxes = torch.cat([bboxes_x, bboxes_y, bboxes_w, bboxes_h], dim = -1)\n bboxes = bboxes.unsqueeze(2) # [n_batch, n_bboxes, 4] -> [n_batch, n_bboxes, 1, 4]\n obj_conf = x[..., 4:]\n scores = obj_conf\n num_det, det_boxes, det_scores, det_classes = TRT_NMS.apply(bboxes, scores, self.background_class, self.box_coding,\n self.iou_threshold, self.max_obj,\n self.plugin_version, self.score_activation,\n self.score_threshold)\n return num_det, det_boxes, det_scores, det_classes\n\nclass End2End(nn.Module):\n '''export onnx or tensorrt model with NMS operation.'''\n def __init__(self, model, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None, device=None, n_classes=80):\n super().__init__()\n device = device if device else torch.device('cpu')\n assert isinstance(max_wh,(int)) or max_wh is None\n self.model = model.to(device)\n self.model.model[-1].end2end = True\n self.patch_model = ONNX_TRT if max_wh is None else ONNX_ORT\n self.end2end = self.patch_model(max_obj, iou_thres, score_thres, max_wh, device, n_classes)\n self.end2end.eval()\n\n def forward(self, x):\n x = self.model(x)\n x = self.end2end(x)\n return x\n\n\ndef attempt_load(weights, device=None, inplace=True, fuse=True):\n # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a\n from asone.detectors.yolov9.yolov9.models.yolo import Detect, Model\n\n model = Ensemble()\n for w in weights if isinstance(weights, list) else [weights]:\n ckpt = torch.load(attempt_download(w), map_location='cpu') # load\n ckpt = (ckpt.get('ema') or ckpt['model']).to(device).float() # FP32 model\n\n # Model compatibility updates\n if not hasattr(ckpt, 'stride'):\n ckpt.stride = torch.tensor([32.])\n if hasattr(ckpt, 'names') and isinstance(ckpt.names, (list, tuple)):\n ckpt.names = dict(enumerate(ckpt.names)) # convert to dict\n\n model.append(ckpt.fuse().eval() if fuse and hasattr(ckpt, 'fuse') else ckpt.eval()) # model in eval mode\n\n # Module compatibility updates\n for m in model.modules():\n t = type(m)\n if t in (nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU, Detect, Model):\n m.inplace = inplace # torch 1.7.0 compatibility\n # if t is Detect and not isinstance(m.anchor_grid, list):\n # delattr(m, 'anchor_grid')\n # setattr(m, 'anchor_grid', [torch.zeros(1)] * m.nl)\n elif t is nn.Upsample and not hasattr(m, 'recompute_scale_factor'):\n m.recompute_scale_factor = None # torch 1.11.0 compatibility\n\n # Return model\n if len(model) == 1:\n return model[-1]\n\n # Return detection ensemble\n print(f'Ensemble created with {weights}\\n')\n for k in 'names', 'nc', 'yaml':\n setattr(model, k, getattr(model[0], k))\n model.stride = model[torch.argmax(torch.tensor([m.stride.max() for m in model])).int()].stride # max stride\n assert all(model[0].nc == m.nc for m in model), f'Models have different class counts: {[m.nc for m in model]}'\n return model\n" }, { "path": "asone/detectors/yolov9/yolov9/models/tf.py", "content": "import argparse\nimport sys\nfrom copy import deepcopy\nfrom pathlib import Path\n\nFILE = Path(__file__).resolve()\nROOT = FILE.parents[1] # YOLO root directory\n# if str(ROOT) not in sys.path:\n# sys.path.append(str(ROOT)) # add ROOT to PATH\n# ROOT = ROOT.relative_to(Path.cwd()) # relative\n\nimport numpy as np\nimport tensorflow as tf\nimport torch\nimport torch.nn as nn\nfrom tensorflow import keras\n\nfrom asone.detectors.yolov9.yolov9.models.common import (C3, SPP, SPPF, Bottleneck, BottleneckCSP, C3x, Concat, Conv, CrossConv, DWConv,\n DWConvTranspose2d, Focus, autopad)\nfrom asone.detectors.yolov9.yolov9.models.experimental import MixConv2d, attempt_load\nfrom asone.detectors.yolov9.yolov9.models.yolo import Detect, Segment\nfrom asone.detectors.yolov9.yolov9.utils.activations import SiLU\nfrom asone.detectors.yolov9.yolov9.utils.general import LOGGER, make_divisible, print_args\n\n\nclass TFBN(keras.layers.Layer):\n # TensorFlow BatchNormalization wrapper\n def __init__(self, w=None):\n super().__init__()\n self.bn = keras.layers.BatchNormalization(\n beta_initializer=keras.initializers.Constant(w.bias.numpy()),\n gamma_initializer=keras.initializers.Constant(w.weight.numpy()),\n moving_mean_initializer=keras.initializers.Constant(w.running_mean.numpy()),\n moving_variance_initializer=keras.initializers.Constant(w.running_var.numpy()),\n epsilon=w.eps)\n\n def call(self, inputs):\n return self.bn(inputs)\n\n\nclass TFPad(keras.layers.Layer):\n # Pad inputs in spatial dimensions 1 and 2\n def __init__(self, pad):\n super().__init__()\n if isinstance(pad, int):\n self.pad = tf.constant([[0, 0], [pad, pad], [pad, pad], [0, 0]])\n else: # tuple/list\n self.pad = tf.constant([[0, 0], [pad[0], pad[0]], [pad[1], pad[1]], [0, 0]])\n\n def call(self, inputs):\n return tf.pad(inputs, self.pad, mode='constant', constant_values=0)\n\n\nclass TFConv(keras.layers.Layer):\n # Standard convolution\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None):\n # ch_in, ch_out, weights, kernel, stride, padding, groups\n super().__init__()\n assert g == 1, \"TF v2.2 Conv2D does not support 'groups' argument\"\n # TensorFlow convolution padding is inconsistent with PyTorch (e.g. k=3 s=2 'SAME' padding)\n # see https://stackoverflow.com/questions/52975843/comparing-conv2d-with-padding-between-tensorflow-and-pytorch\n conv = keras.layers.Conv2D(\n filters=c2,\n kernel_size=k,\n strides=s,\n padding='SAME' if s == 1 else 'VALID',\n use_bias=not hasattr(w, 'bn'),\n kernel_initializer=keras.initializers.Constant(w.conv.weight.permute(2, 3, 1, 0).numpy()),\n bias_initializer='zeros' if hasattr(w, 'bn') else keras.initializers.Constant(w.conv.bias.numpy()))\n self.conv = conv if s == 1 else keras.Sequential([TFPad(autopad(k, p)), conv])\n self.bn = TFBN(w.bn) if hasattr(w, 'bn') else tf.identity\n self.act = activations(w.act) if act else tf.identity\n\n def call(self, inputs):\n return self.act(self.bn(self.conv(inputs)))\n\n\nclass TFDWConv(keras.layers.Layer):\n # Depthwise convolution\n def __init__(self, c1, c2, k=1, s=1, p=None, act=True, w=None):\n # ch_in, ch_out, weights, kernel, stride, padding, groups\n super().__init__()\n assert c2 % c1 == 0, f'TFDWConv() output={c2} must be a multiple of input={c1} channels'\n conv = keras.layers.DepthwiseConv2D(\n kernel_size=k,\n depth_multiplier=c2 // c1,\n strides=s,\n padding='SAME' if s == 1 else 'VALID',\n use_bias=not hasattr(w, 'bn'),\n depthwise_initializer=keras.initializers.Constant(w.conv.weight.permute(2, 3, 1, 0).numpy()),\n bias_initializer='zeros' if hasattr(w, 'bn') else keras.initializers.Constant(w.conv.bias.numpy()))\n self.conv = conv if s == 1 else keras.Sequential([TFPad(autopad(k, p)), conv])\n self.bn = TFBN(w.bn) if hasattr(w, 'bn') else tf.identity\n self.act = activations(w.act) if act else tf.identity\n\n def call(self, inputs):\n return self.act(self.bn(self.conv(inputs)))\n\n\nclass TFDWConvTranspose2d(keras.layers.Layer):\n # Depthwise ConvTranspose2d\n def __init__(self, c1, c2, k=1, s=1, p1=0, p2=0, w=None):\n # ch_in, ch_out, weights, kernel, stride, padding, groups\n super().__init__()\n assert c1 == c2, f'TFDWConv() output={c2} must be equal to input={c1} channels'\n assert k == 4 and p1 == 1, 'TFDWConv() only valid for k=4 and p1=1'\n weight, bias = w.weight.permute(2, 3, 1, 0).numpy(), w.bias.numpy()\n self.c1 = c1\n self.conv = [\n keras.layers.Conv2DTranspose(filters=1,\n kernel_size=k,\n strides=s,\n padding='VALID',\n output_padding=p2,\n use_bias=True,\n kernel_initializer=keras.initializers.Constant(weight[..., i:i + 1]),\n bias_initializer=keras.initializers.Constant(bias[i])) for i in range(c1)]\n\n def call(self, inputs):\n return tf.concat([m(x) for m, x in zip(self.conv, tf.split(inputs, self.c1, 3))], 3)[:, 1:-1, 1:-1]\n\n\nclass TFFocus(keras.layers.Layer):\n # Focus wh information into c-space\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None):\n # ch_in, ch_out, kernel, stride, padding, groups\n super().__init__()\n self.conv = TFConv(c1 * 4, c2, k, s, p, g, act, w.conv)\n\n def call(self, inputs): # x(b,w,h,c) -> y(b,w/2,h/2,4c)\n # inputs = inputs / 255 # normalize 0-255 to 0-1\n inputs = [inputs[:, ::2, ::2, :], inputs[:, 1::2, ::2, :], inputs[:, ::2, 1::2, :], inputs[:, 1::2, 1::2, :]]\n return self.conv(tf.concat(inputs, 3))\n\n\nclass TFBottleneck(keras.layers.Layer):\n # Standard bottleneck\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5, w=None): # ch_in, ch_out, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv(c_, c2, 3, 1, g=g, w=w.cv2)\n self.add = shortcut and c1 == c2\n\n def call(self, inputs):\n return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs))\n\n\nclass TFCrossConv(keras.layers.Layer):\n # Cross Convolution\n def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False, w=None):\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = TFConv(c1, c_, (1, k), (1, s), w=w.cv1)\n self.cv2 = TFConv(c_, c2, (k, 1), (s, 1), g=g, w=w.cv2)\n self.add = shortcut and c1 == c2\n\n def call(self, inputs):\n return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs))\n\n\nclass TFConv2d(keras.layers.Layer):\n # Substitution for PyTorch nn.Conv2D\n def __init__(self, c1, c2, k, s=1, g=1, bias=True, w=None):\n super().__init__()\n assert g == 1, \"TF v2.2 Conv2D does not support 'groups' argument\"\n self.conv = keras.layers.Conv2D(filters=c2,\n kernel_size=k,\n strides=s,\n padding='VALID',\n use_bias=bias,\n kernel_initializer=keras.initializers.Constant(\n w.weight.permute(2, 3, 1, 0).numpy()),\n bias_initializer=keras.initializers.Constant(w.bias.numpy()) if bias else None)\n\n def call(self, inputs):\n return self.conv(inputs)\n\n\nclass TFBottleneckCSP(keras.layers.Layer):\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):\n # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv2d(c1, c_, 1, 1, bias=False, w=w.cv2)\n self.cv3 = TFConv2d(c_, c_, 1, 1, bias=False, w=w.cv3)\n self.cv4 = TFConv(2 * c_, c2, 1, 1, w=w.cv4)\n self.bn = TFBN(w.bn)\n self.act = lambda x: keras.activations.swish(x)\n self.m = keras.Sequential([TFBottleneck(c_, c_, shortcut, g, e=1.0, w=w.m[j]) for j in range(n)])\n\n def call(self, inputs):\n y1 = self.cv3(self.m(self.cv1(inputs)))\n y2 = self.cv2(inputs)\n return self.cv4(self.act(self.bn(tf.concat((y1, y2), axis=3))))\n\n\nclass TFC3(keras.layers.Layer):\n # CSP Bottleneck with 3 convolutions\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):\n # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2)\n self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3)\n self.m = keras.Sequential([TFBottleneck(c_, c_, shortcut, g, e=1.0, w=w.m[j]) for j in range(n)])\n\n def call(self, inputs):\n return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3))\n\n\nclass TFC3x(keras.layers.Layer):\n # 3 module with cross-convolutions\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None):\n # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n c_ = int(c2 * e) # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2)\n self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3)\n self.m = keras.Sequential([\n TFCrossConv(c_, c_, k=3, s=1, g=g, e=1.0, shortcut=shortcut, w=w.m[j]) for j in range(n)])\n\n def call(self, inputs):\n return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3))\n\n\nclass TFSPP(keras.layers.Layer):\n # Spatial pyramid pooling layer used in YOLOv3-SPP\n def __init__(self, c1, c2, k=(5, 9, 13), w=None):\n super().__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv(c_ * (len(k) + 1), c2, 1, 1, w=w.cv2)\n self.m = [keras.layers.MaxPool2D(pool_size=x, strides=1, padding='SAME') for x in k]\n\n def call(self, inputs):\n x = self.cv1(inputs)\n return self.cv2(tf.concat([x] + [m(x) for m in self.m], 3))\n\n\nclass TFSPPF(keras.layers.Layer):\n # Spatial pyramid pooling-Fast layer\n def __init__(self, c1, c2, k=5, w=None):\n super().__init__()\n c_ = c1 // 2 # hidden channels\n self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1)\n self.cv2 = TFConv(c_ * 4, c2, 1, 1, w=w.cv2)\n self.m = keras.layers.MaxPool2D(pool_size=k, strides=1, padding='SAME')\n\n def call(self, inputs):\n x = self.cv1(inputs)\n y1 = self.m(x)\n y2 = self.m(y1)\n return self.cv2(tf.concat([x, y1, y2, self.m(y2)], 3))\n\n\nclass TFDetect(keras.layers.Layer):\n # TF YOLO Detect layer\n def __init__(self, nc=80, anchors=(), ch=(), imgsz=(640, 640), w=None): # detection layer\n super().__init__()\n self.stride = tf.convert_to_tensor(w.stride.numpy(), dtype=tf.float32)\n self.nc = nc # number of classes\n self.no = nc + 5 # number of outputs per anchor\n self.nl = len(anchors) # number of detection layers\n self.na = len(anchors[0]) // 2 # number of anchors\n self.grid = [tf.zeros(1)] * self.nl # init grid\n self.anchors = tf.convert_to_tensor(w.anchors.numpy(), dtype=tf.float32)\n self.anchor_grid = tf.reshape(self.anchors * tf.reshape(self.stride, [self.nl, 1, 1]), [self.nl, 1, -1, 1, 2])\n self.m = [TFConv2d(x, self.no * self.na, 1, w=w.m[i]) for i, x in enumerate(ch)]\n self.training = False # set to False after building model\n self.imgsz = imgsz\n for i in range(self.nl):\n ny, nx = self.imgsz[0] // self.stride[i], self.imgsz[1] // self.stride[i]\n self.grid[i] = self._make_grid(nx, ny)\n\n def call(self, inputs):\n z = [] # inference output\n x = []\n for i in range(self.nl):\n x.append(self.m[i](inputs[i]))\n # x(bs,20,20,255) to x(bs,3,20,20,85)\n ny, nx = self.imgsz[0] // self.stride[i], self.imgsz[1] // self.stride[i]\n x[i] = tf.reshape(x[i], [-1, ny * nx, self.na, self.no])\n\n if not self.training: # inference\n y = x[i]\n grid = tf.transpose(self.grid[i], [0, 2, 1, 3]) - 0.5\n anchor_grid = tf.transpose(self.anchor_grid[i], [0, 2, 1, 3]) * 4\n xy = (tf.sigmoid(y[..., 0:2]) * 2 + grid) * self.stride[i] # xy\n wh = tf.sigmoid(y[..., 2:4]) ** 2 * anchor_grid\n # Normalize xywh to 0-1 to reduce calibration error\n xy /= tf.constant([[self.imgsz[1], self.imgsz[0]]], dtype=tf.float32)\n wh /= tf.constant([[self.imgsz[1], self.imgsz[0]]], dtype=tf.float32)\n y = tf.concat([xy, wh, tf.sigmoid(y[..., 4:5 + self.nc]), y[..., 5 + self.nc:]], -1)\n z.append(tf.reshape(y, [-1, self.na * ny * nx, self.no]))\n\n return tf.transpose(x, [0, 2, 1, 3]) if self.training else (tf.concat(z, 1),)\n\n @staticmethod\n def _make_grid(nx=20, ny=20):\n # yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\n # return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\n xv, yv = tf.meshgrid(tf.range(nx), tf.range(ny))\n return tf.cast(tf.reshape(tf.stack([xv, yv], 2), [1, 1, ny * nx, 2]), dtype=tf.float32)\n\n\nclass TFSegment(TFDetect):\n # YOLO Segment head for segmentation models\n def __init__(self, nc=80, anchors=(), nm=32, npr=256, ch=(), imgsz=(640, 640), w=None):\n super().__init__(nc, anchors, ch, imgsz, w)\n self.nm = nm # number of masks\n self.npr = npr # number of protos\n self.no = 5 + nc + self.nm # number of outputs per anchor\n self.m = [TFConv2d(x, self.no * self.na, 1, w=w.m[i]) for i, x in enumerate(ch)] # output conv\n self.proto = TFProto(ch[0], self.npr, self.nm, w=w.proto) # protos\n self.detect = TFDetect.call\n\n def call(self, x):\n p = self.proto(x[0])\n # p = TFUpsample(None, scale_factor=4, mode='nearest')(self.proto(x[0])) # (optional) full-size protos\n p = tf.transpose(p, [0, 3, 1, 2]) # from shape(1,160,160,32) to shape(1,32,160,160)\n x = self.detect(self, x)\n return (x, p) if self.training else (x[0], p)\n\n\nclass TFProto(keras.layers.Layer):\n\n def __init__(self, c1, c_=256, c2=32, w=None):\n super().__init__()\n self.cv1 = TFConv(c1, c_, k=3, w=w.cv1)\n self.upsample = TFUpsample(None, scale_factor=2, mode='nearest')\n self.cv2 = TFConv(c_, c_, k=3, w=w.cv2)\n self.cv3 = TFConv(c_, c2, w=w.cv3)\n\n def call(self, inputs):\n return self.cv3(self.cv2(self.upsample(self.cv1(inputs))))\n\n\nclass TFUpsample(keras.layers.Layer):\n # TF version of torch.nn.Upsample()\n def __init__(self, size, scale_factor, mode, w=None): # warning: all arguments needed including 'w'\n super().__init__()\n assert scale_factor % 2 == 0, \"scale_factor must be multiple of 2\"\n self.upsample = lambda x: tf.image.resize(x, (x.shape[1] * scale_factor, x.shape[2] * scale_factor), mode)\n # self.upsample = keras.layers.UpSampling2D(size=scale_factor, interpolation=mode)\n # with default arguments: align_corners=False, half_pixel_centers=False\n # self.upsample = lambda x: tf.raw_ops.ResizeNearestNeighbor(images=x,\n # size=(x.shape[1] * 2, x.shape[2] * 2))\n\n def call(self, inputs):\n return self.upsample(inputs)\n\n\nclass TFConcat(keras.layers.Layer):\n # TF version of torch.concat()\n def __init__(self, dimension=1, w=None):\n super().__init__()\n assert dimension == 1, \"convert only NCHW to NHWC concat\"\n self.d = 3\n\n def call(self, inputs):\n return tf.concat(inputs, self.d)\n\n\ndef parse_model(d, ch, model, imgsz): # model_dict, input_channels(3)\n LOGGER.info(f\"\\n{'':>3}{'from':>18}{'n':>3}{'params':>10} {'module':<40}{'arguments':<30}\")\n anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple']\n na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors\n no = na * (nc + 5) # number of outputs = anchors * (classes + 5)\n\n layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out\n for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args\n m_str = m\n m = eval(m) if isinstance(m, str) else m # eval strings\n for j, a in enumerate(args):\n try:\n args[j] = eval(a) if isinstance(a, str) else a # eval strings\n except NameError:\n pass\n\n n = max(round(n * gd), 1) if n > 1 else n # depth gain\n if m in [\n nn.Conv2d, Conv, DWConv, DWConvTranspose2d, Bottleneck, SPP, SPPF, MixConv2d, Focus, CrossConv,\n BottleneckCSP, C3, C3x]:\n c1, c2 = ch[f], args[0]\n c2 = make_divisible(c2 * gw, 8) if c2 != no else c2\n\n args = [c1, c2, *args[1:]]\n if m in [BottleneckCSP, C3, C3x]:\n args.insert(2, n)\n n = 1\n elif m is nn.BatchNorm2d:\n args = [ch[f]]\n elif m is Concat:\n c2 = sum(ch[-1 if x == -1 else x + 1] for x in f)\n elif m in [Detect, Segment]:\n args.append([ch[x + 1] for x in f])\n if isinstance(args[1], int): # number of anchors\n args[1] = [list(range(args[1] * 2))] * len(f)\n if m is Segment:\n args[3] = make_divisible(args[3] * gw, 8)\n args.append(imgsz)\n else:\n c2 = ch[f]\n\n tf_m = eval('TF' + m_str.replace('nn.', ''))\n m_ = keras.Sequential([tf_m(*args, w=model.model[i][j]) for j in range(n)]) if n > 1 \\\n else tf_m(*args, w=model.model[i]) # module\n\n torch_m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args) # module\n t = str(m)[8:-2].replace('__main__.', '') # module type\n np = sum(x.numel() for x in torch_m_.parameters()) # number params\n m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params\n LOGGER.info(f'{i:>3}{str(f):>18}{str(n):>3}{np:>10} {t:<40}{str(args):<30}') # print\n save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist\n layers.append(m_)\n ch.append(c2)\n return keras.Sequential(layers), sorted(save)\n\n\nclass TFModel:\n # TF YOLO model\n def __init__(self, cfg='yolo.yaml', ch=3, nc=None, model=None, imgsz=(640, 640)): # model, channels, classes\n super().__init__()\n if isinstance(cfg, dict):\n self.yaml = cfg # model dict\n else: # is *.yaml\n import yaml # for torch hub\n self.yaml_file = Path(cfg).name\n with open(cfg) as f:\n self.yaml = yaml.load(f, Loader=yaml.FullLoader) # model dict\n\n # Define model\n if nc and nc != self.yaml['nc']:\n LOGGER.info(f\"Overriding {cfg} nc={self.yaml['nc']} with nc={nc}\")\n self.yaml['nc'] = nc # override yaml value\n self.model, self.savelist = parse_model(deepcopy(self.yaml), ch=[ch], model=model, imgsz=imgsz)\n\n def predict(self,\n inputs,\n tf_nms=False,\n agnostic_nms=False,\n topk_per_class=100,\n topk_all=100,\n iou_thres=0.45,\n conf_thres=0.25):\n y = [] # outputs\n x = inputs\n for m in self.model.layers:\n if m.f != -1: # if not from previous layer\n x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers\n\n x = m(x) # run\n y.append(x if m.i in self.savelist else None) # save output\n\n # Add TensorFlow NMS\n if tf_nms:\n boxes = self._xywh2xyxy(x[0][..., :4])\n probs = x[0][:, :, 4:5]\n classes = x[0][:, :, 5:]\n scores = probs * classes\n if agnostic_nms:\n nms = AgnosticNMS()((boxes, classes, scores), topk_all, iou_thres, conf_thres)\n else:\n boxes = tf.expand_dims(boxes, 2)\n nms = tf.image.combined_non_max_suppression(boxes,\n scores,\n topk_per_class,\n topk_all,\n iou_thres,\n conf_thres,\n clip_boxes=False)\n return (nms,)\n return x # output [1,6300,85] = [xywh, conf, class0, class1, ...]\n # x = x[0] # [x(1,6300,85), ...] to x(6300,85)\n # xywh = x[..., :4] # x(6300,4) boxes\n # conf = x[..., 4:5] # x(6300,1) confidences\n # cls = tf.reshape(tf.cast(tf.argmax(x[..., 5:], axis=1), tf.float32), (-1, 1)) # x(6300,1) classes\n # return tf.concat([conf, cls, xywh], 1)\n\n @staticmethod\n def _xywh2xyxy(xywh):\n # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n x, y, w, h = tf.split(xywh, num_or_size_splits=4, axis=-1)\n return tf.concat([x - w / 2, y - h / 2, x + w / 2, y + h / 2], axis=-1)\n\n\nclass AgnosticNMS(keras.layers.Layer):\n # TF Agnostic NMS\n def call(self, input, topk_all, iou_thres, conf_thres):\n # wrap map_fn to avoid TypeSpec related error https://stackoverflow.com/a/65809989/3036450\n return tf.map_fn(lambda x: self._nms(x, topk_all, iou_thres, conf_thres),\n input,\n fn_output_signature=(tf.float32, tf.float32, tf.float32, tf.int32),\n name='agnostic_nms')\n\n @staticmethod\n def _nms(x, topk_all=100, iou_thres=0.45, conf_thres=0.25): # agnostic NMS\n boxes, classes, scores = x\n class_inds = tf.cast(tf.argmax(classes, axis=-1), tf.float32)\n scores_inp = tf.reduce_max(scores, -1)\n selected_inds = tf.image.non_max_suppression(boxes,\n scores_inp,\n max_output_size=topk_all,\n iou_threshold=iou_thres,\n score_threshold=conf_thres)\n selected_boxes = tf.gather(boxes, selected_inds)\n padded_boxes = tf.pad(selected_boxes,\n paddings=[[0, topk_all - tf.shape(selected_boxes)[0]], [0, 0]],\n mode=\"CONSTANT\",\n constant_values=0.0)\n selected_scores = tf.gather(scores_inp, selected_inds)\n padded_scores = tf.pad(selected_scores,\n paddings=[[0, topk_all - tf.shape(selected_boxes)[0]]],\n mode=\"CONSTANT\",\n constant_values=-1.0)\n selected_classes = tf.gather(class_inds, selected_inds)\n padded_classes = tf.pad(selected_classes,\n paddings=[[0, topk_all - tf.shape(selected_boxes)[0]]],\n mode=\"CONSTANT\",\n constant_values=-1.0)\n valid_detections = tf.shape(selected_inds)[0]\n return padded_boxes, padded_scores, padded_classes, valid_detections\n\n\ndef activations(act=nn.SiLU):\n # Returns TF activation from input PyTorch activation\n if isinstance(act, nn.LeakyReLU):\n return lambda x: keras.activations.relu(x, alpha=0.1)\n elif isinstance(act, nn.Hardswish):\n return lambda x: x * tf.nn.relu6(x + 3) * 0.166666667\n elif isinstance(act, (nn.SiLU, SiLU)):\n return lambda x: keras.activations.swish(x)\n else:\n raise Exception(f'no matching TensorFlow activation found for PyTorch activation {act}')\n\n\ndef representative_dataset_gen(dataset, ncalib=100):\n # Representative dataset generator for use with converter.representative_dataset, returns a generator of np arrays\n for n, (path, img, im0s, vid_cap, string) in enumerate(dataset):\n im = np.transpose(img, [1, 2, 0])\n im = np.expand_dims(im, axis=0).astype(np.float32)\n im /= 255\n yield [im]\n if n >= ncalib:\n break\n\n\ndef run(\n weights=ROOT / 'yolo.pt', # weights path\n imgsz=(640, 640), # inference size h,w\n batch_size=1, # batch size\n dynamic=False, # dynamic batch size\n):\n # PyTorch model\n im = torch.zeros((batch_size, 3, *imgsz)) # BCHW image\n model = attempt_load(weights, device=torch.device('cpu'), inplace=True, fuse=False)\n _ = model(im) # inference\n model.info()\n\n # TensorFlow model\n im = tf.zeros((batch_size, *imgsz, 3)) # BHWC image\n tf_model = TFModel(cfg=model.yaml, model=model, nc=model.nc, imgsz=imgsz)\n _ = tf_model.predict(im) # inference\n\n # Keras model\n im = keras.Input(shape=(*imgsz, 3), batch_size=None if dynamic else batch_size)\n keras_model = keras.Model(inputs=im, outputs=tf_model.predict(im))\n keras_model.summary()\n\n LOGGER.info('PyTorch, TensorFlow and Keras models successfully verified.\\nUse export.py for TF model export.')\n\n\ndef parse_opt():\n parser = argparse.ArgumentParser()\n parser.add_argument('--weights', type=str, default=ROOT / 'yolo.pt', help='weights path')\n parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w')\n parser.add_argument('--batch-size', type=int, default=1, help='batch size')\n parser.add_argument('--dynamic', action='store_true', help='dynamic batch size')\n opt = parser.parse_args()\n opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand\n print_args(vars(opt))\n return opt\n\n\ndef main(opt):\n run(**vars(opt))\n\n\nif __name__ == \"__main__\":\n opt = parse_opt()\n main(opt)\n" }, { "path": "asone/detectors/yolov9/yolov9/models/yolo.py", "content": "import argparse\nimport os\nimport platform\nimport sys\nfrom copy import deepcopy\nfrom pathlib import Path\n\nFILE = Path(__file__).resolve()\nROOT = FILE.parents[1] # YOLO root directory\n# if str(ROOT) not in sys.path:\n# sys.path.append(str(ROOT)) # add ROOT to PATH\nif platform.system() != 'Windows':\n ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative\n\nfrom asone.detectors.yolov9.yolov9.models.common import *\nfrom asone.detectors.yolov9.yolov9.models.experimental import *\nfrom asone.detectors.yolov9.yolov9.utils.general import LOGGER, check_version, check_yaml, make_divisible, print_args\nfrom asone.detectors.yolov9.yolov9.utils.plots import feature_visualization\nfrom asone.detectors.yolov9.yolov9.utils.torch_utils import (fuse_conv_and_bn, initialize_weights, model_info, profile, scale_img, select_device,\n time_sync)\nfrom asone.detectors.yolov9.yolov9.utils.tal.anchor_generator import make_anchors, dist2bbox\n\ntry:\n import thop # for FLOPs computation\nexcept ImportError:\n thop = None\n\n\nclass Detect(nn.Module):\n # YOLO Detect head for detection models\n dynamic = False # force grid reconstruction\n export = False # export mode\n shape = None\n anchors = torch.empty(0) # init\n strides = torch.empty(0) # init\n\n def __init__(self, nc=80, ch=(), inplace=True): # detection layer\n super().__init__()\n self.nc = nc # number of classes\n self.nl = len(ch) # number of detection layers\n self.reg_max = 16\n self.no = nc + self.reg_max * 4 # number of outputs per anchor\n self.inplace = inplace # use inplace ops (e.g. slice assignment)\n self.stride = torch.zeros(self.nl) # strides computed during build\n\n c2, c3 = max((ch[0] // 4, self.reg_max * 4, 16)), max((ch[0], min((self.nc * 2, 128)))) # channels\n self.cv2 = nn.ModuleList(\n nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3), nn.Conv2d(c2, 4 * self.reg_max, 1)) for x in ch)\n self.cv3 = nn.ModuleList(\n nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch)\n self.dfl = DFL(self.reg_max) if self.reg_max > 1 else nn.Identity()\n\n def forward(self, x):\n shape = x[0].shape # BCHW\n for i in range(self.nl):\n x[i] = torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1)\n if self.training:\n return x\n elif self.dynamic or self.shape != shape:\n self.anchors, self.strides = (x.transpose(0, 1) for x in make_anchors(x, self.stride, 0.5))\n self.shape = shape\n\n box, cls = torch.cat([xi.view(shape[0], self.no, -1) for xi in x], 2).split((self.reg_max * 4, self.nc), 1)\n dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n y = torch.cat((dbox, cls.sigmoid()), 1)\n return y if self.export else (y, x)\n\n def bias_init(self):\n # Initialize Detect() biases, WARNING: requires stride availability\n m = self # self.model[-1] # Detect() module\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1\n # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency\n for a, b, s in zip(m.cv2, m.cv3, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n\n\nclass DDetect(nn.Module):\n # YOLO Detect head for detection models\n dynamic = False # force grid reconstruction\n export = False # export mode\n shape = None\n anchors = torch.empty(0) # init\n strides = torch.empty(0) # init\n\n def __init__(self, nc=80, ch=(), inplace=True): # detection layer\n super().__init__()\n self.nc = nc # number of classes\n self.nl = len(ch) # number of detection layers\n self.reg_max = 16\n self.no = nc + self.reg_max * 4 # number of outputs per anchor\n self.inplace = inplace # use inplace ops (e.g. slice assignment)\n self.stride = torch.zeros(self.nl) # strides computed during build\n\n c2, c3 = make_divisible(max((ch[0] // 4, self.reg_max * 4, 16)), 4), max((ch[0], min((self.nc * 2, 128)))) # channels\n self.cv2 = nn.ModuleList(\n nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3, g=4), nn.Conv2d(c2, 4 * self.reg_max, 1, groups=4)) for x in ch)\n self.cv3 = nn.ModuleList(\n nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch)\n self.dfl = DFL(self.reg_max) if self.reg_max > 1 else nn.Identity()\n\n def forward(self, x):\n shape = x[0].shape # BCHW\n for i in range(self.nl):\n x[i] = torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1)\n if self.training:\n return x\n elif self.dynamic or self.shape != shape:\n self.anchors, self.strides = (x.transpose(0, 1) for x in make_anchors(x, self.stride, 0.5))\n self.shape = shape\n\n box, cls = torch.cat([xi.view(shape[0], self.no, -1) for xi in x], 2).split((self.reg_max * 4, self.nc), 1)\n dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n y = torch.cat((dbox, cls.sigmoid()), 1)\n return y if self.export else (y, x)\n\n def bias_init(self):\n # Initialize Detect() biases, WARNING: requires stride availability\n m = self # self.model[-1] # Detect() module\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1\n # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency\n for a, b, s in zip(m.cv2, m.cv3, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n\n\nclass DualDetect(nn.Module):\n # YOLO Detect head for detection models\n dynamic = False # force grid reconstruction\n export = False # export mode\n shape = None\n anchors = torch.empty(0) # init\n strides = torch.empty(0) # init\n\n def __init__(self, nc=80, ch=(), inplace=True): # detection layer\n super().__init__()\n self.nc = nc # number of classes\n self.nl = len(ch) // 2 # number of detection layers\n self.reg_max = 16\n self.no = nc + self.reg_max * 4 # number of outputs per anchor\n self.inplace = inplace # use inplace ops (e.g. slice assignment)\n self.stride = torch.zeros(self.nl) # strides computed during build\n\n c2, c3 = max((ch[0] // 4, self.reg_max * 4, 16)), max((ch[0], min((self.nc * 2, 128)))) # channels\n c4, c5 = max((ch[self.nl] // 4, self.reg_max * 4, 16)), max((ch[self.nl], min((self.nc * 2, 128)))) # channels\n self.cv2 = nn.ModuleList(\n nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3), nn.Conv2d(c2, 4 * self.reg_max, 1)) for x in ch[:self.nl])\n self.cv3 = nn.ModuleList(\n nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch[:self.nl])\n self.cv4 = nn.ModuleList(\n nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, 4 * self.reg_max, 1)) for x in ch[self.nl:])\n self.cv5 = nn.ModuleList(\n nn.Sequential(Conv(x, c5, 3), Conv(c5, c5, 3), nn.Conv2d(c5, self.nc, 1)) for x in ch[self.nl:])\n self.dfl = DFL(self.reg_max)\n self.dfl2 = DFL(self.reg_max)\n\n def forward(self, x):\n shape = x[0].shape # BCHW\n d1 = []\n d2 = []\n for i in range(self.nl):\n d1.append(torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1))\n d2.append(torch.cat((self.cv4[i](x[self.nl+i]), self.cv5[i](x[self.nl+i])), 1))\n if self.training:\n return [d1, d2]\n elif self.dynamic or self.shape != shape:\n self.anchors, self.strides = (d1.transpose(0, 1) for d1 in make_anchors(d1, self.stride, 0.5))\n self.shape = shape\n\n box, cls = torch.cat([di.view(shape[0], self.no, -1) for di in d1], 2).split((self.reg_max * 4, self.nc), 1)\n dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n box2, cls2 = torch.cat([di.view(shape[0], self.no, -1) for di in d2], 2).split((self.reg_max * 4, self.nc), 1)\n dbox2 = dist2bbox(self.dfl2(box2), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n y = [torch.cat((dbox, cls.sigmoid()), 1), torch.cat((dbox2, cls2.sigmoid()), 1)]\n return y if self.export else (y, [d1, d2])\n\n def bias_init(self):\n # Initialize Detect() biases, WARNING: requires stride availability\n m = self # self.model[-1] # Detect() module\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1\n # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency\n for a, b, s in zip(m.cv2, m.cv3, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n for a, b, s in zip(m.cv4, m.cv5, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n\n\nclass DualDDetect(nn.Module):\n # YOLO Detect head for detection models\n dynamic = False # force grid reconstruction\n export = False # export mode\n shape = None\n anchors = torch.empty(0) # init\n strides = torch.empty(0) # init\n\n def __init__(self, nc=80, ch=(), inplace=True): # detection layer\n super().__init__()\n self.nc = nc # number of classes\n self.nl = len(ch) // 2 # number of detection layers\n self.reg_max = 16\n self.no = nc + self.reg_max * 4 # number of outputs per anchor\n self.inplace = inplace # use inplace ops (e.g. slice assignment)\n self.stride = torch.zeros(self.nl) # strides computed during build\n\n c2, c3 = make_divisible(max((ch[0] // 4, self.reg_max * 4, 16)), 4), max((ch[0], min((self.nc * 2, 128)))) # channels\n c4, c5 = make_divisible(max((ch[self.nl] // 4, self.reg_max * 4, 16)), 4), max((ch[self.nl], min((self.nc * 2, 128)))) # channels\n self.cv2 = nn.ModuleList(\n nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3, g=4), nn.Conv2d(c2, 4 * self.reg_max, 1, groups=4)) for x in ch[:self.nl])\n self.cv3 = nn.ModuleList(\n nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch[:self.nl])\n self.cv4 = nn.ModuleList(\n nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3, g=4), nn.Conv2d(c4, 4 * self.reg_max, 1, groups=4)) for x in ch[self.nl:])\n self.cv5 = nn.ModuleList(\n nn.Sequential(Conv(x, c5, 3), Conv(c5, c5, 3), nn.Conv2d(c5, self.nc, 1)) for x in ch[self.nl:])\n self.dfl = DFL(self.reg_max)\n self.dfl2 = DFL(self.reg_max)\n\n def forward(self, x):\n shape = x[0].shape # BCHW\n d1 = []\n d2 = []\n for i in range(self.nl):\n d1.append(torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1))\n d2.append(torch.cat((self.cv4[i](x[self.nl+i]), self.cv5[i](x[self.nl+i])), 1))\n if self.training:\n return [d1, d2]\n elif self.dynamic or self.shape != shape:\n self.anchors, self.strides = (d1.transpose(0, 1) for d1 in make_anchors(d1, self.stride, 0.5))\n self.shape = shape\n\n box, cls = torch.cat([di.view(shape[0], self.no, -1) for di in d1], 2).split((self.reg_max * 4, self.nc), 1)\n dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n box2, cls2 = torch.cat([di.view(shape[0], self.no, -1) for di in d2], 2).split((self.reg_max * 4, self.nc), 1)\n dbox2 = dist2bbox(self.dfl2(box2), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n y = [torch.cat((dbox, cls.sigmoid()), 1), torch.cat((dbox2, cls2.sigmoid()), 1)]\n return y if self.export else (y, [d1, d2])\n #y = torch.cat((dbox2, cls2.sigmoid()), 1)\n #return y if self.export else (y, d2)\n #y1 = torch.cat((dbox, cls.sigmoid()), 1)\n #y2 = torch.cat((dbox2, cls2.sigmoid()), 1)\n #return [y1, y2] if self.export else [(y1, d1), (y2, d2)]\n #return [y1, y2] if self.export else [(y1, y2), (d1, d2)]\n\n def bias_init(self):\n # Initialize Detect() biases, WARNING: requires stride availability\n m = self # self.model[-1] # Detect() module\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1\n # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency\n for a, b, s in zip(m.cv2, m.cv3, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n for a, b, s in zip(m.cv4, m.cv5, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n\n\nclass TripleDetect(nn.Module):\n # YOLO Detect head for detection models\n dynamic = False # force grid reconstruction\n export = False # export mode\n shape = None\n anchors = torch.empty(0) # init\n strides = torch.empty(0) # init\n\n def __init__(self, nc=80, ch=(), inplace=True): # detection layer\n super().__init__()\n self.nc = nc # number of classes\n self.nl = len(ch) // 3 # number of detection layers\n self.reg_max = 16\n self.no = nc + self.reg_max * 4 # number of outputs per anchor\n self.inplace = inplace # use inplace ops (e.g. slice assignment)\n self.stride = torch.zeros(self.nl) # strides computed during build\n\n c2, c3 = max((ch[0] // 4, self.reg_max * 4, 16)), max((ch[0], min((self.nc * 2, 128)))) # channels\n c4, c5 = max((ch[self.nl] // 4, self.reg_max * 4, 16)), max((ch[self.nl], min((self.nc * 2, 128)))) # channels\n c6, c7 = max((ch[self.nl * 2] // 4, self.reg_max * 4, 16)), max((ch[self.nl * 2], min((self.nc * 2, 128)))) # channels\n self.cv2 = nn.ModuleList(\n nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3), nn.Conv2d(c2, 4 * self.reg_max, 1)) for x in ch[:self.nl])\n self.cv3 = nn.ModuleList(\n nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch[:self.nl])\n self.cv4 = nn.ModuleList(\n nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, 4 * self.reg_max, 1)) for x in ch[self.nl:self.nl*2])\n self.cv5 = nn.ModuleList(\n nn.Sequential(Conv(x, c5, 3), Conv(c5, c5, 3), nn.Conv2d(c5, self.nc, 1)) for x in ch[self.nl:self.nl*2])\n self.cv6 = nn.ModuleList(\n nn.Sequential(Conv(x, c6, 3), Conv(c6, c6, 3), nn.Conv2d(c6, 4 * self.reg_max, 1)) for x in ch[self.nl*2:self.nl*3])\n self.cv7 = nn.ModuleList(\n nn.Sequential(Conv(x, c7, 3), Conv(c7, c7, 3), nn.Conv2d(c7, self.nc, 1)) for x in ch[self.nl*2:self.nl*3])\n self.dfl = DFL(self.reg_max)\n self.dfl2 = DFL(self.reg_max)\n self.dfl3 = DFL(self.reg_max)\n\n def forward(self, x):\n shape = x[0].shape # BCHW\n d1 = []\n d2 = []\n d3 = []\n for i in range(self.nl):\n d1.append(torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1))\n d2.append(torch.cat((self.cv4[i](x[self.nl+i]), self.cv5[i](x[self.nl+i])), 1))\n d3.append(torch.cat((self.cv6[i](x[self.nl*2+i]), self.cv7[i](x[self.nl*2+i])), 1))\n if self.training:\n return [d1, d2, d3]\n elif self.dynamic or self.shape != shape:\n self.anchors, self.strides = (d1.transpose(0, 1) for d1 in make_anchors(d1, self.stride, 0.5))\n self.shape = shape\n\n box, cls = torch.cat([di.view(shape[0], self.no, -1) for di in d1], 2).split((self.reg_max * 4, self.nc), 1)\n dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n box2, cls2 = torch.cat([di.view(shape[0], self.no, -1) for di in d2], 2).split((self.reg_max * 4, self.nc), 1)\n dbox2 = dist2bbox(self.dfl2(box2), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n box3, cls3 = torch.cat([di.view(shape[0], self.no, -1) for di in d3], 2).split((self.reg_max * 4, self.nc), 1)\n dbox3 = dist2bbox(self.dfl3(box3), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n y = [torch.cat((dbox, cls.sigmoid()), 1), torch.cat((dbox2, cls2.sigmoid()), 1), torch.cat((dbox3, cls3.sigmoid()), 1)]\n return y if self.export else (y, [d1, d2, d3])\n\n def bias_init(self):\n # Initialize Detect() biases, WARNING: requires stride availability\n m = self # self.model[-1] # Detect() module\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1\n # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency\n for a, b, s in zip(m.cv2, m.cv3, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n for a, b, s in zip(m.cv4, m.cv5, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n for a, b, s in zip(m.cv6, m.cv7, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n\n\nclass TripleDDetect(nn.Module):\n # YOLO Detect head for detection models\n dynamic = False # force grid reconstruction\n export = False # export mode\n shape = None\n anchors = torch.empty(0) # init\n strides = torch.empty(0) # init\n\n def __init__(self, nc=80, ch=(), inplace=True): # detection layer\n super().__init__()\n self.nc = nc # number of classes\n self.nl = len(ch) // 3 # number of detection layers\n self.reg_max = 16\n self.no = nc + self.reg_max * 4 # number of outputs per anchor\n self.inplace = inplace # use inplace ops (e.g. slice assignment)\n self.stride = torch.zeros(self.nl) # strides computed during build\n\n c2, c3 = make_divisible(max((ch[0] // 4, self.reg_max * 4, 16)), 4), \\\n max((ch[0], min((self.nc * 2, 128)))) # channels\n c4, c5 = make_divisible(max((ch[self.nl] // 4, self.reg_max * 4, 16)), 4), \\\n max((ch[self.nl], min((self.nc * 2, 128)))) # channels\n c6, c7 = make_divisible(max((ch[self.nl * 2] // 4, self.reg_max * 4, 16)), 4), \\\n max((ch[self.nl * 2], min((self.nc * 2, 128)))) # channels\n self.cv2 = nn.ModuleList(\n nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3, g=4), \n nn.Conv2d(c2, 4 * self.reg_max, 1, groups=4)) for x in ch[:self.nl])\n self.cv3 = nn.ModuleList(\n nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch[:self.nl])\n self.cv4 = nn.ModuleList(\n nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3, g=4), \n nn.Conv2d(c4, 4 * self.reg_max, 1, groups=4)) for x in ch[self.nl:self.nl*2])\n self.cv5 = nn.ModuleList(\n nn.Sequential(Conv(x, c5, 3), Conv(c5, c5, 3), nn.Conv2d(c5, self.nc, 1)) for x in ch[self.nl:self.nl*2])\n self.cv6 = nn.ModuleList(\n nn.Sequential(Conv(x, c6, 3), Conv(c6, c6, 3, g=4), \n nn.Conv2d(c6, 4 * self.reg_max, 1, groups=4)) for x in ch[self.nl*2:self.nl*3])\n self.cv7 = nn.ModuleList(\n nn.Sequential(Conv(x, c7, 3), Conv(c7, c7, 3), nn.Conv2d(c7, self.nc, 1)) for x in ch[self.nl*2:self.nl*3])\n self.dfl = DFL(self.reg_max)\n self.dfl2 = DFL(self.reg_max)\n self.dfl3 = DFL(self.reg_max)\n\n def forward(self, x):\n shape = x[0].shape # BCHW\n d1 = []\n d2 = []\n d3 = []\n for i in range(self.nl):\n d1.append(torch.cat((self.cv2[i](x[i]), self.cv3[i](x[i])), 1))\n d2.append(torch.cat((self.cv4[i](x[self.nl+i]), self.cv5[i](x[self.nl+i])), 1))\n d3.append(torch.cat((self.cv6[i](x[self.nl*2+i]), self.cv7[i](x[self.nl*2+i])), 1))\n if self.training:\n return [d1, d2, d3]\n elif self.dynamic or self.shape != shape:\n self.anchors, self.strides = (d1.transpose(0, 1) for d1 in make_anchors(d1, self.stride, 0.5))\n self.shape = shape\n\n box, cls = torch.cat([di.view(shape[0], self.no, -1) for di in d1], 2).split((self.reg_max * 4, self.nc), 1)\n dbox = dist2bbox(self.dfl(box), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n box2, cls2 = torch.cat([di.view(shape[0], self.no, -1) for di in d2], 2).split((self.reg_max * 4, self.nc), 1)\n dbox2 = dist2bbox(self.dfl2(box2), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n box3, cls3 = torch.cat([di.view(shape[0], self.no, -1) for di in d3], 2).split((self.reg_max * 4, self.nc), 1)\n dbox3 = dist2bbox(self.dfl3(box3), self.anchors.unsqueeze(0), xywh=True, dim=1) * self.strides\n #y = [torch.cat((dbox, cls.sigmoid()), 1), torch.cat((dbox2, cls2.sigmoid()), 1), torch.cat((dbox3, cls3.sigmoid()), 1)]\n #return y if self.export else (y, [d1, d2, d3])\n y = torch.cat((dbox3, cls3.sigmoid()), 1)\n return y if self.export else (y, d3)\n\n def bias_init(self):\n # Initialize Detect() biases, WARNING: requires stride availability\n m = self # self.model[-1] # Detect() module\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1\n # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency\n for a, b, s in zip(m.cv2, m.cv3, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n for a, b, s in zip(m.cv4, m.cv5, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n for a, b, s in zip(m.cv6, m.cv7, m.stride): # from\n a[-1].bias.data[:] = 1.0 # box\n b[-1].bias.data[:m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (5 objects and 80 classes per 640 image)\n\n\nclass Segment(Detect):\n # YOLO Segment head for segmentation models\n def __init__(self, nc=80, nm=32, npr=256, ch=(), inplace=True):\n super().__init__(nc, ch, inplace)\n self.nm = nm # number of masks\n self.npr = npr # number of protos\n self.proto = Proto(ch[0], self.npr, self.nm) # protos\n self.detect = Detect.forward\n\n c4 = max(ch[0] // 4, self.nm)\n self.cv4 = nn.ModuleList(nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, self.nm, 1)) for x in ch)\n\n def forward(self, x):\n p = self.proto(x[0])\n bs = p.shape[0]\n\n mc = torch.cat([self.cv4[i](x[i]).view(bs, self.nm, -1) for i in range(self.nl)], 2) # mask coefficients\n x = self.detect(self, x)\n if self.training:\n return x, mc, p\n return (torch.cat([x, mc], 1), p) if self.export else (torch.cat([x[0], mc], 1), (x[1], mc, p))\n\n\nclass Panoptic(Detect):\n # YOLO Panoptic head for panoptic segmentation models\n def __init__(self, nc=80, sem_nc=93, nm=32, npr=256, ch=(), inplace=True):\n super().__init__(nc, ch, inplace)\n self.sem_nc = sem_nc\n self.nm = nm # number of masks\n self.npr = npr # number of protos\n self.proto = Proto(ch[0], self.npr, self.nm) # protos\n self.uconv = UConv(ch[0], ch[0]//4, self.sem_nc+self.nc)\n self.detect = Detect.forward\n\n c4 = max(ch[0] // 4, self.nm)\n self.cv4 = nn.ModuleList(nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, self.nm, 1)) for x in ch)\n\n\n def forward(self, x):\n p = self.proto(x[0])\n s = self.uconv(x[0])\n bs = p.shape[0]\n\n mc = torch.cat([self.cv4[i](x[i]).view(bs, self.nm, -1) for i in range(self.nl)], 2) # mask coefficients\n x = self.detect(self, x)\n if self.training:\n return x, mc, p, s\n return (torch.cat([x, mc], 1), p, s) if self.export else (torch.cat([x[0], mc], 1), (x[1], mc, p, s))\n \n\nclass BaseModel(nn.Module):\n # YOLO base model\n def forward(self, x, profile=False, visualize=False):\n return self._forward_once(x, profile, visualize) # single-scale inference, train\n\n def _forward_once(self, x, profile=False, visualize=False):\n y, dt = [], [] # outputs\n for m in self.model:\n if m.f != -1: # if not from previous layer\n x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers\n if profile:\n self._profile_one_layer(m, x, dt)\n x = m(x) # run\n y.append(x if m.i in self.save else None) # save output\n if visualize:\n feature_visualization(x, m.type, m.i, save_dir=visualize)\n return x\n\n def _profile_one_layer(self, m, x, dt):\n c = m == self.model[-1] # is final layer, copy input as inplace fix\n o = thop.profile(m, inputs=(x.copy() if c else x,), verbose=False)[0] / 1E9 * 2 if thop else 0 # FLOPs\n t = time_sync()\n for _ in range(10):\n m(x.copy() if c else x)\n dt.append((time_sync() - t) * 100)\n if m == self.model[0]:\n LOGGER.info(f\"{'time (ms)':>10s} {'GFLOPs':>10s} {'params':>10s} module\")\n LOGGER.info(f'{dt[-1]:10.2f} {o:10.2f} {m.np:10.0f} {m.type}')\n if c:\n LOGGER.info(f\"{sum(dt):10.2f} {'-':>10s} {'-':>10s} Total\")\n\n def fuse(self): # fuse model Conv2d() + BatchNorm2d() layers\n LOGGER.info('Fusing layers... ')\n for m in self.model.modules():\n if isinstance(m, (RepConvN)) and hasattr(m, 'fuse_convs'):\n m.fuse_convs()\n m.forward = m.forward_fuse # update forward\n if isinstance(m, (Conv, DWConv)) and hasattr(m, 'bn'):\n m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv\n delattr(m, 'bn') # remove batchnorm\n m.forward = m.forward_fuse # update forward\n self.info()\n return self\n\n def info(self, verbose=False, img_size=640): # print model information\n model_info(self, verbose, img_size)\n\n def _apply(self, fn):\n # Apply to(), cpu(), cuda(), half() to model tensors that are not parameters or registered buffers\n self = super()._apply(fn)\n m = self.model[-1] # Detect()\n if isinstance(m, (Detect, DualDetect, TripleDetect, DDetect, DualDDetect, TripleDDetect, Segment, Panoptic)):\n m.stride = fn(m.stride)\n m.anchors = fn(m.anchors)\n m.strides = fn(m.strides)\n # m.grid = list(map(fn, m.grid))\n return self\n\n\nclass DetectionModel(BaseModel):\n # YOLO detection model\n def __init__(self, cfg='yolo.yaml', ch=3, nc=None, anchors=None): # model, input channels, number of classes\n super().__init__()\n if isinstance(cfg, dict):\n self.yaml = cfg # model dict\n else: # is *.yaml\n import yaml # for torch hub\n self.yaml_file = Path(cfg).name\n with open(cfg, encoding='ascii', errors='ignore') as f:\n self.yaml = yaml.safe_load(f) # model dict\n\n # Define model\n ch = self.yaml['ch'] = self.yaml.get('ch', ch) # input channels\n if nc and nc != self.yaml['nc']:\n LOGGER.info(f\"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}\")\n self.yaml['nc'] = nc # override yaml value\n if anchors:\n LOGGER.info(f'Overriding model.yaml anchors with anchors={anchors}')\n self.yaml['anchors'] = round(anchors) # override yaml value\n self.model, self.save = parse_model(deepcopy(self.yaml), ch=[ch]) # model, savelist\n self.names = [str(i) for i in range(self.yaml['nc'])] # default names\n self.inplace = self.yaml.get('inplace', True)\n\n # Build strides, anchors\n m = self.model[-1] # Detect()\n if isinstance(m, (Detect, DDetect, Segment, Panoptic)):\n s = 256 # 2x min stride\n m.inplace = self.inplace\n forward = lambda x: self.forward(x)[0] if isinstance(m, (Segment, Panoptic)) else self.forward(x)\n m.stride = torch.tensor([s / x.shape[-2] for x in forward(torch.zeros(1, ch, s, s))]) # forward\n # check_anchor_order(m)\n # m.anchors /= m.stride.view(-1, 1, 1)\n self.stride = m.stride\n m.bias_init() # only run once\n if isinstance(m, (DualDetect, TripleDetect, DualDDetect, TripleDDetect)):\n s = 256 # 2x min stride\n m.inplace = self.inplace\n #forward = lambda x: self.forward(x)[0][0] if isinstance(m, (DualSegment, DualPanoptic)) else self.forward(x)[0]\n forward = lambda x: self.forward(x)[0]\n m.stride = torch.tensor([s / x.shape[-2] for x in forward(torch.zeros(1, ch, s, s))]) # forward\n # check_anchor_order(m)\n # m.anchors /= m.stride.view(-1, 1, 1)\n self.stride = m.stride\n m.bias_init() # only run once\n\n # Init weights, biases\n initialize_weights(self)\n self.info()\n LOGGER.info('')\n\n def forward(self, x, augment=False, profile=False, visualize=False):\n if augment:\n return self._forward_augment(x) # augmented inference, None\n return self._forward_once(x, profile, visualize) # single-scale inference, train\n\n def _forward_augment(self, x):\n img_size = x.shape[-2:] # height, width\n s = [1, 0.83, 0.67] # scales\n f = [None, 3, None] # flips (2-ud, 3-lr)\n y = [] # outputs\n for si, fi in zip(s, f):\n xi = scale_img(x.flip(fi) if fi else x, si, gs=int(self.stride.max()))\n yi = self._forward_once(xi)[0] # forward\n # cv2.imwrite(f'img_{si}.jpg', 255 * xi[0].cpu().numpy().transpose((1, 2, 0))[:, :, ::-1]) # save\n yi = self._descale_pred(yi, fi, si, img_size)\n y.append(yi)\n y = self._clip_augmented(y) # clip augmented tails\n return torch.cat(y, 1), None # augmented inference, train\n\n def _descale_pred(self, p, flips, scale, img_size):\n # de-scale predictions following augmented inference (inverse operation)\n if self.inplace:\n p[..., :4] /= scale # de-scale\n if flips == 2:\n p[..., 1] = img_size[0] - p[..., 1] # de-flip ud\n elif flips == 3:\n p[..., 0] = img_size[1] - p[..., 0] # de-flip lr\n else:\n x, y, wh = p[..., 0:1] / scale, p[..., 1:2] / scale, p[..., 2:4] / scale # de-scale\n if flips == 2:\n y = img_size[0] - y # de-flip ud\n elif flips == 3:\n x = img_size[1] - x # de-flip lr\n p = torch.cat((x, y, wh, p[..., 4:]), -1)\n return p\n\n def _clip_augmented(self, y):\n # Clip YOLO augmented inference tails\n nl = self.model[-1].nl # number of detection layers (P3-P5)\n g = sum(4 ** x for x in range(nl)) # grid points\n e = 1 # exclude layer count\n i = (y[0].shape[1] // g) * sum(4 ** x for x in range(e)) # indices\n y[0] = y[0][:, :-i] # large\n i = (y[-1].shape[1] // g) * sum(4 ** (nl - 1 - x) for x in range(e)) # indices\n y[-1] = y[-1][:, i:] # small\n return y\n\n\nModel = DetectionModel # retain YOLO 'Model' class for backwards compatibility\n\n\nclass SegmentationModel(DetectionModel):\n # YOLO segmentation model\n def __init__(self, cfg='yolo-seg.yaml', ch=3, nc=None, anchors=None):\n super().__init__(cfg, ch, nc, anchors)\n\n\nclass ClassificationModel(BaseModel):\n # YOLO classification model\n def __init__(self, cfg=None, model=None, nc=1000, cutoff=10): # yaml, model, number of classes, cutoff index\n super().__init__()\n self._from_detection_model(model, nc, cutoff) if model is not None else self._from_yaml(cfg)\n\n def _from_detection_model(self, model, nc=1000, cutoff=10):\n # Create a YOLO classification model from a YOLO detection model\n if isinstance(model, DetectMultiBackend):\n model = model.model # unwrap DetectMultiBackend\n model.model = model.model[:cutoff] # backbone\n m = model.model[-1] # last layer\n ch = m.conv.in_channels if hasattr(m, 'conv') else m.cv1.conv.in_channels # ch into module\n c = Classify(ch, nc) # Classify()\n c.i, c.f, c.type = m.i, m.f, 'models.common.Classify' # index, from, type\n model.model[-1] = c # replace\n self.model = model.model\n self.stride = model.stride\n self.save = []\n self.nc = nc\n\n def _from_yaml(self, cfg):\n # Create a YOLO classification model from a *.yaml file\n self.model = None\n\n\ndef parse_model(d, ch): # model_dict, input_channels(3)\n # Parse a YOLO model.yaml dictionary\n LOGGER.info(f\"\\n{'':>3}{'from':>18}{'n':>3}{'params':>10} {'module':<40}{'arguments':<30}\")\n anchors, nc, gd, gw, act = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple'], d.get('activation')\n if act:\n Conv.default_act = eval(act) # redefine default activation, i.e. Conv.default_act = nn.SiLU()\n RepConvN.default_act = eval(act) # redefine default activation, i.e. Conv.default_act = nn.SiLU()\n LOGGER.info(f\"{colorstr('activation:')} {act}\") # print\n na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors\n no = na * (nc + 5) # number of outputs = anchors * (classes + 5)\n\n layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out\n for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args\n m = eval(m) if isinstance(m, str) else m # eval strings\n for j, a in enumerate(args):\n with contextlib.suppress(NameError):\n args[j] = eval(a) if isinstance(a, str) else a # eval strings\n\n n = n_ = max(round(n * gd), 1) if n > 1 else n # depth gain\n if m in {\n Conv, AConv, ConvTranspose, \n Bottleneck, SPP, SPPF, DWConv, BottleneckCSP, nn.ConvTranspose2d, DWConvTranspose2d, SPPCSPC, ADown,\n RepNCSPELAN4, SPPELAN}:\n c1, c2 = ch[f], args[0]\n if c2 != no: # if not output\n c2 = make_divisible(c2 * gw, 8)\n\n args = [c1, c2, *args[1:]]\n if m in {BottleneckCSP, SPPCSPC}:\n args.insert(2, n) # number of repeats\n n = 1\n elif m is nn.BatchNorm2d:\n args = [ch[f]]\n elif m is Concat:\n c2 = sum(ch[x] for x in f)\n elif m is Shortcut:\n c2 = ch[f[0]]\n elif m is ReOrg:\n c2 = ch[f] * 4\n elif m is CBLinear:\n c2 = args[0]\n c1 = ch[f]\n args = [c1, c2, *args[1:]]\n elif m is CBFuse:\n c2 = ch[f[-1]]\n # TODO: channel, gw, gd\n elif m in {Detect, DualDetect, TripleDetect, DDetect, DualDDetect, TripleDDetect, Segment, Panoptic}:\n args.append([ch[x] for x in f])\n # if isinstance(args[1], int): # number of anchors\n # args[1] = [list(range(args[1] * 2))] * len(f)\n if m in {Segment, Panoptic}:\n args[2] = make_divisible(args[2] * gw, 8)\n elif m is Contract:\n c2 = ch[f] * args[0] ** 2\n elif m is Expand:\n c2 = ch[f] // args[0] ** 2\n else:\n c2 = ch[f]\n\n m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args) # module\n t = str(m)[8:-2].replace('__main__.', '') # module type\n np = sum(x.numel() for x in m_.parameters()) # number params\n m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params\n LOGGER.info(f'{i:>3}{str(f):>18}{n_:>3}{np:10.0f} {t:<40}{str(args):<30}') # print\n save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist\n layers.append(m_)\n if i == 0:\n ch = []\n ch.append(c2)\n return nn.Sequential(*layers), sorted(save)\n\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument('--cfg', type=str, default='yolo.yaml', help='model.yaml')\n parser.add_argument('--batch-size', type=int, default=1, help='total batch size for all GPUs')\n parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')\n parser.add_argument('--profile', action='store_true', help='profile model speed')\n parser.add_argument('--line-profile', action='store_true', help='profile model speed layer by layer')\n parser.add_argument('--test', action='store_true', help='test all yolo*.yaml')\n opt = parser.parse_args()\n opt.cfg = check_yaml(opt.cfg) # check YAML\n print_args(vars(opt))\n device = select_device(opt.device)\n\n # Create model\n im = torch.rand(opt.batch_size, 3, 640, 640).to(device)\n model = Model(opt.cfg).to(device)\n model.eval()\n\n # Options\n if opt.line_profile: # profile layer by layer\n model(im, profile=True)\n\n elif opt.profile: # profile forward-backward\n results = profile(input=im, ops=[model], n=3)\n\n elif opt.test: # test all models\n for cfg in Path(ROOT / 'models').rglob('yolo*.yaml'):\n try:\n _ = Model(cfg)\n except Exception as e:\n print(f'Error in {cfg}: {e}')\n\n else: # report fused model summary\n model.fuse()\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/__init__.py", "content": "import contextlib\nimport platform\nimport threading\n\n\ndef emojis(str=''):\n # Return platform-dependent emoji-safe version of string\n return str.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else str\n\n\nclass TryExcept(contextlib.ContextDecorator):\n # YOLOv5 TryExcept class. Usage: @TryExcept() decorator or 'with TryExcept():' context manager\n def __init__(self, msg=''):\n self.msg = msg\n\n def __enter__(self):\n pass\n\n def __exit__(self, exc_type, value, traceback):\n if value:\n print(emojis(f\"{self.msg}{': ' if self.msg else ''}{value}\"))\n return True\n\n\ndef threaded(func):\n # Multi-threads a target function and returns thread. Usage: @threaded decorator\n def wrapper(*args, **kwargs):\n thread = threading.Thread(target=func, args=args, kwargs=kwargs, daemon=True)\n thread.start()\n return thread\n\n return wrapper\n\n\ndef join_threads(verbose=False):\n # Join all daemon threads, i.e. atexit.register(lambda: join_threads())\n main_thread = threading.current_thread()\n for t in threading.enumerate():\n if t is not main_thread:\n if verbose:\n print(f'Joining thread {t.name}')\n t.join()\n\n\ndef notebook_init(verbose=True):\n # Check system software and hardware\n print('Checking setup...')\n\n import os\n import shutil\n\n from utils.general import check_font, check_requirements, is_colab\n from utils.torch_utils import select_device # imports\n\n check_font()\n\n import psutil\n from IPython import display # to display images and clear console output\n\n if is_colab():\n shutil.rmtree('/content/sample_data', ignore_errors=True) # remove colab /sample_data directory\n\n # System info\n if verbose:\n gb = 1 << 30 # bytes to GiB (1024 ** 3)\n ram = psutil.virtual_memory().total\n total, used, free = shutil.disk_usage(\"/\")\n display.clear_output()\n s = f'({os.cpu_count()} CPUs, {ram / gb:.1f} GB RAM, {(total - free) / gb:.1f}/{total / gb:.1f} GB disk)'\n else:\n s = ''\n\n select_device(newline=False)\n print(emojis(f'Setup complete ✅ {s}'))\n return display\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/activations.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass SiLU(nn.Module):\n # SiLU activation https://arxiv.org/pdf/1606.08415.pdf\n @staticmethod\n def forward(x):\n return x * torch.sigmoid(x)\n\n\nclass Hardswish(nn.Module):\n # Hard-SiLU activation\n @staticmethod\n def forward(x):\n # return x * F.hardsigmoid(x) # for TorchScript and CoreML\n return x * F.hardtanh(x + 3, 0.0, 6.0) / 6.0 # for TorchScript, CoreML and ONNX\n\n\nclass Mish(nn.Module):\n # Mish activation https://github.com/digantamisra98/Mish\n @staticmethod\n def forward(x):\n return x * F.softplus(x).tanh()\n\n\nclass MemoryEfficientMish(nn.Module):\n # Mish activation memory-efficient\n class F(torch.autograd.Function):\n\n @staticmethod\n def forward(ctx, x):\n ctx.save_for_backward(x)\n return x.mul(torch.tanh(F.softplus(x))) # x * tanh(ln(1 + exp(x)))\n\n @staticmethod\n def backward(ctx, grad_output):\n x = ctx.saved_tensors[0]\n sx = torch.sigmoid(x)\n fx = F.softplus(x).tanh()\n return grad_output * (fx + x * sx * (1 - fx * fx))\n\n def forward(self, x):\n return self.F.apply(x)\n\n\nclass FReLU(nn.Module):\n # FReLU activation https://arxiv.org/abs/2007.11824\n def __init__(self, c1, k=3): # ch_in, kernel\n super().__init__()\n self.conv = nn.Conv2d(c1, c1, k, 1, 1, groups=c1, bias=False)\n self.bn = nn.BatchNorm2d(c1)\n\n def forward(self, x):\n return torch.max(x, self.bn(self.conv(x)))\n\n\nclass AconC(nn.Module):\n r\"\"\" ACON activation (activate or not)\n AconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is a learnable parameter\n according to \"Activate or Not: Learning Customized Activation\" .\n \"\"\"\n\n def __init__(self, c1):\n super().__init__()\n self.p1 = nn.Parameter(torch.randn(1, c1, 1, 1))\n self.p2 = nn.Parameter(torch.randn(1, c1, 1, 1))\n self.beta = nn.Parameter(torch.ones(1, c1, 1, 1))\n\n def forward(self, x):\n dpx = (self.p1 - self.p2) * x\n return dpx * torch.sigmoid(self.beta * dpx) + self.p2 * x\n\n\nclass MetaAconC(nn.Module):\n r\"\"\" ACON activation (activate or not)\n MetaAconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is generated by a small network\n according to \"Activate or Not: Learning Customized Activation\" .\n \"\"\"\n\n def __init__(self, c1, k=1, s=1, r=16): # ch_in, kernel, stride, r\n super().__init__()\n c2 = max(r, c1 // r)\n self.p1 = nn.Parameter(torch.randn(1, c1, 1, 1))\n self.p2 = nn.Parameter(torch.randn(1, c1, 1, 1))\n self.fc1 = nn.Conv2d(c1, c2, k, s, bias=True)\n self.fc2 = nn.Conv2d(c2, c1, k, s, bias=True)\n # self.bn1 = nn.BatchNorm2d(c2)\n # self.bn2 = nn.BatchNorm2d(c1)\n\n def forward(self, x):\n y = x.mean(dim=2, keepdims=True).mean(dim=3, keepdims=True)\n # batch-size 1 bug/instabilities https://github.com/ultralytics/yolov5/issues/2891\n # beta = torch.sigmoid(self.bn2(self.fc2(self.bn1(self.fc1(y))))) # bug/unstable\n beta = torch.sigmoid(self.fc2(self.fc1(y))) # bug patch BN layers removed\n dpx = (self.p1 - self.p2) * x\n return dpx * torch.sigmoid(beta * dpx) + self.p2 * x\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/augmentations.py", "content": "import math\nimport random\n\nimport cv2\nimport numpy as np\nimport torch\nimport torchvision.transforms as T\nimport torchvision.transforms.functional as TF\n\nfrom asone.detectors.yolov9.yolov9.utils.general import LOGGER, check_version, colorstr, resample_segments, segment2box, xywhn2xyxy\nfrom asone.detectors.yolov9.yolov9.utils.metrics import bbox_ioa\n\nIMAGENET_MEAN = 0.485, 0.456, 0.406 # RGB mean\nIMAGENET_STD = 0.229, 0.224, 0.225 # RGB standard deviation\n\n\nclass Albumentations:\n # YOLOv5 Albumentations class (optional, only used if package is installed)\n def __init__(self, size=640):\n self.transform = None\n prefix = colorstr('albumentations: ')\n try:\n import albumentations as A\n check_version(A.__version__, '1.0.3', hard=True) # version requirement\n\n T = [\n A.RandomResizedCrop(height=size, width=size, scale=(0.8, 1.0), ratio=(0.9, 1.11), p=0.0),\n A.Blur(p=0.01),\n A.MedianBlur(p=0.01),\n A.ToGray(p=0.01),\n A.CLAHE(p=0.01),\n A.RandomBrightnessContrast(p=0.0),\n A.RandomGamma(p=0.0),\n A.ImageCompression(quality_lower=75, p=0.0)] # transforms\n self.transform = A.Compose(T, bbox_params=A.BboxParams(format='yolo', label_fields=['class_labels']))\n\n LOGGER.info(prefix + ', '.join(f'{x}'.replace('always_apply=False, ', '') for x in T if x.p))\n except ImportError: # package not installed, skip\n pass\n except Exception as e:\n LOGGER.info(f'{prefix}{e}')\n\n def __call__(self, im, labels, p=1.0):\n if self.transform and random.random() < p:\n new = self.transform(image=im, bboxes=labels[:, 1:], class_labels=labels[:, 0]) # transformed\n im, labels = new['image'], np.array([[c, *b] for c, b in zip(new['class_labels'], new['bboxes'])])\n return im, labels\n\n\ndef normalize(x, mean=IMAGENET_MEAN, std=IMAGENET_STD, inplace=False):\n # Denormalize RGB images x per ImageNet stats in BCHW format, i.e. = (x - mean) / std\n return TF.normalize(x, mean, std, inplace=inplace)\n\n\ndef denormalize(x, mean=IMAGENET_MEAN, std=IMAGENET_STD):\n # Denormalize RGB images x per ImageNet stats in BCHW format, i.e. = x * std + mean\n for i in range(3):\n x[:, i] = x[:, i] * std[i] + mean[i]\n return x\n\n\ndef augment_hsv(im, hgain=0.5, sgain=0.5, vgain=0.5):\n # HSV color-space augmentation\n if hgain or sgain or vgain:\n r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains\n hue, sat, val = cv2.split(cv2.cvtColor(im, cv2.COLOR_BGR2HSV))\n dtype = im.dtype # uint8\n\n x = np.arange(0, 256, dtype=r.dtype)\n lut_hue = ((x * r[0]) % 180).astype(dtype)\n lut_sat = np.clip(x * r[1], 0, 255).astype(dtype)\n lut_val = np.clip(x * r[2], 0, 255).astype(dtype)\n\n im_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val)))\n cv2.cvtColor(im_hsv, cv2.COLOR_HSV2BGR, dst=im) # no return needed\n\n\ndef hist_equalize(im, clahe=True, bgr=False):\n # Equalize histogram on BGR image 'im' with im.shape(n,m,3) and range 0-255\n yuv = cv2.cvtColor(im, cv2.COLOR_BGR2YUV if bgr else cv2.COLOR_RGB2YUV)\n if clahe:\n c = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n yuv[:, :, 0] = c.apply(yuv[:, :, 0])\n else:\n yuv[:, :, 0] = cv2.equalizeHist(yuv[:, :, 0]) # equalize Y channel histogram\n return cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR if bgr else cv2.COLOR_YUV2RGB) # convert YUV image to RGB\n\n\ndef replicate(im, labels):\n # Replicate labels\n h, w = im.shape[:2]\n boxes = labels[:, 1:].astype(int)\n x1, y1, x2, y2 = boxes.T\n s = ((x2 - x1) + (y2 - y1)) / 2 # side length (pixels)\n for i in s.argsort()[:round(s.size * 0.5)]: # smallest indices\n x1b, y1b, x2b, y2b = boxes[i]\n bh, bw = y2b - y1b, x2b - x1b\n yc, xc = int(random.uniform(0, h - bh)), int(random.uniform(0, w - bw)) # offset x, y\n x1a, y1a, x2a, y2a = [xc, yc, xc + bw, yc + bh]\n im[y1a:y2a, x1a:x2a] = im[y1b:y2b, x1b:x2b] # im4[ymin:ymax, xmin:xmax]\n labels = np.append(labels, [[labels[i, 0], x1a, y1a, x2a, y2a]], axis=0)\n\n return im, labels\n\n\ndef letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32):\n # Resize and pad image while meeting stride-multiple constraints\n shape = im.shape[:2] # current shape [height, width]\n if isinstance(new_shape, int):\n new_shape = (new_shape, new_shape)\n\n # Scale ratio (new / old)\n r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])\n if not scaleup: # only scale down, do not scale up (for better val mAP)\n r = min(r, 1.0)\n\n # Compute padding\n ratio = r, r # width, height ratios\n new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))\n dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding\n if auto: # minimum rectangle\n dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding\n elif scaleFill: # stretch\n dw, dh = 0.0, 0.0\n new_unpad = (new_shape[1], new_shape[0])\n ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios\n\n dw /= 2 # divide padding into 2 sides\n dh /= 2\n\n if shape[::-1] != new_unpad: # resize\n im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)\n top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))\n left, right = int(round(dw - 0.1)), int(round(dw + 0.1))\n im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border\n return im, ratio, (dw, dh)\n\n\ndef random_perspective(im,\n targets=(),\n segments=(),\n degrees=10,\n translate=.1,\n scale=.1,\n shear=10,\n perspective=0.0,\n border=(0, 0)):\n # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(0.1, 0.1), scale=(0.9, 1.1), shear=(-10, 10))\n # targets = [cls, xyxy]\n\n height = im.shape[0] + border[0] * 2 # shape(h,w,c)\n width = im.shape[1] + border[1] * 2\n\n # Center\n C = np.eye(3)\n C[0, 2] = -im.shape[1] / 2 # x translation (pixels)\n C[1, 2] = -im.shape[0] / 2 # y translation (pixels)\n\n # Perspective\n P = np.eye(3)\n P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y)\n P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x)\n\n # Rotation and Scale\n R = np.eye(3)\n a = random.uniform(-degrees, degrees)\n # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations\n s = random.uniform(1 - scale, 1 + scale)\n # s = 2 ** random.uniform(-scale, scale)\n R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s)\n\n # Shear\n S = np.eye(3)\n S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg)\n S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg)\n\n # Translation\n T = np.eye(3)\n T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width # x translation (pixels)\n T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height # y translation (pixels)\n\n # Combined rotation matrix\n M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT\n if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed\n if perspective:\n im = cv2.warpPerspective(im, M, dsize=(width, height), borderValue=(114, 114, 114))\n else: # affine\n im = cv2.warpAffine(im, M[:2], dsize=(width, height), borderValue=(114, 114, 114))\n\n # Visualize\n # import matplotlib.pyplot as plt\n # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel()\n # ax[0].imshow(im[:, :, ::-1]) # base\n # ax[1].imshow(im2[:, :, ::-1]) # warped\n\n # Transform label coordinates\n n = len(targets)\n if n:\n use_segments = any(x.any() for x in segments)\n new = np.zeros((n, 4))\n if use_segments: # warp segments\n segments = resample_segments(segments) # upsample\n for i, segment in enumerate(segments):\n xy = np.ones((len(segment), 3))\n xy[:, :2] = segment\n xy = xy @ M.T # transform\n xy = xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2] # perspective rescale or affine\n\n # clip\n new[i] = segment2box(xy, width, height)\n\n else: # warp boxes\n xy = np.ones((n * 4, 3))\n xy[:, :2] = targets[:, [1, 2, 3, 4, 1, 4, 3, 2]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1\n xy = xy @ M.T # transform\n xy = (xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2]).reshape(n, 8) # perspective rescale or affine\n\n # create new boxes\n x = xy[:, [0, 2, 4, 6]]\n y = xy[:, [1, 3, 5, 7]]\n new = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T\n\n # clip\n new[:, [0, 2]] = new[:, [0, 2]].clip(0, width)\n new[:, [1, 3]] = new[:, [1, 3]].clip(0, height)\n\n # filter candidates\n i = box_candidates(box1=targets[:, 1:5].T * s, box2=new.T, area_thr=0.01 if use_segments else 0.10)\n targets = targets[i]\n targets[:, 1:5] = new[i]\n\n return im, targets\n\n\ndef copy_paste(im, labels, segments, p=0.5):\n # Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy)\n n = len(segments)\n if p and n:\n h, w, c = im.shape # height, width, channels\n im_new = np.zeros(im.shape, np.uint8)\n\n # calculate ioa first then select indexes randomly\n boxes = np.stack([w - labels[:, 3], labels[:, 2], w - labels[:, 1], labels[:, 4]], axis=-1) # (n, 4)\n ioa = bbox_ioa(boxes, labels[:, 1:5]) # intersection over area\n indexes = np.nonzero((ioa < 0.30).all(1))[0] # (N, )\n n = len(indexes)\n for j in random.sample(list(indexes), k=round(p * n)):\n l, box, s = labels[j], boxes[j], segments[j]\n labels = np.concatenate((labels, [[l[0], *box]]), 0)\n segments.append(np.concatenate((w - s[:, 0:1], s[:, 1:2]), 1))\n cv2.drawContours(im_new, [segments[j].astype(np.int32)], -1, (1, 1, 1), cv2.FILLED)\n\n result = cv2.flip(im, 1) # augment segments (flip left-right)\n i = cv2.flip(im_new, 1).astype(bool)\n im[i] = result[i] # cv2.imwrite('debug.jpg', im) # debug\n\n return im, labels, segments\n\n\ndef cutout(im, labels, p=0.5):\n # Applies image cutout augmentation https://arxiv.org/abs/1708.04552\n if random.random() < p:\n h, w = im.shape[:2]\n scales = [0.5] * 1 + [0.25] * 2 + [0.125] * 4 + [0.0625] * 8 + [0.03125] * 16 # image size fraction\n for s in scales:\n mask_h = random.randint(1, int(h * s)) # create random masks\n mask_w = random.randint(1, int(w * s))\n\n # box\n xmin = max(0, random.randint(0, w) - mask_w // 2)\n ymin = max(0, random.randint(0, h) - mask_h // 2)\n xmax = min(w, xmin + mask_w)\n ymax = min(h, ymin + mask_h)\n\n # apply random color mask\n im[ymin:ymax, xmin:xmax] = [random.randint(64, 191) for _ in range(3)]\n\n # return unobscured labels\n if len(labels) and s > 0.03:\n box = np.array([[xmin, ymin, xmax, ymax]], dtype=np.float32)\n ioa = bbox_ioa(box, xywhn2xyxy(labels[:, 1:5], w, h))[0] # intersection over area\n labels = labels[ioa < 0.60] # remove >60% obscured labels\n\n return labels\n\n\ndef mixup(im, labels, im2, labels2):\n # Applies MixUp augmentation https://arxiv.org/pdf/1710.09412.pdf\n r = np.random.beta(32.0, 32.0) # mixup ratio, alpha=beta=32.0\n im = (im * r + im2 * (1 - r)).astype(np.uint8)\n labels = np.concatenate((labels, labels2), 0)\n return im, labels\n\n\ndef box_candidates(box1, box2, wh_thr=2, ar_thr=100, area_thr=0.1, eps=1e-16): # box1(4,n), box2(4,n)\n # Compute candidate boxes: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio\n w1, h1 = box1[2] - box1[0], box1[3] - box1[1]\n w2, h2 = box2[2] - box2[0], box2[3] - box2[1]\n ar = np.maximum(w2 / (h2 + eps), h2 / (w2 + eps)) # aspect ratio\n return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + eps) > area_thr) & (ar < ar_thr) # candidates\n\n\ndef classify_albumentations(\n augment=True,\n size=224,\n scale=(0.08, 1.0),\n ratio=(0.75, 1.0 / 0.75), # 0.75, 1.33\n hflip=0.5,\n vflip=0.0,\n jitter=0.4,\n mean=IMAGENET_MEAN,\n std=IMAGENET_STD,\n auto_aug=False):\n # YOLOv5 classification Albumentations (optional, only used if package is installed)\n prefix = colorstr('albumentations: ')\n try:\n import albumentations as A\n from albumentations.pytorch import ToTensorV2\n check_version(A.__version__, '1.0.3', hard=True) # version requirement\n if augment: # Resize and crop\n T = [A.RandomResizedCrop(height=size, width=size, scale=scale, ratio=ratio)]\n if auto_aug:\n # TODO: implement AugMix, AutoAug & RandAug in albumentation\n LOGGER.info(f'{prefix}auto augmentations are currently not supported')\n else:\n if hflip > 0:\n T += [A.HorizontalFlip(p=hflip)]\n if vflip > 0:\n T += [A.VerticalFlip(p=vflip)]\n if jitter > 0:\n color_jitter = (float(jitter),) * 3 # repeat value for brightness, contrast, satuaration, 0 hue\n T += [A.ColorJitter(*color_jitter, 0)]\n else: # Use fixed crop for eval set (reproducibility)\n T = [A.SmallestMaxSize(max_size=size), A.CenterCrop(height=size, width=size)]\n T += [A.Normalize(mean=mean, std=std), ToTensorV2()] # Normalize and convert to Tensor\n LOGGER.info(prefix + ', '.join(f'{x}'.replace('always_apply=False, ', '') for x in T if x.p))\n return A.Compose(T)\n\n except ImportError: # package not installed, skip\n LOGGER.warning(f'{prefix}⚠️ not found, install with `pip install albumentations` (recommended)')\n except Exception as e:\n LOGGER.info(f'{prefix}{e}')\n\n\ndef classify_transforms(size=224):\n # Transforms to apply if albumentations not installed\n assert isinstance(size, int), f'ERROR: classify_transforms size {size} must be integer, not (list, tuple)'\n # T.Compose([T.ToTensor(), T.Resize(size), T.CenterCrop(size), T.Normalize(IMAGENET_MEAN, IMAGENET_STD)])\n return T.Compose([CenterCrop(size), ToTensor(), T.Normalize(IMAGENET_MEAN, IMAGENET_STD)])\n\n\nclass LetterBox:\n # YOLOv5 LetterBox class for image preprocessing, i.e. T.Compose([LetterBox(size), ToTensor()])\n def __init__(self, size=(640, 640), auto=False, stride=32):\n super().__init__()\n self.h, self.w = (size, size) if isinstance(size, int) else size\n self.auto = auto # pass max size integer, automatically solve for short side using stride\n self.stride = stride # used with auto\n\n def __call__(self, im): # im = np.array HWC\n imh, imw = im.shape[:2]\n r = min(self.h / imh, self.w / imw) # ratio of new/old\n h, w = round(imh * r), round(imw * r) # resized image\n hs, ws = (math.ceil(x / self.stride) * self.stride for x in (h, w)) if self.auto else self.h, self.w\n top, left = round((hs - h) / 2 - 0.1), round((ws - w) / 2 - 0.1)\n im_out = np.full((self.h, self.w, 3), 114, dtype=im.dtype)\n im_out[top:top + h, left:left + w] = cv2.resize(im, (w, h), interpolation=cv2.INTER_LINEAR)\n return im_out\n\n\nclass CenterCrop:\n # YOLOv5 CenterCrop class for image preprocessing, i.e. T.Compose([CenterCrop(size), ToTensor()])\n def __init__(self, size=640):\n super().__init__()\n self.h, self.w = (size, size) if isinstance(size, int) else size\n\n def __call__(self, im): # im = np.array HWC\n imh, imw = im.shape[:2]\n m = min(imh, imw) # min dimension\n top, left = (imh - m) // 2, (imw - m) // 2\n return cv2.resize(im[top:top + m, left:left + m], (self.w, self.h), interpolation=cv2.INTER_LINEAR)\n\n\nclass ToTensor:\n # YOLOv5 ToTensor class for image preprocessing, i.e. T.Compose([LetterBox(size), ToTensor()])\n def __init__(self, half=False):\n super().__init__()\n self.half = half\n\n def __call__(self, im): # im = np.array HWC in BGR order\n im = np.ascontiguousarray(im.transpose((2, 0, 1))[::-1]) # HWC to CHW -> BGR to RGB -> contiguous\n im = torch.from_numpy(im) # to torch\n im = im.half() if self.half else im.float() # uint8 to fp16/32\n im /= 255.0 # 0-255 to 0.0-1.0\n return im\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/autoanchor.py", "content": "import random\n\nimport numpy as np\nimport torch\nimport yaml\nfrom tqdm import tqdm\n\nfrom asone.detectors.yolov9.yolov9.utils import TryExcept\nfrom asone.detectors.yolov9.yolov9.utils.general import LOGGER, TQDM_BAR_FORMAT, colorstr\n\nPREFIX = colorstr('AutoAnchor: ')\n\n\ndef check_anchor_order(m):\n # Check anchor order against stride order for YOLOv5 Detect() module m, and correct if necessary\n a = m.anchors.prod(-1).mean(-1).view(-1) # mean anchor area per output layer\n da = a[-1] - a[0] # delta a\n ds = m.stride[-1] - m.stride[0] # delta s\n if da and (da.sign() != ds.sign()): # same order\n LOGGER.info(f'{PREFIX}Reversing anchor order')\n m.anchors[:] = m.anchors.flip(0)\n\n\n@TryExcept(f'{PREFIX}ERROR')\ndef check_anchors(dataset, model, thr=4.0, imgsz=640):\n # Check anchor fit to data, recompute if necessary\n m = model.module.model[-1] if hasattr(model, 'module') else model.model[-1] # Detect()\n shapes = imgsz * dataset.shapes / dataset.shapes.max(1, keepdims=True)\n scale = np.random.uniform(0.9, 1.1, size=(shapes.shape[0], 1)) # augment scale\n wh = torch.tensor(np.concatenate([l[:, 3:5] * s for s, l in zip(shapes * scale, dataset.labels)])).float() # wh\n\n def metric(k): # compute metric\n r = wh[:, None] / k[None]\n x = torch.min(r, 1 / r).min(2)[0] # ratio metric\n best = x.max(1)[0] # best_x\n aat = (x > 1 / thr).float().sum(1).mean() # anchors above threshold\n bpr = (best > 1 / thr).float().mean() # best possible recall\n return bpr, aat\n\n stride = m.stride.to(m.anchors.device).view(-1, 1, 1) # model strides\n anchors = m.anchors.clone() * stride # current anchors\n bpr, aat = metric(anchors.cpu().view(-1, 2))\n s = f'\\n{PREFIX}{aat:.2f} anchors/target, {bpr:.3f} Best Possible Recall (BPR). '\n if bpr > 0.98: # threshold to recompute\n LOGGER.info(f'{s}Current anchors are a good fit to dataset ✅')\n else:\n LOGGER.info(f'{s}Anchors are a poor fit to dataset ⚠️, attempting to improve...')\n na = m.anchors.numel() // 2 # number of anchors\n anchors = kmean_anchors(dataset, n=na, img_size=imgsz, thr=thr, gen=1000, verbose=False)\n new_bpr = metric(anchors)[0]\n if new_bpr > bpr: # replace anchors\n anchors = torch.tensor(anchors, device=m.anchors.device).type_as(m.anchors)\n m.anchors[:] = anchors.clone().view_as(m.anchors)\n check_anchor_order(m) # must be in pixel-space (not grid-space)\n m.anchors /= stride\n s = f'{PREFIX}Done ✅ (optional: update model *.yaml to use these anchors in the future)'\n else:\n s = f'{PREFIX}Done ⚠️ (original anchors better than new anchors, proceeding with original anchors)'\n LOGGER.info(s)\n\n\ndef kmean_anchors(dataset='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen=1000, verbose=True):\n \"\"\" Creates kmeans-evolved anchors from training dataset\n\n Arguments:\n dataset: path to data.yaml, or a loaded dataset\n n: number of anchors\n img_size: image size used for training\n thr: anchor-label wh ratio threshold hyperparameter hyp['anchor_t'] used for training, default=4.0\n gen: generations to evolve anchors using genetic algorithm\n verbose: print all results\n\n Return:\n k: kmeans evolved anchors\n\n Usage:\n from utils.autoanchor import *; _ = kmean_anchors()\n \"\"\"\n from scipy.cluster.vq import kmeans\n\n npr = np.random\n thr = 1 / thr\n\n def metric(k, wh): # compute metrics\n r = wh[:, None] / k[None]\n x = torch.min(r, 1 / r).min(2)[0] # ratio metric\n # x = wh_iou(wh, torch.tensor(k)) # iou metric\n return x, x.max(1)[0] # x, best_x\n\n def anchor_fitness(k): # mutation fitness\n _, best = metric(torch.tensor(k, dtype=torch.float32), wh)\n return (best * (best > thr).float()).mean() # fitness\n\n def print_results(k, verbose=True):\n k = k[np.argsort(k.prod(1))] # sort small to large\n x, best = metric(k, wh0)\n bpr, aat = (best > thr).float().mean(), (x > thr).float().mean() * n # best possible recall, anch > thr\n s = f'{PREFIX}thr={thr:.2f}: {bpr:.4f} best possible recall, {aat:.2f} anchors past thr\\n' \\\n f'{PREFIX}n={n}, img_size={img_size}, metric_all={x.mean():.3f}/{best.mean():.3f}-mean/best, ' \\\n f'past_thr={x[x > thr].mean():.3f}-mean: '\n for x in k:\n s += '%i,%i, ' % (round(x[0]), round(x[1]))\n if verbose:\n LOGGER.info(s[:-2])\n return k\n\n if isinstance(dataset, str): # *.yaml file\n with open(dataset, errors='ignore') as f:\n data_dict = yaml.safe_load(f) # model dict\n from asone.detectors.yolov9.yolov9.utils.dataloaders import LoadImagesAndLabels\n dataset = LoadImagesAndLabels(data_dict['train'], augment=True, rect=True)\n\n # Get label wh\n shapes = img_size * dataset.shapes / dataset.shapes.max(1, keepdims=True)\n wh0 = np.concatenate([l[:, 3:5] * s for s, l in zip(shapes, dataset.labels)]) # wh\n\n # Filter\n i = (wh0 < 3.0).any(1).sum()\n if i:\n LOGGER.info(f'{PREFIX}WARNING ⚠️ Extremely small objects found: {i} of {len(wh0)} labels are <3 pixels in size')\n wh = wh0[(wh0 >= 2.0).any(1)].astype(np.float32) # filter > 2 pixels\n # wh = wh * (npr.rand(wh.shape[0], 1) * 0.9 + 0.1) # multiply by random scale 0-1\n\n # Kmeans init\n try:\n LOGGER.info(f'{PREFIX}Running kmeans for {n} anchors on {len(wh)} points...')\n assert n <= len(wh) # apply overdetermined constraint\n s = wh.std(0) # sigmas for whitening\n k = kmeans(wh / s, n, iter=30)[0] * s # points\n assert n == len(k) # kmeans may return fewer points than requested if wh is insufficient or too similar\n except Exception:\n LOGGER.warning(f'{PREFIX}WARNING ⚠️ switching strategies from kmeans to random init')\n k = np.sort(npr.rand(n * 2)).reshape(n, 2) * img_size # random init\n wh, wh0 = (torch.tensor(x, dtype=torch.float32) for x in (wh, wh0))\n k = print_results(k, verbose=False)\n\n # Plot\n # k, d = [None] * 20, [None] * 20\n # for i in tqdm(range(1, 21)):\n # k[i-1], d[i-1] = kmeans(wh / s, i) # points, mean distance\n # fig, ax = plt.subplots(1, 2, figsize=(14, 7), tight_layout=True)\n # ax = ax.ravel()\n # ax[0].plot(np.arange(1, 21), np.array(d) ** 2, marker='.')\n # fig, ax = plt.subplots(1, 2, figsize=(14, 7)) # plot wh\n # ax[0].hist(wh[wh[:, 0]<100, 0],400)\n # ax[1].hist(wh[wh[:, 1]<100, 1],400)\n # fig.savefig('wh.png', dpi=200)\n\n # Evolve\n f, sh, mp, s = anchor_fitness(k), k.shape, 0.9, 0.1 # fitness, generations, mutation prob, sigma\n pbar = tqdm(range(gen), bar_format=TQDM_BAR_FORMAT) # progress bar\n for _ in pbar:\n v = np.ones(sh)\n while (v == 1).all(): # mutate until a change occurs (prevent duplicates)\n v = ((npr.random(sh) < mp) * random.random() * npr.randn(*sh) * s + 1).clip(0.3, 3.0)\n kg = (k.copy() * v).clip(min=2.0)\n fg = anchor_fitness(kg)\n if fg > f:\n f, k = fg, kg.copy()\n pbar.desc = f'{PREFIX}Evolving anchors with Genetic Algorithm: fitness = {f:.4f}'\n if verbose:\n print_results(k, verbose)\n\n return print_results(k).astype(np.float32)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/autobatch.py", "content": "from copy import deepcopy\n\nimport numpy as np\nimport torch\n\nfrom asone.detectors.yolov9.yolov9.utils.general import LOGGER, colorstr\nfrom asone.detectors.yolov9.yolov9.utils.torch_utils import profile\n\n\ndef check_train_batch_size(model, imgsz=640, amp=True):\n # Check YOLOv5 training batch size\n with torch.cuda.amp.autocast(amp):\n return autobatch(deepcopy(model).train(), imgsz) # compute optimal batch size\n\n\ndef autobatch(model, imgsz=640, fraction=0.8, batch_size=16):\n # Automatically estimate best YOLOv5 batch size to use `fraction` of available CUDA memory\n # Usage:\n # import torch\n # from utils.autobatch import autobatch\n # model = torch.hub.load('ultralytics/yolov5', 'yolov5s', autoshape=False)\n # print(autobatch(model))\n\n # Check device\n prefix = colorstr('AutoBatch: ')\n LOGGER.info(f'{prefix}Computing optimal batch size for --imgsz {imgsz}')\n device = next(model.parameters()).device # get model device\n if device.type == 'cpu':\n LOGGER.info(f'{prefix}CUDA not detected, using default CPU batch-size {batch_size}')\n return batch_size\n if torch.backends.cudnn.benchmark:\n LOGGER.info(f'{prefix} ⚠️ Requires torch.backends.cudnn.benchmark=False, using default batch-size {batch_size}')\n return batch_size\n\n # Inspect CUDA memory\n gb = 1 << 30 # bytes to GiB (1024 ** 3)\n d = str(device).upper() # 'CUDA:0'\n properties = torch.cuda.get_device_properties(device) # device properties\n t = properties.total_memory / gb # GiB total\n r = torch.cuda.memory_reserved(device) / gb # GiB reserved\n a = torch.cuda.memory_allocated(device) / gb # GiB allocated\n f = t - (r + a) # GiB free\n LOGGER.info(f'{prefix}{d} ({properties.name}) {t:.2f}G total, {r:.2f}G reserved, {a:.2f}G allocated, {f:.2f}G free')\n\n # Profile batch sizes\n batch_sizes = [1, 2, 4, 8, 16]\n try:\n img = [torch.empty(b, 3, imgsz, imgsz) for b in batch_sizes]\n results = profile(img, model, n=3, device=device)\n except Exception as e:\n LOGGER.warning(f'{prefix}{e}')\n\n # Fit a solution\n y = [x[2] for x in results if x] # memory [2]\n p = np.polyfit(batch_sizes[:len(y)], y, deg=1) # first degree polynomial fit\n b = int((f * fraction - p[1]) / p[0]) # y intercept (optimal batch size)\n if None in results: # some sizes failed\n i = results.index(None) # first fail index\n if b >= batch_sizes[i]: # y intercept above failure point\n b = batch_sizes[max(i - 1, 0)] # select prior safe point\n if b < 1 or b > 1024: # b outside of safe range\n b = batch_size\n LOGGER.warning(f'{prefix}WARNING ⚠️ CUDA anomaly detected, recommend restart environment and retry command.')\n\n fraction = (np.polyval(p, b) + r + a) / t # actual fraction predicted\n LOGGER.info(f'{prefix}Using batch-size {b} for {d} {t * fraction:.2f}G/{t:.2f}G ({fraction * 100:.0f}%) ✅')\n return b\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/callbacks.py", "content": "import threading\n\n\nclass Callbacks:\n \"\"\"\"\n Handles all registered callbacks for YOLOv5 Hooks\n \"\"\"\n\n def __init__(self):\n # Define the available callbacks\n self._callbacks = {\n 'on_pretrain_routine_start': [],\n 'on_pretrain_routine_end': [],\n 'on_train_start': [],\n 'on_train_epoch_start': [],\n 'on_train_batch_start': [],\n 'optimizer_step': [],\n 'on_before_zero_grad': [],\n 'on_train_batch_end': [],\n 'on_train_epoch_end': [],\n 'on_val_start': [],\n 'on_val_batch_start': [],\n 'on_val_image_end': [],\n 'on_val_batch_end': [],\n 'on_val_end': [],\n 'on_fit_epoch_end': [], # fit = train + val\n 'on_model_save': [],\n 'on_train_end': [],\n 'on_params_update': [],\n 'teardown': [],}\n self.stop_training = False # set True to interrupt training\n\n def register_action(self, hook, name='', callback=None):\n \"\"\"\n Register a new action to a callback hook\n\n Args:\n hook: The callback hook name to register the action to\n name: The name of the action for later reference\n callback: The callback to fire\n \"\"\"\n assert hook in self._callbacks, f\"hook '{hook}' not found in callbacks {self._callbacks}\"\n assert callable(callback), f\"callback '{callback}' is not callable\"\n self._callbacks[hook].append({'name': name, 'callback': callback})\n\n def get_registered_actions(self, hook=None):\n \"\"\"\"\n Returns all the registered actions by callback hook\n\n Args:\n hook: The name of the hook to check, defaults to all\n \"\"\"\n return self._callbacks[hook] if hook else self._callbacks\n\n def run(self, hook, *args, thread=False, **kwargs):\n \"\"\"\n Loop through the registered actions and fire all callbacks on main thread\n\n Args:\n hook: The name of the hook to check, defaults to all\n args: Arguments to receive from YOLOv5\n thread: (boolean) Run callbacks in daemon thread\n kwargs: Keyword Arguments to receive from YOLOv5\n \"\"\"\n\n assert hook in self._callbacks, f\"hook '{hook}' not found in callbacks {self._callbacks}\"\n for logger in self._callbacks[hook]:\n if thread:\n threading.Thread(target=logger['callback'], args=args, kwargs=kwargs, daemon=True).start()\n else:\n logger['callback'](*args, **kwargs)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/coco_utils.py", "content": "import cv2\n\nfrom pycocotools.coco import COCO\nfrom pycocotools import mask as maskUtils\n\n# coco id: https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/\nall_instances_ids = [\n 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,\n 11, 13, 14, 15, 16, 17, 18, 19, 20,\n 21, 22, 23, 24, 25, 27, 28,\n 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,\n 41, 42, 43, 44, 46, 47, 48, 49, 50,\n 51, 52, 53, 54, 55, 56, 57, 58, 59, 60,\n 61, 62, 63, 64, 65, 67, 70,\n 72, 73, 74, 75, 76, 77, 78, 79, 80,\n 81, 82, 84, 85, 86, 87, 88, 89, 90,\n]\n\nall_stuff_ids = [\n 92, 93, 94, 95, 96, 97, 98, 99, 100,\n 101, 102, 103, 104, 105, 106, 107, 108, 109, 110,\n 111, 112, 113, 114, 115, 116, 117, 118, 119, 120,\n 121, 122, 123, 124, 125, 126, 127, 128, 129, 130,\n 131, 132, 133, 134, 135, 136, 137, 138, 139, 140,\n 141, 142, 143, 144, 145, 146, 147, 148, 149, 150,\n 151, 152, 153, 154, 155, 156, 157, 158, 159, 160,\n 161, 162, 163, 164, 165, 166, 167, 168, 169, 170,\n 171, 172, 173, 174, 175, 176, 177, 178, 179, 180,\n 181, 182,\n # other\n 183,\n # unlabeled\n 0,\n]\n\n# panoptic id: https://github.com/cocodataset/panopticapi/blob/master/panoptic_coco_categories.json\npanoptic_stuff_ids = [\n 92, 93, 95, 100,\n 107, 109,\n 112, 118, 119,\n 122, 125, 128, 130,\n 133, 138,\n 141, 144, 145, 147, 148, 149,\n 151, 154, 155, 156, 159,\n 161, 166, 168,\n 171, 175, 176, 177, 178, 180,\n 181, 184, 185, 186, 187, 188, 189, 190,\n 191, 192, 193, 194, 195, 196, 197, 198, 199, 200,\n # unlabeled\n 0,\n]\n\ndef getCocoIds(name = 'semantic'):\n if 'instances' == name:\n return all_instances_ids\n elif 'stuff' == name:\n return all_stuff_ids\n elif 'panoptic' == name:\n return all_instances_ids + panoptic_stuff_ids\n else: # semantic\n return all_instances_ids + all_stuff_ids\n\ndef getMappingId(index, name = 'semantic'):\n ids = getCocoIds(name = name)\n return ids[index]\n\ndef getMappingIndex(id, name = 'semantic'):\n ids = getCocoIds(name = name)\n return ids.index(id)\n\n# convert ann to rle encoded string\ndef annToRLE(ann, img_size):\n h, w = img_size\n segm = ann['segmentation']\n if list == type(segm):\n # polygon -- a single object might consist of multiple parts\n # we merge all parts into one mask rle code\n rles = maskUtils.frPyObjects(segm, h, w)\n rle = maskUtils.merge(rles)\n elif list == type(segm['counts']):\n # uncompressed RLE\n rle = maskUtils.frPyObjects(segm, h, w)\n else:\n # rle\n rle = ann['segmentation']\n return rle\n\n# decode ann to mask martix\ndef annToMask(ann, img_size):\n rle = annToRLE(ann, img_size)\n m = maskUtils.decode(rle)\n return m\n\n# convert mask to polygans\ndef convert_to_polys(mask):\n # opencv 3.2\n contours, hierarchy = cv2.findContours((mask).astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n\n # before opencv 3.2\n # contours, hierarchy = cv2.findContours((mask).astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n\n segmentation = []\n for contour in contours:\n contour = contour.flatten().tolist()\n if 4 < len(contour):\n segmentation.append(contour)\n\n return segmentation\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/dataloaders.py", "content": "import contextlib\nimport glob\nimport hashlib\nimport json\nimport math\nimport os\nimport random\nimport shutil\nimport time\nfrom itertools import repeat\nfrom multiprocessing.pool import Pool, ThreadPool\nfrom pathlib import Path\nfrom threading import Thread\nfrom urllib.parse import urlparse\n\nimport numpy as np\nimport psutil\nimport torch\nimport torch.nn.functional as F\nimport torchvision\nimport yaml\nfrom PIL import ExifTags, Image, ImageOps\nfrom torch.utils.data import DataLoader, Dataset, dataloader, distributed\nfrom tqdm import tqdm\n\nfrom asone.detectors.yolov9.yolov9.utils.augmentations import (Albumentations, augment_hsv, classify_albumentations, classify_transforms, copy_paste,\n letterbox, mixup, random_perspective)\nfrom asone.detectors.yolov9.yolov9.utils.general import (DATASETS_DIR, LOGGER, NUM_THREADS, TQDM_BAR_FORMAT, check_dataset, check_requirements,\n check_yaml, clean_str, cv2, is_colab, is_kaggle, segments2boxes, unzip_file, xyn2xy,\n xywh2xyxy, xywhn2xyxy, xyxy2xywhn)\nfrom asone.detectors.yolov9.yolov9.utils.torch_utils import torch_distributed_zero_first\n\n# Parameters\nHELP_URL = 'See https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data'\nIMG_FORMATS = 'bmp', 'dng', 'jpeg', 'jpg', 'mpo', 'png', 'tif', 'tiff', 'webp', 'pfm' # include image suffixes\nVID_FORMATS = 'asf', 'avi', 'gif', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mpg', 'ts', 'wmv' # include video suffixes\nLOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html\nRANK = int(os.getenv('RANK', -1))\nPIN_MEMORY = str(os.getenv('PIN_MEMORY', True)).lower() == 'true' # global pin_memory for dataloaders\n\n# Get orientation exif tag\nfor orientation in ExifTags.TAGS.keys():\n if ExifTags.TAGS[orientation] == 'Orientation':\n break\n\n\ndef get_hash(paths):\n # Returns a single hash value of a list of paths (files or dirs)\n size = sum(os.path.getsize(p) for p in paths if os.path.exists(p)) # sizes\n h = hashlib.md5(str(size).encode()) # hash sizes\n h.update(''.join(paths).encode()) # hash paths\n return h.hexdigest() # return hash\n\n\ndef exif_size(img):\n # Returns exif-corrected PIL size\n s = img.size # (width, height)\n with contextlib.suppress(Exception):\n rotation = dict(img._getexif().items())[orientation]\n if rotation in [6, 8]: # rotation 270 or 90\n s = (s[1], s[0])\n return s\n\n\ndef exif_transpose(image):\n \"\"\"\n Transpose a PIL image accordingly if it has an EXIF Orientation tag.\n Inplace version of https://github.com/python-pillow/Pillow/blob/master/src/PIL/ImageOps.py exif_transpose()\n\n :param image: The image to transpose.\n :return: An image.\n \"\"\"\n exif = image.getexif()\n orientation = exif.get(0x0112, 1) # default 1\n if orientation > 1:\n method = {\n 2: Image.FLIP_LEFT_RIGHT,\n 3: Image.ROTATE_180,\n 4: Image.FLIP_TOP_BOTTOM,\n 5: Image.TRANSPOSE,\n 6: Image.ROTATE_270,\n 7: Image.TRANSVERSE,\n 8: Image.ROTATE_90}.get(orientation)\n if method is not None:\n image = image.transpose(method)\n del exif[0x0112]\n image.info[\"exif\"] = exif.tobytes()\n return image\n\n\ndef seed_worker(worker_id):\n # Set dataloader worker seed https://pytorch.org/docs/stable/notes/randomness.html#dataloader\n worker_seed = torch.initial_seed() % 2 ** 32\n np.random.seed(worker_seed)\n random.seed(worker_seed)\n\n\ndef create_dataloader(path,\n imgsz,\n batch_size,\n stride,\n single_cls=False,\n hyp=None,\n augment=False,\n cache=False,\n pad=0.0,\n rect=False,\n rank=-1,\n workers=8,\n image_weights=False,\n close_mosaic=False,\n quad=False,\n min_items=0,\n prefix='',\n shuffle=False):\n if rect and shuffle:\n LOGGER.warning('WARNING ⚠️ --rect is incompatible with DataLoader shuffle, setting shuffle=False')\n shuffle = False\n with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP\n dataset = LoadImagesAndLabels(\n path,\n imgsz,\n batch_size,\n augment=augment, # augmentation\n hyp=hyp, # hyperparameters\n rect=rect, # rectangular batches\n cache_images=cache,\n single_cls=single_cls,\n stride=int(stride),\n pad=pad,\n image_weights=image_weights,\n min_items=min_items,\n prefix=prefix)\n\n batch_size = min(batch_size, len(dataset))\n nd = torch.cuda.device_count() # number of CUDA devices\n nw = min([os.cpu_count() // max(nd, 1), batch_size if batch_size > 1 else 0, workers]) # number of workers\n sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle)\n #loader = DataLoader if image_weights else InfiniteDataLoader # only DataLoader allows for attribute updates\n loader = DataLoader if image_weights or close_mosaic else InfiniteDataLoader\n generator = torch.Generator()\n generator.manual_seed(6148914691236517205 + RANK)\n return loader(dataset,\n batch_size=batch_size,\n shuffle=shuffle and sampler is None,\n num_workers=nw,\n sampler=sampler,\n pin_memory=PIN_MEMORY,\n collate_fn=LoadImagesAndLabels.collate_fn4 if quad else LoadImagesAndLabels.collate_fn,\n worker_init_fn=seed_worker,\n generator=generator), dataset\n\n\nclass InfiniteDataLoader(dataloader.DataLoader):\n \"\"\" Dataloader that reuses workers\n\n Uses same syntax as vanilla DataLoader\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n object.__setattr__(self, 'batch_sampler', _RepeatSampler(self.batch_sampler))\n self.iterator = super().__iter__()\n\n def __len__(self):\n return len(self.batch_sampler.sampler)\n\n def __iter__(self):\n for _ in range(len(self)):\n yield next(self.iterator)\n\n\nclass _RepeatSampler:\n \"\"\" Sampler that repeats forever\n\n Args:\n sampler (Sampler)\n \"\"\"\n\n def __init__(self, sampler):\n self.sampler = sampler\n\n def __iter__(self):\n while True:\n yield from iter(self.sampler)\n\n\nclass LoadScreenshots:\n # YOLOv5 screenshot dataloader, i.e. `python detect.py --source \"screen 0 100 100 512 256\"`\n def __init__(self, source, img_size=640, stride=32, auto=True, transforms=None):\n # source = [screen_number left top width height] (pixels)\n check_requirements('mss')\n import mss\n\n source, *params = source.split()\n self.screen, left, top, width, height = 0, None, None, None, None # default to full screen 0\n if len(params) == 1:\n self.screen = int(params[0])\n elif len(params) == 4:\n left, top, width, height = (int(x) for x in params)\n elif len(params) == 5:\n self.screen, left, top, width, height = (int(x) for x in params)\n self.img_size = img_size\n self.stride = stride\n self.transforms = transforms\n self.auto = auto\n self.mode = 'stream'\n self.frame = 0\n self.sct = mss.mss()\n\n # Parse monitor shape\n monitor = self.sct.monitors[self.screen]\n self.top = monitor[\"top\"] if top is None else (monitor[\"top\"] + top)\n self.left = monitor[\"left\"] if left is None else (monitor[\"left\"] + left)\n self.width = width or monitor[\"width\"]\n self.height = height or monitor[\"height\"]\n self.monitor = {\"left\": self.left, \"top\": self.top, \"width\": self.width, \"height\": self.height}\n\n def __iter__(self):\n return self\n\n def __next__(self):\n # mss screen capture: get raw pixels from the screen as np array\n im0 = np.array(self.sct.grab(self.monitor))[:, :, :3] # [:, :, :3] BGRA to BGR\n s = f\"screen {self.screen} (LTWH): {self.left},{self.top},{self.width},{self.height}: \"\n\n if self.transforms:\n im = self.transforms(im0) # transforms\n else:\n im = letterbox(im0, self.img_size, stride=self.stride, auto=self.auto)[0] # padded resize\n im = im.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n im = np.ascontiguousarray(im) # contiguous\n self.frame += 1\n return str(self.screen), im, im0, None, s # screen, img, original img, im0s, s\n\n\nclass LoadImages:\n # YOLOv5 image/video dataloader, i.e. `python detect.py --source image.jpg/vid.mp4`\n def __init__(self, path, img_size=640, stride=32, auto=True, transforms=None, vid_stride=1):\n files = []\n for p in sorted(path) if isinstance(path, (list, tuple)) else [path]:\n p = str(Path(p).resolve())\n if '*' in p:\n files.extend(sorted(glob.glob(p, recursive=True))) # glob\n elif os.path.isdir(p):\n files.extend(sorted(glob.glob(os.path.join(p, '*.*')))) # dir\n elif os.path.isfile(p):\n files.append(p) # files\n else:\n raise FileNotFoundError(f'{p} does not exist')\n\n images = [x for x in files if x.split('.')[-1].lower() in IMG_FORMATS]\n videos = [x for x in files if x.split('.')[-1].lower() in VID_FORMATS]\n ni, nv = len(images), len(videos)\n\n self.img_size = img_size\n self.stride = stride\n self.files = images + videos\n self.nf = ni + nv # number of files\n self.video_flag = [False] * ni + [True] * nv\n self.mode = 'image'\n self.auto = auto\n self.transforms = transforms # optional\n self.vid_stride = vid_stride # video frame-rate stride\n if any(videos):\n self._new_video(videos[0]) # new video\n else:\n self.cap = None\n assert self.nf > 0, f'No images or videos found in {p}. ' \\\n f'Supported formats are:\\nimages: {IMG_FORMATS}\\nvideos: {VID_FORMATS}'\n\n def __iter__(self):\n self.count = 0\n return self\n\n def __next__(self):\n if self.count == self.nf:\n raise StopIteration\n path = self.files[self.count]\n\n if self.video_flag[self.count]:\n # Read video\n self.mode = 'video'\n for _ in range(self.vid_stride):\n self.cap.grab()\n ret_val, im0 = self.cap.retrieve()\n while not ret_val:\n self.count += 1\n self.cap.release()\n if self.count == self.nf: # last video\n raise StopIteration\n path = self.files[self.count]\n self._new_video(path)\n ret_val, im0 = self.cap.read()\n\n self.frame += 1\n # im0 = self._cv2_rotate(im0) # for use if cv2 autorotation is False\n s = f'video {self.count + 1}/{self.nf} ({self.frame}/{self.frames}) {path}: '\n\n else:\n # Read image\n self.count += 1\n im0 = cv2.imread(path) # BGR\n assert im0 is not None, f'Image Not Found {path}'\n s = f'image {self.count}/{self.nf} {path}: '\n\n if self.transforms:\n im = self.transforms(im0) # transforms\n else:\n im = letterbox(im0, self.img_size, stride=self.stride, auto=self.auto)[0] # padded resize\n im = im.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n im = np.ascontiguousarray(im) # contiguous\n\n return path, im, im0, self.cap, s\n\n def _new_video(self, path):\n # Create a new video capture object\n self.frame = 0\n self.cap = cv2.VideoCapture(path)\n self.frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT) / self.vid_stride)\n self.orientation = int(self.cap.get(cv2.CAP_PROP_ORIENTATION_META)) # rotation degrees\n # self.cap.set(cv2.CAP_PROP_ORIENTATION_AUTO, 0) # disable https://github.com/ultralytics/yolov5/issues/8493\n\n def _cv2_rotate(self, im):\n # Rotate a cv2 video manually\n if self.orientation == 0:\n return cv2.rotate(im, cv2.ROTATE_90_CLOCKWISE)\n elif self.orientation == 180:\n return cv2.rotate(im, cv2.ROTATE_90_COUNTERCLOCKWISE)\n elif self.orientation == 90:\n return cv2.rotate(im, cv2.ROTATE_180)\n return im\n\n def __len__(self):\n return self.nf # number of files\n\n\nclass LoadStreams:\n # YOLOv5 streamloader, i.e. `python detect.py --source 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP streams`\n def __init__(self, sources='streams.txt', img_size=640, stride=32, auto=True, transforms=None, vid_stride=1):\n torch.backends.cudnn.benchmark = True # faster for fixed-size inference\n self.mode = 'stream'\n self.img_size = img_size\n self.stride = stride\n self.vid_stride = vid_stride # video frame-rate stride\n sources = Path(sources).read_text().rsplit() if os.path.isfile(sources) else [sources]\n n = len(sources)\n self.sources = [clean_str(x) for x in sources] # clean source names for later\n self.imgs, self.fps, self.frames, self.threads = [None] * n, [0] * n, [0] * n, [None] * n\n for i, s in enumerate(sources): # index, source\n # Start thread to read frames from video stream\n st = f'{i + 1}/{n}: {s}... '\n if urlparse(s).hostname in ('www.youtube.com', 'youtube.com', 'youtu.be'): # if source is YouTube video\n # YouTube format i.e. 'https://www.youtube.com/watch?v=Zgi9g1ksQHc' or 'https://youtu.be/Zgi9g1ksQHc'\n check_requirements(('pafy', 'youtube_dl==2020.12.2'))\n import pafy\n s = pafy.new(s).getbest(preftype=\"mp4\").url # YouTube URL\n s = eval(s) if s.isnumeric() else s # i.e. s = '0' local webcam\n if s == 0:\n assert not is_colab(), '--source 0 webcam unsupported on Colab. Rerun command in a local environment.'\n assert not is_kaggle(), '--source 0 webcam unsupported on Kaggle. Rerun command in a local environment.'\n cap = cv2.VideoCapture(s)\n assert cap.isOpened(), f'{st}Failed to open {s}'\n w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n fps = cap.get(cv2.CAP_PROP_FPS) # warning: may return 0 or nan\n self.frames[i] = max(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)), 0) or float('inf') # infinite stream fallback\n self.fps[i] = max((fps if math.isfinite(fps) else 0) % 100, 0) or 30 # 30 FPS fallback\n\n _, self.imgs[i] = cap.read() # guarantee first frame\n self.threads[i] = Thread(target=self.update, args=([i, cap, s]), daemon=True)\n LOGGER.info(f\"{st} Success ({self.frames[i]} frames {w}x{h} at {self.fps[i]:.2f} FPS)\")\n self.threads[i].start()\n LOGGER.info('') # newline\n\n # check for common shapes\n s = np.stack([letterbox(x, img_size, stride=stride, auto=auto)[0].shape for x in self.imgs])\n self.rect = np.unique(s, axis=0).shape[0] == 1 # rect inference if all shapes equal\n self.auto = auto and self.rect\n self.transforms = transforms # optional\n if not self.rect:\n LOGGER.warning('WARNING ⚠️ Stream shapes differ. For optimal performance supply similarly-shaped streams.')\n\n def update(self, i, cap, stream):\n # Read stream `i` frames in daemon thread\n n, f = 0, self.frames[i] # frame number, frame array\n while cap.isOpened() and n < f:\n n += 1\n cap.grab() # .read() = .grab() followed by .retrieve()\n if n % self.vid_stride == 0:\n success, im = cap.retrieve()\n if success:\n self.imgs[i] = im\n else:\n LOGGER.warning('WARNING ⚠️ Video stream unresponsive, please check your IP camera connection.')\n self.imgs[i] = np.zeros_like(self.imgs[i])\n cap.open(stream) # re-open stream if signal was lost\n time.sleep(0.0) # wait time\n\n def __iter__(self):\n self.count = -1\n return self\n\n def __next__(self):\n self.count += 1\n if not all(x.is_alive() for x in self.threads) or cv2.waitKey(1) == ord('q'): # q to quit\n cv2.destroyAllWindows()\n raise StopIteration\n\n im0 = self.imgs.copy()\n if self.transforms:\n im = np.stack([self.transforms(x) for x in im0]) # transforms\n else:\n im = np.stack([letterbox(x, self.img_size, stride=self.stride, auto=self.auto)[0] for x in im0]) # resize\n im = im[..., ::-1].transpose((0, 3, 1, 2)) # BGR to RGB, BHWC to BCHW\n im = np.ascontiguousarray(im) # contiguous\n\n return self.sources, im, im0, None, ''\n\n def __len__(self):\n return len(self.sources) # 1E12 frames = 32 streams at 30 FPS for 30 years\n\n\ndef img2label_paths(img_paths):\n # Define label paths as a function of image paths\n sa, sb = f'{os.sep}images{os.sep}', f'{os.sep}labels{os.sep}' # /images/, /labels/ substrings\n return [sb.join(x.rsplit(sa, 1)).rsplit('.', 1)[0] + '.txt' for x in img_paths]\n\n\nclass LoadImagesAndLabels(Dataset):\n # YOLOv5 train_loader/val_loader, loads images and labels for training and validation\n cache_version = 0.6 # dataset labels *.cache version\n rand_interp_methods = [cv2.INTER_NEAREST, cv2.INTER_LINEAR, cv2.INTER_CUBIC, cv2.INTER_AREA, cv2.INTER_LANCZOS4]\n\n def __init__(self,\n path,\n img_size=640,\n batch_size=16,\n augment=False,\n hyp=None,\n rect=False,\n image_weights=False,\n cache_images=False,\n single_cls=False,\n stride=32,\n pad=0.0,\n min_items=0,\n prefix=''):\n self.img_size = img_size\n self.augment = augment\n self.hyp = hyp\n self.image_weights = image_weights\n self.rect = False if image_weights else rect\n self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training)\n self.mosaic_border = [-img_size // 2, -img_size // 2]\n self.stride = stride\n self.path = path\n self.albumentations = Albumentations(size=img_size) if augment else None\n\n try:\n f = [] # image files\n for p in path if isinstance(path, list) else [path]:\n p = Path(p) # os-agnostic\n if p.is_dir(): # dir\n f += glob.glob(str(p / '**' / '*.*'), recursive=True)\n # f = list(p.rglob('*.*')) # pathlib\n elif p.is_file(): # file\n with open(p) as t:\n t = t.read().strip().splitlines()\n parent = str(p.parent) + os.sep\n f += [x.replace('./', parent, 1) if x.startswith('./') else x for x in t] # to global path\n # f += [p.parent / x.lstrip(os.sep) for x in t] # to global path (pathlib)\n else:\n raise FileNotFoundError(f'{prefix}{p} does not exist')\n self.im_files = sorted(x.replace('/', os.sep) for x in f if x.split('.')[-1].lower() in IMG_FORMATS)\n # self.img_files = sorted([x for x in f if x.suffix[1:].lower() in IMG_FORMATS]) # pathlib\n assert self.im_files, f'{prefix}No images found'\n except Exception as e:\n raise Exception(f'{prefix}Error loading data from {path}: {e}\\n{HELP_URL}') from e\n\n # Check cache\n self.label_files = img2label_paths(self.im_files) # labels\n cache_path = (p if p.is_file() else Path(self.label_files[0]).parent).with_suffix('.cache')\n try:\n cache, exists = np.load(cache_path, allow_pickle=True).item(), True # load dict\n assert cache['version'] == self.cache_version # matches current version\n assert cache['hash'] == get_hash(self.label_files + self.im_files) # identical hash\n except Exception:\n cache, exists = self.cache_labels(cache_path, prefix), False # run cache ops\n\n # Display cache\n nf, nm, ne, nc, n = cache.pop('results') # found, missing, empty, corrupt, total\n if exists and LOCAL_RANK in {-1, 0}:\n d = f\"Scanning {cache_path}... {nf} images, {nm + ne} backgrounds, {nc} corrupt\"\n tqdm(None, desc=prefix + d, total=n, initial=n, bar_format=TQDM_BAR_FORMAT) # display cache results\n if cache['msgs']:\n LOGGER.info('\\n'.join(cache['msgs'])) # display warnings\n assert nf > 0 or not augment, f'{prefix}No labels found in {cache_path}, can not start training. {HELP_URL}'\n\n # Read cache\n [cache.pop(k) for k in ('hash', 'version', 'msgs')] # remove items\n labels, shapes, self.segments = zip(*cache.values())\n nl = len(np.concatenate(labels, 0)) # number of labels\n assert nl > 0 or not augment, f'{prefix}All labels empty in {cache_path}, can not start training. {HELP_URL}'\n self.labels = list(labels)\n self.shapes = np.array(shapes)\n self.im_files = list(cache.keys()) # update\n self.label_files = img2label_paths(cache.keys()) # update\n\n # Filter images\n if min_items:\n include = np.array([len(x) >= min_items for x in self.labels]).nonzero()[0].astype(int)\n LOGGER.info(f'{prefix}{n - len(include)}/{n} images filtered from dataset')\n self.im_files = [self.im_files[i] for i in include]\n self.label_files = [self.label_files[i] for i in include]\n self.labels = [self.labels[i] for i in include]\n self.segments = [self.segments[i] for i in include]\n self.shapes = self.shapes[include] # wh\n\n # Create indices\n n = len(self.shapes) # number of images\n bi = np.floor(np.arange(n) / batch_size).astype(int) # batch index\n nb = bi[-1] + 1 # number of batches\n self.batch = bi # batch index of image\n self.n = n\n self.indices = range(n)\n\n # Update labels\n include_class = [] # filter labels to include only these classes (optional)\n include_class_array = np.array(include_class).reshape(1, -1)\n for i, (label, segment) in enumerate(zip(self.labels, self.segments)):\n if include_class:\n j = (label[:, 0:1] == include_class_array).any(1)\n self.labels[i] = label[j]\n if segment:\n self.segments[i] = segment[j]\n if single_cls: # single-class training, merge all classes into 0\n self.labels[i][:, 0] = 0\n\n # Rectangular Training\n if self.rect:\n # Sort by aspect ratio\n s = self.shapes # wh\n ar = s[:, 1] / s[:, 0] # aspect ratio\n irect = ar.argsort()\n self.im_files = [self.im_files[i] for i in irect]\n self.label_files = [self.label_files[i] for i in irect]\n self.labels = [self.labels[i] for i in irect]\n self.segments = [self.segments[i] for i in irect]\n self.shapes = s[irect] # wh\n ar = ar[irect]\n\n # Set training image shapes\n shapes = [[1, 1]] * nb\n for i in range(nb):\n ari = ar[bi == i]\n mini, maxi = ari.min(), ari.max()\n if maxi < 1:\n shapes[i] = [maxi, 1]\n elif mini > 1:\n shapes[i] = [1, 1 / mini]\n\n self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(int) * stride\n\n # Cache images into RAM/disk for faster training\n if cache_images == 'ram' and not self.check_cache_ram(prefix=prefix):\n cache_images = False\n self.ims = [None] * n\n self.npy_files = [Path(f).with_suffix('.npy') for f in self.im_files]\n if cache_images:\n b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes\n self.im_hw0, self.im_hw = [None] * n, [None] * n\n fcn = self.cache_images_to_disk if cache_images == 'disk' else self.load_image\n results = ThreadPool(NUM_THREADS).imap(fcn, range(n))\n pbar = tqdm(enumerate(results), total=n, bar_format=TQDM_BAR_FORMAT, disable=LOCAL_RANK > 0)\n for i, x in pbar:\n if cache_images == 'disk':\n b += self.npy_files[i].stat().st_size\n else: # 'ram'\n self.ims[i], self.im_hw0[i], self.im_hw[i] = x # im, hw_orig, hw_resized = load_image(self, i)\n b += self.ims[i].nbytes\n pbar.desc = f'{prefix}Caching images ({b / gb:.1f}GB {cache_images})'\n pbar.close()\n\n def check_cache_ram(self, safety_margin=0.1, prefix=''):\n # Check image caching requirements vs available memory\n b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes\n n = min(self.n, 30) # extrapolate from 30 random images\n for _ in range(n):\n im = cv2.imread(random.choice(self.im_files)) # sample image\n ratio = self.img_size / max(im.shape[0], im.shape[1]) # max(h, w) # ratio\n b += im.nbytes * ratio ** 2\n mem_required = b * self.n / n # GB required to cache dataset into RAM\n mem = psutil.virtual_memory()\n cache = mem_required * (1 + safety_margin) < mem.available # to cache or not to cache, that is the question\n if not cache:\n LOGGER.info(f\"{prefix}{mem_required / gb:.1f}GB RAM required, \"\n f\"{mem.available / gb:.1f}/{mem.total / gb:.1f}GB available, \"\n f\"{'caching images ✅' if cache else 'not caching images ⚠️'}\")\n return cache\n\n def cache_labels(self, path=Path('./labels.cache'), prefix=''):\n # Cache dataset labels, check images and read shapes\n x = {} # dict\n nm, nf, ne, nc, msgs = 0, 0, 0, 0, [] # number missing, found, empty, corrupt, messages\n desc = f\"{prefix}Scanning {path.parent / path.stem}...\"\n with Pool(NUM_THREADS) as pool:\n pbar = tqdm(pool.imap(verify_image_label, zip(self.im_files, self.label_files, repeat(prefix))),\n desc=desc,\n total=len(self.im_files),\n bar_format=TQDM_BAR_FORMAT)\n for im_file, lb, shape, segments, nm_f, nf_f, ne_f, nc_f, msg in pbar:\n nm += nm_f\n nf += nf_f\n ne += ne_f\n nc += nc_f\n if im_file:\n x[im_file] = [lb, shape, segments]\n if msg:\n msgs.append(msg)\n pbar.desc = f\"{desc} {nf} images, {nm + ne} backgrounds, {nc} corrupt\"\n\n pbar.close()\n if msgs:\n LOGGER.info('\\n'.join(msgs))\n if nf == 0:\n LOGGER.warning(f'{prefix}WARNING ⚠️ No labels found in {path}. {HELP_URL}')\n x['hash'] = get_hash(self.label_files + self.im_files)\n x['results'] = nf, nm, ne, nc, len(self.im_files)\n x['msgs'] = msgs # warnings\n x['version'] = self.cache_version # cache version\n try:\n np.save(path, x) # save cache for next time\n path.with_suffix('.cache.npy').rename(path) # remove .npy suffix\n LOGGER.info(f'{prefix}New cache created: {path}')\n except Exception as e:\n LOGGER.warning(f'{prefix}WARNING ⚠️ Cache directory {path.parent} is not writeable: {e}') # not writeable\n return x\n\n def __len__(self):\n return len(self.im_files)\n\n # def __iter__(self):\n # self.count = -1\n # print('ran dataset iter')\n # #self.shuffled_vector = np.random.permutation(self.nF) if self.augment else np.arange(self.nF)\n # return self\n\n def __getitem__(self, index):\n index = self.indices[index] # linear, shuffled, or image_weights\n\n hyp = self.hyp\n mosaic = self.mosaic and random.random() < hyp['mosaic']\n if mosaic:\n # Load mosaic\n img, labels = self.load_mosaic(index)\n shapes = None\n\n # MixUp augmentation\n if random.random() < hyp['mixup']:\n img, labels = mixup(img, labels, *self.load_mosaic(random.randint(0, self.n - 1)))\n\n else:\n # Load image\n img, (h0, w0), (h, w) = self.load_image(index)\n\n # Letterbox\n shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape\n img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment)\n shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling\n\n labels = self.labels[index].copy()\n if labels.size: # normalized xywh to pixel xyxy format\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], ratio[0] * w, ratio[1] * h, padw=pad[0], padh=pad[1])\n\n if self.augment:\n img, labels = random_perspective(img,\n labels,\n degrees=hyp['degrees'],\n translate=hyp['translate'],\n scale=hyp['scale'],\n shear=hyp['shear'],\n perspective=hyp['perspective'])\n\n nl = len(labels) # number of labels\n if nl:\n labels[:, 1:5] = xyxy2xywhn(labels[:, 1:5], w=img.shape[1], h=img.shape[0], clip=True, eps=1E-3)\n\n if self.augment:\n # Albumentations\n img, labels = self.albumentations(img, labels)\n nl = len(labels) # update after albumentations\n\n # HSV color-space\n augment_hsv(img, hgain=hyp['hsv_h'], sgain=hyp['hsv_s'], vgain=hyp['hsv_v'])\n\n # Flip up-down\n if random.random() < hyp['flipud']:\n img = np.flipud(img)\n if nl:\n labels[:, 2] = 1 - labels[:, 2]\n\n # Flip left-right\n if random.random() < hyp['fliplr']:\n img = np.fliplr(img)\n if nl:\n labels[:, 1] = 1 - labels[:, 1]\n\n # Cutouts\n # labels = cutout(img, labels, p=0.5)\n # nl = len(labels) # update after cutout\n\n labels_out = torch.zeros((nl, 6))\n if nl:\n labels_out[:, 1:] = torch.from_numpy(labels)\n\n # Convert\n img = img.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n img = np.ascontiguousarray(img)\n\n return torch.from_numpy(img), labels_out, self.im_files[index], shapes\n\n def load_image(self, i):\n # Loads 1 image from dataset index 'i', returns (im, original hw, resized hw)\n im, f, fn = self.ims[i], self.im_files[i], self.npy_files[i],\n if im is None: # not cached in RAM\n if fn.exists(): # load npy\n im = np.load(fn)\n else: # read image\n im = cv2.imread(f) # BGR\n assert im is not None, f'Image Not Found {f}'\n h0, w0 = im.shape[:2] # orig hw\n r = self.img_size / max(h0, w0) # ratio\n if r != 1: # if sizes are not equal\n interp = cv2.INTER_LINEAR if (self.augment or r > 1) else cv2.INTER_AREA\n im = cv2.resize(im, (int(w0 * r), int(h0 * r)), interpolation=interp)\n return im, (h0, w0), im.shape[:2] # im, hw_original, hw_resized\n return self.ims[i], self.im_hw0[i], self.im_hw[i] # im, hw_original, hw_resized\n\n def cache_images_to_disk(self, i):\n # Saves an image as an *.npy file for faster loading\n f = self.npy_files[i]\n if not f.exists():\n np.save(f.as_posix(), cv2.imread(self.im_files[i]))\n\n def load_mosaic(self, index):\n # YOLOv5 4-mosaic loader. Loads 1 image + 3 random images into a 4-image mosaic\n labels4, segments4 = [], []\n s = self.img_size\n yc, xc = (int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border) # mosaic center x, y\n indices = [index] + random.choices(self.indices, k=3) # 3 additional image indices\n random.shuffle(indices)\n for i, index in enumerate(indices):\n # Load image\n img, _, (h, w) = self.load_image(index)\n\n # place img in img4\n if i == 0: # top left\n img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\n x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image)\n x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image)\n elif i == 1: # top right\n x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n elif i == 2: # bottom left\n x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h)\n elif i == 3: # bottom right\n x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n\n img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]\n padw = x1a - x1b\n padh = y1a - y1b\n\n # Labels\n labels, segments = self.labels[index].copy(), self.segments[index].copy()\n if labels.size:\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh) # normalized xywh to pixel xyxy format\n segments = [xyn2xy(x, w, h, padw, padh) for x in segments]\n labels4.append(labels)\n segments4.extend(segments)\n\n # Concat/clip labels\n labels4 = np.concatenate(labels4, 0)\n for x in (labels4[:, 1:], *segments4):\n np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective()\n # img4, labels4 = replicate(img4, labels4) # replicate\n\n # Augment\n img4, labels4, segments4 = copy_paste(img4, labels4, segments4, p=self.hyp['copy_paste'])\n img4, labels4 = random_perspective(img4,\n labels4,\n segments4,\n degrees=self.hyp['degrees'],\n translate=self.hyp['translate'],\n scale=self.hyp['scale'],\n shear=self.hyp['shear'],\n perspective=self.hyp['perspective'],\n border=self.mosaic_border) # border to remove\n\n return img4, labels4\n\n def load_mosaic9(self, index):\n # YOLOv5 9-mosaic loader. Loads 1 image + 8 random images into a 9-image mosaic\n labels9, segments9 = [], []\n s = self.img_size\n indices = [index] + random.choices(self.indices, k=8) # 8 additional image indices\n random.shuffle(indices)\n hp, wp = -1, -1 # height, width previous\n for i, index in enumerate(indices):\n # Load image\n img, _, (h, w) = self.load_image(index)\n\n # place img in img9\n if i == 0: # center\n img9 = np.full((s * 3, s * 3, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\n h0, w0 = h, w\n c = s, s, s + w, s + h # xmin, ymin, xmax, ymax (base) coordinates\n elif i == 1: # top\n c = s, s - h, s + w, s\n elif i == 2: # top right\n c = s + wp, s - h, s + wp + w, s\n elif i == 3: # right\n c = s + w0, s, s + w0 + w, s + h\n elif i == 4: # bottom right\n c = s + w0, s + hp, s + w0 + w, s + hp + h\n elif i == 5: # bottom\n c = s + w0 - w, s + h0, s + w0, s + h0 + h\n elif i == 6: # bottom left\n c = s + w0 - wp - w, s + h0, s + w0 - wp, s + h0 + h\n elif i == 7: # left\n c = s - w, s + h0 - h, s, s + h0\n elif i == 8: # top left\n c = s - w, s + h0 - hp - h, s, s + h0 - hp\n\n padx, pady = c[:2]\n x1, y1, x2, y2 = (max(x, 0) for x in c) # allocate coords\n\n # Labels\n labels, segments = self.labels[index].copy(), self.segments[index].copy()\n if labels.size:\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padx, pady) # normalized xywh to pixel xyxy format\n segments = [xyn2xy(x, w, h, padx, pady) for x in segments]\n labels9.append(labels)\n segments9.extend(segments)\n\n # Image\n img9[y1:y2, x1:x2] = img[y1 - pady:, x1 - padx:] # img9[ymin:ymax, xmin:xmax]\n hp, wp = h, w # height, width previous\n\n # Offset\n yc, xc = (int(random.uniform(0, s)) for _ in self.mosaic_border) # mosaic center x, y\n img9 = img9[yc:yc + 2 * s, xc:xc + 2 * s]\n\n # Concat/clip labels\n labels9 = np.concatenate(labels9, 0)\n labels9[:, [1, 3]] -= xc\n labels9[:, [2, 4]] -= yc\n c = np.array([xc, yc]) # centers\n segments9 = [x - c for x in segments9]\n\n for x in (labels9[:, 1:], *segments9):\n np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective()\n # img9, labels9 = replicate(img9, labels9) # replicate\n\n # Augment\n img9, labels9, segments9 = copy_paste(img9, labels9, segments9, p=self.hyp['copy_paste'])\n img9, labels9 = random_perspective(img9,\n labels9,\n segments9,\n degrees=self.hyp['degrees'],\n translate=self.hyp['translate'],\n scale=self.hyp['scale'],\n shear=self.hyp['shear'],\n perspective=self.hyp['perspective'],\n border=self.mosaic_border) # border to remove\n\n return img9, labels9\n\n @staticmethod\n def collate_fn(batch):\n im, label, path, shapes = zip(*batch) # transposed\n for i, lb in enumerate(label):\n lb[:, 0] = i # add target image index for build_targets()\n return torch.stack(im, 0), torch.cat(label, 0), path, shapes\n\n @staticmethod\n def collate_fn4(batch):\n im, label, path, shapes = zip(*batch) # transposed\n n = len(shapes) // 4\n im4, label4, path4, shapes4 = [], [], path[:n], shapes[:n]\n\n ho = torch.tensor([[0.0, 0, 0, 1, 0, 0]])\n wo = torch.tensor([[0.0, 0, 1, 0, 0, 0]])\n s = torch.tensor([[1, 1, 0.5, 0.5, 0.5, 0.5]]) # scale\n for i in range(n): # zidane torch.zeros(16,3,720,1280) # BCHW\n i *= 4\n if random.random() < 0.5:\n im1 = F.interpolate(im[i].unsqueeze(0).float(), scale_factor=2.0, mode='bilinear',\n align_corners=False)[0].type(im[i].type())\n lb = label[i]\n else:\n im1 = torch.cat((torch.cat((im[i], im[i + 1]), 1), torch.cat((im[i + 2], im[i + 3]), 1)), 2)\n lb = torch.cat((label[i], label[i + 1] + ho, label[i + 2] + wo, label[i + 3] + ho + wo), 0) * s\n im4.append(im1)\n label4.append(lb)\n\n for i, lb in enumerate(label4):\n lb[:, 0] = i # add target image index for build_targets()\n\n return torch.stack(im4, 0), torch.cat(label4, 0), path4, shapes4\n\n\n# Ancillary functions --------------------------------------------------------------------------------------------------\ndef flatten_recursive(path=DATASETS_DIR / 'coco128'):\n # Flatten a recursive directory by bringing all files to top level\n new_path = Path(f'{str(path)}_flat')\n if os.path.exists(new_path):\n shutil.rmtree(new_path) # delete output folder\n os.makedirs(new_path) # make new output folder\n for file in tqdm(glob.glob(f'{str(Path(path))}/**/*.*', recursive=True)):\n shutil.copyfile(file, new_path / Path(file).name)\n\n\ndef extract_boxes(path=DATASETS_DIR / 'coco128'): # from utils.dataloaders import *; extract_boxes()\n # Convert detection dataset into classification dataset, with one directory per class\n path = Path(path) # images dir\n shutil.rmtree(path / 'classification') if (path / 'classification').is_dir() else None # remove existing\n files = list(path.rglob('*.*'))\n n = len(files) # number of files\n for im_file in tqdm(files, total=n):\n if im_file.suffix[1:] in IMG_FORMATS:\n # image\n im = cv2.imread(str(im_file))[..., ::-1] # BGR to RGB\n h, w = im.shape[:2]\n\n # labels\n lb_file = Path(img2label_paths([str(im_file)])[0])\n if Path(lb_file).exists():\n with open(lb_file) as f:\n lb = np.array([x.split() for x in f.read().strip().splitlines()], dtype=np.float32) # labels\n\n for j, x in enumerate(lb):\n c = int(x[0]) # class\n f = (path / 'classifier') / f'{c}' / f'{path.stem}_{im_file.stem}_{j}.jpg' # new filename\n if not f.parent.is_dir():\n f.parent.mkdir(parents=True)\n\n b = x[1:] * [w, h, w, h] # box\n # b[2:] = b[2:].max() # rectangle to square\n b[2:] = b[2:] * 1.2 + 3 # pad\n b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(int)\n\n b[[0, 2]] = np.clip(b[[0, 2]], 0, w) # clip boxes outside of image\n b[[1, 3]] = np.clip(b[[1, 3]], 0, h)\n assert cv2.imwrite(str(f), im[b[1]:b[3], b[0]:b[2]]), f'box failure in {f}'\n\n\ndef autosplit(path=DATASETS_DIR / 'coco128/images', weights=(0.9, 0.1, 0.0), annotated_only=False):\n \"\"\" Autosplit a dataset into train/val/test splits and save path/autosplit_*.txt files\n Usage: from utils.dataloaders import *; autosplit()\n Arguments\n path: Path to images directory\n weights: Train, val, test weights (list, tuple)\n annotated_only: Only use images with an annotated txt file\n \"\"\"\n path = Path(path) # images dir\n files = sorted(x for x in path.rglob('*.*') if x.suffix[1:].lower() in IMG_FORMATS) # image files only\n n = len(files) # number of files\n random.seed(0) # for reproducibility\n indices = random.choices([0, 1, 2], weights=weights, k=n) # assign each image to a split\n\n txt = ['autosplit_train.txt', 'autosplit_val.txt', 'autosplit_test.txt'] # 3 txt files\n for x in txt:\n if (path.parent / x).exists():\n (path.parent / x).unlink() # remove existing\n\n print(f'Autosplitting images from {path}' + ', using *.txt labeled images only' * annotated_only)\n for i, img in tqdm(zip(indices, files), total=n):\n if not annotated_only or Path(img2label_paths([str(img)])[0]).exists(): # check label\n with open(path.parent / txt[i], 'a') as f:\n f.write(f'./{img.relative_to(path.parent).as_posix()}' + '\\n') # add image to txt file\n\n\ndef verify_image_label(args):\n # Verify one image-label pair\n im_file, lb_file, prefix = args\n nm, nf, ne, nc, msg, segments = 0, 0, 0, 0, '', [] # number (missing, found, empty, corrupt), message, segments\n try:\n # verify images\n im = Image.open(im_file)\n im.verify() # PIL verify\n shape = exif_size(im) # image size\n assert (shape[0] > 9) & (shape[1] > 9), f'image size {shape} <10 pixels'\n assert im.format.lower() in IMG_FORMATS, f'invalid image format {im.format}'\n if im.format.lower() in ('jpg', 'jpeg'):\n with open(im_file, 'rb') as f:\n f.seek(-2, 2)\n if f.read() != b'\\xff\\xd9': # corrupt JPEG\n ImageOps.exif_transpose(Image.open(im_file)).save(im_file, 'JPEG', subsampling=0, quality=100)\n msg = f'{prefix}WARNING ⚠️ {im_file}: corrupt JPEG restored and saved'\n\n # verify labels\n if os.path.isfile(lb_file):\n nf = 1 # label found\n with open(lb_file) as f:\n lb = [x.split() for x in f.read().strip().splitlines() if len(x)]\n if any(len(x) > 6 for x in lb): # is segment\n classes = np.array([x[0] for x in lb], dtype=np.float32)\n segments = [np.array(x[1:], dtype=np.float32).reshape(-1, 2) for x in lb] # (cls, xy1...)\n lb = np.concatenate((classes.reshape(-1, 1), segments2boxes(segments)), 1) # (cls, xywh)\n lb = np.array(lb, dtype=np.float32)\n nl = len(lb)\n if nl:\n assert lb.shape[1] == 5, f'labels require 5 columns, {lb.shape[1]} columns detected'\n assert (lb >= 0).all(), f'negative label values {lb[lb < 0]}'\n assert (lb[:, 1:] <= 1).all(), f'non-normalized or out of bounds coordinates {lb[:, 1:][lb[:, 1:] > 1]}'\n _, i = np.unique(lb, axis=0, return_index=True)\n if len(i) < nl: # duplicate row check\n lb = lb[i] # remove duplicates\n if segments:\n segments = [segments[x] for x in i]\n msg = f'{prefix}WARNING ⚠️ {im_file}: {nl - len(i)} duplicate labels removed'\n else:\n ne = 1 # label empty\n lb = np.zeros((0, 5), dtype=np.float32)\n else:\n nm = 1 # label missing\n lb = np.zeros((0, 5), dtype=np.float32)\n return im_file, lb, shape, segments, nm, nf, ne, nc, msg\n except Exception as e:\n nc = 1\n msg = f'{prefix}WARNING ⚠️ {im_file}: ignoring corrupt image/label: {e}'\n return [None, None, None, None, nm, nf, ne, nc, msg]\n\n\nclass HUBDatasetStats():\n \"\"\" Class for generating HUB dataset JSON and `-hub` dataset directory\n\n Arguments\n path: Path to data.yaml or data.zip (with data.yaml inside data.zip)\n autodownload: Attempt to download dataset if not found locally\n\n Usage\n from utils.dataloaders import HUBDatasetStats\n stats = HUBDatasetStats('coco128.yaml', autodownload=True) # usage 1\n stats = HUBDatasetStats('path/to/coco128.zip') # usage 2\n stats.get_json(save=False)\n stats.process_images()\n \"\"\"\n\n def __init__(self, path='coco128.yaml', autodownload=False):\n # Initialize class\n zipped, data_dir, yaml_path = self._unzip(Path(path))\n try:\n with open(check_yaml(yaml_path), errors='ignore') as f:\n data = yaml.safe_load(f) # data dict\n if zipped:\n data['path'] = data_dir\n except Exception as e:\n raise Exception(\"error/HUB/dataset_stats/yaml_load\") from e\n\n check_dataset(data, autodownload) # download dataset if missing\n self.hub_dir = Path(data['path'] + '-hub')\n self.im_dir = self.hub_dir / 'images'\n self.im_dir.mkdir(parents=True, exist_ok=True) # makes /images\n self.stats = {'nc': data['nc'], 'names': list(data['names'].values())} # statistics dictionary\n self.data = data\n\n @staticmethod\n def _find_yaml(dir):\n # Return data.yaml file\n files = list(dir.glob('*.yaml')) or list(dir.rglob('*.yaml')) # try root level first and then recursive\n assert files, f'No *.yaml file found in {dir}'\n if len(files) > 1:\n files = [f for f in files if f.stem == dir.stem] # prefer *.yaml files that match dir name\n assert files, f'Multiple *.yaml files found in {dir}, only 1 *.yaml file allowed'\n assert len(files) == 1, f'Multiple *.yaml files found: {files}, only 1 *.yaml file allowed in {dir}'\n return files[0]\n\n def _unzip(self, path):\n # Unzip data.zip\n if not str(path).endswith('.zip'): # path is data.yaml\n return False, None, path\n assert Path(path).is_file(), f'Error unzipping {path}, file not found'\n unzip_file(path, path=path.parent)\n dir = path.with_suffix('') # dataset directory == zip name\n assert dir.is_dir(), f'Error unzipping {path}, {dir} not found. path/to/abc.zip MUST unzip to path/to/abc/'\n return True, str(dir), self._find_yaml(dir) # zipped, data_dir, yaml_path\n\n def _hub_ops(self, f, max_dim=1920):\n # HUB ops for 1 image 'f': resize and save at reduced quality in /dataset-hub for web/app viewing\n f_new = self.im_dir / Path(f).name # dataset-hub image filename\n try: # use PIL\n im = Image.open(f)\n r = max_dim / max(im.height, im.width) # ratio\n if r < 1.0: # image too large\n im = im.resize((int(im.width * r), int(im.height * r)))\n im.save(f_new, 'JPEG', quality=50, optimize=True) # save\n except Exception as e: # use OpenCV\n LOGGER.info(f'WARNING ⚠️ HUB ops PIL failure {f}: {e}')\n im = cv2.imread(f)\n im_height, im_width = im.shape[:2]\n r = max_dim / max(im_height, im_width) # ratio\n if r < 1.0: # image too large\n im = cv2.resize(im, (int(im_width * r), int(im_height * r)), interpolation=cv2.INTER_AREA)\n cv2.imwrite(str(f_new), im)\n\n def get_json(self, save=False, verbose=False):\n # Return dataset JSON for Ultralytics HUB\n def _round(labels):\n # Update labels to integer class and 6 decimal place floats\n return [[int(c), *(round(x, 4) for x in points)] for c, *points in labels]\n\n for split in 'train', 'val', 'test':\n if self.data.get(split) is None:\n self.stats[split] = None # i.e. no test set\n continue\n dataset = LoadImagesAndLabels(self.data[split]) # load dataset\n x = np.array([\n np.bincount(label[:, 0].astype(int), minlength=self.data['nc'])\n for label in tqdm(dataset.labels, total=dataset.n, desc='Statistics')]) # shape(128x80)\n self.stats[split] = {\n 'instance_stats': {\n 'total': int(x.sum()),\n 'per_class': x.sum(0).tolist()},\n 'image_stats': {\n 'total': dataset.n,\n 'unlabelled': int(np.all(x == 0, 1).sum()),\n 'per_class': (x > 0).sum(0).tolist()},\n 'labels': [{\n str(Path(k).name): _round(v.tolist())} for k, v in zip(dataset.im_files, dataset.labels)]}\n\n # Save, print and return\n if save:\n stats_path = self.hub_dir / 'stats.json'\n print(f'Saving {stats_path.resolve()}...')\n with open(stats_path, 'w') as f:\n json.dump(self.stats, f) # save stats.json\n if verbose:\n print(json.dumps(self.stats, indent=2, sort_keys=False))\n return self.stats\n\n def process_images(self):\n # Compress images for Ultralytics HUB\n for split in 'train', 'val', 'test':\n if self.data.get(split) is None:\n continue\n dataset = LoadImagesAndLabels(self.data[split]) # load dataset\n desc = f'{split} images'\n for _ in tqdm(ThreadPool(NUM_THREADS).imap(self._hub_ops, dataset.im_files), total=dataset.n, desc=desc):\n pass\n print(f'Done. All images saved to {self.im_dir}')\n return self.im_dir\n\n\n# Classification dataloaders -------------------------------------------------------------------------------------------\nclass ClassificationDataset(torchvision.datasets.ImageFolder):\n \"\"\"\n YOLOv5 Classification Dataset.\n Arguments\n root: Dataset path\n transform: torchvision transforms, used by default\n album_transform: Albumentations transforms, used if installed\n \"\"\"\n\n def __init__(self, root, augment, imgsz, cache=False):\n super().__init__(root=root)\n self.torch_transforms = classify_transforms(imgsz)\n self.album_transforms = classify_albumentations(augment, imgsz) if augment else None\n self.cache_ram = cache is True or cache == 'ram'\n self.cache_disk = cache == 'disk'\n self.samples = [list(x) + [Path(x[0]).with_suffix('.npy'), None] for x in self.samples] # file, index, npy, im\n\n def __getitem__(self, i):\n f, j, fn, im = self.samples[i] # filename, index, filename.with_suffix('.npy'), image\n if self.cache_ram and im is None:\n im = self.samples[i][3] = cv2.imread(f)\n elif self.cache_disk:\n if not fn.exists(): # load npy\n np.save(fn.as_posix(), cv2.imread(f))\n im = np.load(fn)\n else: # read image\n im = cv2.imread(f) # BGR\n if self.album_transforms:\n sample = self.album_transforms(image=cv2.cvtColor(im, cv2.COLOR_BGR2RGB))[\"image\"]\n else:\n sample = self.torch_transforms(im)\n return sample, j\n\n\ndef create_classification_dataloader(path,\n imgsz=224,\n batch_size=16,\n augment=True,\n cache=False,\n rank=-1,\n workers=8,\n shuffle=True):\n # Returns Dataloader object to be used with YOLOv5 Classifier\n with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP\n dataset = ClassificationDataset(root=path, imgsz=imgsz, augment=augment, cache=cache)\n batch_size = min(batch_size, len(dataset))\n nd = torch.cuda.device_count()\n nw = min([os.cpu_count() // max(nd, 1), batch_size if batch_size > 1 else 0, workers])\n sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle)\n generator = torch.Generator()\n generator.manual_seed(6148914691236517205 + RANK)\n return InfiniteDataLoader(dataset,\n batch_size=batch_size,\n shuffle=shuffle and sampler is None,\n num_workers=nw,\n sampler=sampler,\n pin_memory=PIN_MEMORY,\n worker_init_fn=seed_worker,\n generator=generator) # or DataLoader(persistent_workers=True)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/downloads.py", "content": "import logging\nimport os\nimport subprocess\nimport urllib\nfrom pathlib import Path\n\nimport requests\nimport torch\n\n\ndef is_url(url, check=True):\n # Check if string is URL and check if URL exists\n try:\n url = str(url)\n result = urllib.parse.urlparse(url)\n assert all([result.scheme, result.netloc]) # check if is url\n return (urllib.request.urlopen(url).getcode() == 200) if check else True # check if exists online\n except (AssertionError, urllib.request.HTTPError):\n return False\n\n\ndef gsutil_getsize(url=''):\n # gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du\n s = subprocess.check_output(f'gsutil du {url}', shell=True).decode('utf-8')\n return eval(s.split(' ')[0]) if len(s) else 0 # bytes\n\n\ndef url_getsize(url='https://ultralytics.com/images/bus.jpg'):\n # Return downloadable file size in bytes\n response = requests.head(url, allow_redirects=True)\n return int(response.headers.get('content-length', -1))\n\n\ndef safe_download(file, url, url2=None, min_bytes=1E0, error_msg=''):\n # Attempts to download file from url or url2, checks and removes incomplete downloads < min_bytes\n from asone.detectors.yolov9.yolov9.utils.general import LOGGER\n\n file = Path(file)\n assert_msg = f\"Downloaded file '{file}' does not exist or size is < min_bytes={min_bytes}\"\n try: # url1\n LOGGER.info(f'Downloading {url} to {file}...')\n torch.hub.download_url_to_file(url, str(file), progress=LOGGER.level <= logging.INFO)\n assert file.exists() and file.stat().st_size > min_bytes, assert_msg # check\n except Exception as e: # url2\n if file.exists():\n file.unlink() # remove partial downloads\n LOGGER.info(f'ERROR: {e}\\nRe-attempting {url2 or url} to {file}...')\n os.system(f\"curl -# -L '{url2 or url}' -o '{file}' --retry 3 -C -\") # curl download, retry and resume on fail\n finally:\n if not file.exists() or file.stat().st_size < min_bytes: # check\n if file.exists():\n file.unlink() # remove partial downloads\n LOGGER.info(f\"ERROR: {assert_msg}\\n{error_msg}\")\n LOGGER.info('')\n\n\ndef attempt_download(file, repo='ultralytics/yolov5', release='v7.0'):\n # Attempt file download from GitHub release assets if not found locally. release = 'latest', 'v7.0', etc.\n from asone.detectors.yolov9.yolov9.utils.general import LOGGER\n\n def github_assets(repository, version='latest'):\n # Return GitHub repo tag (i.e. 'v7.0') and assets (i.e. ['yolov5s.pt', 'yolov5m.pt', ...])\n if version != 'latest':\n version = f'tags/{version}' # i.e. tags/v7.0\n response = requests.get(f'https://api.github.com/repos/{repository}/releases/{version}').json() # github api\n return response['tag_name'], [x['name'] for x in response['assets']] # tag, assets\n\n file = Path(str(file).strip().replace(\"'\", ''))\n if not file.exists():\n # URL specified\n name = Path(urllib.parse.unquote(str(file))).name # decode '%2F' to '/' etc.\n if str(file).startswith(('http:/', 'https:/')): # download\n url = str(file).replace(':/', '://') # Pathlib turns :// -> :/\n file = name.split('?')[0] # parse authentication https://url.com/file.txt?auth...\n if Path(file).is_file():\n LOGGER.info(f'Found {url} locally at {file}') # file already exists\n else:\n safe_download(file=file, url=url, min_bytes=1E5)\n return file\n\n # GitHub assets\n assets = [f'yolov5{size}{suffix}.pt' for size in 'nsmlx' for suffix in ('', '6', '-cls', '-seg')] # default\n try:\n tag, assets = github_assets(repo, release)\n except Exception:\n try:\n tag, assets = github_assets(repo) # latest release\n except Exception:\n try:\n tag = subprocess.check_output('git tag', shell=True, stderr=subprocess.STDOUT).decode().split()[-1]\n except Exception:\n tag = release\n\n file.parent.mkdir(parents=True, exist_ok=True) # make parent dir (if required)\n if name in assets:\n url3 = 'https://drive.google.com/drive/folders/1EFQTEUeXWSFww0luse2jB9M1QNZQGwNl' # backup gdrive mirror\n safe_download(\n file,\n url=f'https://github.com/{repo}/releases/download/{tag}/{name}',\n min_bytes=1E5,\n error_msg=f'{file} missing, try downloading from https://github.com/{repo}/releases/{tag} or {url3}')\n\n return str(file)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/general.py", "content": "import contextlib\nimport glob\nimport inspect\nimport logging\nimport logging.config\nimport math\nimport os\nimport platform\nimport random\nimport re\nimport signal\nimport sys\nimport time\nimport urllib\nfrom copy import deepcopy\nfrom datetime import datetime\nfrom itertools import repeat\nfrom multiprocessing.pool import ThreadPool\nfrom pathlib import Path\nfrom subprocess import check_output\nfrom tarfile import is_tarfile\nfrom typing import Optional\nfrom zipfile import ZipFile, is_zipfile\n\nimport cv2\nimport IPython\nimport numpy as np\nimport pandas as pd\nimport pkg_resources as pkg\nimport torch\nimport torchvision\nimport yaml\n\nfrom asone.detectors.yolov9.yolov9.utils import TryExcept, emojis\nfrom asone.detectors.yolov9.yolov9.utils.downloads import gsutil_getsize\nfrom asone.detectors.yolov9.yolov9.utils.metrics import box_iou, fitness\n\nFILE = Path(__file__).resolve()\nROOT = FILE.parents[1] # YOLO root directory\nRANK = int(os.getenv('RANK', -1))\n\n# Settings\nNUM_THREADS = min(8, max(1, os.cpu_count() - 1)) # number of YOLOv5 multiprocessing threads\nDATASETS_DIR = Path(os.getenv('YOLOv5_DATASETS_DIR', ROOT.parent / 'datasets')) # global datasets directory\nAUTOINSTALL = str(os.getenv('YOLOv5_AUTOINSTALL', True)).lower() == 'true' # global auto-install mode\nVERBOSE = str(os.getenv('YOLOv5_VERBOSE', True)).lower() == 'true' # global verbose mode\nTQDM_BAR_FORMAT = '{l_bar}{bar:10}| {n_fmt}/{total_fmt} {elapsed}' # tqdm bar format\nFONT = 'Arial.ttf' # https://ultralytics.com/assets/Arial.ttf\n\ntorch.set_printoptions(linewidth=320, precision=5, profile='long')\nnp.set_printoptions(linewidth=320, formatter={'float_kind': '{:11.5g}'.format}) # format short g, %precision=5\npd.options.display.max_columns = 10\ncv2.setNumThreads(0) # prevent OpenCV from multithreading (incompatible with PyTorch DataLoader)\nos.environ['NUMEXPR_MAX_THREADS'] = str(NUM_THREADS) # NumExpr max threads\nos.environ['OMP_NUM_THREADS'] = '1' if platform.system() == 'darwin' else str(NUM_THREADS) # OpenMP (PyTorch and SciPy)\n\n\ndef is_ascii(s=''):\n # Is string composed of all ASCII (no UTF) characters? (note str().isascii() introduced in python 3.7)\n s = str(s) # convert list, tuple, None, etc. to str\n return len(s.encode().decode('ascii', 'ignore')) == len(s)\n\n\ndef is_chinese(s='人工智能'):\n # Is string composed of any Chinese characters?\n return bool(re.search('[\\u4e00-\\u9fff]', str(s)))\n\n\ndef is_colab():\n # Is environment a Google Colab instance?\n return 'google.colab' in sys.modules\n\n\ndef is_notebook():\n # Is environment a Jupyter notebook? Verified on Colab, Jupyterlab, Kaggle, Paperspace\n ipython_type = str(type(IPython.get_ipython()))\n return 'colab' in ipython_type or 'zmqshell' in ipython_type\n\n\ndef is_kaggle():\n # Is environment a Kaggle Notebook?\n return os.environ.get('PWD') == '/kaggle/working' and os.environ.get('KAGGLE_URL_BASE') == 'https://www.kaggle.com'\n\n\ndef is_docker() -> bool:\n \"\"\"Check if the process runs inside a docker container.\"\"\"\n if Path(\"/.dockerenv\").exists():\n return True\n try: # check if docker is in control groups\n with open(\"/proc/self/cgroup\") as file:\n return any(\"docker\" in line for line in file)\n except OSError:\n return False\n\n\ndef is_writeable(dir, test=False):\n # Return True if directory has write permissions, test opening a file with write permissions if test=True\n if not test:\n return os.access(dir, os.W_OK) # possible issues on Windows\n file = Path(dir) / 'tmp.txt'\n try:\n with open(file, 'w'): # open file with write permissions\n pass\n file.unlink() # remove file\n return True\n except OSError:\n return False\n\n\nLOGGING_NAME = \"yolov5\"\n\n\ndef set_logging(name=LOGGING_NAME, verbose=True):\n # sets up logging for the given name\n rank = int(os.getenv('RANK', -1)) # rank in world for Multi-GPU trainings\n level = logging.INFO if verbose and rank in {-1, 0} else logging.ERROR\n logging.config.dictConfig({\n \"version\": 1,\n \"disable_existing_loggers\": False,\n \"formatters\": {\n name: {\n \"format\": \"%(message)s\"}},\n \"handlers\": {\n name: {\n \"class\": \"logging.StreamHandler\",\n \"formatter\": name,\n \"level\": level,}},\n \"loggers\": {\n name: {\n \"level\": level,\n \"handlers\": [name],\n \"propagate\": False,}}})\n\n\nset_logging(LOGGING_NAME) # run before defining LOGGER\nLOGGER = logging.getLogger(LOGGING_NAME) # define globally (used in train.py, val.py, detect.py, etc.)\nif platform.system() == 'Windows':\n for fn in LOGGER.info, LOGGER.warning:\n setattr(LOGGER, fn.__name__, lambda x: fn(emojis(x))) # emoji safe logging\n\n\ndef user_config_dir(dir='Ultralytics', env_var='YOLOV5_CONFIG_DIR'):\n # Return path of user configuration directory. Prefer environment variable if exists. Make dir if required.\n env = os.getenv(env_var)\n if env:\n path = Path(env) # use environment variable\n else:\n cfg = {'Windows': 'AppData/Roaming', 'Linux': '.config', 'Darwin': 'Library/Application Support'} # 3 OS dirs\n path = Path.home() / cfg.get(platform.system(), '') # OS-specific config dir\n path = (path if is_writeable(path) else Path('/tmp')) / dir # GCP and AWS lambda fix, only /tmp is writeable\n path.mkdir(exist_ok=True) # make if required\n return path\n\n\nCONFIG_DIR = user_config_dir() # Ultralytics settings dir\n\n\nclass Profile(contextlib.ContextDecorator):\n # YOLO Profile class. Usage: @Profile() decorator or 'with Profile():' context manager\n def __init__(self, t=0.0):\n self.t = t\n self.cuda = torch.cuda.is_available()\n\n def __enter__(self):\n self.start = self.time()\n return self\n\n def __exit__(self, type, value, traceback):\n self.dt = self.time() - self.start # delta-time\n self.t += self.dt # accumulate dt\n\n def time(self):\n if self.cuda:\n torch.cuda.synchronize()\n return time.time()\n\n\nclass Timeout(contextlib.ContextDecorator):\n # YOLO Timeout class. Usage: @Timeout(seconds) decorator or 'with Timeout(seconds):' context manager\n def __init__(self, seconds, *, timeout_msg='', suppress_timeout_errors=True):\n self.seconds = int(seconds)\n self.timeout_message = timeout_msg\n self.suppress = bool(suppress_timeout_errors)\n\n def _timeout_handler(self, signum, frame):\n raise TimeoutError(self.timeout_message)\n\n def __enter__(self):\n if platform.system() != 'Windows': # not supported on Windows\n signal.signal(signal.SIGALRM, self._timeout_handler) # Set handler for SIGALRM\n signal.alarm(self.seconds) # start countdown for SIGALRM to be raised\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n if platform.system() != 'Windows':\n signal.alarm(0) # Cancel SIGALRM if it's scheduled\n if self.suppress and exc_type is TimeoutError: # Suppress TimeoutError\n return True\n\n\nclass WorkingDirectory(contextlib.ContextDecorator):\n # Usage: @WorkingDirectory(dir) decorator or 'with WorkingDirectory(dir):' context manager\n def __init__(self, new_dir):\n self.dir = new_dir # new dir\n self.cwd = Path.cwd().resolve() # current dir\n\n def __enter__(self):\n os.chdir(self.dir)\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n os.chdir(self.cwd)\n\n\ndef methods(instance):\n # Get class/instance methods\n return [f for f in dir(instance) if callable(getattr(instance, f)) and not f.startswith(\"__\")]\n\n\ndef print_args(args: Optional[dict] = None, show_file=True, show_func=False):\n # Print function arguments (optional args dict)\n x = inspect.currentframe().f_back # previous frame\n file, _, func, _, _ = inspect.getframeinfo(x)\n if args is None: # get args automatically\n args, _, _, frm = inspect.getargvalues(x)\n args = {k: v for k, v in frm.items() if k in args}\n try:\n file = Path(file).resolve().relative_to(ROOT).with_suffix('')\n except ValueError:\n file = Path(file).stem\n s = (f'{file}: ' if show_file else '') + (f'{func}: ' if show_func else '')\n LOGGER.info(colorstr(s) + ', '.join(f'{k}={v}' for k, v in args.items()))\n\n\ndef init_seeds(seed=0, deterministic=False):\n # Initialize random number generator (RNG) seeds https://pytorch.org/docs/stable/notes/randomness.html\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) # for Multi-GPU, exception safe\n # torch.backends.cudnn.benchmark = True # AutoBatch problem https://github.com/ultralytics/yolov5/issues/9287\n if deterministic and check_version(torch.__version__, '1.12.0'): # https://github.com/ultralytics/yolov5/pull/8213\n torch.use_deterministic_algorithms(True)\n torch.backends.cudnn.deterministic = True\n os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8'\n os.environ['PYTHONHASHSEED'] = str(seed)\n\n\ndef intersect_dicts(da, db, exclude=()):\n # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values\n return {k: v for k, v in da.items() if k in db and all(x not in k for x in exclude) and v.shape == db[k].shape}\n\n\ndef get_default_args(func):\n # Get func() default arguments\n signature = inspect.signature(func)\n return {k: v.default for k, v in signature.parameters.items() if v.default is not inspect.Parameter.empty}\n\n\ndef get_latest_run(search_dir='.'):\n # Return path to most recent 'last.pt' in /runs (i.e. to --resume from)\n last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True)\n return max(last_list, key=os.path.getctime) if last_list else ''\n\n\ndef file_age(path=__file__):\n # Return days since last file update\n dt = (datetime.now() - datetime.fromtimestamp(Path(path).stat().st_mtime)) # delta\n return dt.days # + dt.seconds / 86400 # fractional days\n\n\ndef file_date(path=__file__):\n # Return human-readable file modification date, i.e. '2021-3-26'\n t = datetime.fromtimestamp(Path(path).stat().st_mtime)\n return f'{t.year}-{t.month}-{t.day}'\n\n\ndef file_size(path):\n # Return file/dir size (MB)\n mb = 1 << 20 # bytes to MiB (1024 ** 2)\n path = Path(path)\n if path.is_file():\n return path.stat().st_size / mb\n elif path.is_dir():\n return sum(f.stat().st_size for f in path.glob('**/*') if f.is_file()) / mb\n else:\n return 0.0\n\n\ndef check_online():\n # Check internet connectivity\n import socket\n\n def run_once():\n # Check once\n try:\n socket.create_connection((\"1.1.1.1\", 443), 5) # check host accessibility\n return True\n except OSError:\n return False\n\n return run_once() or run_once() # check twice to increase robustness to intermittent connectivity issues\n\n\ndef git_describe(path=ROOT): # path must be a directory\n # Return human-readable git description, i.e. v5.0-5-g3e25f1e https://git-scm.com/docs/git-describe\n try:\n assert (Path(path) / '.git').is_dir()\n return check_output(f'git -C {path} describe --tags --long --always', shell=True).decode()[:-1]\n except Exception:\n return ''\n\n\n@TryExcept()\n@WorkingDirectory(ROOT)\ndef check_git_status(repo='WongKinYiu/yolov9', branch='main'):\n # YOLO status check, recommend 'git pull' if code is out of date\n url = f'https://github.com/{repo}'\n msg = f', for updates see {url}'\n s = colorstr('github: ') # string\n assert Path('.git').exists(), s + 'skipping check (not a git repository)' + msg\n assert check_online(), s + 'skipping check (offline)' + msg\n\n splits = re.split(pattern=r'\\s', string=check_output('git remote -v', shell=True).decode())\n matches = [repo in s for s in splits]\n if any(matches):\n remote = splits[matches.index(True) - 1]\n else:\n remote = 'ultralytics'\n check_output(f'git remote add {remote} {url}', shell=True)\n check_output(f'git fetch {remote}', shell=True, timeout=5) # git fetch\n local_branch = check_output('git rev-parse --abbrev-ref HEAD', shell=True).decode().strip() # checked out\n n = int(check_output(f'git rev-list {local_branch}..{remote}/{branch} --count', shell=True)) # commits behind\n if n > 0:\n pull = 'git pull' if remote == 'origin' else f'git pull {remote} {branch}'\n s += f\"⚠️ YOLO is out of date by {n} commit{'s' * (n > 1)}. Use `{pull}` or `git clone {url}` to update.\"\n else:\n s += f'up to date with {url} ✅'\n LOGGER.info(s)\n\n\n@WorkingDirectory(ROOT)\ndef check_git_info(path='.'):\n # YOLO git info check, return {remote, branch, commit}\n check_requirements('gitpython')\n import git\n try:\n repo = git.Repo(path)\n remote = repo.remotes.origin.url.replace('.git', '') # i.e. 'https://github.com/WongKinYiu/yolov9'\n commit = repo.head.commit.hexsha # i.e. '3134699c73af83aac2a481435550b968d5792c0d'\n try:\n branch = repo.active_branch.name # i.e. 'main'\n except TypeError: # not on any branch\n branch = None # i.e. 'detached HEAD' state\n return {'remote': remote, 'branch': branch, 'commit': commit}\n except git.exc.InvalidGitRepositoryError: # path is not a git dir\n return {'remote': None, 'branch': None, 'commit': None}\n\n\ndef check_python(minimum='3.7.0'):\n # Check current python version vs. required python version\n check_version(platform.python_version(), minimum, name='Python ', hard=True)\n\n\ndef check_version(current='0.0.0', minimum='0.0.0', name='version ', pinned=False, hard=False, verbose=False):\n # Check version vs. required version\n current, minimum = (pkg.parse_version(x) for x in (current, minimum))\n result = (current == minimum) if pinned else (current >= minimum) # bool\n s = f'WARNING ⚠️ {name}{minimum} is required by YOLO, but {name}{current} is currently installed' # string\n if hard:\n assert result, emojis(s) # assert min requirements met\n if verbose and not result:\n LOGGER.warning(s)\n return result\n\n\n@TryExcept()\ndef check_requirements(requirements=ROOT / 'requirements.txt', exclude=(), install=True, cmds=''):\n # Check installed dependencies meet YOLO requirements (pass *.txt file or list of packages or single package str)\n prefix = colorstr('red', 'bold', 'requirements:')\n check_python() # check python version\n if isinstance(requirements, Path): # requirements.txt file\n file = requirements.resolve()\n assert file.exists(), f\"{prefix} {file} not found, check failed.\"\n with file.open() as f:\n requirements = [f'{x.name}{x.specifier}' for x in pkg.parse_requirements(f) if x.name not in exclude]\n elif isinstance(requirements, str):\n requirements = [requirements]\n\n s = ''\n n = 0\n for r in requirements:\n try:\n pkg.require(r)\n except (pkg.VersionConflict, pkg.DistributionNotFound): # exception if requirements not met\n s += f'\"{r}\" '\n n += 1\n\n if s and install and AUTOINSTALL: # check environment variable\n LOGGER.info(f\"{prefix} YOLO requirement{'s' * (n > 1)} {s}not found, attempting AutoUpdate...\")\n try:\n # assert check_online(), \"AutoUpdate skipped (offline)\"\n LOGGER.info(check_output(f'pip install {s} {cmds}', shell=True).decode())\n source = file if 'file' in locals() else requirements\n s = f\"{prefix} {n} package{'s' * (n > 1)} updated per {source}\\n\" \\\n f\"{prefix} ⚠️ {colorstr('bold', 'Restart runtime or rerun command for updates to take effect')}\\n\"\n LOGGER.info(s)\n except Exception as e:\n LOGGER.warning(f'{prefix} ❌ {e}')\n\n\ndef check_img_size(imgsz, s=32, floor=0):\n # Verify image size is a multiple of stride s in each dimension\n if isinstance(imgsz, int): # integer i.e. img_size=640\n new_size = max(make_divisible(imgsz, int(s)), floor)\n else: # list i.e. img_size=[640, 480]\n imgsz = list(imgsz) # convert to list if tuple\n new_size = [max(make_divisible(x, int(s)), floor) for x in imgsz]\n if new_size != imgsz:\n LOGGER.warning(f'WARNING ⚠️ --img-size {imgsz} must be multiple of max stride {s}, updating to {new_size}')\n return new_size\n\n\ndef check_imshow(warn=False):\n # Check if environment supports image displays\n try:\n assert not is_notebook()\n assert not is_docker()\n cv2.imshow('test', np.zeros((1, 1, 3)))\n cv2.waitKey(1)\n cv2.destroyAllWindows()\n cv2.waitKey(1)\n return True\n except Exception as e:\n if warn:\n LOGGER.warning(f'WARNING ⚠️ Environment does not support cv2.imshow() or PIL Image.show()\\n{e}')\n return False\n\n\ndef check_suffix(file='yolo.pt', suffix=('.pt',), msg=''):\n # Check file(s) for acceptable suffix\n if file and suffix:\n if isinstance(suffix, str):\n suffix = [suffix]\n for f in file if isinstance(file, (list, tuple)) else [file]:\n s = Path(f).suffix.lower() # file suffix\n if len(s):\n assert s in suffix, f\"{msg}{f} acceptable suffix is {suffix}\"\n\n\ndef check_yaml(file, suffix=('.yaml', '.yml')):\n # Search/download YAML file (if necessary) and return path, checking suffix\n return check_file(file, suffix)\n\n\ndef check_file(file, suffix=''):\n # Search/download file (if necessary) and return path\n check_suffix(file, suffix) # optional\n file = str(file) # convert to str()\n if os.path.isfile(file) or not file: # exists\n return file\n elif file.startswith(('http:/', 'https:/')): # download\n url = file # warning: Pathlib turns :// -> :/\n file = Path(urllib.parse.unquote(file).split('?')[0]).name # '%2F' to '/', split https://url.com/file.txt?auth\n if os.path.isfile(file):\n LOGGER.info(f'Found {url} locally at {file}') # file already exists\n else:\n LOGGER.info(f'Downloading {url} to {file}...')\n torch.hub.download_url_to_file(url, file)\n assert Path(file).exists() and Path(file).stat().st_size > 0, f'File download failed: {url}' # check\n return file\n elif file.startswith('clearml://'): # ClearML Dataset ID\n assert 'clearml' in sys.modules, \"ClearML is not installed, so cannot use ClearML dataset. Try running 'pip install clearml'.\"\n return file\n else: # search\n files = []\n for d in 'data', 'models', 'utils': # search directories\n files.extend(glob.glob(str(ROOT / d / '**' / file), recursive=True)) # find file\n assert len(files), f'File not found: {file}' # assert file was found\n assert len(files) == 1, f\"Multiple files match '{file}', specify exact path: {files}\" # assert unique\n return files[0] # return file\n\n\ndef check_font(font=FONT, progress=False):\n # Download font to CONFIG_DIR if necessary\n font = Path(font)\n file = CONFIG_DIR / font.name\n if not font.exists() and not file.exists():\n url = f'https://ultralytics.com/assets/{font.name}'\n LOGGER.info(f'Downloading {url} to {file}...')\n torch.hub.download_url_to_file(url, str(file), progress=progress)\n\n\ndef check_dataset(data, autodownload=True):\n # Download, check and/or unzip dataset if not found locally\n\n # Download (optional)\n extract_dir = ''\n if isinstance(data, (str, Path)) and (is_zipfile(data) or is_tarfile(data)):\n download(data, dir=f'{DATASETS_DIR}/{Path(data).stem}', unzip=True, delete=False, curl=False, threads=1)\n data = next((DATASETS_DIR / Path(data).stem).rglob('*.yaml'))\n extract_dir, autodownload = data.parent, False\n\n # Read yaml (optional)\n if isinstance(data, (str, Path)):\n data = yaml_load(data) # dictionary\n\n # Checks\n for k in 'train', 'val', 'names':\n assert k in data, emojis(f\"data.yaml '{k}:' field missing ❌\")\n if isinstance(data['names'], (list, tuple)): # old array format\n data['names'] = dict(enumerate(data['names'])) # convert to dict\n assert all(isinstance(k, int) for k in data['names'].keys()), 'data.yaml names keys must be integers, i.e. 2: car'\n data['nc'] = len(data['names'])\n\n # Resolve paths\n path = Path(extract_dir or data.get('path') or '') # optional 'path' default to '.'\n if not path.is_absolute():\n path = (ROOT / path).resolve()\n data['path'] = path # download scripts\n for k in 'train', 'val', 'test':\n if data.get(k): # prepend path\n if isinstance(data[k], str):\n x = (path / data[k]).resolve()\n if not x.exists() and data[k].startswith('../'):\n x = (path / data[k][3:]).resolve()\n data[k] = str(x)\n else:\n data[k] = [str((path / x).resolve()) for x in data[k]]\n\n # Parse yaml\n train, val, test, s = (data.get(x) for x in ('train', 'val', 'test', 'download'))\n if val:\n val = [Path(x).resolve() for x in (val if isinstance(val, list) else [val])] # val path\n if not all(x.exists() for x in val):\n LOGGER.info('\\nDataset not found ⚠️, missing paths %s' % [str(x) for x in val if not x.exists()])\n if not s or not autodownload:\n raise Exception('Dataset not found ❌')\n t = time.time()\n if s.startswith('http') and s.endswith('.zip'): # URL\n f = Path(s).name # filename\n LOGGER.info(f'Downloading {s} to {f}...')\n torch.hub.download_url_to_file(s, f)\n Path(DATASETS_DIR).mkdir(parents=True, exist_ok=True) # create root\n unzip_file(f, path=DATASETS_DIR) # unzip\n Path(f).unlink() # remove zip\n r = None # success\n elif s.startswith('bash '): # bash script\n LOGGER.info(f'Running {s} ...')\n r = os.system(s)\n else: # python script\n r = exec(s, {'yaml': data}) # return None\n dt = f'({round(time.time() - t, 1)}s)'\n s = f\"success ✅ {dt}, saved to {colorstr('bold', DATASETS_DIR)}\" if r in (0, None) else f\"failure {dt} ❌\"\n LOGGER.info(f\"Dataset download {s}\")\n check_font('Arial.ttf' if is_ascii(data['names']) else 'Arial.Unicode.ttf', progress=True) # download fonts\n return data # dictionary\n\n\ndef check_amp(model):\n # Check PyTorch Automatic Mixed Precision (AMP) functionality. Return True on correct operation\n from asone.detectors.yolov9.yolov9.models.common import AutoShape, DetectMultiBackend\n\n def amp_allclose(model, im):\n # All close FP32 vs AMP results\n m = AutoShape(model, verbose=False) # model\n a = m(im).xywhn[0] # FP32 inference\n m.amp = True\n b = m(im).xywhn[0] # AMP inference\n return a.shape == b.shape and torch.allclose(a, b, atol=0.1) # close to 10% absolute tolerance\n\n prefix = colorstr('AMP: ')\n device = next(model.parameters()).device # get model device\n if device.type in ('cpu', 'mps'):\n return False # AMP only used on CUDA devices\n f = ROOT / 'data' / 'images' / 'bus.jpg' # image to check\n im = f if f.exists() else 'https://ultralytics.com/images/bus.jpg' if check_online() else np.ones((640, 640, 3))\n try:\n #assert amp_allclose(deepcopy(model), im) or amp_allclose(DetectMultiBackend('yolo.pt', device), im)\n LOGGER.info(f'{prefix}checks passed ✅')\n return True\n except Exception:\n help_url = 'https://github.com/ultralytics/yolov5/issues/7908'\n LOGGER.warning(f'{prefix}checks failed ❌, disabling Automatic Mixed Precision. See {help_url}')\n return False\n\n\ndef yaml_load(file='data.yaml'):\n # Single-line safe yaml loading\n with open(file, errors='ignore') as f:\n return yaml.safe_load(f)\n\n\ndef yaml_save(file='data.yaml', data={}):\n # Single-line safe yaml saving\n with open(file, 'w') as f:\n yaml.safe_dump({k: str(v) if isinstance(v, Path) else v for k, v in data.items()}, f, sort_keys=False)\n\n\ndef unzip_file(file, path=None, exclude=('.DS_Store', '__MACOSX')):\n # Unzip a *.zip file to path/, excluding files containing strings in exclude list\n if path is None:\n path = Path(file).parent # default path\n with ZipFile(file) as zipObj:\n for f in zipObj.namelist(): # list all archived filenames in the zip\n if all(x not in f for x in exclude):\n zipObj.extract(f, path=path)\n\n\ndef url2file(url):\n # Convert URL to filename, i.e. https://url.com/file.txt?auth -> file.txt\n url = str(Path(url)).replace(':/', '://') # Pathlib turns :// -> :/\n return Path(urllib.parse.unquote(url)).name.split('?')[0] # '%2F' to '/', split https://url.com/file.txt?auth\n\n\ndef download(url, dir='.', unzip=True, delete=True, curl=False, threads=1, retry=3):\n # Multithreaded file download and unzip function, used in data.yaml for autodownload\n def download_one(url, dir):\n # Download 1 file\n success = True\n if os.path.isfile(url):\n f = Path(url) # filename\n else: # does not exist\n f = dir / Path(url).name\n LOGGER.info(f'Downloading {url} to {f}...')\n for i in range(retry + 1):\n if curl:\n s = 'sS' if threads > 1 else '' # silent\n r = os.system(\n f'curl -# -{s}L \"{url}\" -o \"{f}\" --retry 9 -C -') # curl download with retry, continue\n success = r == 0\n else:\n torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download\n success = f.is_file()\n if success:\n break\n elif i < retry:\n LOGGER.warning(f'⚠️ Download failure, retrying {i + 1}/{retry} {url}...')\n else:\n LOGGER.warning(f'❌ Failed to download {url}...')\n\n if unzip and success and (f.suffix == '.gz' or is_zipfile(f) or is_tarfile(f)):\n LOGGER.info(f'Unzipping {f}...')\n if is_zipfile(f):\n unzip_file(f, dir) # unzip\n elif is_tarfile(f):\n os.system(f'tar xf {f} --directory {f.parent}') # unzip\n elif f.suffix == '.gz':\n os.system(f'tar xfz {f} --directory {f.parent}') # unzip\n if delete:\n f.unlink() # remove zip\n\n dir = Path(dir)\n dir.mkdir(parents=True, exist_ok=True) # make directory\n if threads > 1:\n pool = ThreadPool(threads)\n pool.imap(lambda x: download_one(*x), zip(url, repeat(dir))) # multithreaded\n pool.close()\n pool.join()\n else:\n for u in [url] if isinstance(url, (str, Path)) else url:\n download_one(u, dir)\n\n\ndef make_divisible(x, divisor):\n # Returns nearest x divisible by divisor\n if isinstance(divisor, torch.Tensor):\n divisor = int(divisor.max()) # to int\n return math.ceil(x / divisor) * divisor\n\n\ndef clean_str(s):\n # Cleans a string by replacing special characters with underscore _\n return re.sub(pattern=\"[|@#!¡·$€%&()=?¿^*;:,¨´><+]\", repl=\"_\", string=s)\n\n\ndef one_cycle(y1=0.0, y2=1.0, steps=100):\n # lambda function for sinusoidal ramp from y1 to y2 https://arxiv.org/pdf/1812.01187.pdf\n return lambda x: ((1 - math.cos(x * math.pi / steps)) / 2) * (y2 - y1) + y1\n\n\ndef one_flat_cycle(y1=0.0, y2=1.0, steps=100):\n # lambda function for sinusoidal ramp from y1 to y2 https://arxiv.org/pdf/1812.01187.pdf\n #return lambda x: ((1 - math.cos(x * math.pi / steps)) / 2) * (y2 - y1) + y1\n return lambda x: ((1 - math.cos((x - (steps // 2)) * math.pi / (steps // 2))) / 2) * (y2 - y1) + y1 if (x > (steps // 2)) else y1\n\n\ndef colorstr(*input):\n # Colors a string https://en.wikipedia.org/wiki/ANSI_escape_code, i.e. colorstr('blue', 'hello world')\n *args, string = input if len(input) > 1 else ('blue', 'bold', input[0]) # color arguments, string\n colors = {\n 'black': '\\033[30m', # basic colors\n 'red': '\\033[31m',\n 'green': '\\033[32m',\n 'yellow': '\\033[33m',\n 'blue': '\\033[34m',\n 'magenta': '\\033[35m',\n 'cyan': '\\033[36m',\n 'white': '\\033[37m',\n 'bright_black': '\\033[90m', # bright colors\n 'bright_red': '\\033[91m',\n 'bright_green': '\\033[92m',\n 'bright_yellow': '\\033[93m',\n 'bright_blue': '\\033[94m',\n 'bright_magenta': '\\033[95m',\n 'bright_cyan': '\\033[96m',\n 'bright_white': '\\033[97m',\n 'end': '\\033[0m', # misc\n 'bold': '\\033[1m',\n 'underline': '\\033[4m'}\n return ''.join(colors[x] for x in args) + f'{string}' + colors['end']\n\n\ndef labels_to_class_weights(labels, nc=80):\n # Get class weights (inverse frequency) from training labels\n if labels[0] is None: # no labels loaded\n return torch.Tensor()\n\n labels = np.concatenate(labels, 0) # labels.shape = (866643, 5) for COCO\n classes = labels[:, 0].astype(int) # labels = [class xywh]\n weights = np.bincount(classes, minlength=nc) # occurrences per class\n\n # Prepend gridpoint count (for uCE training)\n # gpi = ((320 / 32 * np.array([1, 2, 4])) ** 2 * 3).sum() # gridpoints per image\n # weights = np.hstack([gpi * len(labels) - weights.sum() * 9, weights * 9]) ** 0.5 # prepend gridpoints to start\n\n weights[weights == 0] = 1 # replace empty bins with 1\n weights = 1 / weights # number of targets per class\n weights /= weights.sum() # normalize\n return torch.from_numpy(weights).float()\n\n\ndef labels_to_image_weights(labels, nc=80, class_weights=np.ones(80)):\n # Produces image weights based on class_weights and image contents\n # Usage: index = random.choices(range(n), weights=image_weights, k=1) # weighted image sample\n class_counts = np.array([np.bincount(x[:, 0].astype(int), minlength=nc) for x in labels])\n return (class_weights.reshape(1, nc) * class_counts).sum(1)\n\n\ndef coco80_to_coco91_class(): # converts 80-index (val2014) to 91-index (paper)\n # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/\n # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\\n')\n # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\\n')\n # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco\n # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet\n return [\n 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34,\n 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,\n 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90]\n\n\ndef xyxy2xywh(x):\n # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = (x[..., 0] + x[..., 2]) / 2 # x center\n y[..., 1] = (x[..., 1] + x[..., 3]) / 2 # y center\n y[..., 2] = x[..., 2] - x[..., 0] # width\n y[..., 3] = x[..., 3] - x[..., 1] # height\n return y\n\n\ndef xywh2xyxy(x):\n # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = x[..., 0] - x[..., 2] / 2 # top left x\n y[..., 1] = x[..., 1] - x[..., 3] / 2 # top left y\n y[..., 2] = x[..., 0] + x[..., 2] / 2 # bottom right x\n y[..., 3] = x[..., 1] + x[..., 3] / 2 # bottom right y\n return y\n\n\ndef xywhn2xyxy(x, w=640, h=640, padw=0, padh=0):\n # Convert nx4 boxes from [x, y, w, h] normalized to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = w * (x[..., 0] - x[..., 2] / 2) + padw # top left x\n y[..., 1] = h * (x[..., 1] - x[..., 3] / 2) + padh # top left y\n y[..., 2] = w * (x[..., 0] + x[..., 2] / 2) + padw # bottom right x\n y[..., 3] = h * (x[..., 1] + x[..., 3] / 2) + padh # bottom right y\n return y\n\n\ndef xyxy2xywhn(x, w=640, h=640, clip=False, eps=0.0):\n # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] normalized where xy1=top-left, xy2=bottom-right\n if clip:\n clip_boxes(x, (h - eps, w - eps)) # warning: inplace clip\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = ((x[..., 0] + x[..., 2]) / 2) / w # x center\n y[..., 1] = ((x[..., 1] + x[..., 3]) / 2) / h # y center\n y[..., 2] = (x[..., 2] - x[..., 0]) / w # width\n y[..., 3] = (x[..., 3] - x[..., 1]) / h # height\n return y\n\n\ndef xyn2xy(x, w=640, h=640, padw=0, padh=0):\n # Convert normalized segments into pixel segments, shape (n,2)\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = w * x[..., 0] + padw # top left x\n y[..., 1] = h * x[..., 1] + padh # top left y\n return y\n\n\ndef segment2box(segment, width=640, height=640):\n # Convert 1 segment label to 1 box label, applying inside-image constraint, i.e. (xy1, xy2, ...) to (xyxy)\n x, y = segment.T # segment xy\n inside = (x >= 0) & (y >= 0) & (x <= width) & (y <= height)\n x, y, = x[inside], y[inside]\n return np.array([x.min(), y.min(), x.max(), y.max()]) if any(x) else np.zeros((1, 4)) # xyxy\n\n\ndef segments2boxes(segments):\n # Convert segment labels to box labels, i.e. (cls, xy1, xy2, ...) to (cls, xywh)\n boxes = []\n for s in segments:\n x, y = s.T # segment xy\n boxes.append([x.min(), y.min(), x.max(), y.max()]) # cls, xyxy\n return xyxy2xywh(np.array(boxes)) # cls, xywh\n\n\ndef resample_segments(segments, n=1000):\n # Up-sample an (n,2) segment\n for i, s in enumerate(segments):\n s = np.concatenate((s, s[0:1, :]), axis=0)\n x = np.linspace(0, len(s) - 1, n)\n xp = np.arange(len(s))\n segments[i] = np.concatenate([np.interp(x, xp, s[:, i]) for i in range(2)]).reshape(2, -1).T # segment xy\n return segments\n\n\ndef scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None):\n # Rescale boxes (xyxy) from img1_shape to img0_shape\n if ratio_pad is None: # calculate from img0_shape\n gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new\n pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding\n else:\n gain = ratio_pad[0][0]\n pad = ratio_pad[1]\n\n boxes[:, [0, 2]] -= pad[0] # x padding\n boxes[:, [1, 3]] -= pad[1] # y padding\n boxes[:, :4] /= gain\n clip_boxes(boxes, img0_shape)\n return boxes\n\n\ndef scale_segments(img1_shape, segments, img0_shape, ratio_pad=None, normalize=False):\n # Rescale coords (xyxy) from img1_shape to img0_shape\n if ratio_pad is None: # calculate from img0_shape\n gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new\n pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding\n else:\n gain = ratio_pad[0][0]\n pad = ratio_pad[1]\n\n segments[:, 0] -= pad[0] # x padding\n segments[:, 1] -= pad[1] # y padding\n segments /= gain\n clip_segments(segments, img0_shape)\n if normalize:\n segments[:, 0] /= img0_shape[1] # width\n segments[:, 1] /= img0_shape[0] # height\n return segments\n\n\ndef clip_boxes(boxes, shape):\n # Clip boxes (xyxy) to image shape (height, width)\n if isinstance(boxes, torch.Tensor): # faster individually\n boxes[:, 0].clamp_(0, shape[1]) # x1\n boxes[:, 1].clamp_(0, shape[0]) # y1\n boxes[:, 2].clamp_(0, shape[1]) # x2\n boxes[:, 3].clamp_(0, shape[0]) # y2\n else: # np.array (faster grouped)\n boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, shape[1]) # x1, x2\n boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, shape[0]) # y1, y2\n\n\ndef clip_segments(segments, shape):\n # Clip segments (xy1,xy2,...) to image shape (height, width)\n if isinstance(segments, torch.Tensor): # faster individually\n segments[:, 0].clamp_(0, shape[1]) # x\n segments[:, 1].clamp_(0, shape[0]) # y\n else: # np.array (faster grouped)\n segments[:, 0] = segments[:, 0].clip(0, shape[1]) # x\n segments[:, 1] = segments[:, 1].clip(0, shape[0]) # y\n\n\ndef non_max_suppression(\n prediction,\n conf_thres=0.25,\n iou_thres=0.45,\n classes=None,\n agnostic=False,\n multi_label=False,\n labels=(),\n max_det=300,\n nm=0, # number of masks\n):\n \"\"\"Non-Maximum Suppression (NMS) on inference results to reject overlapping detections\n\n Returns:\n list of detections, on (n,6) tensor per image [xyxy, conf, cls]\n \"\"\"\n\n if isinstance(prediction, (list, tuple)): # YOLO model in validation model, output = (inference_out, loss_out)\n prediction = prediction[0] # select only inference output\n\n device = prediction.device\n mps = 'mps' in device.type # Apple MPS\n if mps: # MPS not fully supported yet, convert tensors to CPU before NMS\n prediction = prediction.cpu()\n bs = prediction.shape[0] # batch size\n nc = prediction.shape[1] - nm - 4 # number of classes\n mi = 4 + nc # mask start index\n xc = prediction[:, 4:mi].amax(1) > conf_thres # candidates\n\n # Checks\n assert 0 <= conf_thres <= 1, f'Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0'\n assert 0 <= iou_thres <= 1, f'Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0'\n\n # Settings\n # min_wh = 2 # (pixels) minimum box width and height\n max_wh = 7680 # (pixels) maximum box width and height\n max_nms = 30000 # maximum number of boxes into torchvision.ops.nms()\n time_limit = 2.5 + 0.05 * bs # seconds to quit after\n redundant = True # require redundant detections\n multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img)\n merge = False # use merge-NMS\n\n t = time.time()\n output = [torch.zeros((0, 6 + nm), device=prediction.device)] * bs\n for xi, x in enumerate(prediction): # image index, image inference\n # Apply constraints\n # x[((x[:, 2:4] < min_wh) | (x[:, 2:4] > max_wh)).any(1), 4] = 0 # width-height\n x = x.T[xc[xi]] # confidence\n\n # Cat apriori labels if autolabelling\n if labels and len(labels[xi]):\n lb = labels[xi]\n v = torch.zeros((len(lb), nc + nm + 5), device=x.device)\n v[:, :4] = lb[:, 1:5] # box\n v[range(len(lb)), lb[:, 0].long() + 4] = 1.0 # cls\n x = torch.cat((x, v), 0)\n\n # If none remain process next image\n if not x.shape[0]:\n continue\n\n # Detections matrix nx6 (xyxy, conf, cls)\n box, cls, mask = x.split((4, nc, nm), 1)\n box = xywh2xyxy(box) # center_x, center_y, width, height) to (x1, y1, x2, y2)\n if multi_label:\n i, j = (cls > conf_thres).nonzero(as_tuple=False).T\n x = torch.cat((box[i], x[i, 4 + j, None], j[:, None].float(), mask[i]), 1)\n else: # best class only\n conf, j = cls.max(1, keepdim=True)\n x = torch.cat((box, conf, j.float(), mask), 1)[conf.view(-1) > conf_thres]\n\n # Filter by class\n if classes is not None:\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\n\n # Apply finite constraint\n # if not torch.isfinite(x).all():\n # x = x[torch.isfinite(x).all(1)]\n\n # Check shape\n n = x.shape[0] # number of boxes\n if not n: # no boxes\n continue\n elif n > max_nms: # excess boxes\n x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence\n else:\n x = x[x[:, 4].argsort(descending=True)] # sort by confidence\n\n # Batched NMS\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\n i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS\n if i.shape[0] > max_det: # limit detections\n i = i[:max_det]\n if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)\n # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\n weights = iou * scores[None] # box weights\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\n if redundant:\n i = i[iou.sum(1) > 1] # require redundancy\n\n output[xi] = x[i]\n if mps:\n output[xi] = output[xi].to(device)\n if (time.time() - t) > time_limit:\n LOGGER.warning(f'WARNING ⚠️ NMS time limit {time_limit:.3f}s exceeded')\n break # time limit exceeded\n\n return output\n\n\ndef strip_optimizer(f='best.pt', s=''): # from utils.general import *; strip_optimizer()\n # Strip optimizer from 'f' to finalize training, optionally save as 's'\n x = torch.load(f, map_location=torch.device('cpu'))\n if x.get('ema'):\n x['model'] = x['ema'] # replace model with ema\n for k in 'optimizer', 'best_fitness', 'ema', 'updates': # keys\n x[k] = None\n x['epoch'] = -1\n x['model'].half() # to FP16\n for p in x['model'].parameters():\n p.requires_grad = False\n torch.save(x, s or f)\n mb = os.path.getsize(s or f) / 1E6 # filesize\n LOGGER.info(f\"Optimizer stripped from {f},{f' saved as {s},' if s else ''} {mb:.1f}MB\")\n\n\ndef print_mutation(keys, results, hyp, save_dir, bucket, prefix=colorstr('evolve: ')):\n evolve_csv = save_dir / 'evolve.csv'\n evolve_yaml = save_dir / 'hyp_evolve.yaml'\n keys = tuple(keys) + tuple(hyp.keys()) # [results + hyps]\n keys = tuple(x.strip() for x in keys)\n vals = results + tuple(hyp.values())\n n = len(keys)\n\n # Download (optional)\n if bucket:\n url = f'gs://{bucket}/evolve.csv'\n if gsutil_getsize(url) > (evolve_csv.stat().st_size if evolve_csv.exists() else 0):\n os.system(f'gsutil cp {url} {save_dir}') # download evolve.csv if larger than local\n\n # Log to evolve.csv\n s = '' if evolve_csv.exists() else (('%20s,' * n % keys).rstrip(',') + '\\n') # add header\n with open(evolve_csv, 'a') as f:\n f.write(s + ('%20.5g,' * n % vals).rstrip(',') + '\\n')\n\n # Save yaml\n with open(evolve_yaml, 'w') as f:\n data = pd.read_csv(evolve_csv)\n data = data.rename(columns=lambda x: x.strip()) # strip keys\n i = np.argmax(fitness(data.values[:, :4])) #\n generations = len(data)\n f.write('# YOLO Hyperparameter Evolution Results\\n' + f'# Best generation: {i}\\n' +\n f'# Last generation: {generations - 1}\\n' + '# ' + ', '.join(f'{x.strip():>20s}' for x in keys[:7]) +\n '\\n' + '# ' + ', '.join(f'{x:>20.5g}' for x in data.values[i, :7]) + '\\n\\n')\n yaml.safe_dump(data.loc[i][7:].to_dict(), f, sort_keys=False)\n\n # Print to screen\n LOGGER.info(prefix + f'{generations} generations finished, current result:\\n' + prefix +\n ', '.join(f'{x.strip():>20s}' for x in keys) + '\\n' + prefix + ', '.join(f'{x:20.5g}'\n for x in vals) + '\\n\\n')\n\n if bucket:\n os.system(f'gsutil cp {evolve_csv} {evolve_yaml} gs://{bucket}') # upload\n\n\ndef apply_classifier(x, model, img, im0):\n # Apply a second stage classifier to YOLO outputs\n # Example model = torchvision.models.__dict__['efficientnet_b0'](pretrained=True).to(device).eval()\n im0 = [im0] if isinstance(im0, np.ndarray) else im0\n for i, d in enumerate(x): # per image\n if d is not None and len(d):\n d = d.clone()\n\n # Reshape and pad cutouts\n b = xyxy2xywh(d[:, :4]) # boxes\n b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # rectangle to square\n b[:, 2:] = b[:, 2:] * 1.3 + 30 # pad\n d[:, :4] = xywh2xyxy(b).long()\n\n # Rescale boxes from img_size to im0 size\n scale_boxes(img.shape[2:], d[:, :4], im0[i].shape)\n\n # Classes\n pred_cls1 = d[:, 5].long()\n ims = []\n for a in d:\n cutout = im0[i][int(a[1]):int(a[3]), int(a[0]):int(a[2])]\n im = cv2.resize(cutout, (224, 224)) # BGR\n\n im = im[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\n im = np.ascontiguousarray(im, dtype=np.float32) # uint8 to float32\n im /= 255 # 0 - 255 to 0.0 - 1.0\n ims.append(im)\n\n pred_cls2 = model(torch.Tensor(ims).to(d.device)).argmax(1) # classifier prediction\n x[i] = x[i][pred_cls1 == pred_cls2] # retain matching class detections\n\n return x\n\n\ndef increment_path(path, exist_ok=False, sep='', mkdir=False):\n # Increment file or directory path, i.e. runs/exp --> runs/exp{sep}2, runs/exp{sep}3, ... etc.\n path = Path(path) # os-agnostic\n if path.exists() and not exist_ok:\n path, suffix = (path.with_suffix(''), path.suffix) if path.is_file() else (path, '')\n\n # Method 1\n for n in range(2, 9999):\n p = f'{path}{sep}{n}{suffix}' # increment path\n if not os.path.exists(p): #\n break\n path = Path(p)\n\n # Method 2 (deprecated)\n # dirs = glob.glob(f\"{path}{sep}*\") # similar paths\n # matches = [re.search(rf\"{path.stem}{sep}(\\d+)\", d) for d in dirs]\n # i = [int(m.groups()[0]) for m in matches if m] # indices\n # n = max(i) + 1 if i else 2 # increment number\n # path = Path(f\"{path}{sep}{n}{suffix}\") # increment path\n\n if mkdir:\n path.mkdir(parents=True, exist_ok=True) # make directory\n\n return path\n\n\n# OpenCV Chinese-friendly functions ------------------------------------------------------------------------------------\nimshow_ = cv2.imshow # copy to avoid recursion errors\n\n\ndef imread(path, flags=cv2.IMREAD_COLOR):\n return cv2.imdecode(np.fromfile(path, np.uint8), flags)\n\n\ndef imwrite(path, im):\n try:\n cv2.imencode(Path(path).suffix, im)[1].tofile(path)\n return True\n except Exception:\n return False\n\n\ndef imshow(path, im):\n imshow_(path.encode('unicode_escape').decode(), im)\n\n\ncv2.imread, cv2.imwrite, cv2.imshow = imread, imwrite, imshow # redefine\n\n# Variables ------------------------------------------------------------------------------------------------------------\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/lion.py", "content": "\"\"\"PyTorch implementation of the Lion optimizer.\"\"\"\nimport torch\nfrom torch.optim.optimizer import Optimizer\n\n\nclass Lion(Optimizer):\n r\"\"\"Implements Lion algorithm.\"\"\"\n\n def __init__(self, params, lr=1e-4, betas=(0.9, 0.99), weight_decay=0.0):\n \"\"\"Initialize the hyperparameters.\n Args:\n params (iterable): iterable of parameters to optimize or dicts defining\n parameter groups\n lr (float, optional): learning rate (default: 1e-4)\n betas (Tuple[float, float], optional): coefficients used for computing\n running averages of gradient and its square (default: (0.9, 0.99))\n weight_decay (float, optional): weight decay coefficient (default: 0)\n \"\"\"\n\n if not 0.0 <= lr:\n raise ValueError('Invalid learning rate: {}'.format(lr))\n if not 0.0 <= betas[0] < 1.0:\n raise ValueError('Invalid beta parameter at index 0: {}'.format(betas[0]))\n if not 0.0 <= betas[1] < 1.0:\n raise ValueError('Invalid beta parameter at index 1: {}'.format(betas[1]))\n defaults = dict(lr=lr, betas=betas, weight_decay=weight_decay)\n super().__init__(params, defaults)\n\n @torch.no_grad()\n def step(self, closure=None):\n \"\"\"Performs a single optimization step.\n Args:\n closure (callable, optional): A closure that reevaluates the model\n and returns the loss.\n Returns:\n the loss.\n \"\"\"\n loss = None\n if closure is not None:\n with torch.enable_grad():\n loss = closure()\n\n for group in self.param_groups:\n for p in group['params']:\n if p.grad is None:\n continue\n\n # Perform stepweight decay\n p.data.mul_(1 - group['lr'] * group['weight_decay'])\n\n grad = p.grad\n state = self.state[p]\n # State initialization\n if len(state) == 0:\n # Exponential moving average of gradient values\n state['exp_avg'] = torch.zeros_like(p)\n\n exp_avg = state['exp_avg']\n beta1, beta2 = group['betas']\n\n # Weight update\n update = exp_avg * beta1 + grad * (1 - beta1)\n p.add_(torch.sign(update), alpha=-group['lr'])\n # Decay the momentum running average coefficient\n exp_avg.mul_(beta2).add_(grad, alpha=1 - beta2)\n\n return loss" }, { "path": "asone/detectors/yolov9/yolov9/utils/loss.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom asone.detectors.yolov9.yolov9.utils.metrics import bbox_iou\nfrom asone.detectors.yolov9.yolov9.utils.torch_utils import de_parallel\n\n\ndef smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441\n # return positive, negative label smoothing BCE targets\n return 1.0 - 0.5 * eps, 0.5 * eps\n\n\nclass BCEBlurWithLogitsLoss(nn.Module):\n # BCEwithLogitLoss() with reduced missing label effects.\n def __init__(self, alpha=0.05):\n super().__init__()\n self.loss_fcn = nn.BCEWithLogitsLoss(reduction='none') # must be nn.BCEWithLogitsLoss()\n self.alpha = alpha\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n pred = torch.sigmoid(pred) # prob from logits\n dx = pred - true # reduce only missing label effects\n # dx = (pred - true).abs() # reduce missing label and false label effects\n alpha_factor = 1 - torch.exp((dx - 1) / (self.alpha + 1e-4))\n loss *= alpha_factor\n return loss.mean()\n\n\nclass FocalLoss(nn.Module):\n # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\n super().__init__()\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\n self.gamma = gamma\n self.alpha = alpha\n self.reduction = loss_fcn.reduction\n self.loss_fcn.reduction = 'none' # required to apply FL to each element\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n # p_t = torch.exp(-loss)\n # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability\n\n # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py\n pred_prob = torch.sigmoid(pred) # prob from logits\n p_t = true * pred_prob + (1 - true) * (1 - pred_prob)\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\n modulating_factor = (1.0 - p_t) ** self.gamma\n loss *= alpha_factor * modulating_factor\n\n if self.reduction == 'mean':\n return loss.mean()\n elif self.reduction == 'sum':\n return loss.sum()\n else: # 'none'\n return loss\n\n\nclass QFocalLoss(nn.Module):\n # Wraps Quality focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\n super().__init__()\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\n self.gamma = gamma\n self.alpha = alpha\n self.reduction = loss_fcn.reduction\n self.loss_fcn.reduction = 'none' # required to apply FL to each element\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n\n pred_prob = torch.sigmoid(pred) # prob from logits\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\n modulating_factor = torch.abs(true - pred_prob) ** self.gamma\n loss *= alpha_factor * modulating_factor\n\n if self.reduction == 'mean':\n return loss.mean()\n elif self.reduction == 'sum':\n return loss.sum()\n else: # 'none'\n return loss\n\n\nclass ComputeLoss:\n sort_obj_iou = False\n\n # Compute losses\n def __init__(self, model, autobalance=False):\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device))\n BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device))\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h['fl_gamma'] # focal loss gamma\n if g > 0:\n BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.ssi = list(m.stride).index(16) if autobalance else 0 # stride 16 index\n self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, 1.0, h, autobalance\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.anchors = m.anchors\n self.device = device\n\n def __call__(self, p, targets): # predictions, targets\n bs = p[0].shape[0] # batch size\n loss = torch.zeros(3, device=self.device) # [box, obj, cls] losses\n tcls, tbox, indices = self.build_targets(p, targets) # targets\n\n # Losses\n for i, pi in enumerate(p): # layer index, layer predictions\n b, gj, gi = indices[i] # image, anchor, gridy, gridx\n tobj = torch.zeros((pi.shape[0], pi.shape[2], pi.shape[3]), dtype=pi.dtype, device=self.device) # tgt obj\n\n n_labels = b.shape[0] # number of labels\n if n_labels:\n # pxy, pwh, _, pcls = pi[b, a, gj, gi].tensor_split((2, 4, 5), dim=1) # faster, requires torch 1.8.0\n pxy, pwh, _, pcls = pi[b, :, gj, gi].split((2, 2, 1, self.nc), 1) # target-subset of predictions\n\n # Regression\n # pwh = (pwh.sigmoid() * 2) ** 2 * anchors[i]\n # pwh = (0.0 + (pwh - 1.09861).sigmoid() * 4) * anchors[i]\n # pwh = (0.33333 + (pwh - 1.09861).sigmoid() * 2.66667) * anchors[i]\n # pwh = (0.25 + (pwh - 1.38629).sigmoid() * 3.75) * anchors[i]\n # pwh = (0.20 + (pwh - 1.60944).sigmoid() * 4.8) * anchors[i]\n # pwh = (0.16667 + (pwh - 1.79175).sigmoid() * 5.83333) * anchors[i]\n pxy = pxy.sigmoid() * 1.6 - 0.3\n pwh = (0.2 + pwh.sigmoid() * 4.8) * self.anchors[i]\n pbox = torch.cat((pxy, pwh), 1) # predicted box\n iou = bbox_iou(pbox, tbox[i], CIoU=True).squeeze() # iou(prediction, target)\n loss[0] += (1.0 - iou).mean() # box loss\n\n # Objectness\n iou = iou.detach().clamp(0).type(tobj.dtype)\n if self.sort_obj_iou:\n j = iou.argsort()\n b, gj, gi, iou = b[j], gj[j], gi[j], iou[j]\n if self.gr < 1:\n iou = (1.0 - self.gr) + self.gr * iou\n tobj[b, gj, gi] = iou # iou ratio\n\n # Classification\n if self.nc > 1: # cls loss (only if multiple classes)\n t = torch.full_like(pcls, self.cn, device=self.device) # targets\n t[range(n_labels), tcls[i]] = self.cp\n loss[2] += self.BCEcls(pcls, t) # cls loss\n\n obji = self.BCEobj(pi[:, 4], tobj)\n loss[1] += obji * self.balance[i] # obj loss\n if self.autobalance:\n self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item()\n\n if self.autobalance:\n self.balance = [x / self.balance[self.ssi] for x in self.balance]\n loss[0] *= self.hyp['box']\n loss[1] *= self.hyp['obj']\n loss[2] *= self.hyp['cls']\n return loss.sum() * bs, loss.detach() # [box, obj, cls] losses\n\n def build_targets(self, p, targets):\n # Build targets for compute_loss(), input targets(image,class,x,y,w,h)\n nt = targets.shape[0] # number of anchors, targets\n tcls, tbox, indices = [], [], []\n gain = torch.ones(6, device=self.device) # normalized to gridspace gain\n\n g = 0.3 # bias\n off = torch.tensor(\n [\n [0, 0],\n [1, 0],\n [0, 1],\n [-1, 0],\n [0, -1], # j,k,l,m\n # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm\n ],\n device=self.device).float() * g # offsets\n\n for i in range(self.nl):\n shape = p[i].shape\n gain[2:6] = torch.tensor(shape)[[3, 2, 3, 2]] # xyxy gain\n\n # Match targets to anchors\n t = targets * gain # shape(3,n,7)\n if nt:\n # Matches\n r = t[..., 4:6] / self.anchors[i] # wh ratio\n j = torch.max(r, 1 / r).max(1)[0] < self.hyp['anchor_t'] # compare\n # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2))\n t = t[j] # filter\n\n # Offsets\n gxy = t[:, 2:4] # grid xy\n gxi = gain[[2, 3]] - gxy # inverse\n j, k = ((gxy % 1 < g) & (gxy > 1)).T\n l, m = ((gxi % 1 < g) & (gxi > 1)).T\n j = torch.stack((torch.ones_like(j), j, k, l, m))\n t = t.repeat((5, 1, 1))[j]\n offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j]\n else:\n t = targets[0]\n offsets = 0\n\n # Define\n bc, gxy, gwh = t.chunk(3, 1) # (image, class), grid xy, grid wh\n b, c = bc.long().T # image, class\n gij = (gxy - offsets).long()\n gi, gj = gij.T # grid indices\n\n # Append\n indices.append((b, gj.clamp_(0, shape[2] - 1), gi.clamp_(0, shape[3] - 1))) # image, grid_y, grid_x indices\n tbox.append(torch.cat((gxy - gij, gwh), 1)) # box\n tcls.append(c) # class\n\n return tcls, tbox, indices\n\n\nclass ComputeLoss_NEW:\n sort_obj_iou = False\n\n # Compute losses\n def __init__(self, model, autobalance=False):\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device))\n BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device))\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h['fl_gamma'] # focal loss gamma\n if g > 0:\n BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.ssi = list(m.stride).index(16) if autobalance else 0 # stride 16 index\n self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, 1.0, h, autobalance\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.anchors = m.anchors\n self.device = device\n self.BCE_base = nn.BCEWithLogitsLoss(reduction='none')\n\n def __call__(self, p, targets): # predictions, targets\n tcls, tbox, indices = self.build_targets(p, targets) # targets\n bs = p[0].shape[0] # batch size\n n_labels = targets.shape[0] # number of labels\n loss = torch.zeros(3, device=self.device) # [box, obj, cls] losses\n\n # Compute all losses\n all_loss = []\n for i, pi in enumerate(p): # layer index, layer predictions\n b, gj, gi = indices[i] # image, anchor, gridy, gridx\n if n_labels:\n pxy, pwh, pobj, pcls = pi[b, :, gj, gi].split((2, 2, 1, self.nc), 2) # target-subset of predictions\n\n # Regression\n pbox = torch.cat((pxy.sigmoid() * 1.6 - 0.3, (0.2 + pwh.sigmoid() * 4.8) * self.anchors[i]), 2)\n iou = bbox_iou(pbox, tbox[i], CIoU=True).squeeze() # iou(predicted_box, target_box)\n obj_target = iou.detach().clamp(0).type(pi.dtype) # objectness targets\n\n all_loss.append([(1.0 - iou) * self.hyp['box'],\n self.BCE_base(pobj.squeeze(), torch.ones_like(obj_target)) * self.hyp['obj'],\n self.BCE_base(pcls, F.one_hot(tcls[i], self.nc).float()).mean(2) * self.hyp['cls'],\n obj_target,\n tbox[i][..., 2] > 0.0]) # valid\n\n # Lowest 3 losses per label\n n_assign = 4 # top n matches\n cat_loss = [torch.cat(x, 1) for x in zip(*all_loss)]\n ij = torch.zeros_like(cat_loss[0]).bool() # top 3 mask\n sum_loss = cat_loss[0] + cat_loss[2]\n for col in torch.argsort(sum_loss, dim=1).T[:n_assign]:\n # ij[range(n_labels), col] = True\n ij[range(n_labels), col] = cat_loss[4][range(n_labels), col]\n loss[0] = cat_loss[0][ij].mean() * self.nl # box loss\n loss[2] = cat_loss[2][ij].mean() * self.nl # cls loss\n\n # Obj loss\n for i, (h, pi) in enumerate(zip(ij.chunk(self.nl, 1), p)): # layer index, layer predictions\n b, gj, gi = indices[i] # image, anchor, gridy, gridx\n tobj = torch.zeros((pi.shape[0], pi.shape[2], pi.shape[3]), dtype=pi.dtype, device=self.device) # obj\n if n_labels: # if any labels\n tobj[b[h], gj[h], gi[h]] = all_loss[i][3][h]\n loss[1] += self.BCEobj(pi[:, 4], tobj) * (self.balance[i] * self.hyp['obj'])\n\n return loss.sum() * bs, loss.detach() # [box, obj, cls] losses\n\n def build_targets(self, p, targets):\n # Build targets for compute_loss(), input targets(image,class,x,y,w,h)\n nt = targets.shape[0] # number of anchors, targets\n tcls, tbox, indices = [], [], []\n gain = torch.ones(6, device=self.device) # normalized to gridspace gain\n\n g = 0.3 # bias\n off = torch.tensor(\n [\n [0, 0],\n [1, 0],\n [0, 1],\n [-1, 0],\n [0, -1], # j,k,l,m\n # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm\n ],\n device=self.device).float() # offsets\n\n for i in range(self.nl):\n shape = p[i].shape\n gain[2:6] = torch.tensor(shape)[[3, 2, 3, 2]] # xyxy gain\n\n # Match targets to anchors\n t = targets * gain # shape(3,n,7)\n if nt:\n # # Matches\n r = t[..., 4:6] / self.anchors[i] # wh ratio\n a = torch.max(r, 1 / r).max(1)[0] < self.hyp['anchor_t'] # compare\n # a = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2))\n # t = t[a] # filter\n\n # # Offsets\n gxy = t[:, 2:4] # grid xy\n gxi = gain[[2, 3]] - gxy # inverse\n j, k = ((gxy % 1 < g) & (gxy > 1)).T\n l, m = ((gxi % 1 < g) & (gxi > 1)).T\n j = torch.stack((torch.ones_like(j), j, k, l, m)) & a\n t = t.repeat((5, 1, 1))\n offsets = torch.zeros_like(gxy)[None] + off[:, None]\n t[..., 4:6][~j] = 0.0 # move unsuitable targets far away\n else:\n t = targets[0]\n offsets = 0\n\n # Define\n bc, gxy, gwh = t.chunk(3, 2) # (image, class), grid xy, grid wh\n b, c = bc.long().transpose(0, 2).contiguous() # image, class\n gij = (gxy - offsets).long()\n gi, gj = gij.transpose(0, 2).contiguous() # grid indices\n\n # Append\n indices.append((b, gj.clamp_(0, shape[2] - 1), gi.clamp_(0, shape[3] - 1))) # image, grid_y, grid_x indices\n tbox.append(torch.cat((gxy - gij, gwh), 2).permute(1, 0, 2).contiguous()) # box\n tcls.append(c) # class\n\n # # Unique\n # n1 = torch.cat((b.view(-1, 1), tbox[i].view(-1, 4)), 1).shape[0]\n # n2 = tbox[i].view(-1, 4).unique(dim=0).shape[0]\n # print(f'targets-unique {n1}-{n2} diff={n1-n2}')\n\n return tcls, tbox, indices\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/loss_tal.py", "content": "import os\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom asone.detectors.yolov9.yolov9.utils.general import xywh2xyxy\nfrom asone.detectors.yolov9.yolov9.utils.metrics import bbox_iou\nfrom asone.detectors.yolov9.yolov9.utils.tal.anchor_generator import dist2bbox, make_anchors, bbox2dist\nfrom asone.detectors.yolov9.yolov9.utils.tal.assigner import TaskAlignedAssigner\nfrom asone.detectors.yolov9.yolov9.utils.torch_utils import de_parallel\n\n\ndef smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441\n # return positive, negative label smoothing BCE targets\n return 1.0 - 0.5 * eps, 0.5 * eps\n\n\nclass VarifocalLoss(nn.Module):\n # Varifocal loss by Zhang et al. https://arxiv.org/abs/2008.13367\n def __init__(self):\n super().__init__()\n\n def forward(self, pred_score, gt_score, label, alpha=0.75, gamma=2.0):\n weight = alpha * pred_score.sigmoid().pow(gamma) * (1 - label) + gt_score * label\n with torch.cuda.amp.autocast(enabled=False):\n loss = (F.binary_cross_entropy_with_logits(pred_score.float(), gt_score.float(),\n reduction=\"none\") * weight).sum()\n return loss\n\n\nclass FocalLoss(nn.Module):\n # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\n super().__init__()\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\n self.gamma = gamma\n self.alpha = alpha\n self.reduction = loss_fcn.reduction\n self.loss_fcn.reduction = \"none\" # required to apply FL to each element\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n # p_t = torch.exp(-loss)\n # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability\n\n # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py\n pred_prob = torch.sigmoid(pred) # prob from logits\n p_t = true * pred_prob + (1 - true) * (1 - pred_prob)\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\n modulating_factor = (1.0 - p_t) ** self.gamma\n loss *= alpha_factor * modulating_factor\n\n if self.reduction == \"mean\":\n return loss.mean()\n elif self.reduction == \"sum\":\n return loss.sum()\n else: # 'none'\n return loss\n\n\nclass BboxLoss(nn.Module):\n def __init__(self, reg_max, use_dfl=False):\n super().__init__()\n self.reg_max = reg_max\n self.use_dfl = use_dfl\n\n def forward(self, pred_dist, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask):\n # iou loss\n bbox_mask = fg_mask.unsqueeze(-1).repeat([1, 1, 4]) # (b, h*w, 4)\n pred_bboxes_pos = torch.masked_select(pred_bboxes, bbox_mask).view(-1, 4)\n target_bboxes_pos = torch.masked_select(target_bboxes, bbox_mask).view(-1, 4)\n bbox_weight = torch.masked_select(target_scores.sum(-1), fg_mask).unsqueeze(-1)\n \n iou = bbox_iou(pred_bboxes_pos, target_bboxes_pos, xywh=False, CIoU=True)\n loss_iou = 1.0 - iou\n\n loss_iou *= bbox_weight\n loss_iou = loss_iou.sum() / target_scores_sum\n\n # dfl loss\n if self.use_dfl:\n dist_mask = fg_mask.unsqueeze(-1).repeat([1, 1, (self.reg_max + 1) * 4])\n pred_dist_pos = torch.masked_select(pred_dist, dist_mask).view(-1, 4, self.reg_max + 1)\n target_ltrb = bbox2dist(anchor_points, target_bboxes, self.reg_max)\n target_ltrb_pos = torch.masked_select(target_ltrb, bbox_mask).view(-1, 4)\n loss_dfl = self._df_loss(pred_dist_pos, target_ltrb_pos) * bbox_weight\n loss_dfl = loss_dfl.sum() / target_scores_sum\n else:\n loss_dfl = torch.tensor(0.0).to(pred_dist.device)\n\n return loss_iou, loss_dfl, iou\n\n def _df_loss(self, pred_dist, target):\n target_left = target.to(torch.long)\n target_right = target_left + 1\n weight_left = target_right.to(torch.float) - target\n weight_right = 1 - weight_left\n loss_left = F.cross_entropy(pred_dist.view(-1, self.reg_max + 1), target_left.view(-1), reduction=\"none\").view(\n target_left.shape) * weight_left\n loss_right = F.cross_entropy(pred_dist.view(-1, self.reg_max + 1), target_right.view(-1),\n reduction=\"none\").view(target_left.shape) * weight_right\n return (loss_left + loss_right).mean(-1, keepdim=True)\n\n\nclass ComputeLoss:\n # Compute losses\n def __init__(self, model, use_dfl=True):\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h[\"cls_pw\"]], device=device), reduction='none')\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get(\"label_smoothing\", 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h[\"fl_gamma\"] # focal loss gamma\n if g > 0:\n BCEcls = FocalLoss(BCEcls, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.BCEcls = BCEcls\n self.hyp = h\n self.stride = m.stride # model strides\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.no = m.no\n self.reg_max = m.reg_max\n self.device = device\n\n self.assigner = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.bbox_loss = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.proj = torch.arange(m.reg_max).float().to(device) # / 120.0\n self.use_dfl = use_dfl\n\n def preprocess(self, targets, batch_size, scale_tensor):\n if targets.shape[0] == 0:\n out = torch.zeros(batch_size, 0, 5, device=self.device)\n else:\n i = targets[:, 0] # image index\n _, counts = i.unique(return_counts=True)\n out = torch.zeros(batch_size, counts.max(), 5, device=self.device)\n for j in range(batch_size):\n matches = i == j\n n = matches.sum()\n if n:\n out[j, :n] = targets[matches, 1:]\n out[..., 1:5] = xywh2xyxy(out[..., 1:5].mul_(scale_tensor))\n return out\n\n def bbox_decode(self, anchor_points, pred_dist):\n if self.use_dfl:\n b, a, c = pred_dist.shape # batch, anchors, channels\n pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = pred_dist.view(b, a, c // 4, 4).transpose(2,3).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = (pred_dist.view(b, a, c // 4, 4).softmax(2) * self.proj.type(pred_dist.dtype).view(1, 1, -1, 1)).sum(2)\n return dist2bbox(pred_dist, anchor_points, xywh=False)\n\n def __call__(self, p, targets, img=None, epoch=0):\n loss = torch.zeros(3, device=self.device) # box, cls, dfl\n feats = p[1] if isinstance(p, tuple) else p\n pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores = pred_scores.permute(0, 2, 1).contiguous()\n pred_distri = pred_distri.permute(0, 2, 1).contiguous()\n\n dtype = pred_scores.dtype\n batch_size, grid_size = pred_scores.shape[:2]\n imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w)\n anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5)\n\n # targets\n targets = self.preprocess(targets, batch_size, scale_tensor=imgsz[[1, 0, 1, 0]])\n gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy\n mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0)\n\n # pboxes\n pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4)\n\n target_labels, target_bboxes, target_scores, fg_mask = self.assigner(\n pred_scores.detach().sigmoid(),\n (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n target_bboxes /= stride_tensor\n target_scores_sum = max(target_scores.sum(), 1)\n\n # cls loss\n # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way\n loss[1] = self.BCEcls(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n\n # bbox loss\n if fg_mask.sum():\n loss[0], loss[2], iou = self.bbox_loss(pred_distri,\n pred_bboxes,\n anchor_points,\n target_bboxes,\n target_scores,\n target_scores_sum,\n fg_mask)\n\n loss[0] *= 7.5 # box gain\n loss[1] *= 0.5 # cls gain\n loss[2] *= 1.5 # dfl gain\n\n return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/loss_tal_dual.py", "content": "import os\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom asone.detectors.yolov9.yolov9.utils.general import xywh2xyxy\nfrom asone.detectors.yolov9.yolov9.utils.metrics import bbox_iou\nfrom asone.detectors.yolov9.yolov9.utils.tal.anchor_generator import dist2bbox, make_anchors, bbox2dist\nfrom asone.detectors.yolov9.yolov9.utils.tal.assigner import TaskAlignedAssigner\nfrom asone.detectors.yolov9.yolov9.utils.torch_utils import de_parallel\n\n\ndef smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441\n # return positive, negative label smoothing BCE targets\n return 1.0 - 0.5 * eps, 0.5 * eps\n\n\nclass VarifocalLoss(nn.Module):\n # Varifocal loss by Zhang et al. https://arxiv.org/abs/2008.13367\n def __init__(self):\n super().__init__()\n\n def forward(self, pred_score, gt_score, label, alpha=0.75, gamma=2.0):\n weight = alpha * pred_score.sigmoid().pow(gamma) * (1 - label) + gt_score * label\n with torch.cuda.amp.autocast(enabled=False):\n loss = (F.binary_cross_entropy_with_logits(pred_score.float(), gt_score.float(),\n reduction=\"none\") * weight).sum()\n return loss\n\n\nclass FocalLoss(nn.Module):\n # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\n super().__init__()\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\n self.gamma = gamma\n self.alpha = alpha\n self.reduction = loss_fcn.reduction\n self.loss_fcn.reduction = \"none\" # required to apply FL to each element\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n # p_t = torch.exp(-loss)\n # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability\n\n # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py\n pred_prob = torch.sigmoid(pred) # prob from logits\n p_t = true * pred_prob + (1 - true) * (1 - pred_prob)\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\n modulating_factor = (1.0 - p_t) ** self.gamma\n loss *= alpha_factor * modulating_factor\n\n if self.reduction == \"mean\":\n return loss.mean()\n elif self.reduction == \"sum\":\n return loss.sum()\n else: # 'none'\n return loss\n\n\nclass BboxLoss(nn.Module):\n def __init__(self, reg_max, use_dfl=False):\n super().__init__()\n self.reg_max = reg_max\n self.use_dfl = use_dfl\n\n def forward(self, pred_dist, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask):\n # iou loss\n bbox_mask = fg_mask.unsqueeze(-1).repeat([1, 1, 4]) # (b, h*w, 4)\n pred_bboxes_pos = torch.masked_select(pred_bboxes, bbox_mask).view(-1, 4)\n target_bboxes_pos = torch.masked_select(target_bboxes, bbox_mask).view(-1, 4)\n bbox_weight = torch.masked_select(target_scores.sum(-1), fg_mask).unsqueeze(-1)\n \n iou = bbox_iou(pred_bboxes_pos, target_bboxes_pos, xywh=False, CIoU=True)\n loss_iou = 1.0 - iou\n\n loss_iou *= bbox_weight\n loss_iou = loss_iou.sum() / target_scores_sum\n\n # dfl loss\n if self.use_dfl:\n dist_mask = fg_mask.unsqueeze(-1).repeat([1, 1, (self.reg_max + 1) * 4])\n pred_dist_pos = torch.masked_select(pred_dist, dist_mask).view(-1, 4, self.reg_max + 1)\n target_ltrb = bbox2dist(anchor_points, target_bboxes, self.reg_max)\n target_ltrb_pos = torch.masked_select(target_ltrb, bbox_mask).view(-1, 4)\n loss_dfl = self._df_loss(pred_dist_pos, target_ltrb_pos) * bbox_weight\n loss_dfl = loss_dfl.sum() / target_scores_sum\n else:\n loss_dfl = torch.tensor(0.0).to(pred_dist.device)\n\n return loss_iou, loss_dfl, iou\n\n def _df_loss(self, pred_dist, target):\n target_left = target.to(torch.long)\n target_right = target_left + 1\n weight_left = target_right.to(torch.float) - target\n weight_right = 1 - weight_left\n loss_left = F.cross_entropy(pred_dist.view(-1, self.reg_max + 1), target_left.view(-1), reduction=\"none\").view(\n target_left.shape) * weight_left\n loss_right = F.cross_entropy(pred_dist.view(-1, self.reg_max + 1), target_right.view(-1),\n reduction=\"none\").view(target_left.shape) * weight_right\n return (loss_left + loss_right).mean(-1, keepdim=True)\n\n\nclass ComputeLoss:\n # Compute losses\n def __init__(self, model, use_dfl=True):\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h[\"cls_pw\"]], device=device), reduction='none')\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get(\"label_smoothing\", 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h[\"fl_gamma\"] # focal loss gamma\n if g > 0:\n BCEcls = FocalLoss(BCEcls, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.BCEcls = BCEcls\n self.hyp = h\n self.stride = m.stride # model strides\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.no = m.no\n self.reg_max = m.reg_max\n self.device = device\n\n self.assigner = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.assigner2 = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.bbox_loss = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.bbox_loss2 = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.proj = torch.arange(m.reg_max).float().to(device) # / 120.0\n self.use_dfl = use_dfl\n\n def preprocess(self, targets, batch_size, scale_tensor):\n if targets.shape[0] == 0:\n out = torch.zeros(batch_size, 0, 5, device=self.device)\n else:\n i = targets[:, 0] # image index\n _, counts = i.unique(return_counts=True)\n out = torch.zeros(batch_size, counts.max(), 5, device=self.device)\n for j in range(batch_size):\n matches = i == j\n n = matches.sum()\n if n:\n out[j, :n] = targets[matches, 1:]\n out[..., 1:5] = xywh2xyxy(out[..., 1:5].mul_(scale_tensor))\n return out\n\n def bbox_decode(self, anchor_points, pred_dist):\n if self.use_dfl:\n b, a, c = pred_dist.shape # batch, anchors, channels\n pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = pred_dist.view(b, a, c // 4, 4).transpose(2,3).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = (pred_dist.view(b, a, c // 4, 4).softmax(2) * self.proj.type(pred_dist.dtype).view(1, 1, -1, 1)).sum(2)\n return dist2bbox(pred_dist, anchor_points, xywh=False)\n\n def __call__(self, p, targets, img=None, epoch=0):\n loss = torch.zeros(3, device=self.device) # box, cls, dfl\n feats = p[1][0] if isinstance(p, tuple) else p[0]\n feats2 = p[1][1] if isinstance(p, tuple) else p[1]\n \n pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores = pred_scores.permute(0, 2, 1).contiguous()\n pred_distri = pred_distri.permute(0, 2, 1).contiguous()\n \n pred_distri2, pred_scores2 = torch.cat([xi.view(feats2[0].shape[0], self.no, -1) for xi in feats2], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores2 = pred_scores2.permute(0, 2, 1).contiguous()\n pred_distri2 = pred_distri2.permute(0, 2, 1).contiguous()\n\n dtype = pred_scores.dtype\n batch_size, grid_size = pred_scores.shape[:2]\n imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w)\n anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5)\n\n # targets\n targets = self.preprocess(targets, batch_size, scale_tensor=imgsz[[1, 0, 1, 0]])\n gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy\n mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0)\n\n # pboxes\n pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4)\n pred_bboxes2 = self.bbox_decode(anchor_points, pred_distri2) # xyxy, (b, h*w, 4)\n\n target_labels, target_bboxes, target_scores, fg_mask = self.assigner(\n pred_scores.detach().sigmoid(),\n (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n target_labels2, target_bboxes2, target_scores2, fg_mask2 = self.assigner2(\n pred_scores2.detach().sigmoid(),\n (pred_bboxes2.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n target_bboxes /= stride_tensor\n target_scores_sum = max(target_scores.sum(), 1)\n target_bboxes2 /= stride_tensor\n target_scores_sum2 = max(target_scores2.sum(), 1)\n\n # cls loss\n # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way\n loss[1] = self.BCEcls(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n loss[1] *= 0.25\n loss[1] += self.BCEcls(pred_scores2, target_scores2.to(dtype)).sum() / target_scores_sum2 # BCE\n\n # bbox loss\n if fg_mask.sum():\n loss[0], loss[2], iou = self.bbox_loss(pred_distri,\n pred_bboxes,\n anchor_points,\n target_bboxes,\n target_scores,\n target_scores_sum,\n fg_mask)\n loss[0] *= 0.25\n loss[2] *= 0.25\n if fg_mask2.sum():\n loss0_, loss2_, iou2 = self.bbox_loss2(pred_distri2,\n pred_bboxes2,\n anchor_points,\n target_bboxes2,\n target_scores2,\n target_scores_sum2,\n fg_mask2)\n loss[0] += loss0_\n loss[2] += loss2_\n\n loss[0] *= 7.5 # box gain\n loss[1] *= 0.5 # cls gain\n loss[2] *= 1.5 # dfl gain\n\n return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl)\n\n\nclass ComputeLossLH:\n # Compute losses\n def __init__(self, model, use_dfl=True):\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h[\"cls_pw\"]], device=device), reduction='none')\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get(\"label_smoothing\", 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h[\"fl_gamma\"] # focal loss gamma\n if g > 0:\n BCEcls = FocalLoss(BCEcls, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.BCEcls = BCEcls\n self.hyp = h\n self.stride = m.stride # model strides\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.no = m.no\n self.reg_max = m.reg_max\n self.device = device\n\n self.assigner = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.bbox_loss = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.proj = torch.arange(m.reg_max).float().to(device) # / 120.0\n self.use_dfl = use_dfl\n\n def preprocess(self, targets, batch_size, scale_tensor):\n if targets.shape[0] == 0:\n out = torch.zeros(batch_size, 0, 5, device=self.device)\n else:\n i = targets[:, 0] # image index\n _, counts = i.unique(return_counts=True)\n out = torch.zeros(batch_size, counts.max(), 5, device=self.device)\n for j in range(batch_size):\n matches = i == j\n n = matches.sum()\n if n:\n out[j, :n] = targets[matches, 1:]\n out[..., 1:5] = xywh2xyxy(out[..., 1:5].mul_(scale_tensor))\n return out\n\n def bbox_decode(self, anchor_points, pred_dist):\n if self.use_dfl:\n b, a, c = pred_dist.shape # batch, anchors, channels\n pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = pred_dist.view(b, a, c // 4, 4).transpose(2,3).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = (pred_dist.view(b, a, c // 4, 4).softmax(2) * self.proj.type(pred_dist.dtype).view(1, 1, -1, 1)).sum(2)\n return dist2bbox(pred_dist, anchor_points, xywh=False)\n\n def __call__(self, p, targets, img=None, epoch=0):\n loss = torch.zeros(3, device=self.device) # box, cls, dfl\n feats = p[1][0] if isinstance(p, tuple) else p[0]\n feats2 = p[1][1] if isinstance(p, tuple) else p[1]\n \n pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores = pred_scores.permute(0, 2, 1).contiguous()\n pred_distri = pred_distri.permute(0, 2, 1).contiguous()\n \n pred_distri2, pred_scores2 = torch.cat([xi.view(feats2[0].shape[0], self.no, -1) for xi in feats2], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores2 = pred_scores2.permute(0, 2, 1).contiguous()\n pred_distri2 = pred_distri2.permute(0, 2, 1).contiguous()\n\n dtype = pred_scores.dtype\n batch_size, grid_size = pred_scores.shape[:2]\n imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w)\n anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5)\n\n # targets\n targets = self.preprocess(targets, batch_size, scale_tensor=imgsz[[1, 0, 1, 0]])\n gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy\n mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0)\n\n # pboxes\n pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4)\n pred_bboxes2 = self.bbox_decode(anchor_points, pred_distri2) # xyxy, (b, h*w, 4)\n\n target_labels, target_bboxes, target_scores, fg_mask = self.assigner(\n pred_scores2.detach().sigmoid(),\n (pred_bboxes2.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n target_bboxes /= stride_tensor\n target_scores_sum = target_scores.sum()\n\n # cls loss\n # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way\n loss[1] = self.BCEcls(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n loss[1] *= 0.25\n loss[1] += self.BCEcls(pred_scores2, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n\n # bbox loss\n if fg_mask.sum():\n loss[0], loss[2], iou = self.bbox_loss(pred_distri,\n pred_bboxes,\n anchor_points,\n target_bboxes,\n target_scores,\n target_scores_sum,\n fg_mask)\n loss[0] *= 0.25\n loss[2] *= 0.25\n if fg_mask.sum():\n loss0_, loss2_, iou2 = self.bbox_loss(pred_distri2,\n pred_bboxes2,\n anchor_points,\n target_bboxes,\n target_scores,\n target_scores_sum,\n fg_mask)\n loss[0] += loss0_\n loss[2] += loss2_\n\n loss[0] *= 7.5 # box gain\n loss[1] *= 0.5 # cls gain\n loss[2] *= 1.5 # dfl gain\n\n return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/loss_tal_triple.py", "content": "import os\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom asone.detectors.yolov9.yolov9.utils.general import xywh2xyxy\nfrom asone.detectors.yolov9.yolov9.utils.metrics import bbox_iou\nfrom asone.detectors.yolov9.yolov9.utils.tal.anchor_generator import dist2bbox, make_anchors, bbox2dist\nfrom asone.detectors.yolov9.yolov9.utils.tal.assigner import TaskAlignedAssigner\nfrom asone.detectors.yolov9.yolov9.utils.torch_utils import de_parallel\n\n\ndef smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441\n # return positive, negative label smoothing BCE targets\n return 1.0 - 0.5 * eps, 0.5 * eps\n\n\nclass VarifocalLoss(nn.Module):\n # Varifocal loss by Zhang et al. https://arxiv.org/abs/2008.13367\n def __init__(self):\n super().__init__()\n\n def forward(self, pred_score, gt_score, label, alpha=0.75, gamma=2.0):\n weight = alpha * pred_score.sigmoid().pow(gamma) * (1 - label) + gt_score * label\n with torch.cuda.amp.autocast(enabled=False):\n loss = (F.binary_cross_entropy_with_logits(pred_score.float(), gt_score.float(),\n reduction=\"none\") * weight).sum()\n return loss\n\n\nclass FocalLoss(nn.Module):\n # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\n super().__init__()\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\n self.gamma = gamma\n self.alpha = alpha\n self.reduction = loss_fcn.reduction\n self.loss_fcn.reduction = \"none\" # required to apply FL to each element\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n # p_t = torch.exp(-loss)\n # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability\n\n # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py\n pred_prob = torch.sigmoid(pred) # prob from logits\n p_t = true * pred_prob + (1 - true) * (1 - pred_prob)\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\n modulating_factor = (1.0 - p_t) ** self.gamma\n loss *= alpha_factor * modulating_factor\n\n if self.reduction == \"mean\":\n return loss.mean()\n elif self.reduction == \"sum\":\n return loss.sum()\n else: # 'none'\n return loss\n\n\nclass BboxLoss(nn.Module):\n def __init__(self, reg_max, use_dfl=False):\n super().__init__()\n self.reg_max = reg_max\n self.use_dfl = use_dfl\n\n def forward(self, pred_dist, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask):\n # iou loss\n bbox_mask = fg_mask.unsqueeze(-1).repeat([1, 1, 4]) # (b, h*w, 4)\n pred_bboxes_pos = torch.masked_select(pred_bboxes, bbox_mask).view(-1, 4)\n target_bboxes_pos = torch.masked_select(target_bboxes, bbox_mask).view(-1, 4)\n bbox_weight = torch.masked_select(target_scores.sum(-1), fg_mask).unsqueeze(-1)\n \n iou = bbox_iou(pred_bboxes_pos, target_bboxes_pos, xywh=False, CIoU=True)\n loss_iou = 1.0 - iou\n\n loss_iou *= bbox_weight\n loss_iou = loss_iou.sum() / target_scores_sum\n\n # dfl loss\n if self.use_dfl:\n dist_mask = fg_mask.unsqueeze(-1).repeat([1, 1, (self.reg_max + 1) * 4])\n pred_dist_pos = torch.masked_select(pred_dist, dist_mask).view(-1, 4, self.reg_max + 1)\n target_ltrb = bbox2dist(anchor_points, target_bboxes, self.reg_max)\n target_ltrb_pos = torch.masked_select(target_ltrb, bbox_mask).view(-1, 4)\n loss_dfl = self._df_loss(pred_dist_pos, target_ltrb_pos) * bbox_weight\n loss_dfl = loss_dfl.sum() / target_scores_sum\n else:\n loss_dfl = torch.tensor(0.0).to(pred_dist.device)\n\n return loss_iou, loss_dfl, iou\n\n def _df_loss(self, pred_dist, target):\n target_left = target.to(torch.long)\n target_right = target_left + 1\n weight_left = target_right.to(torch.float) - target\n weight_right = 1 - weight_left\n loss_left = F.cross_entropy(pred_dist.view(-1, self.reg_max + 1), target_left.view(-1), reduction=\"none\").view(\n target_left.shape) * weight_left\n loss_right = F.cross_entropy(pred_dist.view(-1, self.reg_max + 1), target_right.view(-1),\n reduction=\"none\").view(target_left.shape) * weight_right\n return (loss_left + loss_right).mean(-1, keepdim=True)\n\n\nclass ComputeLoss:\n # Compute losses\n def __init__(self, model, use_dfl=True):\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h[\"cls_pw\"]], device=device), reduction='none')\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get(\"label_smoothing\", 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h[\"fl_gamma\"] # focal loss gamma\n if g > 0:\n BCEcls = FocalLoss(BCEcls, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.BCEcls = BCEcls\n self.hyp = h\n self.stride = m.stride # model strides\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.no = m.no\n self.reg_max = m.reg_max\n self.device = device\n\n self.assigner = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.assigner2 = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.assigner3 = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.bbox_loss = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.bbox_loss2 = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.bbox_loss3 = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.proj = torch.arange(m.reg_max).float().to(device) # / 120.0\n self.use_dfl = use_dfl\n\n def preprocess(self, targets, batch_size, scale_tensor):\n if targets.shape[0] == 0:\n out = torch.zeros(batch_size, 0, 5, device=self.device)\n else:\n i = targets[:, 0] # image index\n _, counts = i.unique(return_counts=True)\n out = torch.zeros(batch_size, counts.max(), 5, device=self.device)\n for j in range(batch_size):\n matches = i == j\n n = matches.sum()\n if n:\n out[j, :n] = targets[matches, 1:]\n out[..., 1:5] = xywh2xyxy(out[..., 1:5].mul_(scale_tensor))\n return out\n\n def bbox_decode(self, anchor_points, pred_dist):\n if self.use_dfl:\n b, a, c = pred_dist.shape # batch, anchors, channels\n pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = pred_dist.view(b, a, c // 4, 4).transpose(2,3).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = (pred_dist.view(b, a, c // 4, 4).softmax(2) * self.proj.type(pred_dist.dtype).view(1, 1, -1, 1)).sum(2)\n return dist2bbox(pred_dist, anchor_points, xywh=False)\n\n def __call__(self, p, targets, img=None, epoch=0):\n loss = torch.zeros(3, device=self.device) # box, cls, dfl\n feats = p[1][0] if isinstance(p, tuple) else p[0]\n feats2 = p[1][1] if isinstance(p, tuple) else p[1]\n feats3 = p[1][2] if isinstance(p, tuple) else p[2]\n \n pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores = pred_scores.permute(0, 2, 1).contiguous()\n pred_distri = pred_distri.permute(0, 2, 1).contiguous()\n \n pred_distri2, pred_scores2 = torch.cat([xi.view(feats2[0].shape[0], self.no, -1) for xi in feats2], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores2 = pred_scores2.permute(0, 2, 1).contiguous()\n pred_distri2 = pred_distri2.permute(0, 2, 1).contiguous()\n \n pred_distri3, pred_scores3 = torch.cat([xi.view(feats3[0].shape[0], self.no, -1) for xi in feats3], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores3 = pred_scores3.permute(0, 2, 1).contiguous()\n pred_distri3 = pred_distri3.permute(0, 2, 1).contiguous()\n\n dtype = pred_scores.dtype\n batch_size, grid_size = pred_scores.shape[:2]\n imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w)\n anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5)\n\n # targets\n targets = self.preprocess(targets, batch_size, scale_tensor=imgsz[[1, 0, 1, 0]])\n gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy\n mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0)\n\n # pboxes\n pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4)\n pred_bboxes2 = self.bbox_decode(anchor_points, pred_distri2) # xyxy, (b, h*w, 4)\n pred_bboxes3 = self.bbox_decode(anchor_points, pred_distri3) # xyxy, (b, h*w, 4)\n\n target_labels, target_bboxes, target_scores, fg_mask = self.assigner(\n pred_scores.detach().sigmoid(),\n (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n target_labels2, target_bboxes2, target_scores2, fg_mask2 = self.assigner2(\n pred_scores2.detach().sigmoid(),\n (pred_bboxes2.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n target_labels3, target_bboxes3, target_scores3, fg_mask3 = self.assigner3(\n pred_scores3.detach().sigmoid(),\n (pred_bboxes3.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n target_bboxes /= stride_tensor\n target_scores_sum = max(target_scores.sum(), 1)\n target_bboxes2 /= stride_tensor\n target_scores_sum2 = max(target_scores2.sum(), 1)\n target_bboxes3 /= stride_tensor\n target_scores_sum3 = max(target_scores3.sum(), 1)\n\n # cls loss\n # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way\n loss[1] = 0.25 * self.BCEcls(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n loss[1] += 0.25 * self.BCEcls(pred_scores2, target_scores2.to(dtype)).sum() / target_scores_sum2 # BCE\n loss[1] += self.BCEcls(pred_scores3, target_scores3.to(dtype)).sum() / target_scores_sum3 # BCE\n\n # bbox loss\n if fg_mask.sum():\n loss[0], loss[2], iou = self.bbox_loss(pred_distri,\n pred_bboxes,\n anchor_points,\n target_bboxes,\n target_scores,\n target_scores_sum,\n fg_mask)\n loss[0] *= 0.25\n loss[2] *= 0.25\n if fg_mask2.sum():\n loss0_, loss2_, iou2 = self.bbox_loss2(pred_distri2,\n pred_bboxes2,\n anchor_points,\n target_bboxes2,\n target_scores2,\n target_scores_sum2,\n fg_mask2)\n loss[0] += 0.25 * loss0_\n loss[2] += 0.25 * loss2_\n if fg_mask3.sum():\n loss0__, loss2__, iou3 = self.bbox_loss3(pred_distri3,\n pred_bboxes3,\n anchor_points,\n target_bboxes3,\n target_scores3,\n target_scores_sum3,\n fg_mask3)\n loss[0] += loss0__\n loss[2] += loss2__\n\n loss[0] *= 7.5 # box gain\n loss[1] *= 0.5 # cls gain\n loss[2] *= 1.5 # dfl gain\n\n return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/metrics.py", "content": "import math\nimport warnings\nfrom pathlib import Path\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\n\nfrom asone.detectors.yolov9.yolov9.utils import TryExcept, threaded\n\n\ndef fitness(x):\n # Model fitness as a weighted combination of metrics\n w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]\n return (x[:, :4] * w).sum(1)\n\n\ndef smooth(y, f=0.05):\n # Box filter of fraction f\n nf = round(len(y) * f * 2) // 2 + 1 # number of filter elements (must be odd)\n p = np.ones(nf // 2) # ones padding\n yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded\n return np.convolve(yp, np.ones(nf) / nf, mode='valid') # y-smoothed\n\n\ndef ap_per_class(tp, conf, pred_cls, target_cls, plot=False, save_dir='.', names=(), eps=1e-16, prefix=\"\"):\n \"\"\" Compute the average precision, given the recall and precision curves.\n Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.\n # Arguments\n tp: True positives (nparray, nx1 or nx10).\n conf: Objectness value from 0-1 (nparray).\n pred_cls: Predicted object classes (nparray).\n target_cls: True object classes (nparray).\n plot: Plot precision-recall curve at mAP@0.5\n save_dir: Plot save directory\n # Returns\n The average precision as computed in py-faster-rcnn.\n \"\"\"\n\n # Sort by objectness\n i = np.argsort(-conf)\n tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]\n\n # Find unique classes\n unique_classes, nt = np.unique(target_cls, return_counts=True)\n nc = unique_classes.shape[0] # number of classes, number of detections\n\n # Create Precision-Recall curve and compute AP for each class\n px, py = np.linspace(0, 1, 1000), [] # for plotting\n ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000))\n for ci, c in enumerate(unique_classes):\n i = pred_cls == c\n n_l = nt[ci] # number of labels\n n_p = i.sum() # number of predictions\n if n_p == 0 or n_l == 0:\n continue\n\n # Accumulate FPs and TPs\n fpc = (1 - tp[i]).cumsum(0)\n tpc = tp[i].cumsum(0)\n\n # Recall\n recall = tpc / (n_l + eps) # recall curve\n r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases\n\n # Precision\n precision = tpc / (tpc + fpc) # precision curve\n p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score\n\n # AP from recall-precision curve\n for j in range(tp.shape[1]):\n ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j])\n if plot and j == 0:\n py.append(np.interp(px, mrec, mpre)) # precision at mAP@0.5\n\n # Compute F1 (harmonic mean of precision and recall)\n f1 = 2 * p * r / (p + r + eps)\n names = [v for k, v in names.items() if k in unique_classes] # list: only classes that have data\n names = dict(enumerate(names)) # to dict\n if plot:\n plot_pr_curve(px, py, ap, Path(save_dir) / f'{prefix}PR_curve.png', names)\n plot_mc_curve(px, f1, Path(save_dir) / f'{prefix}F1_curve.png', names, ylabel='F1')\n plot_mc_curve(px, p, Path(save_dir) / f'{prefix}P_curve.png', names, ylabel='Precision')\n plot_mc_curve(px, r, Path(save_dir) / f'{prefix}R_curve.png', names, ylabel='Recall')\n\n i = smooth(f1.mean(0), 0.1).argmax() # max F1 index\n p, r, f1 = p[:, i], r[:, i], f1[:, i]\n tp = (r * nt).round() # true positives\n fp = (tp / (p + eps) - tp).round() # false positives\n return tp, fp, p, r, f1, ap, unique_classes.astype(int)\n\n\ndef compute_ap(recall, precision):\n \"\"\" Compute the average precision, given the recall and precision curves\n # Arguments\n recall: The recall curve (list)\n precision: The precision curve (list)\n # Returns\n Average precision, precision curve, recall curve\n \"\"\"\n\n # Append sentinel values to beginning and end\n mrec = np.concatenate(([0.0], recall, [1.0]))\n mpre = np.concatenate(([1.0], precision, [0.0]))\n\n # Compute the precision envelope\n mpre = np.flip(np.maximum.accumulate(np.flip(mpre)))\n\n # Integrate area under curve\n method = 'interp' # methods: 'continuous', 'interp'\n if method == 'interp':\n x = np.linspace(0, 1, 101) # 101-point interp (COCO)\n ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate\n else: # 'continuous'\n i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes\n ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve\n\n return ap, mpre, mrec\n\n\nclass ConfusionMatrix:\n # Updated version of https://github.com/kaanakan/object_detection_confusion_matrix\n def __init__(self, nc, conf=0.25, iou_thres=0.45):\n self.matrix = np.zeros((nc + 1, nc + 1))\n self.nc = nc # number of classes\n self.conf = conf\n self.iou_thres = iou_thres\n\n def process_batch(self, detections, labels):\n \"\"\"\n Return intersection-over-union (Jaccard index) of boxes.\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\n Arguments:\n detections (Array[N, 6]), x1, y1, x2, y2, conf, class\n labels (Array[M, 5]), class, x1, y1, x2, y2\n Returns:\n None, updates confusion matrix accordingly\n \"\"\"\n if detections is None:\n gt_classes = labels.int()\n for gc in gt_classes:\n self.matrix[self.nc, gc] += 1 # background FN\n return\n\n detections = detections[detections[:, 4] > self.conf]\n gt_classes = labels[:, 0].int()\n detection_classes = detections[:, 5].int()\n iou = box_iou(labels[:, 1:], detections[:, :4])\n\n x = torch.where(iou > self.iou_thres)\n if x[0].shape[0]:\n matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy()\n if x[0].shape[0] > 1:\n matches = matches[matches[:, 2].argsort()[::-1]]\n matches = matches[np.unique(matches[:, 1], return_index=True)[1]]\n matches = matches[matches[:, 2].argsort()[::-1]]\n matches = matches[np.unique(matches[:, 0], return_index=True)[1]]\n else:\n matches = np.zeros((0, 3))\n\n n = matches.shape[0] > 0\n m0, m1, _ = matches.transpose().astype(int)\n for i, gc in enumerate(gt_classes):\n j = m0 == i\n if n and sum(j) == 1:\n self.matrix[detection_classes[m1[j]], gc] += 1 # correct\n else:\n self.matrix[self.nc, gc] += 1 # true background\n\n if n:\n for i, dc in enumerate(detection_classes):\n if not any(m1 == i):\n self.matrix[dc, self.nc] += 1 # predicted background\n\n def matrix(self):\n return self.matrix\n\n def tp_fp(self):\n tp = self.matrix.diagonal() # true positives\n fp = self.matrix.sum(1) - tp # false positives\n # fn = self.matrix.sum(0) - tp # false negatives (missed detections)\n return tp[:-1], fp[:-1] # remove background class\n\n @TryExcept('WARNING ⚠️ ConfusionMatrix plot failure')\n def plot(self, normalize=True, save_dir='', names=()):\n import seaborn as sn\n\n array = self.matrix / ((self.matrix.sum(0).reshape(1, -1) + 1E-9) if normalize else 1) # normalize columns\n array[array < 0.005] = np.nan # don't annotate (would appear as 0.00)\n\n fig, ax = plt.subplots(1, 1, figsize=(12, 9), tight_layout=True)\n nc, nn = self.nc, len(names) # number of classes, names\n sn.set(font_scale=1.0 if nc < 50 else 0.8) # for label size\n labels = (0 < nn < 99) and (nn == nc) # apply names to ticklabels\n ticklabels = (names + ['background']) if labels else \"auto\"\n with warnings.catch_warnings():\n warnings.simplefilter('ignore') # suppress empty matrix RuntimeWarning: All-NaN slice encountered\n sn.heatmap(array,\n ax=ax,\n annot=nc < 30,\n annot_kws={\n \"size\": 8},\n cmap='Blues',\n fmt='.2f',\n square=True,\n vmin=0.0,\n xticklabels=ticklabels,\n yticklabels=ticklabels).set_facecolor((1, 1, 1))\n ax.set_ylabel('True')\n ax.set_ylabel('Predicted')\n ax.set_title('Confusion Matrix')\n fig.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250)\n plt.close(fig)\n\n def print(self):\n for i in range(self.nc + 1):\n print(' '.join(map(str, self.matrix[i])))\n \n\nclass WIoU_Scale:\n ''' monotonous: {\n None: origin v1\n True: monotonic FM v2\n False: non-monotonic FM v3\n }\n momentum: The momentum of running mean'''\n \n iou_mean = 1.\n monotonous = False\n _momentum = 1 - 0.5 ** (1 / 7000)\n _is_train = True\n\n def __init__(self, iou):\n self.iou = iou\n self._update(self)\n \n @classmethod\n def _update(cls, self):\n if cls._is_train: cls.iou_mean = (1 - cls._momentum) * cls.iou_mean + \\\n cls._momentum * self.iou.detach().mean().item()\n \n @classmethod\n def _scaled_loss(cls, self, gamma=1.9, delta=3):\n if isinstance(self.monotonous, bool):\n if self.monotonous:\n return (self.iou.detach() / self.iou_mean).sqrt()\n else:\n beta = self.iou.detach() / self.iou_mean\n alpha = delta * torch.pow(gamma, beta - delta)\n return beta / alpha\n return 1\n\n\ndef bbox_iou(box1, box2, xywh=True, GIoU=False, DIoU=False, CIoU=False, MDPIoU=False, feat_h=640, feat_w=640, eps=1e-7):\n # Returns Intersection over Union (IoU) of box1(1,4) to box2(n,4)\n\n # Get the coordinates of bounding boxes\n if xywh: # transform from xywh to xyxy\n (x1, y1, w1, h1), (x2, y2, w2, h2) = box1.chunk(4, -1), box2.chunk(4, -1)\n w1_, h1_, w2_, h2_ = w1 / 2, h1 / 2, w2 / 2, h2 / 2\n b1_x1, b1_x2, b1_y1, b1_y2 = x1 - w1_, x1 + w1_, y1 - h1_, y1 + h1_\n b2_x1, b2_x2, b2_y1, b2_y2 = x2 - w2_, x2 + w2_, y2 - h2_, y2 + h2_\n else: # x1, y1, x2, y2 = box1\n b1_x1, b1_y1, b1_x2, b1_y2 = box1.chunk(4, -1)\n b2_x1, b2_y1, b2_x2, b2_y2 = box2.chunk(4, -1)\n w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps\n w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps\n\n # Intersection area\n inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \\\n (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0)\n\n # Union Area\n union = w1 * h1 + w2 * h2 - inter + eps\n\n # IoU\n iou = inter / union\n if CIoU or DIoU or GIoU:\n cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1) # convex (smallest enclosing box) width\n ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height\n if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1\n c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared\n rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 + (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center dist ** 2\n if CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47\n v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2)\n with torch.no_grad():\n alpha = v / (v - iou + (1 + eps))\n return iou - (rho2 / c2 + v * alpha) # CIoU\n return iou - rho2 / c2 # DIoU\n c_area = cw * ch + eps # convex area\n return iou - (c_area - union) / c_area # GIoU https://arxiv.org/pdf/1902.09630.pdf\n elif MDPIoU:\n d1 = (b2_x1 - b1_x1) ** 2 + (b2_y1 - b1_y1) ** 2\n d2 = (b2_x2 - b1_x2) ** 2 + (b2_y2 - b1_y2) ** 2\n mpdiou_hw_pow = feat_h ** 2 + feat_w ** 2\n return iou - d1 / mpdiou_hw_pow - d2 / mpdiou_hw_pow # MPDIoU\n return iou # IoU\n\n\ndef box_iou(box1, box2, eps=1e-7):\n # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py\n \"\"\"\n Return intersection-over-union (Jaccard index) of boxes.\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\n Arguments:\n box1 (Tensor[N, 4])\n box2 (Tensor[M, 4])\n Returns:\n iou (Tensor[N, M]): the NxM matrix containing the pairwise\n IoU values for every element in boxes1 and boxes2\n \"\"\"\n\n # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)\n (a1, a2), (b1, b2) = box1.unsqueeze(1).chunk(2, 2), box2.unsqueeze(0).chunk(2, 2)\n inter = (torch.min(a2, b2) - torch.max(a1, b1)).clamp(0).prod(2)\n\n # IoU = inter / (area1 + area2 - inter)\n return inter / ((a2 - a1).prod(2) + (b2 - b1).prod(2) - inter + eps)\n\n\ndef bbox_ioa(box1, box2, eps=1e-7):\n \"\"\"Returns the intersection over box2 area given box1, box2. Boxes are x1y1x2y2\n box1: np.array of shape(nx4)\n box2: np.array of shape(mx4)\n returns: np.array of shape(nxm)\n \"\"\"\n\n # Get the coordinates of bounding boxes\n b1_x1, b1_y1, b1_x2, b1_y2 = box1.T\n b2_x1, b2_y1, b2_x2, b2_y2 = box2.T\n\n # Intersection area\n inter_area = (np.minimum(b1_x2[:, None], b2_x2) - np.maximum(b1_x1[:, None], b2_x1)).clip(0) * \\\n (np.minimum(b1_y2[:, None], b2_y2) - np.maximum(b1_y1[:, None], b2_y1)).clip(0)\n\n # box2 area\n box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + eps\n\n # Intersection over box2 area\n return inter_area / box2_area\n\n\ndef wh_iou(wh1, wh2, eps=1e-7):\n # Returns the nxm IoU matrix. wh1 is nx2, wh2 is mx2\n wh1 = wh1[:, None] # [N,1,2]\n wh2 = wh2[None] # [1,M,2]\n inter = torch.min(wh1, wh2).prod(2) # [N,M]\n return inter / (wh1.prod(2) + wh2.prod(2) - inter + eps) # iou = inter / (area1 + area2 - inter)\n\n\n# Plots ----------------------------------------------------------------------------------------------------------------\n\n\n@threaded\ndef plot_pr_curve(px, py, ap, save_dir=Path('pr_curve.png'), names=()):\n # Precision-recall curve\n fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)\n py = np.stack(py, axis=1)\n\n if 0 < len(names) < 21: # display per-class legend if < 21 classes\n for i, y in enumerate(py.T):\n ax.plot(px, y, linewidth=1, label=f'{names[i]} {ap[i, 0]:.3f}') # plot(recall, precision)\n else:\n ax.plot(px, py, linewidth=1, color='grey') # plot(recall, precision)\n\n ax.plot(px, py.mean(1), linewidth=3, color='blue', label='all classes %.3f mAP@0.5' % ap[:, 0].mean())\n ax.set_xlabel('Recall')\n ax.set_ylabel('Precision')\n ax.set_xlim(0, 1)\n ax.set_ylim(0, 1)\n ax.legend(bbox_to_anchor=(1.04, 1), loc=\"upper left\")\n ax.set_title('Precision-Recall Curve')\n fig.savefig(save_dir, dpi=250)\n plt.close(fig)\n\n\n@threaded\ndef plot_mc_curve(px, py, save_dir=Path('mc_curve.png'), names=(), xlabel='Confidence', ylabel='Metric'):\n # Metric-confidence curve\n fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)\n\n if 0 < len(names) < 21: # display per-class legend if < 21 classes\n for i, y in enumerate(py):\n ax.plot(px, y, linewidth=1, label=f'{names[i]}') # plot(confidence, metric)\n else:\n ax.plot(px, py.T, linewidth=1, color='grey') # plot(confidence, metric)\n\n y = smooth(py.mean(0), 0.05)\n ax.plot(px, y, linewidth=3, color='blue', label=f'all classes {y.max():.2f} at {px[y.argmax()]:.3f}')\n ax.set_xlabel(xlabel)\n ax.set_ylabel(ylabel)\n ax.set_xlim(0, 1)\n ax.set_ylim(0, 1)\n ax.legend(bbox_to_anchor=(1.04, 1), loc=\"upper left\")\n ax.set_title(f'{ylabel}-Confidence Curve')\n fig.savefig(save_dir, dpi=250)\n plt.close(fig)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/plots.py", "content": "import contextlib\nimport math\nimport os\nfrom copy import copy\nfrom pathlib import Path\nfrom urllib.error import URLError\n\nimport cv2\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sn\nimport torch\nfrom PIL import Image, ImageDraw, ImageFont\n\nfrom asone.detectors.yolov9.yolov9.utils import TryExcept, threaded\nfrom asone.detectors.yolov9.yolov9.utils.general import (CONFIG_DIR, FONT, LOGGER, check_font, check_requirements, clip_boxes, increment_path,\n is_ascii, xywh2xyxy, xyxy2xywh)\nfrom asone.detectors.yolov9.yolov9.utils.metrics import fitness\nfrom asone.detectors.yolov9.yolov9.utils.segment.general import scale_image\n\n# Settings\nRANK = int(os.getenv('RANK', -1))\nmatplotlib.rc('font', **{'size': 11})\nmatplotlib.use('Agg') # for writing to files only\n\n\nclass Colors:\n # Ultralytics color palette https://ultralytics.com/\n def __init__(self):\n # hex = matplotlib.colors.TABLEAU_COLORS.values()\n hexs = ('FF3838', 'FF9D97', 'FF701F', 'FFB21D', 'CFD231', '48F90A', '92CC17', '3DDB86', '1A9334', '00D4BB',\n '2C99A8', '00C2FF', '344593', '6473FF', '0018EC', '8438FF', '520085', 'CB38FF', 'FF95C8', 'FF37C7')\n self.palette = [self.hex2rgb(f'#{c}') for c in hexs]\n self.n = len(self.palette)\n\n def __call__(self, i, bgr=False):\n c = self.palette[int(i) % self.n]\n return (c[2], c[1], c[0]) if bgr else c\n\n @staticmethod\n def hex2rgb(h): # rgb order (PIL)\n return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4))\n\n\ncolors = Colors() # create instance for 'from utils.plots import colors'\n\n\ndef check_pil_font(font=FONT, size=10):\n # Return a PIL TrueType Font, downloading to CONFIG_DIR if necessary\n font = Path(font)\n font = font if font.exists() else (CONFIG_DIR / font.name)\n try:\n return ImageFont.truetype(str(font) if font.exists() else font.name, size)\n except Exception: # download if missing\n try:\n check_font(font)\n return ImageFont.truetype(str(font), size)\n except TypeError:\n check_requirements('Pillow>=8.4.0') # known issue https://github.com/ultralytics/yolov5/issues/5374\n except URLError: # not online\n return ImageFont.load_default()\n\n\nclass Annotator:\n # YOLOv5 Annotator for train/val mosaics and jpgs and detect/hub inference annotations\n def __init__(self, im, line_width=None, font_size=None, font='Arial.ttf', pil=False, example='abc'):\n assert im.data.contiguous, 'Image not contiguous. Apply np.ascontiguousarray(im) to Annotator() input images.'\n non_ascii = not is_ascii(example) # non-latin labels, i.e. asian, arabic, cyrillic\n self.pil = pil or non_ascii\n if self.pil: # use PIL\n self.im = im if isinstance(im, Image.Image) else Image.fromarray(im)\n self.draw = ImageDraw.Draw(self.im)\n self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,\n size=font_size or max(round(sum(self.im.size) / 2 * 0.035), 12))\n else: # use cv2\n self.im = im\n self.lw = line_width or max(round(sum(im.shape) / 2 * 0.003), 2) # line width\n\n def box_label(self, box, label='', color=(128, 128, 128), txt_color=(255, 255, 255)):\n # Add one xyxy box to image with label\n if self.pil or not is_ascii(label):\n self.draw.rectangle(box, width=self.lw, outline=color) # box\n if label:\n w, h = self.font.getsize(label) # text width, height\n outside = box[1] - h >= 0 # label fits outside box\n self.draw.rectangle(\n (box[0], box[1] - h if outside else box[1], box[0] + w + 1,\n box[1] + 1 if outside else box[1] + h + 1),\n fill=color,\n )\n # self.draw.text((box[0], box[1]), label, fill=txt_color, font=self.font, anchor='ls') # for PIL>8.0\n self.draw.text((box[0], box[1] - h if outside else box[1]), label, fill=txt_color, font=self.font)\n else: # cv2\n p1, p2 = (int(box[0]), int(box[1])), (int(box[2]), int(box[3]))\n cv2.rectangle(self.im, p1, p2, color, thickness=self.lw, lineType=cv2.LINE_AA)\n if label:\n tf = max(self.lw - 1, 1) # font thickness\n w, h = cv2.getTextSize(label, 0, fontScale=self.lw / 3, thickness=tf)[0] # text width, height\n outside = p1[1] - h >= 3\n p2 = p1[0] + w, p1[1] - h - 3 if outside else p1[1] + h + 3\n cv2.rectangle(self.im, p1, p2, color, -1, cv2.LINE_AA) # filled\n cv2.putText(self.im,\n label, (p1[0], p1[1] - 2 if outside else p1[1] + h + 2),\n 0,\n self.lw / 3,\n txt_color,\n thickness=tf,\n lineType=cv2.LINE_AA)\n\n def masks(self, masks, colors, im_gpu=None, alpha=0.5):\n \"\"\"Plot masks at once.\n Args:\n masks (tensor): predicted masks on cuda, shape: [n, h, w]\n colors (List[List[Int]]): colors for predicted masks, [[r, g, b] * n]\n im_gpu (tensor): img is in cuda, shape: [3, h, w], range: [0, 1]\n alpha (float): mask transparency: 0.0 fully transparent, 1.0 opaque\n \"\"\"\n if self.pil:\n # convert to numpy first\n self.im = np.asarray(self.im).copy()\n if im_gpu is None:\n # Add multiple masks of shape(h,w,n) with colors list([r,g,b], [r,g,b], ...)\n if len(masks) == 0:\n return\n if isinstance(masks, torch.Tensor):\n masks = torch.as_tensor(masks, dtype=torch.uint8)\n masks = masks.permute(1, 2, 0).contiguous()\n masks = masks.cpu().numpy()\n # masks = np.ascontiguousarray(masks.transpose(1, 2, 0))\n masks = scale_image(masks.shape[:2], masks, self.im.shape)\n masks = np.asarray(masks, dtype=np.float32)\n colors = np.asarray(colors, dtype=np.float32) # shape(n,3)\n s = masks.sum(2, keepdims=True).clip(0, 1) # add all masks together\n masks = (masks @ colors).clip(0, 255) # (h,w,n) @ (n,3) = (h,w,3)\n self.im[:] = masks * alpha + self.im * (1 - s * alpha)\n else:\n if len(masks) == 0:\n self.im[:] = im_gpu.permute(1, 2, 0).contiguous().cpu().numpy() * 255\n colors = torch.tensor(colors, device=im_gpu.device, dtype=torch.float32) / 255.0\n colors = colors[:, None, None] # shape(n,1,1,3)\n masks = masks.unsqueeze(3) # shape(n,h,w,1)\n masks_color = masks * (colors * alpha) # shape(n,h,w,3)\n\n inv_alph_masks = (1 - masks * alpha).cumprod(0) # shape(n,h,w,1)\n mcs = (masks_color * inv_alph_masks).sum(0) * 2 # mask color summand shape(n,h,w,3)\n\n im_gpu = im_gpu.flip(dims=[0]) # flip channel\n im_gpu = im_gpu.permute(1, 2, 0).contiguous() # shape(h,w,3)\n im_gpu = im_gpu * inv_alph_masks[-1] + mcs\n im_mask = (im_gpu * 255).byte().cpu().numpy()\n self.im[:] = scale_image(im_gpu.shape, im_mask, self.im.shape)\n if self.pil:\n # convert im back to PIL and update draw\n self.fromarray(self.im)\n\n def rectangle(self, xy, fill=None, outline=None, width=1):\n # Add rectangle to image (PIL-only)\n self.draw.rectangle(xy, fill, outline, width)\n\n def text(self, xy, text, txt_color=(255, 255, 255), anchor='top'):\n # Add text to image (PIL-only)\n if anchor == 'bottom': # start y from font bottom\n w, h = self.font.getsize(text) # text width, height\n xy[1] += 1 - h\n self.draw.text(xy, text, fill=txt_color, font=self.font)\n\n def fromarray(self, im):\n # Update self.im from a numpy array\n self.im = im if isinstance(im, Image.Image) else Image.fromarray(im)\n self.draw = ImageDraw.Draw(self.im)\n\n def result(self):\n # Return annotated image as array\n return np.asarray(self.im)\n\n\ndef feature_visualization(x, module_type, stage, n=32, save_dir=Path('runs/detect/exp')):\n \"\"\"\n x: Features to be visualized\n module_type: Module type\n stage: Module stage within model\n n: Maximum number of feature maps to plot\n save_dir: Directory to save results\n \"\"\"\n if 'Detect' not in module_type:\n batch, channels, height, width = x.shape # batch, channels, height, width\n if height > 1 and width > 1:\n f = save_dir / f\"stage{stage}_{module_type.split('.')[-1]}_features.png\" # filename\n\n blocks = torch.chunk(x[0].cpu(), channels, dim=0) # select batch index 0, block by channels\n n = min(n, channels) # number of plots\n fig, ax = plt.subplots(math.ceil(n / 8), 8, tight_layout=True) # 8 rows x n/8 cols\n ax = ax.ravel()\n plt.subplots_adjust(wspace=0.05, hspace=0.05)\n for i in range(n):\n ax[i].imshow(blocks[i].squeeze()) # cmap='gray'\n ax[i].axis('off')\n\n LOGGER.info(f'Saving {f}... ({n}/{channels})')\n plt.savefig(f, dpi=300, bbox_inches='tight')\n plt.close()\n np.save(str(f.with_suffix('.npy')), x[0].cpu().numpy()) # npy save\n\n\ndef hist2d(x, y, n=100):\n # 2d histogram used in labels.png and evolve.png\n xedges, yedges = np.linspace(x.min(), x.max(), n), np.linspace(y.min(), y.max(), n)\n hist, xedges, yedges = np.histogram2d(x, y, (xedges, yedges))\n xidx = np.clip(np.digitize(x, xedges) - 1, 0, hist.shape[0] - 1)\n yidx = np.clip(np.digitize(y, yedges) - 1, 0, hist.shape[1] - 1)\n return np.log(hist[xidx, yidx])\n\n\ndef butter_lowpass_filtfilt(data, cutoff=1500, fs=50000, order=5):\n from scipy.signal import butter, filtfilt\n\n # https://stackoverflow.com/questions/28536191/how-to-filter-smooth-with-scipy-numpy\n def butter_lowpass(cutoff, fs, order):\n nyq = 0.5 * fs\n normal_cutoff = cutoff / nyq\n return butter(order, normal_cutoff, btype='low', analog=False)\n\n b, a = butter_lowpass(cutoff, fs, order=order)\n return filtfilt(b, a, data) # forward-backward filter\n\n\ndef output_to_target(output, max_det=300):\n # Convert model output to target format [batch_id, class_id, x, y, w, h, conf] for plotting\n targets = []\n for i, o in enumerate(output):\n box, conf, cls = o[:max_det, :6].cpu().split((4, 1, 1), 1)\n j = torch.full((conf.shape[0], 1), i)\n targets.append(torch.cat((j, cls, xyxy2xywh(box), conf), 1))\n return torch.cat(targets, 0).numpy()\n\n\n@threaded\ndef plot_images(images, targets, paths=None, fname='images.jpg', names=None):\n # Plot image grid with labels\n if isinstance(images, torch.Tensor):\n images = images.cpu().float().numpy()\n if isinstance(targets, torch.Tensor):\n targets = targets.cpu().numpy()\n\n max_size = 1920 # max image size\n max_subplots = 16 # max image subplots, i.e. 4x4\n bs, _, h, w = images.shape # batch size, _, height, width\n bs = min(bs, max_subplots) # limit plot images\n ns = np.ceil(bs ** 0.5) # number of subplots (square)\n if np.max(images[0]) <= 1:\n images *= 255 # de-normalise (optional)\n\n # Build Image\n mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init\n for i, im in enumerate(images):\n if i == max_subplots: # if last batch has fewer images than we expect\n break\n x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin\n im = im.transpose(1, 2, 0)\n mosaic[y:y + h, x:x + w, :] = im\n\n # Resize (optional)\n scale = max_size / ns / max(h, w)\n if scale < 1:\n h = math.ceil(scale * h)\n w = math.ceil(scale * w)\n mosaic = cv2.resize(mosaic, tuple(int(x * ns) for x in (w, h)))\n\n # Annotate\n fs = int((h + w) * ns * 0.01) # font size\n annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)\n for i in range(i + 1):\n x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin\n annotator.rectangle([x, y, x + w, y + h], None, (255, 255, 255), width=2) # borders\n if paths:\n annotator.text((x + 5, y + 5), text=Path(paths[i]).name[:40], txt_color=(220, 220, 220)) # filenames\n if len(targets) > 0:\n ti = targets[targets[:, 0] == i] # image targets\n boxes = xywh2xyxy(ti[:, 2:6]).T\n classes = ti[:, 1].astype('int')\n labels = ti.shape[1] == 6 # labels if no conf column\n conf = None if labels else ti[:, 6] # check for confidence presence (label vs pred)\n\n if boxes.shape[1]:\n if boxes.max() <= 1.01: # if normalized with tolerance 0.01\n boxes[[0, 2]] *= w # scale to pixels\n boxes[[1, 3]] *= h\n elif scale < 1: # absolute coords need scale if image scales\n boxes *= scale\n boxes[[0, 2]] += x\n boxes[[1, 3]] += y\n for j, box in enumerate(boxes.T.tolist()):\n cls = classes[j]\n color = colors(cls)\n cls = names[cls] if names else cls\n if labels or conf[j] > 0.25: # 0.25 conf thresh\n label = f'{cls}' if labels else f'{cls} {conf[j]:.1f}'\n annotator.box_label(box, label, color=color)\n annotator.im.save(fname) # save\n\n\ndef plot_lr_scheduler(optimizer, scheduler, epochs=300, save_dir=''):\n # Plot LR simulating training for full epochs\n optimizer, scheduler = copy(optimizer), copy(scheduler) # do not modify originals\n y = []\n for _ in range(epochs):\n scheduler.step()\n y.append(optimizer.param_groups[0]['lr'])\n plt.plot(y, '.-', label='LR')\n plt.xlabel('epoch')\n plt.ylabel('LR')\n plt.grid()\n plt.xlim(0, epochs)\n plt.ylim(0)\n plt.savefig(Path(save_dir) / 'LR.png', dpi=200)\n plt.close()\n\n\ndef plot_val_txt(): # from utils.plots import *; plot_val()\n # Plot val.txt histograms\n x = np.loadtxt('val.txt', dtype=np.float32)\n box = xyxy2xywh(x[:, :4])\n cx, cy = box[:, 0], box[:, 1]\n\n fig, ax = plt.subplots(1, 1, figsize=(6, 6), tight_layout=True)\n ax.hist2d(cx, cy, bins=600, cmax=10, cmin=0)\n ax.set_aspect('equal')\n plt.savefig('hist2d.png', dpi=300)\n\n fig, ax = plt.subplots(1, 2, figsize=(12, 6), tight_layout=True)\n ax[0].hist(cx, bins=600)\n ax[1].hist(cy, bins=600)\n plt.savefig('hist1d.png', dpi=200)\n\n\ndef plot_targets_txt(): # from utils.plots import *; plot_targets_txt()\n # Plot targets.txt histograms\n x = np.loadtxt('targets.txt', dtype=np.float32).T\n s = ['x targets', 'y targets', 'width targets', 'height targets']\n fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)\n ax = ax.ravel()\n for i in range(4):\n ax[i].hist(x[i], bins=100, label=f'{x[i].mean():.3g} +/- {x[i].std():.3g}')\n ax[i].legend()\n ax[i].set_title(s[i])\n plt.savefig('targets.jpg', dpi=200)\n\n\ndef plot_val_study(file='', dir='', x=None): # from utils.plots import *; plot_val_study()\n # Plot file=study.txt generated by val.py (or plot all study*.txt in dir)\n save_dir = Path(file).parent if file else Path(dir)\n plot2 = False # plot additional results\n if plot2:\n ax = plt.subplots(2, 4, figsize=(10, 6), tight_layout=True)[1].ravel()\n\n fig2, ax2 = plt.subplots(1, 1, figsize=(8, 4), tight_layout=True)\n # for f in [save_dir / f'study_coco_{x}.txt' for x in ['yolov5n6', 'yolov5s6', 'yolov5m6', 'yolov5l6', 'yolov5x6']]:\n for f in sorted(save_dir.glob('study*.txt')):\n y = np.loadtxt(f, dtype=np.float32, usecols=[0, 1, 2, 3, 7, 8, 9], ndmin=2).T\n x = np.arange(y.shape[1]) if x is None else np.array(x)\n if plot2:\n s = ['P', 'R', 'mAP@.5', 'mAP@.5:.95', 't_preprocess (ms/img)', 't_inference (ms/img)', 't_NMS (ms/img)']\n for i in range(7):\n ax[i].plot(x, y[i], '.-', linewidth=2, markersize=8)\n ax[i].set_title(s[i])\n\n j = y[3].argmax() + 1\n ax2.plot(y[5, 1:j],\n y[3, 1:j] * 1E2,\n '.-',\n linewidth=2,\n markersize=8,\n label=f.stem.replace('study_coco_', '').replace('yolo', 'YOLO'))\n\n ax2.plot(1E3 / np.array([209, 140, 97, 58, 35, 18]), [34.6, 40.5, 43.0, 47.5, 49.7, 51.5],\n 'k.-',\n linewidth=2,\n markersize=8,\n alpha=.25,\n label='EfficientDet')\n\n ax2.grid(alpha=0.2)\n ax2.set_yticks(np.arange(20, 60, 5))\n ax2.set_xlim(0, 57)\n ax2.set_ylim(25, 55)\n ax2.set_xlabel('GPU Speed (ms/img)')\n ax2.set_ylabel('COCO AP val')\n ax2.legend(loc='lower right')\n f = save_dir / 'study.png'\n print(f'Saving {f}...')\n plt.savefig(f, dpi=300)\n\n\n@TryExcept() # known issue https://github.com/ultralytics/yolov5/issues/5395\ndef plot_labels(labels, names=(), save_dir=Path('')):\n # plot dataset labels\n LOGGER.info(f\"Plotting labels to {save_dir / 'labels.jpg'}... \")\n c, b = labels[:, 0], labels[:, 1:].transpose() # classes, boxes\n nc = int(c.max() + 1) # number of classes\n x = pd.DataFrame(b.transpose(), columns=['x', 'y', 'width', 'height'])\n\n # seaborn correlogram\n sn.pairplot(x, corner=True, diag_kind='auto', kind='hist', diag_kws=dict(bins=50), plot_kws=dict(pmax=0.9))\n plt.savefig(save_dir / 'labels_correlogram.jpg', dpi=200)\n plt.close()\n\n # matplotlib labels\n matplotlib.use('svg') # faster\n ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)[1].ravel()\n y = ax[0].hist(c, bins=np.linspace(0, nc, nc + 1) - 0.5, rwidth=0.8)\n with contextlib.suppress(Exception): # color histogram bars by class\n [y[2].patches[i].set_color([x / 255 for x in colors(i)]) for i in range(nc)] # known issue #3195\n ax[0].set_ylabel('instances')\n if 0 < len(names) < 30:\n ax[0].set_xticks(range(len(names)))\n ax[0].set_xticklabels(list(names.values()), rotation=90, fontsize=10)\n else:\n ax[0].set_xlabel('classes')\n sn.histplot(x, x='x', y='y', ax=ax[2], bins=50, pmax=0.9)\n sn.histplot(x, x='width', y='height', ax=ax[3], bins=50, pmax=0.9)\n\n # rectangles\n labels[:, 1:3] = 0.5 # center\n labels[:, 1:] = xywh2xyxy(labels[:, 1:]) * 2000\n img = Image.fromarray(np.ones((2000, 2000, 3), dtype=np.uint8) * 255)\n for cls, *box in labels[:1000]:\n ImageDraw.Draw(img).rectangle(box, width=1, outline=colors(cls)) # plot\n ax[1].imshow(img)\n ax[1].axis('off')\n\n for a in [0, 1, 2, 3]:\n for s in ['top', 'right', 'left', 'bottom']:\n ax[a].spines[s].set_visible(False)\n\n plt.savefig(save_dir / 'labels.jpg', dpi=200)\n matplotlib.use('Agg')\n plt.close()\n\n\ndef imshow_cls(im, labels=None, pred=None, names=None, nmax=25, verbose=False, f=Path('images.jpg')):\n # Show classification image grid with labels (optional) and predictions (optional)\n from asone.detectors.yolov9.yolov9.utils.augmentations import denormalize\n\n names = names or [f'class{i}' for i in range(1000)]\n blocks = torch.chunk(denormalize(im.clone()).cpu().float(), len(im),\n dim=0) # select batch index 0, block by channels\n n = min(len(blocks), nmax) # number of plots\n m = min(8, round(n ** 0.5)) # 8 x 8 default\n fig, ax = plt.subplots(math.ceil(n / m), m) # 8 rows x n/8 cols\n ax = ax.ravel() if m > 1 else [ax]\n # plt.subplots_adjust(wspace=0.05, hspace=0.05)\n for i in range(n):\n ax[i].imshow(blocks[i].squeeze().permute((1, 2, 0)).numpy().clip(0.0, 1.0))\n ax[i].axis('off')\n if labels is not None:\n s = names[labels[i]] + (f'—{names[pred[i]]}' if pred is not None else '')\n ax[i].set_title(s, fontsize=8, verticalalignment='top')\n plt.savefig(f, dpi=300, bbox_inches='tight')\n plt.close()\n if verbose:\n LOGGER.info(f\"Saving {f}\")\n if labels is not None:\n LOGGER.info('True: ' + ' '.join(f'{names[i]:3s}' for i in labels[:nmax]))\n if pred is not None:\n LOGGER.info('Predicted:' + ' '.join(f'{names[i]:3s}' for i in pred[:nmax]))\n return f\n\n\ndef plot_evolve(evolve_csv='path/to/evolve.csv'): # from utils.plots import *; plot_evolve()\n # Plot evolve.csv hyp evolution results\n evolve_csv = Path(evolve_csv)\n data = pd.read_csv(evolve_csv)\n keys = [x.strip() for x in data.columns]\n x = data.values\n f = fitness(x)\n j = np.argmax(f) # max fitness index\n plt.figure(figsize=(10, 12), tight_layout=True)\n matplotlib.rc('font', **{'size': 8})\n print(f'Best results from row {j} of {evolve_csv}:')\n for i, k in enumerate(keys[7:]):\n v = x[:, 7 + i]\n mu = v[j] # best single result\n plt.subplot(6, 5, i + 1)\n plt.scatter(v, f, c=hist2d(v, f, 20), cmap='viridis', alpha=.8, edgecolors='none')\n plt.plot(mu, f.max(), 'k+', markersize=15)\n plt.title(f'{k} = {mu:.3g}', fontdict={'size': 9}) # limit to 40 characters\n if i % 5 != 0:\n plt.yticks([])\n print(f'{k:>15}: {mu:.3g}')\n f = evolve_csv.with_suffix('.png') # filename\n plt.savefig(f, dpi=200)\n plt.close()\n print(f'Saved {f}')\n\n\ndef plot_results(file='path/to/results.csv', dir=''):\n # Plot training results.csv. Usage: from utils.plots import *; plot_results('path/to/results.csv')\n save_dir = Path(file).parent if file else Path(dir)\n fig, ax = plt.subplots(2, 5, figsize=(12, 6), tight_layout=True)\n ax = ax.ravel()\n files = list(save_dir.glob('results*.csv'))\n assert len(files), f'No results.csv files found in {save_dir.resolve()}, nothing to plot.'\n for f in files:\n try:\n data = pd.read_csv(f)\n s = [x.strip() for x in data.columns]\n x = data.values[:, 0]\n for i, j in enumerate([1, 2, 3, 4, 5, 8, 9, 10, 6, 7]):\n y = data.values[:, j].astype('float')\n # y[y == 0] = np.nan # don't show zero values\n ax[i].plot(x, y, marker='.', label=f.stem, linewidth=2, markersize=8)\n ax[i].set_title(s[j], fontsize=12)\n # if j in [8, 9, 10]: # share train and val loss y axes\n # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5])\n except Exception as e:\n LOGGER.info(f'Warning: Plotting error for {f}: {e}')\n ax[1].legend()\n fig.savefig(save_dir / 'results.png', dpi=200)\n plt.close()\n\n\ndef profile_idetection(start=0, stop=0, labels=(), save_dir=''):\n # Plot iDetection '*.txt' per-image logs. from utils.plots import *; profile_idetection()\n ax = plt.subplots(2, 4, figsize=(12, 6), tight_layout=True)[1].ravel()\n s = ['Images', 'Free Storage (GB)', 'RAM Usage (GB)', 'Battery', 'dt_raw (ms)', 'dt_smooth (ms)', 'real-world FPS']\n files = list(Path(save_dir).glob('frames*.txt'))\n for fi, f in enumerate(files):\n try:\n results = np.loadtxt(f, ndmin=2).T[:, 90:-30] # clip first and last rows\n n = results.shape[1] # number of rows\n x = np.arange(start, min(stop, n) if stop else n)\n results = results[:, x]\n t = (results[0] - results[0].min()) # set t0=0s\n results[0] = x\n for i, a in enumerate(ax):\n if i < len(results):\n label = labels[fi] if len(labels) else f.stem.replace('frames_', '')\n a.plot(t, results[i], marker='.', label=label, linewidth=1, markersize=5)\n a.set_title(s[i])\n a.set_xlabel('time (s)')\n # if fi == len(files) - 1:\n # a.set_ylim(bottom=0)\n for side in ['top', 'right']:\n a.spines[side].set_visible(False)\n else:\n a.remove()\n except Exception as e:\n print(f'Warning: Plotting error for {f}; {e}')\n ax[1].legend()\n plt.savefig(Path(save_dir) / 'idetection_profile.png', dpi=200)\n\n\ndef save_one_box(xyxy, im, file=Path('im.jpg'), gain=1.02, pad=10, square=False, BGR=False, save=True):\n # Save image crop as {file} with crop size multiple {gain} and {pad} pixels. Save and/or return crop\n xyxy = torch.tensor(xyxy).view(-1, 4)\n b = xyxy2xywh(xyxy) # boxes\n if square:\n b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # attempt rectangle to square\n b[:, 2:] = b[:, 2:] * gain + pad # box wh * gain + pad\n xyxy = xywh2xyxy(b).long()\n clip_boxes(xyxy, im.shape)\n crop = im[int(xyxy[0, 1]):int(xyxy[0, 3]), int(xyxy[0, 0]):int(xyxy[0, 2]), ::(1 if BGR else -1)]\n if save:\n file.parent.mkdir(parents=True, exist_ok=True) # make directory\n f = str(increment_path(file).with_suffix('.jpg'))\n # cv2.imwrite(f, crop) # save BGR, https://github.com/ultralytics/yolov5/issues/7007 chroma subsampling issue\n Image.fromarray(crop[..., ::-1]).save(f, quality=95, subsampling=0) # save RGB\n return crop\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/__init__.py", "content": "# init" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/augmentations.py", "content": "import math\nimport random\n\nimport cv2\nimport numpy as np\n\nfrom ..augmentations import box_candidates\nfrom ..general import resample_segments, segment2box\n\n\ndef mixup(im, labels, segments, im2, labels2, segments2):\n # Applies MixUp augmentation https://arxiv.org/pdf/1710.09412.pdf\n r = np.random.beta(32.0, 32.0) # mixup ratio, alpha=beta=32.0\n im = (im * r + im2 * (1 - r)).astype(np.uint8)\n labels = np.concatenate((labels, labels2), 0)\n segments = np.concatenate((segments, segments2), 0)\n return im, labels, segments\n\n\ndef random_perspective(im,\n targets=(),\n segments=(),\n degrees=10,\n translate=.1,\n scale=.1,\n shear=10,\n perspective=0.0,\n border=(0, 0)):\n # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(.1, .1), scale=(.9, 1.1), shear=(-10, 10))\n # targets = [cls, xyxy]\n\n height = im.shape[0] + border[0] * 2 # shape(h,w,c)\n width = im.shape[1] + border[1] * 2\n\n # Center\n C = np.eye(3)\n C[0, 2] = -im.shape[1] / 2 # x translation (pixels)\n C[1, 2] = -im.shape[0] / 2 # y translation (pixels)\n\n # Perspective\n P = np.eye(3)\n P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y)\n P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x)\n\n # Rotation and Scale\n R = np.eye(3)\n a = random.uniform(-degrees, degrees)\n # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations\n s = random.uniform(1 - scale, 1 + scale)\n # s = 2 ** random.uniform(-scale, scale)\n R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s)\n\n # Shear\n S = np.eye(3)\n S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg)\n S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg)\n\n # Translation\n T = np.eye(3)\n T[0, 2] = (random.uniform(0.5 - translate, 0.5 + translate) * width) # x translation (pixels)\n T[1, 2] = (random.uniform(0.5 - translate, 0.5 + translate) * height) # y translation (pixels)\n\n # Combined rotation matrix\n M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT\n if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed\n if perspective:\n im = cv2.warpPerspective(im, M, dsize=(width, height), borderValue=(114, 114, 114))\n else: # affine\n im = cv2.warpAffine(im, M[:2], dsize=(width, height), borderValue=(114, 114, 114))\n\n # Visualize\n # import matplotlib.pyplot as plt\n # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel()\n # ax[0].imshow(im[:, :, ::-1]) # base\n # ax[1].imshow(im2[:, :, ::-1]) # warped\n\n # Transform label coordinates\n n = len(targets)\n new_segments = []\n if n:\n new = np.zeros((n, 4))\n segments = resample_segments(segments) # upsample\n for i, segment in enumerate(segments):\n xy = np.ones((len(segment), 3))\n xy[:, :2] = segment\n xy = xy @ M.T # transform\n xy = (xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2]) # perspective rescale or affine\n\n # clip\n new[i] = segment2box(xy, width, height)\n new_segments.append(xy)\n\n # filter candidates\n i = box_candidates(box1=targets[:, 1:5].T * s, box2=new.T, area_thr=0.01)\n targets = targets[i]\n targets[:, 1:5] = new[i]\n new_segments = np.array(new_segments)[i]\n\n return im, targets, new_segments\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/dataloaders.py", "content": "import os\nimport random\n\nimport cv2\nimport numpy as np\nimport torch\nfrom torch.utils.data import DataLoader, distributed\n\nfrom ..augmentations import augment_hsv, copy_paste, letterbox\nfrom ..dataloaders import InfiniteDataLoader, LoadImagesAndLabels, seed_worker\nfrom ..general import LOGGER, xyn2xy, xywhn2xyxy, xyxy2xywhn\nfrom ..torch_utils import torch_distributed_zero_first\nfrom .augmentations import mixup, random_perspective\n\nRANK = int(os.getenv('RANK', -1))\n\n\ndef create_dataloader(path,\n imgsz,\n batch_size,\n stride,\n single_cls=False,\n hyp=None,\n augment=False,\n cache=False,\n pad=0.0,\n rect=False,\n rank=-1,\n workers=8,\n image_weights=False,\n close_mosaic=False,\n quad=False,\n prefix='',\n shuffle=False,\n mask_downsample_ratio=1,\n overlap_mask=False):\n if rect and shuffle:\n LOGGER.warning('WARNING ⚠️ --rect is incompatible with DataLoader shuffle, setting shuffle=False')\n shuffle = False\n with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP\n dataset = LoadImagesAndLabelsAndMasks(\n path,\n imgsz,\n batch_size,\n augment=augment, # augmentation\n hyp=hyp, # hyperparameters\n rect=rect, # rectangular batches\n cache_images=cache,\n single_cls=single_cls,\n stride=int(stride),\n pad=pad,\n image_weights=image_weights,\n prefix=prefix,\n downsample_ratio=mask_downsample_ratio,\n overlap=overlap_mask)\n\n batch_size = min(batch_size, len(dataset))\n nd = torch.cuda.device_count() # number of CUDA devices\n nw = min([os.cpu_count() // max(nd, 1), batch_size if batch_size > 1 else 0, workers]) # number of workers\n sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle)\n #loader = DataLoader if image_weights else InfiniteDataLoader # only DataLoader allows for attribute updates\n loader = DataLoader if image_weights or close_mosaic else InfiniteDataLoader\n generator = torch.Generator()\n generator.manual_seed(6148914691236517205 + RANK)\n return loader(\n dataset,\n batch_size=batch_size,\n shuffle=shuffle and sampler is None,\n num_workers=nw,\n sampler=sampler,\n pin_memory=True,\n collate_fn=LoadImagesAndLabelsAndMasks.collate_fn4 if quad else LoadImagesAndLabelsAndMasks.collate_fn,\n worker_init_fn=seed_worker,\n generator=generator,\n ), dataset\n\n\nclass LoadImagesAndLabelsAndMasks(LoadImagesAndLabels): # for training/testing\n\n def __init__(\n self,\n path,\n img_size=640,\n batch_size=16,\n augment=False,\n hyp=None,\n rect=False,\n image_weights=False,\n cache_images=False,\n single_cls=False,\n stride=32,\n pad=0,\n min_items=0,\n prefix=\"\",\n downsample_ratio=1,\n overlap=False,\n ):\n super().__init__(path, img_size, batch_size, augment, hyp, rect, image_weights, cache_images, single_cls,\n stride, pad, min_items, prefix)\n self.downsample_ratio = downsample_ratio\n self.overlap = overlap\n\n def __getitem__(self, index):\n index = self.indices[index] # linear, shuffled, or image_weights\n\n hyp = self.hyp\n mosaic = self.mosaic and random.random() < hyp['mosaic']\n masks = []\n if mosaic:\n # Load mosaic\n img, labels, segments = self.load_mosaic(index)\n shapes = None\n\n # MixUp augmentation\n if random.random() < hyp[\"mixup\"]:\n img, labels, segments = mixup(img, labels, segments, *self.load_mosaic(random.randint(0, self.n - 1)))\n\n else:\n # Load image\n img, (h0, w0), (h, w) = self.load_image(index)\n\n # Letterbox\n shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape\n img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment)\n shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling\n\n labels = self.labels[index].copy()\n # [array, array, ....], array.shape=(num_points, 2), xyxyxyxy\n segments = self.segments[index].copy()\n if len(segments):\n for i_s in range(len(segments)):\n segments[i_s] = xyn2xy(\n segments[i_s],\n ratio[0] * w,\n ratio[1] * h,\n padw=pad[0],\n padh=pad[1],\n )\n if labels.size: # normalized xywh to pixel xyxy format\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], ratio[0] * w, ratio[1] * h, padw=pad[0], padh=pad[1])\n\n if self.augment:\n img, labels, segments = random_perspective(img,\n labels,\n segments=segments,\n degrees=hyp[\"degrees\"],\n translate=hyp[\"translate\"],\n scale=hyp[\"scale\"],\n shear=hyp[\"shear\"],\n perspective=hyp[\"perspective\"])\n\n nl = len(labels) # number of labels\n if nl:\n labels[:, 1:5] = xyxy2xywhn(labels[:, 1:5], w=img.shape[1], h=img.shape[0], clip=True, eps=1e-3)\n if self.overlap:\n masks, sorted_idx = polygons2masks_overlap(img.shape[:2],\n segments,\n downsample_ratio=self.downsample_ratio)\n masks = masks[None] # (640, 640) -> (1, 640, 640)\n labels = labels[sorted_idx]\n else:\n masks = polygons2masks(img.shape[:2], segments, color=1, downsample_ratio=self.downsample_ratio)\n\n masks = (torch.from_numpy(masks) if len(masks) else torch.zeros(1 if self.overlap else nl, img.shape[0] //\n self.downsample_ratio, img.shape[1] //\n self.downsample_ratio))\n # TODO: albumentations support\n if self.augment:\n # Albumentations\n # there are some augmentation that won't change boxes and masks,\n # so just be it for now.\n img, labels = self.albumentations(img, labels)\n nl = len(labels) # update after albumentations\n\n # HSV color-space\n augment_hsv(img, hgain=hyp[\"hsv_h\"], sgain=hyp[\"hsv_s\"], vgain=hyp[\"hsv_v\"])\n\n # Flip up-down\n if random.random() < hyp[\"flipud\"]:\n img = np.flipud(img)\n if nl:\n labels[:, 2] = 1 - labels[:, 2]\n masks = torch.flip(masks, dims=[1])\n\n # Flip left-right\n if random.random() < hyp[\"fliplr\"]:\n img = np.fliplr(img)\n if nl:\n labels[:, 1] = 1 - labels[:, 1]\n masks = torch.flip(masks, dims=[2])\n\n # Cutouts # labels = cutout(img, labels, p=0.5)\n\n labels_out = torch.zeros((nl, 6))\n if nl:\n labels_out[:, 1:] = torch.from_numpy(labels)\n\n # Convert\n img = img.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB\n img = np.ascontiguousarray(img)\n\n return (torch.from_numpy(img), labels_out, self.im_files[index], shapes, masks)\n\n def load_mosaic(self, index):\n # YOLOv5 4-mosaic loader. Loads 1 image + 3 random images into a 4-image mosaic\n labels4, segments4 = [], []\n s = self.img_size\n yc, xc = (int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border) # mosaic center x, y\n\n # 3 additional image indices\n indices = [index] + random.choices(self.indices, k=3) # 3 additional image indices\n for i, index in enumerate(indices):\n # Load image\n img, _, (h, w) = self.load_image(index)\n\n # place img in img4\n if i == 0: # top left\n img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\n x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image)\n x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image)\n elif i == 1: # top right\n x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n elif i == 2: # bottom left\n x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h)\n elif i == 3: # bottom right\n x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n\n img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]\n padw = x1a - x1b\n padh = y1a - y1b\n\n labels, segments = self.labels[index].copy(), self.segments[index].copy()\n\n if labels.size:\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh) # normalized xywh to pixel xyxy format\n segments = [xyn2xy(x, w, h, padw, padh) for x in segments]\n labels4.append(labels)\n segments4.extend(segments)\n\n # Concat/clip labels\n labels4 = np.concatenate(labels4, 0)\n for x in (labels4[:, 1:], *segments4):\n np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective()\n # img4, labels4 = replicate(img4, labels4) # replicate\n\n # Augment\n img4, labels4, segments4 = copy_paste(img4, labels4, segments4, p=self.hyp[\"copy_paste\"])\n img4, labels4, segments4 = random_perspective(img4,\n labels4,\n segments4,\n degrees=self.hyp[\"degrees\"],\n translate=self.hyp[\"translate\"],\n scale=self.hyp[\"scale\"],\n shear=self.hyp[\"shear\"],\n perspective=self.hyp[\"perspective\"],\n border=self.mosaic_border) # border to remove\n return img4, labels4, segments4\n\n @staticmethod\n def collate_fn(batch):\n img, label, path, shapes, masks = zip(*batch) # transposed\n batched_masks = torch.cat(masks, 0)\n for i, l in enumerate(label):\n l[:, 0] = i # add target image index for build_targets()\n return torch.stack(img, 0), torch.cat(label, 0), path, shapes, batched_masks\n\n\ndef polygon2mask(img_size, polygons, color=1, downsample_ratio=1):\n \"\"\"\n Args:\n img_size (tuple): The image size.\n polygons (np.ndarray): [N, M], N is the number of polygons,\n M is the number of points(Be divided by 2).\n \"\"\"\n mask = np.zeros(img_size, dtype=np.uint8)\n polygons = np.asarray(polygons)\n polygons = polygons.astype(np.int32)\n shape = polygons.shape\n polygons = polygons.reshape(shape[0], -1, 2)\n cv2.fillPoly(mask, polygons, color=color)\n nh, nw = (img_size[0] // downsample_ratio, img_size[1] // downsample_ratio)\n # NOTE: fillPoly firstly then resize is trying the keep the same way\n # of loss calculation when mask-ratio=1.\n mask = cv2.resize(mask, (nw, nh))\n return mask\n\n\ndef polygons2masks(img_size, polygons, color, downsample_ratio=1):\n \"\"\"\n Args:\n img_size (tuple): The image size.\n polygons (list[np.ndarray]): each polygon is [N, M],\n N is the number of polygons,\n M is the number of points(Be divided by 2).\n \"\"\"\n masks = []\n for si in range(len(polygons)):\n mask = polygon2mask(img_size, [polygons[si].reshape(-1)], color, downsample_ratio)\n masks.append(mask)\n return np.array(masks)\n\n\ndef polygons2masks_overlap(img_size, segments, downsample_ratio=1):\n \"\"\"Return a (640, 640) overlap mask.\"\"\"\n masks = np.zeros((img_size[0] // downsample_ratio, img_size[1] // downsample_ratio),\n dtype=np.int32 if len(segments) > 255 else np.uint8)\n areas = []\n ms = []\n for si in range(len(segments)):\n mask = polygon2mask(\n img_size,\n [segments[si].reshape(-1)],\n downsample_ratio=downsample_ratio,\n color=1,\n )\n ms.append(mask)\n areas.append(mask.sum())\n areas = np.asarray(areas)\n index = np.argsort(-areas)\n ms = np.array(ms)[index]\n for i in range(len(segments)):\n mask = ms[i] * (i + 1)\n masks = masks + mask\n masks = np.clip(masks, a_min=0, a_max=i + 1)\n return masks, index\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/general.py", "content": "import cv2\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\n\n\ndef crop_mask(masks, boxes):\n \"\"\"\n \"Crop\" predicted masks by zeroing out everything not in the predicted bbox.\n Vectorized by Chong (thanks Chong).\n\n Args:\n - masks should be a size [h, w, n] tensor of masks\n - boxes should be a size [n, 4] tensor of bbox coords in relative point form\n \"\"\"\n\n n, h, w = masks.shape\n x1, y1, x2, y2 = torch.chunk(boxes[:, :, None], 4, 1) # x1 shape(1,1,n)\n r = torch.arange(w, device=masks.device, dtype=x1.dtype)[None, None, :] # rows shape(1,w,1)\n c = torch.arange(h, device=masks.device, dtype=x1.dtype)[None, :, None] # cols shape(h,1,1)\n\n return masks * ((r >= x1) * (r < x2) * (c >= y1) * (c < y2))\n\n\ndef process_mask_upsample(protos, masks_in, bboxes, shape):\n \"\"\"\n Crop after upsample.\n proto_out: [mask_dim, mask_h, mask_w]\n out_masks: [n, mask_dim], n is number of masks after nms\n bboxes: [n, 4], n is number of masks after nms\n shape:input_image_size, (h, w)\n\n return: h, w, n\n \"\"\"\n\n c, mh, mw = protos.shape # CHW\n masks = (masks_in @ protos.float().view(c, -1)).sigmoid().view(-1, mh, mw)\n masks = F.interpolate(masks[None], shape, mode='bilinear', align_corners=False)[0] # CHW\n masks = crop_mask(masks, bboxes) # CHW\n return masks.gt_(0.5)\n\n\ndef process_mask(protos, masks_in, bboxes, shape, upsample=False):\n \"\"\"\n Crop before upsample.\n proto_out: [mask_dim, mask_h, mask_w]\n out_masks: [n, mask_dim], n is number of masks after nms\n bboxes: [n, 4], n is number of masks after nms\n shape:input_image_size, (h, w)\n\n return: h, w, n\n \"\"\"\n\n c, mh, mw = protos.shape # CHW\n ih, iw = shape\n masks = (masks_in @ protos.float().view(c, -1)).sigmoid().view(-1, mh, mw) # CHW\n\n downsampled_bboxes = bboxes.clone()\n downsampled_bboxes[:, 0] *= mw / iw\n downsampled_bboxes[:, 2] *= mw / iw\n downsampled_bboxes[:, 3] *= mh / ih\n downsampled_bboxes[:, 1] *= mh / ih\n\n masks = crop_mask(masks, downsampled_bboxes) # CHW\n if upsample:\n masks = F.interpolate(masks[None], shape, mode='bilinear', align_corners=False)[0] # CHW\n return masks.gt_(0.5)\n\n\ndef scale_image(im1_shape, masks, im0_shape, ratio_pad=None):\n \"\"\"\n img1_shape: model input shape, [h, w]\n img0_shape: origin pic shape, [h, w, 3]\n masks: [h, w, num]\n \"\"\"\n # Rescale coordinates (xyxy) from im1_shape to im0_shape\n if ratio_pad is None: # calculate from im0_shape\n gain = min(im1_shape[0] / im0_shape[0], im1_shape[1] / im0_shape[1]) # gain = old / new\n pad = (im1_shape[1] - im0_shape[1] * gain) / 2, (im1_shape[0] - im0_shape[0] * gain) / 2 # wh padding\n else:\n pad = ratio_pad[1]\n top, left = int(pad[1]), int(pad[0]) # y, x\n bottom, right = int(im1_shape[0] - pad[1]), int(im1_shape[1] - pad[0])\n\n if len(masks.shape) < 2:\n raise ValueError(f'\"len of masks shape\" should be 2 or 3, but got {len(masks.shape)}')\n masks = masks[top:bottom, left:right]\n # masks = masks.permute(2, 0, 1).contiguous()\n # masks = F.interpolate(masks[None], im0_shape[:2], mode='bilinear', align_corners=False)[0]\n # masks = masks.permute(1, 2, 0).contiguous()\n masks = cv2.resize(masks, (im0_shape[1], im0_shape[0]))\n\n if len(masks.shape) == 2:\n masks = masks[:, :, None]\n return masks\n\n\ndef mask_iou(mask1, mask2, eps=1e-7):\n \"\"\"\n mask1: [N, n] m1 means number of predicted objects\n mask2: [M, n] m2 means number of gt objects\n Note: n means image_w x image_h\n\n return: masks iou, [N, M]\n \"\"\"\n intersection = torch.matmul(mask1, mask2.t()).clamp(0)\n union = (mask1.sum(1)[:, None] + mask2.sum(1)[None]) - intersection # (area1 + area2) - intersection\n return intersection / (union + eps)\n\n\ndef masks_iou(mask1, mask2, eps=1e-7):\n \"\"\"\n mask1: [N, n] m1 means number of predicted objects\n mask2: [N, n] m2 means number of gt objects\n Note: n means image_w x image_h\n\n return: masks iou, (N, )\n \"\"\"\n intersection = (mask1 * mask2).sum(1).clamp(0) # (N, )\n union = (mask1.sum(1) + mask2.sum(1))[None] - intersection # (area1 + area2) - intersection\n return intersection / (union + eps)\n\n\ndef masks2segments(masks, strategy='largest'):\n # Convert masks(n,160,160) into segments(n,xy)\n segments = []\n for x in masks.int().cpu().numpy().astype('uint8'):\n c = cv2.findContours(x, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0]\n if c:\n if strategy == 'concat': # concatenate all segments\n c = np.concatenate([x.reshape(-1, 2) for x in c])\n elif strategy == 'largest': # select largest segment\n c = np.array(c[np.array([len(x) for x in c]).argmax()]).reshape(-1, 2)\n else:\n c = np.zeros((0, 2)) # no segments found\n segments.append(c.astype('float32'))\n return segments\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/loss.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom ..general import xywh2xyxy\nfrom ..loss import FocalLoss, smooth_BCE\nfrom ..metrics import bbox_iou\nfrom ..torch_utils import de_parallel\nfrom .general import crop_mask\n\n\nclass ComputeLoss:\n # Compute losses\n def __init__(self, model, autobalance=False, overlap=False):\n self.sort_obj_iou = False\n self.overlap = overlap\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n self.device = device\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device))\n BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device))\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h['fl_gamma'] # focal loss gamma\n if g > 0:\n BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.ssi = list(m.stride).index(16) if autobalance else 0 # stride 16 index\n self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, 1.0, h, autobalance\n self.na = m.na # number of anchors\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.nm = m.nm # number of masks\n self.anchors = m.anchors\n self.device = device\n\n def __call__(self, preds, targets, masks): # predictions, targets, model\n p, proto = preds\n bs, nm, mask_h, mask_w = proto.shape # batch size, number of masks, mask height, mask width\n lcls = torch.zeros(1, device=self.device)\n lbox = torch.zeros(1, device=self.device)\n lobj = torch.zeros(1, device=self.device)\n lseg = torch.zeros(1, device=self.device)\n tcls, tbox, indices, anchors, tidxs, xywhn = self.build_targets(p, targets) # targets\n\n # Losses\n for i, pi in enumerate(p): # layer index, layer predictions\n b, a, gj, gi = indices[i] # image, anchor, gridy, gridx\n tobj = torch.zeros(pi.shape[:4], dtype=pi.dtype, device=self.device) # target obj\n\n n = b.shape[0] # number of targets\n if n:\n pxy, pwh, _, pcls, pmask = pi[b, a, gj, gi].split((2, 2, 1, self.nc, nm), 1) # subset of predictions\n\n # Box regression\n pxy = pxy.sigmoid() * 2 - 0.5\n pwh = (pwh.sigmoid() * 2) ** 2 * anchors[i]\n pbox = torch.cat((pxy, pwh), 1) # predicted box\n iou = bbox_iou(pbox, tbox[i], CIoU=True).squeeze() # iou(prediction, target)\n lbox += (1.0 - iou).mean() # iou loss\n\n # Objectness\n iou = iou.detach().clamp(0).type(tobj.dtype)\n if self.sort_obj_iou:\n j = iou.argsort()\n b, a, gj, gi, iou = b[j], a[j], gj[j], gi[j], iou[j]\n if self.gr < 1:\n iou = (1.0 - self.gr) + self.gr * iou\n tobj[b, a, gj, gi] = iou # iou ratio\n\n # Classification\n if self.nc > 1: # cls loss (only if multiple classes)\n t = torch.full_like(pcls, self.cn, device=self.device) # targets\n t[range(n), tcls[i]] = self.cp\n lcls += self.BCEcls(pcls, t) # BCE\n\n # Mask regression\n if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample\n masks = F.interpolate(masks[None], (mask_h, mask_w), mode=\"nearest\")[0]\n marea = xywhn[i][:, 2:].prod(1) # mask width, height normalized\n mxyxy = xywh2xyxy(xywhn[i] * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=self.device))\n for bi in b.unique():\n j = b == bi # matching index\n if self.overlap:\n mask_gti = torch.where(masks[bi][None] == tidxs[i][j].view(-1, 1, 1), 1.0, 0.0)\n else:\n mask_gti = masks[tidxs[i]][j]\n lseg += self.single_mask_loss(mask_gti, pmask[j], proto[bi], mxyxy[j], marea[j])\n\n obji = self.BCEobj(pi[..., 4], tobj)\n lobj += obji * self.balance[i] # obj loss\n if self.autobalance:\n self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item()\n\n if self.autobalance:\n self.balance = [x / self.balance[self.ssi] for x in self.balance]\n lbox *= self.hyp[\"box\"]\n lobj *= self.hyp[\"obj\"]\n lcls *= self.hyp[\"cls\"]\n lseg *= self.hyp[\"box\"] / bs\n\n loss = lbox + lobj + lcls + lseg\n return loss * bs, torch.cat((lbox, lseg, lobj, lcls)).detach()\n\n def single_mask_loss(self, gt_mask, pred, proto, xyxy, area):\n # Mask loss for one image\n pred_mask = (pred @ proto.view(self.nm, -1)).view(-1, *proto.shape[1:]) # (n,32) @ (32,80,80) -> (n,80,80)\n loss = F.binary_cross_entropy_with_logits(pred_mask, gt_mask, reduction=\"none\")\n return (crop_mask(loss, xyxy).mean(dim=(1, 2)) / area).mean()\n\n def build_targets(self, p, targets):\n # Build targets for compute_loss(), input targets(image,class,x,y,w,h)\n na, nt = self.na, targets.shape[0] # number of anchors, targets\n tcls, tbox, indices, anch, tidxs, xywhn = [], [], [], [], [], []\n gain = torch.ones(8, device=self.device) # normalized to gridspace gain\n ai = torch.arange(na, device=self.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt)\n if self.overlap:\n batch = p[0].shape[0]\n ti = []\n for i in range(batch):\n num = (targets[:, 0] == i).sum() # find number of targets of each image\n ti.append(torch.arange(num, device=self.device).float().view(1, num).repeat(na, 1) + 1) # (na, num)\n ti = torch.cat(ti, 1) # (na, nt)\n else:\n ti = torch.arange(nt, device=self.device).float().view(1, nt).repeat(na, 1)\n targets = torch.cat((targets.repeat(na, 1, 1), ai[..., None], ti[..., None]), 2) # append anchor indices\n\n g = 0.5 # bias\n off = torch.tensor(\n [\n [0, 0],\n [1, 0],\n [0, 1],\n [-1, 0],\n [0, -1], # j,k,l,m\n # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm\n ],\n device=self.device).float() * g # offsets\n\n for i in range(self.nl):\n anchors, shape = self.anchors[i], p[i].shape\n gain[2:6] = torch.tensor(shape)[[3, 2, 3, 2]] # xyxy gain\n\n # Match targets to anchors\n t = targets * gain # shape(3,n,7)\n if nt:\n # Matches\n r = t[..., 4:6] / anchors[:, None] # wh ratio\n j = torch.max(r, 1 / r).max(2)[0] < self.hyp['anchor_t'] # compare\n # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2))\n t = t[j] # filter\n\n # Offsets\n gxy = t[:, 2:4] # grid xy\n gxi = gain[[2, 3]] - gxy # inverse\n j, k = ((gxy % 1 < g) & (gxy > 1)).T\n l, m = ((gxi % 1 < g) & (gxi > 1)).T\n j = torch.stack((torch.ones_like(j), j, k, l, m))\n t = t.repeat((5, 1, 1))[j]\n offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j]\n else:\n t = targets[0]\n offsets = 0\n\n # Define\n bc, gxy, gwh, at = t.chunk(4, 1) # (image, class), grid xy, grid wh, anchors\n (a, tidx), (b, c) = at.long().T, bc.long().T # anchors, image, class\n gij = (gxy - offsets).long()\n gi, gj = gij.T # grid indices\n\n # Append\n indices.append((b, a, gj.clamp_(0, shape[2] - 1), gi.clamp_(0, shape[3] - 1))) # image, anchor, grid\n tbox.append(torch.cat((gxy - gij, gwh), 1)) # box\n anch.append(anchors[a]) # anchors\n tcls.append(c) # class\n tidxs.append(tidx)\n xywhn.append(torch.cat((gxy, gwh), 1) / gain[2:6]) # xywh normalized\n\n return tcls, tbox, indices, anch, tidxs, xywhn\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/loss_tal.py", "content": "import os\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torchvision.ops import sigmoid_focal_loss\n\nfrom utils.general import xywh2xyxy, xyxy2xywh\nfrom utils.metrics import bbox_iou\nfrom utils.segment.tal.anchor_generator import dist2bbox, make_anchors, bbox2dist\nfrom utils.segment.tal.assigner import TaskAlignedAssigner\nfrom utils.torch_utils import de_parallel\nfrom utils.segment.general import crop_mask\n\n\ndef smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441\n # return positive, negative label smoothing BCE targets\n return 1.0 - 0.5 * eps, 0.5 * eps\n\n\nclass VarifocalLoss(nn.Module):\n # Varifocal loss by Zhang et al. https://arxiv.org/abs/2008.13367\n def __init__(self):\n super().__init__()\n\n def forward(self, pred_score, gt_score, label, alpha=0.75, gamma=2.0):\n weight = alpha * pred_score.sigmoid().pow(gamma) * (1 - label) + gt_score * label\n with torch.cuda.amp.autocast(enabled=False):\n loss = (F.binary_cross_entropy_with_logits(pred_score.float(), gt_score.float(),\n reduction=\"none\") * weight).sum()\n return loss\n\n\nclass FocalLoss(nn.Module):\n # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\n super().__init__()\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\n self.gamma = gamma\n self.alpha = alpha\n self.reduction = loss_fcn.reduction\n self.loss_fcn.reduction = \"none\" # required to apply FL to each element\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n # p_t = torch.exp(-loss)\n # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability\n\n # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py\n pred_prob = torch.sigmoid(pred) # prob from logits\n p_t = true * pred_prob + (1 - true) * (1 - pred_prob)\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\n modulating_factor = (1.0 - p_t) ** self.gamma\n loss *= alpha_factor * modulating_factor\n\n if self.reduction == \"mean\":\n return loss.mean()\n elif self.reduction == \"sum\":\n return loss.sum()\n else: # 'none'\n return loss\n\n\nclass BboxLoss(nn.Module):\n def __init__(self, reg_max, use_dfl=False):\n super().__init__()\n self.reg_max = reg_max\n self.use_dfl = use_dfl\n\n def forward(self, pred_dist, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask):\n # iou loss\n bbox_mask = fg_mask.unsqueeze(-1).repeat([1, 1, 4]) # (b, h*w, 4)\n pred_bboxes_pos = torch.masked_select(pred_bboxes, bbox_mask).view(-1, 4)\n target_bboxes_pos = torch.masked_select(target_bboxes, bbox_mask).view(-1, 4)\n bbox_weight = torch.masked_select(target_scores.sum(-1), fg_mask).unsqueeze(-1)\n \n iou = bbox_iou(pred_bboxes_pos, target_bboxes_pos, xywh=False, CIoU=True)\n loss_iou = 1.0 - iou\n\n loss_iou *= bbox_weight\n loss_iou = loss_iou.sum() / target_scores_sum\n\n # dfl loss\n if self.use_dfl:\n dist_mask = fg_mask.unsqueeze(-1).repeat([1, 1, (self.reg_max + 1) * 4])\n pred_dist_pos = torch.masked_select(pred_dist, dist_mask).view(-1, 4, self.reg_max + 1)\n target_ltrb = bbox2dist(anchor_points, target_bboxes, self.reg_max)\n target_ltrb_pos = torch.masked_select(target_ltrb, bbox_mask).view(-1, 4)\n loss_dfl = self._df_loss(pred_dist_pos, target_ltrb_pos) * bbox_weight\n loss_dfl = loss_dfl.sum() / target_scores_sum\n else:\n loss_dfl = torch.tensor(0.0).to(pred_dist.device)\n\n return loss_iou, loss_dfl, iou\n\n def _df_loss(self, pred_dist, target):\n target_left = target.to(torch.long)\n target_right = target_left + 1\n weight_left = target_right.to(torch.float) - target\n weight_right = 1 - weight_left\n loss_left = F.cross_entropy(pred_dist.view(-1, self.reg_max + 1), target_left.view(-1), reduction=\"none\").view(\n target_left.shape) * weight_left\n loss_right = F.cross_entropy(pred_dist.view(-1, self.reg_max + 1), target_right.view(-1),\n reduction=\"none\").view(target_left.shape) * weight_right\n return (loss_left + loss_right).mean(-1, keepdim=True)\n\n\nclass ComputeLoss:\n # Compute losses\n def __init__(self, model, use_dfl=True, overlap=True):\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h[\"cls_pw\"]], device=device), reduction='none')\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get(\"label_smoothing\", 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h[\"fl_gamma\"] # focal loss gamma\n if g > 0:\n BCEcls = FocalLoss(BCEcls, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.BCEcls = BCEcls\n self.hyp = h\n self.stride = m.stride # model strides\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.no = m.no\n self.nm = m.nm\n self.overlap = overlap\n self.reg_max = m.reg_max\n self.device = device\n\n self.assigner = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.bbox_loss = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.proj = torch.arange(m.reg_max).float().to(device) # / 120.0\n self.use_dfl = use_dfl\n\n def preprocess(self, targets, batch_size, scale_tensor):\n if targets.shape[0] == 0:\n out = torch.zeros(batch_size, 0, 5, device=self.device)\n else:\n i = targets[:, 0] # image index\n _, counts = i.unique(return_counts=True)\n out = torch.zeros(batch_size, counts.max(), 5, device=self.device)\n for j in range(batch_size):\n matches = i == j\n n = matches.sum()\n if n:\n out[j, :n] = targets[matches, 1:]\n out[..., 1:5] = xywh2xyxy(out[..., 1:5].mul_(scale_tensor))\n return out\n\n def bbox_decode(self, anchor_points, pred_dist):\n if self.use_dfl:\n b, a, c = pred_dist.shape # batch, anchors, channels\n pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = pred_dist.view(b, a, c // 4, 4).transpose(2,3).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = (pred_dist.view(b, a, c // 4, 4).softmax(2) * self.proj.type(pred_dist.dtype).view(1, 1, -1, 1)).sum(2)\n return dist2bbox(pred_dist, anchor_points, xywh=False)\n\n def __call__(self, p, targets, masks, img=None, epoch=0):\n loss = torch.zeros(4, device=self.device) # box, cls, dfl\n feats, pred_masks, proto = p if len(p) == 3 else p[1]\n batch_size, _, mask_h, mask_w = proto.shape\n pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores = pred_scores.permute(0, 2, 1).contiguous()\n pred_distri = pred_distri.permute(0, 2, 1).contiguous()\n pred_masks = pred_masks.permute(0, 2, 1).contiguous()\n\n dtype = pred_scores.dtype\n batch_size, grid_size = pred_scores.shape[:2]\n imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w)\n anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5)\n\n # targets\n try:\n batch_idx = targets[:, 0].view(-1, 1)\n targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]])\n gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy\n mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0)\n except RuntimeError as e:\n raise TypeError('ERROR.') from e\n\n\n # pboxes\n pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4)\n\n target_labels, target_bboxes, target_scores, fg_mask, target_gt_idx = self.assigner(\n pred_scores.detach().sigmoid(),\n (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n target_scores_sum = target_scores.sum()\n\n # cls loss\n # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way\n loss[2] = self.BCEcls(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n\n # bbox loss\n if fg_mask.sum():\n loss[0], loss[3], _ = self.bbox_loss(pred_distri, \n pred_bboxes, \n anchor_points, \n target_bboxes / stride_tensor,\n target_scores, \n target_scores_sum, \n fg_mask)\n \n # masks loss\n if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample\n masks = F.interpolate(masks[None], (mask_h, mask_w), mode='nearest')[0]\n \n for i in range(batch_size):\n if fg_mask[i].sum():\n mask_idx = target_gt_idx[i][fg_mask[i]]\n if self.overlap:\n gt_mask = torch.where(masks[[i]] == (mask_idx + 1).view(-1, 1, 1), 1.0, 0.0)\n else:\n gt_mask = masks[batch_idx.view(-1) == i][mask_idx]\n xyxyn = target_bboxes[i][fg_mask[i]] / imgsz[[1, 0, 1, 0]]\n marea = xyxy2xywh(xyxyn)[:, 2:].prod(1)\n mxyxy = xyxyn * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=self.device)\n loss[1] += self.single_mask_loss(gt_mask, pred_masks[i][fg_mask[i]], proto[i], mxyxy,\n marea) # seg loss\n\n loss[0] *= 7.5 # box gain\n loss[1] *= 2.5 / batch_size\n loss[2] *= 0.5 # cls gain\n loss[3] *= 1.5 # dfl gain\n\n return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl)\n\n def single_mask_loss(self, gt_mask, pred, proto, xyxy, area):\n # Mask loss for one image\n pred_mask = (pred @ proto.view(self.nm, -1)).view(-1, *proto.shape[1:]) # (n, 32) @ (32,80,80) -> (n,80,80)\n loss = F.binary_cross_entropy_with_logits(pred_mask, gt_mask, reduction='none')\n #loss = sigmoid_focal_loss(pred_mask, gt_mask, alpha = .25, gamma = 2., reduction = 'none')\n \n return (crop_mask(loss, xyxy).mean(dim=(1, 2)) / area).mean()\n \n #p_m = torch.flatten(pred_mask.sigmoid())\n #p_m = torch.flatten(pred_mask.softmax(dim = 1))\n #g_m = torch.flatten(gt_mask)\n #i_m = torch.sum(torch.mul(p_m, g_m))\n #u_m = torch.sum(torch.add(p_m, g_m))\n #d_c = (2. * i_m + 1.) / (u_m + 1.)\n #d_l = (1. - d_c)\n #return d_l\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/loss_tal_dual.py", "content": "import os\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torchvision.ops import sigmoid_focal_loss\n\nfrom utils.general import xywh2xyxy, xyxy2xywh\nfrom utils.metrics import bbox_iou\nfrom utils.segment.tal.anchor_generator import dist2bbox, make_anchors, bbox2dist\nfrom utils.segment.tal.assigner import TaskAlignedAssigner\nfrom utils.torch_utils import de_parallel\nfrom utils.segment.general import crop_mask\n\n\ndef smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441\n # return positive, negative label smoothing BCE targets\n return 1.0 - 0.5 * eps, 0.5 * eps\n\n\nclass VarifocalLoss(nn.Module):\n # Varifocal loss by Zhang et al. https://arxiv.org/abs/2008.13367\n def __init__(self):\n super().__init__()\n\n def forward(self, pred_score, gt_score, label, alpha=0.75, gamma=2.0):\n weight = alpha * pred_score.sigmoid().pow(gamma) * (1 - label) + gt_score * label\n with torch.cuda.amp.autocast(enabled=False):\n loss = (F.binary_cross_entropy_with_logits(pred_score.float(), gt_score.float(),\n reduction=\"none\") * weight).sum()\n return loss\n\n\nclass FocalLoss(nn.Module):\n # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\n super().__init__()\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\n self.gamma = gamma\n self.alpha = alpha\n self.reduction = loss_fcn.reduction\n self.loss_fcn.reduction = \"none\" # required to apply FL to each element\n\n def forward(self, pred, true):\n loss = self.loss_fcn(pred, true)\n # p_t = torch.exp(-loss)\n # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability\n\n # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py\n pred_prob = torch.sigmoid(pred) # prob from logits\n p_t = true * pred_prob + (1 - true) * (1 - pred_prob)\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\n modulating_factor = (1.0 - p_t) ** self.gamma\n loss *= alpha_factor * modulating_factor\n\n if self.reduction == \"mean\":\n return loss.mean()\n elif self.reduction == \"sum\":\n return loss.sum()\n else: # 'none'\n return loss\n\n\nclass BboxLoss(nn.Module):\n def __init__(self, reg_max, use_dfl=False):\n super().__init__()\n self.reg_max = reg_max\n self.use_dfl = use_dfl\n\n def forward(self, pred_dist, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask):\n # iou loss\n bbox_mask = fg_mask.unsqueeze(-1).repeat([1, 1, 4]) # (b, h*w, 4)\n pred_bboxes_pos = torch.masked_select(pred_bboxes, bbox_mask).view(-1, 4)\n target_bboxes_pos = torch.masked_select(target_bboxes, bbox_mask).view(-1, 4)\n bbox_weight = torch.masked_select(target_scores.sum(-1), fg_mask).unsqueeze(-1)\n \n iou = bbox_iou(pred_bboxes_pos, target_bboxes_pos, xywh=False, CIoU=True)\n loss_iou = 1.0 - iou\n\n loss_iou *= bbox_weight\n loss_iou = loss_iou.sum() / target_scores_sum\n\n # dfl loss\n if self.use_dfl:\n dist_mask = fg_mask.unsqueeze(-1).repeat([1, 1, (self.reg_max + 1) * 4])\n pred_dist_pos = torch.masked_select(pred_dist, dist_mask).view(-1, 4, self.reg_max + 1)\n target_ltrb = bbox2dist(anchor_points, target_bboxes, self.reg_max)\n target_ltrb_pos = torch.masked_select(target_ltrb, bbox_mask).view(-1, 4)\n loss_dfl = self._df_loss(pred_dist_pos, target_ltrb_pos) * bbox_weight\n loss_dfl = loss_dfl.sum() / target_scores_sum\n else:\n loss_dfl = torch.tensor(0.0).to(pred_dist.device)\n\n return loss_iou, loss_dfl, iou\n\n def _df_loss(self, pred_dist, target):\n target_left = target.to(torch.long)\n target_right = target_left + 1\n weight_left = target_right.to(torch.float) - target\n weight_right = 1 - weight_left\n loss_left = F.cross_entropy(pred_dist.view(-1, self.reg_max + 1), target_left.view(-1), reduction=\"none\").view(\n target_left.shape) * weight_left\n loss_right = F.cross_entropy(pred_dist.view(-1, self.reg_max + 1), target_right.view(-1),\n reduction=\"none\").view(target_left.shape) * weight_right\n return (loss_left + loss_right).mean(-1, keepdim=True)\n\n\nclass ComputeLoss:\n # Compute losses\n def __init__(self, model, use_dfl=True, overlap=True):\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h[\"cls_pw\"]], device=device), reduction='none')\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get(\"label_smoothing\", 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h[\"fl_gamma\"] # focal loss gamma\n if g > 0:\n BCEcls = FocalLoss(BCEcls, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.BCEcls = BCEcls\n self.hyp = h\n self.stride = m.stride # model strides\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.no = m.no\n self.nm = m.nm\n self.overlap = overlap\n self.reg_max = m.reg_max\n self.device = device\n\n self.assigner = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.assigner2 = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.bbox_loss = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.bbox_loss2 = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.proj = torch.arange(m.reg_max).float().to(device) # / 120.0\n self.use_dfl = use_dfl\n\n def preprocess(self, targets, batch_size, scale_tensor):\n if targets.shape[0] == 0:\n out = torch.zeros(batch_size, 0, 5, device=self.device)\n else:\n i = targets[:, 0] # image index\n _, counts = i.unique(return_counts=True)\n out = torch.zeros(batch_size, counts.max(), 5, device=self.device)\n for j in range(batch_size):\n matches = i == j\n n = matches.sum()\n if n:\n out[j, :n] = targets[matches, 1:]\n out[..., 1:5] = xywh2xyxy(out[..., 1:5].mul_(scale_tensor))\n return out\n\n def bbox_decode(self, anchor_points, pred_dist):\n if self.use_dfl:\n b, a, c = pred_dist.shape # batch, anchors, channels\n pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = pred_dist.view(b, a, c // 4, 4).transpose(2,3).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = (pred_dist.view(b, a, c // 4, 4).softmax(2) * self.proj.type(pred_dist.dtype).view(1, 1, -1, 1)).sum(2)\n return dist2bbox(pred_dist, anchor_points, xywh=False)\n\n def __call__(self, p, targets, masks, img=None, epoch=0):\n loss = torch.zeros(4, device=self.device) # box, cls, dfl\n \n feats_, pred_masks_, proto_ = p if len(p) == 3 else p[1]\n \n feats, pred_masks, proto = feats_[0], pred_masks_[0], proto_[0]\n feats2, pred_masks2, proto2 = feats_[1], pred_masks_[1], proto_[1]\n \n batch_size, _, mask_h, mask_w = proto.shape\n \n pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores = pred_scores.permute(0, 2, 1).contiguous()\n pred_distri = pred_distri.permute(0, 2, 1).contiguous()\n pred_masks = pred_masks.permute(0, 2, 1).contiguous()\n \n pred_distri2, pred_scores2 = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats2], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores2 = pred_scores2.permute(0, 2, 1).contiguous()\n pred_distri2 = pred_distri2.permute(0, 2, 1).contiguous()\n pred_masks2 = pred_masks2.permute(0, 2, 1).contiguous()\n\n dtype = pred_scores.dtype\n batch_size, grid_size = pred_scores.shape[:2]\n imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w)\n anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5)\n\n # targets\n try:\n batch_idx = targets[:, 0].view(-1, 1)\n targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]])\n gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy\n mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0)\n except RuntimeError as e:\n raise TypeError('ERROR.') from e\n\n\n # pboxes\n pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4)\n \n pred_bboxes2 = self.bbox_decode(anchor_points, pred_distri2) # xyxy, (b, h*w, 4)\n\n target_labels, target_bboxes, target_scores, fg_mask, target_gt_idx = self.assigner(\n pred_scores.detach().sigmoid(),\n (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n target_labels2, target_bboxes2, target_scores2, fg_mask2, target_gt_idx2 = self.assigner2(\n pred_scores2.detach().sigmoid(),\n (pred_bboxes2.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n target_scores_sum = target_scores.sum()\n\n target_scores_sum2 = target_scores2.sum()\n\n # cls loss\n # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way\n loss[2] = self.BCEcls(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n loss[2] *= 0.25\n loss[2] += self.BCEcls(pred_scores2, target_scores2.to(dtype)).sum() / target_scores_sum2 # BCE\n\n # bbox loss\n if fg_mask.sum():\n loss[0], loss[3], _ = self.bbox_loss(pred_distri, \n pred_bboxes, \n anchor_points, \n target_bboxes / stride_tensor,\n target_scores, \n target_scores_sum, \n fg_mask)\n \n # masks loss\n if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample\n masks = F.interpolate(masks[None], (mask_h, mask_w), mode='nearest')[0]\n \n for i in range(batch_size):\n if fg_mask[i].sum():\n mask_idx = target_gt_idx[i][fg_mask[i]]\n if self.overlap:\n gt_mask = torch.where(masks[[i]] == (mask_idx + 1).view(-1, 1, 1), 1.0, 0.0)\n else:\n gt_mask = masks[batch_idx.view(-1) == i][mask_idx]\n xyxyn = target_bboxes[i][fg_mask[i]] / imgsz[[1, 0, 1, 0]]\n marea = xyxy2xywh(xyxyn)[:, 2:].prod(1)\n mxyxy = xyxyn * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=self.device)\n loss[1] += self.single_mask_loss(gt_mask, pred_masks[i][fg_mask[i]], proto[i], mxyxy,\n marea) # seg loss\n \n loss[0] *= 0.25\n loss[3] *= 0.25\n loss[1] *= 0.25\n\n # bbox loss\n if fg_mask2.sum():\n loss0_, loss3_, _ = self.bbox_loss2(pred_distri2, \n pred_bboxes2, \n anchor_points, \n target_bboxes2 / stride_tensor,\n target_scores2, \n target_scores_sum2, \n fg_mask2)\n \n # masks loss\n if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample\n masks = F.interpolate(masks[None], (mask_h, mask_w), mode='nearest')[0]\n \n for i in range(batch_size):\n if fg_mask2[i].sum():\n mask_idx = target_gt_idx2[i][fg_mask2[i]]\n if self.overlap:\n gt_mask = torch.where(masks[[i]] == (mask_idx + 1).view(-1, 1, 1), 1.0, 0.0)\n else:\n gt_mask = masks[batch_idx.view(-1) == i][mask_idx]\n xyxyn = target_bboxes2[i][fg_mask2[i]] / imgsz[[1, 0, 1, 0]]\n marea = xyxy2xywh(xyxyn)[:, 2:].prod(1)\n mxyxy = xyxyn * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=self.device)\n loss[1] += self.single_mask_loss(gt_mask, pred_masks2[i][fg_mask2[i]], proto2[i], mxyxy,\n marea) # seg loss\n \n loss[0] += loss0_\n loss[3] += loss3_\n\n loss[0] *= 7.5 # box gain\n loss[1] *= 2.5 / batch_size\n loss[2] *= 0.5 # cls gain\n loss[3] *= 1.5 # dfl gain\n\n return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl)\n\n def single_mask_loss(self, gt_mask, pred, proto, xyxy, area):\n # Mask loss for one image\n pred_mask = (pred @ proto.view(self.nm, -1)).view(-1, *proto.shape[1:]) # (n, 32) @ (32,80,80) -> (n,80,80)\n loss = F.binary_cross_entropy_with_logits(pred_mask, gt_mask, reduction='none')\n #loss = sigmoid_focal_loss(pred_mask, gt_mask, alpha = .25, gamma = 2., reduction = 'none')\n \n #p_m = torch.flatten(pred_mask.softmax(dim = 1))\n #g_m = torch.flatten(gt_mask)\n #i_m = torch.sum(torch.mul(p_m, g_m))\n #u_m = torch.sum(torch.add(p_m, g_m))\n #dice_coef = (2. * i_m + 1.) / (u_m + 1.)\n #dice_loss = (1. - dice_coef)\n return (crop_mask(loss, xyxy).mean(dim=(1, 2)) / area).mean()\n\n\nclass ComputeLossLH:\n # Compute losses\n def __init__(self, model, use_dfl=True, overlap=True):\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h[\"cls_pw\"]], device=device), reduction='none')\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get(\"label_smoothing\", 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h[\"fl_gamma\"] # focal loss gamma\n if g > 0:\n BCEcls = FocalLoss(BCEcls, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.BCEcls = BCEcls\n self.hyp = h\n self.stride = m.stride # model strides\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.no = m.no\n self.nm = m.nm\n self.overlap = overlap\n self.reg_max = m.reg_max\n self.device = device\n\n self.assigner = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.bbox_loss = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.proj = torch.arange(m.reg_max).float().to(device) # / 120.0\n self.use_dfl = use_dfl\n\n def preprocess(self, targets, batch_size, scale_tensor):\n if targets.shape[0] == 0:\n out = torch.zeros(batch_size, 0, 5, device=self.device)\n else:\n i = targets[:, 0] # image index\n _, counts = i.unique(return_counts=True)\n out = torch.zeros(batch_size, counts.max(), 5, device=self.device)\n for j in range(batch_size):\n matches = i == j\n n = matches.sum()\n if n:\n out[j, :n] = targets[matches, 1:]\n out[..., 1:5] = xywh2xyxy(out[..., 1:5].mul_(scale_tensor))\n return out\n\n def bbox_decode(self, anchor_points, pred_dist):\n if self.use_dfl:\n b, a, c = pred_dist.shape # batch, anchors, channels\n pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = pred_dist.view(b, a, c // 4, 4).transpose(2,3).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = (pred_dist.view(b, a, c // 4, 4).softmax(2) * self.proj.type(pred_dist.dtype).view(1, 1, -1, 1)).sum(2)\n return dist2bbox(pred_dist, anchor_points, xywh=False)\n\n def __call__(self, p, targets, masks, img=None, epoch=0):\n loss = torch.zeros(4, device=self.device) # box, cls, dfl\n \n feats_, pred_masks_, proto_ = p if len(p) == 3 else p[1]\n \n feats, pred_masks, proto = feats_[0], pred_masks_[0], proto_[0]\n feats2, pred_masks2, proto2 = feats_[1], pred_masks_[1], proto_[1]\n \n batch_size, _, mask_h, mask_w = proto.shape\n \n pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores = pred_scores.permute(0, 2, 1).contiguous()\n pred_distri = pred_distri.permute(0, 2, 1).contiguous()\n pred_masks = pred_masks.permute(0, 2, 1).contiguous()\n \n pred_distri2, pred_scores2 = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats2], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores2 = pred_scores2.permute(0, 2, 1).contiguous()\n pred_distri2 = pred_distri2.permute(0, 2, 1).contiguous()\n pred_masks2 = pred_masks2.permute(0, 2, 1).contiguous()\n\n dtype = pred_scores.dtype\n batch_size, grid_size = pred_scores.shape[:2]\n imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w)\n anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5)\n\n # targets\n try:\n batch_idx = targets[:, 0].view(-1, 1)\n targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]])\n gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy\n mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0)\n except RuntimeError as e:\n raise TypeError('ERROR.') from e\n\n\n # pboxes\n pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4)\n \n pred_bboxes2 = self.bbox_decode(anchor_points, pred_distri2) # xyxy, (b, h*w, 4)\n\n target_labels, target_bboxes, target_scores, fg_mask, target_gt_idx = self.assigner(\n pred_scores2.detach().sigmoid(),\n (pred_bboxes2.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n target_scores_sum = target_scores.sum()\n\n # cls loss\n # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way\n loss[2] = self.BCEcls(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n loss[2] *= 0.25\n loss[2] += self.BCEcls(pred_scores2, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n\n # bbox loss\n if fg_mask.sum():\n loss[0], loss[3], _ = self.bbox_loss(pred_distri, \n pred_bboxes, \n anchor_points, \n target_bboxes / stride_tensor,\n target_scores, \n target_scores_sum, \n fg_mask)\n \n # masks loss\n if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample\n masks = F.interpolate(masks[None], (mask_h, mask_w), mode='nearest')[0]\n \n for i in range(batch_size):\n if fg_mask[i].sum():\n mask_idx = target_gt_idx[i][fg_mask[i]]\n if self.overlap:\n gt_mask = torch.where(masks[[i]] == (mask_idx + 1).view(-1, 1, 1), 1.0, 0.0)\n else:\n gt_mask = masks[batch_idx.view(-1) == i][mask_idx]\n xyxyn = target_bboxes[i][fg_mask[i]] / imgsz[[1, 0, 1, 0]]\n marea = xyxy2xywh(xyxyn)[:, 2:].prod(1)\n mxyxy = xyxyn * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=self.device)\n loss[1] += self.single_mask_loss(gt_mask, pred_masks[i][fg_mask[i]], proto[i], mxyxy,\n marea) # seg loss\n \n loss[0] *= 0.25\n loss[3] *= 0.25\n loss[1] *= 0.25\n\n # bbox loss\n if fg_mask.sum():\n loss0_, loss3_, _ = self.bbox_loss(pred_distri2, \n pred_bboxes2, \n anchor_points, \n target_bboxes / stride_tensor,\n target_scores, \n target_scores_sum, \n fg_mask)\n \n # masks loss\n if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample\n masks = F.interpolate(masks[None], (mask_h, mask_w), mode='nearest')[0]\n \n for i in range(batch_size):\n if fg_mask[i].sum():\n mask_idx = target_gt_idx[i][fg_mask[i]]\n if self.overlap:\n gt_mask = torch.where(masks[[i]] == (mask_idx + 1).view(-1, 1, 1), 1.0, 0.0)\n else:\n gt_mask = masks[batch_idx.view(-1) == i][mask_idx]\n xyxyn = target_bboxes[i][fg_mask[i]] / imgsz[[1, 0, 1, 0]]\n marea = xyxy2xywh(xyxyn)[:, 2:].prod(1)\n mxyxy = xyxyn * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=self.device)\n loss[1] += self.single_mask_loss(gt_mask, pred_masks2[i][fg_mask[i]], proto2[i], mxyxy,\n marea) # seg loss\n \n loss[0] += loss0_\n loss[3] += loss3_\n\n loss[0] *= 7.5 # box gain\n loss[1] *= 2.5 / batch_size\n loss[2] *= 0.5 # cls gain\n loss[3] *= 1.5 # dfl gain\n\n return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl)\n\n def single_mask_loss(self, gt_mask, pred, proto, xyxy, area):\n # Mask loss for one image\n pred_mask = (pred @ proto.view(self.nm, -1)).view(-1, *proto.shape[1:]) # (n, 32) @ (32,80,80) -> (n,80,80)\n loss = F.binary_cross_entropy_with_logits(pred_mask, gt_mask, reduction='none')\n #loss = sigmoid_focal_loss(pred_mask, gt_mask, alpha = .25, gamma = 2., reduction = 'none')\n \n #p_m = torch.flatten(pred_mask.softmax(dim = 1))\n #g_m = torch.flatten(gt_mask)\n #i_m = torch.sum(torch.mul(p_m, g_m))\n #u_m = torch.sum(torch.add(p_m, g_m))\n #dice_coef = (2. * i_m + 1.) / (u_m + 1.)\n #dice_loss = (1. - dice_coef)\n return (crop_mask(loss, xyxy).mean(dim=(1, 2)) / area).mean()\n\n\nclass ComputeLossLH0:\n # Compute losses\n def __init__(self, model, use_dfl=True, overlap=True):\n device = next(model.parameters()).device # get model device\n h = model.hyp # hyperparameters\n\n # Define criteria\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h[\"cls_pw\"]], device=device), reduction='none')\n\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\n self.cp, self.cn = smooth_BCE(eps=h.get(\"label_smoothing\", 0.0)) # positive, negative BCE targets\n\n # Focal loss\n g = h[\"fl_gamma\"] # focal loss gamma\n if g > 0:\n BCEcls = FocalLoss(BCEcls, g)\n\n m = de_parallel(model).model[-1] # Detect() module\n self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7\n self.BCEcls = BCEcls\n self.hyp = h\n self.stride = m.stride # model strides\n self.nc = m.nc # number of classes\n self.nl = m.nl # number of layers\n self.no = m.no\n self.nm = m.nm\n self.overlap = overlap\n self.reg_max = m.reg_max\n self.device = device\n\n self.assigner = TaskAlignedAssigner(topk=int(os.getenv('YOLOM', 10)),\n num_classes=self.nc,\n alpha=float(os.getenv('YOLOA', 0.5)),\n beta=float(os.getenv('YOLOB', 6.0)))\n self.bbox_loss = BboxLoss(m.reg_max - 1, use_dfl=use_dfl).to(device)\n self.proj = torch.arange(m.reg_max).float().to(device) # / 120.0\n self.use_dfl = use_dfl\n\n def preprocess(self, targets, batch_size, scale_tensor):\n if targets.shape[0] == 0:\n out = torch.zeros(batch_size, 0, 5, device=self.device)\n else:\n i = targets[:, 0] # image index\n _, counts = i.unique(return_counts=True)\n out = torch.zeros(batch_size, counts.max(), 5, device=self.device)\n for j in range(batch_size):\n matches = i == j\n n = matches.sum()\n if n:\n out[j, :n] = targets[matches, 1:]\n out[..., 1:5] = xywh2xyxy(out[..., 1:5].mul_(scale_tensor))\n return out\n\n def bbox_decode(self, anchor_points, pred_dist):\n if self.use_dfl:\n b, a, c = pred_dist.shape # batch, anchors, channels\n pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = pred_dist.view(b, a, c // 4, 4).transpose(2,3).softmax(3).matmul(self.proj.type(pred_dist.dtype))\n # pred_dist = (pred_dist.view(b, a, c // 4, 4).softmax(2) * self.proj.type(pred_dist.dtype).view(1, 1, -1, 1)).sum(2)\n return dist2bbox(pred_dist, anchor_points, xywh=False)\n\n def __call__(self, p, targets, masks, img=None, epoch=0):\n loss = torch.zeros(4, device=self.device) # box, cls, dfl\n \n feats_, pred_masks_, proto_ = p if len(p) == 3 else p[1]\n \n feats, pred_masks, proto = feats_[0], pred_masks_[0], proto_[0]\n feats2, pred_masks2, proto2 = feats_[1], pred_masks_[1], proto_[1]\n \n batch_size, _, mask_h, mask_w = proto.shape\n \n pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores = pred_scores.permute(0, 2, 1).contiguous()\n pred_distri = pred_distri.permute(0, 2, 1).contiguous()\n pred_masks = pred_masks.permute(0, 2, 1).contiguous()\n \n pred_distri2, pred_scores2 = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats2], 2).split(\n (self.reg_max * 4, self.nc), 1)\n pred_scores2 = pred_scores2.permute(0, 2, 1).contiguous()\n pred_distri2 = pred_distri2.permute(0, 2, 1).contiguous()\n pred_masks2 = pred_masks2.permute(0, 2, 1).contiguous()\n\n dtype = pred_scores.dtype\n batch_size, grid_size = pred_scores.shape[:2]\n imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w)\n anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5)\n\n # targets\n try:\n batch_idx = targets[:, 0].view(-1, 1)\n targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]])\n gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy\n mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0)\n except RuntimeError as e:\n raise TypeError('ERROR.') from e\n\n\n # pboxes\n pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4)\n \n pred_bboxes2 = self.bbox_decode(anchor_points, pred_distri2) # xyxy, (b, h*w, 4)\n\n target_labels, target_bboxes, target_scores, fg_mask, target_gt_idx = self.assigner(\n pred_scores2.detach().sigmoid(),\n (pred_bboxes2.detach() * stride_tensor).type(gt_bboxes.dtype),\n anchor_points * stride_tensor,\n gt_labels,\n gt_bboxes,\n mask_gt)\n\n target_scores_sum = target_scores.sum()\n\n # cls loss\n # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way\n loss[2] = self.BCEcls(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n loss[2] *= 0.25\n loss[2] += self.BCEcls(pred_scores2, target_scores.to(dtype)).sum() / target_scores_sum # BCE\n\n # bbox loss\n if fg_mask.sum():\n loss[0], loss[3], _ = self.bbox_loss(pred_distri, \n pred_bboxes, \n anchor_points, \n target_bboxes / stride_tensor,\n target_scores, \n target_scores_sum, \n fg_mask)\n \n # masks loss\n if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample\n masks = F.interpolate(masks[None], (mask_h, mask_w), mode='nearest')[0]\n \n for i in range(batch_size):\n if fg_mask[i].sum():\n mask_idx = target_gt_idx[i][fg_mask[i]]\n if self.overlap:\n gt_mask = torch.where(masks[[i]] == (mask_idx + 1).view(-1, 1, 1), 1.0, 0.0)\n else:\n gt_mask = masks[batch_idx.view(-1) == i][mask_idx]\n xyxyn = target_bboxes[i][fg_mask[i]] / imgsz[[1, 0, 1, 0]]\n marea = xyxy2xywh(xyxyn)[:, 2:].prod(1)\n mxyxy = xyxyn * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=self.device)\n loss[1] += self.single_mask_loss(gt_mask, pred_masks[i][fg_mask[i]], proto[i], mxyxy,\n marea) # seg loss\n \n loss[0] *= 0.25\n loss[3] *= 0.25\n loss[1] *= 0.25\n\n # bbox loss\n if fg_mask.sum():\n loss0_, loss3_, _ = self.bbox_loss(pred_distri2, \n pred_bboxes2, \n anchor_points, \n target_bboxes / stride_tensor,\n target_scores, \n target_scores_sum, \n fg_mask)\n \n # masks loss\n if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample\n masks = F.interpolate(masks[None], (mask_h, mask_w), mode='nearest')[0]\n \n for i in range(batch_size):\n if fg_mask[i].sum():\n mask_idx = target_gt_idx[i][fg_mask[i]]\n if self.overlap:\n gt_mask = torch.where(masks[[i]] == (mask_idx + 1).view(-1, 1, 1), 1.0, 0.0)\n else:\n gt_mask = masks[batch_idx.view(-1) == i][mask_idx]\n xyxyn = target_bboxes[i][fg_mask[i]] / imgsz[[1, 0, 1, 0]]\n marea = xyxy2xywh(xyxyn)[:, 2:].prod(1)\n mxyxy = xyxyn * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=self.device)\n loss[1] += 0. * self.single_mask_loss(gt_mask, pred_masks2[i][fg_mask[i]], proto2[i], mxyxy,\n marea) # seg loss\n \n loss[0] += loss0_\n loss[3] += loss3_\n\n loss[0] *= 7.5 # box gain\n loss[1] *= 2.5 / batch_size\n loss[2] *= 0.5 # cls gain\n loss[3] *= 1.5 # dfl gain\n\n return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl)\n\n def single_mask_loss(self, gt_mask, pred, proto, xyxy, area):\n # Mask loss for one image\n pred_mask = (pred @ proto.view(self.nm, -1)).view(-1, *proto.shape[1:]) # (n, 32) @ (32,80,80) -> (n,80,80)\n loss = F.binary_cross_entropy_with_logits(pred_mask, gt_mask, reduction='none')\n #loss = sigmoid_focal_loss(pred_mask, gt_mask, alpha = .25, gamma = 2., reduction = 'none')\n \n #p_m = torch.flatten(pred_mask.softmax(dim = 1))\n #g_m = torch.flatten(gt_mask)\n #i_m = torch.sum(torch.mul(p_m, g_m))\n #u_m = torch.sum(torch.add(p_m, g_m))\n #dice_coef = (2. * i_m + 1.) / (u_m + 1.)\n #dice_loss = (1. - dice_coef)\n return (crop_mask(loss, xyxy).mean(dim=(1, 2)) / area).mean()\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/metrics.py", "content": "import numpy as np\n\nfrom ..metrics import ap_per_class\n\n\ndef fitness(x):\n # Model fitness as a weighted combination of metrics\n w = [0.0, 0.0, 0.1, 0.9, 0.0, 0.0, 0.1, 0.9]\n return (x[:, :8] * w).sum(1)\n\n\ndef ap_per_class_box_and_mask(\n tp_m,\n tp_b,\n conf,\n pred_cls,\n target_cls,\n plot=False,\n save_dir=\".\",\n names=(),\n):\n \"\"\"\n Args:\n tp_b: tp of boxes.\n tp_m: tp of masks.\n other arguments see `func: ap_per_class`.\n \"\"\"\n results_boxes = ap_per_class(tp_b,\n conf,\n pred_cls,\n target_cls,\n plot=plot,\n save_dir=save_dir,\n names=names,\n prefix=\"Box\")[2:]\n results_masks = ap_per_class(tp_m,\n conf,\n pred_cls,\n target_cls,\n plot=plot,\n save_dir=save_dir,\n names=names,\n prefix=\"Mask\")[2:]\n\n results = {\n \"boxes\": {\n \"p\": results_boxes[0],\n \"r\": results_boxes[1],\n \"ap\": results_boxes[3],\n \"f1\": results_boxes[2],\n \"ap_class\": results_boxes[4]},\n \"masks\": {\n \"p\": results_masks[0],\n \"r\": results_masks[1],\n \"ap\": results_masks[3],\n \"f1\": results_masks[2],\n \"ap_class\": results_masks[4]}}\n return results\n\n\nclass Metric:\n\n def __init__(self) -> None:\n self.p = [] # (nc, )\n self.r = [] # (nc, )\n self.f1 = [] # (nc, )\n self.all_ap = [] # (nc, 10)\n self.ap_class_index = [] # (nc, )\n\n @property\n def ap50(self):\n \"\"\"AP@0.5 of all classes.\n Return:\n (nc, ) or [].\n \"\"\"\n return self.all_ap[:, 0] if len(self.all_ap) else []\n\n @property\n def ap(self):\n \"\"\"AP@0.5:0.95\n Return:\n (nc, ) or [].\n \"\"\"\n return self.all_ap.mean(1) if len(self.all_ap) else []\n\n @property\n def mp(self):\n \"\"\"mean precision of all classes.\n Return:\n float.\n \"\"\"\n return self.p.mean() if len(self.p) else 0.0\n\n @property\n def mr(self):\n \"\"\"mean recall of all classes.\n Return:\n float.\n \"\"\"\n return self.r.mean() if len(self.r) else 0.0\n\n @property\n def map50(self):\n \"\"\"Mean AP@0.5 of all classes.\n Return:\n float.\n \"\"\"\n return self.all_ap[:, 0].mean() if len(self.all_ap) else 0.0\n\n @property\n def map(self):\n \"\"\"Mean AP@0.5:0.95 of all classes.\n Return:\n float.\n \"\"\"\n return self.all_ap.mean() if len(self.all_ap) else 0.0\n\n def mean_results(self):\n \"\"\"Mean of results, return mp, mr, map50, map\"\"\"\n return (self.mp, self.mr, self.map50, self.map)\n\n def class_result(self, i):\n \"\"\"class-aware result, return p[i], r[i], ap50[i], ap[i]\"\"\"\n return (self.p[i], self.r[i], self.ap50[i], self.ap[i])\n\n def get_maps(self, nc):\n maps = np.zeros(nc) + self.map\n for i, c in enumerate(self.ap_class_index):\n maps[c] = self.ap[i]\n return maps\n\n def update(self, results):\n \"\"\"\n Args:\n results: tuple(p, r, ap, f1, ap_class)\n \"\"\"\n p, r, all_ap, f1, ap_class_index = results\n self.p = p\n self.r = r\n self.all_ap = all_ap\n self.f1 = f1\n self.ap_class_index = ap_class_index\n\n\nclass Metrics:\n \"\"\"Metric for boxes and masks.\"\"\"\n\n def __init__(self) -> None:\n self.metric_box = Metric()\n self.metric_mask = Metric()\n\n def update(self, results):\n \"\"\"\n Args:\n results: Dict{'boxes': Dict{}, 'masks': Dict{}}\n \"\"\"\n self.metric_box.update(list(results[\"boxes\"].values()))\n self.metric_mask.update(list(results[\"masks\"].values()))\n\n def mean_results(self):\n return self.metric_box.mean_results() + self.metric_mask.mean_results()\n\n def class_result(self, i):\n return self.metric_box.class_result(i) + self.metric_mask.class_result(i)\n\n def get_maps(self, nc):\n return self.metric_box.get_maps(nc) + self.metric_mask.get_maps(nc)\n\n @property\n def ap_class_index(self):\n # boxes and masks have the same ap_class_index\n return self.metric_box.ap_class_index\n\n\nKEYS = [\n \"train/box_loss\",\n \"train/seg_loss\", # train loss\n \"train/obj_loss\",\n \"train/cls_loss\",\n \"metrics/precision(B)\",\n \"metrics/recall(B)\",\n \"metrics/mAP_0.5(B)\",\n \"metrics/mAP_0.5:0.95(B)\", # metrics\n \"metrics/precision(M)\",\n \"metrics/recall(M)\",\n \"metrics/mAP_0.5(M)\",\n \"metrics/mAP_0.5:0.95(M)\", # metrics\n \"val/box_loss\",\n \"val/seg_loss\", # val loss\n \"val/obj_loss\",\n \"val/cls_loss\",\n \"x/lr0\",\n \"x/lr1\",\n \"x/lr2\",]\n\nBEST_KEYS = [\n \"best/epoch\",\n \"best/precision(B)\",\n \"best/recall(B)\",\n \"best/mAP_0.5(B)\",\n \"best/mAP_0.5:0.95(B)\",\n \"best/precision(M)\",\n \"best/recall(M)\",\n \"best/mAP_0.5(M)\",\n \"best/mAP_0.5:0.95(M)\",]\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/plots.py", "content": "import contextlib\nimport math\nfrom pathlib import Path\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport torch\n\nfrom .. import threaded\nfrom ..general import xywh2xyxy\nfrom ..plots import Annotator, colors\n\n\n@threaded\ndef plot_images_and_masks(images, targets, masks, paths=None, fname='images.jpg', names=None):\n # Plot image grid with labels\n if isinstance(images, torch.Tensor):\n images = images.cpu().float().numpy()\n if isinstance(targets, torch.Tensor):\n targets = targets.cpu().numpy()\n if isinstance(masks, torch.Tensor):\n masks = masks.cpu().numpy().astype(int)\n\n max_size = 1920 # max image size\n max_subplots = 16 # max image subplots, i.e. 4x4\n bs, _, h, w = images.shape # batch size, _, height, width\n bs = min(bs, max_subplots) # limit plot images\n ns = np.ceil(bs ** 0.5) # number of subplots (square)\n if np.max(images[0]) <= 1:\n images *= 255 # de-normalise (optional)\n\n # Build Image\n mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init\n for i, im in enumerate(images):\n if i == max_subplots: # if last batch has fewer images than we expect\n break\n x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin\n im = im.transpose(1, 2, 0)\n mosaic[y:y + h, x:x + w, :] = im\n\n # Resize (optional)\n scale = max_size / ns / max(h, w)\n if scale < 1:\n h = math.ceil(scale * h)\n w = math.ceil(scale * w)\n mosaic = cv2.resize(mosaic, tuple(int(x * ns) for x in (w, h)))\n\n # Annotate\n fs = int((h + w) * ns * 0.01) # font size\n annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)\n for i in range(i + 1):\n x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin\n annotator.rectangle([x, y, x + w, y + h], None, (255, 255, 255), width=2) # borders\n if paths:\n annotator.text((x + 5, y + 5 + h), text=Path(paths[i]).name[:40], txt_color=(220, 220, 220)) # filenames\n if len(targets) > 0:\n idx = targets[:, 0] == i\n ti = targets[idx] # image targets\n\n boxes = xywh2xyxy(ti[:, 2:6]).T\n classes = ti[:, 1].astype('int')\n labels = ti.shape[1] == 6 # labels if no conf column\n conf = None if labels else ti[:, 6] # check for confidence presence (label vs pred)\n\n if boxes.shape[1]:\n if boxes.max() <= 1.01: # if normalized with tolerance 0.01\n boxes[[0, 2]] *= w # scale to pixels\n boxes[[1, 3]] *= h\n elif scale < 1: # absolute coords need scale if image scales\n boxes *= scale\n boxes[[0, 2]] += x\n boxes[[1, 3]] += y\n for j, box in enumerate(boxes.T.tolist()):\n cls = classes[j]\n color = colors(cls)\n cls = names[cls] if names else cls\n if labels or conf[j] > 0.25: # 0.25 conf thresh\n label = f'{cls}' if labels else f'{cls} {conf[j]:.1f}'\n annotator.box_label(box, label, color=color)\n\n # Plot masks\n if len(masks):\n if masks.max() > 1.0: # mean that masks are overlap\n image_masks = masks[[i]] # (1, 640, 640)\n nl = len(ti)\n index = np.arange(nl).reshape(nl, 1, 1) + 1\n image_masks = np.repeat(image_masks, nl, axis=0)\n image_masks = np.where(image_masks == index, 1.0, 0.0)\n else:\n image_masks = masks[idx]\n\n im = np.asarray(annotator.im).copy()\n for j, box in enumerate(boxes.T.tolist()):\n if labels or conf[j] > 0.25: # 0.25 conf thresh\n color = colors(classes[j])\n mh, mw = image_masks[j].shape\n if mh != h or mw != w:\n mask = image_masks[j].astype(np.uint8)\n mask = cv2.resize(mask, (w, h))\n mask = mask.astype(bool)\n else:\n mask = image_masks[j].astype(bool)\n with contextlib.suppress(Exception):\n im[y:y + h, x:x + w, :][mask] = im[y:y + h, x:x + w, :][mask] * 0.4 + np.array(color) * 0.6\n annotator.fromarray(im)\n annotator.im.save(fname) # save\n\n\ndef plot_results_with_masks(file=\"path/to/results.csv\", dir=\"\", best=True):\n # Plot training results.csv. Usage: from utils.plots import *; plot_results('path/to/results.csv')\n save_dir = Path(file).parent if file else Path(dir)\n fig, ax = plt.subplots(2, 8, figsize=(18, 6), tight_layout=True)\n ax = ax.ravel()\n files = list(save_dir.glob(\"results*.csv\"))\n assert len(files), f\"No results.csv files found in {save_dir.resolve()}, nothing to plot.\"\n for f in files:\n try:\n data = pd.read_csv(f)\n index = np.argmax(0.9 * data.values[:, 8] + 0.1 * data.values[:, 7] + 0.9 * data.values[:, 12] +\n 0.1 * data.values[:, 11])\n s = [x.strip() for x in data.columns]\n x = data.values[:, 0]\n for i, j in enumerate([1, 2, 3, 4, 5, 6, 9, 10, 13, 14, 15, 16, 7, 8, 11, 12]):\n y = data.values[:, j]\n # y[y == 0] = np.nan # don't show zero values\n ax[i].plot(x, y, marker=\".\", label=f.stem, linewidth=2, markersize=2)\n if best:\n # best\n ax[i].scatter(index, y[index], color=\"r\", label=f\"best:{index}\", marker=\"*\", linewidth=3)\n ax[i].set_title(s[j] + f\"\\n{round(y[index], 5)}\")\n else:\n # last\n ax[i].scatter(x[-1], y[-1], color=\"r\", label=\"last\", marker=\"*\", linewidth=3)\n ax[i].set_title(s[j] + f\"\\n{round(y[-1], 5)}\")\n # if j in [8, 9, 10]: # share train and val loss y axes\n # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5])\n except Exception as e:\n print(f\"Warning: Plotting error for {f}: {e}\")\n ax[1].legend()\n fig.savefig(save_dir / \"results.png\", dpi=200)\n plt.close()\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/tal/__init__.py", "content": "# init" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/tal/anchor_generator.py", "content": "import torch\n\nfrom utils.general import check_version\n\nTORCH_1_10 = check_version(torch.__version__, '1.10.0')\n\n\ndef make_anchors(feats, strides, grid_cell_offset=0.5):\n \"\"\"Generate anchors from features.\"\"\"\n anchor_points, stride_tensor = [], []\n assert feats is not None\n dtype, device = feats[0].dtype, feats[0].device\n for i, stride in enumerate(strides):\n _, _, h, w = feats[i].shape\n sx = torch.arange(end=w, device=device, dtype=dtype) + grid_cell_offset # shift x\n sy = torch.arange(end=h, device=device, dtype=dtype) + grid_cell_offset # shift y\n sy, sx = torch.meshgrid(sy, sx, indexing='ij') if TORCH_1_10 else torch.meshgrid(sy, sx)\n anchor_points.append(torch.stack((sx, sy), -1).view(-1, 2))\n stride_tensor.append(torch.full((h * w, 1), stride, dtype=dtype, device=device))\n return torch.cat(anchor_points), torch.cat(stride_tensor)\n\n\ndef dist2bbox(distance, anchor_points, xywh=True, dim=-1):\n \"\"\"Transform distance(ltrb) to box(xywh or xyxy).\"\"\"\n lt, rb = torch.split(distance, 2, dim)\n x1y1 = anchor_points - lt\n x2y2 = anchor_points + rb\n if xywh:\n c_xy = (x1y1 + x2y2) / 2\n wh = x2y2 - x1y1\n return torch.cat((c_xy, wh), dim) # xywh bbox\n return torch.cat((x1y1, x2y2), dim) # xyxy bbox\n\n\ndef bbox2dist(anchor_points, bbox, reg_max):\n \"\"\"Transform bbox(xyxy) to dist(ltrb).\"\"\"\n x1y1, x2y2 = torch.split(bbox, 2, -1)\n return torch.cat((anchor_points - x1y1, x2y2 - anchor_points), -1).clamp(0, reg_max - 0.01) # dist (lt, rb)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/segment/tal/assigner.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom utils.metrics import bbox_iou\n\n\ndef select_candidates_in_gts(xy_centers, gt_bboxes, eps=1e-9):\n \"\"\"select the positive anchor center in gt\n\n Args:\n xy_centers (Tensor): shape(h*w, 4)\n gt_bboxes (Tensor): shape(b, n_boxes, 4)\n Return:\n (Tensor): shape(b, n_boxes, h*w)\n \"\"\"\n n_anchors = xy_centers.shape[0]\n bs, n_boxes, _ = gt_bboxes.shape\n lt, rb = gt_bboxes.view(-1, 1, 4).chunk(2, 2) # left-top, right-bottom\n bbox_deltas = torch.cat((xy_centers[None] - lt, rb - xy_centers[None]), dim=2).view(bs, n_boxes, n_anchors, -1)\n # return (bbox_deltas.min(3)[0] > eps).to(gt_bboxes.dtype)\n return bbox_deltas.amin(3).gt_(eps)\n\n\ndef select_highest_overlaps(mask_pos, overlaps, n_max_boxes):\n \"\"\"if an anchor box is assigned to multiple gts,\n the one with the highest iou will be selected.\n\n Args:\n mask_pos (Tensor): shape(b, n_max_boxes, h*w)\n overlaps (Tensor): shape(b, n_max_boxes, h*w)\n Return:\n target_gt_idx (Tensor): shape(b, h*w)\n fg_mask (Tensor): shape(b, h*w)\n mask_pos (Tensor): shape(b, n_max_boxes, h*w)\n \"\"\"\n # (b, n_max_boxes, h*w) -> (b, h*w)\n fg_mask = mask_pos.sum(-2)\n if fg_mask.max() > 1: # one anchor is assigned to multiple gt_bboxes\n mask_multi_gts = (fg_mask.unsqueeze(1) > 1).repeat([1, n_max_boxes, 1]) # (b, n_max_boxes, h*w)\n max_overlaps_idx = overlaps.argmax(1) # (b, h*w)\n is_max_overlaps = F.one_hot(max_overlaps_idx, n_max_boxes) # (b, h*w, n_max_boxes)\n is_max_overlaps = is_max_overlaps.permute(0, 2, 1).to(overlaps.dtype) # (b, n_max_boxes, h*w)\n mask_pos = torch.where(mask_multi_gts, is_max_overlaps, mask_pos) # (b, n_max_boxes, h*w)\n fg_mask = mask_pos.sum(-2)\n # find each grid serve which gt(index)\n target_gt_idx = mask_pos.argmax(-2) # (b, h*w)\n return target_gt_idx, fg_mask, mask_pos\n\n\nclass TaskAlignedAssigner(nn.Module):\n def __init__(self, topk=13, num_classes=80, alpha=1.0, beta=6.0, eps=1e-9):\n super().__init__()\n self.topk = topk\n self.num_classes = num_classes\n self.bg_idx = num_classes\n self.alpha = alpha\n self.beta = beta\n self.eps = eps\n\n @torch.no_grad()\n def forward(self, pd_scores, pd_bboxes, anc_points, gt_labels, gt_bboxes, mask_gt):\n \"\"\"This code referenced to\n https://github.com/Nioolek/PPYOLOE_pytorch/blob/master/ppyoloe/assigner/tal_assigner.py\n\n Args:\n pd_scores (Tensor): shape(bs, num_total_anchors, num_classes)\n pd_bboxes (Tensor): shape(bs, num_total_anchors, 4)\n anc_points (Tensor): shape(num_total_anchors, 2)\n gt_labels (Tensor): shape(bs, n_max_boxes, 1)\n gt_bboxes (Tensor): shape(bs, n_max_boxes, 4)\n mask_gt (Tensor): shape(bs, n_max_boxes, 1)\n Returns:\n target_labels (Tensor): shape(bs, num_total_anchors)\n target_bboxes (Tensor): shape(bs, num_total_anchors, 4)\n target_scores (Tensor): shape(bs, num_total_anchors, num_classes)\n fg_mask (Tensor): shape(bs, num_total_anchors)\n \"\"\"\n self.bs = pd_scores.size(0)\n self.n_max_boxes = gt_bboxes.size(1)\n\n if self.n_max_boxes == 0:\n device = gt_bboxes.device\n return (torch.full_like(pd_scores[..., 0], self.bg_idx).to(device),\n torch.zeros_like(pd_bboxes).to(device),\n torch.zeros_like(pd_scores).to(device),\n torch.zeros_like(pd_scores[..., 0]).to(device),\n torch.zeros_like(pd_scores[..., 0]).to(device))\n\n mask_pos, align_metric, overlaps = self.get_pos_mask(pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points,\n mask_gt)\n\n target_gt_idx, fg_mask, mask_pos = select_highest_overlaps(mask_pos, overlaps, self.n_max_boxes)\n\n # assigned target\n target_labels, target_bboxes, target_scores = self.get_targets(gt_labels, gt_bboxes, target_gt_idx, fg_mask)\n\n # normalize\n align_metric *= mask_pos\n pos_align_metrics = align_metric.amax(axis=-1, keepdim=True) # b, max_num_obj\n pos_overlaps = (overlaps * mask_pos).amax(axis=-1, keepdim=True) # b, max_num_obj\n norm_align_metric = (align_metric * pos_overlaps / (pos_align_metrics + self.eps)).amax(-2).unsqueeze(-1)\n target_scores = target_scores * norm_align_metric\n\n return target_labels, target_bboxes, target_scores, fg_mask.bool(), target_gt_idx\n\n def get_pos_mask(self, pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points, mask_gt):\n\n # get anchor_align metric, (b, max_num_obj, h*w)\n align_metric, overlaps = self.get_box_metrics(pd_scores, pd_bboxes, gt_labels, gt_bboxes)\n # get in_gts mask, (b, max_num_obj, h*w)\n mask_in_gts = select_candidates_in_gts(anc_points, gt_bboxes)\n # get topk_metric mask, (b, max_num_obj, h*w)\n mask_topk = self.select_topk_candidates(align_metric * mask_in_gts,\n topk_mask=mask_gt.repeat([1, 1, self.topk]).bool())\n # merge all mask to a final mask, (b, max_num_obj, h*w)\n mask_pos = mask_topk * mask_in_gts * mask_gt\n\n return mask_pos, align_metric, overlaps\n\n def get_box_metrics(self, pd_scores, pd_bboxes, gt_labels, gt_bboxes):\n\n gt_labels = gt_labels.to(torch.long) # b, max_num_obj, 1\n ind = torch.zeros([2, self.bs, self.n_max_boxes], dtype=torch.long) # 2, b, max_num_obj\n ind[0] = torch.arange(end=self.bs).view(-1, 1).repeat(1, self.n_max_boxes) # b, max_num_obj\n ind[1] = gt_labels.squeeze(-1) # b, max_num_obj\n # get the scores of each grid for each gt cls\n bbox_scores = pd_scores[ind[0], :, ind[1]] # b, max_num_obj, h*w\n\n overlaps = bbox_iou(gt_bboxes.unsqueeze(2), pd_bboxes.unsqueeze(1), xywh=False, CIoU=True).squeeze(3).clamp(0)\n align_metric = bbox_scores.pow(self.alpha) * (overlaps).pow(self.beta)\n return align_metric, overlaps\n\n def select_topk_candidates(self, metrics, largest=True, topk_mask=None):\n \"\"\"\n Args:\n metrics: (b, max_num_obj, h*w).\n topk_mask: (b, max_num_obj, topk) or None\n \"\"\"\n\n num_anchors = metrics.shape[-1] # h*w\n # (b, max_num_obj, topk)\n topk_metrics, topk_idxs = torch.topk(metrics, self.topk, dim=-1, largest=largest)\n if topk_mask is None:\n topk_mask = (topk_metrics.max(-1, keepdim=True) > self.eps).tile([1, 1, self.topk])\n # (b, max_num_obj, topk)\n topk_idxs = torch.where(topk_mask, topk_idxs, 0)\n # (b, max_num_obj, topk, h*w) -> (b, max_num_obj, h*w)\n is_in_topk = F.one_hot(topk_idxs, num_anchors).sum(-2)\n # filter invalid bboxes\n # assigned topk should be unique, this is for dealing with empty labels\n # since empty labels will generate index `0` through `F.one_hot`\n # NOTE: but what if the topk_idxs include `0`?\n is_in_topk = torch.where(is_in_topk > 1, 0, is_in_topk)\n return is_in_topk.to(metrics.dtype)\n\n def get_targets(self, gt_labels, gt_bboxes, target_gt_idx, fg_mask):\n \"\"\"\n Args:\n gt_labels: (b, max_num_obj, 1)\n gt_bboxes: (b, max_num_obj, 4)\n target_gt_idx: (b, h*w)\n fg_mask: (b, h*w)\n \"\"\"\n\n # assigned target labels, (b, 1)\n batch_ind = torch.arange(end=self.bs, dtype=torch.int64, device=gt_labels.device)[..., None]\n target_gt_idx = target_gt_idx + batch_ind * self.n_max_boxes # (b, h*w)\n target_labels = gt_labels.long().flatten()[target_gt_idx] # (b, h*w)\n\n # assigned target boxes, (b, max_num_obj, 4) -> (b, h*w)\n target_bboxes = gt_bboxes.view(-1, 4)[target_gt_idx]\n\n # assigned target scores\n target_labels.clamp(0)\n target_scores = F.one_hot(target_labels, self.num_classes) # (b, h*w, 80)\n fg_scores_mask = fg_mask[:, :, None].repeat(1, 1, self.num_classes) # (b, h*w, 80)\n target_scores = torch.where(fg_scores_mask > 0, target_scores, 0)\n\n return target_labels, target_bboxes, target_scores\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/tal/__init__.py", "content": "# init" }, { "path": "asone/detectors/yolov9/yolov9/utils/tal/anchor_generator.py", "content": "import torch\n\nfrom asone.detectors.yolov9.yolov9.utils.general import check_version\n\nTORCH_1_10 = check_version(torch.__version__, '1.10.0')\n\n\ndef make_anchors(feats, strides, grid_cell_offset=0.5):\n \"\"\"Generate anchors from features.\"\"\"\n anchor_points, stride_tensor = [], []\n assert feats is not None\n dtype, device = feats[0].dtype, feats[0].device\n for i, stride in enumerate(strides):\n _, _, h, w = feats[i].shape\n sx = torch.arange(end=w, device=device, dtype=dtype) + grid_cell_offset # shift x\n sy = torch.arange(end=h, device=device, dtype=dtype) + grid_cell_offset # shift y\n sy, sx = torch.meshgrid(sy, sx, indexing='ij') if TORCH_1_10 else torch.meshgrid(sy, sx)\n anchor_points.append(torch.stack((sx, sy), -1).view(-1, 2))\n stride_tensor.append(torch.full((h * w, 1), stride, dtype=dtype, device=device))\n return torch.cat(anchor_points), torch.cat(stride_tensor)\n\n\ndef dist2bbox(distance, anchor_points, xywh=True, dim=-1):\n \"\"\"Transform distance(ltrb) to box(xywh or xyxy).\"\"\"\n lt, rb = torch.split(distance, 2, dim)\n x1y1 = anchor_points - lt\n x2y2 = anchor_points + rb\n if xywh:\n c_xy = (x1y1 + x2y2) / 2\n wh = x2y2 - x1y1\n return torch.cat((c_xy, wh), dim) # xywh bbox\n return torch.cat((x1y1, x2y2), dim) # xyxy bbox\n\n\ndef bbox2dist(anchor_points, bbox, reg_max):\n \"\"\"Transform bbox(xyxy) to dist(ltrb).\"\"\"\n x1y1, x2y2 = torch.split(bbox, 2, -1)\n return torch.cat((anchor_points - x1y1, x2y2 - anchor_points), -1).clamp(0, reg_max - 0.01) # dist (lt, rb)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/tal/assigner.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom utils.metrics import bbox_iou\n\n\ndef select_candidates_in_gts(xy_centers, gt_bboxes, eps=1e-9):\n \"\"\"select the positive anchor center in gt\n\n Args:\n xy_centers (Tensor): shape(h*w, 4)\n gt_bboxes (Tensor): shape(b, n_boxes, 4)\n Return:\n (Tensor): shape(b, n_boxes, h*w)\n \"\"\"\n n_anchors = xy_centers.shape[0]\n bs, n_boxes, _ = gt_bboxes.shape\n lt, rb = gt_bboxes.view(-1, 1, 4).chunk(2, 2) # left-top, right-bottom\n bbox_deltas = torch.cat((xy_centers[None] - lt, rb - xy_centers[None]), dim=2).view(bs, n_boxes, n_anchors, -1)\n # return (bbox_deltas.min(3)[0] > eps).to(gt_bboxes.dtype)\n return bbox_deltas.amin(3).gt_(eps)\n\n\ndef select_highest_overlaps(mask_pos, overlaps, n_max_boxes):\n \"\"\"if an anchor box is assigned to multiple gts,\n the one with the highest iou will be selected.\n\n Args:\n mask_pos (Tensor): shape(b, n_max_boxes, h*w)\n overlaps (Tensor): shape(b, n_max_boxes, h*w)\n Return:\n target_gt_idx (Tensor): shape(b, h*w)\n fg_mask (Tensor): shape(b, h*w)\n mask_pos (Tensor): shape(b, n_max_boxes, h*w)\n \"\"\"\n # (b, n_max_boxes, h*w) -> (b, h*w)\n fg_mask = mask_pos.sum(-2)\n if fg_mask.max() > 1: # one anchor is assigned to multiple gt_bboxes\n mask_multi_gts = (fg_mask.unsqueeze(1) > 1).repeat([1, n_max_boxes, 1]) # (b, n_max_boxes, h*w)\n max_overlaps_idx = overlaps.argmax(1) # (b, h*w)\n is_max_overlaps = F.one_hot(max_overlaps_idx, n_max_boxes) # (b, h*w, n_max_boxes)\n is_max_overlaps = is_max_overlaps.permute(0, 2, 1).to(overlaps.dtype) # (b, n_max_boxes, h*w)\n mask_pos = torch.where(mask_multi_gts, is_max_overlaps, mask_pos) # (b, n_max_boxes, h*w)\n fg_mask = mask_pos.sum(-2)\n # find each grid serve which gt(index)\n target_gt_idx = mask_pos.argmax(-2) # (b, h*w)\n return target_gt_idx, fg_mask, mask_pos\n\n\nclass TaskAlignedAssigner(nn.Module):\n def __init__(self, topk=13, num_classes=80, alpha=1.0, beta=6.0, eps=1e-9):\n super().__init__()\n self.topk = topk\n self.num_classes = num_classes\n self.bg_idx = num_classes\n self.alpha = alpha\n self.beta = beta\n self.eps = eps\n\n @torch.no_grad()\n def forward(self, pd_scores, pd_bboxes, anc_points, gt_labels, gt_bboxes, mask_gt):\n \"\"\"This code referenced to\n https://github.com/Nioolek/PPYOLOE_pytorch/blob/master/ppyoloe/assigner/tal_assigner.py\n\n Args:\n pd_scores (Tensor): shape(bs, num_total_anchors, num_classes)\n pd_bboxes (Tensor): shape(bs, num_total_anchors, 4)\n anc_points (Tensor): shape(num_total_anchors, 2)\n gt_labels (Tensor): shape(bs, n_max_boxes, 1)\n gt_bboxes (Tensor): shape(bs, n_max_boxes, 4)\n mask_gt (Tensor): shape(bs, n_max_boxes, 1)\n Returns:\n target_labels (Tensor): shape(bs, num_total_anchors)\n target_bboxes (Tensor): shape(bs, num_total_anchors, 4)\n target_scores (Tensor): shape(bs, num_total_anchors, num_classes)\n fg_mask (Tensor): shape(bs, num_total_anchors)\n \"\"\"\n self.bs = pd_scores.size(0)\n self.n_max_boxes = gt_bboxes.size(1)\n\n if self.n_max_boxes == 0:\n device = gt_bboxes.device\n return (torch.full_like(pd_scores[..., 0], self.bg_idx).to(device),\n torch.zeros_like(pd_bboxes).to(device),\n torch.zeros_like(pd_scores).to(device),\n torch.zeros_like(pd_scores[..., 0]).to(device))\n\n mask_pos, align_metric, overlaps = self.get_pos_mask(pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points,\n mask_gt)\n\n target_gt_idx, fg_mask, mask_pos = select_highest_overlaps(mask_pos, overlaps, self.n_max_boxes)\n\n # assigned target\n target_labels, target_bboxes, target_scores = self.get_targets(gt_labels, gt_bboxes, target_gt_idx, fg_mask)\n\n # normalize\n align_metric *= mask_pos\n pos_align_metrics = align_metric.amax(axis=-1, keepdim=True) # b, max_num_obj\n pos_overlaps = (overlaps * mask_pos).amax(axis=-1, keepdim=True) # b, max_num_obj\n norm_align_metric = (align_metric * pos_overlaps / (pos_align_metrics + self.eps)).amax(-2).unsqueeze(-1)\n target_scores = target_scores * norm_align_metric\n\n return target_labels, target_bboxes, target_scores, fg_mask.bool()\n\n def get_pos_mask(self, pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points, mask_gt):\n\n # get anchor_align metric, (b, max_num_obj, h*w)\n align_metric, overlaps = self.get_box_metrics(pd_scores, pd_bboxes, gt_labels, gt_bboxes)\n # get in_gts mask, (b, max_num_obj, h*w)\n mask_in_gts = select_candidates_in_gts(anc_points, gt_bboxes)\n # get topk_metric mask, (b, max_num_obj, h*w)\n mask_topk = self.select_topk_candidates(align_metric * mask_in_gts,\n topk_mask=mask_gt.repeat([1, 1, self.topk]).bool())\n # merge all mask to a final mask, (b, max_num_obj, h*w)\n mask_pos = mask_topk * mask_in_gts * mask_gt\n\n return mask_pos, align_metric, overlaps\n\n def get_box_metrics(self, pd_scores, pd_bboxes, gt_labels, gt_bboxes):\n\n gt_labels = gt_labels.to(torch.long) # b, max_num_obj, 1\n ind = torch.zeros([2, self.bs, self.n_max_boxes], dtype=torch.long) # 2, b, max_num_obj\n ind[0] = torch.arange(end=self.bs).view(-1, 1).repeat(1, self.n_max_boxes) # b, max_num_obj\n ind[1] = gt_labels.squeeze(-1) # b, max_num_obj\n # get the scores of each grid for each gt cls\n bbox_scores = pd_scores[ind[0], :, ind[1]] # b, max_num_obj, h*w\n\n overlaps = bbox_iou(gt_bboxes.unsqueeze(2), pd_bboxes.unsqueeze(1), xywh=False, CIoU=True).squeeze(3).clamp(0)\n align_metric = bbox_scores.pow(self.alpha) * overlaps.pow(self.beta)\n return align_metric, overlaps\n\n def select_topk_candidates(self, metrics, largest=True, topk_mask=None):\n \"\"\"\n Args:\n metrics: (b, max_num_obj, h*w).\n topk_mask: (b, max_num_obj, topk) or None\n \"\"\"\n\n num_anchors = metrics.shape[-1] # h*w\n # (b, max_num_obj, topk)\n topk_metrics, topk_idxs = torch.topk(metrics, self.topk, dim=-1, largest=largest)\n if topk_mask is None:\n topk_mask = (topk_metrics.max(-1, keepdim=True) > self.eps).tile([1, 1, self.topk])\n # (b, max_num_obj, topk)\n topk_idxs = torch.where(topk_mask, topk_idxs, 0)\n # (b, max_num_obj, topk, h*w) -> (b, max_num_obj, h*w)\n is_in_topk = F.one_hot(topk_idxs, num_anchors).sum(-2)\n # filter invalid bboxes\n # assigned topk should be unique, this is for dealing with empty labels\n # since empty labels will generate index `0` through `F.one_hot`\n # NOTE: but what if the topk_idxs include `0`?\n is_in_topk = torch.where(is_in_topk > 1, 0, is_in_topk)\n return is_in_topk.to(metrics.dtype)\n\n def get_targets(self, gt_labels, gt_bboxes, target_gt_idx, fg_mask):\n \"\"\"\n Args:\n gt_labels: (b, max_num_obj, 1)\n gt_bboxes: (b, max_num_obj, 4)\n target_gt_idx: (b, h*w)\n fg_mask: (b, h*w)\n \"\"\"\n\n # assigned target labels, (b, 1)\n batch_ind = torch.arange(end=self.bs, dtype=torch.int64, device=gt_labels.device)[..., None]\n target_gt_idx = target_gt_idx + batch_ind * self.n_max_boxes # (b, h*w)\n target_labels = gt_labels.long().flatten()[target_gt_idx] # (b, h*w)\n\n # assigned target boxes, (b, max_num_obj, 4) -> (b, h*w)\n target_bboxes = gt_bboxes.view(-1, 4)[target_gt_idx]\n\n # assigned target scores\n target_labels.clamp(0)\n target_scores = F.one_hot(target_labels, self.num_classes) # (b, h*w, 80)\n fg_scores_mask = fg_mask[:, :, None].repeat(1, 1, self.num_classes) # (b, h*w, 80)\n target_scores = torch.where(fg_scores_mask > 0, target_scores, 0)\n\n return target_labels, target_bboxes, target_scores\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/torch_utils.py", "content": "import math\nimport os\nimport platform\nimport subprocess\nimport time\nimport warnings\nfrom contextlib import contextmanager\nfrom copy import deepcopy\nfrom pathlib import Path\n\nimport torch\nimport torch.distributed as dist\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn.parallel import DistributedDataParallel as DDP\n\nfrom asone.detectors.yolov9.yolov9.utils.general import LOGGER, check_version, colorstr, file_date, git_describe\nfrom asone.detectors.yolov9.yolov9.utils.lion import Lion\n\nLOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html\nRANK = int(os.getenv('RANK', -1))\nWORLD_SIZE = int(os.getenv('WORLD_SIZE', 1))\n\ntry:\n import thop # for FLOPs computation\nexcept ImportError:\n thop = None\n\n# Suppress PyTorch warnings\nwarnings.filterwarnings('ignore', message='User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\nwarnings.filterwarnings('ignore', category=UserWarning)\n\n\ndef smart_inference_mode(torch_1_9=check_version(torch.__version__, '1.9.0')):\n # Applies torch.inference_mode() decorator if torch>=1.9.0 else torch.no_grad() decorator\n def decorate(fn):\n return (torch.inference_mode if torch_1_9 else torch.no_grad)()(fn)\n\n return decorate\n\n\ndef smartCrossEntropyLoss(label_smoothing=0.0):\n # Returns nn.CrossEntropyLoss with label smoothing enabled for torch>=1.10.0\n if check_version(torch.__version__, '1.10.0'):\n return nn.CrossEntropyLoss(label_smoothing=label_smoothing)\n if label_smoothing > 0:\n LOGGER.warning(f'WARNING ⚠️ label smoothing {label_smoothing} requires torch>=1.10.0')\n return nn.CrossEntropyLoss()\n\n\ndef smart_DDP(model):\n # Model DDP creation with checks\n assert not check_version(torch.__version__, '1.12.0', pinned=True), \\\n 'torch==1.12.0 torchvision==0.13.0 DDP training is not supported due to a known issue. ' \\\n 'Please upgrade or downgrade torch to use DDP. See https://github.com/ultralytics/yolov5/issues/8395'\n if check_version(torch.__version__, '1.11.0'):\n return DDP(model, device_ids=[LOCAL_RANK], output_device=LOCAL_RANK, static_graph=True)\n else:\n return DDP(model, device_ids=[LOCAL_RANK], output_device=LOCAL_RANK)\n\n\ndef reshape_classifier_output(model, n=1000):\n # Update a TorchVision classification model to class count 'n' if required\n from asone.detectors.yolov9.yolov9.models.common import Classify\n name, m = list((model.model if hasattr(model, 'model') else model).named_children())[-1] # last module\n if isinstance(m, Classify): # YOLOv5 Classify() head\n if m.linear.out_features != n:\n m.linear = nn.Linear(m.linear.in_features, n)\n elif isinstance(m, nn.Linear): # ResNet, EfficientNet\n if m.out_features != n:\n setattr(model, name, nn.Linear(m.in_features, n))\n elif isinstance(m, nn.Sequential):\n types = [type(x) for x in m]\n if nn.Linear in types:\n i = types.index(nn.Linear) # nn.Linear index\n if m[i].out_features != n:\n m[i] = nn.Linear(m[i].in_features, n)\n elif nn.Conv2d in types:\n i = types.index(nn.Conv2d) # nn.Conv2d index\n if m[i].out_channels != n:\n m[i] = nn.Conv2d(m[i].in_channels, n, m[i].kernel_size, m[i].stride, bias=m[i].bias is not None)\n\n\n@contextmanager\ndef torch_distributed_zero_first(local_rank: int):\n # Decorator to make all processes in distributed training wait for each local_master to do something\n if local_rank not in [-1, 0]:\n dist.barrier(device_ids=[local_rank])\n yield\n if local_rank == 0:\n dist.barrier(device_ids=[0])\n\n\ndef device_count():\n # Returns number of CUDA devices available. Safe version of torch.cuda.device_count(). Supports Linux and Windows\n assert platform.system() in ('Linux', 'Windows'), 'device_count() only supported on Linux or Windows'\n try:\n cmd = 'nvidia-smi -L | wc -l' if platform.system() == 'Linux' else 'nvidia-smi -L | find /c /v \"\"' # Windows\n return int(subprocess.run(cmd, shell=True, capture_output=True, check=True).stdout.decode().split()[-1])\n except Exception:\n return 0\n\n\ndef select_device(device='', batch_size=0, newline=True):\n # device = None or 'cpu' or 0 or '0' or '0,1,2,3'\n s = f'YOLO 🚀 {git_describe() or file_date()} Python-{platform.python_version()} torch-{torch.__version__} '\n device = str(device).strip().lower().replace('cuda:', '').replace('none', '') # to string, 'cuda:0' to '0'\n cpu = device == 'cpu'\n mps = device == 'mps' # Apple Metal Performance Shaders (MPS)\n if cpu or mps:\n os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # force torch.cuda.is_available() = False\n elif device: # non-cpu device requested\n os.environ['CUDA_VISIBLE_DEVICES'] = device # set environment variable - must be before assert is_available()\n assert torch.cuda.is_available() and torch.cuda.device_count() >= len(device.replace(',', '')), \\\n f\"Invalid CUDA '--device {device}' requested, use '--device cpu' or pass valid CUDA device(s)\"\n\n if not cpu and not mps and torch.cuda.is_available(): # prefer GPU if available\n devices = device.split(',') if device else '0' # range(torch.cuda.device_count()) # i.e. 0,1,6,7\n n = len(devices) # device count\n if n > 1 and batch_size > 0: # check batch_size is divisible by device_count\n assert batch_size % n == 0, f'batch-size {batch_size} not multiple of GPU count {n}'\n space = ' ' * (len(s) + 1)\n for i, d in enumerate(devices):\n p = torch.cuda.get_device_properties(i)\n s += f\"{'' if i == 0 else space}CUDA:{d} ({p.name}, {p.total_memory / (1 << 20):.0f}MiB)\\n\" # bytes to MB\n arg = 'cuda:0'\n elif mps and getattr(torch, 'has_mps', False) and torch.backends.mps.is_available(): # prefer MPS if available\n s += 'MPS\\n'\n arg = 'mps'\n else: # revert to CPU\n s += 'CPU\\n'\n arg = 'cpu'\n\n if not newline:\n s = s.rstrip()\n LOGGER.info(s)\n return torch.device(arg)\n\n\ndef time_sync():\n # PyTorch-accurate time\n if torch.cuda.is_available():\n torch.cuda.synchronize()\n return time.time()\n\n\ndef profile(input, ops, n=10, device=None):\n \"\"\" YOLOv5 speed/memory/FLOPs profiler\n Usage:\n input = torch.randn(16, 3, 640, 640)\n m1 = lambda x: x * torch.sigmoid(x)\n m2 = nn.SiLU()\n profile(input, [m1, m2], n=100) # profile over 100 iterations\n \"\"\"\n results = []\n if not isinstance(device, torch.device):\n device = select_device(device)\n print(f\"{'Params':>12s}{'GFLOPs':>12s}{'GPU_mem (GB)':>14s}{'forward (ms)':>14s}{'backward (ms)':>14s}\"\n f\"{'input':>24s}{'output':>24s}\")\n\n for x in input if isinstance(input, list) else [input]:\n x = x.to(device)\n x.requires_grad = True\n for m in ops if isinstance(ops, list) else [ops]:\n m = m.to(device) if hasattr(m, 'to') else m # device\n m = m.half() if hasattr(m, 'half') and isinstance(x, torch.Tensor) and x.dtype is torch.float16 else m\n tf, tb, t = 0, 0, [0, 0, 0] # dt forward, backward\n try:\n flops = thop.profile(m, inputs=(x,), verbose=False)[0] / 1E9 * 2 # GFLOPs\n except Exception:\n flops = 0\n\n try:\n for _ in range(n):\n t[0] = time_sync()\n y = m(x)\n t[1] = time_sync()\n try:\n _ = (sum(yi.sum() for yi in y) if isinstance(y, list) else y).sum().backward()\n t[2] = time_sync()\n except Exception: # no backward method\n # print(e) # for debug\n t[2] = float('nan')\n tf += (t[1] - t[0]) * 1000 / n # ms per op forward\n tb += (t[2] - t[1]) * 1000 / n # ms per op backward\n mem = torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0 # (GB)\n s_in, s_out = (tuple(x.shape) if isinstance(x, torch.Tensor) else 'list' for x in (x, y)) # shapes\n p = sum(x.numel() for x in m.parameters()) if isinstance(m, nn.Module) else 0 # parameters\n print(f'{p:12}{flops:12.4g}{mem:>14.3f}{tf:14.4g}{tb:14.4g}{str(s_in):>24s}{str(s_out):>24s}')\n results.append([p, flops, mem, tf, tb, s_in, s_out])\n except Exception as e:\n print(e)\n results.append(None)\n torch.cuda.empty_cache()\n return results\n\n\ndef is_parallel(model):\n # Returns True if model is of type DP or DDP\n return type(model) in (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)\n\n\ndef de_parallel(model):\n # De-parallelize a model: returns single-GPU model if model is of type DP or DDP\n return model.module if is_parallel(model) else model\n\n\ndef initialize_weights(model):\n for m in model.modules():\n t = type(m)\n if t is nn.Conv2d:\n pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n elif t is nn.BatchNorm2d:\n m.eps = 1e-3\n m.momentum = 0.03\n elif t in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]:\n m.inplace = True\n\n\ndef find_modules(model, mclass=nn.Conv2d):\n # Finds layer indices matching module class 'mclass'\n return [i for i, m in enumerate(model.module_list) if isinstance(m, mclass)]\n\n\ndef sparsity(model):\n # Return global model sparsity\n a, b = 0, 0\n for p in model.parameters():\n a += p.numel()\n b += (p == 0).sum()\n return b / a\n\n\ndef prune(model, amount=0.3):\n # Prune model to requested global sparsity\n import torch.nn.utils.prune as prune\n for name, m in model.named_modules():\n if isinstance(m, nn.Conv2d):\n prune.l1_unstructured(m, name='weight', amount=amount) # prune\n prune.remove(m, 'weight') # make permanent\n LOGGER.info(f'Model pruned to {sparsity(model):.3g} global sparsity')\n\n\ndef fuse_conv_and_bn(conv, bn):\n # Fuse Conv2d() and BatchNorm2d() layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/\n fusedconv = nn.Conv2d(conv.in_channels,\n conv.out_channels,\n kernel_size=conv.kernel_size,\n stride=conv.stride,\n padding=conv.padding,\n dilation=conv.dilation,\n groups=conv.groups,\n bias=True).requires_grad_(False).to(conv.weight.device)\n\n # Prepare filters\n w_conv = conv.weight.clone().view(conv.out_channels, -1)\n w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))\n fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape))\n\n # Prepare spatial bias\n b_conv = torch.zeros(conv.weight.size(0), device=conv.weight.device) if conv.bias is None else conv.bias\n b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps))\n fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)\n\n return fusedconv\n\n\ndef model_info(model, verbose=False, imgsz=640):\n # Model information. img_size may be int or list, i.e. img_size=640 or img_size=[640, 320]\n n_p = sum(x.numel() for x in model.parameters()) # number parameters\n n_g = sum(x.numel() for x in model.parameters() if x.requires_grad) # number gradients\n if verbose:\n print(f\"{'layer':>5} {'name':>40} {'gradient':>9} {'parameters':>12} {'shape':>20} {'mu':>10} {'sigma':>10}\")\n for i, (name, p) in enumerate(model.named_parameters()):\n name = name.replace('module_list.', '')\n print('%5g %40s %9s %12g %20s %10.3g %10.3g' %\n (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std()))\n\n try: # FLOPs\n p = next(model.parameters())\n stride = max(int(model.stride.max()), 32) if hasattr(model, 'stride') else 32 # max stride\n im = torch.empty((1, p.shape[1], stride, stride), device=p.device) # input image in BCHW format\n flops = thop.profile(deepcopy(model), inputs=(im,), verbose=False)[0] / 1E9 * 2 # stride GFLOPs\n imgsz = imgsz if isinstance(imgsz, list) else [imgsz, imgsz] # expand if int/float\n fs = f', {flops * imgsz[0] / stride * imgsz[1] / stride:.1f} GFLOPs' # 640x640 GFLOPs\n except Exception:\n fs = ''\n\n name = Path(model.yaml_file).stem.replace('yolov5', 'YOLOv5') if hasattr(model, 'yaml_file') else 'Model'\n LOGGER.info(f\"{name} summary: {len(list(model.modules()))} layers, {n_p} parameters, {n_g} gradients{fs}\")\n\n\ndef scale_img(img, ratio=1.0, same_shape=False, gs=32): # img(16,3,256,416)\n # Scales img(bs,3,y,x) by ratio constrained to gs-multiple\n if ratio == 1.0:\n return img\n h, w = img.shape[2:]\n s = (int(h * ratio), int(w * ratio)) # new size\n img = F.interpolate(img, size=s, mode='bilinear', align_corners=False) # resize\n if not same_shape: # pad/crop img\n h, w = (math.ceil(x * ratio / gs) * gs for x in (h, w))\n return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean\n\n\ndef copy_attr(a, b, include=(), exclude=()):\n # Copy attributes from b to a, options to only include [...] and to exclude [...]\n for k, v in b.__dict__.items():\n if (len(include) and k not in include) or k.startswith('_') or k in exclude:\n continue\n else:\n setattr(a, k, v)\n\n\ndef smart_optimizer(model, name='Adam', lr=0.001, momentum=0.9, decay=1e-5):\n # YOLOv5 3-param group optimizer: 0) weights with decay, 1) weights no decay, 2) biases no decay\n g = [], [], [] # optimizer parameter groups\n bn = tuple(v for k, v in nn.__dict__.items() if 'Norm' in k) # normalization layers, i.e. BatchNorm2d()\n #for v in model.modules():\n # for p_name, p in v.named_parameters(recurse=0):\n # if p_name == 'bias': # bias (no decay)\n # g[2].append(p)\n # elif p_name == 'weight' and isinstance(v, bn): # weight (no decay)\n # g[1].append(p)\n # else:\n # g[0].append(p) # weight (with decay)\n \n for v in model.modules():\n if hasattr(v, 'bias') and isinstance(v.bias, nn.Parameter): # bias (no decay)\n g[2].append(v.bias)\n if isinstance(v, bn): # weight (no decay)\n g[1].append(v.weight)\n elif hasattr(v, 'weight') and isinstance(v.weight, nn.Parameter): # weight (with decay)\n g[0].append(v.weight)\n \n if hasattr(v, 'im'):\n if hasattr(v.im, 'implicit'): \n g[1].append(v.im.implicit)\n else:\n for iv in v.im:\n g[1].append(iv.implicit)\n if hasattr(v, 'ia'):\n if hasattr(v.ia, 'implicit'): \n g[1].append(v.ia.implicit)\n else:\n for iv in v.ia:\n g[1].append(iv.implicit)\n \n if hasattr(v, 'im2'):\n if hasattr(v.im2, 'implicit'): \n g[1].append(v.im2.implicit)\n else:\n for iv in v.im2:\n g[1].append(iv.implicit)\n if hasattr(v, 'ia2'):\n if hasattr(v.ia2, 'implicit'): \n g[1].append(v.ia2.implicit)\n else:\n for iv in v.ia2:\n g[1].append(iv.implicit)\n \n if hasattr(v, 'im3'):\n if hasattr(v.im3, 'implicit'): \n g[1].append(v.im3.implicit)\n else:\n for iv in v.im3:\n g[1].append(iv.implicit)\n if hasattr(v, 'ia3'):\n if hasattr(v.ia3, 'implicit'): \n g[1].append(v.ia3.implicit)\n else:\n for iv in v.ia3:\n g[1].append(iv.implicit)\n \n if hasattr(v, 'im4'):\n if hasattr(v.im4, 'implicit'): \n g[1].append(v.im4.implicit)\n else:\n for iv in v.im4:\n g[1].append(iv.implicit)\n if hasattr(v, 'ia4'):\n if hasattr(v.ia4, 'implicit'): \n g[1].append(v.ia4.implicit)\n else:\n for iv in v.ia4:\n g[1].append(iv.implicit)\n \n if hasattr(v, 'im5'):\n if hasattr(v.im5, 'implicit'): \n g[1].append(v.im5.implicit)\n else:\n for iv in v.im5:\n g[1].append(iv.implicit)\n if hasattr(v, 'ia5'):\n if hasattr(v.ia5, 'implicit'): \n g[1].append(v.ia5.implicit)\n else:\n for iv in v.ia5:\n g[1].append(iv.implicit)\n \n if hasattr(v, 'im6'):\n if hasattr(v.im6, 'implicit'): \n g[1].append(v.im6.implicit)\n else:\n for iv in v.im6:\n g[1].append(iv.implicit)\n if hasattr(v, 'ia6'):\n if hasattr(v.ia6, 'implicit'): \n g[1].append(v.ia6.implicit)\n else:\n for iv in v.ia6:\n g[1].append(iv.implicit)\n \n if hasattr(v, 'im7'):\n if hasattr(v.im7, 'implicit'): \n g[1].append(v.im7.implicit)\n else:\n for iv in v.im7:\n g[1].append(iv.implicit)\n if hasattr(v, 'ia7'):\n if hasattr(v.ia7, 'implicit'): \n g[1].append(v.ia7.implicit)\n else:\n for iv in v.ia7:\n g[1].append(iv.implicit)\n\n if name == 'Adam':\n optimizer = torch.optim.Adam(g[2], lr=lr, betas=(momentum, 0.999)) # adjust beta1 to momentum\n elif name == 'AdamW':\n optimizer = torch.optim.AdamW(g[2], lr=lr, betas=(momentum, 0.999), weight_decay=0.0, amsgrad=True)\n elif name == 'RMSProp':\n optimizer = torch.optim.RMSprop(g[2], lr=lr, momentum=momentum)\n elif name == 'SGD':\n optimizer = torch.optim.SGD(g[2], lr=lr, momentum=momentum, nesterov=True)\n elif name == 'LION':\n optimizer = Lion(g[2], lr=lr, betas=(momentum, 0.99), weight_decay=0.0)\n else:\n raise NotImplementedError(f'Optimizer {name} not implemented.')\n\n optimizer.add_param_group({'params': g[0], 'weight_decay': decay}) # add g0 with weight_decay\n optimizer.add_param_group({'params': g[1], 'weight_decay': 0.0}) # add g1 (BatchNorm2d weights)\n LOGGER.info(f\"{colorstr('optimizer:')} {type(optimizer).__name__}(lr={lr}) with parameter groups \"\n f\"{len(g[1])} weight(decay=0.0), {len(g[0])} weight(decay={decay}), {len(g[2])} bias\")\n return optimizer\n\n\ndef smart_hub_load(repo='ultralytics/yolov5', model='yolov5s', **kwargs):\n # YOLOv5 torch.hub.load() wrapper with smart error/issue handling\n if check_version(torch.__version__, '1.9.1'):\n kwargs['skip_validation'] = True # validation causes GitHub API rate limit errors\n if check_version(torch.__version__, '1.12.0'):\n kwargs['trust_repo'] = True # argument required starting in torch 0.12\n try:\n return torch.hub.load(repo, model, **kwargs)\n except Exception:\n return torch.hub.load(repo, model, force_reload=True, **kwargs)\n\n\ndef smart_resume(ckpt, optimizer, ema=None, weights='yolov5s.pt', epochs=300, resume=True):\n # Resume training from a partially trained checkpoint\n best_fitness = 0.0\n start_epoch = ckpt['epoch'] + 1\n if ckpt['optimizer'] is not None:\n optimizer.load_state_dict(ckpt['optimizer']) # optimizer\n best_fitness = ckpt['best_fitness']\n if ema and ckpt.get('ema'):\n ema.ema.load_state_dict(ckpt['ema'].float().state_dict()) # EMA\n ema.updates = ckpt['updates']\n if resume:\n assert start_epoch > 0, f'{weights} training to {epochs} epochs is finished, nothing to resume.\\n' \\\n f\"Start a new training without --resume, i.e. 'python train.py --weights {weights}'\"\n LOGGER.info(f'Resuming training from {weights} from epoch {start_epoch} to {epochs} total epochs')\n if epochs < start_epoch:\n LOGGER.info(f\"{weights} has been trained for {ckpt['epoch']} epochs. Fine-tuning for {epochs} more epochs.\")\n epochs += ckpt['epoch'] # finetune additional epochs\n return best_fitness, start_epoch, epochs\n\n\nclass EarlyStopping:\n # YOLOv5 simple early stopper\n def __init__(self, patience=30):\n self.best_fitness = 0.0 # i.e. mAP\n self.best_epoch = 0\n self.patience = patience or float('inf') # epochs to wait after fitness stops improving to stop\n self.possible_stop = False # possible stop may occur next epoch\n\n def __call__(self, epoch, fitness):\n if fitness >= self.best_fitness: # >= 0 to allow for early zero-fitness stage of training\n self.best_epoch = epoch\n self.best_fitness = fitness\n delta = epoch - self.best_epoch # epochs without improvement\n self.possible_stop = delta >= (self.patience - 1) # possible stop may occur next epoch\n stop = delta >= self.patience # stop training if patience exceeded\n if stop:\n LOGGER.info(f'Stopping training early as no improvement observed in last {self.patience} epochs. '\n f'Best results observed at epoch {self.best_epoch}, best model saved as best.pt.\\n'\n f'To update EarlyStopping(patience={self.patience}) pass a new patience value, '\n f'i.e. `python train.py --patience 300` or use `--patience 0` to disable EarlyStopping.')\n return stop\n\n\nclass ModelEMA:\n \"\"\" Updated Exponential Moving Average (EMA) from https://github.com/rwightman/pytorch-image-models\n Keeps a moving average of everything in the model state_dict (parameters and buffers)\n For EMA details see https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage\n \"\"\"\n\n def __init__(self, model, decay=0.9999, tau=2000, updates=0):\n # Create EMA\n self.ema = deepcopy(de_parallel(model)).eval() # FP32 EMA\n self.updates = updates # number of EMA updates\n self.decay = lambda x: decay * (1 - math.exp(-x / tau)) # decay exponential ramp (to help early epochs)\n for p in self.ema.parameters():\n p.requires_grad_(False)\n\n def update(self, model):\n # Update EMA parameters\n self.updates += 1\n d = self.decay(self.updates)\n\n msd = de_parallel(model).state_dict() # model state_dict\n for k, v in self.ema.state_dict().items():\n if v.dtype.is_floating_point: # true for FP16 and FP32\n v *= d\n v += (1 - d) * msd[k].detach()\n # assert v.dtype == msd[k].dtype == torch.float32, f'{k}: EMA {v.dtype} and model {msd[k].dtype} must be FP32'\n\n def update_attr(self, model, include=(), exclude=('process_group', 'reducer')):\n # Update EMA attributes\n copy_attr(self.ema, model, include, exclude)\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/triton.py", "content": "import typing\nfrom urllib.parse import urlparse\n\nimport torch\n\n\nclass TritonRemoteModel:\n \"\"\" A wrapper over a model served by the Triton Inference Server. It can\n be configured to communicate over GRPC or HTTP. It accepts Torch Tensors\n as input and returns them as outputs.\n \"\"\"\n\n def __init__(self, url: str):\n \"\"\"\n Keyword arguments:\n url: Fully qualified address of the Triton server - for e.g. grpc://localhost:8000\n \"\"\"\n\n parsed_url = urlparse(url)\n if parsed_url.scheme == \"grpc\":\n from tritonclient.grpc import InferenceServerClient, InferInput\n\n self.client = InferenceServerClient(parsed_url.netloc) # Triton GRPC client\n model_repository = self.client.get_model_repository_index()\n self.model_name = model_repository.models[0].name\n self.metadata = self.client.get_model_metadata(self.model_name, as_json=True)\n\n def create_input_placeholders() -> typing.List[InferInput]:\n return [\n InferInput(i['name'], [int(s) for s in i[\"shape\"]], i['datatype']) for i in self.metadata['inputs']]\n\n else:\n from tritonclient.http import InferenceServerClient, InferInput\n\n self.client = InferenceServerClient(parsed_url.netloc) # Triton HTTP client\n model_repository = self.client.get_model_repository_index()\n self.model_name = model_repository[0]['name']\n self.metadata = self.client.get_model_metadata(self.model_name)\n\n def create_input_placeholders() -> typing.List[InferInput]:\n return [\n InferInput(i['name'], [int(s) for s in i[\"shape\"]], i['datatype']) for i in self.metadata['inputs']]\n\n self._create_input_placeholders_fn = create_input_placeholders\n\n @property\n def runtime(self):\n \"\"\"Returns the model runtime\"\"\"\n return self.metadata.get(\"backend\", self.metadata.get(\"platform\"))\n\n def __call__(self, *args, **kwargs) -> typing.Union[torch.Tensor, typing.Tuple[torch.Tensor, ...]]:\n \"\"\" Invokes the model. Parameters can be provided via args or kwargs.\n args, if provided, are assumed to match the order of inputs of the model.\n kwargs are matched with the model input names.\n \"\"\"\n inputs = self._create_inputs(*args, **kwargs)\n response = self.client.infer(model_name=self.model_name, inputs=inputs)\n result = []\n for output in self.metadata['outputs']:\n tensor = torch.as_tensor(response.as_numpy(output['name']))\n result.append(tensor)\n return result[0] if len(result) == 1 else result\n\n def _create_inputs(self, *args, **kwargs):\n args_len, kwargs_len = len(args), len(kwargs)\n if not args_len and not kwargs_len:\n raise RuntimeError(\"No inputs provided.\")\n if args_len and kwargs_len:\n raise RuntimeError(\"Cannot specify args and kwargs at the same time\")\n\n placeholders = self._create_input_placeholders_fn()\n if args_len:\n if args_len != len(placeholders):\n raise RuntimeError(f\"Expected {len(placeholders)} inputs, got {args_len}.\")\n for input, value in zip(placeholders, args):\n input.set_data_from_numpy(value.cpu().numpy())\n else:\n for input in placeholders:\n value = kwargs[input.name]\n input.set_data_from_numpy(value.cpu().numpy())\n return placeholders\n" }, { "path": "asone/detectors/yolov9/yolov9/utils/yolov9_utils.py", "content": "import cv2\nimport numpy as np\nimport torch\nimport torchvision\nimport time\n\nfrom asone.detectors.yolov9.yolov9.utils.metrics import box_iou\n\ndef prepare_input(image, input_shape):\n input_height, input_width = input_shape\n input_img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n # Resize input image\n input_img = cv2.resize(input_img, (input_width, input_height))\n # Scale input pixel values to 0 to 1\n input_img = input_img / 255.0\n input_img = input_img.transpose(2, 0, 1)\n input_tensor = input_img[np.newaxis, :, :, :].astype(np.float32)\n\n return input_tensor\n\ndef process_output(output, ori_shape, input_shape, conf_threshold, iou_threshold):\n predictions = output[0]\n # predictions = np.squeeze(output[0])\n\n # Filter out object confidence scores below threshold\n # obj_conf = predictions[:, 4]\n obj_conf = predictions[:, 6]\n # predictions = predictions[obj_conf > conf_threshold]\n # obj_conf = obj_conf[obj_conf > conf_threshold]\n\n # print(obj_conf[0])\n\n # Multiply class confidence with bounding box confidence\n # predictions[:, 5] *= obj_conf[:, np.newaxis]\n # predictions[:, 6] *= obj_conf\n\n # Get the scores\n # scores = np.max(predictions[:, 5:], axis=1)\n scores = predictions[:, 6]\n\n # Filter out the objects with a low score\n predictions = predictions[obj_conf > conf_threshold]\n scores = scores[scores > conf_threshold]\n\n if len(scores) == 0:\n return []\n\n # Get the class with the highest confidence\n # class_ids = np.argmax(predictions[:, 5:], axis=1)\n class_ids = predictions[:, 5].astype(np.uint16)\n\n # Extract boxes from predictions\n boxes = predictions[:, 1:5]\n\n # Scale boxes to original image dimensions\n boxes = rescale_boxes(boxes, ori_shape, input_shape)\n\n # Convert boxes to xyxy format\n # boxes = xywh2xyxy(boxes)\n\n # Apply non-maxima suppression to suppress weak, overlapping bounding boxes\n indices = nms(boxes, scores, iou_threshold)\n\n dets = []\n for i in indices:\n dets.append([*boxes[i], scores[i], class_ids[i]])\n\n # return boxes[indices], scores[indices], class_ids[indices]\n return np.array(dets)\n\n\ndef rescale_boxes(boxes, ori_shape, input_shape):\n \n input_height, input_width = input_shape\n img_height, img_width = ori_shape\n # Rescale boxes to original image dimensions\n input_shape = np.array([input_width, input_height, input_width, input_height])\n boxes = np.divide(boxes, input_shape, dtype=np.float32)\n boxes *= np.array([img_width, img_height, img_width, img_height])\n return boxes\n\ndef nms(boxes, scores, iou_threshold):\n # Sort by score\n sorted_indices = np.argsort(scores)[::-1]\n\n keep_boxes = []\n while sorted_indices.size > 0:\n # Pick the last box\n box_id = sorted_indices[0]\n keep_boxes.append(box_id)\n\n # Compute IoU of the picked box with the rest\n ious = compute_iou(boxes[box_id, :], boxes[sorted_indices[1:], :])\n\n # Remove boxes with IoU over the threshold\n keep_indices = np.where(ious < iou_threshold)[0]\n\n # print(keep_indices.shape, sorted_indices.shape)\n sorted_indices = sorted_indices[keep_indices + 1]\n\n return keep_boxes\n\n\ndef compute_iou(box, boxes):\n # Compute xmin, ymin, xmax, ymax for both boxes\n xmin = np.maximum(box[0], boxes[:, 0])\n ymin = np.maximum(box[1], boxes[:, 1])\n xmax = np.minimum(box[2], boxes[:, 2])\n ymax = np.minimum(box[3], boxes[:, 3])\n\n # Compute intersection area\n intersection_area = np.maximum(0, xmax - xmin) * np.maximum(0, ymax - ymin)\n\n # Compute union area\n box_area = (box[2] - box[0]) * (box[3] - box[1])\n boxes_area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])\n union_area = box_area + boxes_area - intersection_area\n\n # Compute IoU\n iou = intersection_area / union_area\n\n return iou\n\n\ndef xywh2xyxy(x):\n # Convert bounding box (x, y, w, h) to bounding box (x1, y1, x2, y2)\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[..., 0] = x[..., 0] - x[..., 2] / 2\n y[..., 1] = x[..., 1] - x[..., 3] / 2\n y[..., 2] = x[..., 0] + x[..., 2] / 2\n y[..., 3] = x[..., 1] + x[..., 3] / 2\n return y\n\ndef non_max_suppression(\n prediction,\n conf_thres=0.25,\n iou_thres=0.45,\n classes=None,\n agnostic=False,\n multi_label=False,\n labels=(),\n max_det=300,\n nm=0, # number of masks\n ):\n \"\"\"Non-Maximum Suppression (NMS) on inference results to reject overlapping detections\n\n Returns:\n list of detections, on (n,6) tensor per image [xyxy, conf, cls]\n \"\"\"\n\n if isinstance(prediction, (list, tuple)): # YOLO model in validation model, output = (inference_out, loss_out)\n prediction = prediction[0] # select only inference output\n\n device = prediction.device\n mps = 'mps' in device.type # Apple MPS\n if mps: # MPS not fully supported yet, convert tensors to CPU before NMS\n prediction = prediction.cpu()\n bs = prediction.shape[0] # batch size\n nc = prediction.shape[1] - nm - 4 # number of classes\n mi = 4 + nc # mask start index\n xc = prediction[:, 4:mi].amax(1) > conf_thres # candidates\n\n # Checks\n assert 0 <= conf_thres <= 1, f'Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0'\n assert 0 <= iou_thres <= 1, f'Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0'\n\n # Settings\n # min_wh = 2 # (pixels) minimum box width and height\n max_wh = 7680 # (pixels) maximum box width and height\n max_nms = 30000 # maximum number of boxes into torchvision.ops.nms()\n time_limit = 2.5 + 0.05 * bs # seconds to quit after\n redundant = True # require redundant detections\n multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img)\n merge = False # use merge-NMS\n\n t = time.time()\n output = [torch.zeros((0, 6 + nm), device=prediction.device)] * bs\n for xi, x in enumerate(prediction): # image index, image inference\n # Apply constraints\n # x[((x[:, 2:4] < min_wh) | (x[:, 2:4] > max_wh)).any(1), 4] = 0 # width-height\n x = x.T[xc[xi]] # confidence\n\n # Cat apriori labels if autolabelling\n if labels and len(labels[xi]):\n lb = labels[xi]\n v = torch.zeros((len(lb), nc + nm + 5), device=x.device)\n v[:, :4] = lb[:, 1:5] # box\n v[range(len(lb)), lb[:, 0].long() + 4] = 1.0 # cls\n x = torch.cat((x, v), 0)\n\n # If none remain process next image\n if not x.shape[0]:\n continue\n\n # Detections matrix nx6 (xyxy, conf, cls)\n box, cls, mask = x.split((4, nc, nm), 1)\n box = xywh2xyxy(box) # center_x, center_y, width, height) to (x1, y1, x2, y2)\n if multi_label:\n i, j = (cls > conf_thres).nonzero(as_tuple=False).T\n x = torch.cat((box[i], x[i, 4 + j, None], j[:, None].float(), mask[i]), 1)\n else: # best class only\n conf, j = cls.max(1, keepdim=True)\n x = torch.cat((box, conf, j.float(), mask), 1)[conf.view(-1) > conf_thres]\n\n # Filter by class\n if classes is not None:\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\n\n # Apply finite constraint\n # if not torch.isfinite(x).all():\n # x = x[torch.isfinite(x).all(1)]\n\n # Check shape\n n = x.shape[0] # number of boxes\n if not n: # no boxes\n continue\n elif n > max_nms: # excess boxes\n x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence\n else:\n x = x[x[:, 4].argsort(descending=True)] # sort by confidence\n\n # Batched NMS\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\n i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS\n if i.shape[0] > max_det: # limit detections\n i = i[:max_det]\n if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)\n # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\n weights = iou * scores[None] # box weights\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\n if redundant:\n i = i[iou.sum(1) > 1] # require redundancy\n\n output[xi] = x[i]\n if mps:\n output[xi] = output[xi].to(device)\n\n return output\n\n\n\n\n\n" }, { "path": "asone/detectors/yolov9/yolov9_detector.py", "content": "import os\nimport sys\nimport onnxruntime\nimport torch\nimport coremltools as ct\nfrom asone.utils import get_names\nimport numpy as np\nimport warnings\nfrom asone.detectors.yolov9.yolov9.utils.yolov9_utils import (prepare_input,\n process_output,\n non_max_suppression)\nfrom asone.detectors.yolov9.yolov9.models.experimental import attempt_load\nfrom asone import utils\nfrom PIL import Image\nfrom asone.detectors.utils.coreml_utils import yolo_to_xyxy, generalize_output_format, scale_bboxes\n\nfrom asone.utils.utils import PathResolver\n\n\ndef xywh2xyxy(x):\n # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x\n y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y\n y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x\n y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y\n return y\n\n\n# sys.path.append(os.path.join(os.path.dirname(__file__), 'yolov9'))\nclass YOLOv9Detector:\n def __init__(self,\n weights=None,\n use_onnx=False,\n mlmodel=False,\n use_cuda=True):\n self.use_onnx = use_onnx\n self.mlmodel = mlmodel\n self.device = 'cuda' if use_cuda else 'cpu'\n\n #If incase weighst is a list of paths then select path at first index\n\n weights = str(weights[0] if isinstance(weights, list) else weights)\n\n if not os.path.exists(weights):\n utils.download_weights(weights)\n \n with PathResolver():\n # Load Model\n self.model = self.load_model(use_cuda, weights)\n \n def load_model(self, use_cuda, weights, fp16=False):\n # Device: CUDA and if fp16=True only then half precision floating point works \n self.fp16 = fp16 & ((not self.use_onnx or self.use_onnx) and self.device != 'cpu')\n # Load onnx \n if self.use_onnx: \n if use_cuda:\n providers = ['CUDAExecutionProvider','CPUExecutionProvider']\n else:\n providers = ['CPUExecutionProvider']\n\n model = onnxruntime.InferenceSession(weights, providers=providers)\n # Load coreml\n elif self.mlmodel:\n model = ct.models.MLModel(weights)\n #Load Pytorch\n else: \n model = attempt_load(weights, device=self.device)\n model.half() if self.fp16 else model.float()\n return model\n\n\n def detect(self, image: list,\n input_shape: tuple = (640, 640),\n conf_thres: float = 0.25,\n iou_thres: float = 0.45,\n max_det: int = 1000,\n filter_classes: bool = None,\n agnostic_nms: bool = True,\n with_p6: bool = False,\n return_image=False) -> list:\n\n # Preprocess input image and also copying original image for later use\n original_image = image.copy()\n img_height, img_width = original_image.shape[:2]\n processed_image = prepare_input(image, input_shape)\n \n # Perform Inference on the Image\n if self.use_onnx:\n # Run ONNX model \n input_name = self.model.get_inputs()[0].name\n prediction = self.model.run([self.model.get_outputs()[0].name], {\n input_name: processed_image})\n # Run Coreml model \n elif self.mlmodel:\n h ,w = image.shape[:2]\n pred = self.model.predict({\"image\":Image.fromarray(image).resize(input_shape)})\n xyxy = yolo_to_xyxy(pred['coordinates'], input_shape)\n out = generalize_output_format(xyxy, pred['confidence'], conf_thres)\n if out != []:\n detections = scale_bboxes(out, image.shape[:2], input_shape)\n else:\n detections = np.empty((0, 6))\n \n if filter_classes:\n class_names = get_names()\n\n filter_class_idx = []\n if filter_classes:\n for _class in filter_classes:\n if _class.lower() in class_names:\n filter_class_idx.append(class_names.index(_class.lower()))\n else:\n warnings.warn(f\"class {_class} not found in model classes list.\")\n\n detections = detections[np.in1d(detections[:,5].astype(int), filter_class_idx)]\n \n return detections, {'width':w, 'height':h}\n # Run Pytorch model\n else:\n processed_image = torch.from_numpy(processed_image).to(self.device)\n # Change image floating point precision if fp16 set to true\n processed_image = processed_image.half() if self.fp16 else processed_image.float() \n\n with torch.no_grad():\n prediction = self.model(processed_image, augment=False)[0]\n\n \n detection = []\n # Postprocess prediction\n if self.use_onnx:\n \n detection = process_output(prediction,\n original_image.shape[:2],\n input_shape,\n conf_thres,\n iou_thres)\n else:\n detection = non_max_suppression(prediction,\n conf_thres,\n iou_thres,\n agnostic=agnostic_nms)[0]\n \n detection = detection.detach().cpu().numpy()\n # detection = yolo_to_xyxy(detection, input_shape)\n # print(detection)\n \n # Rescaling Bounding Boxes\n detection[:, :4] /= np.array([input_shape[1], input_shape[0], input_shape[1], input_shape[0]])\n detection[:, :4] *= np.array([img_width, img_height, img_width, img_height])\n\n image_info = {\n 'width': original_image.shape[1],\n 'height': original_image.shape[0],\n }\n\n if len(detection) > 0:\n self.boxes = detection[:, :4]\n self.scores = detection[:, 4:5]\n self.class_ids = detection[:, 5:6]\n\n if filter_classes:\n class_names = get_names()\n\n filter_class_idx = []\n if filter_classes:\n for _class in filter_classes:\n if _class.lower() in class_names:\n filter_class_idx.append(class_names.index(_class.lower()))\n else:\n warnings.warn(f\"class {_class} not found in model classes list.\")\n\n detection = detection[np.in1d(detection[:,5].astype(int), filter_class_idx)]\n\n if return_image:\n return detection, original_image\n else: \n return detection, image_info\n" }, { "path": "asone/detectors/yolox/__init__.py", "content": "from .yolox_detector import YOLOxDetector\n__all__ = ['YOLOxDetector']" }, { "path": "asone/detectors/yolox/exps/__init__.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n" }, { "path": "asone/detectors/yolox/exps/yolov3.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nimport torch.nn as nn\n\nfrom asone.detectors.yolox.yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.depth = 1.0\n self.width = 1.0\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n\n def get_model(self, sublinear=False):\n def init_yolo(M):\n for m in M.modules():\n if isinstance(m, nn.BatchNorm2d):\n m.eps = 1e-3\n m.momentum = 0.03\n if \"model\" not in self.__dict__:\n from asone.detectors.yolox.yolox.models import YOLOX, YOLOFPN, YOLOXHead\n backbone = YOLOFPN()\n head = YOLOXHead(self.num_classes, self.width, in_channels=[128, 256, 512], act=\"lrelu\")\n self.model = YOLOX(backbone, head)\n self.model.apply(init_yolo)\n self.model.head.initialize_biases(1e-2)\n\n return self.model\n" }, { "path": "asone/detectors/yolox/exps/yolox_l.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom asone.detectors.yolox.yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.depth = 1.0\n self.width = 1.0\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n" }, { "path": "asone/detectors/yolox/exps/yolox_m.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom asone.detectors.yolox.yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.depth = 0.67\n self.width = 0.75\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n" }, { "path": "asone/detectors/yolox/exps/yolox_nano.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nimport torch.nn as nn\n\nfrom asone.detectors.yolox.yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.depth = 0.33\n self.width = 0.25\n self.input_size = (416, 416)\n self.random_size = (10, 20)\n self.mosaic_scale = (0.5, 1.5)\n self.test_size = (416, 416)\n self.mosaic_prob = 0.5\n self.enable_mixup = False\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n\n def get_model(self, sublinear=False):\n\n def init_yolo(M):\n for m in M.modules():\n if isinstance(m, nn.BatchNorm2d):\n m.eps = 1e-3\n m.momentum = 0.03\n if \"model\" not in self.__dict__:\n from asone.detectors.yolox.yolox.models import YOLOX, YOLOPAFPN, YOLOXHead\n in_channels = [256, 512, 1024]\n # NANO model use depthwise = True, which is main difference.\n backbone = YOLOPAFPN(\n self.depth, self.width, in_channels=in_channels,\n act=self.act, depthwise=True,\n )\n head = YOLOXHead(\n self.num_classes, self.width, in_channels=in_channels,\n act=self.act, depthwise=True\n )\n self.model = YOLOX(backbone, head)\n\n self.model.apply(init_yolo)\n self.model.head.initialize_biases(1e-2)\n return self.model\n" }, { "path": "asone/detectors/yolox/exps/yolox_s.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom asone.detectors.yolox.yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.depth = 0.33\n self.width = 0.50\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n" }, { "path": "asone/detectors/yolox/exps/yolox_tiny.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom asone.detectors.yolox.yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.depth = 0.33\n self.width = 0.375\n self.input_size = (416, 416)\n self.mosaic_scale = (0.5, 1.5)\n self.random_size = (10, 20)\n self.test_size = (416, 416)\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n self.enable_mixup = False\n" }, { "path": "asone/detectors/yolox/exps/yolox_x.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom asone.detectors.yolox.yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n def __init__(self):\n super(Exp, self).__init__()\n self.depth = 1.33\n self.width = 1.25\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n" }, { "path": "asone/detectors/yolox/yolox/__init__.py", "content": "" }, { "path": "asone/detectors/yolox/yolox/core/__init__.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nfrom .launch import launch\nfrom .trainer import Trainer\n" }, { "path": "asone/detectors/yolox/yolox/core/launch.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Code are based on\n# https://github.com/facebookresearch/detectron2/blob/master/detectron2/engine/launch.py\n# Copyright (c) Facebook, Inc. and its affiliates.\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport sys\nfrom datetime import timedelta\nfrom loguru import logger\n\nimport torch\nimport torch.distributed as dist\nimport torch.multiprocessing as mp\n\nimport yolox.utils.dist as comm\n\n__all__ = [\"launch\"]\n\n\nDEFAULT_TIMEOUT = timedelta(minutes=30)\n\n\ndef _find_free_port():\n \"\"\"\n Find an available port of current machine / node.\n \"\"\"\n import socket\n\n sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n # Binding to port 0 will cause the OS to find an available port for us\n sock.bind((\"\", 0))\n port = sock.getsockname()[1]\n sock.close()\n # NOTE: there is still a chance the port could be taken by other processes.\n return port\n\n\ndef launch(\n main_func,\n num_gpus_per_machine,\n num_machines=1,\n machine_rank=0,\n backend=\"nccl\",\n dist_url=None,\n args=(),\n timeout=DEFAULT_TIMEOUT,\n):\n \"\"\"\n Args:\n main_func: a function that will be called by `main_func(*args)`\n num_machines (int): the total number of machines\n machine_rank (int): the rank of this machine (one per machine)\n dist_url (str): url to connect to for distributed training, including protocol\n e.g. \"tcp://127.0.0.1:8686\".\n Can be set to auto to automatically select a free port on localhost\n args (tuple): arguments passed to main_func\n \"\"\"\n world_size = num_machines * num_gpus_per_machine\n if world_size > 1:\n # https://github.com/pytorch/pytorch/pull/14391\n # TODO prctl in spawned processes\n\n if dist_url == \"auto\":\n assert (\n num_machines == 1\n ), \"dist_url=auto cannot work with distributed training.\"\n port = _find_free_port()\n dist_url = f\"tcp://127.0.0.1:{port}\"\n\n start_method = \"spawn\"\n cache = vars(args[1]).get(\"cache\", False)\n\n # To use numpy memmap for caching image into RAM, we have to use fork method\n if cache:\n assert sys.platform != \"win32\", (\n \"As Windows platform doesn't support fork method, \"\n \"do not add --cache in your training command.\"\n )\n start_method = \"fork\"\n\n mp.start_processes(\n _distributed_worker,\n nprocs=num_gpus_per_machine,\n args=(\n main_func,\n world_size,\n num_gpus_per_machine,\n machine_rank,\n backend,\n dist_url,\n args,\n ),\n daemon=False,\n start_method=start_method,\n )\n else:\n main_func(*args)\n\n\ndef _distributed_worker(\n local_rank,\n main_func,\n world_size,\n num_gpus_per_machine,\n machine_rank,\n backend,\n dist_url,\n args,\n timeout=DEFAULT_TIMEOUT,\n):\n assert (\n torch.cuda.is_available()\n ), \"cuda is not available. Please check your installation.\"\n global_rank = machine_rank * num_gpus_per_machine + local_rank\n logger.info(\"Rank {} initialization finished.\".format(global_rank))\n try:\n dist.init_process_group(\n backend=backend,\n init_method=dist_url,\n world_size=world_size,\n rank=global_rank,\n timeout=timeout,\n )\n except Exception:\n logger.error(\"Process group URL: {}\".format(dist_url))\n raise\n\n # Setup the local process group (which contains ranks within the same machine)\n assert comm._LOCAL_PROCESS_GROUP is None\n num_machines = world_size // num_gpus_per_machine\n for i in range(num_machines):\n ranks_on_i = list(\n range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine)\n )\n pg = dist.new_group(ranks_on_i)\n if i == machine_rank:\n comm._LOCAL_PROCESS_GROUP = pg\n\n # synchronize is needed here to prevent a possible timeout after calling init_process_group\n # See: https://github.com/facebookresearch/maskrcnn-benchmark/issues/172\n comm.synchronize()\n\n assert num_gpus_per_machine <= torch.cuda.device_count()\n torch.cuda.set_device(local_rank)\n\n main_func(*args)\n" }, { "path": "asone/detectors/yolox/yolox/core/trainer.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport datetime\nimport os\nimport time\nfrom loguru import logger\n\nimport torch\nfrom torch.nn.parallel import DistributedDataParallel as DDP\nfrom torch.utils.tensorboard import SummaryWriter\n\nfrom yolox.data import DataPrefetcher\nfrom yolox.exp import Exp\nfrom yolox.utils import (\n MeterBuffer,\n ModelEMA,\n WandbLogger,\n adjust_status,\n all_reduce_norm,\n get_local_rank,\n get_model_info,\n get_rank,\n get_world_size,\n gpu_mem_usage,\n is_parallel,\n load_ckpt,\n mem_usage,\n occupy_mem,\n save_checkpoint,\n setup_logger,\n synchronize\n)\n\n\nclass Trainer:\n def __init__(self, exp: Exp, args):\n # init function only defines some basic attr, other attrs like model, optimizer are built in\n # before_train methods.\n self.exp = exp\n self.args = args\n\n # training related attr\n self.max_epoch = exp.max_epoch\n self.amp_training = args.fp16\n self.scaler = torch.cuda.amp.GradScaler(enabled=args.fp16)\n self.is_distributed = get_world_size() > 1\n self.rank = get_rank()\n self.local_rank = get_local_rank()\n self.device = \"cuda:{}\".format(self.local_rank)\n self.use_model_ema = exp.ema\n self.save_history_ckpt = exp.save_history_ckpt\n\n # data/dataloader related attr\n self.data_type = torch.float16 if args.fp16 else torch.float32\n self.input_size = exp.input_size\n self.best_ap = 0\n\n # metric record\n self.meter = MeterBuffer(window_size=exp.print_interval)\n self.file_name = os.path.join(exp.output_dir, args.experiment_name)\n\n if self.rank == 0:\n os.makedirs(self.file_name, exist_ok=True)\n\n setup_logger(\n self.file_name,\n distributed_rank=self.rank,\n filename=\"train_log.txt\",\n mode=\"a\",\n )\n\n def train(self):\n self.before_train()\n try:\n self.train_in_epoch()\n except Exception:\n raise\n finally:\n self.after_train()\n\n def train_in_epoch(self):\n for self.epoch in range(self.start_epoch, self.max_epoch):\n self.before_epoch()\n self.train_in_iter()\n self.after_epoch()\n\n def train_in_iter(self):\n for self.iter in range(self.max_iter):\n self.before_iter()\n self.train_one_iter()\n self.after_iter()\n\n def train_one_iter(self):\n iter_start_time = time.time()\n\n inps, targets = self.prefetcher.next()\n inps = inps.to(self.data_type)\n targets = targets.to(self.data_type)\n targets.requires_grad = False\n inps, targets = self.exp.preprocess(inps, targets, self.input_size)\n data_end_time = time.time()\n\n with torch.cuda.amp.autocast(enabled=self.amp_training):\n outputs = self.model(inps, targets)\n\n loss = outputs[\"total_loss\"]\n\n self.optimizer.zero_grad()\n self.scaler.scale(loss).backward()\n self.scaler.step(self.optimizer)\n self.scaler.update()\n\n if self.use_model_ema:\n self.ema_model.update(self.model)\n\n lr = self.lr_scheduler.update_lr(self.progress_in_iter + 1)\n for param_group in self.optimizer.param_groups:\n param_group[\"lr\"] = lr\n\n iter_end_time = time.time()\n self.meter.update(\n iter_time=iter_end_time - iter_start_time,\n data_time=data_end_time - iter_start_time,\n lr=lr,\n **outputs,\n )\n\n def before_train(self):\n logger.info(\"args: {}\".format(self.args))\n logger.info(\"exp value:\\n{}\".format(self.exp))\n\n # model related init\n torch.cuda.set_device(self.local_rank)\n model = self.exp.get_model()\n logger.info(\n \"Model Summary: {}\".format(get_model_info(model, self.exp.test_size))\n )\n model.to(self.device)\n\n # solver related init\n self.optimizer = self.exp.get_optimizer(self.args.batch_size)\n\n # value of epoch will be set in `resume_train`\n model = self.resume_train(model)\n\n # data related init\n self.no_aug = self.start_epoch >= self.max_epoch - self.exp.no_aug_epochs\n self.train_loader = self.exp.get_data_loader(\n batch_size=self.args.batch_size,\n is_distributed=self.is_distributed,\n no_aug=self.no_aug,\n cache_img=self.args.cache,\n )\n logger.info(\"init prefetcher, this might take one minute or less...\")\n self.prefetcher = DataPrefetcher(self.train_loader)\n # max_iter means iters per epoch\n self.max_iter = len(self.train_loader)\n\n self.lr_scheduler = self.exp.get_lr_scheduler(\n self.exp.basic_lr_per_img * self.args.batch_size, self.max_iter\n )\n if self.args.occupy:\n occupy_mem(self.local_rank)\n\n if self.is_distributed:\n model = DDP(model, device_ids=[self.local_rank], broadcast_buffers=False)\n\n if self.use_model_ema:\n self.ema_model = ModelEMA(model, 0.9998)\n self.ema_model.updates = self.max_iter * self.start_epoch\n\n self.model = model\n\n self.evaluator = self.exp.get_evaluator(\n batch_size=self.args.batch_size, is_distributed=self.is_distributed\n )\n # Tensorboard and Wandb loggers\n if self.rank == 0:\n if self.args.logger == \"tensorboard\":\n self.tblogger = SummaryWriter(os.path.join(self.file_name, \"tensorboard\"))\n elif self.args.logger == \"wandb\":\n self.wandb_logger = WandbLogger.initialize_wandb_logger(\n self.args,\n self.exp,\n self.evaluator.dataloader.dataset\n )\n else:\n raise ValueError(\"logger must be either 'tensorboard' or 'wandb'\")\n\n logger.info(\"Training start...\")\n logger.info(\"\\n{}\".format(model))\n\n def after_train(self):\n logger.info(\n \"Training of experiment is done and the best AP is {:.2f}\".format(self.best_ap * 100)\n )\n if self.rank == 0:\n if self.args.logger == \"wandb\":\n self.wandb_logger.finish()\n\n def before_epoch(self):\n logger.info(\"---> start train epoch{}\".format(self.epoch + 1))\n\n if self.epoch + 1 == self.max_epoch - self.exp.no_aug_epochs or self.no_aug:\n logger.info(\"--->No mosaic aug now!\")\n self.train_loader.close_mosaic()\n logger.info(\"--->Add additional L1 loss now!\")\n if self.is_distributed:\n self.model.module.head.use_l1 = True\n else:\n self.model.head.use_l1 = True\n self.exp.eval_interval = 1\n if not self.no_aug:\n self.save_ckpt(ckpt_name=\"last_mosaic_epoch\")\n\n def after_epoch(self):\n self.save_ckpt(ckpt_name=\"latest\")\n\n if (self.epoch + 1) % self.exp.eval_interval == 0:\n all_reduce_norm(self.model)\n self.evaluate_and_save_model()\n\n def before_iter(self):\n pass\n\n def after_iter(self):\n \"\"\"\n `after_iter` contains two parts of logic:\n * log information\n * reset setting of resize\n \"\"\"\n # log needed information\n if (self.iter + 1) % self.exp.print_interval == 0:\n # TODO check ETA logic\n left_iters = self.max_iter * self.max_epoch - (self.progress_in_iter + 1)\n eta_seconds = self.meter[\"iter_time\"].global_avg * left_iters\n eta_str = \"ETA: {}\".format(datetime.timedelta(seconds=int(eta_seconds)))\n\n progress_str = \"epoch: {}/{}, iter: {}/{}\".format(\n self.epoch + 1, self.max_epoch, self.iter + 1, self.max_iter\n )\n loss_meter = self.meter.get_filtered_meter(\"loss\")\n loss_str = \", \".join(\n [\"{}: {:.1f}\".format(k, v.latest) for k, v in loss_meter.items()]\n )\n\n time_meter = self.meter.get_filtered_meter(\"time\")\n time_str = \", \".join(\n [\"{}: {:.3f}s\".format(k, v.avg) for k, v in time_meter.items()]\n )\n\n mem_str = \"gpu mem: {:.0f}Mb, mem: {:.1f}Gb\".format(gpu_mem_usage(), mem_usage())\n\n logger.info(\n \"{}, {}, {}, {}, lr: {:.3e}\".format(\n progress_str,\n mem_str,\n time_str,\n loss_str,\n self.meter[\"lr\"].latest,\n )\n + (\", size: {:d}, {}\".format(self.input_size[0], eta_str))\n )\n\n if self.rank == 0:\n if self.args.logger == \"tensorboard\":\n self.tblogger.add_scalar(\n \"train/lr\", self.meter[\"lr\"].latest, self.progress_in_iter)\n for k, v in loss_meter.items():\n self.tblogger.add_scalar(\n f\"train/{k}\", v.latest, self.progress_in_iter)\n if self.args.logger == \"wandb\":\n metrics = {\"train/\" + k: v.latest for k, v in loss_meter.items()}\n metrics.update({\n \"train/lr\": self.meter[\"lr\"].latest\n })\n self.wandb_logger.log_metrics(metrics, step=self.progress_in_iter)\n\n self.meter.clear_meters()\n\n # random resizing\n if (self.progress_in_iter + 1) % 10 == 0:\n self.input_size = self.exp.random_resize(\n self.train_loader, self.epoch, self.rank, self.is_distributed\n )\n\n @property\n def progress_in_iter(self):\n return self.epoch * self.max_iter + self.iter\n\n def resume_train(self, model):\n if self.args.resume:\n logger.info(\"resume training\")\n if self.args.ckpt is None:\n ckpt_file = os.path.join(self.file_name, \"latest\" + \"_ckpt.pth\")\n else:\n ckpt_file = self.args.ckpt\n\n ckpt = torch.load(ckpt_file, map_location=self.device)\n # resume the model/optimizer state dict\n model.load_state_dict(ckpt[\"model\"])\n self.optimizer.load_state_dict(ckpt[\"optimizer\"])\n self.best_ap = ckpt.pop(\"best_ap\", 0)\n # resume the training states variables\n start_epoch = (\n self.args.start_epoch - 1\n if self.args.start_epoch is not None\n else ckpt[\"start_epoch\"]\n )\n self.start_epoch = start_epoch\n logger.info(\n \"loaded checkpoint '{}' (epoch {})\".format(\n self.args.resume, self.start_epoch\n )\n ) # noqa\n else:\n if self.args.ckpt is not None:\n logger.info(\"loading checkpoint for fine tuning\")\n ckpt_file = self.args.ckpt\n ckpt = torch.load(ckpt_file, map_location=self.device)[\"model\"]\n model = load_ckpt(model, ckpt)\n self.start_epoch = 0\n\n return model\n\n def evaluate_and_save_model(self):\n if self.use_model_ema:\n evalmodel = self.ema_model.ema\n else:\n evalmodel = self.model\n if is_parallel(evalmodel):\n evalmodel = evalmodel.module\n\n with adjust_status(evalmodel, training=False):\n (ap50_95, ap50, summary), predictions = self.exp.eval(\n evalmodel, self.evaluator, self.is_distributed, return_outputs=True\n )\n\n update_best_ckpt = ap50_95 > self.best_ap\n self.best_ap = max(self.best_ap, ap50_95)\n\n if self.rank == 0:\n if self.args.logger == \"tensorboard\":\n self.tblogger.add_scalar(\"val/COCOAP50\", ap50, self.epoch + 1)\n self.tblogger.add_scalar(\"val/COCOAP50_95\", ap50_95, self.epoch + 1)\n if self.args.logger == \"wandb\":\n self.wandb_logger.log_metrics({\n \"val/COCOAP50\": ap50,\n \"val/COCOAP50_95\": ap50_95,\n \"train/epoch\": self.epoch + 1,\n })\n self.wandb_logger.log_images(predictions)\n logger.info(\"\\n\" + summary)\n synchronize()\n\n self.save_ckpt(\"last_epoch\", update_best_ckpt, ap=ap50_95)\n if self.save_history_ckpt:\n self.save_ckpt(f\"epoch_{self.epoch + 1}\", ap=ap50_95)\n\n def save_ckpt(self, ckpt_name, update_best_ckpt=False, ap=None):\n if self.rank == 0:\n save_model = self.ema_model.ema if self.use_model_ema else self.model\n logger.info(\"Save weights to {}\".format(self.file_name))\n ckpt_state = {\n \"start_epoch\": self.epoch + 1,\n \"model\": save_model.state_dict(),\n \"optimizer\": self.optimizer.state_dict(),\n \"best_ap\": self.best_ap,\n \"curr_ap\": ap,\n }\n save_checkpoint(\n ckpt_state,\n update_best_ckpt,\n self.file_name,\n ckpt_name,\n )\n\n if self.args.logger == \"wandb\":\n self.wandb_logger.save_checkpoint(\n self.file_name,\n ckpt_name,\n update_best_ckpt,\n metadata={\n \"epoch\": self.epoch + 1,\n \"optimizer\": self.optimizer.state_dict(),\n \"best_ap\": self.best_ap,\n \"curr_ap\": ap\n }\n )\n" }, { "path": "asone/detectors/yolox/yolox/data/__init__.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nfrom .data_augment import TrainTransform, ValTransform\nfrom .data_prefetcher import DataPrefetcher\nfrom .dataloading import DataLoader, get_yolox_datadir, worker_init_reset_seed\nfrom .datasets import *\nfrom .samplers import InfiniteSampler, YoloBatchSampler\n" }, { "path": "asone/detectors/yolox/yolox/data/data_augment.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\"\"\"\nData augmentation functionality. Passed as callable transformations to\nDataset classes.\n\nThe data augmentation procedures were interpreted from @weiliu89's SSD paper\nhttp://arxiv.org/abs/1512.02325\n\"\"\"\n\nimport math\nimport random\n\nimport cv2\nimport numpy as np\n\nfrom yolox.utils import xyxy2cxcywh\n\n\ndef augment_hsv(img, hgain=5, sgain=30, vgain=30):\n hsv_augs = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] # random gains\n hsv_augs *= np.random.randint(0, 2, 3) # random selection of h, s, v\n hsv_augs = hsv_augs.astype(np.int16)\n img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV).astype(np.int16)\n\n img_hsv[..., 0] = (img_hsv[..., 0] + hsv_augs[0]) % 180\n img_hsv[..., 1] = np.clip(img_hsv[..., 1] + hsv_augs[1], 0, 255)\n img_hsv[..., 2] = np.clip(img_hsv[..., 2] + hsv_augs[2], 0, 255)\n\n cv2.cvtColor(img_hsv.astype(img.dtype), cv2.COLOR_HSV2BGR, dst=img) # no return needed\n\n\ndef get_aug_params(value, center=0):\n if isinstance(value, float):\n return random.uniform(center - value, center + value)\n elif len(value) == 2:\n return random.uniform(value[0], value[1])\n else:\n raise ValueError(\n \"Affine params should be either a sequence containing two values\\\n or single float values. Got {}\".format(value)\n )\n\n\ndef get_affine_matrix(\n target_size,\n degrees=10,\n translate=0.1,\n scales=0.1,\n shear=10,\n):\n twidth, theight = target_size\n\n # Rotation and Scale\n angle = get_aug_params(degrees)\n scale = get_aug_params(scales, center=1.0)\n\n if scale <= 0.0:\n raise ValueError(\"Argument scale should be positive\")\n\n R = cv2.getRotationMatrix2D(angle=angle, center=(0, 0), scale=scale)\n\n M = np.ones([2, 3])\n # Shear\n shear_x = math.tan(get_aug_params(shear) * math.pi / 180)\n shear_y = math.tan(get_aug_params(shear) * math.pi / 180)\n\n M[0] = R[0] + shear_y * R[1]\n M[1] = R[1] + shear_x * R[0]\n\n # Translation\n translation_x = get_aug_params(translate) * twidth # x translation (pixels)\n translation_y = get_aug_params(translate) * theight # y translation (pixels)\n\n M[0, 2] = translation_x\n M[1, 2] = translation_y\n\n return M, scale\n\n\ndef apply_affine_to_bboxes(targets, target_size, M, scale):\n num_gts = len(targets)\n\n # warp corner points\n twidth, theight = target_size\n corner_points = np.ones((4 * num_gts, 3))\n corner_points[:, :2] = targets[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(\n 4 * num_gts, 2\n ) # x1y1, x2y2, x1y2, x2y1\n corner_points = corner_points @ M.T # apply affine transform\n corner_points = corner_points.reshape(num_gts, 8)\n\n # create new boxes\n corner_xs = corner_points[:, 0::2]\n corner_ys = corner_points[:, 1::2]\n new_bboxes = (\n np.concatenate(\n (corner_xs.min(1), corner_ys.min(1), corner_xs.max(1), corner_ys.max(1))\n )\n .reshape(4, num_gts)\n .T\n )\n\n # clip boxes\n new_bboxes[:, 0::2] = new_bboxes[:, 0::2].clip(0, twidth)\n new_bboxes[:, 1::2] = new_bboxes[:, 1::2].clip(0, theight)\n\n targets[:, :4] = new_bboxes\n\n return targets\n\n\ndef random_affine(\n img,\n targets=(),\n target_size=(640, 640),\n degrees=10,\n translate=0.1,\n scales=0.1,\n shear=10,\n):\n M, scale = get_affine_matrix(target_size, degrees, translate, scales, shear)\n\n img = cv2.warpAffine(img, M, dsize=target_size, borderValue=(114, 114, 114))\n\n # Transform label coordinates\n if len(targets) > 0:\n targets = apply_affine_to_bboxes(targets, target_size, M, scale)\n\n return img, targets\n\n\ndef _mirror(image, boxes, prob=0.5):\n _, width, _ = image.shape\n if random.random() < prob:\n image = image[:, ::-1]\n boxes[:, 0::2] = width - boxes[:, 2::-2]\n return image, boxes\n\n\ndef preproc(img, input_size, swap=(2, 0, 1)):\n if len(img.shape) == 3:\n padded_img = np.ones((input_size[0], input_size[1], 3), dtype=np.uint8) * 114\n else:\n padded_img = np.ones(input_size, dtype=np.uint8) * 114\n\n r = min(input_size[0] / img.shape[0], input_size[1] / img.shape[1])\n resized_img = cv2.resize(\n img,\n (int(img.shape[1] * r), int(img.shape[0] * r)),\n interpolation=cv2.INTER_LINEAR,\n ).astype(np.uint8)\n padded_img[: int(img.shape[0] * r), : int(img.shape[1] * r)] = resized_img\n\n padded_img = padded_img.transpose(swap)\n padded_img = np.ascontiguousarray(padded_img, dtype=np.float32)\n return padded_img, r\n\n\nclass TrainTransform:\n def __init__(self, max_labels=50, flip_prob=0.5, hsv_prob=1.0):\n self.max_labels = max_labels\n self.flip_prob = flip_prob\n self.hsv_prob = hsv_prob\n\n def __call__(self, image, targets, input_dim):\n boxes = targets[:, :4].copy()\n labels = targets[:, 4].copy()\n if len(boxes) == 0:\n targets = np.zeros((self.max_labels, 5), dtype=np.float32)\n image, r_o = preproc(image, input_dim)\n return image, targets\n\n image_o = image.copy()\n targets_o = targets.copy()\n height_o, width_o, _ = image_o.shape\n boxes_o = targets_o[:, :4]\n labels_o = targets_o[:, 4]\n # bbox_o: [xyxy] to [c_x,c_y,w,h]\n boxes_o = xyxy2cxcywh(boxes_o)\n\n if random.random() < self.hsv_prob:\n augment_hsv(image)\n image_t, boxes = _mirror(image, boxes, self.flip_prob)\n height, width, _ = image_t.shape\n image_t, r_ = preproc(image_t, input_dim)\n # boxes [xyxy] 2 [cx,cy,w,h]\n boxes = xyxy2cxcywh(boxes)\n boxes *= r_\n\n mask_b = np.minimum(boxes[:, 2], boxes[:, 3]) > 1\n boxes_t = boxes[mask_b]\n labels_t = labels[mask_b]\n\n if len(boxes_t) == 0:\n image_t, r_o = preproc(image_o, input_dim)\n boxes_o *= r_o\n boxes_t = boxes_o\n labels_t = labels_o\n\n labels_t = np.expand_dims(labels_t, 1)\n\n targets_t = np.hstack((labels_t, boxes_t))\n padded_labels = np.zeros((self.max_labels, 5))\n padded_labels[range(len(targets_t))[: self.max_labels]] = targets_t[\n : self.max_labels\n ]\n padded_labels = np.ascontiguousarray(padded_labels, dtype=np.float32)\n return image_t, padded_labels\n\n\nclass ValTransform:\n \"\"\"\n Defines the transformations that should be applied to test PIL image\n for input into the network\n\n dimension -> tensorize -> color adj\n\n Arguments:\n resize (int): input dimension to SSD\n rgb_means ((int,int,int)): average RGB of the dataset\n (104,117,123)\n swap ((int,int,int)): final order of channels\n\n Returns:\n transform (transform) : callable transform to be applied to test/val\n data\n \"\"\"\n\n def __init__(self, swap=(2, 0, 1), legacy=False):\n self.swap = swap\n self.legacy = legacy\n\n # assume input is cv2 img for now\n def __call__(self, img, res, input_size):\n img, _ = preproc(img, input_size, self.swap)\n if self.legacy:\n img = img[::-1, :, :].copy()\n img /= 255.0\n img -= np.array([0.485, 0.456, 0.406]).reshape(3, 1, 1)\n img /= np.array([0.229, 0.224, 0.225]).reshape(3, 1, 1)\n return img, np.zeros((1, 5))\n" }, { "path": "asone/detectors/yolox/yolox/data/data_prefetcher.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport torch\n\n\nclass DataPrefetcher:\n \"\"\"\n DataPrefetcher is inspired by code of following file:\n https://github.com/NVIDIA/apex/blob/master/examples/imagenet/main_amp.py\n It could speedup your pytorch dataloader. For more information, please check\n https://github.com/NVIDIA/apex/issues/304#issuecomment-493562789.\n \"\"\"\n\n def __init__(self, loader):\n self.loader = iter(loader)\n self.stream = torch.cuda.Stream()\n self.input_cuda = self._input_cuda_for_image\n self.record_stream = DataPrefetcher._record_stream_for_image\n self.preload()\n\n def preload(self):\n try:\n self.next_input, self.next_target, _, _ = next(self.loader)\n except StopIteration:\n self.next_input = None\n self.next_target = None\n return\n\n with torch.cuda.stream(self.stream):\n self.input_cuda()\n self.next_target = self.next_target.cuda(non_blocking=True)\n\n def next(self):\n torch.cuda.current_stream().wait_stream(self.stream)\n input = self.next_input\n target = self.next_target\n if input is not None:\n self.record_stream(input)\n if target is not None:\n target.record_stream(torch.cuda.current_stream())\n self.preload()\n return input, target\n\n def _input_cuda_for_image(self):\n self.next_input = self.next_input.cuda(non_blocking=True)\n\n @staticmethod\n def _record_stream_for_image(input):\n input.record_stream(torch.cuda.current_stream())\n" }, { "path": "asone/detectors/yolox/yolox/data/dataloading.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\nimport random\nimport uuid\n\nimport numpy as np\n\nimport torch\nfrom torch.utils.data.dataloader import DataLoader as torchDataLoader\nfrom torch.utils.data.dataloader import default_collate\n\nfrom .samplers import YoloBatchSampler\n\n\ndef get_yolox_datadir():\n \"\"\"\n get dataset dir of YOLOX. If environment variable named `YOLOX_DATADIR` is set,\n this function will return value of the environment variable. Otherwise, use data\n \"\"\"\n yolox_datadir = os.getenv(\"YOLOX_DATADIR\", None)\n if yolox_datadir is None:\n import yolox\n\n yolox_path = os.path.dirname(os.path.dirname(yolox.__file__))\n yolox_datadir = os.path.join(yolox_path, \"datasets\")\n return yolox_datadir\n\n\nclass DataLoader(torchDataLoader):\n \"\"\"\n Lightnet dataloader that enables on the fly resizing of the images.\n See :class:`torch.utils.data.DataLoader` for more information on the arguments.\n Check more on the following website:\n https://gitlab.com/EAVISE/lightnet/-/blob/master/lightnet/data/_dataloading.py\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self.__initialized = False\n shuffle = False\n batch_sampler = None\n if len(args) > 5:\n shuffle = args[2]\n sampler = args[3]\n batch_sampler = args[4]\n elif len(args) > 4:\n shuffle = args[2]\n sampler = args[3]\n if \"batch_sampler\" in kwargs:\n batch_sampler = kwargs[\"batch_sampler\"]\n elif len(args) > 3:\n shuffle = args[2]\n if \"sampler\" in kwargs:\n sampler = kwargs[\"sampler\"]\n if \"batch_sampler\" in kwargs:\n batch_sampler = kwargs[\"batch_sampler\"]\n else:\n if \"shuffle\" in kwargs:\n shuffle = kwargs[\"shuffle\"]\n if \"sampler\" in kwargs:\n sampler = kwargs[\"sampler\"]\n if \"batch_sampler\" in kwargs:\n batch_sampler = kwargs[\"batch_sampler\"]\n\n # Use custom BatchSampler\n if batch_sampler is None:\n if sampler is None:\n if shuffle:\n sampler = torch.utils.data.sampler.RandomSampler(self.dataset)\n # sampler = torch.utils.data.DistributedSampler(self.dataset)\n else:\n sampler = torch.utils.data.sampler.SequentialSampler(self.dataset)\n batch_sampler = YoloBatchSampler(\n sampler,\n self.batch_size,\n self.drop_last,\n input_dimension=self.dataset.input_dim,\n )\n # batch_sampler = IterationBasedBatchSampler(batch_sampler, num_iterations =\n\n self.batch_sampler = batch_sampler\n\n self.__initialized = True\n\n def close_mosaic(self):\n self.batch_sampler.mosaic = False\n\n\ndef list_collate(batch):\n \"\"\"\n Function that collates lists or tuples together into one list (of lists/tuples).\n Use this as the collate function in a Dataloader, if you want to have a list of\n items as an output, as opposed to tensors (eg. Brambox.boxes).\n \"\"\"\n items = list(zip(*batch))\n\n for i in range(len(items)):\n if isinstance(items[i][0], (list, tuple)):\n items[i] = list(items[i])\n else:\n items[i] = default_collate(items[i])\n\n return items\n\n\ndef worker_init_reset_seed(worker_id):\n seed = uuid.uuid4().int % 2**32\n random.seed(seed)\n torch.set_rng_state(torch.manual_seed(seed).get_state())\n np.random.seed(seed)\n" }, { "path": "asone/detectors/yolox/yolox/data/datasets/__init__.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nfrom .coco import COCODataset\nfrom .coco_classes import COCO_CLASSES\nfrom .datasets_wrapper import CacheDataset, ConcatDataset, Dataset, MixConcatDataset\nfrom .mosaicdetection import MosaicDetection\nfrom .voc import VOCDetection\n" }, { "path": "asone/detectors/yolox/yolox/data/datasets/coco.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\nimport copy\nimport os\n\nimport cv2\nimport numpy as np\nfrom pycocotools.coco import COCO\n\nfrom ..dataloading import get_yolox_datadir\nfrom .datasets_wrapper import CacheDataset, cache_read_img\n\n\ndef remove_useless_info(coco):\n \"\"\"\n Remove useless info in coco dataset. COCO object is modified inplace.\n This function is mainly used for saving memory (save about 30% mem).\n \"\"\"\n if isinstance(coco, COCO):\n dataset = coco.dataset\n dataset.pop(\"info\", None)\n dataset.pop(\"licenses\", None)\n for img in dataset[\"images\"]:\n img.pop(\"license\", None)\n img.pop(\"coco_url\", None)\n img.pop(\"date_captured\", None)\n img.pop(\"flickr_url\", None)\n if \"annotations\" in coco.dataset:\n for anno in coco.dataset[\"annotations\"]:\n anno.pop(\"segmentation\", None)\n\n\nclass COCODataset(CacheDataset):\n \"\"\"\n COCO dataset class.\n \"\"\"\n\n def __init__(\n self,\n data_dir=None,\n json_file=\"instances_train2017.json\",\n name=\"train2017\",\n img_size=(416, 416),\n preproc=None,\n cache=False,\n cache_type=\"ram\",\n ):\n \"\"\"\n COCO dataset initialization. Annotation data are read into memory by COCO API.\n Args:\n data_dir (str): dataset root directory\n json_file (str): COCO json file name\n name (str): COCO data name (e.g. 'train2017' or 'val2017')\n img_size (int): target image size after pre-processing\n preproc: data augmentation strategy\n \"\"\"\n if data_dir is None:\n data_dir = os.path.join(get_yolox_datadir(), \"COCO\")\n self.data_dir = data_dir\n self.json_file = json_file\n\n self.coco = COCO(os.path.join(self.data_dir, \"annotations\", self.json_file))\n remove_useless_info(self.coco)\n self.ids = self.coco.getImgIds()\n self.num_imgs = len(self.ids)\n self.class_ids = sorted(self.coco.getCatIds())\n self.cats = self.coco.loadCats(self.coco.getCatIds())\n self._classes = tuple([c[\"name\"] for c in self.cats])\n self.name = name\n self.img_size = img_size\n self.preproc = preproc\n self.annotations = self._load_coco_annotations()\n\n path_filename = [os.path.join(name, anno[3]) for anno in self.annotations]\n super().__init__(\n input_dimension=img_size,\n num_imgs=self.num_imgs,\n data_dir=data_dir,\n cache_dir_name=f\"cache_{name}\",\n path_filename=path_filename,\n cache=cache,\n cache_type=cache_type\n )\n\n def __len__(self):\n return self.num_imgs\n\n def _load_coco_annotations(self):\n return [self.load_anno_from_ids(_ids) for _ids in self.ids]\n\n def load_anno_from_ids(self, id_):\n im_ann = self.coco.loadImgs(id_)[0]\n width = im_ann[\"width\"]\n height = im_ann[\"height\"]\n anno_ids = self.coco.getAnnIds(imgIds=[int(id_)], iscrowd=False)\n annotations = self.coco.loadAnns(anno_ids)\n objs = []\n for obj in annotations:\n x1 = np.max((0, obj[\"bbox\"][0]))\n y1 = np.max((0, obj[\"bbox\"][1]))\n x2 = np.min((width, x1 + np.max((0, obj[\"bbox\"][2]))))\n y2 = np.min((height, y1 + np.max((0, obj[\"bbox\"][3]))))\n if obj[\"area\"] > 0 and x2 >= x1 and y2 >= y1:\n obj[\"clean_bbox\"] = [x1, y1, x2, y2]\n objs.append(obj)\n\n num_objs = len(objs)\n\n res = np.zeros((num_objs, 5))\n for ix, obj in enumerate(objs):\n cls = self.class_ids.index(obj[\"category_id\"])\n res[ix, 0:4] = obj[\"clean_bbox\"]\n res[ix, 4] = cls\n\n r = min(self.img_size[0] / height, self.img_size[1] / width)\n res[:, :4] *= r\n\n img_info = (height, width)\n resized_info = (int(height * r), int(width * r))\n\n file_name = (\n im_ann[\"file_name\"]\n if \"file_name\" in im_ann\n else \"{:012}\".format(id_) + \".jpg\"\n )\n\n return (res, img_info, resized_info, file_name)\n\n def load_anno(self, index):\n return self.annotations[index][0]\n\n def load_resized_img(self, index):\n img = self.load_image(index)\n r = min(self.img_size[0] / img.shape[0], self.img_size[1] / img.shape[1])\n resized_img = cv2.resize(\n img,\n (int(img.shape[1] * r), int(img.shape[0] * r)),\n interpolation=cv2.INTER_LINEAR,\n ).astype(np.uint8)\n return resized_img\n\n def load_image(self, index):\n file_name = self.annotations[index][3]\n\n img_file = os.path.join(self.data_dir, self.name, file_name)\n\n img = cv2.imread(img_file)\n assert img is not None, f\"file named {img_file} not found\"\n\n return img\n\n @cache_read_img(use_cache=True)\n def read_img(self, index):\n return self.load_resized_img(index)\n\n def pull_item(self, index):\n id_ = self.ids[index]\n label, origin_image_size, _, _ = self.annotations[index]\n img = self.read_img(index)\n\n return img, copy.deepcopy(label), origin_image_size, np.array([id_])\n\n @CacheDataset.mosaic_getitem\n def __getitem__(self, index):\n \"\"\"\n One image / label pair for the given index is picked up and pre-processed.\n\n Args:\n index (int): data index\n\n Returns:\n img (numpy.ndarray): pre-processed image\n padded_labels (torch.Tensor): pre-processed label data.\n The shape is :math:`[max_labels, 5]`.\n each label consists of [class, xc, yc, w, h]:\n class (float): class index.\n xc, yc (float) : center of bbox whose values range from 0 to 1.\n w, h (float) : size of bbox whose values range from 0 to 1.\n info_img : tuple of h, w.\n h, w (int): original shape of the image\n img_id (int): same as the input index. Used for evaluation.\n \"\"\"\n img, target, img_info, img_id = self.pull_item(index)\n\n if self.preproc is not None:\n img, target = self.preproc(img, target, self.input_dim)\n return img, target, img_info, img_id\n" }, { "path": "asone/detectors/yolox/yolox/data/datasets/coco_classes.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nCOCO_CLASSES = (\n \"person\",\n \"bicycle\",\n \"car\",\n \"motorcycle\",\n \"airplane\",\n \"bus\",\n \"train\",\n \"truck\",\n \"boat\",\n \"traffic light\",\n \"fire hydrant\",\n \"stop sign\",\n \"parking meter\",\n \"bench\",\n \"bird\",\n \"cat\",\n \"dog\",\n \"horse\",\n \"sheep\",\n \"cow\",\n \"elephant\",\n \"bear\",\n \"zebra\",\n \"giraffe\",\n \"backpack\",\n \"umbrella\",\n \"handbag\",\n \"tie\",\n \"suitcase\",\n \"frisbee\",\n \"skis\",\n \"snowboard\",\n \"sports ball\",\n \"kite\",\n \"baseball bat\",\n \"baseball glove\",\n \"skateboard\",\n \"surfboard\",\n \"tennis racket\",\n \"bottle\",\n \"wine glass\",\n \"cup\",\n \"fork\",\n \"knife\",\n \"spoon\",\n \"bowl\",\n \"banana\",\n \"apple\",\n \"sandwich\",\n \"orange\",\n \"broccoli\",\n \"carrot\",\n \"hot dog\",\n \"pizza\",\n \"donut\",\n \"cake\",\n \"chair\",\n \"couch\",\n \"potted plant\",\n \"bed\",\n \"dining table\",\n \"toilet\",\n \"tv\",\n \"laptop\",\n \"mouse\",\n \"remote\",\n \"keyboard\",\n \"cell phone\",\n \"microwave\",\n \"oven\",\n \"toaster\",\n \"sink\",\n \"refrigerator\",\n \"book\",\n \"clock\",\n \"vase\",\n \"scissors\",\n \"teddy bear\",\n \"hair drier\",\n \"toothbrush\",\n)\n" }, { "path": "asone/detectors/yolox/yolox/data/datasets/datasets_wrapper.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport bisect\nimport copy\nimport os\nimport random\nfrom abc import ABCMeta, abstractmethod\nfrom functools import partial, wraps\nfrom multiprocessing.pool import ThreadPool\nimport psutil\nfrom loguru import logger\nfrom tqdm import tqdm\n\nimport numpy as np\n\nfrom torch.utils.data.dataset import ConcatDataset as torchConcatDataset\nfrom torch.utils.data.dataset import Dataset as torchDataset\n\n\nclass ConcatDataset(torchConcatDataset):\n def __init__(self, datasets):\n super(ConcatDataset, self).__init__(datasets)\n if hasattr(self.datasets[0], \"input_dim\"):\n self._input_dim = self.datasets[0].input_dim\n self.input_dim = self.datasets[0].input_dim\n\n def pull_item(self, idx):\n if idx < 0:\n if -idx > len(self):\n raise ValueError(\n \"absolute value of index should not exceed dataset length\"\n )\n idx = len(self) + idx\n dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx)\n if dataset_idx == 0:\n sample_idx = idx\n else:\n sample_idx = idx - self.cumulative_sizes[dataset_idx - 1]\n return self.datasets[dataset_idx].pull_item(sample_idx)\n\n\nclass MixConcatDataset(torchConcatDataset):\n def __init__(self, datasets):\n super(MixConcatDataset, self).__init__(datasets)\n if hasattr(self.datasets[0], \"input_dim\"):\n self._input_dim = self.datasets[0].input_dim\n self.input_dim = self.datasets[0].input_dim\n\n def __getitem__(self, index):\n\n if not isinstance(index, int):\n idx = index[1]\n if idx < 0:\n if -idx > len(self):\n raise ValueError(\n \"absolute value of index should not exceed dataset length\"\n )\n idx = len(self) + idx\n dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx)\n if dataset_idx == 0:\n sample_idx = idx\n else:\n sample_idx = idx - self.cumulative_sizes[dataset_idx - 1]\n if not isinstance(index, int):\n index = (index[0], sample_idx, index[2])\n\n return self.datasets[dataset_idx][index]\n\n\nclass Dataset(torchDataset):\n \"\"\" This class is a subclass of the base :class:`torch.utils.data.Dataset`,\n that enables on the fly resizing of the ``input_dim``.\n\n Args:\n input_dimension (tuple): (width,height) tuple with default dimensions of the network\n \"\"\"\n\n def __init__(self, input_dimension, mosaic=True):\n super().__init__()\n self.__input_dim = input_dimension[:2]\n self.enable_mosaic = mosaic\n\n @property\n def input_dim(self):\n \"\"\"\n Dimension that can be used by transforms to set the correct image size, etc.\n This allows transforms to have a single source of truth\n for the input dimension of the network.\n\n Return:\n list: Tuple containing the current width,height\n \"\"\"\n if hasattr(self, \"_input_dim\"):\n return self._input_dim\n return self.__input_dim\n\n @staticmethod\n def mosaic_getitem(getitem_fn):\n \"\"\"\n Decorator method that needs to be used around the ``__getitem__`` method. |br|\n This decorator enables the closing mosaic\n\n Example:\n >>> class CustomSet(ln.data.Dataset):\n ... def __len__(self):\n ... return 10\n ... @ln.data.Dataset.mosaic_getitem\n ... def __getitem__(self, index):\n ... return self.enable_mosaic\n \"\"\"\n\n @wraps(getitem_fn)\n def wrapper(self, index):\n if not isinstance(index, int):\n self.enable_mosaic = index[0]\n index = index[1]\n\n ret_val = getitem_fn(self, index)\n\n return ret_val\n\n return wrapper\n\n\nclass CacheDataset(Dataset, metaclass=ABCMeta):\n \"\"\" This class is a subclass of the base :class:`yolox.data.datasets.Dataset`,\n that enables cache images to ram or disk.\n\n Args:\n input_dimension (tuple): (width,height) tuple with default dimensions of the network\n num_imgs (int): datset size\n data_dir (str): the root directory of the dataset, e.g. `/path/to/COCO`.\n cache_dir_name (str): the name of the directory to cache to disk,\n e.g. `\"custom_cache\"`. The files cached to disk will be saved\n under `/path/to/COCO/custom_cache`.\n path_filename (str): a list of paths to the data relative to the `data_dir`,\n e.g. if you have data `/path/to/COCO/train/1.jpg`, `/path/to/COCO/train/2.jpg`,\n then `path_filename = ['train/1.jpg', ' train/2.jpg']`.\n cache (bool): whether to cache the images to ram or disk.\n cache_type (str): the type of cache,\n \"ram\" : Caching imgs to ram for fast training.\n \"disk\": Caching imgs to disk for fast training.\n \"\"\"\n\n def __init__(\n self,\n input_dimension,\n num_imgs=None,\n data_dir=None,\n cache_dir_name=None,\n path_filename=None,\n cache=False,\n cache_type=\"ram\",\n ):\n super().__init__(input_dimension)\n self.cache = cache\n self.cache_type = cache_type\n\n if self.cache and self.cache_type == \"disk\":\n self.cache_dir = os.path.join(data_dir, cache_dir_name)\n self.path_filename = path_filename\n\n if self.cache and self.cache_type == \"ram\":\n self.imgs = None\n\n if self.cache:\n self.cache_images(\n num_imgs=num_imgs,\n data_dir=data_dir,\n cache_dir_name=cache_dir_name,\n path_filename=path_filename,\n )\n\n def __del__(self):\n if self.cache and self.cache_type == \"ram\":\n del self.imgs\n\n @abstractmethod\n def read_img(self, index):\n \"\"\"\n Given index, return the corresponding image\n\n Args:\n index (int): image index\n \"\"\"\n raise NotImplementedError\n\n def cache_images(\n self,\n num_imgs=None,\n data_dir=None,\n cache_dir_name=None,\n path_filename=None,\n ):\n assert num_imgs is not None, \"num_imgs must be specified as the size of the dataset\"\n if self.cache_type == \"disk\":\n assert (data_dir and cache_dir_name and path_filename) is not None, \\\n \"data_dir, cache_name and path_filename must be specified if cache_type is disk\"\n self.path_filename = path_filename\n\n mem = psutil.virtual_memory()\n mem_required = self.cal_cache_occupy(num_imgs)\n gb = 1 << 30\n\n if self.cache_type == \"ram\":\n if mem_required > mem.available:\n self.cache = False\n else:\n logger.info(\n f\"{mem_required / gb:.1f}GB RAM required, \"\n f\"{mem.available / gb:.1f}/{mem.total / gb:.1f}GB RAM available, \"\n f\"Since the first thing we do is cache, \"\n f\"there is no guarantee that the remaining memory space is sufficient\"\n )\n\n if self.cache and self.imgs is None:\n if self.cache_type == 'ram':\n self.imgs = [None] * num_imgs\n logger.info(\"You are using cached images in RAM to accelerate training!\")\n else: # 'disk'\n if not os.path.exists(self.cache_dir):\n os.mkdir(self.cache_dir)\n logger.warning(\n f\"\\n*******************************************************************\\n\"\n f\"You are using cached images in DISK to accelerate training.\\n\"\n f\"This requires large DISK space.\\n\"\n f\"Make sure you have {mem_required / gb:.1f} \"\n f\"available DISK space for training your dataset.\\n\"\n f\"*******************************************************************\\\\n\"\n )\n else:\n logger.info(f\"Found disk cache at {self.cache_dir}\")\n return\n\n logger.info(\n \"Caching images...\\n\"\n \"This might take some time for your dataset\"\n )\n\n num_threads = min(8, max(1, os.cpu_count() - 1))\n b = 0\n load_imgs = ThreadPool(num_threads).imap(\n partial(self.read_img, use_cache=False),\n range(num_imgs)\n )\n pbar = tqdm(enumerate(load_imgs), total=num_imgs)\n for i, x in pbar: # x = self.read_img(self, i, use_cache=False)\n if self.cache_type == 'ram':\n self.imgs[i] = x\n else: # 'disk'\n cache_filename = f'{self.path_filename[i].split(\".\")[0]}.npy'\n cache_path_filename = os.path.join(self.cache_dir, cache_filename)\n os.makedirs(os.path.dirname(cache_path_filename), exist_ok=True)\n np.save(cache_path_filename, x)\n b += x.nbytes\n pbar.desc = \\\n f'Caching images ({b / gb:.1f}/{mem_required / gb:.1f}GB {self.cache_type})'\n pbar.close()\n\n def cal_cache_occupy(self, num_imgs):\n cache_bytes = 0\n num_samples = min(num_imgs, 32)\n for _ in range(num_samples):\n img = self.read_img(index=random.randint(0, num_imgs - 1), use_cache=False)\n cache_bytes += img.nbytes\n mem_required = cache_bytes * num_imgs / num_samples\n return mem_required\n\n\ndef cache_read_img(use_cache=True):\n def decorator(read_img_fn):\n \"\"\"\n Decorate the read_img function to cache the image\n\n Args:\n read_img_fn: read_img function\n use_cache (bool, optional): For the decorated read_img function,\n whether to read the image from cache.\n Defaults to True.\n \"\"\"\n @wraps(read_img_fn)\n def wrapper(self, index, use_cache=use_cache):\n cache = self.cache and use_cache\n if cache:\n if self.cache_type == \"ram\":\n img = self.imgs[index]\n img = copy.deepcopy(img)\n elif self.cache_type == \"disk\":\n img = np.load(\n os.path.join(\n self.cache_dir, f\"{self.path_filename[index].split('.')[0]}.npy\"))\n else:\n raise ValueError(f\"Unknown cache type: {self.cache_type}\")\n else:\n img = read_img_fn(self, index)\n return img\n return wrapper\n return decorator\n" }, { "path": "asone/detectors/yolox/yolox/data/datasets/mosaicdetection.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport random\n\nimport cv2\nimport numpy as np\n\nfrom yolox.utils import adjust_box_anns, get_local_rank\n\nfrom ..data_augment import random_affine\nfrom .datasets_wrapper import Dataset\n\n\ndef get_mosaic_coordinate(mosaic_image, mosaic_index, xc, yc, w, h, input_h, input_w):\n # TODO update doc\n # index0 to top left part of image\n if mosaic_index == 0:\n x1, y1, x2, y2 = max(xc - w, 0), max(yc - h, 0), xc, yc\n small_coord = w - (x2 - x1), h - (y2 - y1), w, h\n # index1 to top right part of image\n elif mosaic_index == 1:\n x1, y1, x2, y2 = xc, max(yc - h, 0), min(xc + w, input_w * 2), yc\n small_coord = 0, h - (y2 - y1), min(w, x2 - x1), h\n # index2 to bottom left part of image\n elif mosaic_index == 2:\n x1, y1, x2, y2 = max(xc - w, 0), yc, xc, min(input_h * 2, yc + h)\n small_coord = w - (x2 - x1), 0, w, min(y2 - y1, h)\n # index2 to bottom right part of image\n elif mosaic_index == 3:\n x1, y1, x2, y2 = xc, yc, min(xc + w, input_w * 2), min(input_h * 2, yc + h) # noqa\n small_coord = 0, 0, min(w, x2 - x1), min(y2 - y1, h)\n return (x1, y1, x2, y2), small_coord\n\n\nclass MosaicDetection(Dataset):\n \"\"\"Detection dataset wrapper that performs mixup for normal dataset.\"\"\"\n\n def __init__(\n self, dataset, img_size, mosaic=True, preproc=None,\n degrees=10.0, translate=0.1, mosaic_scale=(0.5, 1.5),\n mixup_scale=(0.5, 1.5), shear=2.0, enable_mixup=True,\n mosaic_prob=1.0, mixup_prob=1.0, *args\n ):\n \"\"\"\n\n Args:\n dataset(Dataset) : Pytorch dataset object.\n img_size (tuple):\n mosaic (bool): enable mosaic augmentation or not.\n preproc (func):\n degrees (float):\n translate (float):\n mosaic_scale (tuple):\n mixup_scale (tuple):\n shear (float):\n enable_mixup (bool):\n *args(tuple) : Additional arguments for mixup random sampler.\n \"\"\"\n super().__init__(img_size, mosaic=mosaic)\n self._dataset = dataset\n self.preproc = preproc\n self.degrees = degrees\n self.translate = translate\n self.scale = mosaic_scale\n self.shear = shear\n self.mixup_scale = mixup_scale\n self.enable_mosaic = mosaic\n self.enable_mixup = enable_mixup\n self.mosaic_prob = mosaic_prob\n self.mixup_prob = mixup_prob\n self.local_rank = get_local_rank()\n\n def __len__(self):\n return len(self._dataset)\n\n @Dataset.mosaic_getitem\n def __getitem__(self, idx):\n if self.enable_mosaic and random.random() < self.mosaic_prob:\n mosaic_labels = []\n input_dim = self._dataset.input_dim\n input_h, input_w = input_dim[0], input_dim[1]\n\n # yc, xc = s, s # mosaic center x, y\n yc = int(random.uniform(0.5 * input_h, 1.5 * input_h))\n xc = int(random.uniform(0.5 * input_w, 1.5 * input_w))\n\n # 3 additional image indices\n indices = [idx] + [random.randint(0, len(self._dataset) - 1) for _ in range(3)]\n\n for i_mosaic, index in enumerate(indices):\n img, _labels, _, img_id = self._dataset.pull_item(index)\n h0, w0 = img.shape[:2] # orig hw\n scale = min(1. * input_h / h0, 1. * input_w / w0)\n img = cv2.resize(\n img, (int(w0 * scale), int(h0 * scale)), interpolation=cv2.INTER_LINEAR\n )\n # generate output mosaic image\n (h, w, c) = img.shape[:3]\n if i_mosaic == 0:\n mosaic_img = np.full((input_h * 2, input_w * 2, c), 114, dtype=np.uint8)\n\n # suffix l means large image, while s means small image in mosaic aug.\n (l_x1, l_y1, l_x2, l_y2), (s_x1, s_y1, s_x2, s_y2) = get_mosaic_coordinate(\n mosaic_img, i_mosaic, xc, yc, w, h, input_h, input_w\n )\n\n mosaic_img[l_y1:l_y2, l_x1:l_x2] = img[s_y1:s_y2, s_x1:s_x2]\n padw, padh = l_x1 - s_x1, l_y1 - s_y1\n\n labels = _labels.copy()\n # Normalized xywh to pixel xyxy format\n if _labels.size > 0:\n labels[:, 0] = scale * _labels[:, 0] + padw\n labels[:, 1] = scale * _labels[:, 1] + padh\n labels[:, 2] = scale * _labels[:, 2] + padw\n labels[:, 3] = scale * _labels[:, 3] + padh\n mosaic_labels.append(labels)\n\n if len(mosaic_labels):\n mosaic_labels = np.concatenate(mosaic_labels, 0)\n np.clip(mosaic_labels[:, 0], 0, 2 * input_w, out=mosaic_labels[:, 0])\n np.clip(mosaic_labels[:, 1], 0, 2 * input_h, out=mosaic_labels[:, 1])\n np.clip(mosaic_labels[:, 2], 0, 2 * input_w, out=mosaic_labels[:, 2])\n np.clip(mosaic_labels[:, 3], 0, 2 * input_h, out=mosaic_labels[:, 3])\n\n mosaic_img, mosaic_labels = random_affine(\n mosaic_img,\n mosaic_labels,\n target_size=(input_w, input_h),\n degrees=self.degrees,\n translate=self.translate,\n scales=self.scale,\n shear=self.shear,\n )\n\n # -----------------------------------------------------------------\n # CopyPaste: https://arxiv.org/abs/2012.07177\n # -----------------------------------------------------------------\n if (\n self.enable_mixup\n and not len(mosaic_labels) == 0\n and random.random() < self.mixup_prob\n ):\n mosaic_img, mosaic_labels = self.mixup(mosaic_img, mosaic_labels, self.input_dim)\n mix_img, padded_labels = self.preproc(mosaic_img, mosaic_labels, self.input_dim)\n img_info = (mix_img.shape[1], mix_img.shape[0])\n\n # -----------------------------------------------------------------\n # img_info and img_id are not used for training.\n # They are also hard to be specified on a mosaic image.\n # -----------------------------------------------------------------\n return mix_img, padded_labels, img_info, img_id\n\n else:\n self._dataset._input_dim = self.input_dim\n img, label, img_info, img_id = self._dataset.pull_item(idx)\n img, label = self.preproc(img, label, self.input_dim)\n return img, label, img_info, img_id\n\n def mixup(self, origin_img, origin_labels, input_dim):\n jit_factor = random.uniform(*self.mixup_scale)\n FLIP = random.uniform(0, 1) > 0.5\n cp_labels = []\n while len(cp_labels) == 0:\n cp_index = random.randint(0, self.__len__() - 1)\n cp_labels = self._dataset.load_anno(cp_index)\n img, cp_labels, _, _ = self._dataset.pull_item(cp_index)\n\n if len(img.shape) == 3:\n cp_img = np.ones((input_dim[0], input_dim[1], 3), dtype=np.uint8) * 114\n else:\n cp_img = np.ones(input_dim, dtype=np.uint8) * 114\n\n cp_scale_ratio = min(input_dim[0] / img.shape[0], input_dim[1] / img.shape[1])\n resized_img = cv2.resize(\n img,\n (int(img.shape[1] * cp_scale_ratio), int(img.shape[0] * cp_scale_ratio)),\n interpolation=cv2.INTER_LINEAR,\n )\n\n cp_img[\n : int(img.shape[0] * cp_scale_ratio), : int(img.shape[1] * cp_scale_ratio)\n ] = resized_img\n\n cp_img = cv2.resize(\n cp_img,\n (int(cp_img.shape[1] * jit_factor), int(cp_img.shape[0] * jit_factor)),\n )\n cp_scale_ratio *= jit_factor\n\n if FLIP:\n cp_img = cp_img[:, ::-1, :]\n\n origin_h, origin_w = cp_img.shape[:2]\n target_h, target_w = origin_img.shape[:2]\n padded_img = np.zeros(\n (max(origin_h, target_h), max(origin_w, target_w), 3), dtype=np.uint8\n )\n padded_img[:origin_h, :origin_w] = cp_img\n\n x_offset, y_offset = 0, 0\n if padded_img.shape[0] > target_h:\n y_offset = random.randint(0, padded_img.shape[0] - target_h - 1)\n if padded_img.shape[1] > target_w:\n x_offset = random.randint(0, padded_img.shape[1] - target_w - 1)\n padded_cropped_img = padded_img[\n y_offset: y_offset + target_h, x_offset: x_offset + target_w\n ]\n\n cp_bboxes_origin_np = adjust_box_anns(\n cp_labels[:, :4].copy(), cp_scale_ratio, 0, 0, origin_w, origin_h\n )\n if FLIP:\n cp_bboxes_origin_np[:, 0::2] = (\n origin_w - cp_bboxes_origin_np[:, 0::2][:, ::-1]\n )\n cp_bboxes_transformed_np = cp_bboxes_origin_np.copy()\n cp_bboxes_transformed_np[:, 0::2] = np.clip(\n cp_bboxes_transformed_np[:, 0::2] - x_offset, 0, target_w\n )\n cp_bboxes_transformed_np[:, 1::2] = np.clip(\n cp_bboxes_transformed_np[:, 1::2] - y_offset, 0, target_h\n )\n\n cls_labels = cp_labels[:, 4:5].copy()\n box_labels = cp_bboxes_transformed_np\n labels = np.hstack((box_labels, cls_labels))\n origin_labels = np.vstack((origin_labels, labels))\n origin_img = origin_img.astype(np.float32)\n origin_img = 0.5 * origin_img + 0.5 * padded_cropped_img.astype(np.float32)\n\n return origin_img.astype(np.uint8), origin_labels\n" }, { "path": "asone/detectors/yolox/yolox/data/datasets/voc.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Code are based on\n# https://github.com/fmassa/vision/blob/voc_dataset/torchvision/datasets/voc.py\n# Copyright (c) Francisco Massa.\n# Copyright (c) Ellis Brown, Max deGroot.\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\nimport os.path\nimport pickle\nimport xml.etree.ElementTree as ET\n\nimport cv2\nimport numpy as np\n\nfrom yolox.evaluators.voc_eval import voc_eval\n\nfrom .datasets_wrapper import CacheDataset, cache_read_img\nfrom .voc_classes import VOC_CLASSES\n\n\nclass AnnotationTransform(object):\n\n \"\"\"Transforms a VOC annotation into a Tensor of bbox coords and label index\n Initilized with a dictionary lookup of classnames to indexes\n\n Arguments:\n class_to_ind (dict, optional): dictionary lookup of classnames -> indexes\n (default: alphabetic indexing of VOC's 20 classes)\n keep_difficult (bool, optional): keep difficult instances or not\n (default: False)\n height (int): height\n width (int): width\n \"\"\"\n\n def __init__(self, class_to_ind=None, keep_difficult=True):\n self.class_to_ind = class_to_ind or dict(\n zip(VOC_CLASSES, range(len(VOC_CLASSES)))\n )\n self.keep_difficult = keep_difficult\n\n def __call__(self, target):\n \"\"\"\n Arguments:\n target (annotation) : the target annotation to be made usable\n will be an ET.Element\n Returns:\n a list containing lists of bounding boxes [bbox coords, class name]\n \"\"\"\n res = np.empty((0, 5))\n for obj in target.iter(\"object\"):\n difficult = obj.find(\"difficult\")\n if difficult is not None:\n difficult = int(difficult.text) == 1\n else:\n difficult = False\n if not self.keep_difficult and difficult:\n continue\n name = obj.find(\"name\").text.strip()\n bbox = obj.find(\"bndbox\")\n\n pts = [\"xmin\", \"ymin\", \"xmax\", \"ymax\"]\n bndbox = []\n for i, pt in enumerate(pts):\n cur_pt = int(float(bbox.find(pt).text)) - 1\n # scale height or width\n # cur_pt = cur_pt / width if i % 2 == 0 else cur_pt / height\n bndbox.append(cur_pt)\n label_idx = self.class_to_ind[name]\n bndbox.append(label_idx)\n res = np.vstack((res, bndbox)) # [xmin, ymin, xmax, ymax, label_ind]\n # img_id = target.find('filename').text[:-4]\n\n width = int(target.find(\"size\").find(\"width\").text)\n height = int(target.find(\"size\").find(\"height\").text)\n img_info = (height, width)\n\n return res, img_info\n\n\nclass VOCDetection(CacheDataset):\n\n \"\"\"\n VOC Detection Dataset Object\n\n input is image, target is annotation\n\n Args:\n root (string): filepath to VOCdevkit folder.\n image_set (string): imageset to use (eg. 'train', 'val', 'test')\n transform (callable, optional): transformation to perform on the\n input image\n target_transform (callable, optional): transformation to perform on the\n target `annotation`\n (eg: take in caption string, return tensor of word indices)\n dataset_name (string, optional): which dataset to load\n (default: 'VOC2007')\n \"\"\"\n\n def __init__(\n self,\n data_dir,\n image_sets=[(\"2007\", \"trainval\"), (\"2012\", \"trainval\")],\n img_size=(416, 416),\n preproc=None,\n target_transform=AnnotationTransform(),\n dataset_name=\"VOC0712\",\n cache=False,\n cache_type=\"ram\",\n ):\n self.root = data_dir\n self.image_set = image_sets\n self.img_size = img_size\n self.preproc = preproc\n self.target_transform = target_transform\n self.name = dataset_name\n self._annopath = os.path.join(\"%s\", \"Annotations\", \"%s.xml\")\n self._imgpath = os.path.join(\"%s\", \"JPEGImages\", \"%s.jpg\")\n self._classes = VOC_CLASSES\n self.cats = [\n {\"id\": idx, \"name\": val} for idx, val in enumerate(VOC_CLASSES)\n ]\n self.class_ids = list(range(len(VOC_CLASSES)))\n self.ids = list()\n for (year, name) in image_sets:\n self._year = year\n rootpath = os.path.join(self.root, \"VOC\" + year)\n for line in open(\n os.path.join(rootpath, \"ImageSets\", \"Main\", name + \".txt\")\n ):\n self.ids.append((rootpath, line.strip()))\n self.num_imgs = len(self.ids)\n\n self.annotations = self._load_coco_annotations()\n\n path_filename = [\n (self._imgpath % self.ids[i]).split(self.root + \"/\")[1]\n for i in range(self.num_imgs)\n ]\n super().__init__(\n input_dimension=img_size,\n num_imgs=self.num_imgs,\n data_dir=self.root,\n cache_dir_name=f\"cache_{self.name}\",\n path_filename=path_filename,\n cache=cache,\n cache_type=cache_type\n )\n\n def __len__(self):\n return self.num_imgs\n\n def _load_coco_annotations(self):\n return [self.load_anno_from_ids(_ids) for _ids in range(self.num_imgs)]\n\n def load_anno_from_ids(self, index):\n img_id = self.ids[index]\n target = ET.parse(self._annopath % img_id).getroot()\n\n assert self.target_transform is not None\n res, img_info = self.target_transform(target)\n height, width = img_info\n\n r = min(self.img_size[0] / height, self.img_size[1] / width)\n res[:, :4] *= r\n resized_info = (int(height * r), int(width * r))\n\n return (res, img_info, resized_info)\n\n def load_anno(self, index):\n return self.annotations[index][0]\n\n def load_resized_img(self, index):\n img = self.load_image(index)\n r = min(self.img_size[0] / img.shape[0], self.img_size[1] / img.shape[1])\n resized_img = cv2.resize(\n img,\n (int(img.shape[1] * r), int(img.shape[0] * r)),\n interpolation=cv2.INTER_LINEAR,\n ).astype(np.uint8)\n\n return resized_img\n\n def load_image(self, index):\n img_id = self.ids[index]\n img = cv2.imread(self._imgpath % img_id, cv2.IMREAD_COLOR)\n assert img is not None, f\"file named {self._imgpath % img_id} not found\"\n\n return img\n\n @cache_read_img(use_cache=True)\n def read_img(self, index):\n return self.load_resized_img(index)\n\n def pull_item(self, index):\n \"\"\"Returns the original image and target at an index for mixup\n\n Note: not using self.__getitem__(), as any transformations passed in\n could mess up this functionality.\n\n Argument:\n index (int): index of img to show\n Return:\n img, target\n \"\"\"\n target, img_info, _ = self.annotations[index]\n img = self.read_img(index)\n\n return img, target, img_info, index\n\n @CacheDataset.mosaic_getitem\n def __getitem__(self, index):\n img, target, img_info, img_id = self.pull_item(index)\n\n if self.preproc is not None:\n img, target = self.preproc(img, target, self.input_dim)\n\n return img, target, img_info, img_id\n\n def evaluate_detections(self, all_boxes, output_dir=None):\n \"\"\"\n all_boxes is a list of length number-of-classes.\n Each list element is a list of length number-of-images.\n Each of those list elements is either an empty list []\n or a numpy array of detection.\n\n all_boxes[class][image] = [] or np.array of shape #dets x 5\n \"\"\"\n self._write_voc_results_file(all_boxes)\n IouTh = np.linspace(\n 0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True\n )\n mAPs = []\n for iou in IouTh:\n mAP = self._do_python_eval(output_dir, iou)\n mAPs.append(mAP)\n\n print(\"--------------------------------------------------------------\")\n print(\"map_5095:\", np.mean(mAPs))\n print(\"map_50:\", mAPs[0])\n print(\"--------------------------------------------------------------\")\n return np.mean(mAPs), mAPs[0]\n\n def _get_voc_results_file_template(self):\n filename = \"comp4_det_test\" + \"_{:s}.txt\"\n filedir = os.path.join(self.root, \"results\", \"VOC\" + self._year, \"Main\")\n if not os.path.exists(filedir):\n os.makedirs(filedir)\n path = os.path.join(filedir, filename)\n return path\n\n def _write_voc_results_file(self, all_boxes):\n for cls_ind, cls in enumerate(VOC_CLASSES):\n cls_ind = cls_ind\n if cls == \"__background__\":\n continue\n print(\"Writing {} VOC results file\".format(cls))\n filename = self._get_voc_results_file_template().format(cls)\n with open(filename, \"wt\") as f:\n for im_ind, index in enumerate(self.ids):\n index = index[1]\n dets = all_boxes[cls_ind][im_ind]\n if dets == []:\n continue\n for k in range(dets.shape[0]):\n f.write(\n \"{:s} {:.3f} {:.1f} {:.1f} {:.1f} {:.1f}\\n\".format(\n index,\n dets[k, -1],\n dets[k, 0] + 1,\n dets[k, 1] + 1,\n dets[k, 2] + 1,\n dets[k, 3] + 1,\n )\n )\n\n def _do_python_eval(self, output_dir=\"output\", iou=0.5):\n rootpath = os.path.join(self.root, \"VOC\" + self._year)\n name = self.image_set[0][1]\n annopath = os.path.join(rootpath, \"Annotations\", \"{:s}.xml\")\n imagesetfile = os.path.join(rootpath, \"ImageSets\", \"Main\", name + \".txt\")\n cachedir = os.path.join(\n self.root, \"annotations_cache\", \"VOC\" + self._year, name\n )\n if not os.path.exists(cachedir):\n os.makedirs(cachedir)\n aps = []\n # The PASCAL VOC metric changed in 2010\n use_07_metric = True if int(self._year) < 2010 else False\n print(\"Eval IoU : {:.2f}\".format(iou))\n if output_dir is not None and not os.path.isdir(output_dir):\n os.mkdir(output_dir)\n for i, cls in enumerate(VOC_CLASSES):\n\n if cls == \"__background__\":\n continue\n\n filename = self._get_voc_results_file_template().format(cls)\n rec, prec, ap = voc_eval(\n filename,\n annopath,\n imagesetfile,\n cls,\n cachedir,\n ovthresh=iou,\n use_07_metric=use_07_metric,\n )\n aps += [ap]\n if iou == 0.5:\n print(\"AP for {} = {:.4f}\".format(cls, ap))\n if output_dir is not None:\n with open(os.path.join(output_dir, cls + \"_pr.pkl\"), \"wb\") as f:\n pickle.dump({\"rec\": rec, \"prec\": prec, \"ap\": ap}, f)\n if iou == 0.5:\n print(\"Mean AP = {:.4f}\".format(np.mean(aps)))\n print(\"~~~~~~~~\")\n print(\"Results:\")\n for ap in aps:\n print(\"{:.3f}\".format(ap))\n print(\"{:.3f}\".format(np.mean(aps)))\n print(\"~~~~~~~~\")\n print(\"\")\n print(\"--------------------------------------------------------------\")\n print(\"Results computed with the **unofficial** Python eval code.\")\n print(\"Results should be very close to the official MATLAB eval code.\")\n print(\"Recompute with `./tools/reval.py --matlab ...` for your paper.\")\n print(\"-- Thanks, The Management\")\n print(\"--------------------------------------------------------------\")\n\n return np.mean(aps)\n" }, { "path": "asone/detectors/yolox/yolox/data/datasets/voc_classes.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\n# VOC_CLASSES = ( '__background__', # always index 0\nVOC_CLASSES = (\n \"aeroplane\",\n \"bicycle\",\n \"bird\",\n \"boat\",\n \"bottle\",\n \"bus\",\n \"car\",\n \"cat\",\n \"chair\",\n \"cow\",\n \"diningtable\",\n \"dog\",\n \"horse\",\n \"motorbike\",\n \"person\",\n \"pottedplant\",\n \"sheep\",\n \"sofa\",\n \"train\",\n \"tvmonitor\",\n)\n" }, { "path": "asone/detectors/yolox/yolox/data/samplers.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport itertools\nfrom typing import Optional\n\nimport torch\nimport torch.distributed as dist\nfrom torch.utils.data.sampler import BatchSampler as torchBatchSampler\nfrom torch.utils.data.sampler import Sampler\n\n\nclass YoloBatchSampler(torchBatchSampler):\n \"\"\"\n This batch sampler will generate mini-batches of (mosaic, index) tuples from another sampler.\n It works just like the :class:`torch.utils.data.sampler.BatchSampler`,\n but it will turn on/off the mosaic aug.\n \"\"\"\n\n def __init__(self, *args, mosaic=True, **kwargs):\n super().__init__(*args, **kwargs)\n self.mosaic = mosaic\n\n def __iter__(self):\n for batch in super().__iter__():\n yield [(self.mosaic, idx) for idx in batch]\n\n\nclass InfiniteSampler(Sampler):\n \"\"\"\n In training, we only care about the \"infinite stream\" of training data.\n So this sampler produces an infinite stream of indices and\n all workers cooperate to correctly shuffle the indices and sample different indices.\n The samplers in each worker effectively produces `indices[worker_id::num_workers]`\n where `indices` is an infinite stream of indices consisting of\n `shuffle(range(size)) + shuffle(range(size)) + ...` (if shuffle is True)\n or `range(size) + range(size) + ...` (if shuffle is False)\n \"\"\"\n\n def __init__(\n self,\n size: int,\n shuffle: bool = True,\n seed: Optional[int] = 0,\n rank=0,\n world_size=1,\n ):\n \"\"\"\n Args:\n size (int): the total number of data of the underlying dataset to sample from\n shuffle (bool): whether to shuffle the indices or not\n seed (int): the initial seed of the shuffle. Must be the same\n across all workers. If None, will use a random seed shared\n among workers (require synchronization among all workers).\n \"\"\"\n self._size = size\n assert size > 0\n self._shuffle = shuffle\n self._seed = int(seed)\n\n if dist.is_available() and dist.is_initialized():\n self._rank = dist.get_rank()\n self._world_size = dist.get_world_size()\n else:\n self._rank = rank\n self._world_size = world_size\n\n def __iter__(self):\n start = self._rank\n yield from itertools.islice(\n self._infinite_indices(), start, None, self._world_size\n )\n\n def _infinite_indices(self):\n g = torch.Generator()\n g.manual_seed(self._seed)\n while True:\n if self._shuffle:\n yield from torch.randperm(self._size, generator=g)\n else:\n yield from torch.arange(self._size)\n\n def __len__(self):\n return self._size // self._world_size\n" }, { "path": "asone/detectors/yolox/yolox/evaluators/__init__.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nfrom .coco_evaluator import COCOEvaluator\nfrom .voc_evaluator import VOCEvaluator\n" }, { "path": "asone/detectors/yolox/yolox/evaluators/coco_evaluator.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport contextlib\nimport io\nimport itertools\nimport json\nimport tempfile\nimport time\nfrom collections import ChainMap, defaultdict\nfrom loguru import logger\nfrom tabulate import tabulate\nfrom tqdm import tqdm\n\nimport numpy as np\n\nimport torch\n\nfrom yolox.data.datasets import COCO_CLASSES\nfrom yolox.utils import (\n gather,\n is_main_process,\n postprocess,\n synchronize,\n time_synchronized,\n xyxy2xywh\n)\n\n\ndef per_class_AR_table(coco_eval, class_names=COCO_CLASSES, headers=[\"class\", \"AR\"], colums=6):\n per_class_AR = {}\n recalls = coco_eval.eval[\"recall\"]\n # dimension of recalls: [TxKxAxM]\n # recall has dims (iou, cls, area range, max dets)\n assert len(class_names) == recalls.shape[1]\n\n for idx, name in enumerate(class_names):\n recall = recalls[:, idx, 0, -1]\n recall = recall[recall > -1]\n ar = np.mean(recall) if recall.size else float(\"nan\")\n per_class_AR[name] = float(ar * 100)\n\n num_cols = min(colums, len(per_class_AR) * len(headers))\n result_pair = [x for pair in per_class_AR.items() for x in pair]\n row_pair = itertools.zip_longest(*[result_pair[i::num_cols] for i in range(num_cols)])\n table_headers = headers * (num_cols // len(headers))\n table = tabulate(\n row_pair, tablefmt=\"pipe\", floatfmt=\".3f\", headers=table_headers, numalign=\"left\",\n )\n return table\n\n\ndef per_class_AP_table(coco_eval, class_names=COCO_CLASSES, headers=[\"class\", \"AP\"], colums=6):\n per_class_AP = {}\n precisions = coco_eval.eval[\"precision\"]\n # dimension of precisions: [TxRxKxAxM]\n # precision has dims (iou, recall, cls, area range, max dets)\n assert len(class_names) == precisions.shape[2]\n\n for idx, name in enumerate(class_names):\n # area range index 0: all area ranges\n # max dets index -1: typically 100 per image\n precision = precisions[:, :, idx, 0, -1]\n precision = precision[precision > -1]\n ap = np.mean(precision) if precision.size else float(\"nan\")\n per_class_AP[name] = float(ap * 100)\n\n num_cols = min(colums, len(per_class_AP) * len(headers))\n result_pair = [x for pair in per_class_AP.items() for x in pair]\n row_pair = itertools.zip_longest(*[result_pair[i::num_cols] for i in range(num_cols)])\n table_headers = headers * (num_cols // len(headers))\n table = tabulate(\n row_pair, tablefmt=\"pipe\", floatfmt=\".3f\", headers=table_headers, numalign=\"left\",\n )\n return table\n\n\nclass COCOEvaluator:\n \"\"\"\n COCO AP Evaluation class. All the data in the val2017 dataset are processed\n and evaluated by COCO API.\n \"\"\"\n\n def __init__(\n self,\n dataloader,\n img_size: int,\n confthre: float,\n nmsthre: float,\n num_classes: int,\n testdev: bool = False,\n per_class_AP: bool = True,\n per_class_AR: bool = True,\n ):\n \"\"\"\n Args:\n dataloader (Dataloader): evaluate dataloader.\n img_size: image size after preprocess. images are resized\n to squares whose shape is (img_size, img_size).\n confthre: confidence threshold ranging from 0 to 1, which\n is defined in the config file.\n nmsthre: IoU threshold of non-max supression ranging from 0 to 1.\n per_class_AP: Show per class AP during evalution or not. Default to True.\n per_class_AR: Show per class AR during evalution or not. Default to True.\n \"\"\"\n self.dataloader = dataloader\n self.img_size = img_size\n self.confthre = confthre\n self.nmsthre = nmsthre\n self.num_classes = num_classes\n self.testdev = testdev\n self.per_class_AP = per_class_AP\n self.per_class_AR = per_class_AR\n\n def evaluate(\n self, model, distributed=False, half=False, trt_file=None,\n decoder=None, test_size=None, return_outputs=False\n ):\n \"\"\"\n COCO average precision (AP) Evaluation. Iterate inference on the test dataset\n and the results are evaluated by COCO API.\n\n NOTE: This function will change training mode to False, please save states if needed.\n\n Args:\n model : model to evaluate.\n\n Returns:\n ap50_95 (float) : COCO AP of IoU=50:95\n ap50 (float) : COCO AP of IoU=50\n summary (sr): summary info of evaluation.\n \"\"\"\n # TODO half to amp_test\n tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor\n model = model.eval()\n if half:\n model = model.half()\n ids = []\n data_list = []\n output_data = defaultdict()\n progress_bar = tqdm if is_main_process() else iter\n\n inference_time = 0\n nms_time = 0\n n_samples = max(len(self.dataloader) - 1, 1)\n\n if trt_file is not None:\n from torch2trt import TRTModule\n\n model_trt = TRTModule()\n model_trt.load_state_dict(torch.load(trt_file))\n\n x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()\n model(x)\n model = model_trt\n\n for cur_iter, (imgs, _, info_imgs, ids) in enumerate(\n progress_bar(self.dataloader)\n ):\n with torch.no_grad():\n imgs = imgs.type(tensor_type)\n\n # skip the last iters since batchsize might be not enough for batch inference\n is_time_record = cur_iter < len(self.dataloader) - 1\n if is_time_record:\n start = time.time()\n\n outputs = model(imgs)\n if decoder is not None:\n outputs = decoder(outputs, dtype=outputs.type())\n\n if is_time_record:\n infer_end = time_synchronized()\n inference_time += infer_end - start\n\n outputs = postprocess(\n outputs, self.num_classes, self.confthre, self.nmsthre\n )\n if is_time_record:\n nms_end = time_synchronized()\n nms_time += nms_end - infer_end\n\n data_list_elem, image_wise_data = self.convert_to_coco_format(\n outputs, info_imgs, ids, return_outputs=True)\n data_list.extend(data_list_elem)\n output_data.update(image_wise_data)\n\n statistics = torch.cuda.FloatTensor([inference_time, nms_time, n_samples])\n if distributed:\n # different process/device might have different speed,\n # to make sure the process will not be stucked, sync func is used here.\n synchronize()\n data_list = gather(data_list, dst=0)\n output_data = gather(output_data, dst=0)\n data_list = list(itertools.chain(*data_list))\n output_data = dict(ChainMap(*output_data))\n torch.distributed.reduce(statistics, dst=0)\n\n eval_results = self.evaluate_prediction(data_list, statistics)\n synchronize()\n\n if return_outputs:\n return eval_results, output_data\n return eval_results\n\n def convert_to_coco_format(self, outputs, info_imgs, ids, return_outputs=False):\n data_list = []\n image_wise_data = defaultdict(dict)\n for (output, img_h, img_w, img_id) in zip(\n outputs, info_imgs[0], info_imgs[1], ids\n ):\n if output is None:\n continue\n output = output.cpu()\n\n bboxes = output[:, 0:4]\n\n # preprocessing: resize\n scale = min(\n self.img_size[0] / float(img_h), self.img_size[1] / float(img_w)\n )\n bboxes /= scale\n cls = output[:, 6]\n scores = output[:, 4] * output[:, 5]\n\n image_wise_data.update({\n int(img_id): {\n \"bboxes\": [box.numpy().tolist() for box in bboxes],\n \"scores\": [score.numpy().item() for score in scores],\n \"categories\": [\n self.dataloader.dataset.class_ids[int(cls[ind])]\n for ind in range(bboxes.shape[0])\n ],\n }\n })\n\n bboxes = xyxy2xywh(bboxes)\n\n for ind in range(bboxes.shape[0]):\n label = self.dataloader.dataset.class_ids[int(cls[ind])]\n pred_data = {\n \"image_id\": int(img_id),\n \"category_id\": label,\n \"bbox\": bboxes[ind].numpy().tolist(),\n \"score\": scores[ind].numpy().item(),\n \"segmentation\": [],\n } # COCO json format\n data_list.append(pred_data)\n\n if return_outputs:\n return data_list, image_wise_data\n return data_list\n\n def evaluate_prediction(self, data_dict, statistics):\n if not is_main_process():\n return 0, 0, None\n\n logger.info(\"Evaluate in main process...\")\n\n annType = [\"segm\", \"bbox\", \"keypoints\"]\n\n inference_time = statistics[0].item()\n nms_time = statistics[1].item()\n n_samples = statistics[2].item()\n\n a_infer_time = 1000 * inference_time / (n_samples * self.dataloader.batch_size)\n a_nms_time = 1000 * nms_time / (n_samples * self.dataloader.batch_size)\n\n time_info = \", \".join(\n [\n \"Average {} time: {:.2f} ms\".format(k, v)\n for k, v in zip(\n [\"forward\", \"NMS\", \"inference\"],\n [a_infer_time, a_nms_time, (a_infer_time + a_nms_time)],\n )\n ]\n )\n\n info = time_info + \"\\n\"\n\n # Evaluate the Dt (detection) json comparing with the ground truth\n if len(data_dict) > 0:\n cocoGt = self.dataloader.dataset.coco\n # TODO: since pycocotools can't process dict in py36, write data to json file.\n if self.testdev:\n json.dump(data_dict, open(\"./yolox_testdev_2017.json\", \"w\"))\n cocoDt = cocoGt.loadRes(\"./yolox_testdev_2017.json\")\n else:\n _, tmp = tempfile.mkstemp()\n json.dump(data_dict, open(tmp, \"w\"))\n cocoDt = cocoGt.loadRes(tmp)\n try:\n from yolox.layers import COCOeval_opt as COCOeval\n except ImportError:\n from pycocotools.cocoeval import COCOeval\n\n logger.warning(\"Use standard COCOeval.\")\n\n cocoEval = COCOeval(cocoGt, cocoDt, annType[1])\n cocoEval.evaluate()\n cocoEval.accumulate()\n redirect_string = io.StringIO()\n with contextlib.redirect_stdout(redirect_string):\n cocoEval.summarize()\n info += redirect_string.getvalue()\n cat_ids = list(cocoGt.cats.keys())\n cat_names = [cocoGt.cats[catId]['name'] for catId in sorted(cat_ids)]\n if self.per_class_AP:\n AP_table = per_class_AP_table(cocoEval, class_names=cat_names)\n info += \"per class AP:\\n\" + AP_table + \"\\n\"\n if self.per_class_AR:\n AR_table = per_class_AR_table(cocoEval, class_names=cat_names)\n info += \"per class AR:\\n\" + AR_table + \"\\n\"\n return cocoEval.stats[0], cocoEval.stats[1], info\n else:\n return 0, 0, info\n" }, { "path": "asone/detectors/yolox/yolox/evaluators/voc_eval.py", "content": "#!/usr/bin/env python3\n# Code are based on\n# https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/datasets/voc_eval.py\n# Copyright (c) Bharath Hariharan.\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\nimport pickle\nimport xml.etree.ElementTree as ET\n\nimport numpy as np\n\n\ndef parse_rec(filename):\n \"\"\"Parse a PASCAL VOC xml file\"\"\"\n tree = ET.parse(filename)\n objects = []\n for obj in tree.findall(\"object\"):\n obj_struct = {}\n obj_struct[\"name\"] = obj.find(\"name\").text\n obj_struct[\"pose\"] = obj.find(\"pose\").text\n obj_struct[\"truncated\"] = int(obj.find(\"truncated\").text)\n obj_struct[\"difficult\"] = int(obj.find(\"difficult\").text)\n bbox = obj.find(\"bndbox\")\n obj_struct[\"bbox\"] = [\n int(bbox.find(\"xmin\").text),\n int(bbox.find(\"ymin\").text),\n int(bbox.find(\"xmax\").text),\n int(bbox.find(\"ymax\").text),\n ]\n objects.append(obj_struct)\n\n return objects\n\n\ndef voc_ap(rec, prec, use_07_metric=False):\n \"\"\"\n Compute VOC AP given precision and recall.\n If use_07_metric is true, uses the\n VOC 07 11 point method (default:False).\n \"\"\"\n if use_07_metric:\n # 11 point metric\n ap = 0.0\n for t in np.arange(0.0, 1.1, 0.1):\n if np.sum(rec >= t) == 0:\n p = 0\n else:\n p = np.max(prec[rec >= t])\n ap = ap + p / 11.0\n else:\n # correct AP calculation\n # first append sentinel values at the end\n mrec = np.concatenate(([0.0], rec, [1.0]))\n mpre = np.concatenate(([0.0], prec, [0.0]))\n\n # compute the precision envelope\n for i in range(mpre.size - 1, 0, -1):\n mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])\n\n # to calculate area under PR curve, look for points\n # where X axis (recall) changes value\n i = np.where(mrec[1:] != mrec[:-1])[0]\n\n # and sum (\\Delta recall) * prec\n ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])\n return ap\n\n\ndef voc_eval(\n detpath,\n annopath,\n imagesetfile,\n classname,\n cachedir,\n ovthresh=0.5,\n use_07_metric=False,\n):\n # first load gt\n if not os.path.isdir(cachedir):\n os.mkdir(cachedir)\n cachefile = os.path.join(cachedir, \"annots.pkl\")\n # read list of images\n with open(imagesetfile, \"r\") as f:\n lines = f.readlines()\n imagenames = [x.strip() for x in lines]\n\n if not os.path.isfile(cachefile):\n # load annots\n recs = {}\n for i, imagename in enumerate(imagenames):\n recs[imagename] = parse_rec(annopath.format(imagename))\n if i % 100 == 0:\n print(f\"Reading annotation for {i + 1}/{len(imagenames)}\")\n # save\n print(f\"Saving cached annotations to {cachefile}\")\n with open(cachefile, \"wb\") as f:\n pickle.dump(recs, f)\n else:\n # load\n with open(cachefile, \"rb\") as f:\n recs = pickle.load(f)\n\n # extract gt objects for this class\n class_recs = {}\n npos = 0\n for imagename in imagenames:\n R = [obj for obj in recs[imagename] if obj[\"name\"] == classname]\n bbox = np.array([x[\"bbox\"] for x in R])\n difficult = np.array([x[\"difficult\"] for x in R]).astype(bool)\n det = [False] * len(R)\n npos = npos + sum(~difficult)\n class_recs[imagename] = {\"bbox\": bbox, \"difficult\": difficult, \"det\": det}\n\n # read dets\n detfile = detpath.format(classname)\n with open(detfile, \"r\") as f:\n lines = f.readlines()\n\n if len(lines) == 0:\n return 0, 0, 0\n\n splitlines = [x.strip().split(\" \") for x in lines]\n image_ids = [x[0] for x in splitlines]\n confidence = np.array([float(x[1]) for x in splitlines])\n BB = np.array([[float(z) for z in x[2:]] for x in splitlines])\n\n # sort by confidence\n sorted_ind = np.argsort(-confidence)\n BB = BB[sorted_ind, :]\n image_ids = [image_ids[x] for x in sorted_ind]\n\n # go down dets and mark TPs and FPs\n nd = len(image_ids)\n tp = np.zeros(nd)\n fp = np.zeros(nd)\n for d in range(nd):\n R = class_recs[image_ids[d]]\n bb = BB[d, :].astype(float)\n ovmax = -np.inf\n BBGT = R[\"bbox\"].astype(float)\n\n if BBGT.size > 0:\n # compute overlaps\n # intersection\n ixmin = np.maximum(BBGT[:, 0], bb[0])\n iymin = np.maximum(BBGT[:, 1], bb[1])\n ixmax = np.minimum(BBGT[:, 2], bb[2])\n iymax = np.minimum(BBGT[:, 3], bb[3])\n iw = np.maximum(ixmax - ixmin + 1.0, 0.0)\n ih = np.maximum(iymax - iymin + 1.0, 0.0)\n inters = iw * ih\n\n # union\n uni = (\n (bb[2] - bb[0] + 1.0) * (bb[3] - bb[1] + 1.0)\n + (BBGT[:, 2] - BBGT[:, 0] + 1.0) * (BBGT[:, 3] - BBGT[:, 1] + 1.0) - inters\n )\n\n overlaps = inters / uni\n ovmax = np.max(overlaps)\n jmax = np.argmax(overlaps)\n\n if ovmax > ovthresh:\n if not R[\"difficult\"][jmax]:\n if not R[\"det\"][jmax]:\n tp[d] = 1.0\n R[\"det\"][jmax] = 1\n else:\n fp[d] = 1.0\n else:\n fp[d] = 1.0\n\n # compute precision recall\n fp = np.cumsum(fp)\n tp = np.cumsum(tp)\n rec = tp / float(npos)\n # avoid divide by zero in case the first detection matches a difficult\n # ground truth\n prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)\n ap = voc_ap(rec, prec, use_07_metric)\n\n return rec, prec, ap\n" }, { "path": "asone/detectors/yolox/yolox/evaluators/voc_evaluator.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport sys\nimport tempfile\nimport time\nfrom collections import ChainMap\nfrom loguru import logger\nfrom tqdm import tqdm\n\nimport numpy as np\n\nimport torch\n\nfrom yolox.utils import gather, is_main_process, postprocess, synchronize, time_synchronized\n\n\nclass VOCEvaluator:\n \"\"\"\n VOC AP Evaluation class.\n \"\"\"\n\n def __init__(self, dataloader, img_size, confthre, nmsthre, num_classes):\n \"\"\"\n Args:\n dataloader (Dataloader): evaluate dataloader.\n img_size (int): image size after preprocess. images are resized\n to squares whose shape is (img_size, img_size).\n confthre (float): confidence threshold ranging from 0 to 1, which\n is defined in the config file.\n nmsthre (float): IoU threshold of non-max supression ranging from 0 to 1.\n \"\"\"\n self.dataloader = dataloader\n self.img_size = img_size\n self.confthre = confthre\n self.nmsthre = nmsthre\n self.num_classes = num_classes\n self.num_images = len(dataloader.dataset)\n\n def evaluate(\n self, model, distributed=False, half=False, trt_file=None,\n decoder=None, test_size=None, return_outputs=False,\n ):\n \"\"\"\n VOC average precision (AP) Evaluation. Iterate inference on the test dataset\n and the results are evaluated by COCO API.\n\n NOTE: This function will change training mode to False, please save states if needed.\n\n Args:\n model : model to evaluate.\n\n Returns:\n ap50_95 (float) : COCO style AP of IoU=50:95\n ap50 (float) : VOC 2007 metric AP of IoU=50\n summary (sr): summary info of evaluation.\n \"\"\"\n # TODO half to amp_test\n tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor\n model = model.eval()\n if half:\n model = model.half()\n ids = []\n data_list = {}\n progress_bar = tqdm if is_main_process() else iter\n\n inference_time = 0\n nms_time = 0\n n_samples = max(len(self.dataloader) - 1, 1)\n\n if trt_file is not None:\n from torch2trt import TRTModule\n\n model_trt = TRTModule()\n model_trt.load_state_dict(torch.load(trt_file))\n\n x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()\n model(x)\n model = model_trt\n\n for cur_iter, (imgs, _, info_imgs, ids) in enumerate(progress_bar(self.dataloader)):\n with torch.no_grad():\n imgs = imgs.type(tensor_type)\n\n # skip the last iters since batchsize might be not enough for batch inference\n is_time_record = cur_iter < len(self.dataloader) - 1\n if is_time_record:\n start = time.time()\n\n outputs = model(imgs)\n if decoder is not None:\n outputs = decoder(outputs, dtype=outputs.type())\n\n if is_time_record:\n infer_end = time_synchronized()\n inference_time += infer_end - start\n\n outputs = postprocess(\n outputs, self.num_classes, self.confthre, self.nmsthre\n )\n if is_time_record:\n nms_end = time_synchronized()\n nms_time += nms_end - infer_end\n\n data_list.update(self.convert_to_voc_format(outputs, info_imgs, ids))\n\n statistics = torch.cuda.FloatTensor([inference_time, nms_time, n_samples])\n if distributed:\n data_list = gather(data_list, dst=0)\n data_list = ChainMap(*data_list)\n torch.distributed.reduce(statistics, dst=0)\n\n eval_results = self.evaluate_prediction(data_list, statistics)\n synchronize()\n if return_outputs:\n return eval_results, data_list\n return eval_results\n\n def convert_to_voc_format(self, outputs, info_imgs, ids):\n predictions = {}\n for output, img_h, img_w, img_id in zip(outputs, info_imgs[0], info_imgs[1], ids):\n if output is None:\n predictions[int(img_id)] = (None, None, None)\n continue\n output = output.cpu()\n\n bboxes = output[:, 0:4]\n\n # preprocessing: resize\n scale = min(self.img_size[0] / float(img_h), self.img_size[1] / float(img_w))\n bboxes /= scale\n\n cls = output[:, 6]\n scores = output[:, 4] * output[:, 5]\n\n predictions[int(img_id)] = (bboxes, cls, scores)\n return predictions\n\n def evaluate_prediction(self, data_dict, statistics):\n if not is_main_process():\n return 0, 0, None\n\n logger.info(\"Evaluate in main process...\")\n\n inference_time = statistics[0].item()\n nms_time = statistics[1].item()\n n_samples = statistics[2].item()\n\n a_infer_time = 1000 * inference_time / (n_samples * self.dataloader.batch_size)\n a_nms_time = 1000 * nms_time / (n_samples * self.dataloader.batch_size)\n\n time_info = \", \".join(\n [\n \"Average {} time: {:.2f} ms\".format(k, v)\n for k, v in zip(\n [\"forward\", \"NMS\", \"inference\"],\n [a_infer_time, a_nms_time, (a_infer_time + a_nms_time)],\n )\n ]\n )\n info = time_info + \"\\n\"\n\n all_boxes = [\n [[] for _ in range(self.num_images)] for _ in range(self.num_classes)\n ]\n for img_num in range(self.num_images):\n bboxes, cls, scores = data_dict[img_num]\n if bboxes is None:\n for j in range(self.num_classes):\n all_boxes[j][img_num] = np.empty([0, 5], dtype=np.float32)\n continue\n for j in range(self.num_classes):\n mask_c = cls == j\n if sum(mask_c) == 0:\n all_boxes[j][img_num] = np.empty([0, 5], dtype=np.float32)\n continue\n\n c_dets = torch.cat((bboxes, scores.unsqueeze(1)), dim=1)\n all_boxes[j][img_num] = c_dets[mask_c].numpy()\n\n sys.stdout.write(f\"im_eval: {img_num + 1}/{self.num_images} \\r\")\n sys.stdout.flush()\n\n with tempfile.TemporaryDirectory() as tempdir:\n mAP50, mAP70 = self.dataloader.dataset.evaluate_detections(all_boxes, tempdir)\n return mAP50, mAP70, info\n" }, { "path": "asone/detectors/yolox/yolox/exp/__init__.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nfrom .base_exp import BaseExp\nfrom .build import get_exp\nfrom .yolox_base import Exp\n" }, { "path": "asone/detectors/yolox/yolox/exp/base_exp.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport ast\nimport pprint\nfrom abc import ABCMeta, abstractmethod\nfrom typing import Dict\nfrom tabulate import tabulate\nfrom asone.detectors.yolox.yolox.utils import LRScheduler\n\nimport torch\nfrom torch.nn import Module\n\n\n\n\nclass BaseExp(metaclass=ABCMeta):\n \"\"\"Basic class for any experiment.\"\"\"\n\n def __init__(self):\n self.seed = None\n self.output_dir = \"./YOLOX_outputs\"\n self.print_interval = 100\n self.eval_interval = 10\n\n @abstractmethod\n def get_model(self) -> Module:\n pass\n\n @abstractmethod\n def get_data_loader(\n self, batch_size: int, is_distributed: bool\n ) -> Dict[str, torch.utils.data.DataLoader]:\n pass\n\n @abstractmethod\n def get_optimizer(self, batch_size: int) -> torch.optim.Optimizer:\n pass\n\n @abstractmethod\n def get_lr_scheduler(\n self, lr: float, iters_per_epoch: int, **kwargs\n ) -> LRScheduler:\n pass\n\n @abstractmethod\n def get_evaluator(self):\n pass\n\n @abstractmethod\n def eval(self, model, evaluator, weights):\n pass\n\n def __repr__(self):\n table_header = [\"keys\", \"values\"]\n exp_table = [\n (str(k), pprint.pformat(v))\n for k, v in vars(self).items()\n if not k.startswith(\"_\")\n ]\n return tabulate(exp_table, headers=table_header, tablefmt=\"fancy_grid\")\n\n def merge(self, cfg_list):\n assert len(cfg_list) % 2 == 0\n for k, v in zip(cfg_list[0::2], cfg_list[1::2]):\n # only update value with same key\n if hasattr(self, k):\n src_value = getattr(self, k)\n src_type = type(src_value)\n if src_value is not None and src_type != type(v):\n try:\n v = src_type(v)\n except Exception:\n v = ast.literal_eval(v)\n setattr(self, k, v)\n" }, { "path": "asone/detectors/yolox/yolox/exp/build.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport importlib\nimport os\nimport sys\n\n\ndef get_exp_by_file(exp_file):\n try:\n _dir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))\n _dir = os.path.join(_dir, 'exps')\n sys.path.append(_dir)\n\n current_exp = importlib.import_module(os.path.basename(exp_file).split(\".\")[0])\n exp = current_exp.Exp()\n except Exception:\n raise ImportError(\"{} doesn't contains class named 'Exp'\".format(exp_file))\n return exp\n\n\ndef get_exp_by_name(exp_name):\n exp = exp_name.replace(\"-\", \"_\") # convert string like \"yolox-s\" to \"yolox_s\"\n module_name = \".\".join([\"yolox\", \"exp\", \"default\", exp])\n exp_object = importlib.import_module(module_name).Exp()\n return exp_object\n\n\ndef get_exp(exp_file=None, exp_name=None):\n \"\"\"\n get Exp object by file or name. If exp_file and exp_name\n are both provided, get Exp by exp_file.\n\n Args:\n exp_file (str): file path of experiment.\n exp_name (str): name of experiment. \"yolo-s\",\n \"\"\"\n assert (\n exp_file is not None or exp_name is not None\n ), \"plz provide exp file or exp name.\"\n if exp_file is not None:\n return get_exp_by_file(exp_file)\n else:\n return get_exp_by_name(exp_name)\n" }, { "path": "asone/detectors/yolox/yolox/exp/default/__init__.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\n# This file is used for package installation and find default exp file\n\nimport importlib\nimport sys\nfrom pathlib import Path\n\n_EXP_PATH = Path(__file__).resolve().parent.parent.parent.parent / \"exps\" / \"default\"\n\nif _EXP_PATH.is_dir():\n # This is true only for in-place installation (pip install -e, setup.py develop),\n # where setup(package_dir=) does not work: https://github.com/pypa/setuptools/issues/230\n\n class _ExpFinder(importlib.abc.MetaPathFinder):\n \n def find_spec(self, name, path, target=None):\n if not name.startswith(\"yolox.exp.default\"):\n return\n project_name = name.split(\".\")[-1] + \".py\"\n target_file = _EXP_PATH / project_name\n if not target_file.is_file():\n return\n return importlib.util.spec_from_file_location(name, target_file)\n\n sys.meta_path.append(_ExpFinder())\n" }, { "path": "asone/detectors/yolox/yolox/exp/yolox_base.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport os\nimport random\n\nimport torch\nimport torch.distributed as dist\nimport torch.nn as nn\n\nfrom asone.detectors.yolox.yolox.exp.base_exp import BaseExp\n\n\nclass Exp(BaseExp):\n def __init__(self):\n super().__init__()\n\n # ---------------- model config ---------------- #\n # detect classes number of model\n self.num_classes = 80\n # factor of model depth\n self.depth = 1.00\n # factor of model width\n self.width = 1.00\n # activation name. For example, if using \"relu\", then \"silu\" will be replaced to \"relu\".\n self.act = \"silu\"\n\n # ---------------- dataloader config ---------------- #\n # set worker to 4 for shorter dataloader init time\n # If your training process cost many memory, reduce this value.\n self.data_num_workers = 4\n self.input_size = (640, 640) # (height, width)\n # Actual multiscale ranges: [640 - 5 * 32, 640 + 5 * 32].\n # To disable multiscale training, set the value to 0.\n self.multiscale_range = 5\n # You can uncomment this line to specify a multiscale range\n # self.random_size = (14, 26)\n # dir of dataset images, if data_dir is None, this project will use `datasets` dir\n self.data_dir = None\n # name of annotation file for training\n self.train_ann = \"instances_train2017.json\"\n # name of annotation file for evaluation\n self.val_ann = \"instances_val2017.json\"\n # name of annotation file for testing\n self.test_ann = \"instances_test2017.json\"\n\n # --------------- transform config ----------------- #\n # prob of applying mosaic aug\n self.mosaic_prob = 1.0\n # prob of applying mixup aug\n self.mixup_prob = 1.0\n # prob of applying hsv aug\n self.hsv_prob = 1.0\n # prob of applying flip aug\n self.flip_prob = 0.5\n # rotation angle range, for example, if set to 2, the true range is (-2, 2)\n self.degrees = 10.0\n # translate range, for example, if set to 0.1, the true range is (-0.1, 0.1)\n self.translate = 0.1\n self.mosaic_scale = (0.1, 2)\n # apply mixup aug or not\n self.enable_mixup = True\n self.mixup_scale = (0.5, 1.5)\n # shear angle range, for example, if set to 2, the true range is (-2, 2)\n self.shear = 2.0\n\n # -------------- training config --------------------- #\n # epoch number used for warmup\n self.warmup_epochs = 5\n # max training epoch\n self.max_epoch = 300\n # minimum learning rate during warmup\n self.warmup_lr = 0\n self.min_lr_ratio = 0.05\n # learning rate for one image. During training, lr will multiply batchsize.\n self.basic_lr_per_img = 0.01 / 64.0\n # name of LRScheduler\n self.scheduler = \"yoloxwarmcos\"\n # last #epoch to close augmention like mosaic\n self.no_aug_epochs = 15\n # apply EMA during training\n self.ema = True\n\n # weight decay of optimizer\n self.weight_decay = 5e-4\n # momentum of optimizer\n self.momentum = 0.9\n # log period in iter, for example,\n # if set to 1, user could see log every iteration.\n self.print_interval = 10\n # eval period in epoch, for example,\n # if set to 1, model will be evaluate after every epoch.\n self.eval_interval = 10\n # save history checkpoint or not.\n # If set to False, yolox will only save latest and best ckpt.\n self.save_history_ckpt = True\n # name of experiment\n self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n\n # ----------------- testing config ------------------ #\n # output image size during evaluation/test\n self.test_size = (640, 640)\n # confidence threshold during evaluation/test,\n # boxes whose scores are less than test_conf will be filtered\n self.test_conf = 0.01\n # nms threshold\n self.nmsthre = 0.65\n\n def get_model(self):\n from asone.detectors.yolox.yolox.models import YOLOX, YOLOPAFPN, YOLOXHead\n\n def init_yolo(M):\n for m in M.modules():\n if isinstance(m, nn.BatchNorm2d):\n m.eps = 1e-3\n m.momentum = 0.03\n\n if getattr(self, \"model\", None) is None:\n in_channels = [256, 512, 1024]\n backbone = YOLOPAFPN(self.depth, self.width, in_channels=in_channels, act=self.act)\n head = YOLOXHead(self.num_classes, self.width, in_channels=in_channels, act=self.act)\n self.model = YOLOX(backbone, head)\n\n self.model.apply(init_yolo)\n self.model.head.initialize_biases(1e-2)\n self.model.train()\n return self.model\n\n def get_data_loader(self, batch_size, is_distributed, no_aug=False, cache_img=False):\n from asone.detectors.yolox.yolox.data import (\n COCODataset,\n TrainTransform,\n YoloBatchSampler,\n DataLoader,\n InfiniteSampler,\n MosaicDetection,\n worker_init_reset_seed,\n )\n from asone.detectors.yolox.yolox.utils import wait_for_the_master\n\n with wait_for_the_master():\n dataset = COCODataset(\n data_dir=self.data_dir,\n json_file=self.train_ann,\n img_size=self.input_size,\n preproc=TrainTransform(\n max_labels=50,\n flip_prob=self.flip_prob,\n hsv_prob=self.hsv_prob),\n cache=cache_img,\n )\n\n dataset = MosaicDetection(\n dataset,\n mosaic=not no_aug,\n img_size=self.input_size,\n preproc=TrainTransform(\n max_labels=120,\n flip_prob=self.flip_prob,\n hsv_prob=self.hsv_prob),\n degrees=self.degrees,\n translate=self.translate,\n mosaic_scale=self.mosaic_scale,\n mixup_scale=self.mixup_scale,\n shear=self.shear,\n enable_mixup=self.enable_mixup,\n mosaic_prob=self.mosaic_prob,\n mixup_prob=self.mixup_prob,\n )\n\n self.dataset = dataset\n\n if is_distributed:\n batch_size = batch_size // dist.get_world_size()\n\n sampler = InfiniteSampler(len(self.dataset), seed=self.seed if self.seed else 0)\n\n batch_sampler = YoloBatchSampler(\n sampler=sampler,\n batch_size=batch_size,\n drop_last=False,\n mosaic=not no_aug,\n )\n\n dataloader_kwargs = {\"num_workers\": self.data_num_workers, \"pin_memory\": True}\n dataloader_kwargs[\"batch_sampler\"] = batch_sampler\n\n # Make sure each process has different random seed, especially for 'fork' method.\n # Check https://github.com/pytorch/pytorch/issues/63311 for more details.\n dataloader_kwargs[\"worker_init_fn\"] = worker_init_reset_seed\n\n train_loader = DataLoader(self.dataset, **dataloader_kwargs)\n\n return train_loader\n\n def random_resize(self, data_loader, epoch, rank, is_distributed):\n tensor = torch.LongTensor(2).cuda()\n\n if rank == 0:\n size_factor = self.input_size[1] * 1.0 / self.input_size[0]\n if not hasattr(self, 'random_size'):\n min_size = int(self.input_size[0] / 32) - self.multiscale_range\n max_size = int(self.input_size[0] / 32) + self.multiscale_range\n self.random_size = (min_size, max_size)\n size = random.randint(*self.random_size)\n size = (int(32 * size), 32 * int(size * size_factor))\n tensor[0] = size[0]\n tensor[1] = size[1]\n\n if is_distributed:\n dist.barrier()\n dist.broadcast(tensor, 0)\n\n input_size = (tensor[0].item(), tensor[1].item())\n return input_size\n\n def preprocess(self, inputs, targets, tsize):\n scale_y = tsize[0] / self.input_size[0]\n scale_x = tsize[1] / self.input_size[1]\n if scale_x != 1 or scale_y != 1:\n inputs = nn.functional.interpolate(\n inputs, size=tsize, mode=\"bilinear\", align_corners=False\n )\n targets[..., 1::2] = targets[..., 1::2] * scale_x\n targets[..., 2::2] = targets[..., 2::2] * scale_y\n return inputs, targets\n\n def get_optimizer(self, batch_size):\n if \"optimizer\" not in self.__dict__:\n if self.warmup_epochs > 0:\n lr = self.warmup_lr\n else:\n lr = self.basic_lr_per_img * batch_size\n\n pg0, pg1, pg2 = [], [], [] # optimizer parameter groups\n\n for k, v in self.model.named_modules():\n if hasattr(v, \"bias\") and isinstance(v.bias, nn.Parameter):\n pg2.append(v.bias) # biases\n if isinstance(v, nn.BatchNorm2d) or \"bn\" in k:\n pg0.append(v.weight) # no decay\n elif hasattr(v, \"weight\") and isinstance(v.weight, nn.Parameter):\n pg1.append(v.weight) # apply decay\n\n optimizer = torch.optim.SGD(\n pg0, lr=lr, momentum=self.momentum, nesterov=True\n )\n optimizer.add_param_group(\n {\"params\": pg1, \"weight_decay\": self.weight_decay}\n ) # add pg1 with weight_decay\n optimizer.add_param_group({\"params\": pg2})\n self.optimizer = optimizer\n\n return self.optimizer\n\n def get_lr_scheduler(self, lr, iters_per_epoch):\n from asone.detectors.yolox.yolox.utils import LRScheduler\n\n scheduler = LRScheduler(\n self.scheduler,\n lr,\n iters_per_epoch,\n self.max_epoch,\n warmup_epochs=self.warmup_epochs,\n warmup_lr_start=self.warmup_lr,\n no_aug_epochs=self.no_aug_epochs,\n min_lr_ratio=self.min_lr_ratio,\n )\n return scheduler\n\n def get_eval_loader(self, batch_size, is_distributed, testdev=False, legacy=False):\n from asone.detectors.yolox.yolox.data import COCODataset, ValTransform\n\n valdataset = COCODataset(\n data_dir=self.data_dir,\n json_file=self.val_ann if not testdev else self.test_ann,\n name=\"val2017\" if not testdev else \"test2017\",\n img_size=self.test_size,\n preproc=ValTransform(legacy=legacy),\n )\n\n if is_distributed:\n batch_size = batch_size // dist.get_world_size()\n sampler = torch.utils.data.distributed.DistributedSampler(\n valdataset, shuffle=False\n )\n else:\n sampler = torch.utils.data.SequentialSampler(valdataset)\n\n dataloader_kwargs = {\n \"num_workers\": self.data_num_workers,\n \"pin_memory\": True,\n \"sampler\": sampler,\n }\n dataloader_kwargs[\"batch_size\"] = batch_size\n val_loader = torch.utils.data.DataLoader(valdataset, **dataloader_kwargs)\n\n return val_loader\n\n def get_evaluator(self, batch_size, is_distributed, testdev=False, legacy=False):\n from asone.detectors.yolox.yolox.evaluators import COCOEvaluator\n\n val_loader = self.get_eval_loader(batch_size, is_distributed, testdev, legacy)\n evaluator = COCOEvaluator(\n dataloader=val_loader,\n img_size=self.test_size,\n confthre=self.test_conf,\n nmsthre=self.nmsthre,\n num_classes=self.num_classes,\n testdev=testdev,\n )\n return evaluator\n\n def get_trainer(self, args):\n from asone.detectors.yolox.yolox.core import Trainer\n trainer = Trainer(self, args)\n # NOTE: trainer shouldn't be an attribute of exp object\n return trainer\n\n def eval(self, model, evaluator, is_distributed, half=False, return_outputs=False):\n return evaluator.evaluate(model, is_distributed, half, return_outputs=return_outputs)\n" }, { "path": "asone/detectors/yolox/yolox/models/__init__.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nfrom .build import *\nfrom .darknet import CSPDarknet, Darknet\nfrom .losses import IOUloss\nfrom .yolo_fpn import YOLOFPN\nfrom .yolo_head import YOLOXHead\nfrom .yolo_pafpn import YOLOPAFPN\nfrom .yolox import YOLOX\n" }, { "path": "asone/detectors/yolox/yolox/models/build.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n\nimport torch\nfrom torch import nn\nfrom torch.hub import load_state_dict_from_url\n\n__all__ = [\n \"create_yolox_model\",\n \"yolox_nano\",\n \"yolox_tiny\",\n \"yolox_s\",\n \"yolox_m\",\n \"yolox_l\",\n \"yolox_x\",\n \"yolov3\",\n \"yolox_custom\"\n]\n\n_CKPT_ROOT_URL = \"https://github.com/Megvii-BaseDetection/YOLOX/releases/download\"\n_CKPT_FULL_PATH = {\n \"yolox-nano\": f\"{_CKPT_ROOT_URL}/0.1.1rc0/yolox_nano.pth\",\n \"yolox-tiny\": f\"{_CKPT_ROOT_URL}/0.1.1rc0/yolox_tiny.pth\",\n \"yolox-s\": f\"{_CKPT_ROOT_URL}/0.1.1rc0/yolox_s.pth\",\n \"yolox-m\": f\"{_CKPT_ROOT_URL}/0.1.1rc0/yolox_m.pth\",\n \"yolox-l\": f\"{_CKPT_ROOT_URL}/0.1.1rc0/yolox_l.pth\",\n \"yolox-x\": f\"{_CKPT_ROOT_URL}/0.1.1rc0/yolox_x.pth\",\n \"yolov3\": f\"{_CKPT_ROOT_URL}/0.1.1rc0/yolox_darknet.pth\",\n}\n\n\ndef create_yolox_model(name: str, pretrained: bool = True, num_classes: int = 80, device=None,\n exp_path: str = None, ckpt_path: str = None) -> nn.Module:\n \"\"\"creates and loads a YOLOX model\n\n Args:\n name (str): name of model. for example, \"yolox-s\", \"yolox-tiny\" or \"yolox_custom\"\n if you want to load your own model.\n pretrained (bool): load pretrained weights into the model. Default to True.\n device (str): default device to for model. Default to None.\n num_classes (int): number of model classes. Default to 80.\n exp_path (str): path to your own experiment file. Required if name=\"yolox_custom\"\n ckpt_path (str): path to your own ckpt. Required if name=\"yolox_custom\" and you want to\n load a pretrained model\n\n\n Returns:\n YOLOX model (nn.Module)\n \"\"\"\n from asone.detectors.yolox.yolox.exp import get_exp, Exp\n\n if device is None:\n device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n device = torch.device(device)\n\n assert name in _CKPT_FULL_PATH or name == \"yolox_custom\", \\\n f\"user should use one of value in {_CKPT_FULL_PATH.keys()} or \\\"yolox_custom\\\"\"\n if name in _CKPT_FULL_PATH:\n exp: Exp = get_exp(exp_name=name)\n exp.num_classes = num_classes\n yolox_model = exp.get_model()\n if pretrained and num_classes == 80:\n weights_url = _CKPT_FULL_PATH[name]\n ckpt = load_state_dict_from_url(weights_url, map_location=\"cpu\")\n if \"model\" in ckpt:\n ckpt = ckpt[\"model\"]\n yolox_model.load_state_dict(ckpt)\n else:\n assert exp_path is not None, \"for a \\\"yolox_custom\\\" model exp_path must be provided\"\n exp: Exp = get_exp(exp_file=exp_path)\n yolox_model = exp.get_model()\n if ckpt_path:\n ckpt = torch.load(ckpt_path, map_location=\"cpu\")\n if \"model\" in ckpt:\n ckpt = ckpt[\"model\"]\n yolox_model.load_state_dict(ckpt)\n\n yolox_model.to(device)\n return yolox_model\n\n\ndef yolox_nano(pretrained: bool = True, num_classes: int = 80, device: str = None) -> nn.Module:\n return create_yolox_model(\"yolox-nano\", pretrained, num_classes, device)\n\n\ndef yolox_tiny(pretrained: bool = True, num_classes: int = 80, device: str = None) -> nn.Module:\n return create_yolox_model(\"yolox-tiny\", pretrained, num_classes, device)\n\n\ndef yolox_s(pretrained: bool = True, num_classes: int = 80, device: str = None) -> nn.Module:\n return create_yolox_model(\"yolox-s\", pretrained, num_classes, device)\n\n\ndef yolox_m(pretrained: bool = True, num_classes: int = 80, device: str = None) -> nn.Module:\n return create_yolox_model(\"yolox-m\", pretrained, num_classes, device)\n\n\ndef yolox_l(pretrained: bool = True, num_classes: int = 80, device: str = None) -> nn.Module:\n return create_yolox_model(\"yolox-l\", pretrained, num_classes, device)\n\n\ndef yolox_x(pretrained: bool = True, num_classes: int = 80, device: str = None) -> nn.Module:\n return create_yolox_model(\"yolox-x\", pretrained, num_classes, device)\n\n\ndef yolov3(pretrained: bool = True, num_classes: int = 80, device: str = None) -> nn.Module:\n return create_yolox_model(\"yolov3\", pretrained, num_classes, device)\n\n\ndef yolox_custom(ckpt_path: str = None, exp_path: str = None, device: str = None) -> nn.Module:\n return create_yolox_model(\"yolox_custom\", ckpt_path=ckpt_path, exp_path=exp_path, device=device)\n" }, { "path": "asone/detectors/yolox/yolox/models/darknet.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nfrom torch import nn\n\nfrom asone.detectors.yolox.yolox.models.network_blocks import BaseConv, CSPLayer, DWConv, Focus, ResLayer, SPPBottleneck\n\n\nclass Darknet(nn.Module):\n # number of blocks from dark2 to dark5.\n depth2blocks = {21: [1, 2, 2, 1], 53: [2, 8, 8, 4]}\n\n def __init__(\n self,\n depth,\n in_channels=3,\n stem_out_channels=32,\n out_features=(\"dark3\", \"dark4\", \"dark5\"),\n ):\n \"\"\"\n Args:\n depth (int): depth of darknet used in model, usually use [21, 53] for this param.\n in_channels (int): number of input channels, for example, use 3 for RGB image.\n stem_out_channels (int): number of output channels of darknet stem.\n It decides channels of darknet layer2 to layer5.\n out_features (Tuple[str]): desired output layer name.\n \"\"\"\n super().__init__()\n assert out_features, \"please provide output features of Darknet\"\n self.out_features = out_features\n self.stem = nn.Sequential(\n BaseConv(in_channels, stem_out_channels, ksize=3, stride=1, act=\"lrelu\"),\n *self.make_group_layer(stem_out_channels, num_blocks=1, stride=2),\n )\n in_channels = stem_out_channels * 2 # 64\n\n num_blocks = Darknet.depth2blocks[depth]\n # create darknet with `stem_out_channels` and `num_blocks` layers.\n # to make model structure more clear, we don't use `for` statement in python.\n self.dark2 = nn.Sequential(\n *self.make_group_layer(in_channels, num_blocks[0], stride=2)\n )\n in_channels *= 2 # 128\n self.dark3 = nn.Sequential(\n *self.make_group_layer(in_channels, num_blocks[1], stride=2)\n )\n in_channels *= 2 # 256\n self.dark4 = nn.Sequential(\n *self.make_group_layer(in_channels, num_blocks[2], stride=2)\n )\n in_channels *= 2 # 512\n\n self.dark5 = nn.Sequential(\n *self.make_group_layer(in_channels, num_blocks[3], stride=2),\n *self.make_spp_block([in_channels, in_channels * 2], in_channels * 2),\n )\n\n def make_group_layer(self, in_channels: int, num_blocks: int, stride: int = 1):\n \"starts with conv layer then has `num_blocks` `ResLayer`\"\n return [\n BaseConv(in_channels, in_channels * 2, ksize=3, stride=stride, act=\"lrelu\"),\n *[(ResLayer(in_channels * 2)) for _ in range(num_blocks)],\n ]\n\n def make_spp_block(self, filters_list, in_filters):\n m = nn.Sequential(\n *[\n BaseConv(in_filters, filters_list[0], 1, stride=1, act=\"lrelu\"),\n BaseConv(filters_list[0], filters_list[1], 3, stride=1, act=\"lrelu\"),\n SPPBottleneck(\n in_channels=filters_list[1],\n out_channels=filters_list[0],\n activation=\"lrelu\",\n ),\n BaseConv(filters_list[0], filters_list[1], 3, stride=1, act=\"lrelu\"),\n BaseConv(filters_list[1], filters_list[0], 1, stride=1, act=\"lrelu\"),\n ]\n )\n return m\n\n def forward(self, x):\n outputs = {}\n x = self.stem(x)\n outputs[\"stem\"] = x\n x = self.dark2(x)\n outputs[\"dark2\"] = x\n x = self.dark3(x)\n outputs[\"dark3\"] = x\n x = self.dark4(x)\n outputs[\"dark4\"] = x\n x = self.dark5(x)\n outputs[\"dark5\"] = x\n return {k: v for k, v in outputs.items() if k in self.out_features}\n\n\nclass CSPDarknet(nn.Module):\n def __init__(\n self,\n dep_mul,\n wid_mul,\n out_features=(\"dark3\", \"dark4\", \"dark5\"),\n depthwise=False,\n act=\"silu\",\n ):\n super().__init__()\n assert out_features, \"please provide output features of Darknet\"\n self.out_features = out_features\n Conv = DWConv if depthwise else BaseConv\n\n base_channels = int(wid_mul * 64) # 64\n base_depth = max(round(dep_mul * 3), 1) # 3\n\n # stem\n self.stem = Focus(3, base_channels, ksize=3, act=act)\n\n # dark2\n self.dark2 = nn.Sequential(\n Conv(base_channels, base_channels * 2, 3, 2, act=act),\n CSPLayer(\n base_channels * 2,\n base_channels * 2,\n n=base_depth,\n depthwise=depthwise,\n act=act,\n ),\n )\n\n # dark3\n self.dark3 = nn.Sequential(\n Conv(base_channels * 2, base_channels * 4, 3, 2, act=act),\n CSPLayer(\n base_channels * 4,\n base_channels * 4,\n n=base_depth * 3,\n depthwise=depthwise,\n act=act,\n ),\n )\n\n # dark4\n self.dark4 = nn.Sequential(\n Conv(base_channels * 4, base_channels * 8, 3, 2, act=act),\n CSPLayer(\n base_channels * 8,\n base_channels * 8,\n n=base_depth * 3,\n depthwise=depthwise,\n act=act,\n ),\n )\n\n # dark5\n self.dark5 = nn.Sequential(\n Conv(base_channels * 8, base_channels * 16, 3, 2, act=act),\n SPPBottleneck(base_channels * 16, base_channels * 16, activation=act),\n CSPLayer(\n base_channels * 16,\n base_channels * 16,\n n=base_depth,\n shortcut=False,\n depthwise=depthwise,\n act=act,\n ),\n )\n\n def forward(self, x):\n outputs = {}\n x = self.stem(x)\n outputs[\"stem\"] = x\n x = self.dark2(x)\n outputs[\"dark2\"] = x\n x = self.dark3(x)\n outputs[\"dark3\"] = x\n x = self.dark4(x)\n outputs[\"dark4\"] = x\n x = self.dark5(x)\n outputs[\"dark5\"] = x\n return {k: v for k, v in outputs.items() if k in self.out_features}\n" }, { "path": "asone/detectors/yolox/yolox/models/losses.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport torch\nimport torch.nn as nn\n\n\nclass IOUloss(nn.Module):\n def __init__(self, reduction=\"none\", loss_type=\"iou\"):\n super(IOUloss, self).__init__()\n self.reduction = reduction\n self.loss_type = loss_type\n\n def forward(self, pred, target):\n assert pred.shape[0] == target.shape[0]\n\n pred = pred.view(-1, 4)\n target = target.view(-1, 4)\n tl = torch.max(\n (pred[:, :2] - pred[:, 2:] / 2), (target[:, :2] - target[:, 2:] / 2)\n )\n br = torch.min(\n (pred[:, :2] + pred[:, 2:] / 2), (target[:, :2] + target[:, 2:] / 2)\n )\n\n area_p = torch.prod(pred[:, 2:], 1)\n area_g = torch.prod(target[:, 2:], 1)\n\n en = (tl < br).type(tl.type()).prod(dim=1)\n area_i = torch.prod(br - tl, 1) * en\n area_u = area_p + area_g - area_i\n iou = (area_i) / (area_u + 1e-16)\n\n if self.loss_type == \"iou\":\n loss = 1 - iou ** 2\n elif self.loss_type == \"giou\":\n c_tl = torch.min(\n (pred[:, :2] - pred[:, 2:] / 2), (target[:, :2] - target[:, 2:] / 2)\n )\n c_br = torch.max(\n (pred[:, :2] + pred[:, 2:] / 2), (target[:, :2] + target[:, 2:] / 2)\n )\n area_c = torch.prod(c_br - c_tl, 1)\n giou = iou - (area_c - area_u) / area_c.clamp(1e-16)\n loss = 1 - giou.clamp(min=-1.0, max=1.0)\n\n if self.reduction == \"mean\":\n loss = loss.mean()\n elif self.reduction == \"sum\":\n loss = loss.sum()\n\n return loss\n" }, { "path": "asone/detectors/yolox/yolox/models/network_blocks.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport torch\nimport torch.nn as nn\n\n\nclass SiLU(nn.Module):\n \"\"\"export-friendly version of nn.SiLU()\"\"\"\n\n @staticmethod\n def forward(x):\n return x * torch.sigmoid(x)\n\n\ndef get_activation(name=\"silu\", inplace=True):\n if name == \"silu\":\n module = nn.SiLU(inplace=inplace)\n elif name == \"relu\":\n module = nn.ReLU(inplace=inplace)\n elif name == \"lrelu\":\n module = nn.LeakyReLU(0.1, inplace=inplace)\n else:\n raise AttributeError(\"Unsupported act type: {}\".format(name))\n return module\n\n\nclass BaseConv(nn.Module):\n \"\"\"A Conv2d -> Batchnorm -> silu/leaky relu block\"\"\"\n\n def __init__(\n self, in_channels, out_channels, ksize, stride, groups=1, bias=False, act=\"silu\"\n ):\n super().__init__()\n # same padding\n pad = (ksize - 1) // 2\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n kernel_size=ksize,\n stride=stride,\n padding=pad,\n groups=groups,\n bias=bias,\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.act = get_activation(act, inplace=True)\n\n def forward(self, x):\n return self.act(self.bn(self.conv(x)))\n\n def fuseforward(self, x):\n return self.act(self.conv(x))\n\n\nclass DWConv(nn.Module):\n \"\"\"Depthwise Conv + Conv\"\"\"\n\n def __init__(self, in_channels, out_channels, ksize, stride=1, act=\"silu\"):\n super().__init__()\n self.dconv = BaseConv(\n in_channels,\n in_channels,\n ksize=ksize,\n stride=stride,\n groups=in_channels,\n act=act,\n )\n self.pconv = BaseConv(\n in_channels, out_channels, ksize=1, stride=1, groups=1, act=act\n )\n\n def forward(self, x):\n x = self.dconv(x)\n return self.pconv(x)\n\n\nclass Bottleneck(nn.Module):\n # Standard bottleneck\n def __init__(\n self,\n in_channels,\n out_channels,\n shortcut=True,\n expansion=0.5,\n depthwise=False,\n act=\"silu\",\n ):\n super().__init__()\n hidden_channels = int(out_channels * expansion)\n Conv = DWConv if depthwise else BaseConv\n self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act)\n self.conv2 = Conv(hidden_channels, out_channels, 3, stride=1, act=act)\n self.use_add = shortcut and in_channels == out_channels\n\n def forward(self, x):\n y = self.conv2(self.conv1(x))\n if self.use_add:\n y = y + x\n return y\n\n\nclass ResLayer(nn.Module):\n \"Residual layer with `in_channels` inputs.\"\n\n def __init__(self, in_channels: int):\n super().__init__()\n mid_channels = in_channels // 2\n self.layer1 = BaseConv(\n in_channels, mid_channels, ksize=1, stride=1, act=\"lrelu\"\n )\n self.layer2 = BaseConv(\n mid_channels, in_channels, ksize=3, stride=1, act=\"lrelu\"\n )\n\n def forward(self, x):\n out = self.layer2(self.layer1(x))\n return x + out\n\n\nclass SPPBottleneck(nn.Module):\n \"\"\"Spatial pyramid pooling layer used in YOLOv3-SPP\"\"\"\n\n def __init__(\n self, in_channels, out_channels, kernel_sizes=(5, 9, 13), activation=\"silu\"\n ):\n super().__init__()\n hidden_channels = in_channels // 2\n self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=activation)\n self.m = nn.ModuleList(\n [\n nn.MaxPool2d(kernel_size=ks, stride=1, padding=ks // 2)\n for ks in kernel_sizes\n ]\n )\n conv2_channels = hidden_channels * (len(kernel_sizes) + 1)\n self.conv2 = BaseConv(conv2_channels, out_channels, 1, stride=1, act=activation)\n\n def forward(self, x):\n x = self.conv1(x)\n x = torch.cat([x] + [m(x) for m in self.m], dim=1)\n x = self.conv2(x)\n return x\n\n\nclass CSPLayer(nn.Module):\n \"\"\"C3 in yolov5, CSP Bottleneck with 3 convolutions\"\"\"\n\n def __init__(\n self,\n in_channels,\n out_channels,\n n=1,\n shortcut=True,\n expansion=0.5,\n depthwise=False,\n act=\"silu\",\n ):\n \"\"\"\n Args:\n in_channels (int): input channels.\n out_channels (int): output channels.\n n (int): number of Bottlenecks. Default value: 1.\n \"\"\"\n # ch_in, ch_out, number, shortcut, groups, expansion\n super().__init__()\n hidden_channels = int(out_channels * expansion) # hidden channels\n self.conv1 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act)\n self.conv2 = BaseConv(in_channels, hidden_channels, 1, stride=1, act=act)\n self.conv3 = BaseConv(2 * hidden_channels, out_channels, 1, stride=1, act=act)\n module_list = [\n Bottleneck(\n hidden_channels, hidden_channels, shortcut, 1.0, depthwise, act=act\n )\n for _ in range(n)\n ]\n self.m = nn.Sequential(*module_list)\n\n def forward(self, x):\n x_1 = self.conv1(x)\n x_2 = self.conv2(x)\n x_1 = self.m(x_1)\n x = torch.cat((x_1, x_2), dim=1)\n return self.conv3(x)\n\n\nclass Focus(nn.Module):\n \"\"\"Focus width and height information into channel space.\"\"\"\n\n def __init__(self, in_channels, out_channels, ksize=1, stride=1, act=\"silu\"):\n super().__init__()\n self.conv = BaseConv(in_channels * 4, out_channels, ksize, stride, act=act)\n\n def forward(self, x):\n # shape of x (b,c,w,h) -> y(b,4c,w/2,h/2)\n patch_top_left = x[..., ::2, ::2]\n patch_top_right = x[..., ::2, 1::2]\n patch_bot_left = x[..., 1::2, ::2]\n patch_bot_right = x[..., 1::2, 1::2]\n x = torch.cat(\n (\n patch_top_left,\n patch_bot_left,\n patch_top_right,\n patch_bot_right,\n ),\n dim=1,\n )\n return self.conv(x)\n" }, { "path": "asone/detectors/yolox/yolox/models/yolo_fpn.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport torch\nimport torch.nn as nn\n\nfrom asone.detectors.yolox.yolox.models.darknet import Darknet\nfrom asone.detectors.yolox.yolox.models.network_blocks import BaseConv\n\n\nclass YOLOFPN(nn.Module):\n \"\"\"\n YOLOFPN module. Darknet 53 is the default backbone of this model.\n \"\"\"\n\n def __init__(\n self,\n depth=53,\n in_features=[\"dark3\", \"dark4\", \"dark5\"],\n ):\n super().__init__()\n\n self.backbone = Darknet(depth)\n self.in_features = in_features\n\n # out 1\n self.out1_cbl = self._make_cbl(512, 256, 1)\n self.out1 = self._make_embedding([256, 512], 512 + 256)\n\n # out 2\n self.out2_cbl = self._make_cbl(256, 128, 1)\n self.out2 = self._make_embedding([128, 256], 256 + 128)\n\n # upsample\n self.upsample = nn.Upsample(scale_factor=2, mode=\"nearest\")\n\n def _make_cbl(self, _in, _out, ks):\n return BaseConv(_in, _out, ks, stride=1, act=\"lrelu\")\n\n def _make_embedding(self, filters_list, in_filters):\n m = nn.Sequential(\n *[\n self._make_cbl(in_filters, filters_list[0], 1),\n self._make_cbl(filters_list[0], filters_list[1], 3),\n self._make_cbl(filters_list[1], filters_list[0], 1),\n self._make_cbl(filters_list[0], filters_list[1], 3),\n self._make_cbl(filters_list[1], filters_list[0], 1),\n ]\n )\n return m\n\n def load_pretrained_model(self, filename=\"./weights/darknet53.mix.pth\"):\n with open(filename, \"rb\") as f:\n state_dict = torch.load(f, map_location=\"cpu\")\n print(\"loading pretrained weights...\")\n self.backbone.load_state_dict(state_dict)\n\n def forward(self, inputs):\n \"\"\"\n Args:\n inputs (Tensor): input image.\n\n Returns:\n Tuple[Tensor]: FPN output features..\n \"\"\"\n # backbone\n out_features = self.backbone(inputs)\n x2, x1, x0 = [out_features[f] for f in self.in_features]\n\n # yolo branch 1\n x1_in = self.out1_cbl(x0)\n x1_in = self.upsample(x1_in)\n x1_in = torch.cat([x1_in, x1], 1)\n out_dark4 = self.out1(x1_in)\n\n # yolo branch 2\n x2_in = self.out2_cbl(out_dark4)\n x2_in = self.upsample(x2_in)\n x2_in = torch.cat([x2_in, x2], 1)\n out_dark3 = self.out2(x2_in)\n\n outputs = (out_dark3, out_dark4, x0)\n return outputs\n" }, { "path": "asone/detectors/yolox/yolox/models/yolo_head.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport math\nfrom loguru import logger\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom asone.detectors.yolox.yolox.utils import bboxes_iou, meshgrid\n\nfrom asone.detectors.yolox.yolox.models.losses import IOUloss\nfrom asone.detectors.yolox.yolox.models.network_blocks import BaseConv, DWConv\n\n\nclass YOLOXHead(nn.Module):\n def __init__(\n self,\n num_classes,\n width=1.0,\n strides=[8, 16, 32],\n in_channels=[256, 512, 1024],\n act=\"silu\",\n depthwise=False,\n ):\n \"\"\"\n Args:\n act (str): activation type of conv. Defalut value: \"silu\".\n depthwise (bool): whether apply depthwise conv in conv branch. Defalut value: False.\n \"\"\"\n super().__init__()\n\n self.n_anchors = 1\n self.num_classes = num_classes\n self.decode_in_inference = True # for deploy, set to False\n\n self.cls_convs = nn.ModuleList()\n self.reg_convs = nn.ModuleList()\n self.cls_preds = nn.ModuleList()\n self.reg_preds = nn.ModuleList()\n self.obj_preds = nn.ModuleList()\n self.stems = nn.ModuleList()\n Conv = DWConv if depthwise else BaseConv\n\n for i in range(len(in_channels)):\n self.stems.append(\n BaseConv(\n in_channels=int(in_channels[i] * width),\n out_channels=int(256 * width),\n ksize=1,\n stride=1,\n act=act,\n )\n )\n self.cls_convs.append(\n nn.Sequential(\n *[\n Conv(\n in_channels=int(256 * width),\n out_channels=int(256 * width),\n ksize=3,\n stride=1,\n act=act,\n ),\n Conv(\n in_channels=int(256 * width),\n out_channels=int(256 * width),\n ksize=3,\n stride=1,\n act=act,\n ),\n ]\n )\n )\n self.reg_convs.append(\n nn.Sequential(\n *[\n Conv(\n in_channels=int(256 * width),\n out_channels=int(256 * width),\n ksize=3,\n stride=1,\n act=act,\n ),\n Conv(\n in_channels=int(256 * width),\n out_channels=int(256 * width),\n ksize=3,\n stride=1,\n act=act,\n ),\n ]\n )\n )\n self.cls_preds.append(\n nn.Conv2d(\n in_channels=int(256 * width),\n out_channels=self.n_anchors * self.num_classes,\n kernel_size=1,\n stride=1,\n padding=0,\n )\n )\n self.reg_preds.append(\n nn.Conv2d(\n in_channels=int(256 * width),\n out_channels=4,\n kernel_size=1,\n stride=1,\n padding=0,\n )\n )\n self.obj_preds.append(\n nn.Conv2d(\n in_channels=int(256 * width),\n out_channels=self.n_anchors * 1,\n kernel_size=1,\n stride=1,\n padding=0,\n )\n )\n\n self.use_l1 = False\n self.l1_loss = nn.L1Loss(reduction=\"none\")\n self.bcewithlog_loss = nn.BCEWithLogitsLoss(reduction=\"none\")\n self.iou_loss = IOUloss(reduction=\"none\")\n self.strides = strides\n self.grids = [torch.zeros(1)] * len(in_channels)\n\n def initialize_biases(self, prior_prob):\n for conv in self.cls_preds:\n b = conv.bias.view(self.n_anchors, -1)\n b.data.fill_(-math.log((1 - prior_prob) / prior_prob))\n conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n\n for conv in self.obj_preds:\n b = conv.bias.view(self.n_anchors, -1)\n b.data.fill_(-math.log((1 - prior_prob) / prior_prob))\n conv.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\n\n def forward(self, xin, labels=None, imgs=None):\n outputs = []\n origin_preds = []\n x_shifts = []\n y_shifts = []\n expanded_strides = []\n\n for k, (cls_conv, reg_conv, stride_this_level, x) in enumerate(\n zip(self.cls_convs, self.reg_convs, self.strides, xin)\n ):\n x = self.stems[k](x)\n cls_x = x\n reg_x = x\n\n cls_feat = cls_conv(cls_x)\n cls_output = self.cls_preds[k](cls_feat)\n\n reg_feat = reg_conv(reg_x)\n reg_output = self.reg_preds[k](reg_feat)\n obj_output = self.obj_preds[k](reg_feat)\n\n if self.training:\n output = torch.cat([reg_output, obj_output, cls_output], 1)\n output, grid = self.get_output_and_grid(\n output, k, stride_this_level, xin[0].type()\n )\n x_shifts.append(grid[:, :, 0])\n y_shifts.append(grid[:, :, 1])\n expanded_strides.append(\n torch.zeros(1, grid.shape[1])\n .fill_(stride_this_level)\n .type_as(xin[0])\n )\n if self.use_l1:\n batch_size = reg_output.shape[0]\n hsize, wsize = reg_output.shape[-2:]\n reg_output = reg_output.view(\n batch_size, self.n_anchors, 4, hsize, wsize\n )\n reg_output = reg_output.permute(0, 1, 3, 4, 2).reshape(\n batch_size, -1, 4\n )\n origin_preds.append(reg_output.clone())\n\n else:\n output = torch.cat(\n [reg_output, obj_output.sigmoid(), cls_output.sigmoid()], 1\n )\n\n outputs.append(output)\n\n if self.training:\n return self.get_losses(\n imgs,\n x_shifts,\n y_shifts,\n expanded_strides,\n labels,\n torch.cat(outputs, 1),\n origin_preds,\n dtype=xin[0].dtype,\n )\n else:\n self.hw = [x.shape[-2:] for x in outputs]\n # [batch, n_anchors_all, 85]\n outputs = torch.cat(\n [x.flatten(start_dim=2) for x in outputs], dim=2\n ).permute(0, 2, 1)\n if self.decode_in_inference:\n return self.decode_outputs(outputs, dtype=xin[0].type())\n else:\n return outputs\n\n def get_output_and_grid(self, output, k, stride, dtype):\n grid = self.grids[k]\n\n batch_size = output.shape[0]\n n_ch = 5 + self.num_classes\n hsize, wsize = output.shape[-2:]\n if grid.shape[2:4] != output.shape[2:4]:\n yv, xv = meshgrid([torch.arange(hsize), torch.arange(wsize)])\n grid = torch.stack((xv, yv), 2).view(1, 1, hsize, wsize, 2).type(dtype)\n self.grids[k] = grid\n\n output = output.view(batch_size, self.n_anchors, n_ch, hsize, wsize)\n output = output.permute(0, 1, 3, 4, 2).reshape(\n batch_size, self.n_anchors * hsize * wsize, -1\n )\n grid = grid.view(1, -1, 2)\n output[..., :2] = (output[..., :2] + grid) * stride\n output[..., 2:4] = torch.exp(output[..., 2:4]) * stride\n return output, grid\n\n def decode_outputs(self, outputs, dtype):\n grids = []\n strides = []\n for (hsize, wsize), stride in zip(self.hw, self.strides):\n yv, xv = meshgrid([torch.arange(hsize), torch.arange(wsize)])\n grid = torch.stack((xv, yv), 2).view(1, -1, 2)\n grids.append(grid)\n shape = grid.shape[:2]\n strides.append(torch.full((*shape, 1), stride))\n\n grids = torch.cat(grids, dim=1).type(dtype)\n strides = torch.cat(strides, dim=1).type(dtype)\n\n outputs[..., :2] = (outputs[..., :2] + grids) * strides\n outputs[..., 2:4] = torch.exp(outputs[..., 2:4]) * strides\n return outputs\n\n def get_losses(\n self,\n imgs,\n x_shifts,\n y_shifts,\n expanded_strides,\n labels,\n outputs,\n origin_preds,\n dtype,\n ):\n bbox_preds = outputs[:, :, :4] # [batch, n_anchors_all, 4]\n obj_preds = outputs[:, :, 4].unsqueeze(-1) # [batch, n_anchors_all, 1]\n cls_preds = outputs[:, :, 5:] # [batch, n_anchors_all, n_cls]\n\n # calculate targets\n nlabel = (labels.sum(dim=2) > 0).sum(dim=1) # number of objects\n\n total_num_anchors = outputs.shape[1]\n x_shifts = torch.cat(x_shifts, 1) # [1, n_anchors_all]\n y_shifts = torch.cat(y_shifts, 1) # [1, n_anchors_all]\n expanded_strides = torch.cat(expanded_strides, 1)\n if self.use_l1:\n origin_preds = torch.cat(origin_preds, 1)\n\n cls_targets = []\n reg_targets = []\n l1_targets = []\n obj_targets = []\n fg_masks = []\n\n num_fg = 0.0\n num_gts = 0.0\n\n for batch_idx in range(outputs.shape[0]):\n num_gt = int(nlabel[batch_idx])\n num_gts += num_gt\n if num_gt == 0:\n cls_target = outputs.new_zeros((0, self.num_classes))\n reg_target = outputs.new_zeros((0, 4))\n l1_target = outputs.new_zeros((0, 4))\n obj_target = outputs.new_zeros((total_num_anchors, 1))\n fg_mask = outputs.new_zeros(total_num_anchors).bool()\n else:\n gt_bboxes_per_image = labels[batch_idx, :num_gt, 1:5]\n gt_classes = labels[batch_idx, :num_gt, 0]\n bboxes_preds_per_image = bbox_preds[batch_idx]\n\n try:\n (\n gt_matched_classes,\n fg_mask,\n pred_ious_this_matching,\n matched_gt_inds,\n num_fg_img,\n ) = self.get_assignments( # noqa\n batch_idx,\n num_gt,\n total_num_anchors,\n gt_bboxes_per_image,\n gt_classes,\n bboxes_preds_per_image,\n expanded_strides,\n x_shifts,\n y_shifts,\n cls_preds,\n bbox_preds,\n obj_preds,\n labels,\n imgs,\n )\n except RuntimeError as e:\n # TODO: the string might change, consider a better way\n if \"CUDA out of memory. \" not in str(e):\n raise # RuntimeError might not caused by CUDA OOM\n\n logger.error(\n \"OOM RuntimeError is raised due to the huge memory cost during label assignment. \\\n CPU mode is applied in this batch. If you want to avoid this issue, \\\n try to reduce the batch size or image size.\"\n )\n torch.cuda.empty_cache()\n (\n gt_matched_classes,\n fg_mask,\n pred_ious_this_matching,\n matched_gt_inds,\n num_fg_img,\n ) = self.get_assignments( # noqa\n batch_idx,\n num_gt,\n total_num_anchors,\n gt_bboxes_per_image,\n gt_classes,\n bboxes_preds_per_image,\n expanded_strides,\n x_shifts,\n y_shifts,\n cls_preds,\n bbox_preds,\n obj_preds,\n labels,\n imgs,\n \"cpu\",\n )\n\n torch.cuda.empty_cache()\n num_fg += num_fg_img\n\n cls_target = F.one_hot(\n gt_matched_classes.to(torch.int64), self.num_classes\n ) * pred_ious_this_matching.unsqueeze(-1)\n obj_target = fg_mask.unsqueeze(-1)\n reg_target = gt_bboxes_per_image[matched_gt_inds]\n if self.use_l1:\n l1_target = self.get_l1_target(\n outputs.new_zeros((num_fg_img, 4)),\n gt_bboxes_per_image[matched_gt_inds],\n expanded_strides[0][fg_mask],\n x_shifts=x_shifts[0][fg_mask],\n y_shifts=y_shifts[0][fg_mask],\n )\n\n cls_targets.append(cls_target)\n reg_targets.append(reg_target)\n obj_targets.append(obj_target.to(dtype))\n fg_masks.append(fg_mask)\n if self.use_l1:\n l1_targets.append(l1_target)\n\n cls_targets = torch.cat(cls_targets, 0)\n reg_targets = torch.cat(reg_targets, 0)\n obj_targets = torch.cat(obj_targets, 0)\n fg_masks = torch.cat(fg_masks, 0)\n if self.use_l1:\n l1_targets = torch.cat(l1_targets, 0)\n\n num_fg = max(num_fg, 1)\n loss_iou = (\n self.iou_loss(bbox_preds.view(-1, 4)[fg_masks], reg_targets)\n ).sum() / num_fg\n loss_obj = (\n self.bcewithlog_loss(obj_preds.view(-1, 1), obj_targets)\n ).sum() / num_fg\n loss_cls = (\n self.bcewithlog_loss(\n cls_preds.view(-1, self.num_classes)[fg_masks], cls_targets\n )\n ).sum() / num_fg\n if self.use_l1:\n loss_l1 = (\n self.l1_loss(origin_preds.view(-1, 4)[fg_masks], l1_targets)\n ).sum() / num_fg\n else:\n loss_l1 = 0.0\n\n reg_weight = 5.0\n loss = reg_weight * loss_iou + loss_obj + loss_cls + loss_l1\n\n return (\n loss,\n reg_weight * loss_iou,\n loss_obj,\n loss_cls,\n loss_l1,\n num_fg / max(num_gts, 1),\n )\n\n def get_l1_target(self, l1_target, gt, stride, x_shifts, y_shifts, eps=1e-8):\n l1_target[:, 0] = gt[:, 0] / stride - x_shifts\n l1_target[:, 1] = gt[:, 1] / stride - y_shifts\n l1_target[:, 2] = torch.log(gt[:, 2] / stride + eps)\n l1_target[:, 3] = torch.log(gt[:, 3] / stride + eps)\n return l1_target\n\n @torch.no_grad()\n def get_assignments(\n self,\n batch_idx,\n num_gt,\n total_num_anchors,\n gt_bboxes_per_image,\n gt_classes,\n bboxes_preds_per_image,\n expanded_strides,\n x_shifts,\n y_shifts,\n cls_preds,\n bbox_preds,\n obj_preds,\n labels,\n imgs,\n mode=\"gpu\",\n ):\n\n if mode == \"cpu\":\n print(\"------------CPU Mode for This Batch-------------\")\n gt_bboxes_per_image = gt_bboxes_per_image.cpu().float()\n bboxes_preds_per_image = bboxes_preds_per_image.cpu().float()\n gt_classes = gt_classes.cpu().float()\n expanded_strides = expanded_strides.cpu().float()\n x_shifts = x_shifts.cpu()\n y_shifts = y_shifts.cpu()\n\n fg_mask, is_in_boxes_and_center = self.get_in_boxes_info(\n gt_bboxes_per_image,\n expanded_strides,\n x_shifts,\n y_shifts,\n total_num_anchors,\n num_gt,\n )\n\n bboxes_preds_per_image = bboxes_preds_per_image[fg_mask]\n cls_preds_ = cls_preds[batch_idx][fg_mask]\n obj_preds_ = obj_preds[batch_idx][fg_mask]\n num_in_boxes_anchor = bboxes_preds_per_image.shape[0]\n\n if mode == \"cpu\":\n gt_bboxes_per_image = gt_bboxes_per_image.cpu()\n bboxes_preds_per_image = bboxes_preds_per_image.cpu()\n\n pair_wise_ious = bboxes_iou(gt_bboxes_per_image, bboxes_preds_per_image, False)\n\n gt_cls_per_image = (\n F.one_hot(gt_classes.to(torch.int64), self.num_classes)\n .float()\n .unsqueeze(1)\n .repeat(1, num_in_boxes_anchor, 1)\n )\n pair_wise_ious_loss = -torch.log(pair_wise_ious + 1e-8)\n\n if mode == \"cpu\":\n cls_preds_, obj_preds_ = cls_preds_.cpu(), obj_preds_.cpu()\n\n with torch.cuda.amp.autocast(enabled=False):\n cls_preds_ = (\n cls_preds_.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()\n * obj_preds_.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()\n )\n pair_wise_cls_loss = F.binary_cross_entropy(\n cls_preds_.sqrt_(), gt_cls_per_image, reduction=\"none\"\n ).sum(-1)\n del cls_preds_\n\n cost = (\n pair_wise_cls_loss\n + 3.0 * pair_wise_ious_loss\n + 100000.0 * (~is_in_boxes_and_center)\n )\n\n (\n num_fg,\n gt_matched_classes,\n pred_ious_this_matching,\n matched_gt_inds,\n ) = self.dynamic_k_matching(cost, pair_wise_ious, gt_classes, num_gt, fg_mask)\n del pair_wise_cls_loss, cost, pair_wise_ious, pair_wise_ious_loss\n\n if mode == \"cpu\":\n gt_matched_classes = gt_matched_classes.cuda()\n fg_mask = fg_mask.cuda()\n pred_ious_this_matching = pred_ious_this_matching.cuda()\n matched_gt_inds = matched_gt_inds.cuda()\n\n return (\n gt_matched_classes,\n fg_mask,\n pred_ious_this_matching,\n matched_gt_inds,\n num_fg,\n )\n\n def get_in_boxes_info(\n self,\n gt_bboxes_per_image,\n expanded_strides,\n x_shifts,\n y_shifts,\n total_num_anchors,\n num_gt,\n ):\n expanded_strides_per_image = expanded_strides[0]\n x_shifts_per_image = x_shifts[0] * expanded_strides_per_image\n y_shifts_per_image = y_shifts[0] * expanded_strides_per_image\n x_centers_per_image = (\n (x_shifts_per_image + 0.5 * expanded_strides_per_image)\n .unsqueeze(0)\n .repeat(num_gt, 1)\n ) # [n_anchor] -> [n_gt, n_anchor]\n y_centers_per_image = (\n (y_shifts_per_image + 0.5 * expanded_strides_per_image)\n .unsqueeze(0)\n .repeat(num_gt, 1)\n )\n\n gt_bboxes_per_image_l = (\n (gt_bboxes_per_image[:, 0] - 0.5 * gt_bboxes_per_image[:, 2])\n .unsqueeze(1)\n .repeat(1, total_num_anchors)\n )\n gt_bboxes_per_image_r = (\n (gt_bboxes_per_image[:, 0] + 0.5 * gt_bboxes_per_image[:, 2])\n .unsqueeze(1)\n .repeat(1, total_num_anchors)\n )\n gt_bboxes_per_image_t = (\n (gt_bboxes_per_image[:, 1] - 0.5 * gt_bboxes_per_image[:, 3])\n .unsqueeze(1)\n .repeat(1, total_num_anchors)\n )\n gt_bboxes_per_image_b = (\n (gt_bboxes_per_image[:, 1] + 0.5 * gt_bboxes_per_image[:, 3])\n .unsqueeze(1)\n .repeat(1, total_num_anchors)\n )\n\n b_l = x_centers_per_image - gt_bboxes_per_image_l\n b_r = gt_bboxes_per_image_r - x_centers_per_image\n b_t = y_centers_per_image - gt_bboxes_per_image_t\n b_b = gt_bboxes_per_image_b - y_centers_per_image\n bbox_deltas = torch.stack([b_l, b_t, b_r, b_b], 2)\n\n is_in_boxes = bbox_deltas.min(dim=-1).values > 0.0\n is_in_boxes_all = is_in_boxes.sum(dim=0) > 0\n # in fixed center\n\n center_radius = 2.5\n\n gt_bboxes_per_image_l = (gt_bboxes_per_image[:, 0]).unsqueeze(1).repeat(\n 1, total_num_anchors\n ) - center_radius * expanded_strides_per_image.unsqueeze(0)\n gt_bboxes_per_image_r = (gt_bboxes_per_image[:, 0]).unsqueeze(1).repeat(\n 1, total_num_anchors\n ) + center_radius * expanded_strides_per_image.unsqueeze(0)\n gt_bboxes_per_image_t = (gt_bboxes_per_image[:, 1]).unsqueeze(1).repeat(\n 1, total_num_anchors\n ) - center_radius * expanded_strides_per_image.unsqueeze(0)\n gt_bboxes_per_image_b = (gt_bboxes_per_image[:, 1]).unsqueeze(1).repeat(\n 1, total_num_anchors\n ) + center_radius * expanded_strides_per_image.unsqueeze(0)\n\n c_l = x_centers_per_image - gt_bboxes_per_image_l\n c_r = gt_bboxes_per_image_r - x_centers_per_image\n c_t = y_centers_per_image - gt_bboxes_per_image_t\n c_b = gt_bboxes_per_image_b - y_centers_per_image\n center_deltas = torch.stack([c_l, c_t, c_r, c_b], 2)\n is_in_centers = center_deltas.min(dim=-1).values > 0.0\n is_in_centers_all = is_in_centers.sum(dim=0) > 0\n\n # in boxes and in centers\n is_in_boxes_anchor = is_in_boxes_all | is_in_centers_all\n\n is_in_boxes_and_center = (\n is_in_boxes[:, is_in_boxes_anchor] & is_in_centers[:, is_in_boxes_anchor]\n )\n return is_in_boxes_anchor, is_in_boxes_and_center\n\n def dynamic_k_matching(self, cost, pair_wise_ious, gt_classes, num_gt, fg_mask):\n # Dynamic K\n # ---------------------------------------------------------------\n matching_matrix = torch.zeros_like(cost, dtype=torch.uint8)\n\n ious_in_boxes_matrix = pair_wise_ious\n n_candidate_k = min(10, ious_in_boxes_matrix.size(1))\n topk_ious, _ = torch.topk(ious_in_boxes_matrix, n_candidate_k, dim=1)\n dynamic_ks = torch.clamp(topk_ious.sum(1).int(), min=1)\n dynamic_ks = dynamic_ks.tolist()\n for gt_idx in range(num_gt):\n _, pos_idx = torch.topk(\n cost[gt_idx], k=dynamic_ks[gt_idx], largest=False\n )\n matching_matrix[gt_idx][pos_idx] = 1\n\n del topk_ious, dynamic_ks, pos_idx\n\n anchor_matching_gt = matching_matrix.sum(0)\n if (anchor_matching_gt > 1).sum() > 0:\n _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0)\n matching_matrix[:, anchor_matching_gt > 1] *= 0\n matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1\n fg_mask_inboxes = matching_matrix.sum(0) > 0\n num_fg = fg_mask_inboxes.sum().item()\n\n fg_mask[fg_mask.clone()] = fg_mask_inboxes\n\n matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0)\n gt_matched_classes = gt_classes[matched_gt_inds]\n\n pred_ious_this_matching = (matching_matrix * pair_wise_ious).sum(0)[\n fg_mask_inboxes\n ]\n return num_fg, gt_matched_classes, pred_ious_this_matching, matched_gt_inds\n" }, { "path": "asone/detectors/yolox/yolox/models/yolo_pafpn.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport torch\nimport torch.nn as nn\n\nfrom asone.detectors.yolox.yolox.models.darknet import CSPDarknet\nfrom asone.detectors.yolox.yolox.models.network_blocks import BaseConv, CSPLayer, DWConv\n\n\nclass YOLOPAFPN(nn.Module):\n \"\"\"\n YOLOv3 model. Darknet 53 is the default backbone of this model.\n \"\"\"\n\n def __init__(\n self,\n depth=1.0,\n width=1.0,\n in_features=(\"dark3\", \"dark4\", \"dark5\"),\n in_channels=[256, 512, 1024],\n depthwise=False,\n act=\"silu\",\n ):\n super().__init__()\n self.backbone = CSPDarknet(depth, width, depthwise=depthwise, act=act)\n self.in_features = in_features\n self.in_channels = in_channels\n Conv = DWConv if depthwise else BaseConv\n\n self.upsample = nn.Upsample(scale_factor=2, mode=\"nearest\")\n self.lateral_conv0 = BaseConv(\n int(in_channels[2] * width), int(in_channels[1] * width), 1, 1, act=act\n )\n self.C3_p4 = CSPLayer(\n int(2 * in_channels[1] * width),\n int(in_channels[1] * width),\n round(3 * depth),\n False,\n depthwise=depthwise,\n act=act,\n ) # cat\n\n self.reduce_conv1 = BaseConv(\n int(in_channels[1] * width), int(in_channels[0] * width), 1, 1, act=act\n )\n self.C3_p3 = CSPLayer(\n int(2 * in_channels[0] * width),\n int(in_channels[0] * width),\n round(3 * depth),\n False,\n depthwise=depthwise,\n act=act,\n )\n\n # bottom-up conv\n self.bu_conv2 = Conv(\n int(in_channels[0] * width), int(in_channels[0] * width), 3, 2, act=act\n )\n self.C3_n3 = CSPLayer(\n int(2 * in_channels[0] * width),\n int(in_channels[1] * width),\n round(3 * depth),\n False,\n depthwise=depthwise,\n act=act,\n )\n\n # bottom-up conv\n self.bu_conv1 = Conv(\n int(in_channels[1] * width), int(in_channels[1] * width), 3, 2, act=act\n )\n self.C3_n4 = CSPLayer(\n int(2 * in_channels[1] * width),\n int(in_channels[2] * width),\n round(3 * depth),\n False,\n depthwise=depthwise,\n act=act,\n )\n\n def forward(self, input):\n \"\"\"\n Args:\n inputs: input images.\n\n Returns:\n Tuple[Tensor]: FPN feature.\n \"\"\"\n\n # backbone\n out_features = self.backbone(input)\n features = [out_features[f] for f in self.in_features]\n [x2, x1, x0] = features\n\n fpn_out0 = self.lateral_conv0(x0) # 1024->512/32\n f_out0 = self.upsample(fpn_out0) # 512/16\n f_out0 = torch.cat([f_out0, x1], 1) # 512->1024/16\n f_out0 = self.C3_p4(f_out0) # 1024->512/16\n\n fpn_out1 = self.reduce_conv1(f_out0) # 512->256/16\n f_out1 = self.upsample(fpn_out1) # 256/8\n f_out1 = torch.cat([f_out1, x2], 1) # 256->512/8\n pan_out2 = self.C3_p3(f_out1) # 512->256/8\n\n p_out1 = self.bu_conv2(pan_out2) # 256->256/16\n p_out1 = torch.cat([p_out1, fpn_out1], 1) # 256->512/16\n pan_out1 = self.C3_n3(p_out1) # 512->512/16\n\n p_out0 = self.bu_conv1(pan_out1) # 512->512/32\n p_out0 = torch.cat([p_out0, fpn_out0], 1) # 512->1024/32\n pan_out0 = self.C3_n4(p_out0) # 1024->1024/32\n\n outputs = (pan_out2, pan_out1, pan_out0)\n return outputs\n" }, { "path": "asone/detectors/yolox/yolox/models/yolox.py", "content": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport torch.nn as nn\n\nfrom asone.detectors.yolox.yolox.models.yolo_head import YOLOXHead\nfrom asone.detectors.yolox.yolox.models.yolo_pafpn import YOLOPAFPN\n\n\nclass YOLOX(nn.Module):\n \"\"\"\n YOLOX model module. The module list is defined by create_yolov3_modules function.\n The network returns loss values from three YOLO layers during training\n and detection results during test.\n \"\"\"\n\n def __init__(self, backbone=None, head=None):\n super().__init__()\n if backbone is None:\n backbone = YOLOPAFPN()\n if head is None:\n head = YOLOXHead(80)\n\n self.backbone = backbone\n self.head = head\n\n def forward(self, x, targets=None):\n # fpn output content features of [dark3, dark4, dark5]\n fpn_outs = self.backbone(x)\n\n if self.training:\n assert targets is not None\n loss, iou_loss, conf_loss, cls_loss, l1_loss, num_fg = self.head(\n fpn_outs, targets, x\n )\n outputs = {\n \"total_loss\": loss,\n \"iou_loss\": iou_loss,\n \"l1_loss\": l1_loss,\n \"conf_loss\": conf_loss,\n \"cls_loss\": cls_loss,\n \"num_fg\": num_fg,\n }\n else:\n outputs = self.head(fpn_outs)\n\n return outputs\n" }, { "path": "asone/detectors/yolox/yolox/utils/__init__.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nfrom .allreduce_norm import *\nfrom .boxes import *\nfrom .checkpoint import load_ckpt, save_checkpoint\nfrom .compat import meshgrid\nfrom .demo_utils import *\nfrom .dist import *\nfrom .ema import *\nfrom .logger import WandbLogger, setup_logger\nfrom .lr_scheduler import LRScheduler\nfrom .metric import *\nfrom .model_utils import *\nfrom .setup_env import *\nfrom .visualize import *\n" }, { "path": "asone/detectors/yolox/yolox/utils/allreduce_norm.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport pickle\nfrom collections import OrderedDict\n\nimport torch\nfrom torch import distributed as dist\nfrom torch import nn\n\nfrom asone.detectors.yolox.yolox.utils.dist import _get_global_gloo_group, get_world_size\n\nASYNC_NORM = (\n nn.BatchNorm1d,\n nn.BatchNorm2d,\n nn.BatchNorm3d,\n nn.InstanceNorm1d,\n nn.InstanceNorm2d,\n nn.InstanceNorm3d,\n)\n\n__all__ = [\n \"get_async_norm_states\",\n \"pyobj2tensor\",\n \"tensor2pyobj\",\n \"all_reduce\",\n \"all_reduce_norm\",\n]\n\n\ndef get_async_norm_states(module):\n async_norm_states = OrderedDict()\n for name, child in module.named_modules():\n if isinstance(child, ASYNC_NORM):\n for k, v in child.state_dict().items():\n async_norm_states[\".\".join([name, k])] = v\n return async_norm_states\n\n\ndef pyobj2tensor(pyobj, device=\"cuda\"):\n \"\"\"serialize picklable python object to tensor\"\"\"\n storage = torch.ByteStorage.from_buffer(pickle.dumps(pyobj))\n return torch.ByteTensor(storage).to(device=device)\n\n\ndef tensor2pyobj(tensor):\n \"\"\"deserialize tensor to picklable python object\"\"\"\n return pickle.loads(tensor.cpu().numpy().tobytes())\n\n\ndef _get_reduce_op(op_name):\n return {\n \"sum\": dist.ReduceOp.SUM,\n \"mean\": dist.ReduceOp.SUM,\n }[op_name.lower()]\n\n\ndef all_reduce(py_dict, op=\"sum\", group=None):\n \"\"\"\n Apply all reduce function for python dict object.\n NOTE: make sure that every py_dict has the same keys and values are in the same shape.\n\n Args:\n py_dict (dict): dict to apply all reduce op.\n op (str): operator, could be \"sum\" or \"mean\".\n \"\"\"\n world_size = get_world_size()\n if world_size == 1:\n return py_dict\n if group is None:\n group = _get_global_gloo_group()\n if dist.get_world_size(group) == 1:\n return py_dict\n\n # all reduce logic across different devices.\n py_key = list(py_dict.keys())\n py_key_tensor = pyobj2tensor(py_key)\n dist.broadcast(py_key_tensor, src=0)\n py_key = tensor2pyobj(py_key_tensor)\n\n tensor_shapes = [py_dict[k].shape for k in py_key]\n tensor_numels = [py_dict[k].numel() for k in py_key]\n\n flatten_tensor = torch.cat([py_dict[k].flatten() for k in py_key])\n dist.all_reduce(flatten_tensor, op=_get_reduce_op(op))\n if op == \"mean\":\n flatten_tensor /= world_size\n\n split_tensors = [\n x.reshape(shape)\n for x, shape in zip(torch.split(flatten_tensor, tensor_numels), tensor_shapes)\n ]\n return OrderedDict({k: v for k, v in zip(py_key, split_tensors)})\n\n\ndef all_reduce_norm(module):\n \"\"\"\n All reduce norm statistics in different devices.\n \"\"\"\n states = get_async_norm_states(module)\n states = all_reduce(states, op=\"mean\")\n module.load_state_dict(states, strict=False)\n" }, { "path": "asone/detectors/yolox/yolox/utils/boxes.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport numpy as np\n\nimport torch\nimport torchvision\n\n__all__ = [\n \"filter_box\",\n \"postprocess\",\n \"bboxes_iou\",\n \"matrix_iou\",\n \"adjust_box_anns\",\n \"xyxy2xywh\",\n \"xyxy2cxcywh\",\n]\n\n\ndef filter_box(output, scale_range):\n \"\"\"\n output: (N, 5+class) shape\n \"\"\"\n min_scale, max_scale = scale_range\n w = output[:, 2] - output[:, 0]\n h = output[:, 3] - output[:, 1]\n keep = (w * h > min_scale * min_scale) & (w * h < max_scale * max_scale)\n return output[keep]\n\n\ndef postprocess(prediction, num_classes, conf_thre=0.7, nms_thre=0.45, class_agnostic=False):\n box_corner = prediction.new(prediction.shape)\n box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2\n box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2\n box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2\n box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2\n prediction[:, :, :4] = box_corner[:, :, :4]\n\n output = [None for _ in range(len(prediction))]\n for i, image_pred in enumerate(prediction):\n\n # If none are remaining => process next image\n if not image_pred.size(0):\n continue\n # Get score and class with highest confidence\n class_conf, class_pred = torch.max(image_pred[:, 5: 5 + num_classes], 1, keepdim=True)\n\n conf_mask = (image_pred[:, 4] * class_conf.squeeze() >= conf_thre).squeeze()\n # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred)\n detections = torch.cat((image_pred[:, :5], class_conf, class_pred.float()), 1)\n detections = detections[conf_mask]\n # print(detections.shape)\n # exit()\n if not detections.size(0):\n continue\n if class_agnostic:\n nms_out_index = torchvision.ops.nms(\n detections[:, :4],\n detections[:, 4] * detections[:, 5],\n nms_thre,\n )\n else:\n nms_out_index = torchvision.ops.batched_nms(\n detections[:, :4],\n detections[:, 4] * detections[:, 5],\n detections[:, 6],\n nms_thre,\n )\n\n detections = detections[nms_out_index]\n if output[i] is None:\n output[i] = detections\n else:\n output[i] = torch.cat((output[i], detections))\n\n return output\n\n\ndef bboxes_iou(bboxes_a, bboxes_b, xyxy=True):\n if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4:\n raise IndexError\n\n if xyxy:\n tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2])\n br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:])\n area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1)\n area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1)\n else:\n tl = torch.max(\n (bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2),\n (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2),\n )\n br = torch.min(\n (bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2),\n (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2),\n )\n\n area_a = torch.prod(bboxes_a[:, 2:], 1)\n area_b = torch.prod(bboxes_b[:, 2:], 1)\n en = (tl < br).type(tl.type()).prod(dim=2)\n area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all())\n return area_i / (area_a[:, None] + area_b - area_i)\n\n\ndef matrix_iou(a, b):\n \"\"\"\n return iou of a and b, numpy version for data augenmentation\n \"\"\"\n lt = np.maximum(a[:, np.newaxis, :2], b[:, :2])\n rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:])\n\n area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2)\n area_a = np.prod(a[:, 2:] - a[:, :2], axis=1)\n area_b = np.prod(b[:, 2:] - b[:, :2], axis=1)\n return area_i / (area_a[:, np.newaxis] + area_b - area_i + 1e-12)\n\n\ndef adjust_box_anns(bbox, scale_ratio, padw, padh, w_max, h_max):\n bbox[:, 0::2] = np.clip(bbox[:, 0::2] * scale_ratio + padw, 0, w_max)\n bbox[:, 1::2] = np.clip(bbox[:, 1::2] * scale_ratio + padh, 0, h_max)\n return bbox\n\n\ndef xyxy2xywh(bboxes):\n bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0]\n bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1]\n return bboxes\n\n\ndef xyxy2cxcywh(bboxes):\n bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0]\n bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1]\n bboxes[:, 0] = bboxes[:, 0] + bboxes[:, 2] * 0.5\n bboxes[:, 1] = bboxes[:, 1] + bboxes[:, 3] * 0.5\n return bboxes\n" }, { "path": "asone/detectors/yolox/yolox/utils/checkpoint.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\nimport os\nimport shutil\nfrom loguru import logger\n\nimport torch\n\n\ndef load_ckpt(model, ckpt):\n model_state_dict = model.state_dict()\n load_dict = {}\n for key_model, v in model_state_dict.items():\n if key_model not in ckpt:\n logger.warning(\n \"{} is not in the ckpt. Please double check and see if this is desired.\".format(\n key_model\n )\n )\n continue\n v_ckpt = ckpt[key_model]\n if v.shape != v_ckpt.shape:\n logger.warning(\n \"Shape of {} in checkpoint is {}, while shape of {} in model is {}.\".format(\n key_model, v_ckpt.shape, key_model, v.shape\n )\n )\n continue\n load_dict[key_model] = v_ckpt\n\n model.load_state_dict(load_dict, strict=False)\n return model\n\n\ndef save_checkpoint(state, is_best, save_dir, model_name=\"\"):\n if not os.path.exists(save_dir):\n os.makedirs(save_dir)\n filename = os.path.join(save_dir, model_name + \"_ckpt.pth\")\n torch.save(state, filename)\n if is_best:\n best_filename = os.path.join(save_dir, \"best_ckpt.pth\")\n shutil.copyfile(filename, best_filename)\n" }, { "path": "asone/detectors/yolox/yolox/utils/compat.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n\nimport torch\n\n_TORCH_VER = [int(x) for x in torch.__version__.split(\".\")[:2]]\n\n__all__ = [\"meshgrid\"]\n\n\ndef meshgrid(*tensors):\n if _TORCH_VER >= [1, 10]:\n return torch.meshgrid(*tensors, indexing=\"ij\")\n else:\n return torch.meshgrid(*tensors)\n" }, { "path": "asone/detectors/yolox/yolox/utils/demo_utils.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport os\n\nimport numpy as np\n\n__all__ = [\"mkdir\", \"nms\", \"multiclass_nms\", \"demo_postprocess\"]\n\n\ndef mkdir(path):\n if not os.path.exists(path):\n os.makedirs(path)\n\n\ndef nms(boxes, scores, nms_thr):\n \"\"\"Single class NMS implemented in Numpy.\"\"\"\n x1 = boxes[:, 0]\n y1 = boxes[:, 1]\n x2 = boxes[:, 2]\n y2 = boxes[:, 3]\n\n areas = (x2 - x1 + 1) * (y2 - y1 + 1)\n order = scores.argsort()[::-1]\n\n keep = []\n while order.size > 0:\n i = order[0]\n keep.append(i)\n xx1 = np.maximum(x1[i], x1[order[1:]])\n yy1 = np.maximum(y1[i], y1[order[1:]])\n xx2 = np.minimum(x2[i], x2[order[1:]])\n yy2 = np.minimum(y2[i], y2[order[1:]])\n\n w = np.maximum(0.0, xx2 - xx1 + 1)\n h = np.maximum(0.0, yy2 - yy1 + 1)\n inter = w * h\n ovr = inter / (areas[i] + areas[order[1:]] - inter)\n\n inds = np.where(ovr <= nms_thr)[0]\n order = order[inds + 1]\n\n return keep\n\n\ndef multiclass_nms(boxes, scores, nms_thr, score_thr, class_agnostic=True):\n \"\"\"Multiclass NMS implemented in Numpy\"\"\"\n if class_agnostic:\n nms_method = multiclass_nms_class_agnostic\n else:\n nms_method = multiclass_nms_class_aware\n return nms_method(boxes, scores, nms_thr, score_thr)\n\n\ndef multiclass_nms_class_aware(boxes, scores, nms_thr, score_thr):\n \"\"\"Multiclass NMS implemented in Numpy. Class-aware version.\"\"\"\n final_dets = []\n num_classes = scores.shape[1]\n for cls_ind in range(num_classes):\n cls_scores = scores[:, cls_ind]\n valid_score_mask = cls_scores > score_thr\n if valid_score_mask.sum() == 0:\n continue\n else:\n valid_scores = cls_scores[valid_score_mask]\n valid_boxes = boxes[valid_score_mask]\n keep = nms(valid_boxes, valid_scores, nms_thr)\n if len(keep) > 0:\n cls_inds = np.ones((len(keep), 1)) * cls_ind\n dets = np.concatenate(\n [valid_boxes[keep], valid_scores[keep, None], cls_inds], 1\n )\n final_dets.append(dets)\n if len(final_dets) == 0:\n return None\n return np.concatenate(final_dets, 0)\n\n\ndef multiclass_nms_class_agnostic(boxes, scores, nms_thr, score_thr):\n \"\"\"Multiclass NMS implemented in Numpy. Class-agnostic version.\"\"\"\n cls_inds = scores.argmax(1)\n cls_scores = scores[np.arange(len(cls_inds)), cls_inds]\n\n valid_score_mask = cls_scores > score_thr\n if valid_score_mask.sum() == 0:\n return None\n valid_scores = cls_scores[valid_score_mask]\n valid_boxes = boxes[valid_score_mask]\n valid_cls_inds = cls_inds[valid_score_mask]\n keep = nms(valid_boxes, valid_scores, nms_thr)\n if keep:\n dets = np.concatenate(\n [valid_boxes[keep], valid_scores[keep, None], valid_cls_inds[keep, None]], 1\n )\n return dets\n\n\ndef demo_postprocess(outputs, img_size, p6=False):\n\n grids = []\n expanded_strides = []\n\n if not p6:\n strides = [8, 16, 32]\n else:\n strides = [8, 16, 32, 64]\n\n hsizes = [img_size[0] // stride for stride in strides]\n wsizes = [img_size[1] // stride for stride in strides]\n\n for hsize, wsize, stride in zip(hsizes, wsizes, strides):\n xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize))\n grid = np.stack((xv, yv), 2).reshape(1, -1, 2)\n grids.append(grid)\n shape = grid.shape[:2]\n expanded_strides.append(np.full((*shape, 1), stride))\n\n grids = np.concatenate(grids, 1)\n expanded_strides = np.concatenate(expanded_strides, 1)\n outputs[..., :2] = (outputs[..., :2] + grids) * expanded_strides\n outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * expanded_strides\n\n return outputs\n" }, { "path": "asone/detectors/yolox/yolox/utils/dist.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# This file mainly comes from\n# https://github.com/facebookresearch/detectron2/blob/master/detectron2/utils/comm.py\n# Copyright (c) Facebook, Inc. and its affiliates.\n# Copyright (c) Megvii Inc. All rights reserved.\n\"\"\"\nThis file contains primitives for multi-gpu communication.\nThis is useful when doing distributed training.\n\"\"\"\n\nimport functools\nimport os\nimport pickle\nimport time\nfrom contextlib import contextmanager\nfrom loguru import logger\n\nimport numpy as np\n\nimport torch\nfrom torch import distributed as dist\n\n__all__ = [\n \"get_num_devices\",\n \"wait_for_the_master\",\n \"is_main_process\",\n \"synchronize\",\n \"get_world_size\",\n \"get_rank\",\n \"get_local_rank\",\n \"get_local_size\",\n \"time_synchronized\",\n \"gather\",\n \"all_gather\",\n]\n\n_LOCAL_PROCESS_GROUP = None\n\n\ndef get_num_devices():\n gpu_list = os.getenv('CUDA_VISIBLE_DEVICES', None)\n if gpu_list is not None:\n return len(gpu_list.split(','))\n else:\n devices_list_info = os.popen(\"nvidia-smi -L\")\n devices_list_info = devices_list_info.read().strip().split(\"\\n\")\n return len(devices_list_info)\n\n\n@contextmanager\ndef wait_for_the_master(local_rank: int = None):\n \"\"\"\n Make all processes waiting for the master to do some task.\n\n Args:\n local_rank (int): the rank of the current process. Default to None.\n If None, it will use the rank of the current process.\n \"\"\"\n if local_rank is None:\n local_rank = get_local_rank()\n\n if local_rank > 0:\n dist.barrier()\n yield\n if local_rank == 0:\n if not dist.is_available():\n return\n if not dist.is_initialized():\n return\n else:\n dist.barrier()\n\n\ndef synchronize():\n \"\"\"\n Helper function to synchronize (barrier) among all processes when using distributed training\n \"\"\"\n if not dist.is_available():\n return\n if not dist.is_initialized():\n return\n world_size = dist.get_world_size()\n if world_size == 1:\n return\n dist.barrier()\n\n\ndef get_world_size() -> int:\n if not dist.is_available():\n return 1\n if not dist.is_initialized():\n return 1\n return dist.get_world_size()\n\n\ndef get_rank() -> int:\n if not dist.is_available():\n return 0\n if not dist.is_initialized():\n return 0\n return dist.get_rank()\n\n\ndef get_local_rank() -> int:\n \"\"\"\n Returns:\n The rank of the current process within the local (per-machine) process group.\n \"\"\"\n if _LOCAL_PROCESS_GROUP is None:\n return get_rank()\n\n if not dist.is_available():\n return 0\n if not dist.is_initialized():\n return 0\n return dist.get_rank(group=_LOCAL_PROCESS_GROUP)\n\n\ndef get_local_size() -> int:\n \"\"\"\n Returns:\n The size of the per-machine process group, i.e. the number of processes per machine.\n \"\"\"\n if not dist.is_available():\n return 1\n if not dist.is_initialized():\n return 1\n return dist.get_world_size(group=_LOCAL_PROCESS_GROUP)\n\n\ndef is_main_process() -> bool:\n return get_rank() == 0\n\n\n@functools.lru_cache()\ndef _get_global_gloo_group():\n \"\"\"\n Return a process group based on gloo backend, containing all the ranks\n The result is cached.\n \"\"\"\n if dist.get_backend() == \"nccl\":\n return dist.new_group(backend=\"gloo\")\n else:\n return dist.group.WORLD\n\n\ndef _serialize_to_tensor(data, group):\n backend = dist.get_backend(group)\n assert backend in [\"gloo\", \"nccl\"]\n device = torch.device(\"cpu\" if backend == \"gloo\" else \"cuda\")\n\n buffer = pickle.dumps(data)\n if len(buffer) > 1024 ** 3:\n logger.warning(\n \"Rank {} trying to all-gather {:.2f} GB of data on device {}\".format(\n get_rank(), len(buffer) / (1024 ** 3), device\n )\n )\n storage = torch.ByteStorage.from_buffer(buffer)\n tensor = torch.ByteTensor(storage).to(device=device)\n return tensor\n\n\ndef _pad_to_largest_tensor(tensor, group):\n \"\"\"\n Returns:\n list[int]: size of the tensor, on each rank\n Tensor: padded tensor that has the max size\n \"\"\"\n world_size = dist.get_world_size(group=group)\n assert (\n world_size >= 1\n ), \"comm.gather/all_gather must be called from ranks within the given group!\"\n local_size = torch.tensor([tensor.numel()], dtype=torch.int64, device=tensor.device)\n size_list = [\n torch.zeros([1], dtype=torch.int64, device=tensor.device)\n for _ in range(world_size)\n ]\n dist.all_gather(size_list, local_size, group=group)\n size_list = [int(size.item()) for size in size_list]\n\n max_size = max(size_list)\n\n # we pad the tensor because torch all_gather does not support\n # gathering tensors of different shapes\n if local_size != max_size:\n padding = torch.zeros(\n (max_size - local_size,), dtype=torch.uint8, device=tensor.device\n )\n tensor = torch.cat((tensor, padding), dim=0)\n return size_list, tensor\n\n\ndef all_gather(data, group=None):\n \"\"\"\n Run all_gather on arbitrary picklable data (not necessarily tensors).\n\n Args:\n data: any picklable object\n group: a torch process group. By default, will use a group which\n contains all ranks on gloo backend.\n Returns:\n list[data]: list of data gathered from each rank\n \"\"\"\n if get_world_size() == 1:\n return [data]\n if group is None:\n group = _get_global_gloo_group()\n if dist.get_world_size(group) == 1:\n return [data]\n\n tensor = _serialize_to_tensor(data, group)\n\n size_list, tensor = _pad_to_largest_tensor(tensor, group)\n max_size = max(size_list)\n\n # receiving Tensor from all ranks\n tensor_list = [\n torch.empty((max_size,), dtype=torch.uint8, device=tensor.device)\n for _ in size_list\n ]\n dist.all_gather(tensor_list, tensor, group=group)\n\n data_list = []\n for size, tensor in zip(size_list, tensor_list):\n buffer = tensor.cpu().numpy().tobytes()[:size]\n data_list.append(pickle.loads(buffer))\n\n return data_list\n\n\ndef gather(data, dst=0, group=None):\n \"\"\"\n Run gather on arbitrary picklable data (not necessarily tensors).\n\n Args:\n data: any picklable object\n dst (int): destination rank\n group: a torch process group. By default, will use a group which\n contains all ranks on gloo backend.\n\n Returns:\n list[data]: on dst, a list of data gathered from each rank. Otherwise,\n an empty list.\n \"\"\"\n if get_world_size() == 1:\n return [data]\n if group is None:\n group = _get_global_gloo_group()\n if dist.get_world_size(group=group) == 1:\n return [data]\n rank = dist.get_rank(group=group)\n\n tensor = _serialize_to_tensor(data, group)\n size_list, tensor = _pad_to_largest_tensor(tensor, group)\n\n # receiving Tensor from all ranks\n if rank == dst:\n max_size = max(size_list)\n tensor_list = [\n torch.empty((max_size,), dtype=torch.uint8, device=tensor.device)\n for _ in size_list\n ]\n dist.gather(tensor, tensor_list, dst=dst, group=group)\n\n data_list = []\n for size, tensor in zip(size_list, tensor_list):\n buffer = tensor.cpu().numpy().tobytes()[:size]\n data_list.append(pickle.loads(buffer))\n return data_list\n else:\n dist.gather(tensor, [], dst=dst, group=group)\n return []\n\n\ndef shared_random_seed():\n \"\"\"\n Returns:\n int: a random number that is the same across all workers.\n If workers need a shared RNG, they can use this shared seed to\n create one.\n All workers must call this function, otherwise it will deadlock.\n \"\"\"\n ints = np.random.randint(2 ** 31)\n all_ints = all_gather(ints)\n return all_ints[0]\n\n\ndef time_synchronized():\n \"\"\"pytorch-accurate time\"\"\"\n if torch.cuda.is_available():\n torch.cuda.synchronize()\n return time.time()\n" }, { "path": "asone/detectors/yolox/yolox/utils/ema.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\nimport math\nfrom copy import deepcopy\n\nimport torch\nimport torch.nn as nn\n\n__all__ = [\"ModelEMA\", \"is_parallel\"]\n\n\ndef is_parallel(model):\n \"\"\"check if model is in parallel mode.\"\"\"\n parallel_type = (\n nn.parallel.DataParallel,\n nn.parallel.DistributedDataParallel,\n )\n return isinstance(model, parallel_type)\n\n\nclass ModelEMA:\n \"\"\"\n Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models\n Keep a moving average of everything in the model state_dict (parameters and buffers).\n This is intended to allow functionality like\n https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage\n A smoothed version of the weights is necessary for some training schemes to perform well.\n This class is sensitive where it is initialized in the sequence of model init,\n GPU assignment and distributed training wrappers.\n \"\"\"\n\n def __init__(self, model, decay=0.9999, updates=0):\n \"\"\"\n Args:\n model (nn.Module): model to apply EMA.\n decay (float): ema decay reate.\n updates (int): counter of EMA updates.\n \"\"\"\n # Create EMA(FP32)\n self.ema = deepcopy(model.module if is_parallel(model) else model).eval()\n self.updates = updates\n # decay exponential ramp (to help early epochs)\n self.decay = lambda x: decay * (1 - math.exp(-x / 2000))\n for p in self.ema.parameters():\n p.requires_grad_(False)\n\n def update(self, model):\n # Update EMA parameters\n with torch.no_grad():\n self.updates += 1\n d = self.decay(self.updates)\n\n msd = (\n model.module.state_dict() if is_parallel(model) else model.state_dict()\n ) # model state_dict\n for k, v in self.ema.state_dict().items():\n if v.dtype.is_floating_point:\n v *= d\n v += (1.0 - d) * msd[k].detach()\n" }, { "path": "asone/detectors/yolox/yolox/utils/logger.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport inspect\nimport os\nimport sys\nfrom collections import defaultdict\nfrom loguru import logger\n\nimport cv2\nimport numpy as np\n\nimport torch\n\n\ndef get_caller_name(depth=0):\n \"\"\"\n Args:\n depth (int): Depth of caller conext, use 0 for caller depth.\n Default value: 0.\n\n Returns:\n str: module name of the caller\n \"\"\"\n # the following logic is a little bit faster than inspect.stack() logic\n frame = inspect.currentframe().f_back\n for _ in range(depth):\n frame = frame.f_back\n\n return frame.f_globals[\"__name__\"]\n\n\nclass StreamToLoguru:\n \"\"\"\n stream object that redirects writes to a logger instance.\n \"\"\"\n\n def __init__(self, level=\"INFO\", caller_names=(\"apex\", \"pycocotools\")):\n \"\"\"\n Args:\n level(str): log level string of loguru. Default value: \"INFO\".\n caller_names(tuple): caller names of redirected module.\n Default value: (apex, pycocotools).\n \"\"\"\n self.level = level\n self.linebuf = \"\"\n self.caller_names = caller_names\n\n def write(self, buf):\n full_name = get_caller_name(depth=1)\n module_name = full_name.rsplit(\".\", maxsplit=-1)[0]\n if module_name in self.caller_names:\n for line in buf.rstrip().splitlines():\n # use caller level log\n logger.opt(depth=2).log(self.level, line.rstrip())\n else:\n sys.__stdout__.write(buf)\n\n def flush(self):\n pass\n\n def isatty(self):\n # when using colab, jax is installed by default and issue like\n # https://github.com/Megvii-BaseDetection/YOLOX/issues/1437 might be raised\n # due to missing attribute like`isatty`.\n # For more details, checked the following link:\n # https://github.com/google/jax/blob/10720258ea7fb5bde997dfa2f3f71135ab7a6733/jax/_src/pretty_printer.py#L54 # noqa\n return True\n\n\ndef redirect_sys_output(log_level=\"INFO\"):\n redirect_logger = StreamToLoguru(log_level)\n sys.stderr = redirect_logger\n sys.stdout = redirect_logger\n\n\ndef setup_logger(save_dir, distributed_rank=0, filename=\"log.txt\", mode=\"a\"):\n \"\"\"setup logger for training and testing.\n Args:\n save_dir(str): location to save log file\n distributed_rank(int): device rank when multi-gpu environment\n filename (string): log save name.\n mode(str): log file write mode, `append` or `override`. default is `a`.\n\n Return:\n logger instance.\n \"\"\"\n loguru_format = (\n \"{time:YYYY-MM-DD HH:mm:ss} | \"\n \"{level: <8} | \"\n \"{name}:{line} - {message}\"\n )\n\n logger.remove()\n save_file = os.path.join(save_dir, filename)\n if mode == \"o\" and os.path.exists(save_file):\n os.remove(save_file)\n # only keep logger in rank0 process\n if distributed_rank == 0:\n logger.add(\n sys.stderr,\n format=loguru_format,\n level=\"INFO\",\n enqueue=True,\n )\n logger.add(save_file)\n\n # redirect stdout/stderr to loguru\n redirect_sys_output(\"INFO\")\n\n\nclass WandbLogger(object):\n \"\"\"\n Log training runs, datasets, models, and predictions to Weights & Biases.\n This logger sends information to W&B at wandb.ai.\n By default, this information includes hyperparameters,\n system configuration and metrics, model metrics,\n and basic data metrics and analyses.\n\n For more information, please refer to:\n https://docs.wandb.ai/guides/track\n https://docs.wandb.ai/guides/integrations/other/yolox\n \"\"\"\n def __init__(self,\n project=None,\n name=None,\n id=None,\n entity=None,\n save_dir=None,\n config=None,\n val_dataset=None,\n num_eval_images=100,\n log_checkpoints=False,\n **kwargs):\n \"\"\"\n Args:\n project (str): wandb project name.\n name (str): wandb run name.\n id (str): wandb run id.\n entity (str): wandb entity name.\n save_dir (str): save directory.\n config (dict): config dict.\n val_dataset (Dataset): validation dataset.\n num_eval_images (int): number of images from the validation set to log.\n log_checkpoints (bool): log checkpoints\n **kwargs: other kwargs.\n\n Usage:\n Any arguments for wandb.init can be provided on the command line using\n the prefix `wandb-`.\n Example\n ```\n python tools/train.py .... --logger wandb wandb-project \\\n wandb-name \\\n wandb-id \\\n wandb-save_dir \\\n wandb-num_eval_imges \\\n wandb-log_checkpoints \n ```\n The val_dataset argument is not open to the command line.\n \"\"\"\n try:\n import wandb\n self.wandb = wandb\n except ModuleNotFoundError:\n raise ModuleNotFoundError(\n \"wandb is not installed.\"\n \"Please install wandb using pip install wandb\"\n )\n\n self.project = project\n self.name = name\n self.id = id\n self.save_dir = save_dir\n self.config = config\n self.kwargs = kwargs\n self.entity = entity\n self._run = None\n self.val_artifact = None\n if num_eval_images == -1:\n self.num_log_images = len(val_dataset)\n else:\n self.num_log_images = min(num_eval_images, len(val_dataset))\n self.log_checkpoints = (log_checkpoints == \"True\" or log_checkpoints == \"true\")\n self._wandb_init = dict(\n project=self.project,\n name=self.name,\n id=self.id,\n entity=self.entity,\n dir=self.save_dir,\n resume=\"allow\"\n )\n self._wandb_init.update(**kwargs)\n\n _ = self.run\n\n if self.config:\n self.run.config.update(self.config)\n self.run.define_metric(\"train/epoch\")\n self.run.define_metric(\"val/*\", step_metric=\"train/epoch\")\n self.run.define_metric(\"train/step\")\n self.run.define_metric(\"train/*\", step_metric=\"train/step\")\n\n if val_dataset and self.num_log_images != 0:\n self.cats = val_dataset.cats\n self.id_to_class = {\n cls['id']: cls['name'] for cls in self.cats\n }\n self._log_validation_set(val_dataset)\n\n @property\n def run(self):\n if self._run is None:\n if self.wandb.run is not None:\n logger.info(\n \"There is a wandb run already in progress \"\n \"and newly created instances of `WandbLogger` will reuse\"\n \" this run. If this is not desired, call `wandb.finish()`\"\n \"before instantiating `WandbLogger`.\"\n )\n self._run = self.wandb.run\n else:\n self._run = self.wandb.init(**self._wandb_init)\n return self._run\n\n def _log_validation_set(self, val_dataset):\n \"\"\"\n Log validation set to wandb.\n\n Args:\n val_dataset (Dataset): validation dataset.\n \"\"\"\n if self.val_artifact is None:\n self.val_artifact = self.wandb.Artifact(name=\"validation_images\", type=\"dataset\")\n self.val_table = self.wandb.Table(columns=[\"id\", \"input\"])\n\n for i in range(self.num_log_images):\n data_point = val_dataset[i]\n img = data_point[0]\n id = data_point[3]\n img = np.transpose(img, (1, 2, 0))\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n self.val_table.add_data(\n id.item(),\n self.wandb.Image(img)\n )\n\n self.val_artifact.add(self.val_table, \"validation_images_table\")\n self.run.use_artifact(self.val_artifact)\n self.val_artifact.wait()\n\n def log_metrics(self, metrics, step=None):\n \"\"\"\n Args:\n metrics (dict): metrics dict.\n step (int): step number.\n \"\"\"\n\n for k, v in metrics.items():\n if isinstance(v, torch.Tensor):\n metrics[k] = v.item()\n\n if step is not None:\n metrics.update({\"train/step\": step})\n self.run.log(metrics)\n else:\n self.run.log(metrics)\n\n def log_images(self, predictions):\n if len(predictions) == 0 or self.val_artifact is None or self.num_log_images == 0:\n return\n\n table_ref = self.val_artifact.get(\"validation_images_table\")\n\n columns = [\"id\", \"predicted\"]\n for cls in self.cats:\n columns.append(cls[\"name\"])\n\n result_table = self.wandb.Table(columns=columns)\n for idx, val in table_ref.iterrows():\n\n avg_scores = defaultdict(int)\n num_occurrences = defaultdict(int)\n\n if val[0] in predictions:\n prediction = predictions[val[0]]\n boxes = []\n\n for i in range(len(prediction[\"bboxes\"])):\n bbox = prediction[\"bboxes\"][i]\n x0 = bbox[0]\n y0 = bbox[1]\n x1 = bbox[2]\n y1 = bbox[3]\n box = {\n \"position\": {\n \"minX\": min(x0, x1),\n \"minY\": min(y0, y1),\n \"maxX\": max(x0, x1),\n \"maxY\": max(y0, y1)\n },\n \"class_id\": prediction[\"categories\"][i],\n \"domain\": \"pixel\"\n }\n avg_scores[\n self.id_to_class[prediction[\"categories\"][i]]\n ] += prediction[\"scores\"][i]\n num_occurrences[self.id_to_class[prediction[\"categories\"][i]]] += 1\n boxes.append(box)\n else:\n boxes = []\n\n average_class_score = []\n for cls in self.cats:\n if cls[\"name\"] not in num_occurrences:\n score = 0\n else:\n score = avg_scores[cls[\"name\"]] / num_occurrences[cls[\"name\"]]\n average_class_score.append(score)\n result_table.add_data(\n idx,\n self.wandb.Image(val[1], boxes={\n \"prediction\": {\n \"box_data\": boxes,\n \"class_labels\": self.id_to_class\n }\n }\n ),\n *average_class_score\n )\n\n self.wandb.log({\"val_results/result_table\": result_table})\n\n def save_checkpoint(self, save_dir, model_name, is_best, metadata=None):\n \"\"\"\n Args:\n save_dir (str): save directory.\n model_name (str): model name.\n is_best (bool): whether the model is the best model.\n metadata (dict): metadata to save corresponding to the checkpoint.\n \"\"\"\n\n if not self.log_checkpoints:\n return\n\n if \"epoch\" in metadata:\n epoch = metadata[\"epoch\"]\n else:\n epoch = None\n\n filename = os.path.join(save_dir, model_name + \"_ckpt.pth\")\n artifact = self.wandb.Artifact(\n name=f\"run_{self.run.id}_model\",\n type=\"model\",\n metadata=metadata\n )\n artifact.add_file(filename, name=\"model_ckpt.pth\")\n\n aliases = [\"latest\"]\n\n if is_best:\n aliases.append(\"best\")\n\n if epoch:\n aliases.append(f\"epoch-{epoch}\")\n\n self.run.log_artifact(artifact, aliases=aliases)\n\n def finish(self):\n self.run.finish()\n\n @classmethod\n def initialize_wandb_logger(cls, args, exp, val_dataset):\n wandb_params = dict()\n prefix = \"wandb-\"\n for k, v in zip(args.opts[0::2], args.opts[1::2]):\n if k.startswith(\"wandb-\"):\n try:\n wandb_params.update({k[len(prefix):]: int(v)})\n except ValueError:\n wandb_params.update({k[len(prefix):]: v})\n\n return cls(config=vars(exp), val_dataset=val_dataset, **wandb_params)\n" }, { "path": "asone/detectors/yolox/yolox/utils/lr_scheduler.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport math\nfrom functools import partial\n\n\nclass LRScheduler:\n def __init__(self, name, lr, iters_per_epoch, total_epochs, **kwargs):\n \"\"\"\n Supported lr schedulers: [cos, warmcos, multistep]\n\n Args:\n lr (float): learning rate.\n iters_per_peoch (int): number of iterations in one epoch.\n total_epochs (int): number of epochs in training.\n kwargs (dict):\n - cos: None\n - warmcos: [warmup_epochs, warmup_lr_start (default 1e-6)]\n - multistep: [milestones (epochs), gamma (default 0.1)]\n \"\"\"\n\n self.lr = lr\n self.iters_per_epoch = iters_per_epoch\n self.total_epochs = total_epochs\n self.total_iters = iters_per_epoch * total_epochs\n\n self.__dict__.update(kwargs)\n\n self.lr_func = self._get_lr_func(name)\n\n def update_lr(self, iters):\n return self.lr_func(iters)\n\n def _get_lr_func(self, name):\n if name == \"cos\": # cosine lr schedule\n lr_func = partial(cos_lr, self.lr, self.total_iters)\n elif name == \"warmcos\":\n warmup_total_iters = self.iters_per_epoch * self.warmup_epochs\n warmup_lr_start = getattr(self, \"warmup_lr_start\", 1e-6)\n lr_func = partial(\n warm_cos_lr,\n self.lr,\n self.total_iters,\n warmup_total_iters,\n warmup_lr_start,\n )\n elif name == \"yoloxwarmcos\":\n warmup_total_iters = self.iters_per_epoch * self.warmup_epochs\n no_aug_iters = self.iters_per_epoch * self.no_aug_epochs\n warmup_lr_start = getattr(self, \"warmup_lr_start\", 0)\n min_lr_ratio = getattr(self, \"min_lr_ratio\", 0.2)\n lr_func = partial(\n yolox_warm_cos_lr,\n self.lr,\n min_lr_ratio,\n self.total_iters,\n warmup_total_iters,\n warmup_lr_start,\n no_aug_iters,\n )\n elif name == \"yoloxsemiwarmcos\":\n warmup_lr_start = getattr(self, \"warmup_lr_start\", 0)\n min_lr_ratio = getattr(self, \"min_lr_ratio\", 0.2)\n warmup_total_iters = self.iters_per_epoch * self.warmup_epochs\n no_aug_iters = self.iters_per_epoch * self.no_aug_epochs\n normal_iters = self.iters_per_epoch * self.semi_epoch\n semi_iters = self.iters_per_epoch_semi * (\n self.total_epochs - self.semi_epoch - self.no_aug_epochs\n )\n lr_func = partial(\n yolox_semi_warm_cos_lr,\n self.lr,\n min_lr_ratio,\n warmup_lr_start,\n self.total_iters,\n normal_iters,\n no_aug_iters,\n warmup_total_iters,\n semi_iters,\n self.iters_per_epoch,\n self.iters_per_epoch_semi,\n )\n elif name == \"multistep\": # stepwise lr schedule\n milestones = [\n int(self.total_iters * milestone / self.total_epochs)\n for milestone in self.milestones\n ]\n gamma = getattr(self, \"gamma\", 0.1)\n lr_func = partial(multistep_lr, self.lr, milestones, gamma)\n else:\n raise ValueError(\"Scheduler version {} not supported.\".format(name))\n return lr_func\n\n\ndef cos_lr(lr, total_iters, iters):\n \"\"\"Cosine learning rate\"\"\"\n lr *= 0.5 * (1.0 + math.cos(math.pi * iters / total_iters))\n return lr\n\n\ndef warm_cos_lr(lr, total_iters, warmup_total_iters, warmup_lr_start, iters):\n \"\"\"Cosine learning rate with warm up.\"\"\"\n if iters <= warmup_total_iters:\n lr = (lr - warmup_lr_start) * iters / float(\n warmup_total_iters\n ) + warmup_lr_start\n else:\n lr *= 0.5 * (\n 1.0\n + math.cos(\n math.pi\n * (iters - warmup_total_iters)\n / (total_iters - warmup_total_iters)\n )\n )\n return lr\n\n\ndef yolox_warm_cos_lr(\n lr,\n min_lr_ratio,\n total_iters,\n warmup_total_iters,\n warmup_lr_start,\n no_aug_iter,\n iters,\n):\n \"\"\"Cosine learning rate with warm up.\"\"\"\n min_lr = lr * min_lr_ratio\n if iters <= warmup_total_iters:\n # lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start\n lr = (lr - warmup_lr_start) * pow(\n iters / float(warmup_total_iters), 2\n ) + warmup_lr_start\n elif iters >= total_iters - no_aug_iter:\n lr = min_lr\n else:\n lr = min_lr + 0.5 * (lr - min_lr) * (\n 1.0\n + math.cos(\n math.pi\n * (iters - warmup_total_iters)\n / (total_iters - warmup_total_iters - no_aug_iter)\n )\n )\n return lr\n\n\ndef yolox_semi_warm_cos_lr(\n lr,\n min_lr_ratio,\n warmup_lr_start,\n total_iters,\n normal_iters,\n no_aug_iters,\n warmup_total_iters,\n semi_iters,\n iters_per_epoch,\n iters_per_epoch_semi,\n iters,\n):\n \"\"\"Cosine learning rate with warm up.\"\"\"\n min_lr = lr * min_lr_ratio\n if iters <= warmup_total_iters:\n # lr = (lr - warmup_lr_start) * iters / float(warmup_total_iters) + warmup_lr_start\n lr = (lr - warmup_lr_start) * pow(\n iters / float(warmup_total_iters), 2\n ) + warmup_lr_start\n elif iters >= normal_iters + semi_iters:\n lr = min_lr\n elif iters <= normal_iters:\n lr = min_lr + 0.5 * (lr - min_lr) * (\n 1.0\n + math.cos(\n math.pi\n * (iters - warmup_total_iters)\n / (total_iters - warmup_total_iters - no_aug_iters)\n )\n )\n else:\n lr = min_lr + 0.5 * (lr - min_lr) * (\n 1.0\n + math.cos(\n math.pi\n * (\n normal_iters\n - warmup_total_iters\n + (iters - normal_iters)\n * iters_per_epoch\n * 1.0\n / iters_per_epoch_semi\n )\n / (total_iters - warmup_total_iters - no_aug_iters)\n )\n )\n return lr\n\n\ndef multistep_lr(lr, milestones, gamma, iters):\n \"\"\"MultiStep learning rate\"\"\"\n for milestone in milestones:\n lr *= gamma if iters >= milestone else 1.0\n return lr\n" }, { "path": "asone/detectors/yolox/yolox/utils/metric.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\nimport functools\nimport os\nimport time\nfrom collections import defaultdict, deque\n\nimport numpy as np\n\nimport torch\n\n__all__ = [\n \"AverageMeter\",\n \"MeterBuffer\",\n \"get_total_and_free_memory_in_Mb\",\n \"occupy_mem\",\n \"gpu_mem_usage\",\n]\n\n\ndef get_total_and_free_memory_in_Mb(cuda_device):\n devices_info_str = os.popen(\n \"nvidia-smi --query-gpu=memory.total,memory.used --format=csv,nounits,noheader\"\n )\n devices_info = devices_info_str.read().strip().split(\"\\n\")\n if \"CUDA_VISIBLE_DEVICES\" in os.environ:\n visible_devices = os.environ[\"CUDA_VISIBLE_DEVICES\"].split(',')\n cuda_device = int(visible_devices[cuda_device])\n total, used = devices_info[int(cuda_device)].split(\",\")\n return int(total), int(used)\n\n\ndef occupy_mem(cuda_device, mem_ratio=0.9):\n \"\"\"\n pre-allocate gpu memory for training to avoid memory Fragmentation.\n \"\"\"\n total, used = get_total_and_free_memory_in_Mb(cuda_device)\n max_mem = int(total * mem_ratio)\n block_mem = max_mem - used\n x = torch.cuda.FloatTensor(256, 1024, block_mem)\n del x\n time.sleep(5)\n\n\ndef gpu_mem_usage():\n \"\"\"\n Compute the GPU memory usage for the current device (MB).\n \"\"\"\n mem_usage_bytes = torch.cuda.max_memory_allocated()\n return mem_usage_bytes / (1024 * 1024)\n\n\nclass AverageMeter:\n \"\"\"Track a series of values and provide access to smoothed values over a\n window or the global series average.\n \"\"\"\n\n def __init__(self, window_size=50):\n self._deque = deque(maxlen=window_size)\n self._total = 0.0\n self._count = 0\n\n def update(self, value):\n self._deque.append(value)\n self._count += 1\n self._total += value\n\n @property\n def median(self):\n d = np.array(list(self._deque))\n return np.median(d)\n\n @property\n def avg(self):\n # if deque is empty, nan will be returned.\n d = np.array(list(self._deque))\n return d.mean()\n\n @property\n def global_avg(self):\n return self._total / max(self._count, 1e-5)\n\n @property\n def latest(self):\n return self._deque[-1] if len(self._deque) > 0 else None\n\n @property\n def total(self):\n return self._total\n\n def reset(self):\n self._deque.clear()\n self._total = 0.0\n self._count = 0\n\n def clear(self):\n self._deque.clear()\n\n\nclass MeterBuffer(defaultdict):\n \"\"\"Computes and stores the average and current value\"\"\"\n\n def __init__(self, window_size=20):\n factory = functools.partial(AverageMeter, window_size=window_size)\n super().__init__(factory)\n\n def reset(self):\n for v in self.values():\n v.reset()\n\n def get_filtered_meter(self, filter_key=\"time\"):\n return {k: v for k, v in self.items() if filter_key in k}\n\n def update(self, values=None, **kwargs):\n if values is None:\n values = {}\n values.update(kwargs)\n for k, v in values.items():\n if isinstance(v, torch.Tensor):\n v = v.detach()\n self[k].update(v)\n\n def clear_meters(self):\n for v in self.values():\n v.clear()\n" }, { "path": "asone/detectors/yolox/yolox/utils/model_utils.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport contextlib\nfrom copy import deepcopy\nfrom typing import Sequence\n\nimport torch\nimport torch.nn as nn\n\n__all__ = [\n \"fuse_conv_and_bn\",\n \"fuse_model\",\n \"get_model_info\",\n \"replace_module\",\n \"freeze_module\",\n \"adjust_status\",\n]\n\n\ndef get_model_info(model: nn.Module, tsize: Sequence[int]) -> str:\n from thop import profile\n\n stride = 64\n img = torch.zeros((1, 3, stride, stride), device=next(model.parameters()).device)\n flops, params = profile(deepcopy(model), inputs=(img,), verbose=False)\n params /= 1e6\n flops /= 1e9\n flops *= tsize[0] * tsize[1] / stride / stride * 2 # Gflops\n info = \"Params: {:.2f}M, Gflops: {:.2f}\".format(params, flops)\n return info\n\n\ndef fuse_conv_and_bn(conv: nn.Conv2d, bn: nn.BatchNorm2d) -> nn.Conv2d:\n \"\"\"\n Fuse convolution and batchnorm layers.\n check more info on https://tehnokv.com/posts/fusing-batchnorm-and-conv/\n\n Args:\n conv (nn.Conv2d): convolution to fuse.\n bn (nn.BatchNorm2d): batchnorm to fuse.\n\n Returns:\n nn.Conv2d: fused convolution behaves the same as the input conv and bn.\n \"\"\"\n fusedconv = (\n nn.Conv2d(\n conv.in_channels,\n conv.out_channels,\n kernel_size=conv.kernel_size,\n stride=conv.stride,\n padding=conv.padding,\n groups=conv.groups,\n bias=True,\n )\n .requires_grad_(False)\n .to(conv.weight.device)\n )\n\n # prepare filters\n w_conv = conv.weight.clone().view(conv.out_channels, -1)\n w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))\n fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape))\n\n # prepare spatial bias\n b_conv = (\n torch.zeros(conv.weight.size(0), device=conv.weight.device)\n if conv.bias is None\n else conv.bias\n )\n b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(\n torch.sqrt(bn.running_var + bn.eps)\n )\n fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)\n\n return fusedconv\n\n\ndef fuse_model(model: nn.Module) -> nn.Module:\n \"\"\"fuse conv and bn in model\n\n Args:\n model (nn.Module): model to fuse\n\n Returns:\n nn.Module: fused model\n \"\"\"\n from asone.detectors.yolox.yolox.models.network_blocks import BaseConv\n\n for m in model.modules():\n if type(m) is BaseConv and hasattr(m, \"bn\"):\n m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv\n delattr(m, \"bn\") # remove batchnorm\n m.forward = m.fuseforward # update forward\n return model\n\n\ndef replace_module(module, replaced_module_type, new_module_type, replace_func=None) -> nn.Module:\n \"\"\"\n Replace given type in module to a new type. mostly used in deploy.\n\n Args:\n module (nn.Module): model to apply replace operation.\n replaced_module_type (Type): module type to be replaced.\n new_module_type (Type)\n replace_func (function): python function to describe replace logic. Defalut value None.\n\n Returns:\n model (nn.Module): module that already been replaced.\n \"\"\"\n\n def default_replace_func(replaced_module_type, new_module_type):\n return new_module_type()\n\n if replace_func is None:\n replace_func = default_replace_func\n\n model = module\n if isinstance(module, replaced_module_type):\n model = replace_func(replaced_module_type, new_module_type)\n else: # recurrsively replace\n for name, child in module.named_children():\n new_child = replace_module(child, replaced_module_type, new_module_type)\n if new_child is not child: # child is already replaced\n model.add_module(name, new_child)\n\n return model\n\n\ndef freeze_module(module: nn.Module, name=None) -> nn.Module:\n \"\"\"freeze module inplace\n\n Args:\n module (nn.Module): module to freeze.\n name (str, optional): name to freeze. If not given, freeze the whole module.\n Note that fuzzy match is not supported. Defaults to None.\n\n Examples:\n freeze the backbone of model\n >>> freeze_moudle(model.backbone)\n\n or freeze the backbone of model by name\n >>> freeze_moudle(model, name=\"backbone\")\n \"\"\"\n for param_name, parameter in module.named_parameters():\n if name is None or name in param_name:\n parameter.requires_grad = False\n\n # ensure module like BN and dropout are freezed\n for module_name, sub_module in module.named_modules():\n # actually there are no needs to call eval for every single sub_module\n if name is None or name in module_name:\n sub_module.eval()\n\n return module\n\n\n@contextlib.contextmanager\ndef adjust_status(module: nn.Module, training: bool = False) -> nn.Module:\n \"\"\"Adjust module to training/eval mode temporarily.\n\n Args:\n module (nn.Module): module to adjust status.\n training (bool): training mode to set. True for train mode, False fro eval mode.\n\n Examples:\n >>> with adjust_status(model, training=False):\n ... model(data)\n \"\"\"\n status = {}\n\n def backup_status(module):\n for m in module.modules():\n # save prev status to dict\n status[m] = m.training\n m.training = training\n\n def recover_status(module):\n for m in module.modules():\n # recover prev status from dict\n m.training = status.pop(m)\n\n backup_status(module)\n yield module\n recover_status(module)\n" }, { "path": "asone/detectors/yolox/yolox/utils/setup_env.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport os\nimport subprocess\nfrom loguru import logger\n\nimport cv2\n\nfrom asone.detectors.yolox.yolox.utils.dist import get_world_size, is_main_process\n\n__all__ = [\"configure_nccl\", \"configure_module\", \"configure_omp\"]\n\n\ndef configure_nccl():\n \"\"\"Configure multi-machine environment variables of NCCL.\"\"\"\n os.environ[\"NCCL_LAUNCH_MODE\"] = \"PARALLEL\"\n os.environ[\"NCCL_IB_HCA\"] = subprocess.getoutput(\n \"pushd /sys/class/infiniband/ > /dev/null; for i in mlx5_*; \"\n \"do cat $i/ports/1/gid_attrs/types/* 2>/dev/null \"\n \"| grep v >/dev/null && echo $i ; done; popd > /dev/null\"\n )\n os.environ[\"NCCL_IB_GID_INDEX\"] = \"3\"\n os.environ[\"NCCL_IB_TC\"] = \"106\"\n\n\ndef configure_omp(num_threads=1):\n \"\"\"\n If OMP_NUM_THREADS is not configured and world_size is greater than 1,\n Configure OMP_NUM_THREADS environment variables of NCCL to `num_thread`.\n\n Args:\n num_threads (int): value of `OMP_NUM_THREADS` to set.\n \"\"\"\n # We set OMP_NUM_THREADS=1 by default, which achieves the best speed on our machines\n # feel free to change it for better performance.\n if \"OMP_NUM_THREADS\" not in os.environ and get_world_size() > 1:\n os.environ[\"OMP_NUM_THREADS\"] = str(num_threads)\n if is_main_process():\n logger.info(\n \"\\n***************************************************************\\n\"\n \"We set `OMP_NUM_THREADS` for each process to {} to speed up.\\n\"\n \"please further tune the variable for optimal performance.\\n\"\n \"***************************************************************\".format(\n os.environ[\"OMP_NUM_THREADS\"]\n )\n )\n\n\ndef configure_module(ulimit_value=8192):\n \"\"\"\n Configure pytorch module environment. setting of ulimit and cv2 will be set.\n\n Args:\n ulimit_value(int): default open file number on linux. Default value: 8192.\n \"\"\"\n # system setting\n try:\n import resource\n\n rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)\n resource.setrlimit(resource.RLIMIT_NOFILE, (ulimit_value, rlimit[1]))\n except Exception:\n # Exception might be raised in Windows OS or rlimit reaches max limit number.\n # However, set rlimit value might not be necessary.\n pass\n\n # cv2\n # multiprocess might be harmful on performance of torch dataloader\n os.environ[\"OPENCV_OPENCL_RUNTIME\"] = \"disabled\"\n try:\n cv2.setNumThreads(0)\n cv2.ocl.setUseOpenCL(False)\n except Exception:\n # cv2 version mismatch might rasie exceptions.\n pass\n" }, { "path": "asone/detectors/yolox/yolox/utils/visualize.py", "content": "#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii Inc. All rights reserved.\n\nimport cv2\nimport numpy as np\n\n__all__ = [\"vis\"]\n\n\ndef vis(img, boxes, scores, cls_ids, conf=0.5, class_names=None):\n\n for i in range(len(boxes)):\n box = boxes[i]\n cls_id = int(cls_ids[i])\n score = scores[i]\n if score < conf:\n continue\n x0 = int(box[0])\n y0 = int(box[1])\n x1 = int(box[2])\n y1 = int(box[3])\n\n color = (_COLORS[cls_id] * 255).astype(np.uint8).tolist()\n text = '{}:{:.1f}%'.format(class_names[cls_id], score * 100)\n txt_color = (0, 0, 0) if np.mean(_COLORS[cls_id]) > 0.5 else (255, 255, 255)\n font = cv2.FONT_HERSHEY_SIMPLEX\n\n txt_size = cv2.getTextSize(text, font, 0.4, 1)[0]\n cv2.rectangle(img, (x0, y0), (x1, y1), color, 2)\n\n txt_bk_color = (_COLORS[cls_id] * 255 * 0.7).astype(np.uint8).tolist()\n cv2.rectangle(\n img,\n (x0, y0 + 1),\n (x0 + txt_size[0] + 1, y0 + int(1.5*txt_size[1])),\n txt_bk_color,\n -1\n )\n cv2.putText(img, text, (x0, y0 + txt_size[1]), font, 0.4, txt_color, thickness=1)\n\n return img\n\n\n_COLORS = np.array(\n [\n 0.000, 0.447, 0.741,\n 0.850, 0.325, 0.098,\n 0.929, 0.694, 0.125,\n 0.494, 0.184, 0.556,\n 0.466, 0.674, 0.188,\n 0.301, 0.745, 0.933,\n 0.635, 0.078, 0.184,\n 0.300, 0.300, 0.300,\n 0.600, 0.600, 0.600,\n 1.000, 0.000, 0.000,\n 1.000, 0.500, 0.000,\n 0.749, 0.749, 0.000,\n 0.000, 1.000, 0.000,\n 0.000, 0.000, 1.000,\n 0.667, 0.000, 1.000,\n 0.333, 0.333, 0.000,\n 0.333, 0.667, 0.000,\n 0.333, 1.000, 0.000,\n 0.667, 0.333, 0.000,\n 0.667, 0.667, 0.000,\n 0.667, 1.000, 0.000,\n 1.000, 0.333, 0.000,\n 1.000, 0.667, 0.000,\n 1.000, 1.000, 0.000,\n 0.000, 0.333, 0.500,\n 0.000, 0.667, 0.500,\n 0.000, 1.000, 0.500,\n 0.333, 0.000, 0.500,\n 0.333, 0.333, 0.500,\n 0.333, 0.667, 0.500,\n 0.333, 1.000, 0.500,\n 0.667, 0.000, 0.500,\n 0.667, 0.333, 0.500,\n 0.667, 0.667, 0.500,\n 0.667, 1.000, 0.500,\n 1.000, 0.000, 0.500,\n 1.000, 0.333, 0.500,\n 1.000, 0.667, 0.500,\n 1.000, 1.000, 0.500,\n 0.000, 0.333, 1.000,\n 0.000, 0.667, 1.000,\n 0.000, 1.000, 1.000,\n 0.333, 0.000, 1.000,\n 0.333, 0.333, 1.000,\n 0.333, 0.667, 1.000,\n 0.333, 1.000, 1.000,\n 0.667, 0.000, 1.000,\n 0.667, 0.333, 1.000,\n 0.667, 0.667, 1.000,\n 0.667, 1.000, 1.000,\n 1.000, 0.000, 1.000,\n 1.000, 0.333, 1.000,\n 1.000, 0.667, 1.000,\n 0.333, 0.000, 0.000,\n 0.500, 0.000, 0.000,\n 0.667, 0.000, 0.000,\n 0.833, 0.000, 0.000,\n 1.000, 0.000, 0.000,\n 0.000, 0.167, 0.000,\n 0.000, 0.333, 0.000,\n 0.000, 0.500, 0.000,\n 0.000, 0.667, 0.000,\n 0.000, 0.833, 0.000,\n 0.000, 1.000, 0.000,\n 0.000, 0.000, 0.167,\n 0.000, 0.000, 0.333,\n 0.000, 0.000, 0.500,\n 0.000, 0.000, 0.667,\n 0.000, 0.000, 0.833,\n 0.000, 0.000, 1.000,\n 0.000, 0.000, 0.000,\n 0.143, 0.143, 0.143,\n 0.286, 0.286, 0.286,\n 0.429, 0.429, 0.429,\n 0.571, 0.571, 0.571,\n 0.714, 0.714, 0.714,\n 0.857, 0.857, 0.857,\n 0.000, 0.447, 0.741,\n 0.314, 0.717, 0.741,\n 0.50, 0.5, 0\n ]\n).astype(np.float32).reshape(-1, 3)\n" }, { "path": "asone/detectors/yolox/yolox_detector.py", "content": "\nimport os\nfrom asone.utils import get_names\nimport numpy as np\nimport warnings\n\nimport torch\nimport onnxruntime\n\nfrom asone import utils\nfrom asone.detectors.yolox.yolox.utils import fuse_model, postprocess\nfrom asone.detectors.yolox.yolox.exp import get_exp\nfrom asone.detectors.yolox.yolox_utils import preprocess, multiclass_nms, demo_postprocess\n\nfrom asone.utils.utils import PathResolver\n\n\nclass YOLOxDetector:\n def __init__(self,\n model_name=None,\n exp_file=None,\n weights=None,\n use_onnx=False,\n use_cuda=False\n ):\n\n self.use_onnx = use_onnx\n self.device = 'cuda' if use_cuda else 'cpu'\n\n if not os.path.exists(weights):\n utils.download_weights(weights)\n\n self.weights_name = os.path.basename(weights)\n\n if model_name is None:\n model_name = 'yolox-s'\n\n if exp_file is None:\n exp_file = os.path.join(\"exps\", \"default\", \"yolox_s.py\")\n with PathResolver():\n # Load Model\n if self.use_onnx:\n self.model = self.load_onnx_model(use_cuda, weights)\n else:\n self.model = self.load_torch_model(weights, exp_file, model_name)\n\n def load_onnx_model(self, use_cuda, weights):\n # Load onnx\n if use_cuda:\n providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']\n else:\n providers = ['CPUExecutionProvider']\n model = onnxruntime.InferenceSession(weights, providers=providers)\n return model\n\n def load_torch_model(self, weights,\n exp_file, model_name,\n fp16=True, fuse=False):\n # Device: CUDA and if fp16=True only then half precision floating point works\n self.fp16 = bool(fp16) & (\n (not self.use_onnx or self.use_onnx) and self.device != 'cpu')\n exp = get_exp(exp_file, model_name)\n\n ckpt = torch.load(weights, map_location=\"cpu\")\n\n # get number of classes from weights\n # head.cls_preds.0.weight weights contains number of classes so simply extract it and with in exp file.\n exp.num_classes = ckpt['model']['head.cls_preds.0.weight'].size()[0]\n self.classes = exp.num_classes\n model = exp.get_model()\n if self.device == \"cuda\":\n model.cuda()\n if self.fp16: # to FP16\n model.half()\n model.eval()\n\n # load the model state dict\n model.load_state_dict(ckpt[\"model\"])\n if fuse:\n model = fuse_model(model)\n return model\n\n def detect(self,\n image: list,\n input_shape: tuple = (640, 640),\n conf_thres: float = 0.25,\n iou_thres: float = 0.45,\n max_det: int = 1000,\n filter_classes: bool = None,\n agnostic_nms: bool = True,\n with_p6: bool = False,\n return_image=False\n ) -> list:\n\n if self.weights_name in ['yolox_tiny.onnx', 'yolox_nano.onnx']:\n input_shape = (416, 416)\n\n self.input_shape = input_shape\n\n # Image Preprocess for onnx models\n if self.use_onnx:\n processed_image, ratio = preprocess(image, self.input_shape)\n else:\n processed_image, ratio = preprocess(image, self.input_shape)\n processed_image = torch.from_numpy(processed_image).unsqueeze(0)\n processed_image = processed_image.float()\n if self.device == \"cuda\":\n processed_image = processed_image.cuda()\n if self.fp16:\n processed_image = processed_image.half()\n\n detection = []\n # Inference\n if self.use_onnx: # Run ONNX model\n # Model Input and Output\n model_inputs = {self.model.get_inputs(\n )[0].name: processed_image[None, :, :, :]}\n detection = self.model.run(None, model_inputs)[0]\n # Postprrocessing\n detection = demo_postprocess(\n detection, self.input_shape, p6=with_p6)[0]\n boxes = detection[:, :4]\n scores = detection[:, 4:5] * detection[:, 5:]\n boxes_xyxy = np.ones_like(boxes)\n boxes_xyxy[:, 0] = boxes[:, 0] - boxes[:, 2]/2.\n boxes_xyxy[:, 1] = boxes[:, 1] - boxes[:, 3]/2.\n boxes_xyxy[:, 2] = boxes[:, 0] + boxes[:, 2]/2.\n boxes_xyxy[:, 3] = boxes[:, 1] + boxes[:, 3]/2.\n boxes_xyxy /= ratio\n detection = multiclass_nms(\n boxes_xyxy, scores, nms_thr=iou_thres, score_thr=conf_thres)\n\n # Run Pytorch model\n else:\n with torch.no_grad():\n prediction = self.model(processed_image)\n prediction = postprocess(prediction,\n self.classes,\n conf_thres,\n iou_thres,\n class_agnostic=agnostic_nms\n )[0]\n if prediction is not None:\n prediction = prediction.detach().cpu().numpy()\n bboxes = prediction[:, 0:4]\n # Postprocessing\n bboxes /= ratio\n cls = prediction[:, 6]\n scores = prediction[:, 4] * prediction[:, 5]\n for box in range(len(bboxes)):\n pred = np.append(bboxes[box], scores[box])\n pred = np.append(pred, cls[box])\n detection.append(pred)\n detection = np.array(detection)\n else:\n detection = prediction\n\n if filter_classes:\n class_names = get_names()\n\n filter_class_idx = []\n if filter_classes:\n for _class in filter_classes:\n if _class.lower() in class_names:\n filter_class_idx.append(\n class_names.index(_class.lower()))\n else:\n warnings.warn(\n f\"class {_class} not found in model classes list.\")\n\n detection = detection[np.in1d(\n detection[:, 5].astype(int), filter_class_idx)]\n\n image_info = {\n 'width': image.shape[1],\n 'height': image.shape[0],\n }\n \n if return_image:\n return detection, image\n else: \n return detection, image_info\n \n" }, { "path": "asone/detectors/yolox/yolox_utils.py", "content": "import cv2\nimport numpy as np\n\ndef preprocess(img, input_size, swap=(2, 0, 1)):\n if len(img.shape) == 3:\n padded_img = np.ones((input_size[0], input_size[1], 3), dtype=np.uint8) * 114\n else:\n padded_img = np.ones(input_size, dtype=np.uint8) * 114\n\n r = min(input_size[0] / img.shape[0], input_size[1] / img.shape[1])\n resized_img = cv2.resize(\n img,\n (int(img.shape[1] * r), int(img.shape[0] * r)),\n interpolation=cv2.INTER_LINEAR,\n ).astype(np.uint8)\n padded_img[: int(img.shape[0] * r), : int(img.shape[1] * r)] = resized_img\n\n padded_img = padded_img.transpose(swap)\n padded_img = np.ascontiguousarray(padded_img, dtype=np.float32)\n return padded_img, r\n\ndef nms(boxes, scores, nms_thr):\n \"\"\"Single class NMS implemented in Numpy.\"\"\"\n x1 = boxes[:, 0]\n y1 = boxes[:, 1]\n x2 = boxes[:, 2]\n y2 = boxes[:, 3]\n\n areas = (x2 - x1 + 1) * (y2 - y1 + 1)\n order = scores.argsort()[::-1]\n\n keep = []\n while order.size > 0:\n i = order[0]\n keep.append(i)\n xx1 = np.maximum(x1[i], x1[order[1:]])\n yy1 = np.maximum(y1[i], y1[order[1:]])\n xx2 = np.minimum(x2[i], x2[order[1:]])\n yy2 = np.minimum(y2[i], y2[order[1:]])\n\n w = np.maximum(0.0, xx2 - xx1 + 1)\n h = np.maximum(0.0, yy2 - yy1 + 1)\n inter = w * h\n ovr = inter / (areas[i] + areas[order[1:]] - inter)\n\n inds = np.where(ovr <= nms_thr)[0]\n order = order[inds + 1]\n\n return keep\n\n\ndef multiclass_nms(boxes, scores, nms_thr, score_thr, class_agnostic=True):\n \"\"\"Multiclass NMS implemented in Numpy\"\"\"\n if class_agnostic:\n nms_method = multiclass_nms_class_agnostic\n else:\n nms_method = multiclass_nms_class_aware\n return nms_method(boxes, scores, nms_thr, score_thr)\n\n\ndef multiclass_nms_class_aware(boxes, scores, nms_thr, score_thr):\n \"\"\"Multiclass NMS implemented in Numpy. Class-aware version.\"\"\"\n final_dets = []\n num_classes = scores.shape[1]\n for cls_ind in range(num_classes):\n cls_scores = scores[:, cls_ind]\n valid_score_mask = cls_scores > score_thr\n if valid_score_mask.sum() == 0:\n continue\n else:\n valid_scores = cls_scores[valid_score_mask]\n valid_boxes = boxes[valid_score_mask]\n keep = nms(valid_boxes, valid_scores, nms_thr)\n if len(keep) > 0:\n cls_inds = np.ones((len(keep), 1)) * cls_ind\n dets = np.concatenate(\n [valid_boxes[keep], valid_scores[keep, None], cls_inds], 1\n )\n final_dets.append(dets)\n if len(final_dets) == 0:\n return None\n return np.concatenate(final_dets, 0)\n\n\ndef multiclass_nms_class_agnostic(boxes, scores, nms_thr, score_thr):\n \"\"\"Multiclass NMS implemented in Numpy. Class-agnostic version.\"\"\"\n cls_inds = scores.argmax(1)\n cls_scores = scores[np.arange(len(cls_inds)), cls_inds]\n\n valid_score_mask = cls_scores > score_thr\n if valid_score_mask.sum() == 0:\n return None\n valid_scores = cls_scores[valid_score_mask]\n valid_boxes = boxes[valid_score_mask]\n valid_cls_inds = cls_inds[valid_score_mask]\n keep = nms(valid_boxes, valid_scores, nms_thr)\n if keep:\n dets = np.concatenate(\n [valid_boxes[keep], valid_scores[keep, None], valid_cls_inds[keep, None]], 1\n )\n return dets\n\ndef demo_postprocess(outputs, img_size, p6=False):\n\n grids = []\n expanded_strides = []\n\n if not p6:\n strides = [8, 16, 32]\n else:\n strides = [8, 16, 32, 64]\n\n hsizes = [img_size[0] // stride for stride in strides]\n wsizes = [img_size[1] // stride for stride in strides]\n\n for hsize, wsize, stride in zip(hsizes, wsizes, strides):\n xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize))\n grid = np.stack((xv, yv), 2).reshape(1, -1, 2)\n grids.append(grid)\n shape = grid.shape[:2]\n expanded_strides.append(np.full((*shape, 1), stride))\n\n grids = np.concatenate(grids, 1)\n expanded_strides = np.concatenate(expanded_strides, 1)\n outputs[..., :2] = (outputs[..., :2] + grids) * expanded_strides\n outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * expanded_strides\n\n return outputs\n\n" }, { "path": "asone/linux/Instructions/Benchmarking.md", "content": "# Model Zoo\n\n## Hardware Used:\n- CPU: Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz\n- GPU: 8GB (RTX2080) \n\n## Trackers\n\n### DeepSort\n\n| Model | Model Flag | FPS-GPU | FPS-CPU\n|---------------- |-----------| -----------| --------\n|DeepSort-ONNX-Yolov5s|DEEPSORT|13|3.2|\n|DeepSort-Pytorch-Yolov5s|DEEPSORT|13|3.2|\n\n### ByteTrack\n\n| Model | Model Flag | FPS-GPU | FPS-CPU\n|---------------- |-----------| -----------| --------\n|ByteTrack-ONNX-YOLOv5s|BYTETRACK|33.7|17.4|\n|ByteTrack-Pytorch-Sample-YOLOv5s|BYTETRACK|33.7|17.4|\n\n### NorFair\n\n| Model | Model Flag | FPS-GPU | FPS-CPU\n|---------------- |-----------| -----------| --------\n|tryolab-ONNX-YOLOv5s|NORFAIR|25.8|12|\n|tryolab-Pytorch-YOLOv5s|NORFAIR|25.8|12|\n\n### MOTPY\n\n| Model | Model Flag | FPS-GPU | FPS-CPU\n|---------------- |-----------| -----------| --------\n|MOTPY-ONNX-YOLOv7|MOTPY|27.5|4.2|\n|MOTPY-Pytorch-YOLOv7|MOTPY|32.4|3.5|\n\n### StrongSort\n\n| Model | Model Flag | FPS-GPU | FPS-CPU\n|---------------- |-----------| -----------| --------\n|StrongSort-ONNX-YOLOv7|STRONGSORT|7.6|3.1|\n|StrongSort-Pytorch-YOLOv7|STRONGSORT|7.9|3.1|\n\n### OCSORT\n\n| Model | Model Flag | FPS-GPU | FPS-CPU\n|---------------- |-----------| -----------| --------\n|OCSORT-ONNX-YOLOv7|OCSORT|25.7|3.4|\n|OCSORT-Pytorch-YOLOv7|OCSORT|31.4|3.2|\n\n## Detectors\n### YOLOv5\n\n| PyTorch |ONNX |COREML |\n|:-------------------------------:|:-----------------------------:|:-----------------------------:|\n|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOV5X6_PYTORCH 20.8 3.69
YOLOV5S_PYTORCH 57.25 25.4
YOLOV5N_PYTORCH 68 45
YOLOV5M_PYTORCH 54 14
YOLOV5L_PYTORCH 40.06 8.28
YOLOV5X_PYTORCH 28.8 4.32
YOLOV5N6_PYTORCH 63.5 39
YOLOV5S6_PYTORCH 58 23
YOLOV5M6_PYTORCH 49 10
YOLOV5L6_PYTORCH 33 6.5
|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOV5X6_ONNX 2.58 2.46
YOLOV5S_ONNX 17 16.35
YOLOV5N_ONNX 57.25 35.23
YOLOV5M_ONNX 45.8 11.17
YOLOV5L_ONNX 4.07 4.36
YOLOV5X_ONNX 2.32 2.6
YOLOV5N6_ONNX 28.6 32.7
YOLOV5S6_ONNX 17 16.35
YOLOV5M6_ONNX 7.5 7.6
YOLOV5L6_ONNX 3.7 3.98
|
Model Name / Model Flag
YOLOV5X6_MLMODEL
YOLOV5S_MLMODEL
YOLOV5N_MLMODEL
YOLOV5M_MLMODEL
YOLOV5L_MLMODEL
YOLOV5X_MLMODEL
YOLOV5N6_MLMODEL
YOLOV5S6_MLMODEL
YOLOV5M6_MLMODEL
YOLOV5L6_MLMODEL
|\n\n### YOLOv6\n| PyTorch |ONNX |\n|:-------------------------------:|:-----------------------------:|\n|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOV6N_PYTORCH 65.4 35.32
YOLOV6T_PYTORCH 63 15.21
YOLOV6S_PYTORCH 49.24 20
YOLOV6M_PYTORCH 35 9.96
YOLOV6L_PYTORCH 31 6.2
YOLOV6L_RELU_PYTORCH 27 6.3
YOLOV6S_REPOPT_PYTORCH 63.5 39
|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOV6N_ONNX 50 30
YOLOV6T_ONNX 45.8 16
YOLOV6S_ONNX 41 13.8
YOLOV6M_ONNX 25 6.07
YOLOV6L_ONNNX 17.7 3.32
YOLOV6L_RELU_ONNX 19.15 4.36
YOLOV6S_REPOPT_ONNX 63.5 39
|\n\n### YOLOv7\n| PyTorch |ONNX |COREML |POSE |\n|:-------------------------------:|:-----------------------------:|:-----------------------------:|:-----------------------------:|\n|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOV7_TINY_PYTORCH 53 19
YOLOV7_PYTORCH 38 6.83
YOLOV7_X_PYTORCH 28 4.36
YOLOV7_W6_PYTORCH 32.7 7.26
YOLOV7_E6_PYTORCH 15.26 3.07
YOLOV7_D6_PYTORCH 21 3.78
YOLOV7_E6E_PYTORCH 24 3.36
|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOV7_TINY_ONNX 41.6 22
YOLOV7_ONNX 26 3.78
YOLOV7_X_ONNX 19.08 2.35
YOLOV7_W6_ONNX 28.6 5.2
YOLOV7_E6_ONNX 14.3 2.97
YOLOV7_D6_ONNX 18.32 2.58
YOLOV7_E6E_ONNX 15.26 2.09
|
Model Name / Model Flag
YOLOV7_TINY_MLMODEL
YOLOV7_MLMODEL
YOLOV7_X_MLMODEL
YOLOV7_W6_MLMODEL
YOLOV7_E6_MLMODEL
YOLOV7_D6_MLMODEL
YOLOV7_E6E_MLMODEL
|
Model Name / Model Flag
---
---
---
YOLOV7_W6_POSE
---
---
---
|\n\n\n### YOLOR\n| PyTorch |ONNX |\n|:-------------------------------:|:-----------------------------:|\n|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOR_CSP_X_PYTORCH 28.6 1.83
YOLOR_CSP_X_STAR_PYTORCH 30 1.76
YOLOR_CSP_STAR_PYTORCH 38.1 2.86
YOLOR_CSP_PYTORCH 38 2.77
YOLOR_P6_PYTORCH 20 1.57
|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOR_CSP_X_ONNX 15.7 2.53
YOLOR_CSP_X_STAR_ONNX 15.79 2.05
YOLOR_CSP_STAR_ONNX 18.32 3.34
YOLOR_CSP_ONNX 15.7 2.53
YOLOR_P6_ONNX 25.4 5.58
|\n\n### YOLOX\n| PyTorch |ONNX |\n|:-------------------------------:|:-----------------------------:|\n|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOX_L_PYTORCH 2.58 2.31
YOLOX_NANO_PYTORCH 35 32
YOLOX_TINY_PYTORCH 25.4 25.4
YOLOX_DARKNET_PYTORCH 2 1.94
YOLOX_S_PYTORCH 9.54 9.7
YOLOX_M_PYTORCH 4.4 4.36
YOLOX_X_PYTORCH 15.64 1.39
|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOX_L_ONNX 22.9 3.07
YOLOX_NANO_ONNX 59 54
YOLOX_TINY_ONNX 60 35
YOLOX_DARKNET_ONNX 24 3.36
YOLOX_S_ONNX 45 13.8
YOLOX_M_ONNX 32 6.54
YOLOX_X_ONNX 15.79 2.03
|\n\n### YOLOv8\n| PyTorch |ONNX |COREML |POSE |\n|:-------------------------------:|:-----------------------------:|:-----------------------------:|:-----------------------------:|\n|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOV8N_PYTORCH 26.7 17.0
YOLOV8S_PYTORCH 26.4 12.3
YOLOV8M_PYTORCH 25.1 6.8
YOLOV8L_PYTORCH 23.6 4.0
YOLOV8X_PYTORCH 20.7 2.8
|
Model Name / Model Flag FPS-GPU FPS-CPU
YOLOV8N_ONNX 25.1 10.5
YOLOV8S_ONNX 24.5 7.5
YOLOV8M_ONNX 22.9 4.7
YOLOV8l_ONNX 20.4 2.9
YOLOV8X_ONNX 19.0 2.0
|
Model Name / Model Flag
YOLOV8N_MLMODEL
YOLOV8S_MLMODEL
YOLOV8M_MLMODEL
YOLOV8L_MLMODEL
YOLOV8X_MLMODEL
|
Model Name / Model Flag
YOLOV8N_POSE
YOLOV8S_POSE
YOLOV8M_POSE
YOLOV8L_POSE
YOLOV8X_POSE
|\n\n### YOLO-NAS\n| Model Name/Model Flag \t|\n|-----------------------\t|\n| YOLONAS_S_PYTORCH \t|\n| YOLONAS_M_PYTORCH \t|\n| YOLONAS_L_PYTORCH \t|\n\n## OCR\n\n| Model | Model Flag \n|---------------- |-----------| \n|Craft|CRAFT|\n|dbnet18|DBNET18\n\n\nReturn to [Installation Page](../../../README.md) \n" }, { "path": "asone/linux/Instructions/Demo-Detectron2.md", "content": "# ASOne\n\n#### Table of Contents\n\n- [Docker Demo](#docker-demo)\n - [Docker Demo](#docker-demo-1)\n - [Setup Detectron](#setup-detectron)\n - [Demo using docker compose](#demo-using-docker-compose-file)\n - [Test Detectron2](#test-detectron2)\n\n# Docker Installation\n\n- If you haven't installed docker first install it by following provided instructions [here](../)\n\n## Docker Demo\n\n### Setting Up detectron2\n\n1. Clone the Repo\n\n```\ngit clone https://github.com/facebookresearch/detectron2.git\n```\n\n2. Goto the detectron2 directory\n\n```\ncd detectron2\n```\n\n3. Download some sample images in this folder\n\n### Demo Using Docker Compose File\n\n1. Run container without gpu\n\n```\ndocker compose run linux\n```\n\n2. Run container with gpu\n\n```\ndocker compose run linux-gpu\n```\n\n- To test DISPLAY is shared with docker properly:\n\n```\npython main.py\n```\n\n- if an image show up then everything is working properly.\n\n- if you see an error saying `qt.qpa.xcb: could not connect to display` that means your display is not accessible to docker.\n\nTry this:\n```\nsudo xhost +local:docker\n```\n\n- To build and run docker container manually follow instructions for [Manual Build](Manual-Build.md)\n\n\n### Test Detectron2\n\n1. After docker container starts properly, in docker terminal change directory using.\n\n```\ncd detectron2\n```\n\n2. In Docker terminal run demo.py file\n\n```\npython demo/demo.py --input [PATH_TO_TEST_IMAGE] --output [PATH_TO_OUTPUT_IMAGE] \\\n --opts MODEL.DEVICE [DEVICE] \\\n MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl\n```\n\n- `PATH_TO_TEST_IMAGE` = Path of test image\n- `PATH_TO_OUTPUT_IMAGE` = Path of Results\n- `DEVICE` = device to use i.e. `cpu` or `gpu`\n\ne.g.\n```\npython demo/demo.py --input ../test.jpeg --output ../result.jpg \\\n --opts MODEL.DEVICE gpu \\\n MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl\n```\n" }, { "path": "asone/linux/Instructions/Docker-Setup.md", "content": "# Setting ASOne on Docker\n\n1. Clone the repo\n\n```\ngit clone https://github.com/axcelerateai/asone.git\ncd asone\n```\n\n2. If using windows, Run this command in command prompt.\n```\nset PWD=%cd%\n```\n2. Run docker coompose command.\n\n```\n# To test on Linux with GPU \ndocker compose run linux-gpu\n\n# To test on Windows with GPU \ndocker compose run windows-gpu\n```\n\n```\n# To test on Linux with CPU \ndocker compose run linux\n\n# To test on Windows with CPU \ndocker compose run windows\n```\n\n3. In docker terminal.\n\n```\n# if using gpu\npython main.py [VIDEO_PATH]\n\n# if using cpu\npython main.py [VIDEO_PATH] --cpu\n```\n\nReturn to [main page](../../README.md)\n" }, { "path": "asone/linux/Instructions/Driver-Installations.md", "content": "# Driver Installations\n\n\n### Linux\n\nFor systems with `GPU` please verify you have nvidia drivers installed.\n\nRun\n\n```\nnvidia-smi\n```\nDrivers are installed if you see following.\n\n![](../imgs/nvidia-drivers.png)\n\nIf drivers are not installed, you can do so using following command:\n\n```\nsudo apt-get install nvidia-driver-YYY nvidia-dkms-YYY\n```\nwhere,\n- `YYY`= Nvidia driver version\n\ne.g `sudo apt-get install nvidia-driver-510 nvidia-dkms-510`\n\n- `Reboot` your system after installing nvidia-drivers.\n```\nsudo reboot\n```\n\n\nReturn to [Installation Page](../../../README.md) \n" }, { "path": "asone/linux/Instructions/Manual-Build.md", "content": "# ASOne\n\n\n\n# Docker Manual Build\n\n## Docker Installation\n\n- If you haven't installed docker first install it by following provided instructions [here](../)\n\n## Build Image Manually\n\n1. Run the follwoing command to build docker image\n\n```\ndocker build -t [IMAGE_NAME]:[TAG] .\n```\n - `IMAGE_NAME` = Asign a name to image\n - `TAG` = Asign a tag to image\n\ne.g. `docker build -t asone:latest .` \n\n## Run Build Image\n\n1. To run the build image in docker container with `cpu`.\n\n```\ndocker run --env=\"DISPLAY\" --net=host -v [PATH_TO_LOCAL_DIR]:/workspace/ -it [IMAGE_NAME]:[TAG]\n```\n - `IMAGE_NAME` = Asign a name to image\n - `TAG` = Asign a tag to image\n - `PATH_TO_LOCAL_DIR` = Path to detectron2 directory or use `$PWD` if already in that directory\n\n\ne.g `docker run --env=\"DISPLAY\" --net=host -v $PWD:/workspace/ -it asone:latest`\n\n2. To run th ebuild image in docker container with `gpu`\n\n```\ndocker run --gpus all --env=\"DISPLAY\" --net=host -v [PATH_TO_LOCAL_DIR]:/workspace/ -it [IMAGE_NAME]:[TAG]\n```\n - `IMAGE_NAME` = Asign a name to image\n - `TAG` = Asign a tag to image\n - `PATH_TO_LOCAL_DIR` = Path to detectron2 directory or use `$PWD` if already in that directory\n\ne.g `docker run --gpus all --env=\"DISPLAY\" --net=host -v $PWD:/workspace/ -it asone:latest`" }, { "path": "asone/linux/Instructions/Manual-Installation.md", "content": "# ASOne\n\n\n# Docker Manual Installation\n\n## Ubuntu\n\n\n1. Run following command to remove all old versions on docker\n\n```\nsudo apt-get remove docker docker-engine docker.io containerd runc\n```\n\n2. Set up Repository\n\n- Update the apt package index and install packages to allow apt to use a repository over HTTPS:\n\n```\nsudo apt-get update\nsudo apt-get install \\\n ca-certificates \\\n curl \\\n gnupg \\\n lsb-release\n```\n\n- Add Docker’s official GPG key:\n\n```\nsudo mkdir -p /etc/apt/keyrings\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg\n```\n\n- Use the following command to set up the repository:\n\n```\necho \\\n \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \\\n $(lsb_release -cs) stable\" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null\n```\n\n3. Install Docker Engine\n\n- Update the apt package index, and install the latest version of Docker Engine, containerd, and Docker Compose:\n\n```\nsudo apt-get update\nsudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin\n```\n\n4. Install `nvidia-docker` to allow docker interact with GPU.\n\n```\nsudo apt-get install -y nvidia-docker2\nsudo systemctl restart docker\n```\n\n5. Give docker access to devices.\n\n```\nsudo xhost +local:docker\n\nsudo groupadd docker\nsudo gpasswd -a $USER docker\nnewgrp docker\n\n```" }, { "path": "asone/linux/README.md", "content": "# ASOne\n\n#### Table of Contents\n- [Docker Intallation](#docker-installation)\n - [Ubuntu](#ubuntu)\n - [Prerequisite](#prerequisite)\n - [Install Using Shell Script](#install-using-shell-script)\n \n# Docker Installation\n\n## Ubuntu\n#### Prerequisite\n1. For systems with `GPU` please verify you have nvidia drivers installed. run\n\n```\nnvidia-smi\n```\nif you see something like the following. Then you can continue running [shell script](#install-using-shell-script)\n\n![](imgs/nvidia-drivers.png)\n\n or you can install nvidia drivers using following command:\n\n```\nsudo apt-get install nvidia-driver-YYY nvidia-dkms-YYY\n```\n- `YYY`= Nvidia driver version\n\ne.g `sudo apt-get install nvidia-driver-510 nvidia-dkms-510`\n\n- `Reboot` your system after installing nvidia-drivers.\n```\nsudo reboot\n```\n\n#### Install using Shell Script\n\n```\ncd asone-linux\nchmod a+x docker-installation.sh\n./docker-installation.sh\n```\n - For systems with `GPU` run following commands after installing docker.\n\n Setup the package repository and the GPG key:\n ```\n distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \\\n && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \\\n && curl -s -L https://nvidia.github.io/libnvidia-container/experimental/$distribution/libnvidia-container.list | \\\n sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \\\n sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list\n ```\n Install the `nvidia-docker2` package (and dependencies) after updating the package listing:\n ```\n sudo apt-get update\n sudo apt-get install -y nvidia-docker2\n sudo systemctl restart docker\n\n sudo xhost +local:docker\n ``` \n\n- [NOTE] If there is an error while installing docker, try removing apt-lists and resinstalling.\n\n```\nsudo rm -rf /var/lib/apt/lists/*\n./docker-installation.sh\n```\n\n```\n# jump back to main folder\ncd ..\n```\n\nIf everything is done successfully you can return to [Installation Page](../README.md) \n\nIn case shell script keeps failing or you want to install manually follow steps in [Manual Installation](Instructions/Manual-Installation.md)\n" }, { "path": "asone/linux/docker-installation.sh", "content": "#!/bin/bash\n\necho \"[INFO]: Removing previous verions of dockers...\"\necho \"[INFO]: Removing previous verions of dockers...\" > logs.txt\nif sudo apt-get remove docker docker-engine docker.io containerd runc -y >> logs.txt; then\n echo \"[INFO]: Previous docker removed successfully!\"\n echo \"[INFO]: Previous docker removed successfully!\" >> logs.txt\nfi\n\necho \"[INFO]: Updating apt-package index...\"\necho \"[INFO]: Updating apt-package index...\" >> logs.txt\n\n\nif sudo apt-get update -y >> logs.txt; then\n echo \"[INFO]: apt-package index updated successfuly!\"\n echo \"[INFO]: apt-package index updated successfuly!\" >> logs.txt\nelse\n echo \"[ERROR]: Error while updating apt-package index. Check logs.txt file for more info.\"\n echo \"[ERROR]: Error while updating apt-package index.\" >> logs.txt\n # exit 1\nfi\n\necho \"[INFO]: Installing required apt packages...\"\necho \"[INFO]: Installing required apt packages...\" >> logs.txt\n\nif sudo apt-get install \\\n ca-certificates \\\n curl \\\n gnupg \\\n lsb-release -y ; then\n echo \"[INFO]: Required apt packages installed successfully!\"\n echo \"[INFO]: Required apt packages installed successfully!\" >> logs.txt\nelse\n echo \"[ERROR]: Error installing required apt packages. Check logs.txt file for more info.\"\n echo \"[ERROR]: Error installing required apt packages.\" >> logs.txt\n exit 1\nfi\n\necho \"[INFO]: Adding docker GPG key...\"\necho \"[INFO]: Adding docker GPG key...\" >> logs.txt\n\nsudo mkdir -p /etc/apt/keyrings\n\nif curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg >> logs.txt;then\n echo \"[INFO]: Docker GPG key added successfully!\"\n echo \"[INFO]: Docker GPG key added successfully!\" >> logs.txt\nelse\n echo \"[ERROR]: Error adding docker GPG key. Check logs.txt file for more info.\"\n echo \"[ERROR]: Error adding docker GPG key.\" >> logs.txt\n exit 1\nfi\n\n\necho \"[INFO]: Setting docker repository...\"\necho \"[INFO]: Setting docker repository...\" >> logs.txt\nif echo \\\n \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \\\n $(lsb_release -cs) stable\" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null >> logs.txt; then\n echo \"[INFO]: Docker repository setup done.\"\n echo \"[INFO]: Docker repository setup done.\" >> logs.txt\nelse\n echo \"[ERROR]: Error setting up docker repository. Check logs.txt file for more info.\"\n echo \"[ERROR]: Error setting up docker repository.\" >> logs.txt\n exit 1\nfi\n\necho \"[INFO]: Installing Docker Engine...\"\necho \"[INFO]: Installing Docker Engine...\" >> logs.txt\n\nif sudo apt-get update -y >> logs.txt; then\n if sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y >> logs.txt; then\n if sudo docker --version; then \n echo \"[INFO]: Docker Engine instaleld successfully!\"\n echo \"[INFO]: Docker Engine instaleld successfully!\" >> logs.txt\n fi\n else\n echo \"[ERROR]: Error installing docker engine. Check logs.txt file for more info.\"\n echo \"[ERROR]: Error installing docker engine.\" >> logs.txt\n exit 1\n fi\nelse\n echo \"[ERROR]: Error updating apt packages. Check logs.txt file for more info.\"\n echo \"[ERROR]: Error updating apt packages.\" >> logs.txt\n # exit 1\nfi\n\necho \"[INFO]: Adding docker to sudo group...\"\necho \"[INFO]: Adding docker to sudo group...\" >> logs.txt\nsudo xhost +local:docker\n\nsudo groupadd docker\nsudo gpasswd -a $USER docker\nnewgrp docker\n\necho \"[INFO]: Docker Installation and setup completed successfully!\"\necho \"[INFO]: Docker Installation and setup completed successfully!\" >> logs.txt" }, { "path": "asone/linux/main.py", "content": "import cv2\nimport numpy as np\n\nimg = cv2.imread('test-asone.jpeg')\n\ncv2.imshow('RESULT', img)\ncv2.waitKey(0)\n" }, { "path": "asone/pose_estimator.py", "content": "import cv2\nimport numpy as np\nimport os\nfrom .utils import draw_kpts, plot_skeleton_kpts\nimport cv2\nimport time\nimport warnings\n\nfrom asone.pose_estimators.yolov7_pose import Yolov7PoseEstimator\nfrom asone.pose_estimators.yolov8_pose import Yolov8PoseEstimator\nfrom asone.utils import get_weight_path, download_weights\nfrom asone.schemas.output_schemas import ModelOutput\n\nclass PoseEstimator:\n def __init__(self, estimator_flag, weights: str=None, use_cuda=True):\n\n if weights:\n weights = weights\n else:\n weights = get_weight_path(estimator_flag)\n if not os.path.exists(weights):\n download_weights(weights)\n self.estimator = self.get_estimator(estimator_flag, weights, use_cuda)\n self.model_output = ModelOutput()\n\n def get_estimator(self, estimator_flag: int, weights: str, use_cuda: bool):\n \n if estimator_flag in range(149, 155):\n estimator = Yolov7PoseEstimator(weights=weights, use_cuda=use_cuda)\n\n elif estimator_flag in range(144, 149):\n estimator = Yolov8PoseEstimator(weights=weights,\n use_cuda=use_cuda)\n return estimator\n \n def estimate_image(self, frame):\n \n keypoints = self.estimator.estimate(frame)\n return keypoints.cpu().numpy().xy\n \n def estimate_video(self, video_path, save=True, conf_thresh=0.5, display=True):\n # Emit the warning for DeprecationWarning\n with warnings.catch_warnings():\n warnings.simplefilter(\"always\", DeprecationWarning)\n warnings.warn(\"estimate_video function is deprecated. Kindly use video_estimator instead\", DeprecationWarning)\n \n if video_path == 0:\n cap = cv2.VideoCapture(0)\n video_path = 'webcam.mp4'\n else:\n cap = cv2.VideoCapture(video_path)\n \n width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)\n height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)\n FPS = cap.get(cv2.CAP_PROP_FPS)\n\n if save:\n video_writer = cv2.VideoWriter(\n os.path.basename(video_path),\n cv2.VideoWriter_fourcc(*\"mp4v\"),\n FPS,\n (int(width), int(height)),\n )\n \n frame_no = 1\n tic = time.time()\n frame_id = 1\n prevTime = 0\n fframe_num = 0\n while True:\n start_time = time.time()\n\n ret, img = cap.read()\n if not ret:\n break\n frame = img.copy()\n kpts = self.estimator.estimate(img)\n currTime = time.time()\n fps = 1 / (currTime - prevTime)\n prevTime = currTime\n\n if kpts is not None:\n img = draw_kpts(kpts, image=img) \n \n cv2.line(img, (20, 25), (127, 25), [85, 45, 255], 30)\n cv2.putText(img, f'FPS: {int(fps)}', (11, 35), 0, 1, [\n 225, 255, 255], thickness=2, lineType=cv2.LINE_AA)\n frame_id += 1\n frame_no+=1\n if display:\n cv2.imshow('Window', img)\n\n if save:\n video_writer.write(img)\n \n if cv2.waitKey(25) & 0xFF == ord('q'):\n break\n\n yield (kpts), (img if display else frame, frame_id-1, fps)\n \n def video_estimator(self, video_path, save=True, conf_thresh=0.5, display=True):\n \n if video_path == 0:\n cap = cv2.VideoCapture(0)\n video_path = 'webcam.mp4'\n else:\n cap = cv2.VideoCapture(video_path)\n \n width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)\n height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)\n FPS = cap.get(cv2.CAP_PROP_FPS)\n\n if save:\n video_writer = cv2.VideoWriter(\n os.path.basename(video_path),\n cv2.VideoWriter_fourcc(*\"mp4v\"),\n FPS,\n (int(width), int(height)),\n )\n \n frame_no = 1\n tic = time.time()\n frame_id = 1\n prevTime = 0\n fframe_num = 0\n while True:\n start_time = time.time()\n\n ret, img = cap.read()\n if not ret:\n break\n frame = img.copy()\n kpts = self.estimator.estimate(img)\n currTime = time.time()\n fps = 1 / (currTime - prevTime)\n prevTime = currTime\n\n if kpts is not None:\n img = draw_kpts(kpts, image=img) \n \n cv2.line(img, (20, 25), (127, 25), [85, 45, 255], 30)\n cv2.putText(img, f'FPS: {int(fps)}', (11, 35), 0, 1, [\n 225, 255, 255], thickness=2, lineType=cv2.LINE_AA)\n frame_id += 1\n frame_no+=1\n if display:\n cv2.imshow('Window', img)\n\n if save:\n video_writer.write(img)\n \n if cv2.waitKey(25) & 0xFF == ord('q'):\n break\n\n yield self.format_output((kpts), (img if display else frame, frame_id-1, fps))\n \n \n def format_output(self, bbox_details, frame_details):\n\n # Set detections\n self.model_output.dets.bbox = bbox_details\n if frame_details:\n # Set image info\n self.model_output.info.image = frame_details[0]\n self.model_output.info.frame_no = frame_details[1]\n self.model_output.info.fps = frame_details[2]\n\n return self.model_output" }, { "path": "asone/pose_estimators/__init__.py", "content": "from .yolov7_pose import Yolov7PoseEstimator\nfrom .yolov8_pose import Yolov8PoseEstimator\n__all__ = ['Yolov7PoseEstimator', 'Yolov8PoseEstimator']" }, { "path": "asone/pose_estimators/yolov7_pose/__init__.py", "content": "from .yolov7 import Yolov7PoseEstimator\n__all__ = ['Yolov7PoseEstimator']" }, { "path": "asone/pose_estimators/yolov7_pose/main.py", "content": "from pose_estimate import PoseEstimator\n\na = PoseEstimator()\na.run()\n\n\n# parser.add_argument('--poseweights', nargs='+', type=str, default='yolov7-w6-pose.pt', help='model path(s)')\n# parser.add_argument('--source', type=str, default='football1.mp4', help='video/0 for webcam') #video source\n# parser.add_argument('--device', type=str, default='cpu', help='cpu/0,1,2,3(gpu)') #device arugments\n# parser.add_argument('--view-img', action='store_true', help='display results') #display results\n# parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') #save confidence in txt writing\n# parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)') #box linethickness\n# parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels') #box hidelabel\n# parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences') #boxhideconf" }, { "path": "asone/pose_estimators/yolov7_pose/models/__init__.py", "content": "# init" }, { "path": "asone/pose_estimators/yolov7_pose/models/common.py", "content": "import math\r\nfrom copy import copy\r\nfrom pathlib import Path\r\n\r\nimport numpy as np\r\nimport pandas as pd\r\nimport requests\r\nimport torch\r\nimport torch.nn as nn\r\nimport torch.nn.functional as F\r\nfrom torchvision.ops import DeformConv2d\r\nfrom PIL import Image\r\nfrom torch.cuda import amp\r\n\r\nfrom asone.pose_estimators.yolov7_pose.utils.datasets import letterbox\r\nfrom asone.pose_estimators.yolov7_pose.utils.general import non_max_suppression, make_divisible, scale_coords, increment_path, xyxy2xywh\r\nfrom asone.pose_estimators.yolov7_pose.utils.plots import color_list, plot_one_box\r\nfrom asone.pose_estimators.yolov7_pose.utils.torch_utils import time_synchronized\r\n\r\n\r\n##### basic ####\r\n\r\ndef autopad(k, p=None): # kernel, padding\r\n # Pad to 'same'\r\n if p is None:\r\n p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad\r\n return p\r\n\r\n\r\nclass MP(nn.Module):\r\n def __init__(self, k=2):\r\n super(MP, self).__init__()\r\n self.m = nn.MaxPool2d(kernel_size=k, stride=k)\r\n\r\n def forward(self, x):\r\n return self.m(x)\r\n\r\n\r\nclass SP(nn.Module):\r\n def __init__(self, k=3, s=1):\r\n super(SP, self).__init__()\r\n self.m = nn.MaxPool2d(kernel_size=k, stride=s, padding=k // 2)\r\n\r\n def forward(self, x):\r\n return self.m(x)\r\n \r\n \r\nclass ReOrg(nn.Module):\r\n def __init__(self):\r\n super(ReOrg, self).__init__()\r\n\r\n def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2)\r\n return torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1)\r\n\r\n\r\nclass Concat(nn.Module):\r\n def __init__(self, dimension=1):\r\n super(Concat, self).__init__()\r\n self.d = dimension\r\n\r\n def forward(self, x):\r\n return torch.cat(x, self.d)\r\n\r\n\r\nclass Chuncat(nn.Module):\r\n def __init__(self, dimension=1):\r\n super(Chuncat, self).__init__()\r\n self.d = dimension\r\n\r\n def forward(self, x):\r\n x1 = []\r\n x2 = []\r\n for xi in x:\r\n xi1, xi2 = xi.chunk(2, self.d)\r\n x1.append(xi1)\r\n x2.append(xi2)\r\n return torch.cat(x1+x2, self.d)\r\n\r\n\r\nclass Shortcut(nn.Module):\r\n def __init__(self, dimension=0):\r\n super(Shortcut, self).__init__()\r\n self.d = dimension\r\n\r\n def forward(self, x):\r\n return x[0]+x[1]\r\n\r\n\r\nclass Foldcut(nn.Module):\r\n def __init__(self, dimension=0):\r\n super(Foldcut, self).__init__()\r\n self.d = dimension\r\n\r\n def forward(self, x):\r\n x1, x2 = x.chunk(2, self.d)\r\n return x1+x2\r\n\r\n\r\nclass Conv(nn.Module):\r\n # Standard convolution\r\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\r\n super(Conv, self).__init__()\r\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False)\r\n self.bn = nn.BatchNorm2d(c2)\r\n self.act = nn.SiLU() if act is True else (act if isinstance(act, nn.Module) else nn.Identity())\r\n\r\n def forward(self, x):\r\n return self.act(self.bn(self.conv(x)))\r\n\r\n def fuseforward(self, x):\r\n return self.act(self.conv(x))\r\n \r\n\r\nclass RobustConv(nn.Module):\r\n # Robust convolution (use high kernel size 7-11 for: downsampling and other layers). Train for 300 - 450 epochs.\r\n def __init__(self, c1, c2, k=7, s=1, p=None, g=1, act=True, layer_scale_init_value=1e-6): # ch_in, ch_out, kernel, stride, padding, groups\r\n super(RobustConv, self).__init__()\r\n self.conv_dw = Conv(c1, c1, k=k, s=s, p=p, g=c1, act=act)\r\n self.conv1x1 = nn.Conv2d(c1, c2, 1, 1, 0, groups=1, bias=True)\r\n self.gamma = nn.Parameter(layer_scale_init_value * torch.ones(c2)) if layer_scale_init_value > 0 else None\r\n\r\n def forward(self, x):\r\n x = x.to(memory_format=torch.channels_last)\r\n x = self.conv1x1(self.conv_dw(x))\r\n if self.gamma is not None:\r\n x = x.mul(self.gamma.reshape(1, -1, 1, 1)) \r\n return x\r\n\r\n\r\nclass RobustConv2(nn.Module):\r\n # Robust convolution 2 (use [32, 5, 2] or [32, 7, 4] or [32, 11, 8] for one of the paths in CSP).\r\n def __init__(self, c1, c2, k=7, s=4, p=None, g=1, act=True, layer_scale_init_value=1e-6): # ch_in, ch_out, kernel, stride, padding, groups\r\n super(RobustConv2, self).__init__()\r\n self.conv_strided = Conv(c1, c1, k=k, s=s, p=p, g=c1, act=act)\r\n self.conv_deconv = nn.ConvTranspose2d(in_channels=c1, out_channels=c2, kernel_size=s, stride=s, \r\n padding=0, bias=True, dilation=1, groups=1\r\n )\r\n self.gamma = nn.Parameter(layer_scale_init_value * torch.ones(c2)) if layer_scale_init_value > 0 else None\r\n\r\n def forward(self, x):\r\n x = self.conv_deconv(self.conv_strided(x))\r\n if self.gamma is not None:\r\n x = x.mul(self.gamma.reshape(1, -1, 1, 1)) \r\n return x\r\n \r\n\r\ndef DWConv(c1, c2, k=1, s=1, act=True):\r\n # Depthwise convolution\r\n return Conv(c1, c2, k, s, g=math.gcd(c1, c2), act=act)\r\n\r\n\r\nclass GhostConv(nn.Module):\r\n # Ghost Convolution https://github.com/huawei-noah/ghostnet\r\n def __init__(self, c1, c2, k=1, s=1, g=1, act=True): # ch_in, ch_out, kernel, stride, groups\r\n super(GhostConv, self).__init__()\r\n c_ = c2 // 2 # hidden channels\r\n self.cv1 = Conv(c1, c_, k, s, None, g, act)\r\n self.cv2 = Conv(c_, c_, 5, 1, None, c_, act)\r\n\r\n def forward(self, x):\r\n y = self.cv1(x)\r\n return torch.cat([y, self.cv2(y)], 1)\r\n\r\n\r\nclass Stem(nn.Module):\r\n # Stem\r\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\r\n super(Stem, self).__init__()\r\n c_ = int(c2/2) # hidden channels\r\n self.cv1 = Conv(c1, c_, 3, 2)\r\n self.cv2 = Conv(c_, c_, 1, 1)\r\n self.cv3 = Conv(c_, c_, 3, 2)\r\n self.pool = torch.nn.MaxPool2d(2, stride=2)\r\n self.cv4 = Conv(2 * c_, c2, 1, 1)\r\n\r\n def forward(self, x):\r\n x = self.cv1(x)\r\n return self.cv4(torch.cat((self.cv3(self.cv2(x)), self.pool(x)), dim=1))\r\n\r\n\r\nclass DownC(nn.Module):\r\n # Spatial pyramid pooling layer used in YOLOv3-SPP\r\n def __init__(self, c1, c2, n=1, k=2):\r\n super(DownC, self).__init__()\r\n c_ = int(c1) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c_, c2//2, 3, k)\r\n self.cv3 = Conv(c1, c2//2, 1, 1)\r\n self.mp = nn.MaxPool2d(kernel_size=k, stride=k)\r\n\r\n def forward(self, x):\r\n return torch.cat((self.cv2(self.cv1(x)), self.cv3(self.mp(x))), dim=1)\r\n\r\n\r\nclass SPP(nn.Module):\r\n # Spatial pyramid pooling layer used in YOLOv3-SPP\r\n def __init__(self, c1, c2, k=(5, 9, 13)):\r\n super(SPP, self).__init__()\r\n c_ = c1 // 2 # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1)\r\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\r\n\r\n def forward(self, x):\r\n x = self.cv1(x)\r\n return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1))\r\n \r\n\r\nclass Bottleneck(nn.Module):\r\n # Darknet bottleneck\r\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\r\n super(Bottleneck, self).__init__()\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c_, c2, 3, 1, g=g)\r\n self.add = shortcut and c1 == c2\r\n\r\n def forward(self, x):\r\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\r\n\r\n\r\nclass Res(nn.Module):\r\n # ResNet bottleneck\r\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\r\n super(Res, self).__init__()\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c_, c_, 3, 1, g=g)\r\n self.cv3 = Conv(c_, c2, 1, 1)\r\n self.add = shortcut and c1 == c2\r\n\r\n def forward(self, x):\r\n return x + self.cv3(self.cv2(self.cv1(x))) if self.add else self.cv3(self.cv2(self.cv1(x)))\r\n\r\n\r\nclass ResX(Res):\r\n # ResNet bottleneck\r\n def __init__(self, c1, c2, shortcut=True, g=32, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\r\n super().__init__(c1, c2, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n\r\n\r\nclass Ghost(nn.Module):\r\n # Ghost Bottleneck https://github.com/huawei-noah/ghostnet\r\n def __init__(self, c1, c2, k=3, s=1): # ch_in, ch_out, kernel, stride\r\n super(Ghost, self).__init__()\r\n c_ = c2 // 2\r\n self.conv = nn.Sequential(GhostConv(c1, c_, 1, 1), # pw\r\n DWConv(c_, c_, k, s, act=False) if s == 2 else nn.Identity(), # dw\r\n GhostConv(c_, c2, 1, 1, act=False)) # pw-linear\r\n self.shortcut = nn.Sequential(DWConv(c1, c1, k, s, act=False),\r\n Conv(c1, c2, 1, 1, act=False)) if s == 2 else nn.Identity()\r\n\r\n def forward(self, x):\r\n return self.conv(x) + self.shortcut(x)\r\n\r\n##### end of basic #####\r\n\r\n\r\n##### cspnet #####\r\n\r\nclass SPPCSPC(nn.Module):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, k=(5, 9, 13)):\r\n super(SPPCSPC, self).__init__()\r\n c_ = int(2 * c2 * e) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c1, c_, 1, 1)\r\n self.cv3 = Conv(c_, c_, 3, 1)\r\n self.cv4 = Conv(c_, c_, 1, 1)\r\n self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k])\r\n self.cv5 = Conv(4 * c_, c_, 1, 1)\r\n self.cv6 = Conv(c_, c_, 3, 1)\r\n self.cv7 = Conv(2 * c_, c2, 1, 1)\r\n\r\n def forward(self, x):\r\n x1 = self.cv4(self.cv3(self.cv1(x)))\r\n y1 = self.cv6(self.cv5(torch.cat([x1] + [m(x1) for m in self.m], 1)))\r\n y2 = self.cv2(x)\r\n return self.cv7(torch.cat((y1, y2), dim=1))\r\n\r\nclass GhostSPPCSPC(SPPCSPC):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, k=(5, 9, 13)):\r\n super().__init__(c1, c2, n, shortcut, g, e, k)\r\n c_ = int(2 * c2 * e) # hidden channels\r\n self.cv1 = GhostConv(c1, c_, 1, 1)\r\n self.cv2 = GhostConv(c1, c_, 1, 1)\r\n self.cv3 = GhostConv(c_, c_, 3, 1)\r\n self.cv4 = GhostConv(c_, c_, 1, 1)\r\n self.cv5 = GhostConv(4 * c_, c_, 1, 1)\r\n self.cv6 = GhostConv(c_, c_, 3, 1)\r\n self.cv7 = GhostConv(2 * c_, c2, 1, 1)\r\n\r\n\r\nclass GhostStem(Stem):\r\n # Stem\r\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\r\n super().__init__(c1, c2, k, s, p, g, act)\r\n c_ = int(c2/2) # hidden channels\r\n self.cv1 = GhostConv(c1, c_, 3, 2)\r\n self.cv2 = GhostConv(c_, c_, 1, 1)\r\n self.cv3 = GhostConv(c_, c_, 3, 2)\r\n self.cv4 = GhostConv(2 * c_, c2, 1, 1)\r\n \r\n\r\nclass BottleneckCSPA(nn.Module):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super(BottleneckCSPA, self).__init__()\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c1, c_, 1, 1)\r\n self.cv3 = Conv(2 * c_, c2, 1, 1)\r\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n def forward(self, x):\r\n y1 = self.m(self.cv1(x))\r\n y2 = self.cv2(x)\r\n return self.cv3(torch.cat((y1, y2), dim=1))\r\n\r\n\r\nclass BottleneckCSPB(nn.Module):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super(BottleneckCSPB, self).__init__()\r\n c_ = int(c2) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c_, c_, 1, 1)\r\n self.cv3 = Conv(2 * c_, c2, 1, 1)\r\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n def forward(self, x):\r\n x1 = self.cv1(x)\r\n y1 = self.m(x1)\r\n y2 = self.cv2(x1)\r\n return self.cv3(torch.cat((y1, y2), dim=1))\r\n\r\n\r\nclass BottleneckCSPC(nn.Module):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super(BottleneckCSPC, self).__init__()\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c1, c_, 1, 1)\r\n self.cv3 = Conv(c_, c_, 1, 1)\r\n self.cv4 = Conv(2 * c_, c2, 1, 1)\r\n self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n def forward(self, x):\r\n y1 = self.cv3(self.m(self.cv1(x)))\r\n y2 = self.cv2(x)\r\n return self.cv4(torch.cat((y1, y2), dim=1))\r\n\r\n\r\nclass ResCSPA(BottleneckCSPA):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\r\n\r\n\r\nclass ResCSPB(BottleneckCSPB):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2) # hidden channels\r\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\r\n\r\n\r\nclass ResCSPC(BottleneckCSPC):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\r\n\r\n\r\nclass ResXCSPA(ResCSPA):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n\r\nclass ResXCSPB(ResCSPB):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2) # hidden channels\r\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n\r\nclass ResXCSPC(ResCSPC):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n\r\nclass GhostCSPA(BottleneckCSPA):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[Ghost(c_, c_) for _ in range(n)])\r\n\r\n\r\nclass GhostCSPB(BottleneckCSPB):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2) # hidden channels\r\n self.m = nn.Sequential(*[Ghost(c_, c_) for _ in range(n)])\r\n\r\n\r\nclass GhostCSPC(BottleneckCSPC):\r\n # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[Ghost(c_, c_) for _ in range(n)])\r\n\r\n##### end of cspnet #####\r\n\r\n\r\n##### yolor #####\r\n\r\nclass ImplicitA(nn.Module):\r\n def __init__(self, channel, mean=0., std=.02):\r\n super(ImplicitA, self).__init__()\r\n self.channel = channel\r\n self.mean = mean\r\n self.std = std\r\n self.implicit = nn.Parameter(torch.zeros(1, channel, 1, 1))\r\n nn.init.normal_(self.implicit, mean=self.mean, std=self.std)\r\n\r\n def forward(self, x):\r\n return self.implicit + x\r\n \r\n\r\nclass ImplicitM(nn.Module):\r\n def __init__(self, channel, mean=0., std=.02):\r\n super(ImplicitM, self).__init__()\r\n self.channel = channel\r\n self.mean = mean\r\n self.std = std\r\n self.implicit = nn.Parameter(torch.ones(1, channel, 1, 1))\r\n nn.init.normal_(self.implicit, mean=self.mean, std=self.std)\r\n\r\n def forward(self, x):\r\n return self.implicit * x\r\n \r\n##### end of yolor #####\r\n\r\n\r\n##### repvgg #####\r\n\r\nclass RepConv(nn.Module):\r\n # Represented convolution\r\n # https://arxiv.org/abs/2101.03697\r\n\r\n def __init__(self, c1, c2, k=3, s=1, p=None, g=1, act=True, deploy=False):\r\n super(RepConv, self).__init__()\r\n\r\n self.deploy = deploy\r\n self.groups = g\r\n self.in_channels = c1\r\n self.out_channels = c2\r\n\r\n assert k == 3\r\n assert autopad(k, p) == 1\r\n\r\n padding_11 = autopad(k, p) - k // 2\r\n\r\n self.act = nn.SiLU() if act is True else (act if isinstance(act, nn.Module) else nn.Identity())\r\n\r\n if deploy:\r\n self.rbr_reparam = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=True)\r\n\r\n else:\r\n self.rbr_identity = (nn.BatchNorm2d(num_features=c1) if c2 == c1 and s == 1 else None)\r\n\r\n self.rbr_dense = nn.Sequential(\r\n nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False),\r\n nn.BatchNorm2d(num_features=c2),\r\n )\r\n\r\n self.rbr_1x1 = nn.Sequential(\r\n nn.Conv2d( c1, c2, 1, s, padding_11, groups=g, bias=False),\r\n nn.BatchNorm2d(num_features=c2),\r\n )\r\n\r\n def forward(self, inputs):\r\n if hasattr(self, \"rbr_reparam\"):\r\n return self.act(self.rbr_reparam(inputs))\r\n\r\n if self.rbr_identity is None:\r\n id_out = 0\r\n else:\r\n id_out = self.rbr_identity(inputs)\r\n\r\n return self.act(self.rbr_dense(inputs) + self.rbr_1x1(inputs) + id_out)\r\n \r\n def get_equivalent_kernel_bias(self):\r\n kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense)\r\n kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1)\r\n kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity)\r\n return (\r\n kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid,\r\n bias3x3 + bias1x1 + biasid,\r\n )\r\n\r\n def _pad_1x1_to_3x3_tensor(self, kernel1x1):\r\n if kernel1x1 is None:\r\n return 0\r\n else:\r\n return nn.functional.pad(kernel1x1, [1, 1, 1, 1])\r\n\r\n def _fuse_bn_tensor(self, branch):\r\n if branch is None:\r\n return 0, 0\r\n if isinstance(branch, nn.Sequential):\r\n kernel = branch[0].weight\r\n running_mean = branch[1].running_mean\r\n running_var = branch[1].running_var\r\n gamma = branch[1].weight\r\n beta = branch[1].bias\r\n eps = branch[1].eps\r\n else:\r\n assert isinstance(branch, nn.BatchNorm2d)\r\n if not hasattr(self, \"id_tensor\"):\r\n input_dim = self.in_channels // self.groups\r\n kernel_value = np.zeros(\r\n (self.in_channels, input_dim, 3, 3), dtype=np.float32\r\n )\r\n for i in range(self.in_channels):\r\n kernel_value[i, i % input_dim, 1, 1] = 1\r\n self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device)\r\n kernel = self.id_tensor\r\n running_mean = branch.running_mean\r\n running_var = branch.running_var\r\n gamma = branch.weight\r\n beta = branch.bias\r\n eps = branch.eps\r\n std = (running_var + eps).sqrt()\r\n t = (gamma / std).reshape(-1, 1, 1, 1)\r\n return kernel * t, beta - running_mean * gamma / std\r\n\r\n def repvgg_convert(self):\r\n kernel, bias = self.get_equivalent_kernel_bias()\r\n return (\r\n kernel.detach().cpu().numpy(),\r\n bias.detach().cpu().numpy(),\r\n )\r\n\r\n def fuse_conv_bn(self, conv, bn):\r\n\r\n std = (bn.running_var + bn.eps).sqrt()\r\n bias = bn.bias - bn.running_mean * bn.weight / std\r\n\r\n t = (bn.weight / std).reshape(-1, 1, 1, 1)\r\n weights = conv.weight * t\r\n\r\n bn = nn.Identity()\r\n conv = nn.Conv2d(in_channels = conv.in_channels,\r\n out_channels = conv.out_channels,\r\n kernel_size = conv.kernel_size,\r\n stride=conv.stride,\r\n padding = conv.padding,\r\n dilation = conv.dilation,\r\n groups = conv.groups,\r\n bias = True,\r\n padding_mode = conv.padding_mode)\r\n\r\n conv.weight = torch.nn.Parameter(weights)\r\n conv.bias = torch.nn.Parameter(bias)\r\n return conv\r\n\r\n def fuse_repvgg_block(self): \r\n if self.deploy:\r\n return\r\n print(f\"RepConv.fuse_repvgg_block\")\r\n \r\n self.rbr_dense = self.fuse_conv_bn(self.rbr_dense[0], self.rbr_dense[1])\r\n \r\n self.rbr_1x1 = self.fuse_conv_bn(self.rbr_1x1[0], self.rbr_1x1[1])\r\n rbr_1x1_bias = self.rbr_1x1.bias\r\n weight_1x1_expanded = torch.nn.functional.pad(self.rbr_1x1.weight, [1, 1, 1, 1])\r\n \r\n # Fuse self.rbr_identity\r\n if (isinstance(self.rbr_identity, nn.BatchNorm2d) or isinstance(self.rbr_identity, nn.modules.batchnorm.SyncBatchNorm)):\r\n # print(f\"fuse: rbr_identity == BatchNorm2d or SyncBatchNorm\")\r\n identity_conv_1x1 = nn.Conv2d(\r\n in_channels=self.in_channels,\r\n out_channels=self.out_channels,\r\n kernel_size=1,\r\n stride=1,\r\n padding=0,\r\n groups=self.groups, \r\n bias=False)\r\n identity_conv_1x1.weight.data = identity_conv_1x1.weight.data.to(self.rbr_1x1.weight.data.device)\r\n identity_conv_1x1.weight.data = identity_conv_1x1.weight.data.squeeze().squeeze()\r\n # print(f\" identity_conv_1x1.weight = {identity_conv_1x1.weight.shape}\")\r\n identity_conv_1x1.weight.data.fill_(0.0)\r\n identity_conv_1x1.weight.data.fill_diagonal_(1.0)\r\n identity_conv_1x1.weight.data = identity_conv_1x1.weight.data.unsqueeze(2).unsqueeze(3)\r\n # print(f\" identity_conv_1x1.weight = {identity_conv_1x1.weight.shape}\")\r\n\r\n identity_conv_1x1 = self.fuse_conv_bn(identity_conv_1x1, self.rbr_identity)\r\n bias_identity_expanded = identity_conv_1x1.bias\r\n weight_identity_expanded = torch.nn.functional.pad(identity_conv_1x1.weight, [1, 1, 1, 1]) \r\n else:\r\n # print(f\"fuse: rbr_identity != BatchNorm2d, rbr_identity = {self.rbr_identity}\")\r\n bias_identity_expanded = torch.nn.Parameter( torch.zeros_like(rbr_1x1_bias) )\r\n weight_identity_expanded = torch.nn.Parameter( torch.zeros_like(weight_1x1_expanded) ) \r\n \r\n\r\n #print(f\"self.rbr_1x1.weight = {self.rbr_1x1.weight.shape}, \")\r\n #print(f\"weight_1x1_expanded = {weight_1x1_expanded.shape}, \")\r\n #print(f\"self.rbr_dense.weight = {self.rbr_dense.weight.shape}, \")\r\n\r\n self.rbr_dense.weight = torch.nn.Parameter(self.rbr_dense.weight + weight_1x1_expanded + weight_identity_expanded)\r\n self.rbr_dense.bias = torch.nn.Parameter(self.rbr_dense.bias + rbr_1x1_bias + bias_identity_expanded)\r\n \r\n self.rbr_reparam = self.rbr_dense\r\n self.deploy = True\r\n\r\n if self.rbr_identity is not None:\r\n del self.rbr_identity\r\n self.rbr_identity = None\r\n\r\n if self.rbr_1x1 is not None:\r\n del self.rbr_1x1\r\n self.rbr_1x1 = None\r\n\r\n if self.rbr_dense is not None:\r\n del self.rbr_dense\r\n self.rbr_dense = None\r\n\r\n\r\nclass RepBottleneck(Bottleneck):\r\n # Standard bottleneck\r\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\r\n super().__init__(c1, c2, shortcut=True, g=1, e=0.5)\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv2 = RepConv(c_, c2, 3, 1, g=g)\r\n\r\n\r\nclass RepBottleneckCSPA(BottleneckCSPA):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[RepBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n\r\nclass RepBottleneckCSPB(BottleneckCSPB):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2) # hidden channels\r\n self.m = nn.Sequential(*[RepBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n\r\nclass RepBottleneckCSPC(BottleneckCSPC):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[RepBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n\r\nclass RepRes(Res):\r\n # Standard bottleneck\r\n def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\r\n super().__init__(c1, c2, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv2 = RepConv(c_, c_, 3, 1, g=g)\r\n\r\n\r\nclass RepResCSPA(ResCSPA):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[RepRes(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\r\n\r\n\r\nclass RepResCSPB(ResCSPB):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2) # hidden channels\r\n self.m = nn.Sequential(*[RepRes(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\r\n\r\n\r\nclass RepResCSPC(ResCSPC):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[RepRes(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\r\n\r\n\r\nclass RepResX(ResX):\r\n # Standard bottleneck\r\n def __init__(self, c1, c2, shortcut=True, g=32, e=0.5): # ch_in, ch_out, shortcut, groups, expansion\r\n super().__init__(c1, c2, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv2 = RepConv(c_, c_, 3, 1, g=g)\r\n\r\n\r\nclass RepResXCSPA(ResXCSPA):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[RepResX(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\r\n\r\n\r\nclass RepResXCSPB(ResXCSPB):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=False, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2) # hidden channels\r\n self.m = nn.Sequential(*[RepResX(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\r\n\r\n\r\nclass RepResXCSPC(ResXCSPC):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super().__init__(c1, c2, n, shortcut, g, e)\r\n c_ = int(c2 * e) # hidden channels\r\n self.m = nn.Sequential(*[RepResX(c_, c_, shortcut, g, e=0.5) for _ in range(n)])\r\n\r\n##### end of repvgg #####\r\n\r\n\r\n##### transformer #####\r\n\r\nclass TransformerLayer(nn.Module):\r\n # Transformer layer https://arxiv.org/abs/2010.11929 (LayerNorm layers removed for better performance)\r\n def __init__(self, c, num_heads):\r\n super().__init__()\r\n self.q = nn.Linear(c, c, bias=False)\r\n self.k = nn.Linear(c, c, bias=False)\r\n self.v = nn.Linear(c, c, bias=False)\r\n self.ma = nn.MultiheadAttention(embed_dim=c, num_heads=num_heads)\r\n self.fc1 = nn.Linear(c, c, bias=False)\r\n self.fc2 = nn.Linear(c, c, bias=False)\r\n\r\n def forward(self, x):\r\n x = self.ma(self.q(x), self.k(x), self.v(x))[0] + x\r\n x = self.fc2(self.fc1(x)) + x\r\n return x\r\n\r\n\r\nclass TransformerBlock(nn.Module):\r\n # Vision Transformer https://arxiv.org/abs/2010.11929\r\n def __init__(self, c1, c2, num_heads, num_layers):\r\n super().__init__()\r\n self.conv = None\r\n if c1 != c2:\r\n self.conv = Conv(c1, c2)\r\n self.linear = nn.Linear(c2, c2) # learnable position embedding\r\n self.tr = nn.Sequential(*[TransformerLayer(c2, num_heads) for _ in range(num_layers)])\r\n self.c2 = c2\r\n\r\n def forward(self, x):\r\n if self.conv is not None:\r\n x = self.conv(x)\r\n b, _, w, h = x.shape\r\n p = x.flatten(2)\r\n p = p.unsqueeze(0)\r\n p = p.transpose(0, 3)\r\n p = p.squeeze(3)\r\n e = self.linear(p)\r\n x = p + e\r\n\r\n x = self.tr(x)\r\n x = x.unsqueeze(3)\r\n x = x.transpose(0, 3)\r\n x = x.reshape(b, self.c2, w, h)\r\n return x\r\n\r\n##### end of transformer #####\r\n\r\n\r\n##### yolov5 #####\r\n\r\nclass Focus(nn.Module):\r\n # Focus wh information into c-space\r\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups\r\n super(Focus, self).__init__()\r\n self.conv = Conv(c1 * 4, c2, k, s, p, g, act)\r\n # self.contract = Contract(gain=2)\r\n\r\n def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2)\r\n return self.conv(torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1))\r\n # return self.conv(self.contract(x))\r\n \r\n\r\nclass SPPF(nn.Module):\r\n # Spatial Pyramid Pooling - Fast (SPPF) layer for YOLOv5 by Glenn Jocher\r\n def __init__(self, c1, c2, k=5): # equivalent to SPP(k=(5, 9, 13))\r\n super().__init__()\r\n c_ = c1 // 2 # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c_ * 4, c2, 1, 1)\r\n self.m = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2)\r\n\r\n def forward(self, x):\r\n x = self.cv1(x)\r\n y1 = self.m(x)\r\n y2 = self.m(y1)\r\n return self.cv2(torch.cat([x, y1, y2, self.m(y2)], 1))\r\n \r\n \r\nclass Contract(nn.Module):\r\n # Contract width-height into channels, i.e. x(1,64,80,80) to x(1,256,40,40)\r\n def __init__(self, gain=2):\r\n super().__init__()\r\n self.gain = gain\r\n\r\n def forward(self, x):\r\n N, C, H, W = x.size() # assert (H / s == 0) and (W / s == 0), 'Indivisible gain'\r\n s = self.gain\r\n x = x.view(N, C, H // s, s, W // s, s) # x(1,64,40,2,40,2)\r\n x = x.permute(0, 3, 5, 1, 2, 4).contiguous() # x(1,2,2,64,40,40)\r\n return x.view(N, C * s * s, H // s, W // s) # x(1,256,40,40)\r\n\r\n\r\nclass Expand(nn.Module):\r\n # Expand channels into width-height, i.e. x(1,64,80,80) to x(1,16,160,160)\r\n def __init__(self, gain=2):\r\n super().__init__()\r\n self.gain = gain\r\n\r\n def forward(self, x):\r\n N, C, H, W = x.size() # assert C / s ** 2 == 0, 'Indivisible gain'\r\n s = self.gain\r\n x = x.view(N, s, s, C // s ** 2, H, W) # x(1,2,2,16,80,80)\r\n x = x.permute(0, 3, 4, 1, 5, 2).contiguous() # x(1,16,80,2,80,2)\r\n return x.view(N, C // s ** 2, H * s, W * s) # x(1,16,160,160)\r\n\r\n\r\nclass NMS(nn.Module):\r\n # Non-Maximum Suppression (NMS) module\r\n conf = 0.25 # confidence threshold\r\n iou = 0.45 # IoU threshold\r\n classes = None # (optional list) filter by class\r\n\r\n def __init__(self):\r\n super(NMS, self).__init__()\r\n\r\n def forward(self, x):\r\n return non_max_suppression(x[0], conf_thres=self.conf, iou_thres=self.iou, classes=self.classes)\r\n\r\n\r\nclass autoShape(nn.Module):\r\n # input-robust model wrapper for passing cv2/np/PIL/torch inputs. Includes preprocessing, inference and NMS\r\n conf = 0.25 # NMS confidence threshold\r\n iou = 0.45 # NMS IoU threshold\r\n classes = None # (optional list) filter by class\r\n\r\n def __init__(self, model):\r\n super(autoShape, self).__init__()\r\n self.model = model.eval()\r\n\r\n def autoshape(self):\r\n print('autoShape already enabled, skipping... ') # model already converted to model.autoshape()\r\n return self\r\n\r\n @torch.no_grad()\r\n def forward(self, imgs, size=640, augment=False, profile=False):\r\n # Inference from various sources. For height=640, width=1280, RGB images example inputs are:\r\n # filename: imgs = 'data/samples/zidane.jpg'\r\n # URI: = 'https://github.com/ultralytics/yolov5/releases/download/v1.0/zidane.jpg'\r\n # OpenCV: = cv2.imread('image.jpg')[:,:,::-1] # HWC BGR to RGB x(640,1280,3)\r\n # PIL: = Image.open('image.jpg') # HWC x(640,1280,3)\r\n # numpy: = np.zeros((640,1280,3)) # HWC\r\n # torch: = torch.zeros(16,3,320,640) # BCHW (scaled to size=640, 0-1 values)\r\n # multiple: = [Image.open('image1.jpg'), Image.open('image2.jpg'), ...] # list of images\r\n\r\n t = [time_synchronized()]\r\n p = next(self.model.parameters()) # for device and type\r\n if isinstance(imgs, torch.Tensor): # torch\r\n with amp.autocast(enabled=p.device.type != 'cpu'):\r\n return self.model(imgs.to(p.device).type_as(p), augment, profile) # inference\r\n\r\n # Pre-process\r\n n, imgs = (len(imgs), imgs) if isinstance(imgs, list) else (1, [imgs]) # number of images, list of images\r\n shape0, shape1, files = [], [], [] # image and inference shapes, filenames\r\n for i, im in enumerate(imgs):\r\n f = f'image{i}' # filename\r\n if isinstance(im, str): # filename or uri\r\n im, f = np.asarray(Image.open(requests.get(im, stream=True).raw if im.startswith('http') else im)), im\r\n elif isinstance(im, Image.Image): # PIL Image\r\n im, f = np.asarray(im), getattr(im, 'filename', f) or f\r\n files.append(Path(f).with_suffix('.jpg').name)\r\n if im.shape[0] < 5: # image in CHW\r\n im = im.transpose((1, 2, 0)) # reverse dataloader .transpose(2, 0, 1)\r\n im = im[:, :, :3] if im.ndim == 3 else np.tile(im[:, :, None], 3) # enforce 3ch input\r\n s = im.shape[:2] # HWC\r\n shape0.append(s) # image shape\r\n g = (size / max(s)) # gain\r\n shape1.append([y * g for y in s])\r\n imgs[i] = im # update\r\n shape1 = [make_divisible(x, int(self.stride.max())) for x in np.stack(shape1, 0).max(0)] # inference shape\r\n x = [letterbox(im, new_shape=shape1, auto=False)[0] for im in imgs] # pad\r\n x = np.stack(x, 0) if n > 1 else x[0][None] # stack\r\n x = np.ascontiguousarray(x.transpose((0, 3, 1, 2))) # BHWC to BCHW\r\n x = torch.from_numpy(x).to(p.device).type_as(p) / 255. # uint8 to fp16/32\r\n t.append(time_synchronized())\r\n\r\n with amp.autocast(enabled=p.device.type != 'cpu'):\r\n # Inference\r\n y = self.model(x, augment, profile)[0] # forward\r\n t.append(time_synchronized())\r\n\r\n # Post-process\r\n y = non_max_suppression(y, conf_thres=self.conf, iou_thres=self.iou, classes=self.classes) # NMS\r\n for i in range(n):\r\n scale_coords(shape1, y[i][:, :4], shape0[i])\r\n\r\n t.append(time_synchronized())\r\n return Detections(imgs, y, files, t, self.names, x.shape)\r\n\r\n\r\nclass Detections:\r\n # detections class for YOLOv5 inference results\r\n def __init__(self, imgs, pred, files, times=None, names=None, shape=None):\r\n super(Detections, self).__init__()\r\n d = pred[0].device # device\r\n gn = [torch.tensor([*[im.shape[i] for i in [1, 0, 1, 0]], 1., 1.], device=d) for im in imgs] # normalizations\r\n self.imgs = imgs # list of images as numpy arrays\r\n self.pred = pred # list of tensors pred[0] = (xyxy, conf, cls)\r\n self.names = names # class names\r\n self.files = files # image filenames\r\n self.xyxy = pred # xyxy pixels\r\n self.xywh = [xyxy2xywh(x) for x in pred] # xywh pixels\r\n self.xyxyn = [x / g for x, g in zip(self.xyxy, gn)] # xyxy normalized\r\n self.xywhn = [x / g for x, g in zip(self.xywh, gn)] # xywh normalized\r\n self.n = len(self.pred) # number of images (batch size)\r\n self.t = tuple((times[i + 1] - times[i]) * 1000 / self.n for i in range(3)) # timestamps (ms)\r\n self.s = shape # inference BCHW shape\r\n\r\n def display(self, pprint=False, show=False, save=False, render=False, save_dir=''):\r\n colors = color_list()\r\n for i, (img, pred) in enumerate(zip(self.imgs, self.pred)):\r\n str = f'image {i + 1}/{len(self.pred)}: {img.shape[0]}x{img.shape[1]} '\r\n if pred is not None:\r\n for c in pred[:, -1].unique():\r\n n = (pred[:, -1] == c).sum() # detections per class\r\n str += f\"{n} {self.names[int(c)]}{'s' * (n > 1)}, \" # add to string\r\n if show or save or render:\r\n for *box, conf, cls in pred: # xyxy, confidence, class\r\n label = f'{self.names[int(cls)]} {conf:.2f}'\r\n plot_one_box(box, img, label=label, color=colors[int(cls) % 10])\r\n img = Image.fromarray(img.astype(np.uint8)) if isinstance(img, np.ndarray) else img # from np\r\n if pprint:\r\n print(str.rstrip(', '))\r\n if show:\r\n img.show(self.files[i]) # show\r\n if save:\r\n f = self.files[i]\r\n img.save(Path(save_dir) / f) # save\r\n print(f\"{'Saved' * (i == 0)} {f}\", end=',' if i < self.n - 1 else f' to {save_dir}\\n')\r\n if render:\r\n self.imgs[i] = np.asarray(img)\r\n\r\n def print(self):\r\n self.display(pprint=True) # print results\r\n print(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {tuple(self.s)}' % self.t)\r\n\r\n def show(self):\r\n self.display(show=True) # show results\r\n\r\n def save(self, save_dir='runs/hub/exp'):\r\n save_dir = increment_path(save_dir, exist_ok=save_dir != 'runs/hub/exp') # increment save_dir\r\n Path(save_dir).mkdir(parents=True, exist_ok=True)\r\n self.display(save=True, save_dir=save_dir) # save results\r\n\r\n def render(self):\r\n self.display(render=True) # render results\r\n return self.imgs\r\n\r\n def pandas(self):\r\n # return detections as pandas DataFrames, i.e. print(results.pandas().xyxy[0])\r\n new = copy(self) # return copy\r\n ca = 'xmin', 'ymin', 'xmax', 'ymax', 'confidence', 'class', 'name' # xyxy columns\r\n cb = 'xcenter', 'ycenter', 'width', 'height', 'confidence', 'class', 'name' # xywh columns\r\n for k, c in zip(['xyxy', 'xyxyn', 'xywh', 'xywhn'], [ca, ca, cb, cb]):\r\n a = [[x[:5] + [int(x[5]), self.names[int(x[5])]] for x in x.tolist()] for x in getattr(self, k)] # update\r\n setattr(new, k, [pd.DataFrame(x, columns=c) for x in a])\r\n return new\r\n\r\n def tolist(self):\r\n # return a list of Detections objects, i.e. 'for result in results.tolist():'\r\n x = [Detections([self.imgs[i]], [self.pred[i]], self.names, self.s) for i in range(self.n)]\r\n for d in x:\r\n for k in ['imgs', 'pred', 'xyxy', 'xyxyn', 'xywh', 'xywhn']:\r\n setattr(d, k, getattr(d, k)[0]) # pop out of list\r\n return x\r\n\r\n def __len__(self):\r\n return self.n\r\n\r\n\r\nclass Classify(nn.Module):\r\n # Classification head, i.e. x(b,c1,20,20) to x(b,c2)\r\n def __init__(self, c1, c2, k=1, s=1, p=None, g=1): # ch_in, ch_out, kernel, stride, padding, groups\r\n super(Classify, self).__init__()\r\n self.aap = nn.AdaptiveAvgPool2d(1) # to x(b,c1,1,1)\r\n self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g) # to x(b,c2,1,1)\r\n self.flat = nn.Flatten()\r\n\r\n def forward(self, x):\r\n z = torch.cat([self.aap(y) for y in (x if isinstance(x, list) else [x])], 1) # cat if list\r\n return self.flat(self.conv(z)) # flatten to x(b,c2)\r\n\r\n##### end of yolov5 ######\r\n\r\n\r\n##### orepa #####\r\n\r\ndef transI_fusebn(kernel, bn):\r\n gamma = bn.weight\r\n std = (bn.running_var + bn.eps).sqrt()\r\n return kernel * ((gamma / std).reshape(-1, 1, 1, 1)), bn.bias - bn.running_mean * gamma / std\r\n \r\n \r\nclass ConvBN(nn.Module):\r\n def __init__(self, in_channels, out_channels, kernel_size,\r\n stride=1, padding=0, dilation=1, groups=1, deploy=False, nonlinear=None):\r\n super().__init__()\r\n if nonlinear is None:\r\n self.nonlinear = nn.Identity()\r\n else:\r\n self.nonlinear = nonlinear\r\n if deploy:\r\n self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,\r\n stride=stride, padding=padding, dilation=dilation, groups=groups, bias=True)\r\n else:\r\n self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,\r\n stride=stride, padding=padding, dilation=dilation, groups=groups, bias=False)\r\n self.bn = nn.BatchNorm2d(num_features=out_channels)\r\n\r\n def forward(self, x):\r\n if hasattr(self, 'bn'):\r\n return self.nonlinear(self.bn(self.conv(x)))\r\n else:\r\n return self.nonlinear(self.conv(x))\r\n\r\n def switch_to_deploy(self):\r\n kernel, bias = transI_fusebn(self.conv.weight, self.bn)\r\n conv = nn.Conv2d(in_channels=self.conv.in_channels, out_channels=self.conv.out_channels, kernel_size=self.conv.kernel_size,\r\n stride=self.conv.stride, padding=self.conv.padding, dilation=self.conv.dilation, groups=self.conv.groups, bias=True)\r\n conv.weight.data = kernel\r\n conv.bias.data = bias\r\n for para in self.parameters():\r\n para.detach_()\r\n self.__delattr__('conv')\r\n self.__delattr__('bn')\r\n self.conv = conv \r\n\r\nclass OREPA_3x3_RepConv(nn.Module):\r\n\r\n def __init__(self, in_channels, out_channels, kernel_size,\r\n stride=1, padding=0, dilation=1, groups=1,\r\n internal_channels_1x1_3x3=None,\r\n deploy=False, nonlinear=None, single_init=False):\r\n super(OREPA_3x3_RepConv, self).__init__()\r\n self.deploy = deploy\r\n\r\n if nonlinear is None:\r\n self.nonlinear = nn.Identity()\r\n else:\r\n self.nonlinear = nonlinear\r\n\r\n self.kernel_size = kernel_size\r\n self.in_channels = in_channels\r\n self.out_channels = out_channels\r\n self.groups = groups\r\n assert padding == kernel_size // 2\r\n\r\n self.stride = stride\r\n self.padding = padding\r\n self.dilation = dilation\r\n\r\n self.branch_counter = 0\r\n\r\n self.weight_rbr_origin = nn.Parameter(torch.Tensor(out_channels, int(in_channels/self.groups), kernel_size, kernel_size))\r\n nn.init.kaiming_uniform_(self.weight_rbr_origin, a=math.sqrt(1.0))\r\n self.branch_counter += 1\r\n\r\n\r\n if groups < out_channels:\r\n self.weight_rbr_avg_conv = nn.Parameter(torch.Tensor(out_channels, int(in_channels/self.groups), 1, 1))\r\n self.weight_rbr_pfir_conv = nn.Parameter(torch.Tensor(out_channels, int(in_channels/self.groups), 1, 1))\r\n nn.init.kaiming_uniform_(self.weight_rbr_avg_conv, a=1.0)\r\n nn.init.kaiming_uniform_(self.weight_rbr_pfir_conv, a=1.0)\r\n self.weight_rbr_avg_conv.data\r\n self.weight_rbr_pfir_conv.data\r\n self.register_buffer('weight_rbr_avg_avg', torch.ones(kernel_size, kernel_size).mul(1.0/kernel_size/kernel_size))\r\n self.branch_counter += 1\r\n\r\n else:\r\n raise NotImplementedError\r\n self.branch_counter += 1\r\n\r\n if internal_channels_1x1_3x3 is None:\r\n internal_channels_1x1_3x3 = in_channels if groups < out_channels else 2 * in_channels # For mobilenet, it is better to have 2X internal channels\r\n\r\n if internal_channels_1x1_3x3 == in_channels:\r\n self.weight_rbr_1x1_kxk_idconv1 = nn.Parameter(torch.zeros(in_channels, int(in_channels/self.groups), 1, 1))\r\n id_value = np.zeros((in_channels, int(in_channels/self.groups), 1, 1))\r\n for i in range(in_channels):\r\n id_value[i, i % int(in_channels/self.groups), 0, 0] = 1\r\n id_tensor = torch.from_numpy(id_value).type_as(self.weight_rbr_1x1_kxk_idconv1)\r\n self.register_buffer('id_tensor', id_tensor)\r\n\r\n else:\r\n self.weight_rbr_1x1_kxk_conv1 = nn.Parameter(torch.Tensor(internal_channels_1x1_3x3, int(in_channels/self.groups), 1, 1))\r\n nn.init.kaiming_uniform_(self.weight_rbr_1x1_kxk_conv1, a=math.sqrt(1.0))\r\n self.weight_rbr_1x1_kxk_conv2 = nn.Parameter(torch.Tensor(out_channels, int(internal_channels_1x1_3x3/self.groups), kernel_size, kernel_size))\r\n nn.init.kaiming_uniform_(self.weight_rbr_1x1_kxk_conv2, a=math.sqrt(1.0))\r\n self.branch_counter += 1\r\n\r\n expand_ratio = 8\r\n self.weight_rbr_gconv_dw = nn.Parameter(torch.Tensor(in_channels*expand_ratio, 1, kernel_size, kernel_size))\r\n self.weight_rbr_gconv_pw = nn.Parameter(torch.Tensor(out_channels, in_channels*expand_ratio, 1, 1))\r\n nn.init.kaiming_uniform_(self.weight_rbr_gconv_dw, a=math.sqrt(1.0))\r\n nn.init.kaiming_uniform_(self.weight_rbr_gconv_pw, a=math.sqrt(1.0))\r\n self.branch_counter += 1\r\n\r\n if out_channels == in_channels and stride == 1:\r\n self.branch_counter += 1\r\n\r\n self.vector = nn.Parameter(torch.Tensor(self.branch_counter, self.out_channels))\r\n self.bn = nn.BatchNorm2d(out_channels)\r\n\r\n self.fre_init()\r\n\r\n nn.init.constant_(self.vector[0, :], 0.25) #origin\r\n nn.init.constant_(self.vector[1, :], 0.25) #avg\r\n nn.init.constant_(self.vector[2, :], 0.0) #prior\r\n nn.init.constant_(self.vector[3, :], 0.5) #1x1_kxk\r\n nn.init.constant_(self.vector[4, :], 0.5) #dws_conv\r\n\r\n\r\n def fre_init(self):\r\n prior_tensor = torch.Tensor(self.out_channels, self.kernel_size, self.kernel_size)\r\n half_fg = self.out_channels/2\r\n for i in range(self.out_channels):\r\n for h in range(3):\r\n for w in range(3):\r\n if i < half_fg:\r\n prior_tensor[i, h, w] = math.cos(math.pi*(h+0.5)*(i+1)/3)\r\n else:\r\n prior_tensor[i, h, w] = math.cos(math.pi*(w+0.5)*(i+1-half_fg)/3)\r\n\r\n self.register_buffer('weight_rbr_prior', prior_tensor)\r\n\r\n def weight_gen(self):\r\n\r\n weight_rbr_origin = torch.einsum('oihw,o->oihw', self.weight_rbr_origin, self.vector[0, :])\r\n\r\n weight_rbr_avg = torch.einsum('oihw,o->oihw', torch.einsum('oihw,hw->oihw', self.weight_rbr_avg_conv, self.weight_rbr_avg_avg), self.vector[1, :])\r\n \r\n weight_rbr_pfir = torch.einsum('oihw,o->oihw', torch.einsum('oihw,ohw->oihw', self.weight_rbr_pfir_conv, self.weight_rbr_prior), self.vector[2, :])\r\n\r\n weight_rbr_1x1_kxk_conv1 = None\r\n if hasattr(self, 'weight_rbr_1x1_kxk_idconv1'):\r\n weight_rbr_1x1_kxk_conv1 = (self.weight_rbr_1x1_kxk_idconv1 + self.id_tensor).squeeze()\r\n elif hasattr(self, 'weight_rbr_1x1_kxk_conv1'):\r\n weight_rbr_1x1_kxk_conv1 = self.weight_rbr_1x1_kxk_conv1.squeeze()\r\n else:\r\n raise NotImplementedError\r\n weight_rbr_1x1_kxk_conv2 = self.weight_rbr_1x1_kxk_conv2\r\n\r\n if self.groups > 1:\r\n g = self.groups\r\n t, ig = weight_rbr_1x1_kxk_conv1.size()\r\n o, tg, h, w = weight_rbr_1x1_kxk_conv2.size()\r\n weight_rbr_1x1_kxk_conv1 = weight_rbr_1x1_kxk_conv1.view(g, int(t/g), ig)\r\n weight_rbr_1x1_kxk_conv2 = weight_rbr_1x1_kxk_conv2.view(g, int(o/g), tg, h, w)\r\n weight_rbr_1x1_kxk = torch.einsum('gti,gothw->goihw', weight_rbr_1x1_kxk_conv1, weight_rbr_1x1_kxk_conv2).view(o, ig, h, w)\r\n else:\r\n weight_rbr_1x1_kxk = torch.einsum('ti,othw->oihw', weight_rbr_1x1_kxk_conv1, weight_rbr_1x1_kxk_conv2)\r\n\r\n weight_rbr_1x1_kxk = torch.einsum('oihw,o->oihw', weight_rbr_1x1_kxk, self.vector[3, :])\r\n\r\n weight_rbr_gconv = self.dwsc2full(self.weight_rbr_gconv_dw, self.weight_rbr_gconv_pw, self.in_channels)\r\n weight_rbr_gconv = torch.einsum('oihw,o->oihw', weight_rbr_gconv, self.vector[4, :]) \r\n\r\n weight = weight_rbr_origin + weight_rbr_avg + weight_rbr_1x1_kxk + weight_rbr_pfir + weight_rbr_gconv\r\n\r\n return weight\r\n\r\n def dwsc2full(self, weight_dw, weight_pw, groups):\r\n \r\n t, ig, h, w = weight_dw.size()\r\n o, _, _, _ = weight_pw.size()\r\n tg = int(t/groups)\r\n i = int(ig*groups)\r\n weight_dw = weight_dw.view(groups, tg, ig, h, w)\r\n weight_pw = weight_pw.squeeze().view(o, groups, tg)\r\n \r\n weight_dsc = torch.einsum('gtihw,ogt->ogihw', weight_dw, weight_pw)\r\n return weight_dsc.view(o, i, h, w)\r\n\r\n def forward(self, inputs):\r\n weight = self.weight_gen()\r\n out = F.conv2d(inputs, weight, bias=None, stride=self.stride, padding=self.padding, dilation=self.dilation, groups=self.groups)\r\n\r\n return self.nonlinear(self.bn(out))\r\n\r\nclass RepConv_OREPA(nn.Module):\r\n\r\n def __init__(self, c1, c2, k=3, s=1, padding=1, dilation=1, groups=1, padding_mode='zeros', deploy=False, use_se=False, nonlinear=nn.SiLU()):\r\n super(RepConv_OREPA, self).__init__()\r\n self.deploy = deploy\r\n self.groups = groups\r\n self.in_channels = c1\r\n self.out_channels = c2\r\n\r\n self.padding = padding\r\n self.dilation = dilation\r\n self.groups = groups\r\n\r\n assert k == 3\r\n assert padding == 1\r\n\r\n padding_11 = padding - k // 2\r\n\r\n if nonlinear is None:\r\n self.nonlinearity = nn.Identity()\r\n else:\r\n self.nonlinearity = nonlinear\r\n\r\n if use_se:\r\n self.se = SEBlock(self.out_channels, internal_neurons=self.out_channels // 16)\r\n else:\r\n self.se = nn.Identity()\r\n\r\n if deploy:\r\n self.rbr_reparam = nn.Conv2d(in_channels=self.in_channels, out_channels=self.out_channels, kernel_size=k, stride=s,\r\n padding=padding, dilation=dilation, groups=groups, bias=True, padding_mode=padding_mode)\r\n\r\n else:\r\n self.rbr_identity = nn.BatchNorm2d(num_features=self.in_channels) if self.out_channels == self.in_channels and s == 1 else None\r\n self.rbr_dense = OREPA_3x3_RepConv(in_channels=self.in_channels, out_channels=self.out_channels, kernel_size=k, stride=s, padding=padding, groups=groups, dilation=1)\r\n self.rbr_1x1 = ConvBN(in_channels=self.in_channels, out_channels=self.out_channels, kernel_size=1, stride=s, padding=padding_11, groups=groups, dilation=1)\r\n print('RepVGG Block, identity = ', self.rbr_identity)\r\n\r\n\r\n def forward(self, inputs):\r\n if hasattr(self, 'rbr_reparam'):\r\n return self.nonlinearity(self.se(self.rbr_reparam(inputs)))\r\n\r\n if self.rbr_identity is None:\r\n id_out = 0\r\n else:\r\n id_out = self.rbr_identity(inputs)\r\n\r\n out1 = self.rbr_dense(inputs)\r\n out2 = self.rbr_1x1(inputs)\r\n out3 = id_out\r\n out = out1 + out2 + out3\r\n\r\n return self.nonlinearity(self.se(out))\r\n\r\n\r\n # Optional. This improves the accuracy and facilitates quantization.\r\n # 1. Cancel the original weight decay on rbr_dense.conv.weight and rbr_1x1.conv.weight.\r\n # 2. Use like this.\r\n # loss = criterion(....)\r\n # for every RepVGGBlock blk:\r\n # loss += weight_decay_coefficient * 0.5 * blk.get_cust_L2()\r\n # optimizer.zero_grad()\r\n # loss.backward()\r\n\r\n # Not used for OREPA\r\n def get_custom_L2(self):\r\n K3 = self.rbr_dense.weight_gen()\r\n K1 = self.rbr_1x1.conv.weight\r\n t3 = (self.rbr_dense.bn.weight / ((self.rbr_dense.bn.running_var + self.rbr_dense.bn.eps).sqrt())).reshape(-1, 1, 1, 1).detach()\r\n t1 = (self.rbr_1x1.bn.weight / ((self.rbr_1x1.bn.running_var + self.rbr_1x1.bn.eps).sqrt())).reshape(-1, 1, 1, 1).detach()\r\n\r\n l2_loss_circle = (K3 ** 2).sum() - (K3[:, :, 1:2, 1:2] ** 2).sum() # The L2 loss of the \"circle\" of weights in 3x3 kernel. Use regular L2 on them.\r\n eq_kernel = K3[:, :, 1:2, 1:2] * t3 + K1 * t1 # The equivalent resultant central point of 3x3 kernel.\r\n l2_loss_eq_kernel = (eq_kernel ** 2 / (t3 ** 2 + t1 ** 2)).sum() # Normalize for an L2 coefficient comparable to regular L2.\r\n return l2_loss_eq_kernel + l2_loss_circle\r\n\r\n def get_equivalent_kernel_bias(self):\r\n kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense)\r\n kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1)\r\n kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity)\r\n return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid\r\n\r\n def _pad_1x1_to_3x3_tensor(self, kernel1x1):\r\n if kernel1x1 is None:\r\n return 0\r\n else:\r\n return torch.nn.functional.pad(kernel1x1, [1,1,1,1])\r\n\r\n def _fuse_bn_tensor(self, branch):\r\n if branch is None:\r\n return 0, 0\r\n if not isinstance(branch, nn.BatchNorm2d):\r\n if isinstance(branch, OREPA_3x3_RepConv):\r\n kernel = branch.weight_gen()\r\n elif isinstance(branch, ConvBN):\r\n kernel = branch.conv.weight\r\n else:\r\n raise NotImplementedError\r\n running_mean = branch.bn.running_mean\r\n running_var = branch.bn.running_var\r\n gamma = branch.bn.weight\r\n beta = branch.bn.bias\r\n eps = branch.bn.eps\r\n else:\r\n if not hasattr(self, 'id_tensor'):\r\n input_dim = self.in_channels // self.groups\r\n kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), dtype=np.float32)\r\n for i in range(self.in_channels):\r\n kernel_value[i, i % input_dim, 1, 1] = 1\r\n self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device)\r\n kernel = self.id_tensor\r\n running_mean = branch.running_mean\r\n running_var = branch.running_var\r\n gamma = branch.weight\r\n beta = branch.bias\r\n eps = branch.eps\r\n std = (running_var + eps).sqrt()\r\n t = (gamma / std).reshape(-1, 1, 1, 1)\r\n return kernel * t, beta - running_mean * gamma / std\r\n\r\n def switch_to_deploy(self):\r\n if hasattr(self, 'rbr_reparam'):\r\n return\r\n print(f\"RepConv_OREPA.switch_to_deploy\")\r\n kernel, bias = self.get_equivalent_kernel_bias()\r\n self.rbr_reparam = nn.Conv2d(in_channels=self.rbr_dense.in_channels, out_channels=self.rbr_dense.out_channels,\r\n kernel_size=self.rbr_dense.kernel_size, stride=self.rbr_dense.stride,\r\n padding=self.rbr_dense.padding, dilation=self.rbr_dense.dilation, groups=self.rbr_dense.groups, bias=True)\r\n self.rbr_reparam.weight.data = kernel\r\n self.rbr_reparam.bias.data = bias\r\n for para in self.parameters():\r\n para.detach_()\r\n self.__delattr__('rbr_dense')\r\n self.__delattr__('rbr_1x1')\r\n if hasattr(self, 'rbr_identity'):\r\n self.__delattr__('rbr_identity') \r\n\r\n##### end of orepa #####\r\n\r\n\r\n##### swin transformer ##### \r\n \r\nclass WindowAttention(nn.Module):\r\n\r\n def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.):\r\n\r\n super().__init__()\r\n self.dim = dim\r\n self.window_size = window_size # Wh, Ww\r\n self.num_heads = num_heads\r\n head_dim = dim // num_heads\r\n self.scale = qk_scale or head_dim ** -0.5\r\n\r\n # define a parameter table of relative position bias\r\n self.relative_position_bias_table = nn.Parameter(\r\n torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads)) # 2*Wh-1 * 2*Ww-1, nH\r\n\r\n # get pair-wise relative position index for each token inside the window\r\n coords_h = torch.arange(self.window_size[0])\r\n coords_w = torch.arange(self.window_size[1])\r\n coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww\r\n coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww\r\n relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww\r\n relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2\r\n relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0\r\n relative_coords[:, :, 1] += self.window_size[1] - 1\r\n relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1\r\n relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww\r\n self.register_buffer(\"relative_position_index\", relative_position_index)\r\n\r\n self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)\r\n self.attn_drop = nn.Dropout(attn_drop)\r\n self.proj = nn.Linear(dim, dim)\r\n self.proj_drop = nn.Dropout(proj_drop)\r\n\r\n nn.init.normal_(self.relative_position_bias_table, std=.02)\r\n self.softmax = nn.Softmax(dim=-1)\r\n\r\n def forward(self, x, mask=None):\r\n\r\n B_, N, C = x.shape\r\n qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)\r\n q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple)\r\n\r\n q = q * self.scale\r\n attn = (q @ k.transpose(-2, -1))\r\n\r\n relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(\r\n self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH\r\n relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww\r\n attn = attn + relative_position_bias.unsqueeze(0)\r\n\r\n if mask is not None:\r\n nW = mask.shape[0]\r\n attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)\r\n attn = attn.view(-1, self.num_heads, N, N)\r\n attn = self.softmax(attn)\r\n else:\r\n attn = self.softmax(attn)\r\n\r\n attn = self.attn_drop(attn)\r\n\r\n # print(attn.dtype, v.dtype)\r\n try:\r\n x = (attn @ v).transpose(1, 2).reshape(B_, N, C)\r\n except:\r\n #print(attn.dtype, v.dtype)\r\n x = (attn.half() @ v).transpose(1, 2).reshape(B_, N, C)\r\n x = self.proj(x)\r\n x = self.proj_drop(x)\r\n return x\r\n\r\nclass Mlp(nn.Module):\r\n\r\n def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, drop=0.):\r\n super().__init__()\r\n out_features = out_features or in_features\r\n hidden_features = hidden_features or in_features\r\n self.fc1 = nn.Linear(in_features, hidden_features)\r\n self.act = act_layer()\r\n self.fc2 = nn.Linear(hidden_features, out_features)\r\n self.drop = nn.Dropout(drop)\r\n\r\n def forward(self, x):\r\n x = self.fc1(x)\r\n x = self.act(x)\r\n x = self.drop(x)\r\n x = self.fc2(x)\r\n x = self.drop(x)\r\n return x\r\n\r\ndef window_partition(x, window_size):\r\n\r\n B, H, W, C = x.shape\r\n assert H % window_size == 0, 'feature map h and w can not divide by window size'\r\n x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)\r\n windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)\r\n return windows\r\n\r\ndef window_reverse(windows, window_size, H, W):\r\n \r\n B = int(windows.shape[0] / (H * W / window_size / window_size))\r\n x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)\r\n x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)\r\n return x\r\n\r\n\r\nclass SwinTransformerLayer(nn.Module):\r\n\r\n def __init__(self, dim, num_heads, window_size=8, shift_size=0,\r\n mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0.,\r\n act_layer=nn.SiLU, norm_layer=nn.LayerNorm):\r\n super().__init__()\r\n self.dim = dim\r\n self.num_heads = num_heads\r\n self.window_size = window_size\r\n self.shift_size = shift_size\r\n self.mlp_ratio = mlp_ratio\r\n # if min(self.input_resolution) <= self.window_size:\r\n # # if window size is larger than input resolution, we don't partition windows\r\n # self.shift_size = 0\r\n # self.window_size = min(self.input_resolution)\r\n assert 0 <= self.shift_size < self.window_size, \"shift_size must in 0-window_size\"\r\n\r\n self.norm1 = norm_layer(dim)\r\n self.attn = WindowAttention(\r\n dim, window_size=(self.window_size, self.window_size), num_heads=num_heads,\r\n qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)\r\n\r\n self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\r\n self.norm2 = norm_layer(dim)\r\n mlp_hidden_dim = int(dim * mlp_ratio)\r\n self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)\r\n\r\n def create_mask(self, H, W):\r\n # calculate attention mask for SW-MSA\r\n img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1\r\n h_slices = (slice(0, -self.window_size),\r\n slice(-self.window_size, -self.shift_size),\r\n slice(-self.shift_size, None))\r\n w_slices = (slice(0, -self.window_size),\r\n slice(-self.window_size, -self.shift_size),\r\n slice(-self.shift_size, None))\r\n cnt = 0\r\n for h in h_slices:\r\n for w in w_slices:\r\n img_mask[:, h, w, :] = cnt\r\n cnt += 1\r\n\r\n mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1\r\n mask_windows = mask_windows.view(-1, self.window_size * self.window_size)\r\n attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)\r\n attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))\r\n\r\n return attn_mask\r\n\r\n def forward(self, x):\r\n # reshape x[b c h w] to x[b l c]\r\n _, _, H_, W_ = x.shape\r\n\r\n Padding = False\r\n if min(H_, W_) < self.window_size or H_ % self.window_size!=0 or W_ % self.window_size!=0:\r\n Padding = True\r\n # print(f'img_size {min(H_, W_)} is less than (or not divided by) window_size {self.window_size}, Padding.')\r\n pad_r = (self.window_size - W_ % self.window_size) % self.window_size\r\n pad_b = (self.window_size - H_ % self.window_size) % self.window_size\r\n x = F.pad(x, (0, pad_r, 0, pad_b))\r\n\r\n # print('2', x.shape)\r\n B, C, H, W = x.shape\r\n L = H * W\r\n x = x.permute(0, 2, 3, 1).contiguous().view(B, L, C) # b, L, c\r\n\r\n # create mask from init to forward\r\n if self.shift_size > 0:\r\n attn_mask = self.create_mask(H, W).to(x.device)\r\n else:\r\n attn_mask = None\r\n\r\n shortcut = x\r\n x = self.norm1(x)\r\n x = x.view(B, H, W, C)\r\n\r\n # cyclic shift\r\n if self.shift_size > 0:\r\n shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))\r\n else:\r\n shifted_x = x\r\n\r\n # partition windows\r\n x_windows = window_partition(shifted_x, self.window_size) # nW*B, window_size, window_size, C\r\n x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C\r\n\r\n # W-MSA/SW-MSA\r\n attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C\r\n\r\n # merge windows\r\n attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)\r\n shifted_x = window_reverse(attn_windows, self.window_size, H, W) # B H' W' C\r\n\r\n # reverse cyclic shift\r\n if self.shift_size > 0:\r\n x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))\r\n else:\r\n x = shifted_x\r\n x = x.view(B, H * W, C)\r\n\r\n # FFN\r\n x = shortcut + self.drop_path(x)\r\n x = x + self.drop_path(self.mlp(self.norm2(x)))\r\n\r\n x = x.permute(0, 2, 1).contiguous().view(-1, C, H, W) # b c h w\r\n\r\n if Padding:\r\n x = x[:, :, :H_, :W_] # reverse padding\r\n\r\n return x\r\n\r\n\r\nclass SwinTransformerBlock(nn.Module):\r\n def __init__(self, c1, c2, num_heads, num_layers, window_size=8):\r\n super().__init__()\r\n self.conv = None\r\n if c1 != c2:\r\n self.conv = Conv(c1, c2)\r\n\r\n # remove input_resolution\r\n self.blocks = nn.Sequential(*[SwinTransformerLayer(dim=c2, num_heads=num_heads, window_size=window_size,\r\n shift_size=0 if (i % 2 == 0) else window_size // 2) for i in range(num_layers)])\r\n\r\n def forward(self, x):\r\n if self.conv is not None:\r\n x = self.conv(x)\r\n x = self.blocks(x)\r\n return x\r\n\r\n\r\nclass STCSPA(nn.Module):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super(STCSPA, self).__init__()\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c1, c_, 1, 1)\r\n self.cv3 = Conv(2 * c_, c2, 1, 1)\r\n num_heads = c_ // 32\r\n self.m = SwinTransformerBlock(c_, c_, num_heads, n)\r\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n def forward(self, x):\r\n y1 = self.m(self.cv1(x))\r\n y2 = self.cv2(x)\r\n return self.cv3(torch.cat((y1, y2), dim=1))\r\n\r\n\r\nclass STCSPB(nn.Module):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super(STCSPB, self).__init__()\r\n c_ = int(c2) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c_, c_, 1, 1)\r\n self.cv3 = Conv(2 * c_, c2, 1, 1)\r\n num_heads = c_ // 32\r\n self.m = SwinTransformerBlock(c_, c_, num_heads, n)\r\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n def forward(self, x):\r\n x1 = self.cv1(x)\r\n y1 = self.m(x1)\r\n y2 = self.cv2(x1)\r\n return self.cv3(torch.cat((y1, y2), dim=1))\r\n\r\n\r\nclass STCSPC(nn.Module):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super(STCSPC, self).__init__()\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c1, c_, 1, 1)\r\n self.cv3 = Conv(c_, c_, 1, 1)\r\n self.cv4 = Conv(2 * c_, c2, 1, 1)\r\n num_heads = c_ // 32\r\n self.m = SwinTransformerBlock(c_, c_, num_heads, n)\r\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n def forward(self, x):\r\n y1 = self.cv3(self.m(self.cv1(x)))\r\n y2 = self.cv2(x)\r\n return self.cv4(torch.cat((y1, y2), dim=1))\r\n\r\n##### end of swin transformer ##### \r\n\r\n\r\n##### swin transformer v2 ##### \r\n \r\nclass WindowAttention_v2(nn.Module):\r\n\r\n def __init__(self, dim, window_size, num_heads, qkv_bias=True, attn_drop=0., proj_drop=0.,\r\n pretrained_window_size=[0, 0]):\r\n\r\n super().__init__()\r\n self.dim = dim\r\n self.window_size = window_size # Wh, Ww\r\n self.pretrained_window_size = pretrained_window_size\r\n self.num_heads = num_heads\r\n\r\n self.logit_scale = nn.Parameter(torch.log(10 * torch.ones((num_heads, 1, 1))), requires_grad=True)\r\n\r\n # mlp to generate continuous relative position bias\r\n self.cpb_mlp = nn.Sequential(nn.Linear(2, 512, bias=True),\r\n nn.ReLU(inplace=True),\r\n nn.Linear(512, num_heads, bias=False))\r\n\r\n # get relative_coords_table\r\n relative_coords_h = torch.arange(-(self.window_size[0] - 1), self.window_size[0], dtype=torch.float32)\r\n relative_coords_w = torch.arange(-(self.window_size[1] - 1), self.window_size[1], dtype=torch.float32)\r\n relative_coords_table = torch.stack(\r\n torch.meshgrid([relative_coords_h,\r\n relative_coords_w])).permute(1, 2, 0).contiguous().unsqueeze(0) # 1, 2*Wh-1, 2*Ww-1, 2\r\n if pretrained_window_size[0] > 0:\r\n relative_coords_table[:, :, :, 0] /= (pretrained_window_size[0] - 1)\r\n relative_coords_table[:, :, :, 1] /= (pretrained_window_size[1] - 1)\r\n else:\r\n relative_coords_table[:, :, :, 0] /= (self.window_size[0] - 1)\r\n relative_coords_table[:, :, :, 1] /= (self.window_size[1] - 1)\r\n relative_coords_table *= 8 # normalize to -8, 8\r\n relative_coords_table = torch.sign(relative_coords_table) * torch.log2(\r\n torch.abs(relative_coords_table) + 1.0) / np.log2(8)\r\n\r\n self.register_buffer(\"relative_coords_table\", relative_coords_table)\r\n\r\n # get pair-wise relative position index for each token inside the window\r\n coords_h = torch.arange(self.window_size[0])\r\n coords_w = torch.arange(self.window_size[1])\r\n coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww\r\n coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww\r\n relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww\r\n relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2\r\n relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0\r\n relative_coords[:, :, 1] += self.window_size[1] - 1\r\n relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1\r\n relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww\r\n self.register_buffer(\"relative_position_index\", relative_position_index)\r\n\r\n self.qkv = nn.Linear(dim, dim * 3, bias=False)\r\n if qkv_bias:\r\n self.q_bias = nn.Parameter(torch.zeros(dim))\r\n self.v_bias = nn.Parameter(torch.zeros(dim))\r\n else:\r\n self.q_bias = None\r\n self.v_bias = None\r\n self.attn_drop = nn.Dropout(attn_drop)\r\n self.proj = nn.Linear(dim, dim)\r\n self.proj_drop = nn.Dropout(proj_drop)\r\n self.softmax = nn.Softmax(dim=-1)\r\n\r\n def forward(self, x, mask=None):\r\n \r\n B_, N, C = x.shape\r\n qkv_bias = None\r\n if self.q_bias is not None:\r\n qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias))\r\n qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias)\r\n qkv = qkv.reshape(B_, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)\r\n q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple)\r\n\r\n # cosine attention\r\n attn = (F.normalize(q, dim=-1) @ F.normalize(k, dim=-1).transpose(-2, -1))\r\n logit_scale = torch.clamp(self.logit_scale, max=torch.log(torch.tensor(1. / 0.01))).exp()\r\n attn = attn * logit_scale\r\n\r\n relative_position_bias_table = self.cpb_mlp(self.relative_coords_table).view(-1, self.num_heads)\r\n relative_position_bias = relative_position_bias_table[self.relative_position_index.view(-1)].view(\r\n self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH\r\n relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww\r\n relative_position_bias = 16 * torch.sigmoid(relative_position_bias)\r\n attn = attn + relative_position_bias.unsqueeze(0)\r\n\r\n if mask is not None:\r\n nW = mask.shape[0]\r\n attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)\r\n attn = attn.view(-1, self.num_heads, N, N)\r\n attn = self.softmax(attn)\r\n else:\r\n attn = self.softmax(attn)\r\n\r\n attn = self.attn_drop(attn)\r\n\r\n try:\r\n x = (attn @ v).transpose(1, 2).reshape(B_, N, C)\r\n except:\r\n x = (attn.half() @ v).transpose(1, 2).reshape(B_, N, C)\r\n \r\n x = self.proj(x)\r\n x = self.proj_drop(x)\r\n return x\r\n\r\n def extra_repr(self) -> str:\r\n return f'dim={self.dim}, window_size={self.window_size}, ' \\\r\n f'pretrained_window_size={self.pretrained_window_size}, num_heads={self.num_heads}'\r\n\r\n def flops(self, N):\r\n # calculate flops for 1 window with token length of N\r\n flops = 0\r\n # qkv = self.qkv(x)\r\n flops += N * self.dim * 3 * self.dim\r\n # attn = (q @ k.transpose(-2, -1))\r\n flops += self.num_heads * N * (self.dim // self.num_heads) * N\r\n # x = (attn @ v)\r\n flops += self.num_heads * N * N * (self.dim // self.num_heads)\r\n # x = self.proj(x)\r\n flops += N * self.dim * self.dim\r\n return flops\r\n \r\nclass Mlp_v2(nn.Module):\r\n def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, drop=0.):\r\n super().__init__()\r\n out_features = out_features or in_features\r\n hidden_features = hidden_features or in_features\r\n self.fc1 = nn.Linear(in_features, hidden_features)\r\n self.act = act_layer()\r\n self.fc2 = nn.Linear(hidden_features, out_features)\r\n self.drop = nn.Dropout(drop)\r\n\r\n def forward(self, x):\r\n x = self.fc1(x)\r\n x = self.act(x)\r\n x = self.drop(x)\r\n x = self.fc2(x)\r\n x = self.drop(x)\r\n return x\r\n\r\n\r\ndef window_partition_v2(x, window_size):\r\n \r\n B, H, W, C = x.shape\r\n x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)\r\n windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)\r\n return windows\r\n\r\n\r\ndef window_reverse_v2(windows, window_size, H, W):\r\n \r\n B = int(windows.shape[0] / (H * W / window_size / window_size))\r\n x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)\r\n x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)\r\n return x\r\n\r\n\r\nclass SwinTransformerLayer_v2(nn.Module):\r\n\r\n def __init__(self, dim, num_heads, window_size=7, shift_size=0,\r\n mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0., drop_path=0.,\r\n act_layer=nn.SiLU, norm_layer=nn.LayerNorm, pretrained_window_size=0):\r\n super().__init__()\r\n self.dim = dim\r\n #self.input_resolution = input_resolution\r\n self.num_heads = num_heads\r\n self.window_size = window_size\r\n self.shift_size = shift_size\r\n self.mlp_ratio = mlp_ratio\r\n #if min(self.input_resolution) <= self.window_size:\r\n # # if window size is larger than input resolution, we don't partition windows\r\n # self.shift_size = 0\r\n # self.window_size = min(self.input_resolution)\r\n assert 0 <= self.shift_size < self.window_size, \"shift_size must in 0-window_size\"\r\n\r\n self.norm1 = norm_layer(dim)\r\n self.attn = WindowAttention_v2(\r\n dim, window_size=(self.window_size, self.window_size), num_heads=num_heads,\r\n qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop,\r\n pretrained_window_size=(pretrained_window_size, pretrained_window_size))\r\n\r\n self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\r\n self.norm2 = norm_layer(dim)\r\n mlp_hidden_dim = int(dim * mlp_ratio)\r\n self.mlp = Mlp_v2(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)\r\n\r\n def create_mask(self, H, W):\r\n # calculate attention mask for SW-MSA\r\n img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1\r\n h_slices = (slice(0, -self.window_size),\r\n slice(-self.window_size, -self.shift_size),\r\n slice(-self.shift_size, None))\r\n w_slices = (slice(0, -self.window_size),\r\n slice(-self.window_size, -self.shift_size),\r\n slice(-self.shift_size, None))\r\n cnt = 0\r\n for h in h_slices:\r\n for w in w_slices:\r\n img_mask[:, h, w, :] = cnt\r\n cnt += 1\r\n\r\n mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1\r\n mask_windows = mask_windows.view(-1, self.window_size * self.window_size)\r\n attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)\r\n attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))\r\n\r\n return attn_mask\r\n\r\n def forward(self, x):\r\n # reshape x[b c h w] to x[b l c]\r\n _, _, H_, W_ = x.shape\r\n\r\n Padding = False\r\n if min(H_, W_) < self.window_size or H_ % self.window_size!=0 or W_ % self.window_size!=0:\r\n Padding = True\r\n # print(f'img_size {min(H_, W_)} is less than (or not divided by) window_size {self.window_size}, Padding.')\r\n pad_r = (self.window_size - W_ % self.window_size) % self.window_size\r\n pad_b = (self.window_size - H_ % self.window_size) % self.window_size\r\n x = F.pad(x, (0, pad_r, 0, pad_b))\r\n\r\n # print('2', x.shape)\r\n B, C, H, W = x.shape\r\n L = H * W\r\n x = x.permute(0, 2, 3, 1).contiguous().view(B, L, C) # b, L, c\r\n\r\n # create mask from init to forward\r\n if self.shift_size > 0:\r\n attn_mask = self.create_mask(H, W).to(x.device)\r\n else:\r\n attn_mask = None\r\n\r\n shortcut = x\r\n x = x.view(B, H, W, C)\r\n\r\n # cyclic shift\r\n if self.shift_size > 0:\r\n shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))\r\n else:\r\n shifted_x = x\r\n\r\n # partition windows\r\n x_windows = window_partition_v2(shifted_x, self.window_size) # nW*B, window_size, window_size, C\r\n x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C\r\n\r\n # W-MSA/SW-MSA\r\n attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C\r\n\r\n # merge windows\r\n attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)\r\n shifted_x = window_reverse_v2(attn_windows, self.window_size, H, W) # B H' W' C\r\n\r\n # reverse cyclic shift\r\n if self.shift_size > 0:\r\n x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))\r\n else:\r\n x = shifted_x\r\n x = x.view(B, H * W, C)\r\n x = shortcut + self.drop_path(self.norm1(x))\r\n\r\n # FFN\r\n x = x + self.drop_path(self.norm2(self.mlp(x)))\r\n x = x.permute(0, 2, 1).contiguous().view(-1, C, H, W) # b c h w\r\n \r\n if Padding:\r\n x = x[:, :, :H_, :W_] # reverse padding\r\n\r\n return x\r\n\r\n def extra_repr(self) -> str:\r\n return f\"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, \" \\\r\n f\"window_size={self.window_size}, shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}\"\r\n\r\n def flops(self):\r\n flops = 0\r\n H, W = self.input_resolution\r\n # norm1\r\n flops += self.dim * H * W\r\n # W-MSA/SW-MSA\r\n nW = H * W / self.window_size / self.window_size\r\n flops += nW * self.attn.flops(self.window_size * self.window_size)\r\n # mlp\r\n flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio\r\n # norm2\r\n flops += self.dim * H * W\r\n return flops\r\n\r\n\r\nclass SwinTransformer2Block(nn.Module):\r\n def __init__(self, c1, c2, num_heads, num_layers, window_size=7):\r\n super().__init__()\r\n self.conv = None\r\n if c1 != c2:\r\n self.conv = Conv(c1, c2)\r\n\r\n # remove input_resolution\r\n self.blocks = nn.Sequential(*[SwinTransformerLayer_v2(dim=c2, num_heads=num_heads, window_size=window_size,\r\n shift_size=0 if (i % 2 == 0) else window_size // 2) for i in range(num_layers)])\r\n\r\n def forward(self, x):\r\n if self.conv is not None:\r\n x = self.conv(x)\r\n x = self.blocks(x)\r\n return x\r\n\r\n\r\nclass ST2CSPA(nn.Module):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super(ST2CSPA, self).__init__()\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c1, c_, 1, 1)\r\n self.cv3 = Conv(2 * c_, c2, 1, 1)\r\n num_heads = c_ // 32\r\n self.m = SwinTransformer2Block(c_, c_, num_heads, n)\r\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n def forward(self, x):\r\n y1 = self.m(self.cv1(x))\r\n y2 = self.cv2(x)\r\n return self.cv3(torch.cat((y1, y2), dim=1))\r\n\r\n\r\nclass ST2CSPB(nn.Module):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super(ST2CSPB, self).__init__()\r\n c_ = int(c2) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c_, c_, 1, 1)\r\n self.cv3 = Conv(2 * c_, c2, 1, 1)\r\n num_heads = c_ // 32\r\n self.m = SwinTransformer2Block(c_, c_, num_heads, n)\r\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n def forward(self, x):\r\n x1 = self.cv1(x)\r\n y1 = self.m(x1)\r\n y2 = self.cv2(x1)\r\n return self.cv3(torch.cat((y1, y2), dim=1))\r\n\r\n\r\nclass ST2CSPC(nn.Module):\r\n # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks\r\n def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion\r\n super(ST2CSPC, self).__init__()\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv1 = Conv(c1, c_, 1, 1)\r\n self.cv2 = Conv(c1, c_, 1, 1)\r\n self.cv3 = Conv(c_, c_, 1, 1)\r\n self.cv4 = Conv(2 * c_, c2, 1, 1)\r\n num_heads = c_ // 32\r\n self.m = SwinTransformer2Block(c_, c_, num_heads, n)\r\n #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)])\r\n\r\n def forward(self, x):\r\n y1 = self.cv3(self.m(self.cv1(x)))\r\n y2 = self.cv2(x)\r\n return self.cv4(torch.cat((y1, y2), dim=1))\r\n\r\n##### end of swin transformer v2 ##### \r\n" }, { "path": "asone/pose_estimators/yolov7_pose/models/experimental.py", "content": "import numpy as np\r\nimport random\r\nimport torch\r\nimport torch.nn as nn\r\n\r\nfrom asone.pose_estimators.yolov7_pose.models.common import Conv, DWConv\r\nfrom asone.pose_estimators.yolov7_pose.utils.google_utils import attempt_download\r\nfrom asone.pose_estimators.yolov7_pose.utils.yolov7_pose_utils import yolov7_in_syspath\r\n\r\nclass CrossConv(nn.Module):\r\n # Cross Convolution Downsample\r\n def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False):\r\n # ch_in, ch_out, kernel, stride, groups, expansion, shortcut\r\n super(CrossConv, self).__init__()\r\n c_ = int(c2 * e) # hidden channels\r\n self.cv1 = Conv(c1, c_, (1, k), (1, s))\r\n self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g)\r\n self.add = shortcut and c1 == c2\r\n\r\n def forward(self, x):\r\n return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x))\r\n\r\n\r\nclass Sum(nn.Module):\r\n # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070\r\n def __init__(self, n, weight=False): # n: number of inputs\r\n super(Sum, self).__init__()\r\n self.weight = weight # apply weights boolean\r\n self.iter = range(n - 1) # iter object\r\n if weight:\r\n self.w = nn.Parameter(-torch.arange(1., n) / 2, requires_grad=True) # layer weights\r\n\r\n def forward(self, x):\r\n y = x[0] # no weight\r\n if self.weight:\r\n w = torch.sigmoid(self.w) * 2\r\n for i in self.iter:\r\n y = y + x[i + 1] * w[i]\r\n else:\r\n for i in self.iter:\r\n y = y + x[i + 1]\r\n return y\r\n\r\n\r\nclass MixConv2d(nn.Module):\r\n # Mixed Depthwise Conv https://arxiv.org/abs/1907.09595\r\n def __init__(self, c1, c2, k=(1, 3), s=1, equal_ch=True):\r\n super(MixConv2d, self).__init__()\r\n groups = len(k)\r\n if equal_ch: # equal c_ per group\r\n i = torch.linspace(0, groups - 1E-6, c2).floor() # c2 indices\r\n c_ = [(i == g).sum() for g in range(groups)] # intermediate channels\r\n else: # equal weight.numel() per group\r\n b = [c2] + [0] * groups\r\n a = np.eye(groups + 1, groups, k=-1)\r\n a -= np.roll(a, 1, axis=1)\r\n a *= np.array(k) ** 2\r\n a[0] = 1\r\n c_ = np.linalg.lstsq(a, b, rcond=None)[0].round() # solve for equal weight indices, ax = b\r\n\r\n self.m = nn.ModuleList([nn.Conv2d(c1, int(c_[g]), k[g], s, k[g] // 2, bias=False) for g in range(groups)])\r\n self.bn = nn.BatchNorm2d(c2)\r\n self.act = nn.LeakyReLU(0.1, inplace=True)\r\n\r\n def forward(self, x):\r\n return x + self.act(self.bn(torch.cat([m(x) for m in self.m], 1)))\r\n\r\n\r\nclass Ensemble(nn.ModuleList):\r\n # Ensemble of models\r\n def __init__(self):\r\n super(Ensemble, self).__init__()\r\n\r\n def forward(self, x, augment=False):\r\n y = []\r\n for module in self:\r\n y.append(module(x, augment)[0])\r\n # y = torch.stack(y).max(0)[0] # max ensemble\r\n # y = torch.stack(y).mean(0) # mean ensemble\r\n y = torch.cat(y, 1) # nms ensemble\r\n return y, None # inference, train output\r\n\r\n\r\n\r\n\r\n\r\nclass ORT_NMS(torch.autograd.Function):\r\n '''ONNX-Runtime NMS operation'''\r\n @staticmethod\r\n def forward(ctx,\r\n boxes,\r\n scores,\r\n max_output_boxes_per_class=torch.tensor([100]),\r\n iou_threshold=torch.tensor([0.45]),\r\n score_threshold=torch.tensor([0.25])):\r\n device = boxes.device\r\n batch = scores.shape[0]\r\n num_det = random.randint(0, 100)\r\n batches = torch.randint(0, batch, (num_det,)).sort()[0].to(device)\r\n idxs = torch.arange(100, 100 + num_det).to(device)\r\n zeros = torch.zeros((num_det,), dtype=torch.int64).to(device)\r\n selected_indices = torch.cat([batches[None], zeros[None], idxs[None]], 0).T.contiguous()\r\n selected_indices = selected_indices.to(torch.int64)\r\n return selected_indices\r\n\r\n @staticmethod\r\n def symbolic(g, boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold):\r\n return g.op(\"NonMaxSuppression\", boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold)\r\n\r\n\r\nclass TRT_NMS(torch.autograd.Function):\r\n '''TensorRT NMS operation'''\r\n @staticmethod\r\n def forward(\r\n ctx,\r\n boxes,\r\n scores,\r\n background_class=-1,\r\n box_coding=1,\r\n iou_threshold=0.45,\r\n max_output_boxes=100,\r\n plugin_version=\"1\",\r\n score_activation=0,\r\n score_threshold=0.25,\r\n ):\r\n batch_size, num_boxes, num_classes = scores.shape\r\n num_det = torch.randint(0, max_output_boxes, (batch_size, 1), dtype=torch.int32)\r\n det_boxes = torch.randn(batch_size, max_output_boxes, 4)\r\n det_scores = torch.randn(batch_size, max_output_boxes)\r\n det_classes = torch.randint(0, num_classes, (batch_size, max_output_boxes), dtype=torch.int32)\r\n return num_det, det_boxes, det_scores, det_classes\r\n\r\n @staticmethod\r\n def symbolic(g,\r\n boxes,\r\n scores,\r\n background_class=-1,\r\n box_coding=1,\r\n iou_threshold=0.45,\r\n max_output_boxes=100,\r\n plugin_version=\"1\",\r\n score_activation=0,\r\n score_threshold=0.25):\r\n out = g.op(\"TRT::EfficientNMS_TRT\",\r\n boxes,\r\n scores,\r\n background_class_i=background_class,\r\n box_coding_i=box_coding,\r\n iou_threshold_f=iou_threshold,\r\n max_output_boxes_i=max_output_boxes,\r\n plugin_version_s=plugin_version,\r\n score_activation_i=score_activation,\r\n score_threshold_f=score_threshold,\r\n outputs=4)\r\n nums, boxes, scores, classes = out\r\n return nums, boxes, scores, classes\r\n\r\n\r\nclass ONNX_ORT(nn.Module):\r\n '''onnx module with ONNX-Runtime NMS operation.'''\r\n def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=640, device=None):\r\n super().__init__()\r\n self.device = device if device else torch.device(\"cpu\")\r\n self.max_obj = torch.tensor([max_obj]).to(device)\r\n self.iou_threshold = torch.tensor([iou_thres]).to(device)\r\n self.score_threshold = torch.tensor([score_thres]).to(device)\r\n self.max_wh = max_wh # if max_wh != 0 : non-agnostic else : agnostic\r\n self.convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],\r\n dtype=torch.float32,\r\n device=self.device)\r\n\r\n def forward(self, x):\r\n boxes = x[:, :, :4]\r\n conf = x[:, :, 4:5]\r\n scores = x[:, :, 5:]\r\n scores *= conf\r\n boxes @= self.convert_matrix\r\n max_score, category_id = scores.max(2, keepdim=True)\r\n dis = category_id.float() * self.max_wh\r\n nmsbox = boxes + dis\r\n max_score_tp = max_score.transpose(1, 2).contiguous()\r\n selected_indices = ORT_NMS.apply(nmsbox, max_score_tp, self.max_obj, self.iou_threshold, self.score_threshold)\r\n X, Y = selected_indices[:, 0], selected_indices[:, 2]\r\n selected_boxes = boxes[X, Y, :]\r\n selected_categories = category_id[X, Y, :].float()\r\n selected_scores = max_score[X, Y, :]\r\n X = X.unsqueeze(1).float()\r\n return torch.cat([X, selected_boxes, selected_categories, selected_scores], 1)\r\n\r\nclass ONNX_TRT(nn.Module):\r\n '''onnx module with TensorRT NMS operation.'''\r\n def __init__(self, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None ,device=None):\r\n super().__init__()\r\n assert max_wh is None\r\n self.device = device if device else torch.device('cpu')\r\n self.background_class = -1,\r\n self.box_coding = 1,\r\n self.iou_threshold = iou_thres\r\n self.max_obj = max_obj\r\n self.plugin_version = '1'\r\n self.score_activation = 0\r\n self.score_threshold = score_thres\r\n\r\n def forward(self, x):\r\n boxes = x[:, :, :4]\r\n conf = x[:, :, 4:5]\r\n scores = x[:, :, 5:]\r\n scores *= conf\r\n num_det, det_boxes, det_scores, det_classes = TRT_NMS.apply(boxes, scores, self.background_class, self.box_coding,\r\n self.iou_threshold, self.max_obj,\r\n self.plugin_version, self.score_activation,\r\n self.score_threshold)\r\n return num_det, det_boxes, det_scores, det_classes\r\n\r\n\r\nclass End2End(nn.Module):\r\n '''export onnx or tensorrt model with NMS operation.'''\r\n def __init__(self, model, max_obj=100, iou_thres=0.45, score_thres=0.25, max_wh=None, device=None):\r\n super().__init__()\r\n device = device if device else torch.device('cpu')\r\n assert isinstance(max_wh,(int)) or max_wh is None\r\n self.model = model.to(device)\r\n self.model.model[-1].end2end = True\r\n self.patch_model = ONNX_TRT if max_wh is None else ONNX_ORT\r\n self.end2end = self.patch_model(max_obj, iou_thres, score_thres, max_wh, device)\r\n self.end2end.eval()\r\n\r\n def forward(self, x):\r\n x = self.model(x)\r\n x = self.end2end(x)\r\n return x\r\n\r\n\r\n\r\n\r\n\r\ndef attempt_load(weights, map_location=None):\r\n # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a\r\n model = Ensemble()\r\n for w in weights if isinstance(weights, list) else [weights]:\r\n attempt_download(w)\r\n with yolov7_in_syspath():\r\n ckpt = torch.load(w, map_location=map_location) # load\r\n model.append(ckpt['ema' if ckpt.get('ema') else 'model'].float().fuse().eval()) # FP32 model\r\n \r\n # Compatibility updates\r\n for m in model.modules():\r\n if type(m) in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]:\r\n m.inplace = True # pytorch 1.7.0 compatibility\r\n elif type(m) is nn.Upsample:\r\n m.recompute_scale_factor = None # torch 1.11.0 compatibility\r\n elif type(m) is Conv:\r\n m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatibility\r\n \r\n if len(model) == 1:\r\n return model[-1] # return model\r\n else:\r\n print('Ensemble created with %s\\n' % weights)\r\n for k in ['names', 'stride']:\r\n setattr(model, k, getattr(model[-1], k))\r\n return model # return ensemble\r\n\r\n\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/models/yolo.py", "content": "import argparse\r\nimport logging\r\nimport sys\r\nfrom copy import deepcopy\r\n\r\n# sys.path.append('./') # to run '$ python *.py' files in subdirectories\r\nlogger = logging.getLogger(__name__)\r\nimport torch\r\nfrom asone.pose_estimators.yolov7_pose.models.common import *\r\nfrom asone.pose_estimators.yolov7_pose.models.experimental import *\r\nfrom asone.pose_estimators.yolov7_pose.utils.autoanchor import check_anchor_order\r\nfrom asone.pose_estimators.yolov7_pose.utils.general import make_divisible, check_file, set_logging\r\nfrom asone.pose_estimators.yolov7_pose.utils.torch_utils import time_synchronized, fuse_conv_and_bn, model_info, scale_img, initialize_weights, \\\r\n select_device, copy_attr\r\nfrom asone.pose_estimators.yolov7_pose.utils.loss import SigmoidBin\r\n\r\ntry:\r\n import thop # for FLOPS computation\r\nexcept ImportError:\r\n thop = None\r\n\r\n\r\nclass Detect(nn.Module):\r\n stride = None # strides computed during build\r\n export = False # onnx export\r\n end2end = False\r\n include_nms = False \r\n\r\n def __init__(self, nc=80, anchors=(), ch=()): # detection layer\r\n super(Detect, self).__init__()\r\n self.nc = nc # number of classes\r\n self.no = nc + 5 # number of outputs per anchor\r\n self.nl = len(anchors) # number of detection layers\r\n self.na = len(anchors[0]) // 2 # number of anchors\r\n self.grid = [torch.zeros(1)] * self.nl # init grid\r\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\r\n self.register_buffer('anchors', a) # shape(nl,na,2)\r\n self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\r\n self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv\r\n\r\n def forward(self, x):\r\n # x = x.copy() # for profiling\r\n z = [] # inference output\r\n self.training |= self.export\r\n for i in range(self.nl):\r\n x[i] = self.m[i](x[i]) # conv\r\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\r\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()\r\n\r\n if not self.training: # inference\r\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\r\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\r\n y = x[i].sigmoid()\r\n if not torch.onnx.is_in_onnx_export():\r\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy\r\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\r\n else:\r\n xy = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy\r\n wh = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\r\n y = torch.cat((xy, wh, y[..., 4:]), -1)\r\n z.append(y.view(bs, -1, self.no))\r\n\r\n if self.training:\r\n out = x\r\n elif self.end2end:\r\n out = torch.cat(z, 1)\r\n elif self.include_nms:\r\n z = self.convert(z)\r\n out = (z, )\r\n else:\r\n out = (torch.cat(z, 1), x)\r\n\r\n return out\r\n\r\n @staticmethod\r\n def _make_grid(nx=20, ny=20):\r\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\r\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\r\n\r\n def convert(self, z):\r\n z = torch.cat(z, 1)\r\n box = z[:, :, :4]\r\n conf = z[:, :, 4:5]\r\n score = z[:, :, 5:]\r\n score *= conf\r\n convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],\r\n dtype=torch.float32,\r\n device=z.device)\r\n box @= convert_matrix \r\n return (box, score)\r\n\r\n\r\nclass IDetect(nn.Module):\r\n stride = None # strides computed during build\r\n export = False # onnx export\r\n end2end = False\r\n include_nms = False \r\n\r\n def __init__(self, nc=80, anchors=(), ch=()): # detection layer\r\n super(IDetect, self).__init__()\r\n self.nc = nc # number of classes\r\n self.no = nc + 5 # number of outputs per anchor\r\n self.nl = len(anchors) # number of detection layers\r\n self.na = len(anchors[0]) // 2 # number of anchors\r\n self.grid = [torch.zeros(1)] * self.nl # init grid\r\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\r\n self.register_buffer('anchors', a) # shape(nl,na,2)\r\n self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\r\n self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv\r\n \r\n self.ia = nn.ModuleList(ImplicitA(x) for x in ch)\r\n self.im = nn.ModuleList(ImplicitM(self.no * self.na) for _ in ch)\r\n\r\n def forward(self, x):\r\n # x = x.copy() # for profiling\r\n z = [] # inference output\r\n self.training |= self.export\r\n for i in range(self.nl):\r\n x[i] = self.m[i](self.ia[i](x[i])) # conv\r\n x[i] = self.im[i](x[i])\r\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\r\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()\r\n\r\n if not self.training: # inference\r\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\r\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\r\n\r\n y = x[i].sigmoid()\r\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy\r\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\r\n z.append(y.view(bs, -1, self.no))\r\n\r\n return x if self.training else (torch.cat(z, 1), x)\r\n \r\n def fuseforward(self, x):\r\n # x = x.copy() # for profiling\r\n z = [] # inference output\r\n self.training |= self.export\r\n for i in range(self.nl):\r\n x[i] = self.m[i](x[i]) # conv\r\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\r\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()\r\n\r\n if not self.training: # inference\r\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\r\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\r\n\r\n y = x[i].sigmoid()\r\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy\r\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\r\n z.append(y.view(bs, -1, self.no))\r\n\r\n if self.training:\r\n out = x\r\n elif self.end2end:\r\n out = torch.cat(z, 1)\r\n elif self.include_nms:\r\n z = self.convert(z)\r\n out = (z, )\r\n else:\r\n out = (torch.cat(z, 1), x)\r\n\r\n return out\r\n \r\n def fuse(self):\r\n print(\"IDetect.fuse\")\r\n # fuse ImplicitA and Convolution\r\n for i in range(len(self.m)):\r\n c1,c2,_,_ = self.m[i].weight.shape\r\n c1_,c2_, _,_ = self.ia[i].implicit.shape\r\n self.m[i].bias += torch.matmul(self.m[i].weight.reshape(c1,c2),self.ia[i].implicit.reshape(c2_,c1_)).squeeze(1)\r\n\r\n # fuse ImplicitM and Convolution\r\n for i in range(len(self.m)):\r\n c1,c2, _,_ = self.im[i].implicit.shape\r\n self.m[i].bias *= self.im[i].implicit.reshape(c2)\r\n self.m[i].weight *= self.im[i].implicit.transpose(0,1)\r\n \r\n @staticmethod\r\n def _make_grid(nx=20, ny=20):\r\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\r\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\r\n\r\n def convert(self, z):\r\n z = torch.cat(z, 1)\r\n box = z[:, :, :4]\r\n conf = z[:, :, 4:5]\r\n score = z[:, :, 5:]\r\n score *= conf\r\n convert_matrix = torch.tensor([[1, 0, 1, 0], [0, 1, 0, 1], [-0.5, 0, 0.5, 0], [0, -0.5, 0, 0.5]],\r\n dtype=torch.float32,\r\n device=z.device)\r\n box @= convert_matrix \r\n return (box, score)\r\n\r\n\r\nclass IKeypoint(nn.Module):\r\n stride = None # strides computed during build\r\n export = False # onnx export\r\n\r\n def __init__(self, nc=80, anchors=(), nkpt=17, ch=(), inplace=True, dw_conv_kpt=False): # detection layer\r\n super(IKeypoint, self).__init__()\r\n self.nc = nc # number of classes\r\n self.nkpt = nkpt\r\n self.dw_conv_kpt = dw_conv_kpt\r\n self.no_det=(nc + 5) # number of outputs per anchor for box and class\r\n self.no_kpt = 3*self.nkpt ## number of outputs per anchor for keypoints\r\n self.no = self.no_det+self.no_kpt\r\n self.nl = len(anchors) # number of detection layers\r\n self.na = len(anchors[0]) // 2 # number of anchors\r\n self.grid = [torch.zeros(1)] * self.nl # init grid\r\n self.flip_test = False\r\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\r\n self.register_buffer('anchors', a) # shape(nl,na,2)\r\n self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\r\n self.m = nn.ModuleList(nn.Conv2d(x, self.no_det * self.na, 1) for x in ch) # output conv\r\n \r\n self.ia = nn.ModuleList(ImplicitA(x) for x in ch)\r\n self.im = nn.ModuleList(ImplicitM(self.no_det * self.na) for _ in ch)\r\n \r\n if self.nkpt is not None:\r\n if self.dw_conv_kpt: #keypoint head is slightly more complex\r\n self.m_kpt = nn.ModuleList(\r\n nn.Sequential(DWConv(x, x, k=3), Conv(x,x),\r\n DWConv(x, x, k=3), Conv(x, x),\r\n DWConv(x, x, k=3), Conv(x,x),\r\n DWConv(x, x, k=3), Conv(x, x),\r\n DWConv(x, x, k=3), Conv(x, x),\r\n DWConv(x, x, k=3), nn.Conv2d(x, self.no_kpt * self.na, 1)) for x in ch)\r\n else: #keypoint head is a single convolution\r\n self.m_kpt = nn.ModuleList(nn.Conv2d(x, self.no_kpt * self.na, 1) for x in ch)\r\n\r\n self.inplace = inplace # use in-place ops (e.g. slice assignment)\r\n\r\n def forward(self, x):\r\n # x = x.copy() # for profiling\r\n z = [] # inference output\r\n self.training |= self.export\r\n for i in range(self.nl):\r\n if self.nkpt is None or self.nkpt==0:\r\n x[i] = self.im[i](self.m[i](self.ia[i](x[i]))) # conv\r\n else :\r\n x[i] = torch.cat((self.im[i](self.m[i](self.ia[i](x[i]))), self.m_kpt[i](x[i])), axis=1)\r\n\r\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\r\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()\r\n x_det = x[i][..., :6]\r\n x_kpt = x[i][..., 6:]\r\n\r\n if not self.training: # inference\r\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\r\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\r\n kpt_grid_x = self.grid[i][..., 0:1]\r\n kpt_grid_y = self.grid[i][..., 1:2]\r\n\r\n if self.nkpt == 0:\r\n y = x[i].sigmoid()\r\n else:\r\n y = x_det.sigmoid()\r\n\r\n if self.inplace:\r\n xy = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy\r\n wh = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i].view(1, self.na, 1, 1, 2) # wh\r\n if self.nkpt != 0:\r\n x_kpt[..., 0::3] = (x_kpt[..., ::3] * 2. - 0.5 + kpt_grid_x.repeat(1,1,1,1,17)) * self.stride[i] # xy\r\n x_kpt[..., 1::3] = (x_kpt[..., 1::3] * 2. - 0.5 + kpt_grid_y.repeat(1,1,1,1,17)) * self.stride[i] # xy\r\n #x_kpt[..., 0::3] = (x_kpt[..., ::3] + kpt_grid_x.repeat(1,1,1,1,17)) * self.stride[i] # xy\r\n #x_kpt[..., 1::3] = (x_kpt[..., 1::3] + kpt_grid_y.repeat(1,1,1,1,17)) * self.stride[i] # xy\r\n #print('=============')\r\n #print(self.anchor_grid[i].shape)\r\n #print(self.anchor_grid[i][...,0].unsqueeze(4).shape)\r\n #print(x_kpt[..., 0::3].shape)\r\n #x_kpt[..., 0::3] = ((x_kpt[..., 0::3].tanh() * 2.) ** 3 * self.anchor_grid[i][...,0].unsqueeze(4).repeat(1,1,1,1,self.nkpt)) + kpt_grid_x.repeat(1,1,1,1,17) * self.stride[i] # xy\r\n #x_kpt[..., 1::3] = ((x_kpt[..., 1::3].tanh() * 2.) ** 3 * self.anchor_grid[i][...,1].unsqueeze(4).repeat(1,1,1,1,self.nkpt)) + kpt_grid_y.repeat(1,1,1,1,17) * self.stride[i] # xy\r\n #x_kpt[..., 0::3] = (((x_kpt[..., 0::3].sigmoid() * 4.) ** 2 - 8.) * self.anchor_grid[i][...,0].unsqueeze(4).repeat(1,1,1,1,self.nkpt)) + kpt_grid_x.repeat(1,1,1,1,17) * self.stride[i] # xy\r\n #x_kpt[..., 1::3] = (((x_kpt[..., 1::3].sigmoid() * 4.) ** 2 - 8.) * self.anchor_grid[i][...,1].unsqueeze(4).repeat(1,1,1,1,self.nkpt)) + kpt_grid_y.repeat(1,1,1,1,17) * self.stride[i] # xy\r\n x_kpt[..., 2::3] = x_kpt[..., 2::3].sigmoid()\r\n\r\n y = torch.cat((xy, wh, y[..., 4:], x_kpt), dim = -1)\r\n\r\n else: # for YOLOv5 on AWS Inferentia https://github.com/ultralytics/yolov5/pull/2953\r\n xy = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy\r\n wh = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\r\n if self.nkpt != 0:\r\n y[..., 6:] = (y[..., 6:] * 2. - 0.5 + self.grid[i].repeat((1,1,1,1,self.nkpt))) * self.stride[i] # xy\r\n y = torch.cat((xy, wh, y[..., 4:]), -1)\r\n\r\n z.append(y.view(bs, -1, self.no))\r\n\r\n return x if self.training else (torch.cat(z, 1), x)\r\n\r\n @staticmethod\r\n def _make_grid(nx=20, ny=20):\r\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\r\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\r\n\r\n\r\nclass IAuxDetect(nn.Module):\r\n stride = None # strides computed during build\r\n export = False # onnx export\r\n\r\n def __init__(self, nc=80, anchors=(), ch=()): # detection layer\r\n super(IAuxDetect, self).__init__()\r\n self.nc = nc # number of classes\r\n self.no = nc + 5 # number of outputs per anchor\r\n self.nl = len(anchors) # number of detection layers\r\n self.na = len(anchors[0]) // 2 # number of anchors\r\n self.grid = [torch.zeros(1)] * self.nl # init grid\r\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\r\n self.register_buffer('anchors', a) # shape(nl,na,2)\r\n self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\r\n self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch[:self.nl]) # output conv\r\n self.m2 = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch[self.nl:]) # output conv\r\n \r\n self.ia = nn.ModuleList(ImplicitA(x) for x in ch[:self.nl])\r\n self.im = nn.ModuleList(ImplicitM(self.no * self.na) for _ in ch[:self.nl])\r\n\r\n def forward(self, x):\r\n # x = x.copy() # for profiling\r\n z = [] # inference output\r\n self.training |= self.export\r\n for i in range(self.nl):\r\n x[i] = self.m[i](self.ia[i](x[i])) # conv\r\n x[i] = self.im[i](x[i])\r\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\r\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()\r\n \r\n x[i+self.nl] = self.m2[i](x[i+self.nl])\r\n x[i+self.nl] = x[i+self.nl].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()\r\n\r\n if not self.training: # inference\r\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\r\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\r\n\r\n y = x[i].sigmoid()\r\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy\r\n y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\r\n z.append(y.view(bs, -1, self.no))\r\n\r\n return x if self.training else (torch.cat(z, 1), x[:self.nl])\r\n\r\n @staticmethod\r\n def _make_grid(nx=20, ny=20):\r\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\r\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\r\n\r\n\r\nclass IBin(nn.Module):\r\n stride = None # strides computed during build\r\n export = False # onnx export\r\n\r\n def __init__(self, nc=80, anchors=(), ch=(), bin_count=21): # detection layer\r\n super(IBin, self).__init__()\r\n self.nc = nc # number of classes\r\n self.bin_count = bin_count\r\n\r\n self.w_bin_sigmoid = SigmoidBin(bin_count=self.bin_count, min=0.0, max=4.0)\r\n self.h_bin_sigmoid = SigmoidBin(bin_count=self.bin_count, min=0.0, max=4.0)\r\n # classes, x,y,obj\r\n self.no = nc + 3 + \\\r\n self.w_bin_sigmoid.get_length() + self.h_bin_sigmoid.get_length() # w-bce, h-bce\r\n # + self.x_bin_sigmoid.get_length() + self.y_bin_sigmoid.get_length()\r\n \r\n self.nl = len(anchors) # number of detection layers\r\n self.na = len(anchors[0]) // 2 # number of anchors\r\n self.grid = [torch.zeros(1)] * self.nl # init grid\r\n a = torch.tensor(anchors).float().view(self.nl, -1, 2)\r\n self.register_buffer('anchors', a) # shape(nl,na,2)\r\n self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2)\r\n self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv\r\n \r\n self.ia = nn.ModuleList(ImplicitA(x) for x in ch)\r\n self.im = nn.ModuleList(ImplicitM(self.no * self.na) for _ in ch)\r\n\r\n def forward(self, x):\r\n\r\n #self.x_bin_sigmoid.use_fw_regression = True\r\n #self.y_bin_sigmoid.use_fw_regression = True\r\n self.w_bin_sigmoid.use_fw_regression = True\r\n self.h_bin_sigmoid.use_fw_regression = True\r\n \r\n # x = x.copy() # for profiling\r\n z = [] # inference output\r\n self.training |= self.export\r\n for i in range(self.nl):\r\n x[i] = self.m[i](self.ia[i](x[i])) # conv\r\n x[i] = self.im[i](x[i])\r\n bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85)\r\n x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous()\r\n\r\n if not self.training: # inference\r\n if self.grid[i].shape[2:4] != x[i].shape[2:4]:\r\n self.grid[i] = self._make_grid(nx, ny).to(x[i].device)\r\n\r\n y = x[i].sigmoid()\r\n y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy\r\n #y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh\r\n \r\n\r\n #px = (self.x_bin_sigmoid.forward(y[..., 0:12]) + self.grid[i][..., 0]) * self.stride[i]\r\n #py = (self.y_bin_sigmoid.forward(y[..., 12:24]) + self.grid[i][..., 1]) * self.stride[i]\r\n\r\n pw = self.w_bin_sigmoid.forward(y[..., 2:24]) * self.anchor_grid[i][..., 0]\r\n ph = self.h_bin_sigmoid.forward(y[..., 24:46]) * self.anchor_grid[i][..., 1]\r\n\r\n #y[..., 0] = px\r\n #y[..., 1] = py\r\n y[..., 2] = pw\r\n y[..., 3] = ph\r\n \r\n y = torch.cat((y[..., 0:4], y[..., 46:]), dim=-1)\r\n \r\n z.append(y.view(bs, -1, y.shape[-1]))\r\n\r\n return x if self.training else (torch.cat(z, 1), x)\r\n\r\n @staticmethod\r\n def _make_grid(nx=20, ny=20):\r\n yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)])\r\n return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float()\r\n\r\n\r\nclass Model(nn.Module):\r\n def __init__(self, cfg='yolor-csp-c.yaml', ch=3, nc=None, anchors=None): # model, input channels, number of classes\r\n super(Model, self).__init__()\r\n self.traced = False\r\n if isinstance(cfg, dict):\r\n self.yaml = cfg # model dict\r\n else: # is *.yaml\r\n import yaml # for torch hub\r\n self.yaml_file = Path(cfg).name\r\n with open(cfg) as f:\r\n self.yaml = yaml.load(f, Loader=yaml.SafeLoader) # model dict\r\n\r\n # Define model\r\n ch = self.yaml['ch'] = self.yaml.get('ch', ch) # input channels\r\n if nc and nc != self.yaml['nc']:\r\n logger.info(f\"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}\")\r\n self.yaml['nc'] = nc # override yaml value\r\n if anchors:\r\n logger.info(f'Overriding model.yaml anchors with anchors={anchors}')\r\n self.yaml['anchors'] = round(anchors) # override yaml value\r\n self.model, self.save = parse_model(deepcopy(self.yaml), ch=[ch]) # model, savelist\r\n self.names = [str(i) for i in range(self.yaml['nc'])] # default names\r\n # print([x.shape for x in self.forward(torch.zeros(1, ch, 64, 64))])\r\n\r\n # Build strides, anchors\r\n m = self.model[-1] # Detect()\r\n if isinstance(m, Detect):\r\n s = 256 # 2x min stride\r\n m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward\r\n m.anchors /= m.stride.view(-1, 1, 1)\r\n check_anchor_order(m)\r\n self.stride = m.stride\r\n self._initialize_biases() # only run once\r\n # print('Strides: %s' % m.stride.tolist())\r\n if isinstance(m, IDetect):\r\n s = 256 # 2x min stride\r\n m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward\r\n m.anchors /= m.stride.view(-1, 1, 1)\r\n check_anchor_order(m)\r\n self.stride = m.stride\r\n self._initialize_biases() # only run once\r\n # print('Strides: %s' % m.stride.tolist())\r\n if isinstance(m, IAuxDetect):\r\n s = 256 # 2x min stride\r\n m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))[:4]]) # forward\r\n #print(m.stride)\r\n m.anchors /= m.stride.view(-1, 1, 1)\r\n check_anchor_order(m)\r\n self.stride = m.stride\r\n self._initialize_aux_biases() # only run once\r\n # print('Strides: %s' % m.stride.tolist())\r\n if isinstance(m, IBin):\r\n s = 256 # 2x min stride\r\n m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward\r\n m.anchors /= m.stride.view(-1, 1, 1)\r\n check_anchor_order(m)\r\n self.stride = m.stride\r\n self._initialize_biases_bin() # only run once\r\n # print('Strides: %s' % m.stride.tolist())\r\n if isinstance(m, IKeypoint):\r\n s = 256 # 2x min stride\r\n m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward\r\n m.anchors /= m.stride.view(-1, 1, 1)\r\n check_anchor_order(m)\r\n self.stride = m.stride\r\n self._initialize_biases_kpt() # only run once\r\n # print('Strides: %s' % m.stride.tolist())\r\n\r\n # Init weights, biases\r\n initialize_weights(self)\r\n self.info()\r\n logger.info('')\r\n\r\n def forward(self, x, augment=False, profile=False):\r\n if augment:\r\n img_size = x.shape[-2:] # height, width\r\n s = [1, 0.83, 0.67] # scales\r\n f = [None, 3, None] # flips (2-ud, 3-lr)\r\n y = [] # outputs\r\n for si, fi in zip(s, f):\r\n xi = scale_img(x.flip(fi) if fi else x, si, gs=int(self.stride.max()))\r\n yi = self.forward_once(xi)[0] # forward\r\n # cv2.imwrite(f'img_{si}.jpg', 255 * xi[0].cpu().numpy().transpose((1, 2, 0))[:, :, ::-1]) # save\r\n yi[..., :4] /= si # de-scale\r\n if fi == 2:\r\n yi[..., 1] = img_size[0] - yi[..., 1] # de-flip ud\r\n elif fi == 3:\r\n yi[..., 0] = img_size[1] - yi[..., 0] # de-flip lr\r\n y.append(yi)\r\n return torch.cat(y, 1), None # augmented inference, train\r\n else:\r\n return self.forward_once(x, profile) # single-scale inference, train\r\n\r\n def forward_once(self, x, profile=False):\r\n y, dt = [], [] # outputs\r\n for m in self.model:\r\n if m.f != -1: # if not from previous layer\r\n x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers\r\n\r\n if not hasattr(self, 'traced'):\r\n self.traced=False\r\n\r\n if self.traced:\r\n if isinstance(m, Detect) or isinstance(m, IDetect) or isinstance(m, IAuxDetect) or isinstance(m, IKeypoint):\r\n break\r\n\r\n if profile:\r\n c = isinstance(m, (Detect, IDetect, IAuxDetect, IBin))\r\n o = thop.profile(m, inputs=(x.copy() if c else x,), verbose=False)[0] / 1E9 * 2 if thop else 0 # FLOPS\r\n for _ in range(10):\r\n m(x.copy() if c else x)\r\n t = time_synchronized()\r\n for _ in range(10):\r\n m(x.copy() if c else x)\r\n dt.append((time_synchronized() - t) * 100)\r\n print('%10.1f%10.0f%10.1fms %-40s' % (o, m.np, dt[-1], m.type))\r\n\r\n x = m(x) # run\r\n \r\n y.append(x if m.i in self.save else None) # save output\r\n\r\n if profile:\r\n print('%.1fms total' % sum(dt))\r\n return x\r\n\r\n def _initialize_biases(self, cf=None): # initialize biases into Detect(), cf is class frequency\r\n # https://arxiv.org/abs/1708.02002 section 3.3\r\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\r\n m = self.model[-1] # Detect() module\r\n for mi, s in zip(m.m, m.stride): # from\r\n b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85)\r\n b.data[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image)\r\n b.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls\r\n mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\r\n\r\n def _initialize_aux_biases(self, cf=None): # initialize biases into Detect(), cf is class frequency\r\n # https://arxiv.org/abs/1708.02002 section 3.3\r\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\r\n m = self.model[-1] # Detect() module\r\n for mi, mi2, s in zip(m.m, m.m2, m.stride): # from\r\n b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85)\r\n b.data[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image)\r\n b.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls\r\n mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\r\n b2 = mi2.bias.view(m.na, -1) # conv.bias(255) to (3,85)\r\n b2.data[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image)\r\n b2.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls\r\n mi2.bias = torch.nn.Parameter(b2.view(-1), requires_grad=True)\r\n\r\n def _initialize_biases_bin(self, cf=None): # initialize biases into Detect(), cf is class frequency\r\n # https://arxiv.org/abs/1708.02002 section 3.3\r\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\r\n m = self.model[-1] # Bin() module\r\n bc = m.bin_count\r\n for mi, s in zip(m.m, m.stride): # from\r\n b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85)\r\n old = b[:, (0,1,2,bc+3)].data\r\n obj_idx = 2*bc+4\r\n b[:, :obj_idx].data += math.log(0.6 / (bc + 1 - 0.99))\r\n b[:, obj_idx].data += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image)\r\n b[:, (obj_idx+1):].data += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls\r\n b[:, (0,1,2,bc+3)].data = old\r\n mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\r\n\r\n def _initialize_biases_kpt(self, cf=None): # initialize biases into Detect(), cf is class frequency\r\n # https://arxiv.org/abs/1708.02002 section 3.3\r\n # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1.\r\n m = self.model[-1] # Detect() module\r\n for mi, s in zip(m.m, m.stride): # from\r\n b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85)\r\n b.data[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image)\r\n b.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls\r\n mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True)\r\n\r\n def _print_biases(self):\r\n m = self.model[-1] # Detect() module\r\n for mi in m.m: # from\r\n b = mi.bias.detach().view(m.na, -1).T # conv.bias(255) to (3,85)\r\n print(('%6g Conv2d.bias:' + '%10.3g' * 6) % (mi.weight.shape[1], *b[:5].mean(1).tolist(), b[5:].mean()))\r\n\r\n # def _print_weights(self):\r\n # for m in self.model.modules():\r\n # if type(m) is Bottleneck:\r\n # print('%10.3g' % (m.w.detach().sigmoid() * 2)) # shortcut weights\r\n\r\n def fuse(self): # fuse model Conv2d() + BatchNorm2d() layers\r\n print('Fusing layers... ')\r\n for m in self.model.modules():\r\n if isinstance(m, RepConv):\r\n #print(f\" fuse_repvgg_block\")\r\n m.fuse_repvgg_block()\r\n elif isinstance(m, RepConv_OREPA):\r\n #print(f\" switch_to_deploy\")\r\n m.switch_to_deploy()\r\n elif type(m) is Conv and hasattr(m, 'bn'):\r\n m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv\r\n delattr(m, 'bn') # remove batchnorm\r\n m.forward = m.fuseforward # update forward\r\n elif isinstance(m, IDetect):\r\n m.fuse()\r\n m.forward = m.fuseforward\r\n self.info()\r\n return self\r\n\r\n def nms(self, mode=True): # add or remove NMS module\r\n present = type(self.model[-1]) is NMS # last layer is NMS\r\n if mode and not present:\r\n print('Adding NMS... ')\r\n m = NMS() # module\r\n m.f = -1 # from\r\n m.i = self.model[-1].i + 1 # index\r\n self.model.add_module(name='%s' % m.i, module=m) # add\r\n self.eval()\r\n elif not mode and present:\r\n print('Removing NMS... ')\r\n self.model = self.model[:-1] # remove\r\n return self\r\n\r\n def autoshape(self): # add autoShape module\r\n print('Adding autoShape... ')\r\n m = autoShape(self) # wrap model\r\n copy_attr(m, self, include=('yaml', 'nc', 'hyp', 'names', 'stride'), exclude=()) # copy attributes\r\n return m\r\n\r\n def info(self, verbose=False, img_size=640): # print model information\r\n model_info(self, verbose, img_size)\r\n\r\n\r\ndef parse_model(d, ch): # model_dict, input_channels(3)\r\n logger.info('\\n%3s%18s%3s%10s %-40s%-30s' % ('', 'from', 'n', 'params', 'module', 'arguments'))\r\n anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple']\r\n na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors\r\n no = na * (nc + 5) # number of outputs = anchors * (classes + 5)\r\n\r\n layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out\r\n for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args\r\n m = eval(m) if isinstance(m, str) else m # eval strings\r\n for j, a in enumerate(args):\r\n try:\r\n args[j] = eval(a) if isinstance(a, str) else a # eval strings\r\n except:\r\n pass\r\n\r\n n = max(round(n * gd), 1) if n > 1 else n # depth gain\r\n if m in [nn.Conv2d, Conv, RobustConv, RobustConv2, DWConv, GhostConv, RepConv, RepConv_OREPA, DownC, \r\n SPP, SPPF, SPPCSPC, GhostSPPCSPC, MixConv2d, Focus, Stem, GhostStem, CrossConv, \r\n Bottleneck, BottleneckCSPA, BottleneckCSPB, BottleneckCSPC, \r\n RepBottleneck, RepBottleneckCSPA, RepBottleneckCSPB, RepBottleneckCSPC, \r\n Res, ResCSPA, ResCSPB, ResCSPC, \r\n RepRes, RepResCSPA, RepResCSPB, RepResCSPC, \r\n ResX, ResXCSPA, ResXCSPB, ResXCSPC, \r\n RepResX, RepResXCSPA, RepResXCSPB, RepResXCSPC, \r\n Ghost, GhostCSPA, GhostCSPB, GhostCSPC,\r\n SwinTransformerBlock, STCSPA, STCSPB, STCSPC,\r\n SwinTransformer2Block, ST2CSPA, ST2CSPB, ST2CSPC]:\r\n c1, c2 = ch[f], args[0]\r\n if c2 != no: # if not output\r\n c2 = make_divisible(c2 * gw, 8)\r\n\r\n args = [c1, c2, *args[1:]]\r\n if m in [DownC, SPPCSPC, GhostSPPCSPC, \r\n BottleneckCSPA, BottleneckCSPB, BottleneckCSPC, \r\n RepBottleneckCSPA, RepBottleneckCSPB, RepBottleneckCSPC, \r\n ResCSPA, ResCSPB, ResCSPC, \r\n RepResCSPA, RepResCSPB, RepResCSPC, \r\n ResXCSPA, ResXCSPB, ResXCSPC, \r\n RepResXCSPA, RepResXCSPB, RepResXCSPC,\r\n GhostCSPA, GhostCSPB, GhostCSPC,\r\n STCSPA, STCSPB, STCSPC,\r\n ST2CSPA, ST2CSPB, ST2CSPC]:\r\n args.insert(2, n) # number of repeats\r\n n = 1\r\n elif m is nn.BatchNorm2d:\r\n args = [ch[f]]\r\n elif m is Concat:\r\n c2 = sum([ch[x] for x in f])\r\n elif m is Chuncat:\r\n c2 = sum([ch[x] for x in f])\r\n elif m is Shortcut:\r\n c2 = ch[f[0]]\r\n elif m is Foldcut:\r\n c2 = ch[f] // 2\r\n elif m in [Detect, IDetect, IAuxDetect, IBin, IKeypoint]:\r\n args.append([ch[x] for x in f])\r\n if isinstance(args[1], int): # number of anchors\r\n args[1] = [list(range(args[1] * 2))] * len(f)\r\n elif m is ReOrg:\r\n c2 = ch[f] * 4\r\n elif m is Contract:\r\n c2 = ch[f] * args[0] ** 2\r\n elif m is Expand:\r\n c2 = ch[f] // args[0] ** 2\r\n else:\r\n c2 = ch[f]\r\n\r\n m_ = nn.Sequential(*[m(*args) for _ in range(n)]) if n > 1 else m(*args) # module\r\n t = str(m)[8:-2].replace('__main__.', '') # module type\r\n np = sum([x.numel() for x in m_.parameters()]) # number params\r\n m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params\r\n logger.info('%3s%18s%3s%10.0f %-40s%-30s' % (i, f, n, np, t, args)) # print\r\n save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist\r\n layers.append(m_)\r\n if i == 0:\r\n ch = []\r\n ch.append(c2)\r\n return nn.Sequential(*layers), sorted(save)\r\n\r\n\r\nif __name__ == '__main__':\r\n parser = argparse.ArgumentParser()\r\n parser.add_argument('--cfg', type=str, default='yolor-csp-c.yaml', help='model.yaml')\r\n parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu')\r\n parser.add_argument('--profile', action='store_true', help='profile model speed')\r\n opt = parser.parse_args()\r\n opt.cfg = check_file(opt.cfg) # check file\r\n set_logging()\r\n device = select_device(opt.device)\r\n\r\n # Create model\r\n model = Model(opt.cfg).to(device)\r\n model.train()\r\n \r\n if opt.profile:\r\n img = torch.rand(1, 3, 640, 640).to(device)\r\n y = model(img, profile=True)\r\n\r\n # Profile\r\n # img = torch.rand(8 if torch.cuda.is_available() else 1, 3, 640, 640).to(device)\r\n # y = model(img, profile=True)\r\n\r\n # Tensorboard\r\n # from torch.utils.tensorboard import SummaryWriter\r\n # tb_writer = SummaryWriter()\r\n # print(\"Run 'tensorboard --logdir=models/runs' to view tensorboard at http://localhost:6006/\")\r\n # tb_writer.add_graph(model.model, img) # add model to tensorboard\r\n # tb_writer.add_image('test', img[0], dataformats='CWH') # add model to tensorboard\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/requirements.txt", "content": "# Usage: pip install -r requirements.txt\n\n# Base ----------------------------------------\nmatplotlib>=3.2.2\nnumpy>=1.18.5\nopencv-python>=4.1.1\nPillow>=7.1.2\nPyYAML>=5.3.1\nrequests>=2.23.0\nscipy>=1.4.1\ntorch>=1.7.0,!=1.12.0\ntorchvision>=0.8.1,!=0.13.0\ntqdm>=4.41.0\nprotobuf<4.21.3\n\n# Logging -------------------------------------\ntensorboard>=2.4.1\n# wandb\n\n# Plotting ------------------------------------\npandas>=1.1.4\nseaborn>=0.11.0\n\n\n# Extras --------------------------------------\nipython # interactive notebook\npsutil # system utilization\nthop # FLOPs computation\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/__init__.py", "content": "# init" }, { "path": "asone/pose_estimators/yolov7_pose/utils/activations.py", "content": "# Activation functions\r\n\r\nimport torch\r\nimport torch.nn as nn\r\nimport torch.nn.functional as F\r\n\r\n\r\n# SiLU https://arxiv.org/pdf/1606.08415.pdf ----------------------------------------------------------------------------\r\nclass SiLU(nn.Module): # export-friendly version of nn.SiLU()\r\n @staticmethod\r\n def forward(x):\r\n return x * torch.sigmoid(x)\r\n\r\n\r\nclass Hardswish(nn.Module): # export-friendly version of nn.Hardswish()\r\n @staticmethod\r\n def forward(x):\r\n # return x * F.hardsigmoid(x) # for torchscript and CoreML\r\n return x * F.hardtanh(x + 3, 0., 6.) / 6. # for torchscript, CoreML and ONNX\r\n\r\n\r\nclass MemoryEfficientSwish(nn.Module):\r\n class F(torch.autograd.Function):\r\n @staticmethod\r\n def forward(ctx, x):\r\n ctx.save_for_backward(x)\r\n return x * torch.sigmoid(x)\r\n\r\n @staticmethod\r\n def backward(ctx, grad_output):\r\n x = ctx.saved_tensors[0]\r\n sx = torch.sigmoid(x)\r\n return grad_output * (sx * (1 + x * (1 - sx)))\r\n\r\n def forward(self, x):\r\n return self.F.apply(x)\r\n\r\n\r\n# Mish https://github.com/digantamisra98/Mish --------------------------------------------------------------------------\r\nclass Mish(nn.Module):\r\n @staticmethod\r\n def forward(x):\r\n return x * F.softplus(x).tanh()\r\n\r\n\r\nclass MemoryEfficientMish(nn.Module):\r\n class F(torch.autograd.Function):\r\n @staticmethod\r\n def forward(ctx, x):\r\n ctx.save_for_backward(x)\r\n return x.mul(torch.tanh(F.softplus(x))) # x * tanh(ln(1 + exp(x)))\r\n\r\n @staticmethod\r\n def backward(ctx, grad_output):\r\n x = ctx.saved_tensors[0]\r\n sx = torch.sigmoid(x)\r\n fx = F.softplus(x).tanh()\r\n return grad_output * (fx + x * sx * (1 - fx * fx))\r\n\r\n def forward(self, x):\r\n return self.F.apply(x)\r\n\r\n\r\n# FReLU https://arxiv.org/abs/2007.11824 -------------------------------------------------------------------------------\r\nclass FReLU(nn.Module):\r\n def __init__(self, c1, k=3): # ch_in, kernel\r\n super().__init__()\r\n self.conv = nn.Conv2d(c1, c1, k, 1, 1, groups=c1, bias=False)\r\n self.bn = nn.BatchNorm2d(c1)\r\n\r\n def forward(self, x):\r\n return torch.max(x, self.bn(self.conv(x)))\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/add_nms.py", "content": "import numpy as np\r\nimport onnx\r\nfrom onnx import shape_inference\r\ntry:\r\n import onnx_graphsurgeon as gs\r\nexcept Exception as e:\r\n print('Import onnx_graphsurgeon failure: %s' % e)\r\n\r\nimport logging\r\n\r\nLOGGER = logging.getLogger(__name__)\r\n\r\nclass RegisterNMS(object):\r\n def __init__(\r\n self,\r\n onnx_model_path: str,\r\n precision: str = \"fp32\",\r\n ):\r\n\r\n self.graph = gs.import_onnx(onnx.load(onnx_model_path))\r\n assert self.graph\r\n LOGGER.info(\"ONNX graph created successfully\")\r\n # Fold constants via ONNX-GS that PyTorch2ONNX may have missed\r\n self.graph.fold_constants()\r\n self.precision = precision\r\n self.batch_size = 1\r\n def infer(self):\r\n \"\"\"\r\n Sanitize the graph by cleaning any unconnected nodes, do a topological resort,\r\n and fold constant inputs values. When possible, run shape inference on the\r\n ONNX graph to determine tensor shapes.\r\n \"\"\"\r\n for _ in range(3):\r\n count_before = len(self.graph.nodes)\r\n\r\n self.graph.cleanup().toposort()\r\n try:\r\n for node in self.graph.nodes:\r\n for o in node.outputs:\r\n o.shape = None\r\n model = gs.export_onnx(self.graph)\r\n model = shape_inference.infer_shapes(model)\r\n self.graph = gs.import_onnx(model)\r\n except Exception as e:\r\n LOGGER.info(f\"Shape inference could not be performed at this time:\\n{e}\")\r\n try:\r\n self.graph.fold_constants(fold_shapes=True)\r\n except TypeError as e:\r\n LOGGER.error(\r\n \"This version of ONNX GraphSurgeon does not support folding shapes, \"\r\n f\"please upgrade your onnx_graphsurgeon module. Error:\\n{e}\"\r\n )\r\n raise\r\n\r\n count_after = len(self.graph.nodes)\r\n if count_before == count_after:\r\n # No new folding occurred in this iteration, so we can stop for now.\r\n break\r\n\r\n def save(self, output_path):\r\n \"\"\"\r\n Save the ONNX model to the given location.\r\n Args:\r\n output_path: Path pointing to the location where to write\r\n out the updated ONNX model.\r\n \"\"\"\r\n self.graph.cleanup().toposort()\r\n model = gs.export_onnx(self.graph)\r\n onnx.save(model, output_path)\r\n LOGGER.info(f\"Saved ONNX model to {output_path}\")\r\n\r\n def register_nms(\r\n self,\r\n *,\r\n score_thresh: float = 0.25,\r\n nms_thresh: float = 0.45,\r\n detections_per_img: int = 100,\r\n ):\r\n \"\"\"\r\n Register the ``EfficientNMS_TRT`` plugin node.\r\n NMS expects these shapes for its input tensors:\r\n - box_net: [batch_size, number_boxes, 4]\r\n - class_net: [batch_size, number_boxes, number_labels]\r\n Args:\r\n score_thresh (float): The scalar threshold for score (low scoring boxes are removed).\r\n nms_thresh (float): The scalar threshold for IOU (new boxes that have high IOU\r\n overlap with previously selected boxes are removed).\r\n detections_per_img (int): Number of best detections to keep after NMS.\r\n \"\"\"\r\n\r\n self.infer()\r\n # Find the concat node at the end of the network\r\n op_inputs = self.graph.outputs\r\n op = \"EfficientNMS_TRT\"\r\n attrs = {\r\n \"plugin_version\": \"1\",\r\n \"background_class\": -1, # no background class\r\n \"max_output_boxes\": detections_per_img,\r\n \"score_threshold\": score_thresh,\r\n \"iou_threshold\": nms_thresh,\r\n \"score_activation\": False,\r\n \"box_coding\": 0,\r\n }\r\n\r\n if self.precision == \"fp32\":\r\n dtype_output = np.float32\r\n elif self.precision == \"fp16\":\r\n dtype_output = np.float16\r\n else:\r\n raise NotImplementedError(f\"Currently not supports precision: {self.precision}\")\r\n\r\n # NMS Outputs\r\n output_num_detections = gs.Variable(\r\n name=\"num_detections\",\r\n dtype=np.int32,\r\n shape=[self.batch_size, 1],\r\n ) # A scalar indicating the number of valid detections per batch image.\r\n output_boxes = gs.Variable(\r\n name=\"detection_boxes\",\r\n dtype=dtype_output,\r\n shape=[self.batch_size, detections_per_img, 4],\r\n )\r\n output_scores = gs.Variable(\r\n name=\"detection_scores\",\r\n dtype=dtype_output,\r\n shape=[self.batch_size, detections_per_img],\r\n )\r\n output_labels = gs.Variable(\r\n name=\"detection_classes\",\r\n dtype=np.int32,\r\n shape=[self.batch_size, detections_per_img],\r\n )\r\n\r\n op_outputs = [output_num_detections, output_boxes, output_scores, output_labels]\r\n\r\n # Create the NMS Plugin node with the selected inputs. The outputs of the node will also\r\n # become the final outputs of the graph.\r\n self.graph.layer(op=op, name=\"batched_nms\", inputs=op_inputs, outputs=op_outputs, attrs=attrs)\r\n LOGGER.info(f\"Created NMS plugin '{op}' with attributes: {attrs}\")\r\n\r\n self.graph.outputs = op_outputs\r\n\r\n self.infer()\r\n\r\n def save(self, output_path):\r\n \"\"\"\r\n Save the ONNX model to the given location.\r\n Args:\r\n output_path: Path pointing to the location where to write\r\n out the updated ONNX model.\r\n \"\"\"\r\n self.graph.cleanup().toposort()\r\n model = gs.export_onnx(self.graph)\r\n onnx.save(model, output_path)\r\n LOGGER.info(f\"Saved ONNX model to {output_path}\")\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/autoanchor.py", "content": "# Auto-anchor utils\r\n\r\nimport numpy as np\r\nimport torch\r\nimport yaml\r\nfrom scipy.cluster.vq import kmeans\r\nfrom tqdm import tqdm\r\n\r\nfrom asone.pose_estimators.yolov7_pose.utils.general import colorstr\r\n\r\n\r\ndef check_anchor_order(m):\r\n # Check anchor order against stride order for YOLO Detect() module m, and correct if necessary\r\n a = m.anchor_grid.prod(-1).view(-1) # anchor area\r\n da = a[-1] - a[0] # delta a\r\n ds = m.stride[-1] - m.stride[0] # delta s\r\n if da.sign() != ds.sign(): # same order\r\n print('Reversing anchor order')\r\n m.anchors[:] = m.anchors.flip(0)\r\n m.anchor_grid[:] = m.anchor_grid.flip(0)\r\n\r\n\r\ndef check_anchors(dataset, model, thr=4.0, imgsz=640):\r\n # Check anchor fit to data, recompute if necessary\r\n prefix = colorstr('autoanchor: ')\r\n print(f'\\n{prefix}Analyzing anchors... ', end='')\r\n m = model.module.model[-1] if hasattr(model, 'module') else model.model[-1] # Detect()\r\n shapes = imgsz * dataset.shapes / dataset.shapes.max(1, keepdims=True)\r\n scale = np.random.uniform(0.9, 1.1, size=(shapes.shape[0], 1)) # augment scale\r\n wh = torch.tensor(np.concatenate([l[:, 3:5] * s for s, l in zip(shapes * scale, dataset.labels)])).float() # wh\r\n\r\n def metric(k): # compute metric\r\n r = wh[:, None] / k[None]\r\n x = torch.min(r, 1. / r).min(2)[0] # ratio metric\r\n best = x.max(1)[0] # best_x\r\n aat = (x > 1. / thr).float().sum(1).mean() # anchors above threshold\r\n bpr = (best > 1. / thr).float().mean() # best possible recall\r\n return bpr, aat\r\n\r\n anchors = m.anchor_grid.clone().cpu().view(-1, 2) # current anchors\r\n bpr, aat = metric(anchors)\r\n print(f'anchors/target = {aat:.2f}, Best Possible Recall (BPR) = {bpr:.4f}', end='')\r\n if bpr < 0.98: # threshold to recompute\r\n print('. Attempting to improve anchors, please wait...')\r\n na = m.anchor_grid.numel() // 2 # number of anchors\r\n try:\r\n anchors = kmean_anchors(dataset, n=na, img_size=imgsz, thr=thr, gen=1000, verbose=False)\r\n except Exception as e:\r\n print(f'{prefix}ERROR: {e}')\r\n new_bpr = metric(anchors)[0]\r\n if new_bpr > bpr: # replace anchors\r\n anchors = torch.tensor(anchors, device=m.anchors.device).type_as(m.anchors)\r\n m.anchor_grid[:] = anchors.clone().view_as(m.anchor_grid) # for inference\r\n m.anchors[:] = anchors.clone().view_as(m.anchors) / m.stride.to(m.anchors.device).view(-1, 1, 1) # loss\r\n check_anchor_order(m)\r\n print(f'{prefix}New anchors saved to model. Update model *.yaml to use these anchors in the future.')\r\n else:\r\n print(f'{prefix}Original anchors better than new anchors. Proceeding with original anchors.')\r\n print('') # newline\r\n\r\n\r\ndef kmean_anchors(path='./data/coco.yaml', n=9, img_size=640, thr=4.0, gen=1000, verbose=True):\r\n \"\"\" Creates kmeans-evolved anchors from training dataset\r\n\r\n Arguments:\r\n path: path to dataset *.yaml, or a loaded dataset\r\n n: number of anchors\r\n img_size: image size used for training\r\n thr: anchor-label wh ratio threshold hyperparameter hyp['anchor_t'] used for training, default=4.0\r\n gen: generations to evolve anchors using genetic algorithm\r\n verbose: print all results\r\n\r\n Return:\r\n k: kmeans evolved anchors\r\n\r\n Usage:\r\n from utils.autoanchor import *; _ = kmean_anchors()\r\n \"\"\"\r\n thr = 1. / thr\r\n prefix = colorstr('autoanchor: ')\r\n\r\n def metric(k, wh): # compute metrics\r\n r = wh[:, None] / k[None]\r\n x = torch.min(r, 1. / r).min(2)[0] # ratio metric\r\n # x = wh_iou(wh, torch.tensor(k)) # iou metric\r\n return x, x.max(1)[0] # x, best_x\r\n\r\n def anchor_fitness(k): # mutation fitness\r\n _, best = metric(torch.tensor(k, dtype=torch.float32), wh)\r\n return (best * (best > thr).float()).mean() # fitness\r\n\r\n def print_results(k):\r\n k = k[np.argsort(k.prod(1))] # sort small to large\r\n x, best = metric(k, wh0)\r\n bpr, aat = (best > thr).float().mean(), (x > thr).float().mean() * n # best possible recall, anch > thr\r\n print(f'{prefix}thr={thr:.2f}: {bpr:.4f} best possible recall, {aat:.2f} anchors past thr')\r\n print(f'{prefix}n={n}, img_size={img_size}, metric_all={x.mean():.3f}/{best.mean():.3f}-mean/best, '\r\n f'past_thr={x[x > thr].mean():.3f}-mean: ', end='')\r\n for i, x in enumerate(k):\r\n print('%i,%i' % (round(x[0]), round(x[1])), end=', ' if i < len(k) - 1 else '\\n') # use in *.cfg\r\n return k\r\n\r\n if isinstance(path, str): # *.yaml file\r\n with open(path) as f:\r\n data_dict = yaml.load(f, Loader=yaml.SafeLoader) # model dict\r\n from asone.pose_estimators.yolov7_pose.utils.datasets import LoadImagesAndLabels\r\n dataset = LoadImagesAndLabels(data_dict['train'], augment=True, rect=True)\r\n else:\r\n dataset = path # dataset\r\n\r\n # Get label wh\r\n shapes = img_size * dataset.shapes / dataset.shapes.max(1, keepdims=True)\r\n wh0 = np.concatenate([l[:, 3:5] * s for s, l in zip(shapes, dataset.labels)]) # wh\r\n\r\n # Filter\r\n i = (wh0 < 3.0).any(1).sum()\r\n if i:\r\n print(f'{prefix}WARNING: Extremely small objects found. {i} of {len(wh0)} labels are < 3 pixels in size.')\r\n wh = wh0[(wh0 >= 2.0).any(1)] # filter > 2 pixels\r\n # wh = wh * (np.random.rand(wh.shape[0], 1) * 0.9 + 0.1) # multiply by random scale 0-1\r\n\r\n # Kmeans calculation\r\n print(f'{prefix}Running kmeans for {n} anchors on {len(wh)} points...')\r\n s = wh.std(0) # sigmas for whitening\r\n k, dist = kmeans(wh / s, n, iter=30) # points, mean distance\r\n assert len(k) == n, print(f'{prefix}ERROR: scipy.cluster.vq.kmeans requested {n} points but returned only {len(k)}')\r\n k *= s\r\n wh = torch.tensor(wh, dtype=torch.float32) # filtered\r\n wh0 = torch.tensor(wh0, dtype=torch.float32) # unfiltered\r\n k = print_results(k)\r\n\r\n # Plot\r\n # k, d = [None] * 20, [None] * 20\r\n # for i in tqdm(range(1, 21)):\r\n # k[i-1], d[i-1] = kmeans(wh / s, i) # points, mean distance\r\n # fig, ax = plt.subplots(1, 2, figsize=(14, 7), tight_layout=True)\r\n # ax = ax.ravel()\r\n # ax[0].plot(np.arange(1, 21), np.array(d) ** 2, marker='.')\r\n # fig, ax = plt.subplots(1, 2, figsize=(14, 7)) # plot wh\r\n # ax[0].hist(wh[wh[:, 0]<100, 0],400)\r\n # ax[1].hist(wh[wh[:, 1]<100, 1],400)\r\n # fig.savefig('wh.png', dpi=200)\r\n\r\n # Evolve\r\n npr = np.random\r\n f, sh, mp, s = anchor_fitness(k), k.shape, 0.9, 0.1 # fitness, generations, mutation prob, sigma\r\n pbar = tqdm(range(gen), desc=f'{prefix}Evolving anchors with Genetic Algorithm:') # progress bar\r\n for _ in pbar:\r\n v = np.ones(sh)\r\n while (v == 1).all(): # mutate until a change occurs (prevent duplicates)\r\n v = ((npr.random(sh) < mp) * npr.random() * npr.randn(*sh) * s + 1).clip(0.3, 3.0)\r\n kg = (k.copy() * v).clip(min=2.0)\r\n fg = anchor_fitness(kg)\r\n if fg > f:\r\n f, k = fg, kg.copy()\r\n pbar.desc = f'{prefix}Evolving anchors with Genetic Algorithm: fitness = {f:.4f}'\r\n if verbose:\r\n print_results(k)\r\n\r\n return print_results(k)\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/datasets.py", "content": "# Dataset utils and dataloaders\r\n\r\nimport glob\r\nimport logging\r\nimport math\r\nimport os\r\nimport random\r\nimport shutil\r\nimport time\r\nfrom itertools import repeat\r\nfrom multiprocessing.pool import ThreadPool\r\nfrom pathlib import Path\r\nfrom threading import Thread\r\n\r\nimport cv2\r\nimport numpy as np\r\nimport torch\r\nimport torch.nn.functional as F\r\nfrom PIL import Image, ExifTags\r\nfrom torch.utils.data import Dataset\r\nfrom tqdm import tqdm\r\n\r\nimport pickle\r\nfrom copy import deepcopy\r\n#from pycocotools import mask as maskUtils\r\nfrom torchvision.utils import save_image\r\nfrom torchvision.ops import roi_pool, roi_align, ps_roi_pool, ps_roi_align\r\n\r\nfrom asone.pose_estimators.yolov7_pose.utils.general import check_requirements, xyxy2xywh, xywh2xyxy, xywhn2xyxy, xyn2xy, segment2box, segments2boxes, \\\r\n resample_segments, clean_str\r\nfrom asone.pose_estimators.yolov7_pose.utils.torch_utils import torch_distributed_zero_first\r\n\r\n# Parameters\r\nhelp_url = 'https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data'\r\nimg_formats = ['bmp', 'jpg', 'jpeg', 'png', 'tif', 'tiff', 'dng', 'webp', 'mpo'] # acceptable image suffixes\r\nvid_formats = ['mov', 'avi', 'mp4', 'mpg', 'mpeg', 'm4v', 'wmv', 'mkv'] # acceptable video suffixes\r\nlogger = logging.getLogger(__name__)\r\n\r\n# Get orientation exif tag\r\nfor orientation in ExifTags.TAGS.keys():\r\n if ExifTags.TAGS[orientation] == 'Orientation':\r\n break\r\n\r\n\r\ndef get_hash(files):\r\n # Returns a single hash value of a list of files\r\n return sum(os.path.getsize(f) for f in files if os.path.isfile(f))\r\n\r\n\r\ndef exif_size(img):\r\n # Returns exif-corrected PIL size\r\n s = img.size # (width, height)\r\n try:\r\n rotation = dict(img._getexif().items())[orientation]\r\n if rotation == 6: # rotation 270\r\n s = (s[1], s[0])\r\n elif rotation == 8: # rotation 90\r\n s = (s[1], s[0])\r\n except:\r\n pass\r\n\r\n return s\r\n\r\n\r\ndef create_dataloader(path, imgsz, batch_size, stride, opt, hyp=None, augment=False, cache=False, pad=0.0, rect=False,\r\n rank=-1, world_size=1, workers=8, image_weights=False, quad=False, prefix=''):\r\n # Make sure only the first process in DDP process the dataset first, and the following others can use the cache\r\n with torch_distributed_zero_first(rank):\r\n dataset = LoadImagesAndLabels(path, imgsz, batch_size,\r\n augment=augment, # augment images\r\n hyp=hyp, # augmentation hyperparameters\r\n rect=rect, # rectangular training\r\n cache_images=cache,\r\n single_cls=opt.single_cls,\r\n stride=int(stride),\r\n pad=pad,\r\n image_weights=image_weights,\r\n prefix=prefix)\r\n\r\n batch_size = min(batch_size, len(dataset))\r\n nw = min([os.cpu_count() // world_size, batch_size if batch_size > 1 else 0, workers]) # number of workers\r\n sampler = torch.utils.data.distributed.DistributedSampler(dataset) if rank != -1 else None\r\n loader = torch.utils.data.DataLoader if image_weights else InfiniteDataLoader\r\n # Use torch.utils.data.DataLoader() if dataset.properties will update during training else InfiniteDataLoader()\r\n dataloader = loader(dataset,\r\n batch_size=batch_size,\r\n num_workers=nw,\r\n sampler=sampler,\r\n pin_memory=True,\r\n collate_fn=LoadImagesAndLabels.collate_fn4 if quad else LoadImagesAndLabels.collate_fn)\r\n return dataloader, dataset\r\n\r\n\r\nclass InfiniteDataLoader(torch.utils.data.dataloader.DataLoader):\r\n \"\"\" Dataloader that reuses workers\r\n\r\n Uses same syntax as vanilla DataLoader\r\n \"\"\"\r\n\r\n def __init__(self, *args, **kwargs):\r\n super().__init__(*args, **kwargs)\r\n object.__setattr__(self, 'batch_sampler', _RepeatSampler(self.batch_sampler))\r\n self.iterator = super().__iter__()\r\n\r\n def __len__(self):\r\n return len(self.batch_sampler.sampler)\r\n\r\n def __iter__(self):\r\n for i in range(len(self)):\r\n yield next(self.iterator)\r\n\r\n\r\nclass _RepeatSampler(object):\r\n \"\"\" Sampler that repeats forever\r\n\r\n Args:\r\n sampler (Sampler)\r\n \"\"\"\r\n\r\n def __init__(self, sampler):\r\n self.sampler = sampler\r\n\r\n def __iter__(self):\r\n while True:\r\n yield from iter(self.sampler)\r\n\r\n\r\nclass LoadImages: # for inference\r\n def __init__(self, path, img_size=640, stride=32):\r\n p = str(Path(path).absolute()) # os-agnostic absolute path\r\n if '*' in p:\r\n files = sorted(glob.glob(p, recursive=True)) # glob\r\n elif os.path.isdir(p):\r\n files = sorted(glob.glob(os.path.join(p, '*.*'))) # dir\r\n elif os.path.isfile(p):\r\n files = [p] # files\r\n else:\r\n raise Exception(f'ERROR: {p} does not exist')\r\n\r\n images = [x for x in files if x.split('.')[-1].lower() in img_formats]\r\n videos = [x for x in files if x.split('.')[-1].lower() in vid_formats]\r\n ni, nv = len(images), len(videos)\r\n\r\n self.img_size = img_size\r\n self.stride = stride\r\n self.files = images + videos\r\n self.nf = ni + nv # number of files\r\n self.video_flag = [False] * ni + [True] * nv\r\n self.mode = 'image'\r\n if any(videos):\r\n self.new_video(videos[0]) # new video\r\n else:\r\n self.cap = None\r\n assert self.nf > 0, f'No images or videos found in {p}. ' \\\r\n f'Supported formats are:\\nimages: {img_formats}\\nvideos: {vid_formats}'\r\n\r\n def __iter__(self):\r\n self.count = 0\r\n return self\r\n\r\n def __next__(self):\r\n if self.count == self.nf:\r\n raise StopIteration\r\n path = self.files[self.count]\r\n\r\n if self.video_flag[self.count]:\r\n # Read video\r\n self.mode = 'video'\r\n ret_val, img0 = self.cap.read()\r\n if not ret_val:\r\n self.count += 1\r\n self.cap.release()\r\n if self.count == self.nf: # last video\r\n raise StopIteration\r\n else:\r\n path = self.files[self.count]\r\n self.new_video(path)\r\n ret_val, img0 = self.cap.read()\r\n\r\n self.frame += 1\r\n print(f'video {self.count + 1}/{self.nf} ({self.frame}/{self.nframes}) {path}: ', end='')\r\n\r\n else:\r\n # Read image\r\n self.count += 1\r\n img0 = cv2.imread(path) # BGR\r\n assert img0 is not None, 'Image Not Found ' + path\r\n #print(f'image {self.count}/{self.nf} {path}: ', end='')\r\n\r\n # Padded resize\r\n img = letterbox(img0, self.img_size, stride=self.stride)[0]\r\n\r\n # Convert\r\n img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\r\n img = np.ascontiguousarray(img)\r\n\r\n return path, img, img0, self.cap\r\n\r\n def new_video(self, path):\r\n self.frame = 0\r\n self.cap = cv2.VideoCapture(path)\r\n self.nframes = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT))\r\n\r\n def __len__(self):\r\n return self.nf # number of files\r\n\r\n\r\nclass LoadWebcam: # for inference\r\n def __init__(self, pipe='0', img_size=640, stride=32):\r\n self.img_size = img_size\r\n self.stride = stride\r\n\r\n if pipe.isnumeric():\r\n pipe = eval(pipe) # local camera\r\n # pipe = 'rtsp://192.168.1.64/1' # IP camera\r\n # pipe = 'rtsp://username:password@192.168.1.64/1' # IP camera with login\r\n # pipe = 'http://wmccpinetop.axiscam.net/mjpg/video.mjpg' # IP golf camera\r\n\r\n self.pipe = pipe\r\n self.cap = cv2.VideoCapture(pipe) # video capture object\r\n self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 3) # set buffer size\r\n\r\n def __iter__(self):\r\n self.count = -1\r\n return self\r\n\r\n def __next__(self):\r\n self.count += 1\r\n if cv2.waitKey(1) == ord('q'): # q to quit\r\n self.cap.release()\r\n cv2.destroyAllWindows()\r\n raise StopIteration\r\n\r\n # Read frame\r\n if self.pipe == 0: # local camera\r\n ret_val, img0 = self.cap.read()\r\n img0 = cv2.flip(img0, 1) # flip left-right\r\n else: # IP camera\r\n n = 0\r\n while True:\r\n n += 1\r\n self.cap.grab()\r\n if n % 30 == 0: # skip frames\r\n ret_val, img0 = self.cap.retrieve()\r\n if ret_val:\r\n break\r\n\r\n # Print\r\n assert ret_val, f'Camera Error {self.pipe}'\r\n img_path = 'webcam.jpg'\r\n print(f'webcam {self.count}: ', end='')\r\n\r\n # Padded resize\r\n img = letterbox(img0, self.img_size, stride=self.stride)[0]\r\n\r\n # Convert\r\n img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\r\n img = np.ascontiguousarray(img)\r\n\r\n return img_path, img, img0, None\r\n\r\n def __len__(self):\r\n return 0\r\n\r\n\r\nclass LoadStreams: # multiple IP or RTSP cameras\r\n def __init__(self, sources='streams.txt', img_size=640, stride=32):\r\n self.mode = 'stream'\r\n self.img_size = img_size\r\n self.stride = stride\r\n\r\n if os.path.isfile(sources):\r\n with open(sources, 'r') as f:\r\n sources = [x.strip() for x in f.read().strip().splitlines() if len(x.strip())]\r\n else:\r\n sources = [sources]\r\n\r\n n = len(sources)\r\n self.imgs = [None] * n\r\n self.sources = [clean_str(x) for x in sources] # clean source names for later\r\n for i, s in enumerate(sources):\r\n # Start the thread to read frames from the video stream\r\n print(f'{i + 1}/{n}: {s}... ', end='')\r\n url = eval(s) if s.isnumeric() else s\r\n if 'youtube.com/' in str(url) or 'youtu.be/' in str(url): # if source is YouTube video\r\n check_requirements(('pafy', 'youtube_dl'))\r\n import pafy\r\n url = pafy.new(url).getbest(preftype=\"mp4\").url\r\n cap = cv2.VideoCapture(url)\r\n assert cap.isOpened(), f'Failed to open {s}'\r\n w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\r\n h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\r\n self.fps = cap.get(cv2.CAP_PROP_FPS) % 100\r\n\r\n _, self.imgs[i] = cap.read() # guarantee first frame\r\n thread = Thread(target=self.update, args=([i, cap]), daemon=True)\r\n print(f' success ({w}x{h} at {self.fps:.2f} FPS).')\r\n thread.start()\r\n print('') # newline\r\n\r\n # check for common shapes\r\n s = np.stack([letterbox(x, self.img_size, stride=self.stride)[0].shape for x in self.imgs], 0) # shapes\r\n self.rect = np.unique(s, axis=0).shape[0] == 1 # rect inference if all shapes equal\r\n if not self.rect:\r\n print('WARNING: Different stream shapes detected. For optimal performance supply similarly-shaped streams.')\r\n\r\n def update(self, index, cap):\r\n # Read next stream frame in a daemon thread\r\n n = 0\r\n while cap.isOpened():\r\n n += 1\r\n # _, self.imgs[index] = cap.read()\r\n cap.grab()\r\n if n == 4: # read every 4th frame\r\n success, im = cap.retrieve()\r\n self.imgs[index] = im if success else self.imgs[index] * 0\r\n n = 0\r\n time.sleep(1 / self.fps) # wait time\r\n\r\n def __iter__(self):\r\n self.count = -1\r\n return self\r\n\r\n def __next__(self):\r\n self.count += 1\r\n img0 = self.imgs.copy()\r\n if cv2.waitKey(1) == ord('q'): # q to quit\r\n cv2.destroyAllWindows()\r\n raise StopIteration\r\n\r\n # Letterbox\r\n img = [letterbox(x, self.img_size, auto=self.rect, stride=self.stride)[0] for x in img0]\r\n\r\n # Stack\r\n img = np.stack(img, 0)\r\n\r\n # Convert\r\n img = img[:, :, :, ::-1].transpose(0, 3, 1, 2) # BGR to RGB, to bsx3x416x416\r\n img = np.ascontiguousarray(img)\r\n\r\n return self.sources, img, img0, None\r\n\r\n def __len__(self):\r\n return 0 # 1E12 frames = 32 streams at 30 FPS for 30 years\r\n\r\n\r\ndef img2label_paths(img_paths):\r\n # Define label paths as a function of image paths\r\n sa, sb = os.sep + 'images' + os.sep, os.sep + 'labels' + os.sep # /images/, /labels/ substrings\r\n return ['txt'.join(x.replace(sa, sb, 1).rsplit(x.split('.')[-1], 1)) for x in img_paths]\r\n\r\n\r\nclass LoadImagesAndLabels(Dataset): # for training/testing\r\n def __init__(self, path, img_size=640, batch_size=16, augment=False, hyp=None, rect=False, image_weights=False,\r\n cache_images=False, single_cls=False, stride=32, pad=0.0, prefix=''):\r\n self.img_size = img_size\r\n self.augment = augment\r\n self.hyp = hyp\r\n self.image_weights = image_weights\r\n self.rect = False if image_weights else rect\r\n self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training)\r\n self.mosaic_border = [-img_size // 2, -img_size // 2]\r\n self.stride = stride\r\n self.path = path \r\n #self.albumentations = Albumentations() if augment else None\r\n\r\n try:\r\n f = [] # image files\r\n for p in path if isinstance(path, list) else [path]:\r\n p = Path(p) # os-agnostic\r\n if p.is_dir(): # dir\r\n f += glob.glob(str(p / '**' / '*.*'), recursive=True)\r\n # f = list(p.rglob('**/*.*')) # pathlib\r\n elif p.is_file(): # file\r\n with open(p, 'r') as t:\r\n t = t.read().strip().splitlines()\r\n parent = str(p.parent) + os.sep\r\n f += [x.replace('./', parent) if x.startswith('./') else x for x in t] # local to global path\r\n # f += [p.parent / x.lstrip(os.sep) for x in t] # local to global path (pathlib)\r\n else:\r\n raise Exception(f'{prefix}{p} does not exist')\r\n self.img_files = sorted([x.replace('/', os.sep) for x in f if x.split('.')[-1].lower() in img_formats])\r\n # self.img_files = sorted([x for x in f if x.suffix[1:].lower() in img_formats]) # pathlib\r\n assert self.img_files, f'{prefix}No images found'\r\n except Exception as e:\r\n raise Exception(f'{prefix}Error loading data from {path}: {e}\\nSee {help_url}')\r\n\r\n # Check cache\r\n self.label_files = img2label_paths(self.img_files) # labels\r\n cache_path = (p if p.is_file() else Path(self.label_files[0]).parent).with_suffix('.cache') # cached labels\r\n if cache_path.is_file():\r\n cache, exists = torch.load(cache_path), True # load\r\n #if cache['hash'] != get_hash(self.label_files + self.img_files) or 'version' not in cache: # changed\r\n # cache, exists = self.cache_labels(cache_path, prefix), False # re-cache\r\n else:\r\n cache, exists = self.cache_labels(cache_path, prefix), False # cache\r\n\r\n # Display cache\r\n nf, nm, ne, nc, n = cache.pop('results') # found, missing, empty, corrupted, total\r\n if exists:\r\n d = f\"Scanning '{cache_path}' images and labels... {nf} found, {nm} missing, {ne} empty, {nc} corrupted\"\r\n tqdm(None, desc=prefix + d, total=n, initial=n) # display cache results\r\n assert nf > 0 or not augment, f'{prefix}No labels in {cache_path}. Can not train without labels. See {help_url}'\r\n\r\n # Read cache\r\n cache.pop('hash') # remove hash\r\n cache.pop('version') # remove version\r\n labels, shapes, self.segments = zip(*cache.values())\r\n self.labels = list(labels)\r\n self.shapes = np.array(shapes, dtype=np.float64)\r\n self.img_files = list(cache.keys()) # update\r\n self.label_files = img2label_paths(cache.keys()) # update\r\n if single_cls:\r\n for x in self.labels:\r\n x[:, 0] = 0\r\n\r\n n = len(shapes) # number of images\r\n bi = np.floor(np.arange(n) / batch_size).astype(np.int) # batch index\r\n nb = bi[-1] + 1 # number of batches\r\n self.batch = bi # batch index of image\r\n self.n = n\r\n self.indices = range(n)\r\n\r\n # Rectangular Training\r\n if self.rect:\r\n # Sort by aspect ratio\r\n s = self.shapes # wh\r\n ar = s[:, 1] / s[:, 0] # aspect ratio\r\n irect = ar.argsort()\r\n self.img_files = [self.img_files[i] for i in irect]\r\n self.label_files = [self.label_files[i] for i in irect]\r\n self.labels = [self.labels[i] for i in irect]\r\n self.shapes = s[irect] # wh\r\n ar = ar[irect]\r\n\r\n # Set training image shapes\r\n shapes = [[1, 1]] * nb\r\n for i in range(nb):\r\n ari = ar[bi == i]\r\n mini, maxi = ari.min(), ari.max()\r\n if maxi < 1:\r\n shapes[i] = [maxi, 1]\r\n elif mini > 1:\r\n shapes[i] = [1, 1 / mini]\r\n\r\n self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(np.int) * stride\r\n\r\n # Cache images into memory for faster training (WARNING: large datasets may exceed system RAM)\r\n self.imgs = [None] * n\r\n if cache_images:\r\n if cache_images == 'disk':\r\n self.im_cache_dir = Path(Path(self.img_files[0]).parent.as_posix() + '_npy')\r\n self.img_npy = [self.im_cache_dir / Path(f).with_suffix('.npy').name for f in self.img_files]\r\n self.im_cache_dir.mkdir(parents=True, exist_ok=True)\r\n gb = 0 # Gigabytes of cached images\r\n self.img_hw0, self.img_hw = [None] * n, [None] * n\r\n results = ThreadPool(8).imap(lambda x: load_image(*x), zip(repeat(self), range(n)))\r\n pbar = tqdm(enumerate(results), total=n)\r\n for i, x in pbar:\r\n if cache_images == 'disk':\r\n if not self.img_npy[i].exists():\r\n np.save(self.img_npy[i].as_posix(), x[0])\r\n gb += self.img_npy[i].stat().st_size\r\n else:\r\n self.imgs[i], self.img_hw0[i], self.img_hw[i] = x\r\n gb += self.imgs[i].nbytes\r\n pbar.desc = f'{prefix}Caching images ({gb / 1E9:.1f}GB)'\r\n pbar.close()\r\n\r\n def cache_labels(self, path=Path('./labels.cache'), prefix=''):\r\n # Cache dataset labels, check images and read shapes\r\n x = {} # dict\r\n nm, nf, ne, nc = 0, 0, 0, 0 # number missing, found, empty, duplicate\r\n pbar = tqdm(zip(self.img_files, self.label_files), desc='Scanning images', total=len(self.img_files))\r\n for i, (im_file, lb_file) in enumerate(pbar):\r\n try:\r\n # verify images\r\n im = Image.open(im_file)\r\n im.verify() # PIL verify\r\n shape = exif_size(im) # image size\r\n segments = [] # instance segments\r\n assert (shape[0] > 9) & (shape[1] > 9), f'image size {shape} <10 pixels'\r\n assert im.format.lower() in img_formats, f'invalid image format {im.format}'\r\n\r\n # verify labels\r\n if os.path.isfile(lb_file):\r\n nf += 1 # label found\r\n with open(lb_file, 'r') as f:\r\n l = [x.split() for x in f.read().strip().splitlines()]\r\n if any([len(x) > 8 for x in l]): # is segment\r\n classes = np.array([x[0] for x in l], dtype=np.float32)\r\n segments = [np.array(x[1:], dtype=np.float32).reshape(-1, 2) for x in l] # (cls, xy1...)\r\n l = np.concatenate((classes.reshape(-1, 1), segments2boxes(segments)), 1) # (cls, xywh)\r\n l = np.array(l, dtype=np.float32)\r\n if len(l):\r\n assert l.shape[1] == 5, 'labels require 5 columns each'\r\n assert (l >= 0).all(), 'negative labels'\r\n assert (l[:, 1:] <= 1).all(), 'non-normalized or out of bounds coordinate labels'\r\n assert np.unique(l, axis=0).shape[0] == l.shape[0], 'duplicate labels'\r\n else:\r\n ne += 1 # label empty\r\n l = np.zeros((0, 5), dtype=np.float32)\r\n else:\r\n nm += 1 # label missing\r\n l = np.zeros((0, 5), dtype=np.float32)\r\n x[im_file] = [l, shape, segments]\r\n except Exception as e:\r\n nc += 1\r\n print(f'{prefix}WARNING: Ignoring corrupted image and/or label {im_file}: {e}')\r\n\r\n pbar.desc = f\"{prefix}Scanning '{path.parent / path.stem}' images and labels... \" \\\r\n f\"{nf} found, {nm} missing, {ne} empty, {nc} corrupted\"\r\n pbar.close()\r\n\r\n if nf == 0:\r\n print(f'{prefix}WARNING: No labels found in {path}. See {help_url}')\r\n\r\n x['hash'] = get_hash(self.label_files + self.img_files)\r\n x['results'] = nf, nm, ne, nc, i + 1\r\n x['version'] = 0.1 # cache version\r\n torch.save(x, path) # save for next time\r\n logging.info(f'{prefix}New cache created: {path}')\r\n return x\r\n\r\n def __len__(self):\r\n return len(self.img_files)\r\n\r\n # def __iter__(self):\r\n # self.count = -1\r\n # print('ran dataset iter')\r\n # #self.shuffled_vector = np.random.permutation(self.nF) if self.augment else np.arange(self.nF)\r\n # return self\r\n\r\n def __getitem__(self, index):\r\n index = self.indices[index] # linear, shuffled, or image_weights\r\n\r\n hyp = self.hyp\r\n mosaic = self.mosaic and random.random() < hyp['mosaic']\r\n if mosaic:\r\n # Load mosaic\r\n if random.random() < 0.8:\r\n img, labels = load_mosaic(self, index)\r\n else:\r\n img, labels = load_mosaic9(self, index)\r\n shapes = None\r\n\r\n # MixUp https://arxiv.org/pdf/1710.09412.pdf\r\n if random.random() < hyp['mixup']:\r\n if random.random() < 0.8:\r\n img2, labels2 = load_mosaic(self, random.randint(0, len(self.labels) - 1))\r\n else:\r\n img2, labels2 = load_mosaic9(self, random.randint(0, len(self.labels) - 1))\r\n r = np.random.beta(8.0, 8.0) # mixup ratio, alpha=beta=8.0\r\n img = (img * r + img2 * (1 - r)).astype(np.uint8)\r\n labels = np.concatenate((labels, labels2), 0)\r\n\r\n else:\r\n # Load image\r\n img, (h0, w0), (h, w) = load_image(self, index)\r\n\r\n # Letterbox\r\n shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape\r\n img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment)\r\n shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling\r\n\r\n labels = self.labels[index].copy()\r\n if labels.size: # normalized xywh to pixel xyxy format\r\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], ratio[0] * w, ratio[1] * h, padw=pad[0], padh=pad[1])\r\n\r\n if self.augment:\r\n # Augment imagespace\r\n if not mosaic:\r\n img, labels = random_perspective(img, labels,\r\n degrees=hyp['degrees'],\r\n translate=hyp['translate'],\r\n scale=hyp['scale'],\r\n shear=hyp['shear'],\r\n perspective=hyp['perspective'])\r\n \r\n \r\n #img, labels = self.albumentations(img, labels)\r\n\r\n # Augment colorspace\r\n augment_hsv(img, hgain=hyp['hsv_h'], sgain=hyp['hsv_s'], vgain=hyp['hsv_v'])\r\n\r\n # Apply cutouts\r\n # if random.random() < 0.9:\r\n # labels = cutout(img, labels)\r\n \r\n if random.random() < hyp['paste_in']:\r\n sample_labels, sample_images, sample_masks = [], [], [] \r\n while len(sample_labels) < 30:\r\n sample_labels_, sample_images_, sample_masks_ = load_samples(self, random.randint(0, len(self.labels) - 1))\r\n sample_labels += sample_labels_\r\n sample_images += sample_images_\r\n sample_masks += sample_masks_\r\n #print(len(sample_labels))\r\n if len(sample_labels) == 0:\r\n break\r\n labels = pastein(img, labels, sample_labels, sample_images, sample_masks)\r\n\r\n nL = len(labels) # number of labels\r\n if nL:\r\n labels[:, 1:5] = xyxy2xywh(labels[:, 1:5]) # convert xyxy to xywh\r\n labels[:, [2, 4]] /= img.shape[0] # normalized height 0-1\r\n labels[:, [1, 3]] /= img.shape[1] # normalized width 0-1\r\n\r\n if self.augment:\r\n # flip up-down\r\n if random.random() < hyp['flipud']:\r\n img = np.flipud(img)\r\n if nL:\r\n labels[:, 2] = 1 - labels[:, 2]\r\n\r\n # flip left-right\r\n if random.random() < hyp['fliplr']:\r\n img = np.fliplr(img)\r\n if nL:\r\n labels[:, 1] = 1 - labels[:, 1]\r\n\r\n labels_out = torch.zeros((nL, 6))\r\n if nL:\r\n labels_out[:, 1:] = torch.from_numpy(labels)\r\n\r\n # Convert\r\n img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\r\n img = np.ascontiguousarray(img)\r\n\r\n return torch.from_numpy(img), labels_out, self.img_files[index], shapes\r\n\r\n @staticmethod\r\n def collate_fn(batch):\r\n img, label, path, shapes = zip(*batch) # transposed\r\n for i, l in enumerate(label):\r\n l[:, 0] = i # add target image index for build_targets()\r\n return torch.stack(img, 0), torch.cat(label, 0), path, shapes\r\n\r\n @staticmethod\r\n def collate_fn4(batch):\r\n img, label, path, shapes = zip(*batch) # transposed\r\n n = len(shapes) // 4\r\n img4, label4, path4, shapes4 = [], [], path[:n], shapes[:n]\r\n\r\n ho = torch.tensor([[0., 0, 0, 1, 0, 0]])\r\n wo = torch.tensor([[0., 0, 1, 0, 0, 0]])\r\n s = torch.tensor([[1, 1, .5, .5, .5, .5]]) # scale\r\n for i in range(n): # zidane torch.zeros(16,3,720,1280) # BCHW\r\n i *= 4\r\n if random.random() < 0.5:\r\n im = F.interpolate(img[i].unsqueeze(0).float(), scale_factor=2., mode='bilinear', align_corners=False)[\r\n 0].type(img[i].type())\r\n l = label[i]\r\n else:\r\n im = torch.cat((torch.cat((img[i], img[i + 1]), 1), torch.cat((img[i + 2], img[i + 3]), 1)), 2)\r\n l = torch.cat((label[i], label[i + 1] + ho, label[i + 2] + wo, label[i + 3] + ho + wo), 0) * s\r\n img4.append(im)\r\n label4.append(l)\r\n\r\n for i, l in enumerate(label4):\r\n l[:, 0] = i # add target image index for build_targets()\r\n\r\n return torch.stack(img4, 0), torch.cat(label4, 0), path4, shapes4\r\n\r\n\r\n# Ancillary functions --------------------------------------------------------------------------------------------------\r\ndef load_image(self, index):\r\n # loads 1 image from dataset, returns img, original hw, resized hw\r\n img = self.imgs[index]\r\n if img is None: # not cached\r\n path = self.img_files[index]\r\n img = cv2.imread(path) # BGR\r\n assert img is not None, 'Image Not Found ' + path\r\n h0, w0 = img.shape[:2] # orig hw\r\n r = self.img_size / max(h0, w0) # resize image to img_size\r\n if r != 1: # always resize down, only resize up if training with augmentation\r\n interp = cv2.INTER_AREA if r < 1 and not self.augment else cv2.INTER_LINEAR\r\n img = cv2.resize(img, (int(w0 * r), int(h0 * r)), interpolation=interp)\r\n return img, (h0, w0), img.shape[:2] # img, hw_original, hw_resized\r\n else:\r\n return self.imgs[index], self.img_hw0[index], self.img_hw[index] # img, hw_original, hw_resized\r\n\r\n\r\ndef augment_hsv(img, hgain=0.5, sgain=0.5, vgain=0.5):\r\n r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains\r\n hue, sat, val = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV))\r\n dtype = img.dtype # uint8\r\n\r\n x = np.arange(0, 256, dtype=np.int16)\r\n lut_hue = ((x * r[0]) % 180).astype(dtype)\r\n lut_sat = np.clip(x * r[1], 0, 255).astype(dtype)\r\n lut_val = np.clip(x * r[2], 0, 255).astype(dtype)\r\n\r\n img_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))).astype(dtype)\r\n cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR, dst=img) # no return needed\r\n\r\n\r\ndef hist_equalize(img, clahe=True, bgr=False):\r\n # Equalize histogram on BGR image 'img' with img.shape(n,m,3) and range 0-255\r\n yuv = cv2.cvtColor(img, cv2.COLOR_BGR2YUV if bgr else cv2.COLOR_RGB2YUV)\r\n if clahe:\r\n c = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\r\n yuv[:, :, 0] = c.apply(yuv[:, :, 0])\r\n else:\r\n yuv[:, :, 0] = cv2.equalizeHist(yuv[:, :, 0]) # equalize Y channel histogram\r\n return cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR if bgr else cv2.COLOR_YUV2RGB) # convert YUV image to RGB\r\n\r\n\r\ndef load_mosaic(self, index):\r\n # loads images in a 4-mosaic\r\n\r\n labels4, segments4 = [], []\r\n s = self.img_size\r\n yc, xc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border] # mosaic center x, y\r\n indices = [index] + random.choices(self.indices, k=3) # 3 additional image indices\r\n for i, index in enumerate(indices):\r\n # Load image\r\n img, _, (h, w) = load_image(self, index)\r\n\r\n # place img in img4\r\n if i == 0: # top left\r\n img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\r\n x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image)\r\n x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image)\r\n elif i == 1: # top right\r\n x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\r\n x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\r\n elif i == 2: # bottom left\r\n x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\r\n x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h)\r\n elif i == 3: # bottom right\r\n x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\r\n x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\r\n\r\n img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]\r\n padw = x1a - x1b\r\n padh = y1a - y1b\r\n\r\n # Labels\r\n labels, segments = self.labels[index].copy(), self.segments[index].copy()\r\n if labels.size:\r\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh) # normalized xywh to pixel xyxy format\r\n segments = [xyn2xy(x, w, h, padw, padh) for x in segments]\r\n labels4.append(labels)\r\n segments4.extend(segments)\r\n\r\n # Concat/clip labels\r\n labels4 = np.concatenate(labels4, 0)\r\n for x in (labels4[:, 1:], *segments4):\r\n np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective()\r\n # img4, labels4 = replicate(img4, labels4) # replicate\r\n\r\n # Augment\r\n #img4, labels4, segments4 = remove_background(img4, labels4, segments4)\r\n #sample_segments(img4, labels4, segments4, probability=self.hyp['copy_paste'])\r\n img4, labels4, segments4 = copy_paste(img4, labels4, segments4, probability=self.hyp['copy_paste'])\r\n img4, labels4 = random_perspective(img4, labels4, segments4,\r\n degrees=self.hyp['degrees'],\r\n translate=self.hyp['translate'],\r\n scale=self.hyp['scale'],\r\n shear=self.hyp['shear'],\r\n perspective=self.hyp['perspective'],\r\n border=self.mosaic_border) # border to remove\r\n\r\n return img4, labels4\r\n\r\n\r\ndef load_mosaic9(self, index):\r\n # loads images in a 9-mosaic\r\n\r\n labels9, segments9 = [], []\r\n s = self.img_size\r\n indices = [index] + random.choices(self.indices, k=8) # 8 additional image indices\r\n for i, index in enumerate(indices):\r\n # Load image\r\n img, _, (h, w) = load_image(self, index)\r\n\r\n # place img in img9\r\n if i == 0: # center\r\n img9 = np.full((s * 3, s * 3, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\r\n h0, w0 = h, w\r\n c = s, s, s + w, s + h # xmin, ymin, xmax, ymax (base) coordinates\r\n elif i == 1: # top\r\n c = s, s - h, s + w, s\r\n elif i == 2: # top right\r\n c = s + wp, s - h, s + wp + w, s\r\n elif i == 3: # right\r\n c = s + w0, s, s + w0 + w, s + h\r\n elif i == 4: # bottom right\r\n c = s + w0, s + hp, s + w0 + w, s + hp + h\r\n elif i == 5: # bottom\r\n c = s + w0 - w, s + h0, s + w0, s + h0 + h\r\n elif i == 6: # bottom left\r\n c = s + w0 - wp - w, s + h0, s + w0 - wp, s + h0 + h\r\n elif i == 7: # left\r\n c = s - w, s + h0 - h, s, s + h0\r\n elif i == 8: # top left\r\n c = s - w, s + h0 - hp - h, s, s + h0 - hp\r\n\r\n padx, pady = c[:2]\r\n x1, y1, x2, y2 = [max(x, 0) for x in c] # allocate coords\r\n\r\n # Labels\r\n labels, segments = self.labels[index].copy(), self.segments[index].copy()\r\n if labels.size:\r\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padx, pady) # normalized xywh to pixel xyxy format\r\n segments = [xyn2xy(x, w, h, padx, pady) for x in segments]\r\n labels9.append(labels)\r\n segments9.extend(segments)\r\n\r\n # Image\r\n img9[y1:y2, x1:x2] = img[y1 - pady:, x1 - padx:] # img9[ymin:ymax, xmin:xmax]\r\n hp, wp = h, w # height, width previous\r\n\r\n # Offset\r\n yc, xc = [int(random.uniform(0, s)) for _ in self.mosaic_border] # mosaic center x, y\r\n img9 = img9[yc:yc + 2 * s, xc:xc + 2 * s]\r\n\r\n # Concat/clip labels\r\n labels9 = np.concatenate(labels9, 0)\r\n labels9[:, [1, 3]] -= xc\r\n labels9[:, [2, 4]] -= yc\r\n c = np.array([xc, yc]) # centers\r\n segments9 = [x - c for x in segments9]\r\n\r\n for x in (labels9[:, 1:], *segments9):\r\n np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective()\r\n # img9, labels9 = replicate(img9, labels9) # replicate\r\n\r\n # Augment\r\n #img9, labels9, segments9 = remove_background(img9, labels9, segments9)\r\n img9, labels9, segments9 = copy_paste(img9, labels9, segments9, probability=self.hyp['copy_paste'])\r\n img9, labels9 = random_perspective(img9, labels9, segments9,\r\n degrees=self.hyp['degrees'],\r\n translate=self.hyp['translate'],\r\n scale=self.hyp['scale'],\r\n shear=self.hyp['shear'],\r\n perspective=self.hyp['perspective'],\r\n border=self.mosaic_border) # border to remove\r\n\r\n return img9, labels9\r\n\r\n\r\ndef load_samples(self, index):\r\n # loads images in a 4-mosaic\r\n\r\n labels4, segments4 = [], []\r\n s = self.img_size\r\n yc, xc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border] # mosaic center x, y\r\n indices = [index] + random.choices(self.indices, k=3) # 3 additional image indices\r\n for i, index in enumerate(indices):\r\n # Load image\r\n img, _, (h, w) = load_image(self, index)\r\n\r\n # place img in img4\r\n if i == 0: # top left\r\n img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles\r\n x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image)\r\n x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image)\r\n elif i == 1: # top right\r\n x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\r\n x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\r\n elif i == 2: # bottom left\r\n x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\r\n x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h)\r\n elif i == 3: # bottom right\r\n x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\r\n x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\r\n\r\n img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]\r\n padw = x1a - x1b\r\n padh = y1a - y1b\r\n\r\n # Labels\r\n labels, segments = self.labels[index].copy(), self.segments[index].copy()\r\n if labels.size:\r\n labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh) # normalized xywh to pixel xyxy format\r\n segments = [xyn2xy(x, w, h, padw, padh) for x in segments]\r\n labels4.append(labels)\r\n segments4.extend(segments)\r\n\r\n # Concat/clip labels\r\n labels4 = np.concatenate(labels4, 0)\r\n for x in (labels4[:, 1:], *segments4):\r\n np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective()\r\n # img4, labels4 = replicate(img4, labels4) # replicate\r\n\r\n # Augment\r\n #img4, labels4, segments4 = remove_background(img4, labels4, segments4)\r\n sample_labels, sample_images, sample_masks = sample_segments(img4, labels4, segments4, probability=0.5)\r\n\r\n return sample_labels, sample_images, sample_masks\r\n\r\n\r\ndef copy_paste(img, labels, segments, probability=0.5):\r\n # Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy)\r\n n = len(segments)\r\n if probability and n:\r\n h, w, c = img.shape # height, width, channels\r\n im_new = np.zeros(img.shape, np.uint8)\r\n for j in random.sample(range(n), k=round(probability * n)):\r\n l, s = labels[j], segments[j]\r\n box = w - l[3], l[2], w - l[1], l[4]\r\n ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area\r\n if (ioa < 0.30).all(): # allow 30% obscuration of existing labels\r\n labels = np.concatenate((labels, [[l[0], *box]]), 0)\r\n segments.append(np.concatenate((w - s[:, 0:1], s[:, 1:2]), 1))\r\n cv2.drawContours(im_new, [segments[j].astype(np.int32)], -1, (255, 255, 255), cv2.FILLED)\r\n\r\n result = cv2.bitwise_and(src1=img, src2=im_new)\r\n result = cv2.flip(result, 1) # augment segments (flip left-right)\r\n i = result > 0 # pixels to replace\r\n # i[:, :] = result.max(2).reshape(h, w, 1) # act over ch\r\n img[i] = result[i] # cv2.imwrite('debug.jpg', img) # debug\r\n\r\n return img, labels, segments\r\n\r\n\r\ndef remove_background(img, labels, segments):\r\n # Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy)\r\n n = len(segments)\r\n h, w, c = img.shape # height, width, channels\r\n im_new = np.zeros(img.shape, np.uint8)\r\n img_new = np.ones(img.shape, np.uint8) * 114\r\n for j in range(n):\r\n cv2.drawContours(im_new, [segments[j].astype(np.int32)], -1, (255, 255, 255), cv2.FILLED)\r\n\r\n result = cv2.bitwise_and(src1=img, src2=im_new)\r\n \r\n i = result > 0 # pixels to replace\r\n img_new[i] = result[i] # cv2.imwrite('debug.jpg', img) # debug\r\n\r\n return img_new, labels, segments\r\n\r\n\r\ndef sample_segments(img, labels, segments, probability=0.5):\r\n # Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy)\r\n n = len(segments)\r\n sample_labels = []\r\n sample_images = []\r\n sample_masks = []\r\n if probability and n:\r\n h, w, c = img.shape # height, width, channels\r\n for j in random.sample(range(n), k=round(probability * n)):\r\n l, s = labels[j], segments[j]\r\n box = l[1].astype(int).clip(0,w-1), l[2].astype(int).clip(0,h-1), l[3].astype(int).clip(0,w-1), l[4].astype(int).clip(0,h-1) \r\n \r\n #print(box)\r\n if (box[2] <= box[0]) or (box[3] <= box[1]):\r\n continue\r\n \r\n sample_labels.append(l[0])\r\n \r\n mask = np.zeros(img.shape, np.uint8)\r\n \r\n cv2.drawContours(mask, [segments[j].astype(np.int32)], -1, (255, 255, 255), cv2.FILLED)\r\n sample_masks.append(mask[box[1]:box[3],box[0]:box[2],:])\r\n \r\n result = cv2.bitwise_and(src1=img, src2=mask)\r\n i = result > 0 # pixels to replace\r\n mask[i] = result[i] # cv2.imwrite('debug.jpg', img) # debug\r\n #print(box)\r\n sample_images.append(mask[box[1]:box[3],box[0]:box[2],:])\r\n\r\n return sample_labels, sample_images, sample_masks\r\n\r\n\r\ndef replicate(img, labels):\r\n # Replicate labels\r\n h, w = img.shape[:2]\r\n boxes = labels[:, 1:].astype(int)\r\n x1, y1, x2, y2 = boxes.T\r\n s = ((x2 - x1) + (y2 - y1)) / 2 # side length (pixels)\r\n for i in s.argsort()[:round(s.size * 0.5)]: # smallest indices\r\n x1b, y1b, x2b, y2b = boxes[i]\r\n bh, bw = y2b - y1b, x2b - x1b\r\n yc, xc = int(random.uniform(0, h - bh)), int(random.uniform(0, w - bw)) # offset x, y\r\n x1a, y1a, x2a, y2a = [xc, yc, xc + bw, yc + bh]\r\n img[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax]\r\n labels = np.append(labels, [[labels[i, 0], x1a, y1a, x2a, y2a]], axis=0)\r\n\r\n return img, labels\r\n\r\n\r\ndef letterbox(img, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32):\r\n # Resize and pad image while meeting stride-multiple constraints\r\n shape = img.shape[:2] # current shape [height, width]\r\n if isinstance(new_shape, int):\r\n new_shape = (new_shape, new_shape)\r\n\r\n # Scale ratio (new / old)\r\n r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])\r\n if not scaleup: # only scale down, do not scale up (for better test mAP)\r\n r = min(r, 1.0)\r\n\r\n # Compute padding\r\n ratio = r, r # width, height ratios\r\n new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))\r\n dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding\r\n if auto: # minimum rectangle\r\n dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding\r\n elif scaleFill: # stretch\r\n dw, dh = 0.0, 0.0\r\n new_unpad = (new_shape[1], new_shape[0])\r\n ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios\r\n\r\n dw /= 2 # divide padding into 2 sides\r\n dh /= 2\r\n\r\n if shape[::-1] != new_unpad: # resize\r\n img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR)\r\n top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))\r\n left, right = int(round(dw - 0.1)), int(round(dw + 0.1))\r\n img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border\r\n return img, ratio, (dw, dh)\r\n\r\n\r\ndef random_perspective(img, targets=(), segments=(), degrees=10, translate=.1, scale=.1, shear=10, perspective=0.0,\r\n border=(0, 0)):\r\n # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(.1, .1), scale=(.9, 1.1), shear=(-10, 10))\r\n # targets = [cls, xyxy]\r\n\r\n height = img.shape[0] + border[0] * 2 # shape(h,w,c)\r\n width = img.shape[1] + border[1] * 2\r\n\r\n # Center\r\n C = np.eye(3)\r\n C[0, 2] = -img.shape[1] / 2 # x translation (pixels)\r\n C[1, 2] = -img.shape[0] / 2 # y translation (pixels)\r\n\r\n # Perspective\r\n P = np.eye(3)\r\n P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y)\r\n P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x)\r\n\r\n # Rotation and Scale\r\n R = np.eye(3)\r\n a = random.uniform(-degrees, degrees)\r\n # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations\r\n s = random.uniform(1 - scale, 1.1 + scale)\r\n # s = 2 ** random.uniform(-scale, scale)\r\n R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s)\r\n\r\n # Shear\r\n S = np.eye(3)\r\n S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg)\r\n S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg)\r\n\r\n # Translation\r\n T = np.eye(3)\r\n T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width # x translation (pixels)\r\n T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height # y translation (pixels)\r\n\r\n # Combined rotation matrix\r\n M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT\r\n if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed\r\n if perspective:\r\n img = cv2.warpPerspective(img, M, dsize=(width, height), borderValue=(114, 114, 114))\r\n else: # affine\r\n img = cv2.warpAffine(img, M[:2], dsize=(width, height), borderValue=(114, 114, 114))\r\n\r\n # Visualize\r\n # import matplotlib.pyplot as plt\r\n # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel()\r\n # ax[0].imshow(img[:, :, ::-1]) # base\r\n # ax[1].imshow(img2[:, :, ::-1]) # warped\r\n\r\n # Transform label coordinates\r\n n = len(targets)\r\n if n:\r\n use_segments = any(x.any() for x in segments)\r\n new = np.zeros((n, 4))\r\n if use_segments: # warp segments\r\n segments = resample_segments(segments) # upsample\r\n for i, segment in enumerate(segments):\r\n xy = np.ones((len(segment), 3))\r\n xy[:, :2] = segment\r\n xy = xy @ M.T # transform\r\n xy = xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2] # perspective rescale or affine\r\n\r\n # clip\r\n new[i] = segment2box(xy, width, height)\r\n\r\n else: # warp boxes\r\n xy = np.ones((n * 4, 3))\r\n xy[:, :2] = targets[:, [1, 2, 3, 4, 1, 4, 3, 2]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1\r\n xy = xy @ M.T # transform\r\n xy = (xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2]).reshape(n, 8) # perspective rescale or affine\r\n\r\n # create new boxes\r\n x = xy[:, [0, 2, 4, 6]]\r\n y = xy[:, [1, 3, 5, 7]]\r\n new = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T\r\n\r\n # clip\r\n new[:, [0, 2]] = new[:, [0, 2]].clip(0, width)\r\n new[:, [1, 3]] = new[:, [1, 3]].clip(0, height)\r\n\r\n # filter candidates\r\n i = box_candidates(box1=targets[:, 1:5].T * s, box2=new.T, area_thr=0.01 if use_segments else 0.10)\r\n targets = targets[i]\r\n targets[:, 1:5] = new[i]\r\n\r\n return img, targets\r\n\r\n\r\ndef box_candidates(box1, box2, wh_thr=2, ar_thr=20, area_thr=0.1, eps=1e-16): # box1(4,n), box2(4,n)\r\n # Compute candidate boxes: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio\r\n w1, h1 = box1[2] - box1[0], box1[3] - box1[1]\r\n w2, h2 = box2[2] - box2[0], box2[3] - box2[1]\r\n ar = np.maximum(w2 / (h2 + eps), h2 / (w2 + eps)) # aspect ratio\r\n return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + eps) > area_thr) & (ar < ar_thr) # candidates\r\n\r\n\r\ndef bbox_ioa(box1, box2):\r\n # Returns the intersection over box2 area given box1, box2. box1 is 4, box2 is nx4. boxes are x1y1x2y2\r\n box2 = box2.transpose()\r\n\r\n # Get the coordinates of bounding boxes\r\n b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]\r\n b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]\r\n\r\n # Intersection area\r\n inter_area = (np.minimum(b1_x2, b2_x2) - np.maximum(b1_x1, b2_x1)).clip(0) * \\\r\n (np.minimum(b1_y2, b2_y2) - np.maximum(b1_y1, b2_y1)).clip(0)\r\n\r\n # box2 area\r\n box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + 1e-16\r\n\r\n # Intersection over box2 area\r\n return inter_area / box2_area\r\n \r\n\r\ndef cutout(image, labels):\r\n # Applies image cutout augmentation https://arxiv.org/abs/1708.04552\r\n h, w = image.shape[:2]\r\n\r\n # create random masks\r\n scales = [0.5] * 1 + [0.25] * 2 + [0.125] * 4 + [0.0625] * 8 + [0.03125] * 16 # image size fraction\r\n for s in scales:\r\n mask_h = random.randint(1, int(h * s))\r\n mask_w = random.randint(1, int(w * s))\r\n\r\n # box\r\n xmin = max(0, random.randint(0, w) - mask_w // 2)\r\n ymin = max(0, random.randint(0, h) - mask_h // 2)\r\n xmax = min(w, xmin + mask_w)\r\n ymax = min(h, ymin + mask_h)\r\n\r\n # apply random color mask\r\n image[ymin:ymax, xmin:xmax] = [random.randint(64, 191) for _ in range(3)]\r\n\r\n # return unobscured labels\r\n if len(labels) and s > 0.03:\r\n box = np.array([xmin, ymin, xmax, ymax], dtype=np.float32)\r\n ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area\r\n labels = labels[ioa < 0.60] # remove >60% obscured labels\r\n\r\n return labels\r\n \r\n\r\ndef pastein(image, labels, sample_labels, sample_images, sample_masks):\r\n # Applies image cutout augmentation https://arxiv.org/abs/1708.04552\r\n h, w = image.shape[:2]\r\n\r\n # create random masks\r\n scales = [0.75] * 2 + [0.5] * 4 + [0.25] * 4 + [0.125] * 4 + [0.0625] * 6 # image size fraction\r\n for s in scales:\r\n if random.random() < 0.2:\r\n continue\r\n mask_h = random.randint(1, int(h * s))\r\n mask_w = random.randint(1, int(w * s))\r\n\r\n # box\r\n xmin = max(0, random.randint(0, w) - mask_w // 2)\r\n ymin = max(0, random.randint(0, h) - mask_h // 2)\r\n xmax = min(w, xmin + mask_w)\r\n ymax = min(h, ymin + mask_h) \r\n \r\n box = np.array([xmin, ymin, xmax, ymax], dtype=np.float32)\r\n if len(labels):\r\n ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area \r\n else:\r\n ioa = np.zeros(1)\r\n \r\n if (ioa < 0.30).all() and len(sample_labels) and (xmax > xmin+20) and (ymax > ymin+20): # allow 30% obscuration of existing labels\r\n sel_ind = random.randint(0, len(sample_labels)-1)\r\n #print(len(sample_labels))\r\n #print(sel_ind)\r\n #print((xmax-xmin, ymax-ymin))\r\n #print(image[ymin:ymax, xmin:xmax].shape)\r\n #print([[sample_labels[sel_ind], *box]])\r\n #print(labels.shape)\r\n hs, ws, cs = sample_images[sel_ind].shape\r\n r_scale = min((ymax-ymin)/hs, (xmax-xmin)/ws)\r\n r_w = int(ws*r_scale)\r\n r_h = int(hs*r_scale)\r\n \r\n if (r_w > 10) and (r_h > 10):\r\n r_mask = cv2.resize(sample_masks[sel_ind], (r_w, r_h))\r\n r_image = cv2.resize(sample_images[sel_ind], (r_w, r_h))\r\n temp_crop = image[ymin:ymin+r_h, xmin:xmin+r_w]\r\n m_ind = r_mask > 0\r\n if m_ind.astype(np.int).sum() > 60:\r\n temp_crop[m_ind] = r_image[m_ind]\r\n #print(sample_labels[sel_ind])\r\n #print(sample_images[sel_ind].shape)\r\n #print(temp_crop.shape)\r\n box = np.array([xmin, ymin, xmin+r_w, ymin+r_h], dtype=np.float32)\r\n if len(labels):\r\n labels = np.concatenate((labels, [[sample_labels[sel_ind], *box]]), 0)\r\n else:\r\n labels = np.array([[sample_labels[sel_ind], *box]])\r\n \r\n image[ymin:ymin+r_h, xmin:xmin+r_w] = temp_crop\r\n\r\n return labels\r\n\r\nclass Albumentations:\r\n # YOLOv5 Albumentations class (optional, only used if package is installed)\r\n def __init__(self):\r\n self.transform = None\r\n import albumentations as A\r\n\r\n self.transform = A.Compose([\r\n A.CLAHE(p=0.01),\r\n A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.01),\r\n A.RandomGamma(gamma_limit=[80, 120], p=0.01),\r\n A.Blur(p=0.01),\r\n A.MedianBlur(p=0.01),\r\n A.ToGray(p=0.01),\r\n A.ImageCompression(quality_lower=75, p=0.01),],\r\n bbox_params=A.BboxParams(format='pascal_voc', label_fields=['class_labels']))\r\n\r\n #logging.info(colorstr('albumentations: ') + ', '.join(f'{x}' for x in self.transform.transforms if x.p))\r\n\r\n def __call__(self, im, labels, p=1.0):\r\n if self.transform and random.random() < p:\r\n new = self.transform(image=im, bboxes=labels[:, 1:], class_labels=labels[:, 0]) # transformed\r\n im, labels = new['image'], np.array([[c, *b] for c, b in zip(new['class_labels'], new['bboxes'])])\r\n return im, labels\r\n\r\n\r\ndef create_folder(path='./new'):\r\n # Create folder\r\n if os.path.exists(path):\r\n shutil.rmtree(path) # delete output folder\r\n os.makedirs(path) # make new output folder\r\n\r\n\r\ndef flatten_recursive(path='../coco'):\r\n # Flatten a recursive directory by bringing all files to top level\r\n new_path = Path(path + '_flat')\r\n create_folder(new_path)\r\n for file in tqdm(glob.glob(str(Path(path)) + '/**/*.*', recursive=True)):\r\n shutil.copyfile(file, new_path / Path(file).name)\r\n\r\n\r\ndef extract_boxes(path='../coco/'): # from utils.datasets import *; extract_boxes('../coco128')\r\n # Convert detection dataset into classification dataset, with one directory per class\r\n\r\n path = Path(path) # images dir\r\n shutil.rmtree(path / 'classifier') if (path / 'classifier').is_dir() else None # remove existing\r\n files = list(path.rglob('*.*'))\r\n n = len(files) # number of files\r\n for im_file in tqdm(files, total=n):\r\n if im_file.suffix[1:] in img_formats:\r\n # image\r\n im = cv2.imread(str(im_file))[..., ::-1] # BGR to RGB\r\n h, w = im.shape[:2]\r\n\r\n # labels\r\n lb_file = Path(img2label_paths([str(im_file)])[0])\r\n if Path(lb_file).exists():\r\n with open(lb_file, 'r') as f:\r\n lb = np.array([x.split() for x in f.read().strip().splitlines()], dtype=np.float32) # labels\r\n\r\n for j, x in enumerate(lb):\r\n c = int(x[0]) # class\r\n f = (path / 'classifier') / f'{c}' / f'{path.stem}_{im_file.stem}_{j}.jpg' # new filename\r\n if not f.parent.is_dir():\r\n f.parent.mkdir(parents=True)\r\n\r\n b = x[1:] * [w, h, w, h] # box\r\n # b[2:] = b[2:].max() # rectangle to square\r\n b[2:] = b[2:] * 1.2 + 3 # pad\r\n b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(np.int)\r\n\r\n b[[0, 2]] = np.clip(b[[0, 2]], 0, w) # clip boxes outside of image\r\n b[[1, 3]] = np.clip(b[[1, 3]], 0, h)\r\n assert cv2.imwrite(str(f), im[b[1]:b[3], b[0]:b[2]]), f'box failure in {f}'\r\n\r\n\r\ndef autosplit(path='../coco', weights=(0.9, 0.1, 0.0), annotated_only=False):\r\n \"\"\" Autosplit a dataset into train/val/test splits and save path/autosplit_*.txt files\r\n Usage: from utils.datasets import *; autosplit('../coco')\r\n Arguments\r\n path: Path to images directory\r\n weights: Train, val, test weights (list)\r\n annotated_only: Only use images with an annotated txt file\r\n \"\"\"\r\n path = Path(path) # images dir\r\n files = sum([list(path.rglob(f\"*.{img_ext}\")) for img_ext in img_formats], []) # image files only\r\n n = len(files) # number of files\r\n indices = random.choices([0, 1, 2], weights=weights, k=n) # assign each image to a split\r\n\r\n txt = ['autosplit_train.txt', 'autosplit_val.txt', 'autosplit_test.txt'] # 3 txt files\r\n [(path / x).unlink() for x in txt if (path / x).exists()] # remove existing\r\n\r\n print(f'Autosplitting images from {path}' + ', using *.txt labeled images only' * annotated_only)\r\n for i, img in tqdm(zip(indices, files), total=n):\r\n if not annotated_only or Path(img2label_paths([str(img)])[0]).exists(): # check label\r\n with open(path / txt[i], 'a') as f:\r\n f.write(str(img) + '\\n') # add image to txt file\r\n \r\n \r\ndef load_segmentations(self, index):\r\n key = '/work/handsomejw66/coco17/' + self.img_files[index]\r\n #print(key)\r\n # /work/handsomejw66/coco17/\r\n return self.segs[key]\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/general.py", "content": "# YOLOR general utils\r\n\r\nimport glob\r\nimport logging\r\nimport math\r\nimport os\r\nimport platform\r\nimport random\r\nimport re\r\nimport subprocess\r\nimport time\r\nfrom pathlib import Path\r\n\r\nimport cv2\r\nimport numpy as np\r\nimport pandas as pd\r\nimport torch\r\nimport torchvision\r\nimport yaml\r\n\r\nfrom asone.pose_estimators.yolov7_pose.utils.google_utils import gsutil_getsize\r\nfrom asone.pose_estimators.yolov7_pose.utils.metrics import fitness\r\nfrom asone.pose_estimators.yolov7_pose.utils.torch_utils import init_torch_seeds\r\n\r\n# Settings\r\ntorch.set_printoptions(linewidth=320, precision=5, profile='long')\r\nnp.set_printoptions(linewidth=320, formatter={'float_kind': '{:11.5g}'.format}) # format short g, %precision=5\r\npd.options.display.max_columns = 10\r\ncv2.setNumThreads(0) # prevent OpenCV from multithreading (incompatible with PyTorch DataLoader)\r\nos.environ['NUMEXPR_MAX_THREADS'] = str(min(os.cpu_count(), 8)) # NumExpr max threads\r\n\r\n\r\ndef set_logging(rank=-1):\r\n logging.basicConfig(\r\n format=\"%(message)s\",\r\n level=logging.INFO if rank in [-1, 0] else logging.WARN)\r\n\r\n\r\ndef init_seeds(seed=0):\r\n # Initialize random number generator (RNG) seeds\r\n random.seed(seed)\r\n np.random.seed(seed)\r\n init_torch_seeds(seed)\r\n\r\n\r\ndef get_latest_run(search_dir='.'):\r\n # Return path to most recent 'last.pt' in /runs (i.e. to --resume from)\r\n last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True)\r\n return max(last_list, key=os.path.getctime) if last_list else ''\r\n\r\n\r\ndef isdocker():\r\n # Is environment a Docker container\r\n return Path('/workspace').exists() # or Path('/.dockerenv').exists()\r\n\r\n\r\ndef emojis(str=''):\r\n # Return platform-dependent emoji-safe version of string\r\n return str.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else str\r\n\r\n\r\ndef check_online():\r\n # Check internet connectivity\r\n import socket\r\n try:\r\n socket.create_connection((\"1.1.1.1\", 443), 5) # check host accesability\r\n return True\r\n except OSError:\r\n return False\r\n\r\n\r\ndef check_git_status():\r\n # Recommend 'git pull' if code is out of date\r\n print(colorstr('github: '), end='')\r\n try:\r\n assert Path('.git').exists(), 'skipping check (not a git repository)'\r\n assert not isdocker(), 'skipping check (Docker image)'\r\n assert check_online(), 'skipping check (offline)'\r\n\r\n cmd = 'git fetch && git config --get remote.origin.url'\r\n url = subprocess.check_output(cmd, shell=True).decode().strip().rstrip('.git') # github repo url\r\n branch = subprocess.check_output('git rev-parse --abbrev-ref HEAD', shell=True).decode().strip() # checked out\r\n n = int(subprocess.check_output(f'git rev-list {branch}..origin/master --count', shell=True)) # commits behind\r\n if n > 0:\r\n s = f\"⚠️ WARNING: code is out of date by {n} commit{'s' * (n > 1)}. \" \\\r\n f\"Use 'git pull' to update or 'git clone {url}' to download latest.\"\r\n else:\r\n s = f'up to date with {url} ✅'\r\n print(emojis(s)) # emoji-safe\r\n except Exception as e:\r\n print(e)\r\n\r\n\r\ndef check_requirements(requirements='requirements.txt', exclude=()):\r\n # Check installed dependencies meet requirements (pass *.txt file or list of packages)\r\n import pkg_resources as pkg\r\n prefix = colorstr('red', 'bold', 'requirements:')\r\n if isinstance(requirements, (str, Path)): # requirements.txt file\r\n file = Path(requirements)\r\n if not file.exists():\r\n print(f\"{prefix} {file.resolve()} not found, check failed.\")\r\n return\r\n requirements = [f'{x.name}{x.specifier}' for x in pkg.parse_requirements(file.open()) if x.name not in exclude]\r\n else: # list or tuple of packages\r\n requirements = [x for x in requirements if x not in exclude]\r\n\r\n n = 0 # number of packages updates\r\n for r in requirements:\r\n try:\r\n pkg.require(r)\r\n except Exception as e: # DistributionNotFound or VersionConflict if requirements not met\r\n n += 1\r\n print(f\"{prefix} {e.req} not found and is required by YOLOR, attempting auto-update...\")\r\n print(subprocess.check_output(f\"pip install '{e.req}'\", shell=True).decode())\r\n\r\n if n: # if packages updated\r\n source = file.resolve() if 'file' in locals() else requirements\r\n s = f\"{prefix} {n} package{'s' * (n > 1)} updated per {source}\\n\" \\\r\n f\"{prefix} ⚠️ {colorstr('bold', 'Restart runtime or rerun command for updates to take effect')}\\n\"\r\n print(emojis(s)) # emoji-safe\r\n\r\n\r\ndef check_img_size(img_size, s=32):\r\n # Verify img_size is a multiple of stride s\r\n new_size = make_divisible(img_size, int(s)) # ceil gs-multiple\r\n if new_size != img_size:\r\n print('WARNING: --img-size %g must be multiple of max stride %g, updating to %g' % (img_size, s, new_size))\r\n return new_size\r\n\r\n\r\ndef check_imshow():\r\n # Check if environment supports image displays\r\n try:\r\n assert not isdocker(), 'cv2.imshow() is disabled in Docker environments'\r\n cv2.imshow('test', np.zeros((1, 1, 3)))\r\n cv2.waitKey(1)\r\n cv2.destroyAllWindows()\r\n cv2.waitKey(1)\r\n return True\r\n except Exception as e:\r\n print(f'WARNING: Environment does not support cv2.imshow() or PIL Image.show() image displays\\n{e}')\r\n return False\r\n\r\n\r\ndef check_file(file):\r\n # Search for file if not found\r\n if Path(file).is_file() or file == '':\r\n return file\r\n else:\r\n files = glob.glob('./**/' + file, recursive=True) # find file\r\n assert len(files), f'File Not Found: {file}' # assert file was found\r\n assert len(files) == 1, f\"Multiple files match '{file}', specify exact path: {files}\" # assert unique\r\n return files[0] # return file\r\n\r\n\r\ndef check_dataset(dict):\r\n # Download dataset if not found locally\r\n val, s = dict.get('val'), dict.get('download')\r\n if val and len(val):\r\n val = [Path(x).resolve() for x in (val if isinstance(val, list) else [val])] # val path\r\n if not all(x.exists() for x in val):\r\n print('\\nWARNING: Dataset not found, nonexistent paths: %s' % [str(x) for x in val if not x.exists()])\r\n if s and len(s): # download script\r\n print('Downloading %s ...' % s)\r\n if s.startswith('http') and s.endswith('.zip'): # URL\r\n f = Path(s).name # filename\r\n torch.hub.download_url_to_file(s, f)\r\n r = os.system('unzip -q %s -d ../ && rm %s' % (f, f)) # unzip\r\n else: # bash script\r\n r = os.system(s)\r\n print('Dataset autodownload %s\\n' % ('success' if r == 0 else 'failure')) # analyze return value\r\n else:\r\n raise Exception('Dataset not found.')\r\n\r\n\r\ndef make_divisible(x, divisor):\r\n # Returns x evenly divisible by divisor\r\n return math.ceil(x / divisor) * divisor\r\n\r\n\r\ndef clean_str(s):\r\n # Cleans a string by replacing special characters with underscore _\r\n return re.sub(pattern=\"[|@#!¡·$€%&()=?¿^*;:,¨´><+]\", repl=\"_\", string=s)\r\n\r\n\r\ndef one_cycle(y1=0.0, y2=1.0, steps=100):\r\n # lambda function for sinusoidal ramp from y1 to y2\r\n return lambda x: ((1 - math.cos(x * math.pi / steps)) / 2) * (y2 - y1) + y1\r\n\r\n\r\ndef colorstr(*input):\r\n # Colors a string https://en.wikipedia.org/wiki/ANSI_escape_code, i.e. colorstr('blue', 'hello world')\r\n *args, string = input if len(input) > 1 else ('blue', 'bold', input[0]) # color arguments, string\r\n colors = {'black': '\\033[30m', # basic colors\r\n 'red': '\\033[31m',\r\n 'green': '\\033[32m',\r\n 'yellow': '\\033[33m',\r\n 'blue': '\\033[34m',\r\n 'magenta': '\\033[35m',\r\n 'cyan': '\\033[36m',\r\n 'white': '\\033[37m',\r\n 'bright_black': '\\033[90m', # bright colors\r\n 'bright_red': '\\033[91m',\r\n 'bright_green': '\\033[92m',\r\n 'bright_yellow': '\\033[93m',\r\n 'bright_blue': '\\033[94m',\r\n 'bright_magenta': '\\033[95m',\r\n 'bright_cyan': '\\033[96m',\r\n 'bright_white': '\\033[97m',\r\n 'end': '\\033[0m', # misc\r\n 'bold': '\\033[1m',\r\n 'underline': '\\033[4m'}\r\n return ''.join(colors[x] for x in args) + f'{string}' + colors['end']\r\n\r\n\r\ndef labels_to_class_weights(labels, nc=80):\r\n # Get class weights (inverse frequency) from training labels\r\n if labels[0] is None: # no labels loaded\r\n return torch.Tensor()\r\n\r\n labels = np.concatenate(labels, 0) # labels.shape = (866643, 5) for COCO\r\n classes = labels[:, 0].astype(np.int) # labels = [class xywh]\r\n weights = np.bincount(classes, minlength=nc) # occurrences per class\r\n\r\n # Prepend gridpoint count (for uCE training)\r\n # gpi = ((320 / 32 * np.array([1, 2, 4])) ** 2 * 3).sum() # gridpoints per image\r\n # weights = np.hstack([gpi * len(labels) - weights.sum() * 9, weights * 9]) ** 0.5 # prepend gridpoints to start\r\n\r\n weights[weights == 0] = 1 # replace empty bins with 1\r\n weights = 1 / weights # number of targets per class\r\n weights /= weights.sum() # normalize\r\n return torch.from_numpy(weights)\r\n\r\n\r\ndef labels_to_image_weights(labels, nc=80, class_weights=np.ones(80)):\r\n # Produces image weights based on class_weights and image contents\r\n class_counts = np.array([np.bincount(x[:, 0].astype(np.int), minlength=nc) for x in labels])\r\n image_weights = (class_weights.reshape(1, nc) * class_counts).sum(1)\r\n # index = random.choices(range(n), weights=image_weights, k=1) # weight image sample\r\n return image_weights\r\n\r\n\r\ndef coco80_to_coco91_class(): # converts 80-index (val2014) to 91-index (paper)\r\n # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/\r\n # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\\n')\r\n # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\\n')\r\n # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco\r\n # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet\r\n x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34,\r\n 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,\r\n 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90]\r\n return x\r\n\r\n\r\ndef xyxy2xywh(x):\r\n # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right\r\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\r\n y[:, 0] = (x[:, 0] + x[:, 2]) / 2 # x center\r\n y[:, 1] = (x[:, 1] + x[:, 3]) / 2 # y center\r\n y[:, 2] = x[:, 2] - x[:, 0] # width\r\n y[:, 3] = x[:, 3] - x[:, 1] # height\r\n return y\r\n\r\n\r\ndef xywh2xyxy(x):\r\n # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\r\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\r\n y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x\r\n y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y\r\n y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x\r\n y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y\r\n return y\r\n\r\n\r\ndef xywhn2xyxy(x, w=640, h=640, padw=0, padh=0):\r\n # Convert nx4 boxes from [x, y, w, h] normalized to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\r\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\r\n y[:, 0] = w * (x[:, 0] - x[:, 2] / 2) + padw # top left x\r\n y[:, 1] = h * (x[:, 1] - x[:, 3] / 2) + padh # top left y\r\n y[:, 2] = w * (x[:, 0] + x[:, 2] / 2) + padw # bottom right x\r\n y[:, 3] = h * (x[:, 1] + x[:, 3] / 2) + padh # bottom right y\r\n return y\r\n\r\n\r\ndef xyn2xy(x, w=640, h=640, padw=0, padh=0):\r\n # Convert normalized segments into pixel segments, shape (n,2)\r\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\r\n y[:, 0] = w * x[:, 0] + padw # top left x\r\n y[:, 1] = h * x[:, 1] + padh # top left y\r\n return y\r\n\r\n\r\ndef segment2box(segment, width=640, height=640):\r\n # Convert 1 segment label to 1 box label, applying inside-image constraint, i.e. (xy1, xy2, ...) to (xyxy)\r\n x, y = segment.T # segment xy\r\n inside = (x >= 0) & (y >= 0) & (x <= width) & (y <= height)\r\n x, y, = x[inside], y[inside]\r\n return np.array([x.min(), y.min(), x.max(), y.max()]) if any(x) else np.zeros((1, 4)) # xyxy\r\n\r\n\r\ndef segments2boxes(segments):\r\n # Convert segment labels to box labels, i.e. (cls, xy1, xy2, ...) to (cls, xywh)\r\n boxes = []\r\n for s in segments:\r\n x, y = s.T # segment xy\r\n boxes.append([x.min(), y.min(), x.max(), y.max()]) # cls, xyxy\r\n return xyxy2xywh(np.array(boxes)) # cls, xywh\r\n\r\n\r\ndef resample_segments(segments, n=1000):\r\n # Up-sample an (n,2) segment\r\n for i, s in enumerate(segments):\r\n x = np.linspace(0, len(s) - 1, n)\r\n xp = np.arange(len(s))\r\n segments[i] = np.concatenate([np.interp(x, xp, s[:, i]) for i in range(2)]).reshape(2, -1).T # segment xy\r\n return segments\r\n\r\n\r\ndef scale_coords(img1_shape, coords, img0_shape, ratio_pad=None):\r\n # Rescale coords (xyxy) from img1_shape to img0_shape\r\n if ratio_pad is None: # calculate from img0_shape\r\n gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new\r\n pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding\r\n else:\r\n gain = ratio_pad[0][0]\r\n pad = ratio_pad[1]\r\n\r\n coords[:, [0, 2]] -= pad[0] # x padding\r\n coords[:, [1, 3]] -= pad[1] # y padding\r\n coords[:, :4] /= gain\r\n clip_coords(coords, img0_shape)\r\n return coords\r\n\r\n\r\ndef clip_coords(boxes, img_shape):\r\n # Clip bounding xyxy bounding boxes to image shape (height, width)\r\n boxes[:, 0].clamp_(0, img_shape[1]) # x1\r\n boxes[:, 1].clamp_(0, img_shape[0]) # y1\r\n boxes[:, 2].clamp_(0, img_shape[1]) # x2\r\n boxes[:, 3].clamp_(0, img_shape[0]) # y2\r\n\r\n\r\ndef bbox_iou(box1, box2, x1y1x2y2=True, GIoU=False, DIoU=False, CIoU=False, eps=1e-7):\r\n # Returns the IoU of box1 to box2. box1 is 4, box2 is nx4\r\n box2 = box2.T\r\n\r\n # Get the coordinates of bounding boxes\r\n if x1y1x2y2: # x1, y1, x2, y2 = box1\r\n b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]\r\n b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]\r\n else: # transform from xywh to xyxy\r\n b1_x1, b1_x2 = box1[0] - box1[2] / 2, box1[0] + box1[2] / 2\r\n b1_y1, b1_y2 = box1[1] - box1[3] / 2, box1[1] + box1[3] / 2\r\n b2_x1, b2_x2 = box2[0] - box2[2] / 2, box2[0] + box2[2] / 2\r\n b2_y1, b2_y2 = box2[1] - box2[3] / 2, box2[1] + box2[3] / 2\r\n\r\n # Intersection area\r\n inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \\\r\n (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0)\r\n\r\n # Union Area\r\n w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps\r\n w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps\r\n union = w1 * h1 + w2 * h2 - inter + eps\r\n\r\n iou = inter / union\r\n\r\n if GIoU or DIoU or CIoU:\r\n cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1) # convex (smallest enclosing box) width\r\n ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height\r\n if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1\r\n c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared\r\n rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 +\r\n (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center distance squared\r\n if DIoU:\r\n return iou - rho2 / c2 # DIoU\r\n elif CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47\r\n v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / (h2 + eps)) - torch.atan(w1 / (h1 + eps)), 2)\r\n with torch.no_grad():\r\n alpha = v / (v - iou + (1 + eps))\r\n return iou - (rho2 / c2 + v * alpha) # CIoU\r\n else: # GIoU https://arxiv.org/pdf/1902.09630.pdf\r\n c_area = cw * ch + eps # convex area\r\n return iou - (c_area - union) / c_area # GIoU\r\n else:\r\n return iou # IoU\r\n\r\n\r\n\r\n\r\ndef bbox_alpha_iou(box1, box2, x1y1x2y2=False, GIoU=False, DIoU=False, CIoU=False, alpha=2, eps=1e-9):\r\n # Returns tsqrt_he IoU of box1 to box2. box1 is 4, box2 is nx4\r\n box2 = box2.T\r\n\r\n # Get the coordinates of bounding boxes\r\n if x1y1x2y2: # x1, y1, x2, y2 = box1\r\n b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3]\r\n b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3]\r\n else: # transform from xywh to xyxy\r\n b1_x1, b1_x2 = box1[0] - box1[2] / 2, box1[0] + box1[2] / 2\r\n b1_y1, b1_y2 = box1[1] - box1[3] / 2, box1[1] + box1[3] / 2\r\n b2_x1, b2_x2 = box2[0] - box2[2] / 2, box2[0] + box2[2] / 2\r\n b2_y1, b2_y2 = box2[1] - box2[3] / 2, box2[1] + box2[3] / 2\r\n\r\n # Intersection area\r\n inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \\\r\n (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0)\r\n\r\n # Union Area\r\n w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps\r\n w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps\r\n union = w1 * h1 + w2 * h2 - inter + eps\r\n\r\n # change iou into pow(iou+eps)\r\n # iou = inter / union\r\n iou = torch.pow(inter/union + eps, alpha)\r\n # beta = 2 * alpha\r\n if GIoU or DIoU or CIoU:\r\n cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1) # convex (smallest enclosing box) width\r\n ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height\r\n if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1\r\n c2 = (cw ** 2 + ch ** 2) ** alpha + eps # convex diagonal\r\n rho_x = torch.abs(b2_x1 + b2_x2 - b1_x1 - b1_x2)\r\n rho_y = torch.abs(b2_y1 + b2_y2 - b1_y1 - b1_y2)\r\n rho2 = ((rho_x ** 2 + rho_y ** 2) / 4) ** alpha # center distance\r\n if DIoU:\r\n return iou - rho2 / c2 # DIoU\r\n elif CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47\r\n v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2)\r\n with torch.no_grad():\r\n alpha_ciou = v / ((1 + eps) - inter / union + v)\r\n # return iou - (rho2 / c2 + v * alpha_ciou) # CIoU\r\n return iou - (rho2 / c2 + torch.pow(v * alpha_ciou + eps, alpha)) # CIoU\r\n else: # GIoU https://arxiv.org/pdf/1902.09630.pdf\r\n # c_area = cw * ch + eps # convex area\r\n # return iou - (c_area - union) / c_area # GIoU\r\n c_area = torch.max(cw * ch + eps, union) # convex area\r\n return iou - torch.pow((c_area - union) / c_area + eps, alpha) # GIoU\r\n else:\r\n return iou # torch.log(iou+eps) or iou\r\n\r\n\r\ndef box_iou(box1, box2):\r\n # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py\r\n \"\"\"\r\n Return intersection-over-union (Jaccard index) of boxes.\r\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\r\n Arguments:\r\n box1 (Tensor[N, 4])\r\n box2 (Tensor[M, 4])\r\n Returns:\r\n iou (Tensor[N, M]): the NxM matrix containing the pairwise\r\n IoU values for every element in boxes1 and boxes2\r\n \"\"\"\r\n\r\n def box_area(box):\r\n # box = 4xn\r\n return (box[2] - box[0]) * (box[3] - box[1])\r\n\r\n area1 = box_area(box1.T)\r\n area2 = box_area(box2.T)\r\n\r\n # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)\r\n inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2)\r\n return inter / (area1[:, None] + area2 - inter) # iou = inter / (area1 + area2 - inter)\r\n\r\n\r\ndef wh_iou(wh1, wh2):\r\n # Returns the nxm IoU matrix. wh1 is nx2, wh2 is mx2\r\n wh1 = wh1[:, None] # [N,1,2]\r\n wh2 = wh2[None] # [1,M,2]\r\n inter = torch.min(wh1, wh2).prod(2) # [N,M]\r\n return inter / (wh1.prod(2) + wh2.prod(2) - inter) # iou = inter / (area1 + area2 - inter)\r\n\r\n\r\ndef box_giou(box1, box2):\r\n \"\"\"\r\n Return generalized intersection-over-union (Jaccard index) between two sets of boxes.\r\n Both sets of boxes are expected to be in ``(x1, y1, x2, y2)`` format with\r\n ``0 <= x1 < x2`` and ``0 <= y1 < y2``.\r\n Args:\r\n boxes1 (Tensor[N, 4]): first set of boxes\r\n boxes2 (Tensor[M, 4]): second set of boxes\r\n Returns:\r\n Tensor[N, M]: the NxM matrix containing the pairwise generalized IoU values\r\n for every element in boxes1 and boxes2\r\n \"\"\"\r\n\r\n def box_area(box):\r\n # box = 4xn\r\n return (box[2] - box[0]) * (box[3] - box[1])\r\n\r\n area1 = box_area(box1.T)\r\n area2 = box_area(box2.T)\r\n \r\n inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2)\r\n union = (area1[:, None] + area2 - inter)\r\n\r\n iou = inter / union\r\n\r\n lti = torch.min(box1[:, None, :2], box2[:, :2])\r\n rbi = torch.max(box1[:, None, 2:], box2[:, 2:])\r\n\r\n whi = (rbi - lti).clamp(min=0) # [N,M,2]\r\n areai = whi[:, :, 0] * whi[:, :, 1]\r\n\r\n return iou - (areai - union) / areai\r\n\r\n\r\ndef box_ciou(box1, box2, eps: float = 1e-7):\r\n \"\"\"\r\n Return complete intersection-over-union (Jaccard index) between two sets of boxes.\r\n Both sets of boxes are expected to be in ``(x1, y1, x2, y2)`` format with\r\n ``0 <= x1 < x2`` and ``0 <= y1 < y2``.\r\n Args:\r\n boxes1 (Tensor[N, 4]): first set of boxes\r\n boxes2 (Tensor[M, 4]): second set of boxes\r\n eps (float, optional): small number to prevent division by zero. Default: 1e-7\r\n Returns:\r\n Tensor[N, M]: the NxM matrix containing the pairwise complete IoU values\r\n for every element in boxes1 and boxes2\r\n \"\"\"\r\n\r\n def box_area(box):\r\n # box = 4xn\r\n return (box[2] - box[0]) * (box[3] - box[1])\r\n\r\n area1 = box_area(box1.T)\r\n area2 = box_area(box2.T)\r\n \r\n inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2)\r\n union = (area1[:, None] + area2 - inter)\r\n\r\n iou = inter / union\r\n\r\n lti = torch.min(box1[:, None, :2], box2[:, :2])\r\n rbi = torch.max(box1[:, None, 2:], box2[:, 2:])\r\n\r\n whi = (rbi - lti).clamp(min=0) # [N,M,2]\r\n diagonal_distance_squared = (whi[:, :, 0] ** 2) + (whi[:, :, 1] ** 2) + eps\r\n\r\n # centers of boxes\r\n x_p = (box1[:, None, 0] + box1[:, None, 2]) / 2\r\n y_p = (box1[:, None, 1] + box1[:, None, 3]) / 2\r\n x_g = (box2[:, 0] + box2[:, 2]) / 2\r\n y_g = (box2[:, 1] + box2[:, 3]) / 2\r\n # The distance between boxes' centers squared.\r\n centers_distance_squared = (x_p - x_g) ** 2 + (y_p - y_g) ** 2\r\n\r\n w_pred = box1[:, None, 2] - box1[:, None, 0]\r\n h_pred = box1[:, None, 3] - box1[:, None, 1]\r\n\r\n w_gt = box2[:, 2] - box2[:, 0]\r\n h_gt = box2[:, 3] - box2[:, 1]\r\n\r\n v = (4 / (torch.pi ** 2)) * torch.pow((torch.atan(w_gt / h_gt) - torch.atan(w_pred / h_pred)), 2)\r\n with torch.no_grad():\r\n alpha = v / (1 - iou + v + eps)\r\n return iou - (centers_distance_squared / diagonal_distance_squared) - alpha * v\r\n\r\n\r\ndef box_diou(box1, box2, eps: float = 1e-7):\r\n \"\"\"\r\n Return distance intersection-over-union (Jaccard index) between two sets of boxes.\r\n Both sets of boxes are expected to be in ``(x1, y1, x2, y2)`` format with\r\n ``0 <= x1 < x2`` and ``0 <= y1 < y2``.\r\n Args:\r\n boxes1 (Tensor[N, 4]): first set of boxes\r\n boxes2 (Tensor[M, 4]): second set of boxes\r\n eps (float, optional): small number to prevent division by zero. Default: 1e-7\r\n Returns:\r\n Tensor[N, M]: the NxM matrix containing the pairwise distance IoU values\r\n for every element in boxes1 and boxes2\r\n \"\"\"\r\n\r\n def box_area(box):\r\n # box = 4xn\r\n return (box[2] - box[0]) * (box[3] - box[1])\r\n\r\n area1 = box_area(box1.T)\r\n area2 = box_area(box2.T)\r\n \r\n inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2)\r\n union = (area1[:, None] + area2 - inter)\r\n\r\n iou = inter / union\r\n\r\n lti = torch.min(box1[:, None, :2], box2[:, :2])\r\n rbi = torch.max(box1[:, None, 2:], box2[:, 2:])\r\n\r\n whi = (rbi - lti).clamp(min=0) # [N,M,2]\r\n diagonal_distance_squared = (whi[:, :, 0] ** 2) + (whi[:, :, 1] ** 2) + eps\r\n\r\n # centers of boxes\r\n x_p = (box1[:, None, 0] + box1[:, None, 2]) / 2\r\n y_p = (box1[:, None, 1] + box1[:, None, 3]) / 2\r\n x_g = (box2[:, 0] + box2[:, 2]) / 2\r\n y_g = (box2[:, 1] + box2[:, 3]) / 2\r\n # The distance between boxes' centers squared.\r\n centers_distance_squared = (x_p - x_g) ** 2 + (y_p - y_g) ** 2\r\n\r\n # The distance IoU is the IoU penalized by a normalized\r\n # distance between boxes' centers squared.\r\n return iou - (centers_distance_squared / diagonal_distance_squared)\r\n\r\n\r\ndef non_max_suppression(prediction, conf_thres=0.25, iou_thres=0.45, classes=None, agnostic=False, multi_label=False,\r\n labels=()):\r\n \"\"\"Runs Non-Maximum Suppression (NMS) on inference results\r\n\r\n Returns:\r\n list of detections, on (n,6) tensor per image [xyxy, conf, cls]\r\n \"\"\"\r\n\r\n nc = prediction.shape[2] - 5 # number of classes\r\n xc = prediction[..., 4] > conf_thres # candidates\r\n\r\n # Settings\r\n min_wh, max_wh = 2, 4096 # (pixels) minimum and maximum box width and height\r\n max_det = 300 # maximum number of detections per image\r\n max_nms = 30000 # maximum number of boxes into torchvision.ops.nms()\r\n time_limit = 10.0 # seconds to quit after\r\n redundant = True # require redundant detections\r\n multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img)\r\n merge = False # use merge-NMS\r\n\r\n t = time.time()\r\n output = [torch.zeros((0, 6), device=prediction.device)] * prediction.shape[0]\r\n for xi, x in enumerate(prediction): # image index, image inference\r\n # Apply constraints\r\n # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height\r\n x = x[xc[xi]] # confidence\r\n\r\n # Cat apriori labels if autolabelling\r\n if labels and len(labels[xi]):\r\n l = labels[xi]\r\n v = torch.zeros((len(l), nc + 5), device=x.device)\r\n v[:, :4] = l[:, 1:5] # box\r\n v[:, 4] = 1.0 # conf\r\n v[range(len(l)), l[:, 0].long() + 5] = 1.0 # cls\r\n x = torch.cat((x, v), 0)\r\n\r\n # If none remain process next image\r\n if not x.shape[0]:\r\n continue\r\n\r\n # Compute conf\r\n x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf\r\n\r\n # Box (center x, center y, width, height) to (x1, y1, x2, y2)\r\n box = xywh2xyxy(x[:, :4])\r\n\r\n # Detections matrix nx6 (xyxy, conf, cls)\r\n if multi_label:\r\n i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T\r\n x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1)\r\n else: # best class only\r\n conf, j = x[:, 5:].max(1, keepdim=True)\r\n x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres]\r\n\r\n # Filter by class\r\n if classes is not None:\r\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\r\n\r\n # Apply finite constraint\r\n # if not torch.isfinite(x).all():\r\n # x = x[torch.isfinite(x).all(1)]\r\n\r\n # Check shape\r\n n = x.shape[0] # number of boxes\r\n if not n: # no boxes\r\n continue\r\n elif n > max_nms: # excess boxes\r\n x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence\r\n\r\n # Batched NMS\r\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\r\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\r\n i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS\r\n if i.shape[0] > max_det: # limit detections\r\n i = i[:max_det]\r\n if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)\r\n # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\r\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\r\n weights = iou * scores[None] # box weights\r\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\r\n if redundant:\r\n i = i[iou.sum(1) > 1] # require redundancy\r\n\r\n output[xi] = x[i]\r\n if (time.time() - t) > time_limit:\r\n print(f'WARNING: NMS time limit {time_limit}s exceeded')\r\n break # time limit exceeded\r\n\r\n return output\r\n\r\n\r\ndef non_max_suppression_kpt(prediction, conf_thres=0.25, iou_thres=0.45, classes=None, agnostic=False, multi_label=False,\r\n labels=(), kpt_label=False, nc=None, nkpt=None):\r\n \"\"\"Runs Non-Maximum Suppression (NMS) on inference results\r\n\r\n Returns:\r\n list of detections, on (n,6) tensor per image [xyxy, conf, cls]\r\n \"\"\"\r\n if nc is None:\r\n nc = prediction.shape[2] - 5 if not kpt_label else prediction.shape[2] - 56 # number of classes\r\n xc = prediction[..., 4] > conf_thres # candidates\r\n\r\n # Settings\r\n min_wh, max_wh = 2, 4096 # (pixels) minimum and maximum box width and height\r\n max_det = 300 # maximum number of detections per image\r\n max_nms = 30000 # maximum number of boxes into torchvision.ops.nms()\r\n time_limit = 10.0 # seconds to quit after\r\n redundant = True # require redundant detections\r\n multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img)\r\n merge = False # use merge-NMS\r\n\r\n t = time.time()\r\n output = [torch.zeros((0,6), device=prediction.device)] * prediction.shape[0]\r\n for xi, x in enumerate(prediction): # image index, image inference\r\n # Apply constraints\r\n # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height\r\n x = x[xc[xi]] # confidence\r\n\r\n # Cat apriori labels if autolabelling\r\n if labels and len(labels[xi]):\r\n l = labels[xi]\r\n v = torch.zeros((len(l), nc + 5), device=x.device)\r\n v[:, :4] = l[:, 1:5] # box\r\n v[:, 4] = 1.0 # conf\r\n v[range(len(l)), l[:, 0].long() + 5] = 1.0 # cls\r\n x = torch.cat((x, v), 0)\r\n\r\n # If none remain process next image\r\n if not x.shape[0]:\r\n continue\r\n\r\n # Compute conf\r\n x[:, 5:5+nc] *= x[:, 4:5] # conf = obj_conf * cls_conf\r\n\r\n # Box (center x, center y, width, height) to (x1, y1, x2, y2)\r\n box = xywh2xyxy(x[:, :4])\r\n\r\n # Detections matrix nx6 (xyxy, conf, cls)\r\n if multi_label:\r\n i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T\r\n x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1)\r\n else: # best class only\r\n if not kpt_label:\r\n conf, j = x[:, 5:].max(1, keepdim=True)\r\n x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres]\r\n else:\r\n kpts = x[:, 6:]\r\n conf, j = x[:, 5:6].max(1, keepdim=True)\r\n x = torch.cat((box, conf, j.float(), kpts), 1)[conf.view(-1) > conf_thres]\r\n\r\n\r\n # Filter by class\r\n if classes is not None:\r\n x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]\r\n\r\n # Apply finite constraint\r\n # if not torch.isfinite(x).all():\r\n # x = x[torch.isfinite(x).all(1)]\r\n\r\n # Check shape\r\n n = x.shape[0] # number of boxes\r\n if not n: # no boxes\r\n continue\r\n elif n > max_nms: # excess boxes\r\n x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence\r\n\r\n # Batched NMS\r\n c = x[:, 5:6] * (0 if agnostic else max_wh) # classes\r\n boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores\r\n i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS\r\n if i.shape[0] > max_det: # limit detections\r\n i = i[:max_det]\r\n if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean)\r\n # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)\r\n iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix\r\n weights = iou * scores[None] # box weights\r\n x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes\r\n if redundant:\r\n i = i[iou.sum(1) > 1] # require redundancy\r\n\r\n output[xi] = x[i]\r\n if (time.time() - t) > time_limit:\r\n print(f'WARNING: NMS time limit {time_limit}s exceeded')\r\n break # time limit exceeded\r\n\r\n return output\r\n\r\n\r\ndef strip_optimizer(device='cpu',f='yolov7-w6-pose.pt', s=''): # from utils.general import *; strip_optimizer()\r\n # Strip optimizer from 'f' to finalize training, optionally save as 's'\r\n x = torch.load(f, map_location=torch.device('cpu'))\r\n if x.get('ema'):\r\n x['model'] = x['ema'] # replace model with ema\r\n for k in 'optimizer', 'training_results', 'wandb_id', 'ema', 'updates': # keys\r\n x[k] = None\r\n x['epoch'] = -1\r\n if device!='cpu':\r\n x['model'].half() # to FP16\r\n else:\r\n x['model'].float()\r\n for p in x['model'].parameters():\r\n p.requires_grad = False\r\n torch.save(x, s or f)\r\n mb = os.path.getsize(s or f) / 1E6 # filesize\r\n print(f\"Optimizer stripped from {f},{(' saved as %s,' % s) if s else ''} {mb:.1f}MB\")\r\n\r\n\r\ndef print_mutation(hyp, results, yaml_file='hyp_evolved.yaml', bucket=''):\r\n # Print mutation results to evolve.txt (for use with train.py --evolve)\r\n a = '%10s' * len(hyp) % tuple(hyp.keys()) # hyperparam keys\r\n b = '%10.3g' * len(hyp) % tuple(hyp.values()) # hyperparam values\r\n c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3)\r\n print('\\n%s\\n%s\\nEvolved fitness: %s\\n' % (a, b, c))\r\n\r\n if bucket:\r\n url = 'gs://%s/evolve.txt' % bucket\r\n if gsutil_getsize(url) > (os.path.getsize('evolve.txt') if os.path.exists('evolve.txt') else 0):\r\n os.system('gsutil cp %s .' % url) # download evolve.txt if larger than local\r\n\r\n with open('evolve.txt', 'a') as f: # append result\r\n f.write(c + b + '\\n')\r\n x = np.unique(np.loadtxt('evolve.txt', ndmin=2), axis=0) # load unique rows\r\n x = x[np.argsort(-fitness(x))] # sort\r\n np.savetxt('evolve.txt', x, '%10.3g') # save sort by fitness\r\n\r\n # Save yaml\r\n for i, k in enumerate(hyp.keys()):\r\n hyp[k] = float(x[0, i + 7])\r\n with open(yaml_file, 'w') as f:\r\n results = tuple(x[0, :7])\r\n c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3)\r\n f.write('# Hyperparameter Evolution Results\\n# Generations: %g\\n# Metrics: ' % len(x) + c + '\\n\\n')\r\n yaml.dump(hyp, f, sort_keys=False)\r\n\r\n if bucket:\r\n os.system('gsutil cp evolve.txt %s gs://%s' % (yaml_file, bucket)) # upload\r\n\r\n\r\ndef apply_classifier(x, model, img, im0):\r\n # applies a second stage classifier to yolo outputs\r\n im0 = [im0] if isinstance(im0, np.ndarray) else im0\r\n for i, d in enumerate(x): # per image\r\n if d is not None and len(d):\r\n d = d.clone()\r\n\r\n # Reshape and pad cutouts\r\n b = xyxy2xywh(d[:, :4]) # boxes\r\n b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # rectangle to square\r\n b[:, 2:] = b[:, 2:] * 1.3 + 30 # pad\r\n d[:, :4] = xywh2xyxy(b).long()\r\n\r\n # Rescale boxes from img_size to im0 size\r\n scale_coords(img.shape[2:], d[:, :4], im0[i].shape)\r\n\r\n # Classes\r\n pred_cls1 = d[:, 5].long()\r\n ims = []\r\n for j, a in enumerate(d): # per item\r\n cutout = im0[i][int(a[1]):int(a[3]), int(a[0]):int(a[2])]\r\n im = cv2.resize(cutout, (224, 224)) # BGR\r\n # cv2.imwrite('test%i.jpg' % j, cutout)\r\n\r\n im = im[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416\r\n im = np.ascontiguousarray(im, dtype=np.float32) # uint8 to float32\r\n im /= 255.0 # 0 - 255 to 0.0 - 1.0\r\n ims.append(im)\r\n\r\n pred_cls2 = model(torch.Tensor(ims).to(d.device)).argmax(1) # classifier prediction\r\n x[i] = x[i][pred_cls1 == pred_cls2] # retain matching class detections\r\n\r\n return x\r\n\r\n\r\ndef increment_path(path, exist_ok=True, sep=''):\r\n # Increment path, i.e. runs/exp --> runs/exp{sep}0, runs/exp{sep}1 etc.\r\n path = Path(path) # os-agnostic\r\n if (path.exists() and exist_ok) or (not path.exists()):\r\n return str(path)\r\n else:\r\n dirs = glob.glob(f\"{path}{sep}*\") # similar paths\r\n matches = [re.search(rf\"%s{sep}(\\d+)\" % path.stem, d) for d in dirs]\r\n i = [int(m.groups()[0]) for m in matches if m] # indices\r\n n = max(i) + 1 if i else 2 # increment number\r\n return f\"{path}{sep}{n}\" # update path\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/google_utils.py", "content": "# Google utils: https://cloud.google.com/storage/docs/reference/libraries\r\n\r\nimport os\r\nimport platform\r\nimport subprocess\r\nimport time\r\nfrom pathlib import Path\r\n\r\nimport requests\r\nimport torch\r\n\r\n\r\ndef gsutil_getsize(url=''):\r\n # gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du\r\n s = subprocess.check_output(f'gsutil du {url}', shell=True).decode('utf-8')\r\n return eval(s.split(' ')[0]) if len(s) else 0 # bytes\r\n\r\n\r\ndef attempt_download(file, repo='WongKinYiu/yolov7'):\r\n # Attempt file download if does not exist\r\n file = Path(str(file).strip().replace(\"'\", '').lower())\r\n\r\n if not file.exists():\r\n try:\r\n response = requests.get(f'https://api.github.com/repos/{repo}/releases/latest').json() # github api\r\n assets = [x['name'] for x in response['assets']] # release assets\r\n tag = response['tag_name'] # i.e. 'v1.0'\r\n except: # fallback plan\r\n assets = ['yolov7.pt']\r\n tag = subprocess.check_output('git tag', shell=True).decode().split()[-1]\r\n\r\n name = file.name\r\n if name in assets:\r\n msg = f'{file} missing, try downloading from https://github.com/{repo}/releases/'\r\n redundant = False # second download option\r\n try: # GitHub\r\n url = f'https://github.com/{repo}/releases/download/{tag}/{name}'\r\n print(f'Downloading {url} to {file}...')\r\n torch.hub.download_url_to_file(url, file)\r\n assert file.exists() and file.stat().st_size > 1E6 # check\r\n except Exception as e: # GCP\r\n print(f'Download error: {e}')\r\n assert redundant, 'No secondary mirror'\r\n url = f'https://storage.googleapis.com/{repo}/ckpt/{name}'\r\n print(f'Downloading {url} to {file}...')\r\n os.system(f'curl -L {url} -o {file}') # torch.hub.download_url_to_file(url, weights)\r\n finally:\r\n if not file.exists() or file.stat().st_size < 1E6: # check\r\n file.unlink(missing_ok=True) # remove partial downloads\r\n print(f'ERROR: Download failure: {msg}')\r\n print('')\r\n return\r\n\r\n\r\ndef gdrive_download(id='', file='tmp.zip'):\r\n # Downloads a file from Google Drive. from yolov7.utils.google_utils import *; gdrive_download()\r\n t = time.time()\r\n file = Path(file)\r\n cookie = Path('cookie') # gdrive cookie\r\n print(f'Downloading https://drive.google.com/uc?export=download&id={id} as {file}... ', end='')\r\n file.unlink(missing_ok=True) # remove existing file\r\n cookie.unlink(missing_ok=True) # remove existing cookie\r\n\r\n # Attempt file download\r\n out = \"NUL\" if platform.system() == \"Windows\" else \"/dev/null\"\r\n os.system(f'curl -c ./cookie -s -L \"drive.google.com/uc?export=download&id={id}\" > {out}')\r\n if os.path.exists('cookie'): # large file\r\n s = f'curl -Lb ./cookie \"drive.google.com/uc?export=download&confirm={get_token()}&id={id}\" -o {file}'\r\n else: # small file\r\n s = f'curl -s -L -o {file} \"drive.google.com/uc?export=download&id={id}\"'\r\n r = os.system(s) # execute, capture return\r\n cookie.unlink(missing_ok=True) # remove existing cookie\r\n\r\n # Error check\r\n if r != 0:\r\n file.unlink(missing_ok=True) # remove partial\r\n print('Download error ') # raise Exception('Download error')\r\n return r\r\n\r\n # Unzip if archive\r\n if file.suffix == '.zip':\r\n print('unzipping... ', end='')\r\n os.system(f'unzip -q {file}') # unzip\r\n file.unlink() # remove zip to free space\r\n\r\n print(f'Done ({time.time() - t:.1f}s)')\r\n return r\r\n\r\n\r\ndef get_token(cookie=\"./cookie\"):\r\n with open(cookie) as f:\r\n for line in f:\r\n if \"download\" in line:\r\n return line.split()[-1]\r\n return \"\"\r\n\r\n# def upload_blob(bucket_name, source_file_name, destination_blob_name):\r\n# # Uploads a file to a bucket\r\n# # https://cloud.google.com/storage/docs/uploading-objects#storage-upload-object-python\r\n#\r\n# storage_client = storage.Client()\r\n# bucket = storage_client.get_bucket(bucket_name)\r\n# blob = bucket.blob(destination_blob_name)\r\n#\r\n# blob.upload_from_filename(source_file_name)\r\n#\r\n# print('File {} uploaded to {}.'.format(\r\n# source_file_name,\r\n# destination_blob_name))\r\n#\r\n#\r\n# def download_blob(bucket_name, source_blob_name, destination_file_name):\r\n# # Uploads a blob from a bucket\r\n# storage_client = storage.Client()\r\n# bucket = storage_client.get_bucket(bucket_name)\r\n# blob = bucket.blob(source_blob_name)\r\n#\r\n# blob.download_to_filename(destination_file_name)\r\n#\r\n# print('Blob {} downloaded to {}.'.format(\r\n# source_blob_name,\r\n# destination_file_name))\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/loss.py", "content": "# Loss functions\r\n\r\nimport torch\r\nimport torch.nn as nn\r\nimport torch.nn.functional as F\r\n\r\nfrom asone.pose_estimators.yolov7_pose.utils.general import bbox_iou, bbox_alpha_iou, box_iou, box_giou, box_diou, box_ciou, xywh2xyxy\r\nfrom asone.pose_estimators.yolov7_pose.utils.torch_utils import is_parallel\r\n\r\n\r\ndef smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441\r\n # return positive, negative label smoothing BCE targets\r\n return 1.0 - 0.5 * eps, 0.5 * eps\r\n\r\n\r\nclass BCEBlurWithLogitsLoss(nn.Module):\r\n # BCEwithLogitLoss() with reduced missing label effects.\r\n def __init__(self, alpha=0.05):\r\n super(BCEBlurWithLogitsLoss, self).__init__()\r\n self.loss_fcn = nn.BCEWithLogitsLoss(reduction='none') # must be nn.BCEWithLogitsLoss()\r\n self.alpha = alpha\r\n\r\n def forward(self, pred, true):\r\n loss = self.loss_fcn(pred, true)\r\n pred = torch.sigmoid(pred) # prob from logits\r\n dx = pred - true # reduce only missing label effects\r\n # dx = (pred - true).abs() # reduce missing label and false label effects\r\n alpha_factor = 1 - torch.exp((dx - 1) / (self.alpha + 1e-4))\r\n loss *= alpha_factor\r\n return loss.mean()\r\n\r\n\r\nclass SigmoidBin(nn.Module):\r\n stride = None # strides computed during build\r\n export = False # onnx export\r\n\r\n def __init__(self, bin_count=10, min=0.0, max=1.0, reg_scale = 2.0, use_loss_regression=True, use_fw_regression=True, BCE_weight=1.0, smooth_eps=0.0):\r\n super(SigmoidBin, self).__init__()\r\n \r\n self.bin_count = bin_count\r\n self.length = bin_count + 1\r\n self.min = min\r\n self.max = max\r\n self.scale = float(max - min)\r\n self.shift = self.scale / 2.0\r\n\r\n self.use_loss_regression = use_loss_regression\r\n self.use_fw_regression = use_fw_regression\r\n self.reg_scale = reg_scale\r\n self.BCE_weight = BCE_weight\r\n\r\n start = min + (self.scale/2.0) / self.bin_count\r\n end = max - (self.scale/2.0) / self.bin_count\r\n step = self.scale / self.bin_count\r\n self.step = step\r\n #print(f\" start = {start}, end = {end}, step = {step} \")\r\n\r\n bins = torch.range(start, end + 0.0001, step).float() \r\n self.register_buffer('bins', bins) \r\n \r\n\r\n self.cp = 1.0 - 0.5 * smooth_eps\r\n self.cn = 0.5 * smooth_eps\r\n\r\n self.BCEbins = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([BCE_weight]))\r\n self.MSELoss = nn.MSELoss()\r\n\r\n def get_length(self):\r\n return self.length\r\n\r\n def forward(self, pred):\r\n assert pred.shape[-1] == self.length, 'pred.shape[-1]=%d is not equal to self.length=%d' % (pred.shape[-1], self.length)\r\n\r\n pred_reg = (pred[..., 0] * self.reg_scale - self.reg_scale/2.0) * self.step\r\n pred_bin = pred[..., 1:(1+self.bin_count)]\r\n\r\n _, bin_idx = torch.max(pred_bin, dim=-1)\r\n bin_bias = self.bins[bin_idx]\r\n\r\n if self.use_fw_regression:\r\n result = pred_reg + bin_bias\r\n else:\r\n result = bin_bias\r\n result = result.clamp(min=self.min, max=self.max)\r\n\r\n return result\r\n\r\n\r\n def training_loss(self, pred, target):\r\n assert pred.shape[-1] == self.length, 'pred.shape[-1]=%d is not equal to self.length=%d' % (pred.shape[-1], self.length)\r\n assert pred.shape[0] == target.shape[0], 'pred.shape=%d is not equal to the target.shape=%d' % (pred.shape[0], target.shape[0])\r\n device = pred.device\r\n\r\n pred_reg = (pred[..., 0].sigmoid() * self.reg_scale - self.reg_scale/2.0) * self.step\r\n pred_bin = pred[..., 1:(1+self.bin_count)]\r\n\r\n diff_bin_target = torch.abs(target[..., None] - self.bins)\r\n _, bin_idx = torch.min(diff_bin_target, dim=-1)\r\n \r\n bin_bias = self.bins[bin_idx]\r\n bin_bias.requires_grad = False\r\n result = pred_reg + bin_bias\r\n\r\n target_bins = torch.full_like(pred_bin, self.cn, device=device) # targets\r\n n = pred.shape[0] \r\n target_bins[range(n), bin_idx] = self.cp\r\n\r\n loss_bin = self.BCEbins(pred_bin, target_bins) # BCE\r\n\r\n if self.use_loss_regression:\r\n loss_regression = self.MSELoss(result, target) # MSE \r\n loss = loss_bin + loss_regression\r\n else:\r\n loss = loss_bin\r\n\r\n out_result = result.clamp(min=self.min, max=self.max)\r\n\r\n return loss, out_result\r\n\r\n\r\nclass FocalLoss(nn.Module):\r\n # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\r\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\r\n super(FocalLoss, self).__init__()\r\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\r\n self.gamma = gamma\r\n self.alpha = alpha\r\n self.reduction = loss_fcn.reduction\r\n self.loss_fcn.reduction = 'none' # required to apply FL to each element\r\n\r\n def forward(self, pred, true):\r\n loss = self.loss_fcn(pred, true)\r\n # p_t = torch.exp(-loss)\r\n # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability\r\n\r\n # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py\r\n pred_prob = torch.sigmoid(pred) # prob from logits\r\n p_t = true * pred_prob + (1 - true) * (1 - pred_prob)\r\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\r\n modulating_factor = (1.0 - p_t) ** self.gamma\r\n loss *= alpha_factor * modulating_factor\r\n\r\n if self.reduction == 'mean':\r\n return loss.mean()\r\n elif self.reduction == 'sum':\r\n return loss.sum()\r\n else: # 'none'\r\n return loss\r\n\r\n\r\nclass QFocalLoss(nn.Module):\r\n # Wraps Quality focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5)\r\n def __init__(self, loss_fcn, gamma=1.5, alpha=0.25):\r\n super(QFocalLoss, self).__init__()\r\n self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss()\r\n self.gamma = gamma\r\n self.alpha = alpha\r\n self.reduction = loss_fcn.reduction\r\n self.loss_fcn.reduction = 'none' # required to apply FL to each element\r\n\r\n def forward(self, pred, true):\r\n loss = self.loss_fcn(pred, true)\r\n\r\n pred_prob = torch.sigmoid(pred) # prob from logits\r\n alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha)\r\n modulating_factor = torch.abs(true - pred_prob) ** self.gamma\r\n loss *= alpha_factor * modulating_factor\r\n\r\n if self.reduction == 'mean':\r\n return loss.mean()\r\n elif self.reduction == 'sum':\r\n return loss.sum()\r\n else: # 'none'\r\n return loss\r\n\r\nclass RankSort(torch.autograd.Function):\r\n @staticmethod\r\n def forward(ctx, logits, targets, delta_RS=0.50, eps=1e-10): \r\n\r\n classification_grads=torch.zeros(logits.shape).cuda()\r\n \r\n #Filter fg logits\r\n fg_labels = (targets > 0.)\r\n fg_logits = logits[fg_labels]\r\n fg_targets = targets[fg_labels]\r\n fg_num = len(fg_logits)\r\n\r\n #Do not use bg with scores less than minimum fg logit\r\n #since changing its score does not have an effect on precision\r\n threshold_logit = torch.min(fg_logits)-delta_RS\r\n relevant_bg_labels=((targets==0) & (logits>=threshold_logit))\r\n \r\n relevant_bg_logits = logits[relevant_bg_labels] \r\n relevant_bg_grad=torch.zeros(len(relevant_bg_logits)).cuda()\r\n sorting_error=torch.zeros(fg_num).cuda()\r\n ranking_error=torch.zeros(fg_num).cuda()\r\n fg_grad=torch.zeros(fg_num).cuda()\r\n \r\n #sort the fg logits\r\n order=torch.argsort(fg_logits)\r\n #Loops over each positive following the order\r\n for ii in order:\r\n # Difference Transforms (x_ij)\r\n fg_relations=fg_logits-fg_logits[ii] \r\n bg_relations=relevant_bg_logits-fg_logits[ii]\r\n\r\n if delta_RS > 0:\r\n fg_relations=torch.clamp(fg_relations/(2*delta_RS)+0.5,min=0,max=1)\r\n bg_relations=torch.clamp(bg_relations/(2*delta_RS)+0.5,min=0,max=1)\r\n else:\r\n fg_relations = (fg_relations >= 0).float()\r\n bg_relations = (bg_relations >= 0).float()\r\n\r\n # Rank of ii among pos and false positive number (bg with larger scores)\r\n rank_pos=torch.sum(fg_relations)\r\n FP_num=torch.sum(bg_relations)\r\n\r\n # Rank of ii among all examples\r\n rank=rank_pos+FP_num\r\n \r\n # Ranking error of example ii. target_ranking_error is always 0. (Eq. 7)\r\n ranking_error[ii]=FP_num/rank \r\n\r\n # Current sorting error of example ii. (Eq. 7)\r\n current_sorting_error = torch.sum(fg_relations*(1-fg_targets))/rank_pos\r\n\r\n #Find examples in the target sorted order for example ii \r\n iou_relations = (fg_targets >= fg_targets[ii])\r\n target_sorted_order = iou_relations * fg_relations\r\n\r\n #The rank of ii among positives in sorted order\r\n rank_pos_target = torch.sum(target_sorted_order)\r\n\r\n #Compute target sorting error. (Eq. 8)\r\n #Since target ranking error is 0, this is also total target error \r\n target_sorting_error= torch.sum(target_sorted_order*(1-fg_targets))/rank_pos_target\r\n\r\n #Compute sorting error on example ii\r\n sorting_error[ii] = current_sorting_error - target_sorting_error\r\n \r\n #Identity Update for Ranking Error \r\n if FP_num > eps:\r\n #For ii the update is the ranking error\r\n fg_grad[ii] -= ranking_error[ii]\r\n #For negatives, distribute error via ranking pmf (i.e. bg_relations/FP_num)\r\n relevant_bg_grad += (bg_relations*(ranking_error[ii]/FP_num))\r\n\r\n #Find the positives that are misranked (the cause of the error)\r\n #These are the ones with smaller IoU but larger logits\r\n missorted_examples = (~ iou_relations) * fg_relations\r\n\r\n #Denominotor of sorting pmf \r\n sorting_pmf_denom = torch.sum(missorted_examples)\r\n\r\n #Identity Update for Sorting Error \r\n if sorting_pmf_denom > eps:\r\n #For ii the update is the sorting error\r\n fg_grad[ii] -= sorting_error[ii]\r\n #For positives, distribute error via sorting pmf (i.e. missorted_examples/sorting_pmf_denom)\r\n fg_grad += (missorted_examples*(sorting_error[ii]/sorting_pmf_denom))\r\n\r\n #Normalize gradients by number of positives \r\n classification_grads[fg_labels]= (fg_grad/fg_num)\r\n classification_grads[relevant_bg_labels]= (relevant_bg_grad/fg_num)\r\n\r\n ctx.save_for_backward(classification_grads)\r\n\r\n return ranking_error.mean(), sorting_error.mean()\r\n\r\n @staticmethod\r\n def backward(ctx, out_grad1, out_grad2):\r\n g1, =ctx.saved_tensors\r\n return g1*out_grad1, None, None, None\r\n\r\nclass aLRPLoss(torch.autograd.Function):\r\n @staticmethod\r\n def forward(ctx, logits, targets, regression_losses, delta=1., eps=1e-5): \r\n classification_grads=torch.zeros(logits.shape).cuda()\r\n \r\n #Filter fg logits\r\n fg_labels = (targets == 1)\r\n fg_logits = logits[fg_labels]\r\n fg_num = len(fg_logits)\r\n\r\n #Do not use bg with scores less than minimum fg logit\r\n #since changing its score does not have an effect on precision\r\n threshold_logit = torch.min(fg_logits)-delta\r\n\r\n #Get valid bg logits\r\n relevant_bg_labels=((targets==0)&(logits>=threshold_logit))\r\n relevant_bg_logits=logits[relevant_bg_labels] \r\n relevant_bg_grad=torch.zeros(len(relevant_bg_logits)).cuda()\r\n rank=torch.zeros(fg_num).cuda()\r\n prec=torch.zeros(fg_num).cuda()\r\n fg_grad=torch.zeros(fg_num).cuda()\r\n \r\n max_prec=0 \r\n #sort the fg logits\r\n order=torch.argsort(fg_logits)\r\n #Loops over each positive following the order\r\n for ii in order:\r\n #x_ij s as score differences with fgs\r\n fg_relations=fg_logits-fg_logits[ii] \r\n #Apply piecewise linear function and determine relations with fgs\r\n fg_relations=torch.clamp(fg_relations/(2*delta)+0.5,min=0,max=1)\r\n #Discard i=j in the summation in rank_pos\r\n fg_relations[ii]=0\r\n\r\n #x_ij s as score differences with bgs\r\n bg_relations=relevant_bg_logits-fg_logits[ii]\r\n #Apply piecewise linear function and determine relations with bgs\r\n bg_relations=torch.clamp(bg_relations/(2*delta)+0.5,min=0,max=1)\r\n\r\n #Compute the rank of the example within fgs and number of bgs with larger scores\r\n rank_pos=1+torch.sum(fg_relations)\r\n FP_num=torch.sum(bg_relations)\r\n #Store the total since it is normalizer also for aLRP Regression error\r\n rank[ii]=rank_pos+FP_num\r\n \r\n #Compute precision for this example to compute classification loss \r\n prec[ii]=rank_pos/rank[ii] \r\n #For stability, set eps to a infinitesmall value (e.g. 1e-6), then compute grads\r\n if FP_num > eps: \r\n fg_grad[ii] = -(torch.sum(fg_relations*regression_losses)+FP_num)/rank[ii]\r\n relevant_bg_grad += (bg_relations*(-fg_grad[ii]/FP_num)) \r\n \r\n #aLRP with grad formulation fg gradient\r\n classification_grads[fg_labels]= fg_grad\r\n #aLRP with grad formulation bg gradient\r\n classification_grads[relevant_bg_labels]= relevant_bg_grad \r\n \r\n classification_grads /= (fg_num)\r\n \r\n cls_loss=1-prec.mean()\r\n ctx.save_for_backward(classification_grads)\r\n\r\n return cls_loss, rank, order\r\n\r\n @staticmethod\r\n def backward(ctx, out_grad1, out_grad2, out_grad3):\r\n g1, =ctx.saved_tensors\r\n return g1*out_grad1, None, None, None, None\r\n \r\n \r\nclass APLoss(torch.autograd.Function):\r\n @staticmethod\r\n def forward(ctx, logits, targets, delta=1.): \r\n classification_grads=torch.zeros(logits.shape).cuda()\r\n \r\n #Filter fg logits\r\n fg_labels = (targets == 1)\r\n fg_logits = logits[fg_labels]\r\n fg_num = len(fg_logits)\r\n\r\n #Do not use bg with scores less than minimum fg logit\r\n #since changing its score does not have an effect on precision\r\n threshold_logit = torch.min(fg_logits)-delta\r\n\r\n #Get valid bg logits\r\n relevant_bg_labels=((targets==0)&(logits>=threshold_logit))\r\n relevant_bg_logits=logits[relevant_bg_labels] \r\n relevant_bg_grad=torch.zeros(len(relevant_bg_logits)).cuda()\r\n rank=torch.zeros(fg_num).cuda()\r\n prec=torch.zeros(fg_num).cuda()\r\n fg_grad=torch.zeros(fg_num).cuda()\r\n \r\n max_prec=0 \r\n #sort the fg logits\r\n order=torch.argsort(fg_logits)\r\n #Loops over each positive following the order\r\n for ii in order:\r\n #x_ij s as score differences with fgs\r\n fg_relations=fg_logits-fg_logits[ii] \r\n #Apply piecewise linear function and determine relations with fgs\r\n fg_relations=torch.clamp(fg_relations/(2*delta)+0.5,min=0,max=1)\r\n #Discard i=j in the summation in rank_pos\r\n fg_relations[ii]=0\r\n\r\n #x_ij s as score differences with bgs\r\n bg_relations=relevant_bg_logits-fg_logits[ii]\r\n #Apply piecewise linear function and determine relations with bgs\r\n bg_relations=torch.clamp(bg_relations/(2*delta)+0.5,min=0,max=1)\r\n\r\n #Compute the rank of the example within fgs and number of bgs with larger scores\r\n rank_pos=1+torch.sum(fg_relations)\r\n FP_num=torch.sum(bg_relations)\r\n #Store the total since it is normalizer also for aLRP Regression error\r\n rank[ii]=rank_pos+FP_num\r\n \r\n #Compute precision for this example \r\n current_prec=rank_pos/rank[ii]\r\n \r\n #Compute interpolated AP and store gradients for relevant bg examples\r\n if (max_prec<=current_prec):\r\n max_prec=current_prec\r\n relevant_bg_grad += (bg_relations/rank[ii])\r\n else:\r\n relevant_bg_grad += (bg_relations/rank[ii])*(((1-max_prec)/(1-current_prec)))\r\n \r\n #Store fg gradients\r\n fg_grad[ii]=-(1-max_prec)\r\n prec[ii]=max_prec \r\n\r\n #aLRP with grad formulation fg gradient\r\n classification_grads[fg_labels]= fg_grad\r\n #aLRP with grad formulation bg gradient\r\n classification_grads[relevant_bg_labels]= relevant_bg_grad \r\n \r\n classification_grads /= fg_num\r\n \r\n cls_loss=1-prec.mean()\r\n ctx.save_for_backward(classification_grads)\r\n\r\n return cls_loss\r\n\r\n @staticmethod\r\n def backward(ctx, out_grad1):\r\n g1, =ctx.saved_tensors\r\n return g1*out_grad1, None, None\r\n\r\n\r\nclass ComputeLoss:\r\n # Compute losses\r\n def __init__(self, model, autobalance=False):\r\n super(ComputeLoss, self).__init__()\r\n device = next(model.parameters()).device # get model device\r\n h = model.hyp # hyperparameters\r\n\r\n # Define criteria\r\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device))\r\n BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device))\r\n\r\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\r\n self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets\r\n\r\n # Focal loss\r\n g = h['fl_gamma'] # focal loss gamma\r\n if g > 0:\r\n BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)\r\n\r\n det = model.module.model[-1] if is_parallel(model) else model.model[-1] # Detect() module\r\n self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.25, 0.06, .02]) # P3-P7\r\n #self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.25, 0.1, .05]) # P3-P7\r\n #self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.5, 0.4, .1]) # P3-P7\r\n self.ssi = list(det.stride).index(16) if autobalance else 0 # stride 16 index\r\n self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, model.gr, h, autobalance\r\n for k in 'na', 'nc', 'nl', 'anchors':\r\n setattr(self, k, getattr(det, k))\r\n\r\n def __call__(self, p, targets): # predictions, targets, model\r\n device = targets.device\r\n lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)\r\n tcls, tbox, indices, anchors = self.build_targets(p, targets) # targets\r\n\r\n # Losses\r\n for i, pi in enumerate(p): # layer index, layer predictions\r\n b, a, gj, gi = indices[i] # image, anchor, gridy, gridx\r\n tobj = torch.zeros_like(pi[..., 0], device=device) # target obj\r\n\r\n n = b.shape[0] # number of targets\r\n if n:\r\n ps = pi[b, a, gj, gi] # prediction subset corresponding to targets\r\n\r\n # Regression\r\n pxy = ps[:, :2].sigmoid() * 2. - 0.5\r\n pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]\r\n pbox = torch.cat((pxy, pwh), 1) # predicted box\r\n iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, CIoU=True) # iou(prediction, target)\r\n lbox += (1.0 - iou).mean() # iou loss\r\n\r\n # Objectness\r\n tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio\r\n\r\n # Classification\r\n if self.nc > 1: # cls loss (only if multiple classes)\r\n t = torch.full_like(ps[:, 5:], self.cn, device=device) # targets\r\n t[range(n), tcls[i]] = self.cp\r\n #t[t==self.cp] = iou.detach().clamp(0).type(t.dtype)\r\n lcls += self.BCEcls(ps[:, 5:], t) # BCE\r\n\r\n # Append targets to text file\r\n # with open('targets.txt', 'a') as file:\r\n # [file.write('%11.5g ' * 4 % tuple(x) + '\\n') for x in torch.cat((txy[i], twh[i]), 1)]\r\n\r\n obji = self.BCEobj(pi[..., 4], tobj)\r\n lobj += obji * self.balance[i] # obj loss\r\n if self.autobalance:\r\n self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item()\r\n\r\n if self.autobalance:\r\n self.balance = [x / self.balance[self.ssi] for x in self.balance]\r\n lbox *= self.hyp['box']\r\n lobj *= self.hyp['obj']\r\n lcls *= self.hyp['cls']\r\n bs = tobj.shape[0] # batch size\r\n\r\n loss = lbox + lobj + lcls\r\n return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()\r\n\r\n def build_targets(self, p, targets):\r\n # Build targets for compute_loss(), input targets(image,class,x,y,w,h)\r\n na, nt = self.na, targets.shape[0] # number of anchors, targets\r\n tcls, tbox, indices, anch = [], [], [], []\r\n gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain\r\n ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt)\r\n targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices\r\n\r\n g = 0.5 # bias\r\n off = torch.tensor([[0, 0],\r\n [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m\r\n # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm\r\n ], device=targets.device).float() * g # offsets\r\n\r\n for i in range(self.nl):\r\n anchors = self.anchors[i]\r\n gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain\r\n\r\n # Match targets to anchors\r\n t = targets * gain\r\n if nt:\r\n # Matches\r\n r = t[:, :, 4:6] / anchors[:, None] # wh ratio\r\n j = torch.max(r, 1. / r).max(2)[0] < self.hyp['anchor_t'] # compare\r\n # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2))\r\n t = t[j] # filter\r\n\r\n # Offsets\r\n gxy = t[:, 2:4] # grid xy\r\n gxi = gain[[2, 3]] - gxy # inverse\r\n j, k = ((gxy % 1. < g) & (gxy > 1.)).T\r\n l, m = ((gxi % 1. < g) & (gxi > 1.)).T\r\n j = torch.stack((torch.ones_like(j), j, k, l, m))\r\n t = t.repeat((5, 1, 1))[j]\r\n offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j]\r\n else:\r\n t = targets[0]\r\n offsets = 0\r\n\r\n # Define\r\n b, c = t[:, :2].long().T # image, class\r\n gxy = t[:, 2:4] # grid xy\r\n gwh = t[:, 4:6] # grid wh\r\n gij = (gxy - offsets).long()\r\n gi, gj = gij.T # grid xy indices\r\n\r\n # Append\r\n a = t[:, 6].long() # anchor indices\r\n indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices\r\n tbox.append(torch.cat((gxy - gij, gwh), 1)) # box\r\n anch.append(anchors[a]) # anchors\r\n tcls.append(c) # class\r\n\r\n return tcls, tbox, indices, anch\r\n\r\n\r\nclass ComputeLossOTA:\r\n # Compute losses\r\n def __init__(self, model, autobalance=False):\r\n super(ComputeLossOTA, self).__init__()\r\n device = next(model.parameters()).device # get model device\r\n h = model.hyp # hyperparameters\r\n\r\n # Define criteria\r\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device))\r\n BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device))\r\n\r\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\r\n self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets\r\n\r\n # Focal loss\r\n g = h['fl_gamma'] # focal loss gamma\r\n if g > 0:\r\n BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)\r\n\r\n det = model.module.model[-1] if is_parallel(model) else model.model[-1] # Detect() module\r\n self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.25, 0.06, .02]) # P3-P7\r\n self.ssi = list(det.stride).index(16) if autobalance else 0 # stride 16 index\r\n self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, model.gr, h, autobalance\r\n for k in 'na', 'nc', 'nl', 'anchors', 'stride':\r\n setattr(self, k, getattr(det, k))\r\n\r\n def __call__(self, p, targets, imgs): # predictions, targets, model \r\n device = targets.device\r\n lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)\r\n bs, as_, gjs, gis, targets, anchors = self.build_targets(p, targets, imgs)\r\n pre_gen_gains = [torch.tensor(pp.shape, device=device)[[3, 2, 3, 2]] for pp in p] \r\n \r\n\r\n # Losses\r\n for i, pi in enumerate(p): # layer index, layer predictions\r\n b, a, gj, gi = bs[i], as_[i], gjs[i], gis[i] # image, anchor, gridy, gridx\r\n tobj = torch.zeros_like(pi[..., 0], device=device) # target obj\r\n\r\n n = b.shape[0] # number of targets\r\n if n:\r\n ps = pi[b, a, gj, gi] # prediction subset corresponding to targets\r\n\r\n # Regression\r\n grid = torch.stack([gi, gj], dim=1)\r\n pxy = ps[:, :2].sigmoid() * 2. - 0.5\r\n #pxy = ps[:, :2].sigmoid() * 3. - 1.\r\n pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]\r\n pbox = torch.cat((pxy, pwh), 1) # predicted box\r\n selected_tbox = targets[i][:, 2:6] * pre_gen_gains[i]\r\n selected_tbox[:, :2] -= grid\r\n iou = bbox_iou(pbox.T, selected_tbox, x1y1x2y2=False, CIoU=True) # iou(prediction, target)\r\n lbox += (1.0 - iou).mean() # iou loss\r\n\r\n # Objectness\r\n tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio\r\n\r\n # Classification\r\n selected_tcls = targets[i][:, 1].long()\r\n if self.nc > 1: # cls loss (only if multiple classes)\r\n t = torch.full_like(ps[:, 5:], self.cn, device=device) # targets\r\n t[range(n), selected_tcls] = self.cp\r\n lcls += self.BCEcls(ps[:, 5:], t) # BCE\r\n\r\n # Append targets to text file\r\n # with open('targets.txt', 'a') as file:\r\n # [file.write('%11.5g ' * 4 % tuple(x) + '\\n') for x in torch.cat((txy[i], twh[i]), 1)]\r\n\r\n obji = self.BCEobj(pi[..., 4], tobj)\r\n lobj += obji * self.balance[i] # obj loss\r\n if self.autobalance:\r\n self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item()\r\n\r\n if self.autobalance:\r\n self.balance = [x / self.balance[self.ssi] for x in self.balance]\r\n lbox *= self.hyp['box']\r\n lobj *= self.hyp['obj']\r\n lcls *= self.hyp['cls']\r\n bs = tobj.shape[0] # batch size\r\n\r\n loss = lbox + lobj + lcls\r\n return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()\r\n\r\n def build_targets(self, p, targets, imgs):\r\n \r\n #indices, anch = self.find_positive(p, targets)\r\n indices, anch = self.find_3_positive(p, targets)\r\n #indices, anch = self.find_4_positive(p, targets)\r\n #indices, anch = self.find_5_positive(p, targets)\r\n #indices, anch = self.find_9_positive(p, targets)\r\n\r\n matching_bs = [[] for pp in p]\r\n matching_as = [[] for pp in p]\r\n matching_gjs = [[] for pp in p]\r\n matching_gis = [[] for pp in p]\r\n matching_targets = [[] for pp in p]\r\n matching_anchs = [[] for pp in p]\r\n \r\n nl = len(p) \r\n \r\n for batch_idx in range(p[0].shape[0]):\r\n \r\n b_idx = targets[:, 0]==batch_idx\r\n this_target = targets[b_idx]\r\n if this_target.shape[0] == 0:\r\n continue\r\n \r\n txywh = this_target[:, 2:6] * imgs[batch_idx].shape[1]\r\n txyxy = xywh2xyxy(txywh)\r\n\r\n pxyxys = []\r\n p_cls = []\r\n p_obj = []\r\n from_which_layer = []\r\n all_b = []\r\n all_a = []\r\n all_gj = []\r\n all_gi = []\r\n all_anch = []\r\n \r\n for i, pi in enumerate(p):\r\n \r\n b, a, gj, gi = indices[i]\r\n idx = (b == batch_idx)\r\n b, a, gj, gi = b[idx], a[idx], gj[idx], gi[idx] \r\n all_b.append(b)\r\n all_a.append(a)\r\n all_gj.append(gj)\r\n all_gi.append(gi)\r\n all_anch.append(anch[i][idx])\r\n from_which_layer.append(torch.ones(size=(len(b),)) * i)\r\n \r\n fg_pred = pi[b, a, gj, gi] \r\n p_obj.append(fg_pred[:, 4:5])\r\n p_cls.append(fg_pred[:, 5:])\r\n \r\n grid = torch.stack([gi, gj], dim=1)\r\n pxy = (fg_pred[:, :2].sigmoid() * 2. - 0.5 + grid) * self.stride[i] #/ 8.\r\n #pxy = (fg_pred[:, :2].sigmoid() * 3. - 1. + grid) * self.stride[i]\r\n pwh = (fg_pred[:, 2:4].sigmoid() * 2) ** 2 * anch[i][idx] * self.stride[i] #/ 8.\r\n pxywh = torch.cat([pxy, pwh], dim=-1)\r\n pxyxy = xywh2xyxy(pxywh)\r\n pxyxys.append(pxyxy)\r\n \r\n pxyxys = torch.cat(pxyxys, dim=0)\r\n if pxyxys.shape[0] == 0:\r\n continue\r\n p_obj = torch.cat(p_obj, dim=0)\r\n p_cls = torch.cat(p_cls, dim=0)\r\n from_which_layer = torch.cat(from_which_layer, dim=0)\r\n all_b = torch.cat(all_b, dim=0)\r\n all_a = torch.cat(all_a, dim=0)\r\n all_gj = torch.cat(all_gj, dim=0)\r\n all_gi = torch.cat(all_gi, dim=0)\r\n all_anch = torch.cat(all_anch, dim=0)\r\n \r\n pair_wise_iou = box_iou(txyxy, pxyxys)\r\n\r\n pair_wise_iou_loss = -torch.log(pair_wise_iou + 1e-8)\r\n\r\n top_k, _ = torch.topk(pair_wise_iou, min(10, pair_wise_iou.shape[1]), dim=1)\r\n dynamic_ks = torch.clamp(top_k.sum(1).int(), min=1)\r\n\r\n gt_cls_per_image = (\r\n F.one_hot(this_target[:, 1].to(torch.int64), self.nc)\r\n .float()\r\n .unsqueeze(1)\r\n .repeat(1, pxyxys.shape[0], 1)\r\n )\r\n\r\n num_gt = this_target.shape[0]\r\n cls_preds_ = (\r\n p_cls.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()\r\n * p_obj.unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()\r\n )\r\n\r\n y = cls_preds_.sqrt_()\r\n pair_wise_cls_loss = F.binary_cross_entropy_with_logits(\r\n torch.log(y/(1-y)) , gt_cls_per_image, reduction=\"none\"\r\n ).sum(-1)\r\n del cls_preds_\r\n \r\n cost = (\r\n pair_wise_cls_loss\r\n + 3.0 * pair_wise_iou_loss\r\n )\r\n\r\n matching_matrix = torch.zeros_like(cost)\r\n\r\n for gt_idx in range(num_gt):\r\n _, pos_idx = torch.topk(\r\n cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False\r\n )\r\n matching_matrix[gt_idx][pos_idx] = 1.0\r\n\r\n del top_k, dynamic_ks\r\n anchor_matching_gt = matching_matrix.sum(0)\r\n if (anchor_matching_gt > 1).sum() > 0:\r\n _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0)\r\n matching_matrix[:, anchor_matching_gt > 1] *= 0.0\r\n matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0\r\n fg_mask_inboxes = matching_matrix.sum(0) > 0.0\r\n matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0)\r\n \r\n from_which_layer = from_which_layer[fg_mask_inboxes]\r\n all_b = all_b[fg_mask_inboxes]\r\n all_a = all_a[fg_mask_inboxes]\r\n all_gj = all_gj[fg_mask_inboxes]\r\n all_gi = all_gi[fg_mask_inboxes]\r\n all_anch = all_anch[fg_mask_inboxes]\r\n \r\n this_target = this_target[matched_gt_inds]\r\n \r\n for i in range(nl):\r\n layer_idx = from_which_layer == i\r\n matching_bs[i].append(all_b[layer_idx])\r\n matching_as[i].append(all_a[layer_idx])\r\n matching_gjs[i].append(all_gj[layer_idx])\r\n matching_gis[i].append(all_gi[layer_idx])\r\n matching_targets[i].append(this_target[layer_idx])\r\n matching_anchs[i].append(all_anch[layer_idx])\r\n\r\n for i in range(nl):\r\n if matching_targets[i] != []:\r\n matching_bs[i] = torch.cat(matching_bs[i], dim=0)\r\n matching_as[i] = torch.cat(matching_as[i], dim=0)\r\n matching_gjs[i] = torch.cat(matching_gjs[i], dim=0)\r\n matching_gis[i] = torch.cat(matching_gis[i], dim=0)\r\n matching_targets[i] = torch.cat(matching_targets[i], dim=0)\r\n matching_anchs[i] = torch.cat(matching_anchs[i], dim=0)\r\n else:\r\n matching_bs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_as[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_gjs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_gis[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_targets[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_anchs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n\r\n return matching_bs, matching_as, matching_gjs, matching_gis, matching_targets, matching_anchs \r\n\r\n def find_3_positive(self, p, targets):\r\n # Build targets for compute_loss(), input targets(image,class,x,y,w,h)\r\n na, nt = self.na, targets.shape[0] # number of anchors, targets\r\n indices, anch = [], []\r\n gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain\r\n ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt)\r\n targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices\r\n\r\n g = 0.5 # bias\r\n off = torch.tensor([[0, 0],\r\n [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m\r\n # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm\r\n ], device=targets.device).float() * g # offsets\r\n\r\n for i in range(self.nl):\r\n anchors = self.anchors[i]\r\n gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain\r\n\r\n # Match targets to anchors\r\n t = targets * gain\r\n if nt:\r\n # Matches\r\n r = t[:, :, 4:6] / anchors[:, None] # wh ratio\r\n j = torch.max(r, 1. / r).max(2)[0] < self.hyp['anchor_t'] # compare\r\n # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2))\r\n t = t[j] # filter\r\n\r\n # Offsets\r\n gxy = t[:, 2:4] # grid xy\r\n gxi = gain[[2, 3]] - gxy # inverse\r\n j, k = ((gxy % 1. < g) & (gxy > 1.)).T\r\n l, m = ((gxi % 1. < g) & (gxi > 1.)).T\r\n j = torch.stack((torch.ones_like(j), j, k, l, m))\r\n t = t.repeat((5, 1, 1))[j]\r\n offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j]\r\n else:\r\n t = targets[0]\r\n offsets = 0\r\n\r\n # Define\r\n b, c = t[:, :2].long().T # image, class\r\n gxy = t[:, 2:4] # grid xy\r\n gwh = t[:, 4:6] # grid wh\r\n gij = (gxy - offsets).long()\r\n gi, gj = gij.T # grid xy indices\r\n\r\n # Append\r\n a = t[:, 6].long() # anchor indices\r\n indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices\r\n anch.append(anchors[a]) # anchors\r\n\r\n return indices, anch\r\n \r\n\r\nclass ComputeLossBinOTA:\r\n # Compute losses\r\n def __init__(self, model, autobalance=False):\r\n super(ComputeLossBinOTA, self).__init__()\r\n device = next(model.parameters()).device # get model device\r\n h = model.hyp # hyperparameters\r\n\r\n # Define criteria\r\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device))\r\n BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device))\r\n #MSEangle = nn.MSELoss().to(device)\r\n\r\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\r\n self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets\r\n\r\n # Focal loss\r\n g = h['fl_gamma'] # focal loss gamma\r\n if g > 0:\r\n BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)\r\n\r\n det = model.module.model[-1] if is_parallel(model) else model.model[-1] # Detect() module\r\n self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.25, 0.06, .02]) # P3-P7\r\n self.ssi = list(det.stride).index(16) if autobalance else 0 # stride 16 index\r\n self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, model.gr, h, autobalance\r\n for k in 'na', 'nc', 'nl', 'anchors', 'stride', 'bin_count':\r\n setattr(self, k, getattr(det, k))\r\n\r\n #xy_bin_sigmoid = SigmoidBin(bin_count=11, min=-0.5, max=1.5, use_loss_regression=False).to(device)\r\n wh_bin_sigmoid = SigmoidBin(bin_count=self.bin_count, min=0.0, max=4.0, use_loss_regression=False).to(device)\r\n #angle_bin_sigmoid = SigmoidBin(bin_count=31, min=-1.1, max=1.1, use_loss_regression=False).to(device)\r\n self.wh_bin_sigmoid = wh_bin_sigmoid\r\n\r\n def __call__(self, p, targets, imgs): # predictions, targets, model \r\n device = targets.device\r\n lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)\r\n bs, as_, gjs, gis, targets, anchors = self.build_targets(p, targets, imgs)\r\n pre_gen_gains = [torch.tensor(pp.shape, device=device)[[3, 2, 3, 2]] for pp in p] \r\n \r\n\r\n # Losses\r\n for i, pi in enumerate(p): # layer index, layer predictions\r\n b, a, gj, gi = bs[i], as_[i], gjs[i], gis[i] # image, anchor, gridy, gridx\r\n tobj = torch.zeros_like(pi[..., 0], device=device) # target obj\r\n\r\n obj_idx = self.wh_bin_sigmoid.get_length()*2 + 2 # x,y, w-bce, h-bce # xy_bin_sigmoid.get_length()*2\r\n\r\n n = b.shape[0] # number of targets\r\n if n:\r\n ps = pi[b, a, gj, gi] # prediction subset corresponding to targets\r\n\r\n # Regression\r\n grid = torch.stack([gi, gj], dim=1)\r\n selected_tbox = targets[i][:, 2:6] * pre_gen_gains[i]\r\n selected_tbox[:, :2] -= grid\r\n \r\n #pxy = ps[:, :2].sigmoid() * 2. - 0.5\r\n ##pxy = ps[:, :2].sigmoid() * 3. - 1.\r\n #pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]\r\n #pbox = torch.cat((pxy, pwh), 1) # predicted box\r\n\r\n #x_loss, px = xy_bin_sigmoid.training_loss(ps[..., 0:12], tbox[i][..., 0])\r\n #y_loss, py = xy_bin_sigmoid.training_loss(ps[..., 12:24], tbox[i][..., 1])\r\n w_loss, pw = self.wh_bin_sigmoid.training_loss(ps[..., 2:(3+self.bin_count)], selected_tbox[..., 2] / anchors[i][..., 0])\r\n h_loss, ph = self.wh_bin_sigmoid.training_loss(ps[..., (3+self.bin_count):obj_idx], selected_tbox[..., 3] / anchors[i][..., 1])\r\n\r\n pw *= anchors[i][..., 0]\r\n ph *= anchors[i][..., 1]\r\n\r\n px = ps[:, 0].sigmoid() * 2. - 0.5\r\n py = ps[:, 1].sigmoid() * 2. - 0.5\r\n\r\n lbox += w_loss + h_loss # + x_loss + y_loss\r\n\r\n #print(f\"\\n px = {px.shape}, py = {py.shape}, pw = {pw.shape}, ph = {ph.shape} \\n\")\r\n\r\n pbox = torch.cat((px.unsqueeze(1), py.unsqueeze(1), pw.unsqueeze(1), ph.unsqueeze(1)), 1).to(device) # predicted box\r\n\r\n \r\n \r\n \r\n iou = bbox_iou(pbox.T, selected_tbox, x1y1x2y2=False, CIoU=True) # iou(prediction, target)\r\n lbox += (1.0 - iou).mean() # iou loss\r\n\r\n # Objectness\r\n tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio\r\n\r\n # Classification\r\n selected_tcls = targets[i][:, 1].long()\r\n if self.nc > 1: # cls loss (only if multiple classes)\r\n t = torch.full_like(ps[:, (1+obj_idx):], self.cn, device=device) # targets\r\n t[range(n), selected_tcls] = self.cp\r\n lcls += self.BCEcls(ps[:, (1+obj_idx):], t) # BCE\r\n\r\n # Append targets to text file\r\n # with open('targets.txt', 'a') as file:\r\n # [file.write('%11.5g ' * 4 % tuple(x) + '\\n') for x in torch.cat((txy[i], twh[i]), 1)]\r\n\r\n obji = self.BCEobj(pi[..., obj_idx], tobj)\r\n lobj += obji * self.balance[i] # obj loss\r\n if self.autobalance:\r\n self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item()\r\n\r\n if self.autobalance:\r\n self.balance = [x / self.balance[self.ssi] for x in self.balance]\r\n lbox *= self.hyp['box']\r\n lobj *= self.hyp['obj']\r\n lcls *= self.hyp['cls']\r\n bs = tobj.shape[0] # batch size\r\n\r\n loss = lbox + lobj + lcls\r\n return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()\r\n\r\n def build_targets(self, p, targets, imgs):\r\n \r\n #indices, anch = self.find_positive(p, targets)\r\n indices, anch = self.find_3_positive(p, targets)\r\n #indices, anch = self.find_4_positive(p, targets)\r\n #indices, anch = self.find_5_positive(p, targets)\r\n #indices, anch = self.find_9_positive(p, targets)\r\n\r\n matching_bs = [[] for pp in p]\r\n matching_as = [[] for pp in p]\r\n matching_gjs = [[] for pp in p]\r\n matching_gis = [[] for pp in p]\r\n matching_targets = [[] for pp in p]\r\n matching_anchs = [[] for pp in p]\r\n \r\n nl = len(p) \r\n \r\n for batch_idx in range(p[0].shape[0]):\r\n \r\n b_idx = targets[:, 0]==batch_idx\r\n this_target = targets[b_idx]\r\n if this_target.shape[0] == 0:\r\n continue\r\n \r\n txywh = this_target[:, 2:6] * imgs[batch_idx].shape[1]\r\n txyxy = xywh2xyxy(txywh)\r\n\r\n pxyxys = []\r\n p_cls = []\r\n p_obj = []\r\n from_which_layer = []\r\n all_b = []\r\n all_a = []\r\n all_gj = []\r\n all_gi = []\r\n all_anch = []\r\n \r\n for i, pi in enumerate(p):\r\n \r\n obj_idx = self.wh_bin_sigmoid.get_length()*2 + 2\r\n \r\n b, a, gj, gi = indices[i]\r\n idx = (b == batch_idx)\r\n b, a, gj, gi = b[idx], a[idx], gj[idx], gi[idx] \r\n all_b.append(b)\r\n all_a.append(a)\r\n all_gj.append(gj)\r\n all_gi.append(gi)\r\n all_anch.append(anch[i][idx])\r\n from_which_layer.append(torch.ones(size=(len(b),)) * i)\r\n \r\n fg_pred = pi[b, a, gj, gi] \r\n p_obj.append(fg_pred[:, obj_idx:(obj_idx+1)])\r\n p_cls.append(fg_pred[:, (obj_idx+1):])\r\n \r\n grid = torch.stack([gi, gj], dim=1)\r\n pxy = (fg_pred[:, :2].sigmoid() * 2. - 0.5 + grid) * self.stride[i] #/ 8.\r\n #pwh = (fg_pred[:, 2:4].sigmoid() * 2) ** 2 * anch[i][idx] * self.stride[i] #/ 8.\r\n pw = self.wh_bin_sigmoid.forward(fg_pred[..., 2:(3+self.bin_count)].sigmoid()) * anch[i][idx][:, 0] * self.stride[i]\r\n ph = self.wh_bin_sigmoid.forward(fg_pred[..., (3+self.bin_count):obj_idx].sigmoid()) * anch[i][idx][:, 1] * self.stride[i]\r\n \r\n pxywh = torch.cat([pxy, pw.unsqueeze(1), ph.unsqueeze(1)], dim=-1)\r\n pxyxy = xywh2xyxy(pxywh)\r\n pxyxys.append(pxyxy)\r\n \r\n pxyxys = torch.cat(pxyxys, dim=0)\r\n if pxyxys.shape[0] == 0:\r\n continue\r\n p_obj = torch.cat(p_obj, dim=0)\r\n p_cls = torch.cat(p_cls, dim=0)\r\n from_which_layer = torch.cat(from_which_layer, dim=0)\r\n all_b = torch.cat(all_b, dim=0)\r\n all_a = torch.cat(all_a, dim=0)\r\n all_gj = torch.cat(all_gj, dim=0)\r\n all_gi = torch.cat(all_gi, dim=0)\r\n all_anch = torch.cat(all_anch, dim=0)\r\n \r\n pair_wise_iou = box_iou(txyxy, pxyxys)\r\n\r\n pair_wise_iou_loss = -torch.log(pair_wise_iou + 1e-8)\r\n\r\n top_k, _ = torch.topk(pair_wise_iou, min(10, pair_wise_iou.shape[1]), dim=1)\r\n dynamic_ks = torch.clamp(top_k.sum(1).int(), min=1)\r\n\r\n gt_cls_per_image = (\r\n F.one_hot(this_target[:, 1].to(torch.int64), self.nc)\r\n .float()\r\n .unsqueeze(1)\r\n .repeat(1, pxyxys.shape[0], 1)\r\n )\r\n\r\n num_gt = this_target.shape[0] \r\n cls_preds_ = (\r\n p_cls.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()\r\n * p_obj.unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()\r\n )\r\n\r\n y = cls_preds_.sqrt_()\r\n pair_wise_cls_loss = F.binary_cross_entropy_with_logits(\r\n torch.log(y/(1-y)) , gt_cls_per_image, reduction=\"none\"\r\n ).sum(-1)\r\n del cls_preds_\r\n \r\n cost = (\r\n pair_wise_cls_loss\r\n + 3.0 * pair_wise_iou_loss\r\n )\r\n\r\n matching_matrix = torch.zeros_like(cost)\r\n\r\n for gt_idx in range(num_gt):\r\n _, pos_idx = torch.topk(\r\n cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False\r\n )\r\n matching_matrix[gt_idx][pos_idx] = 1.0\r\n\r\n del top_k, dynamic_ks\r\n anchor_matching_gt = matching_matrix.sum(0)\r\n if (anchor_matching_gt > 1).sum() > 0:\r\n _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0)\r\n matching_matrix[:, anchor_matching_gt > 1] *= 0.0\r\n matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0\r\n fg_mask_inboxes = matching_matrix.sum(0) > 0.0\r\n matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0)\r\n \r\n from_which_layer = from_which_layer[fg_mask_inboxes]\r\n all_b = all_b[fg_mask_inboxes]\r\n all_a = all_a[fg_mask_inboxes]\r\n all_gj = all_gj[fg_mask_inboxes]\r\n all_gi = all_gi[fg_mask_inboxes]\r\n all_anch = all_anch[fg_mask_inboxes]\r\n \r\n this_target = this_target[matched_gt_inds]\r\n \r\n for i in range(nl):\r\n layer_idx = from_which_layer == i\r\n matching_bs[i].append(all_b[layer_idx])\r\n matching_as[i].append(all_a[layer_idx])\r\n matching_gjs[i].append(all_gj[layer_idx])\r\n matching_gis[i].append(all_gi[layer_idx])\r\n matching_targets[i].append(this_target[layer_idx])\r\n matching_anchs[i].append(all_anch[layer_idx])\r\n\r\n for i in range(nl):\r\n if matching_targets[i] != []:\r\n matching_bs[i] = torch.cat(matching_bs[i], dim=0)\r\n matching_as[i] = torch.cat(matching_as[i], dim=0)\r\n matching_gjs[i] = torch.cat(matching_gjs[i], dim=0)\r\n matching_gis[i] = torch.cat(matching_gis[i], dim=0)\r\n matching_targets[i] = torch.cat(matching_targets[i], dim=0)\r\n matching_anchs[i] = torch.cat(matching_anchs[i], dim=0)\r\n else:\r\n matching_bs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_as[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_gjs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_gis[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_targets[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_anchs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n\r\n return matching_bs, matching_as, matching_gjs, matching_gis, matching_targets, matching_anchs \r\n\r\n def find_3_positive(self, p, targets):\r\n # Build targets for compute_loss(), input targets(image,class,x,y,w,h)\r\n na, nt = self.na, targets.shape[0] # number of anchors, targets\r\n indices, anch = [], []\r\n gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain\r\n ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt)\r\n targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices\r\n\r\n g = 0.5 # bias\r\n off = torch.tensor([[0, 0],\r\n [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m\r\n # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm\r\n ], device=targets.device).float() * g # offsets\r\n\r\n for i in range(self.nl):\r\n anchors = self.anchors[i]\r\n gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain\r\n\r\n # Match targets to anchors\r\n t = targets * gain\r\n if nt:\r\n # Matches\r\n r = t[:, :, 4:6] / anchors[:, None] # wh ratio\r\n j = torch.max(r, 1. / r).max(2)[0] < self.hyp['anchor_t'] # compare\r\n # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2))\r\n t = t[j] # filter\r\n\r\n # Offsets\r\n gxy = t[:, 2:4] # grid xy\r\n gxi = gain[[2, 3]] - gxy # inverse\r\n j, k = ((gxy % 1. < g) & (gxy > 1.)).T\r\n l, m = ((gxi % 1. < g) & (gxi > 1.)).T\r\n j = torch.stack((torch.ones_like(j), j, k, l, m))\r\n t = t.repeat((5, 1, 1))[j]\r\n offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j]\r\n else:\r\n t = targets[0]\r\n offsets = 0\r\n\r\n # Define\r\n b, c = t[:, :2].long().T # image, class\r\n gxy = t[:, 2:4] # grid xy\r\n gwh = t[:, 4:6] # grid wh\r\n gij = (gxy - offsets).long()\r\n gi, gj = gij.T # grid xy indices\r\n\r\n # Append\r\n a = t[:, 6].long() # anchor indices\r\n indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices\r\n anch.append(anchors[a]) # anchors\r\n\r\n return indices, anch\r\n\r\n\r\nclass ComputeLossAuxOTA:\r\n # Compute losses\r\n def __init__(self, model, autobalance=False):\r\n super(ComputeLossAuxOTA, self).__init__()\r\n device = next(model.parameters()).device # get model device\r\n h = model.hyp # hyperparameters\r\n\r\n # Define criteria\r\n BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device))\r\n BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device))\r\n\r\n # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3\r\n self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets\r\n\r\n # Focal loss\r\n g = h['fl_gamma'] # focal loss gamma\r\n if g > 0:\r\n BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g)\r\n\r\n det = model.module.model[-1] if is_parallel(model) else model.model[-1] # Detect() module\r\n self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.25, 0.06, .02]) # P3-P7\r\n self.ssi = list(det.stride).index(16) if autobalance else 0 # stride 16 index\r\n self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, model.gr, h, autobalance\r\n for k in 'na', 'nc', 'nl', 'anchors', 'stride':\r\n setattr(self, k, getattr(det, k))\r\n\r\n def __call__(self, p, targets, imgs): # predictions, targets, model \r\n device = targets.device\r\n lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device)\r\n bs_aux, as_aux_, gjs_aux, gis_aux, targets_aux, anchors_aux = self.build_targets2(p[:self.nl], targets, imgs)\r\n bs, as_, gjs, gis, targets, anchors = self.build_targets(p[:self.nl], targets, imgs)\r\n pre_gen_gains_aux = [torch.tensor(pp.shape, device=device)[[3, 2, 3, 2]] for pp in p[:self.nl]] \r\n pre_gen_gains = [torch.tensor(pp.shape, device=device)[[3, 2, 3, 2]] for pp in p[:self.nl]] \r\n \r\n\r\n # Losses\r\n for i in range(self.nl): # layer index, layer predictions\r\n pi = p[i]\r\n pi_aux = p[i+self.nl]\r\n b, a, gj, gi = bs[i], as_[i], gjs[i], gis[i] # image, anchor, gridy, gridx\r\n b_aux, a_aux, gj_aux, gi_aux = bs_aux[i], as_aux_[i], gjs_aux[i], gis_aux[i] # image, anchor, gridy, gridx\r\n tobj = torch.zeros_like(pi[..., 0], device=device) # target obj\r\n tobj_aux = torch.zeros_like(pi_aux[..., 0], device=device) # target obj\r\n\r\n n = b.shape[0] # number of targets\r\n if n:\r\n ps = pi[b, a, gj, gi] # prediction subset corresponding to targets\r\n\r\n # Regression\r\n grid = torch.stack([gi, gj], dim=1)\r\n pxy = ps[:, :2].sigmoid() * 2. - 0.5\r\n pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i]\r\n pbox = torch.cat((pxy, pwh), 1) # predicted box\r\n selected_tbox = targets[i][:, 2:6] * pre_gen_gains[i]\r\n selected_tbox[:, :2] -= grid\r\n iou = bbox_iou(pbox.T, selected_tbox, x1y1x2y2=False, CIoU=True) # iou(prediction, target)\r\n lbox += (1.0 - iou).mean() # iou loss\r\n\r\n # Objectness\r\n tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio\r\n\r\n # Classification\r\n selected_tcls = targets[i][:, 1].long()\r\n if self.nc > 1: # cls loss (only if multiple classes)\r\n t = torch.full_like(ps[:, 5:], self.cn, device=device) # targets\r\n t[range(n), selected_tcls] = self.cp\r\n lcls += self.BCEcls(ps[:, 5:], t) # BCE\r\n\r\n # Append targets to text file\r\n # with open('targets.txt', 'a') as file:\r\n # [file.write('%11.5g ' * 4 % tuple(x) + '\\n') for x in torch.cat((txy[i], twh[i]), 1)]\r\n \r\n n_aux = b_aux.shape[0] # number of targets\r\n if n_aux:\r\n ps_aux = pi_aux[b_aux, a_aux, gj_aux, gi_aux] # prediction subset corresponding to targets\r\n grid_aux = torch.stack([gi_aux, gj_aux], dim=1)\r\n pxy_aux = ps_aux[:, :2].sigmoid() * 2. - 0.5\r\n #pxy_aux = ps_aux[:, :2].sigmoid() * 3. - 1.\r\n pwh_aux = (ps_aux[:, 2:4].sigmoid() * 2) ** 2 * anchors_aux[i]\r\n pbox_aux = torch.cat((pxy_aux, pwh_aux), 1) # predicted box\r\n selected_tbox_aux = targets_aux[i][:, 2:6] * pre_gen_gains_aux[i]\r\n selected_tbox_aux[:, :2] -= grid_aux\r\n iou_aux = bbox_iou(pbox_aux.T, selected_tbox_aux, x1y1x2y2=False, CIoU=True) # iou(prediction, target)\r\n lbox += 0.25 * (1.0 - iou_aux).mean() # iou loss\r\n\r\n # Objectness\r\n tobj_aux[b_aux, a_aux, gj_aux, gi_aux] = (1.0 - self.gr) + self.gr * iou_aux.detach().clamp(0).type(tobj_aux.dtype) # iou ratio\r\n\r\n # Classification\r\n selected_tcls_aux = targets_aux[i][:, 1].long()\r\n if self.nc > 1: # cls loss (only if multiple classes)\r\n t_aux = torch.full_like(ps_aux[:, 5:], self.cn, device=device) # targets\r\n t_aux[range(n_aux), selected_tcls_aux] = self.cp\r\n lcls += 0.25 * self.BCEcls(ps_aux[:, 5:], t_aux) # BCE\r\n\r\n obji = self.BCEobj(pi[..., 4], tobj)\r\n obji_aux = self.BCEobj(pi_aux[..., 4], tobj_aux)\r\n lobj += obji * self.balance[i] + 0.25 * obji_aux * self.balance[i] # obj loss\r\n if self.autobalance:\r\n self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item()\r\n\r\n if self.autobalance:\r\n self.balance = [x / self.balance[self.ssi] for x in self.balance]\r\n lbox *= self.hyp['box']\r\n lobj *= self.hyp['obj']\r\n lcls *= self.hyp['cls']\r\n bs = tobj.shape[0] # batch size\r\n\r\n loss = lbox + lobj + lcls\r\n return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach()\r\n\r\n def build_targets(self, p, targets, imgs):\r\n \r\n indices, anch = self.find_3_positive(p, targets)\r\n\r\n matching_bs = [[] for pp in p]\r\n matching_as = [[] for pp in p]\r\n matching_gjs = [[] for pp in p]\r\n matching_gis = [[] for pp in p]\r\n matching_targets = [[] for pp in p]\r\n matching_anchs = [[] for pp in p]\r\n \r\n nl = len(p) \r\n \r\n for batch_idx in range(p[0].shape[0]):\r\n \r\n b_idx = targets[:, 0]==batch_idx\r\n this_target = targets[b_idx]\r\n if this_target.shape[0] == 0:\r\n continue\r\n \r\n txywh = this_target[:, 2:6] * imgs[batch_idx].shape[1]\r\n txyxy = xywh2xyxy(txywh)\r\n\r\n pxyxys = []\r\n p_cls = []\r\n p_obj = []\r\n from_which_layer = []\r\n all_b = []\r\n all_a = []\r\n all_gj = []\r\n all_gi = []\r\n all_anch = []\r\n \r\n for i, pi in enumerate(p):\r\n \r\n b, a, gj, gi = indices[i]\r\n idx = (b == batch_idx)\r\n b, a, gj, gi = b[idx], a[idx], gj[idx], gi[idx] \r\n all_b.append(b)\r\n all_a.append(a)\r\n all_gj.append(gj)\r\n all_gi.append(gi)\r\n all_anch.append(anch[i][idx])\r\n from_which_layer.append(torch.ones(size=(len(b),)) * i)\r\n \r\n fg_pred = pi[b, a, gj, gi] \r\n p_obj.append(fg_pred[:, 4:5])\r\n p_cls.append(fg_pred[:, 5:])\r\n \r\n grid = torch.stack([gi, gj], dim=1)\r\n pxy = (fg_pred[:, :2].sigmoid() * 2. - 0.5 + grid) * self.stride[i] #/ 8.\r\n #pxy = (fg_pred[:, :2].sigmoid() * 3. - 1. + grid) * self.stride[i]\r\n pwh = (fg_pred[:, 2:4].sigmoid() * 2) ** 2 * anch[i][idx] * self.stride[i] #/ 8.\r\n pxywh = torch.cat([pxy, pwh], dim=-1)\r\n pxyxy = xywh2xyxy(pxywh)\r\n pxyxys.append(pxyxy)\r\n \r\n pxyxys = torch.cat(pxyxys, dim=0)\r\n if pxyxys.shape[0] == 0:\r\n continue\r\n p_obj = torch.cat(p_obj, dim=0)\r\n p_cls = torch.cat(p_cls, dim=0)\r\n from_which_layer = torch.cat(from_which_layer, dim=0)\r\n all_b = torch.cat(all_b, dim=0)\r\n all_a = torch.cat(all_a, dim=0)\r\n all_gj = torch.cat(all_gj, dim=0)\r\n all_gi = torch.cat(all_gi, dim=0)\r\n all_anch = torch.cat(all_anch, dim=0)\r\n \r\n pair_wise_iou = box_iou(txyxy, pxyxys)\r\n\r\n pair_wise_iou_loss = -torch.log(pair_wise_iou + 1e-8)\r\n\r\n top_k, _ = torch.topk(pair_wise_iou, min(20, pair_wise_iou.shape[1]), dim=1)\r\n dynamic_ks = torch.clamp(top_k.sum(1).int(), min=1)\r\n\r\n gt_cls_per_image = (\r\n F.one_hot(this_target[:, 1].to(torch.int64), self.nc)\r\n .float()\r\n .unsqueeze(1)\r\n .repeat(1, pxyxys.shape[0], 1)\r\n )\r\n\r\n num_gt = this_target.shape[0]\r\n cls_preds_ = (\r\n p_cls.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()\r\n * p_obj.unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()\r\n )\r\n\r\n y = cls_preds_.sqrt_()\r\n pair_wise_cls_loss = F.binary_cross_entropy_with_logits(\r\n torch.log(y/(1-y)) , gt_cls_per_image, reduction=\"none\"\r\n ).sum(-1)\r\n del cls_preds_\r\n \r\n cost = (\r\n pair_wise_cls_loss\r\n + 3.0 * pair_wise_iou_loss\r\n )\r\n\r\n matching_matrix = torch.zeros_like(cost)\r\n\r\n for gt_idx in range(num_gt):\r\n _, pos_idx = torch.topk(\r\n cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False\r\n )\r\n matching_matrix[gt_idx][pos_idx] = 1.0\r\n\r\n del top_k, dynamic_ks\r\n anchor_matching_gt = matching_matrix.sum(0)\r\n if (anchor_matching_gt > 1).sum() > 0:\r\n _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0)\r\n matching_matrix[:, anchor_matching_gt > 1] *= 0.0\r\n matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0\r\n fg_mask_inboxes = matching_matrix.sum(0) > 0.0\r\n matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0)\r\n \r\n from_which_layer = from_which_layer[fg_mask_inboxes]\r\n all_b = all_b[fg_mask_inboxes]\r\n all_a = all_a[fg_mask_inboxes]\r\n all_gj = all_gj[fg_mask_inboxes]\r\n all_gi = all_gi[fg_mask_inboxes]\r\n all_anch = all_anch[fg_mask_inboxes]\r\n \r\n this_target = this_target[matched_gt_inds]\r\n \r\n for i in range(nl):\r\n layer_idx = from_which_layer == i\r\n matching_bs[i].append(all_b[layer_idx])\r\n matching_as[i].append(all_a[layer_idx])\r\n matching_gjs[i].append(all_gj[layer_idx])\r\n matching_gis[i].append(all_gi[layer_idx])\r\n matching_targets[i].append(this_target[layer_idx])\r\n matching_anchs[i].append(all_anch[layer_idx])\r\n\r\n for i in range(nl):\r\n if matching_targets[i] != []:\r\n matching_bs[i] = torch.cat(matching_bs[i], dim=0)\r\n matching_as[i] = torch.cat(matching_as[i], dim=0)\r\n matching_gjs[i] = torch.cat(matching_gjs[i], dim=0)\r\n matching_gis[i] = torch.cat(matching_gis[i], dim=0)\r\n matching_targets[i] = torch.cat(matching_targets[i], dim=0)\r\n matching_anchs[i] = torch.cat(matching_anchs[i], dim=0)\r\n else:\r\n matching_bs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_as[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_gjs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_gis[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_targets[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_anchs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n\r\n return matching_bs, matching_as, matching_gjs, matching_gis, matching_targets, matching_anchs\r\n\r\n def build_targets2(self, p, targets, imgs):\r\n \r\n indices, anch = self.find_5_positive(p, targets)\r\n\r\n matching_bs = [[] for pp in p]\r\n matching_as = [[] for pp in p]\r\n matching_gjs = [[] for pp in p]\r\n matching_gis = [[] for pp in p]\r\n matching_targets = [[] for pp in p]\r\n matching_anchs = [[] for pp in p]\r\n \r\n nl = len(p) \r\n \r\n for batch_idx in range(p[0].shape[0]):\r\n \r\n b_idx = targets[:, 0]==batch_idx\r\n this_target = targets[b_idx]\r\n if this_target.shape[0] == 0:\r\n continue\r\n \r\n txywh = this_target[:, 2:6] * imgs[batch_idx].shape[1]\r\n txyxy = xywh2xyxy(txywh)\r\n\r\n pxyxys = []\r\n p_cls = []\r\n p_obj = []\r\n from_which_layer = []\r\n all_b = []\r\n all_a = []\r\n all_gj = []\r\n all_gi = []\r\n all_anch = []\r\n \r\n for i, pi in enumerate(p):\r\n \r\n b, a, gj, gi = indices[i]\r\n idx = (b == batch_idx)\r\n b, a, gj, gi = b[idx], a[idx], gj[idx], gi[idx] \r\n all_b.append(b)\r\n all_a.append(a)\r\n all_gj.append(gj)\r\n all_gi.append(gi)\r\n all_anch.append(anch[i][idx])\r\n from_which_layer.append(torch.ones(size=(len(b),)) * i)\r\n \r\n fg_pred = pi[b, a, gj, gi] \r\n p_obj.append(fg_pred[:, 4:5])\r\n p_cls.append(fg_pred[:, 5:])\r\n \r\n grid = torch.stack([gi, gj], dim=1)\r\n pxy = (fg_pred[:, :2].sigmoid() * 2. - 0.5 + grid) * self.stride[i] #/ 8.\r\n #pxy = (fg_pred[:, :2].sigmoid() * 3. - 1. + grid) * self.stride[i]\r\n pwh = (fg_pred[:, 2:4].sigmoid() * 2) ** 2 * anch[i][idx] * self.stride[i] #/ 8.\r\n pxywh = torch.cat([pxy, pwh], dim=-1)\r\n pxyxy = xywh2xyxy(pxywh)\r\n pxyxys.append(pxyxy)\r\n \r\n pxyxys = torch.cat(pxyxys, dim=0)\r\n if pxyxys.shape[0] == 0:\r\n continue\r\n p_obj = torch.cat(p_obj, dim=0)\r\n p_cls = torch.cat(p_cls, dim=0)\r\n from_which_layer = torch.cat(from_which_layer, dim=0)\r\n all_b = torch.cat(all_b, dim=0)\r\n all_a = torch.cat(all_a, dim=0)\r\n all_gj = torch.cat(all_gj, dim=0)\r\n all_gi = torch.cat(all_gi, dim=0)\r\n all_anch = torch.cat(all_anch, dim=0)\r\n \r\n pair_wise_iou = box_iou(txyxy, pxyxys)\r\n\r\n pair_wise_iou_loss = -torch.log(pair_wise_iou + 1e-8)\r\n\r\n top_k, _ = torch.topk(pair_wise_iou, min(20, pair_wise_iou.shape[1]), dim=1)\r\n dynamic_ks = torch.clamp(top_k.sum(1).int(), min=1)\r\n\r\n gt_cls_per_image = (\r\n F.one_hot(this_target[:, 1].to(torch.int64), self.nc)\r\n .float()\r\n .unsqueeze(1)\r\n .repeat(1, pxyxys.shape[0], 1)\r\n )\r\n\r\n num_gt = this_target.shape[0]\r\n cls_preds_ = (\r\n p_cls.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()\r\n * p_obj.unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_()\r\n )\r\n\r\n y = cls_preds_.sqrt_()\r\n pair_wise_cls_loss = F.binary_cross_entropy_with_logits(\r\n torch.log(y/(1-y)) , gt_cls_per_image, reduction=\"none\"\r\n ).sum(-1)\r\n del cls_preds_\r\n \r\n cost = (\r\n pair_wise_cls_loss\r\n + 3.0 * pair_wise_iou_loss\r\n )\r\n\r\n matching_matrix = torch.zeros_like(cost)\r\n\r\n for gt_idx in range(num_gt):\r\n _, pos_idx = torch.topk(\r\n cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False\r\n )\r\n matching_matrix[gt_idx][pos_idx] = 1.0\r\n\r\n del top_k, dynamic_ks\r\n anchor_matching_gt = matching_matrix.sum(0)\r\n if (anchor_matching_gt > 1).sum() > 0:\r\n _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0)\r\n matching_matrix[:, anchor_matching_gt > 1] *= 0.0\r\n matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0\r\n fg_mask_inboxes = matching_matrix.sum(0) > 0.0\r\n matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0)\r\n \r\n from_which_layer = from_which_layer[fg_mask_inboxes]\r\n all_b = all_b[fg_mask_inboxes]\r\n all_a = all_a[fg_mask_inboxes]\r\n all_gj = all_gj[fg_mask_inboxes]\r\n all_gi = all_gi[fg_mask_inboxes]\r\n all_anch = all_anch[fg_mask_inboxes]\r\n \r\n this_target = this_target[matched_gt_inds]\r\n \r\n for i in range(nl):\r\n layer_idx = from_which_layer == i\r\n matching_bs[i].append(all_b[layer_idx])\r\n matching_as[i].append(all_a[layer_idx])\r\n matching_gjs[i].append(all_gj[layer_idx])\r\n matching_gis[i].append(all_gi[layer_idx])\r\n matching_targets[i].append(this_target[layer_idx])\r\n matching_anchs[i].append(all_anch[layer_idx])\r\n\r\n for i in range(nl):\r\n if matching_targets[i] != []:\r\n matching_bs[i] = torch.cat(matching_bs[i], dim=0)\r\n matching_as[i] = torch.cat(matching_as[i], dim=0)\r\n matching_gjs[i] = torch.cat(matching_gjs[i], dim=0)\r\n matching_gis[i] = torch.cat(matching_gis[i], dim=0)\r\n matching_targets[i] = torch.cat(matching_targets[i], dim=0)\r\n matching_anchs[i] = torch.cat(matching_anchs[i], dim=0)\r\n else:\r\n matching_bs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_as[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_gjs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_gis[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_targets[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n matching_anchs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64)\r\n\r\n return matching_bs, matching_as, matching_gjs, matching_gis, matching_targets, matching_anchs \r\n\r\n def find_5_positive(self, p, targets):\r\n # Build targets for compute_loss(), input targets(image,class,x,y,w,h)\r\n na, nt = self.na, targets.shape[0] # number of anchors, targets\r\n indices, anch = [], []\r\n gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain\r\n ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt)\r\n targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices\r\n\r\n g = 1.0 # bias\r\n off = torch.tensor([[0, 0],\r\n [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m\r\n # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm\r\n ], device=targets.device).float() * g # offsets\r\n\r\n for i in range(self.nl):\r\n anchors = self.anchors[i]\r\n gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain\r\n\r\n # Match targets to anchors\r\n t = targets * gain\r\n if nt:\r\n # Matches\r\n r = t[:, :, 4:6] / anchors[:, None] # wh ratio\r\n j = torch.max(r, 1. / r).max(2)[0] < self.hyp['anchor_t'] # compare\r\n # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2))\r\n t = t[j] # filter\r\n\r\n # Offsets\r\n gxy = t[:, 2:4] # grid xy\r\n gxi = gain[[2, 3]] - gxy # inverse\r\n j, k = ((gxy % 1. < g) & (gxy > 1.)).T\r\n l, m = ((gxi % 1. < g) & (gxi > 1.)).T\r\n j = torch.stack((torch.ones_like(j), j, k, l, m))\r\n t = t.repeat((5, 1, 1))[j]\r\n offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j]\r\n else:\r\n t = targets[0]\r\n offsets = 0\r\n\r\n # Define\r\n b, c = t[:, :2].long().T # image, class\r\n gxy = t[:, 2:4] # grid xy\r\n gwh = t[:, 4:6] # grid wh\r\n gij = (gxy - offsets).long()\r\n gi, gj = gij.T # grid xy indices\r\n\r\n # Append\r\n a = t[:, 6].long() # anchor indices\r\n indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices\r\n anch.append(anchors[a]) # anchors\r\n\r\n return indices, anch \r\n\r\n def find_3_positive(self, p, targets):\r\n # Build targets for compute_loss(), input targets(image,class,x,y,w,h)\r\n na, nt = self.na, targets.shape[0] # number of anchors, targets\r\n indices, anch = [], []\r\n gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain\r\n ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt)\r\n targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices\r\n\r\n g = 0.5 # bias\r\n off = torch.tensor([[0, 0],\r\n [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m\r\n # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm\r\n ], device=targets.device).float() * g # offsets\r\n\r\n for i in range(self.nl):\r\n anchors = self.anchors[i]\r\n gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain\r\n\r\n # Match targets to anchors\r\n t = targets * gain\r\n if nt:\r\n # Matches\r\n r = t[:, :, 4:6] / anchors[:, None] # wh ratio\r\n j = torch.max(r, 1. / r).max(2)[0] < self.hyp['anchor_t'] # compare\r\n # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2))\r\n t = t[j] # filter\r\n\r\n # Offsets\r\n gxy = t[:, 2:4] # grid xy\r\n gxi = gain[[2, 3]] - gxy # inverse\r\n j, k = ((gxy % 1. < g) & (gxy > 1.)).T\r\n l, m = ((gxi % 1. < g) & (gxi > 1.)).T\r\n j = torch.stack((torch.ones_like(j), j, k, l, m))\r\n t = t.repeat((5, 1, 1))[j]\r\n offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j]\r\n else:\r\n t = targets[0]\r\n offsets = 0\r\n\r\n # Define\r\n b, c = t[:, :2].long().T # image, class\r\n gxy = t[:, 2:4] # grid xy\r\n gwh = t[:, 4:6] # grid wh\r\n gij = (gxy - offsets).long()\r\n gi, gj = gij.T # grid xy indices\r\n\r\n # Append\r\n a = t[:, 6].long() # anchor indices\r\n indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices\r\n anch.append(anchors[a]) # anchors\r\n\r\n return indices, anch\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/metrics.py", "content": "# Model validation metrics\r\n\r\nfrom pathlib import Path\r\n\r\nimport matplotlib.pyplot as plt\r\nimport numpy as np\r\nimport torch\r\n\r\nfrom asone.pose_estimators.yolov7_pose.utils import general\r\n\r\n\r\ndef fitness(x):\r\n # Model fitness as a weighted combination of metrics\r\n w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95]\r\n return (x[:, :4] * w).sum(1)\r\n\r\n\r\ndef ap_per_class(tp, conf, pred_cls, target_cls, plot=False, save_dir='.', names=()):\r\n \"\"\" Compute the average precision, given the recall and precision curves.\r\n Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.\r\n # Arguments\r\n tp: True positives (nparray, nx1 or nx10).\r\n conf: Objectness value from 0-1 (nparray).\r\n pred_cls: Predicted object classes (nparray).\r\n target_cls: True object classes (nparray).\r\n plot: Plot precision-recall curve at mAP@0.5\r\n save_dir: Plot save directory\r\n # Returns\r\n The average precision as computed in py-faster-rcnn.\r\n \"\"\"\r\n\r\n # Sort by objectness\r\n i = np.argsort(-conf)\r\n tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]\r\n\r\n # Find unique classes\r\n unique_classes = np.unique(target_cls)\r\n nc = unique_classes.shape[0] # number of classes, number of detections\r\n\r\n # Create Precision-Recall curve and compute AP for each class\r\n px, py = np.linspace(0, 1, 1000), [] # for plotting\r\n ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000))\r\n for ci, c in enumerate(unique_classes):\r\n i = pred_cls == c\r\n n_l = (target_cls == c).sum() # number of labels\r\n n_p = i.sum() # number of predictions\r\n\r\n if n_p == 0 or n_l == 0:\r\n continue\r\n else:\r\n # Accumulate FPs and TPs\r\n fpc = (1 - tp[i]).cumsum(0)\r\n tpc = tp[i].cumsum(0)\r\n\r\n # Recall\r\n recall = tpc / (n_l + 1e-16) # recall curve\r\n r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases\r\n\r\n # Precision\r\n precision = tpc / (tpc + fpc) # precision curve\r\n p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score\r\n\r\n # AP from recall-precision curve\r\n for j in range(tp.shape[1]):\r\n ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j])\r\n if plot and j == 0:\r\n py.append(np.interp(px, mrec, mpre)) # precision at mAP@0.5\r\n\r\n # Compute F1 (harmonic mean of precision and recall)\r\n f1 = 2 * p * r / (p + r + 1e-16)\r\n if plot:\r\n plot_pr_curve(px, py, ap, Path(save_dir) / 'PR_curve.png', names)\r\n plot_mc_curve(px, f1, Path(save_dir) / 'F1_curve.png', names, ylabel='F1')\r\n plot_mc_curve(px, p, Path(save_dir) / 'P_curve.png', names, ylabel='Precision')\r\n plot_mc_curve(px, r, Path(save_dir) / 'R_curve.png', names, ylabel='Recall')\r\n\r\n i = f1.mean(0).argmax() # max F1 index\r\n return p[:, i], r[:, i], ap, f1[:, i], unique_classes.astype('int32')\r\n\r\n\r\ndef compute_ap(recall, precision):\r\n \"\"\" Compute the average precision, given the recall and precision curves\r\n # Arguments\r\n recall: The recall curve (list)\r\n precision: The precision curve (list)\r\n # Returns\r\n Average precision, precision curve, recall curve\r\n \"\"\"\r\n\r\n # Append sentinel values to beginning and end\r\n mrec = np.concatenate(([0.], recall, [recall[-1] + 0.01]))\r\n mpre = np.concatenate(([1.], precision, [0.]))\r\n\r\n # Compute the precision envelope\r\n mpre = np.flip(np.maximum.accumulate(np.flip(mpre)))\r\n\r\n # Integrate area under curve\r\n method = 'interp' # methods: 'continuous', 'interp'\r\n if method == 'interp':\r\n x = np.linspace(0, 1, 101) # 101-point interp (COCO)\r\n ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate\r\n else: # 'continuous'\r\n i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes\r\n ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve\r\n\r\n return ap, mpre, mrec\r\n\r\n\r\nclass ConfusionMatrix:\r\n # Updated version of https://github.com/kaanakan/object_detection_confusion_matrix\r\n def __init__(self, nc, conf=0.25, iou_thres=0.45):\r\n self.matrix = np.zeros((nc + 1, nc + 1))\r\n self.nc = nc # number of classes\r\n self.conf = conf\r\n self.iou_thres = iou_thres\r\n\r\n def process_batch(self, detections, labels):\r\n \"\"\"\r\n Return intersection-over-union (Jaccard index) of boxes.\r\n Both sets of boxes are expected to be in (x1, y1, x2, y2) format.\r\n Arguments:\r\n detections (Array[N, 6]), x1, y1, x2, y2, conf, class\r\n labels (Array[M, 5]), class, x1, y1, x2, y2\r\n Returns:\r\n None, updates confusion matrix accordingly\r\n \"\"\"\r\n detections = detections[detections[:, 4] > self.conf]\r\n gt_classes = labels[:, 0].int()\r\n detection_classes = detections[:, 5].int()\r\n iou = general.box_iou(labels[:, 1:], detections[:, :4])\r\n\r\n x = torch.where(iou > self.iou_thres)\r\n if x[0].shape[0]:\r\n matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy()\r\n if x[0].shape[0] > 1:\r\n matches = matches[matches[:, 2].argsort()[::-1]]\r\n matches = matches[np.unique(matches[:, 1], return_index=True)[1]]\r\n matches = matches[matches[:, 2].argsort()[::-1]]\r\n matches = matches[np.unique(matches[:, 0], return_index=True)[1]]\r\n else:\r\n matches = np.zeros((0, 3))\r\n\r\n n = matches.shape[0] > 0\r\n m0, m1, _ = matches.transpose().astype(np.int16)\r\n for i, gc in enumerate(gt_classes):\r\n j = m0 == i\r\n if n and sum(j) == 1:\r\n self.matrix[gc, detection_classes[m1[j]]] += 1 # correct\r\n else:\r\n self.matrix[self.nc, gc] += 1 # background FP\r\n\r\n if n:\r\n for i, dc in enumerate(detection_classes):\r\n if not any(m1 == i):\r\n self.matrix[dc, self.nc] += 1 # background FN\r\n\r\n def matrix(self):\r\n return self.matrix\r\n\r\n def plot(self, save_dir='', names=()):\r\n try:\r\n import seaborn as sn\r\n\r\n array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize\r\n array[array < 0.005] = np.nan # don't annotate (would appear as 0.00)\r\n\r\n fig = plt.figure(figsize=(12, 9), tight_layout=True)\r\n sn.set(font_scale=1.0 if self.nc < 50 else 0.8) # for label size\r\n labels = (0 < len(names) < 99) and len(names) == self.nc # apply names to ticklabels\r\n sn.heatmap(array, annot=self.nc < 30, annot_kws={\"size\": 8}, cmap='Blues', fmt='.2f', square=True,\r\n xticklabels=names + ['background FP'] if labels else \"auto\",\r\n yticklabels=names + ['background FN'] if labels else \"auto\").set_facecolor((1, 1, 1))\r\n fig.axes[0].set_xlabel('True')\r\n fig.axes[0].set_ylabel('Predicted')\r\n fig.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250)\r\n except Exception as e:\r\n pass\r\n\r\n def print(self):\r\n for i in range(self.nc + 1):\r\n print(' '.join(map(str, self.matrix[i])))\r\n\r\n\r\n# Plots ----------------------------------------------------------------------------------------------------------------\r\n\r\ndef plot_pr_curve(px, py, ap, save_dir='pr_curve.png', names=()):\r\n # Precision-recall curve\r\n fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)\r\n py = np.stack(py, axis=1)\r\n\r\n if 0 < len(names) < 21: # display per-class legend if < 21 classes\r\n for i, y in enumerate(py.T):\r\n ax.plot(px, y, linewidth=1, label=f'{names[i]} {ap[i, 0]:.3f}') # plot(recall, precision)\r\n else:\r\n ax.plot(px, py, linewidth=1, color='grey') # plot(recall, precision)\r\n\r\n ax.plot(px, py.mean(1), linewidth=3, color='blue', label='all classes %.3f mAP@0.5' % ap[:, 0].mean())\r\n ax.set_xlabel('Recall')\r\n ax.set_ylabel('Precision')\r\n ax.set_xlim(0, 1)\r\n ax.set_ylim(0, 1)\r\n plt.legend(bbox_to_anchor=(1.04, 1), loc=\"upper left\")\r\n fig.savefig(Path(save_dir), dpi=250)\r\n\r\n\r\ndef plot_mc_curve(px, py, save_dir='mc_curve.png', names=(), xlabel='Confidence', ylabel='Metric'):\r\n # Metric-confidence curve\r\n fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True)\r\n\r\n if 0 < len(names) < 21: # display per-class legend if < 21 classes\r\n for i, y in enumerate(py):\r\n ax.plot(px, y, linewidth=1, label=f'{names[i]}') # plot(confidence, metric)\r\n else:\r\n ax.plot(px, py.T, linewidth=1, color='grey') # plot(confidence, metric)\r\n\r\n y = py.mean(0)\r\n ax.plot(px, y, linewidth=3, color='blue', label=f'all classes {y.max():.2f} at {px[y.argmax()]:.3f}')\r\n ax.set_xlabel(xlabel)\r\n ax.set_ylabel(ylabel)\r\n ax.set_xlim(0, 1)\r\n ax.set_ylim(0, 1)\r\n plt.legend(bbox_to_anchor=(1.04, 1), loc=\"upper left\")\r\n fig.savefig(Path(save_dir), dpi=250)\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/plots.py", "content": "# Plotting utils\r\n\r\nimport glob\r\nimport math\r\nimport os\r\nimport random\r\nfrom copy import copy\r\nfrom pathlib import Path\r\n\r\nimport cv2\r\nimport matplotlib\r\nimport matplotlib.pyplot as plt\r\nimport numpy as np\r\nimport pandas as pd\r\nimport seaborn as sns\r\nimport torch\r\nimport yaml\r\nfrom PIL import Image, ImageDraw, ImageFont\r\nfrom scipy.signal import butter, filtfilt\r\n\r\nfrom asone.pose_estimators.yolov7_pose.utils.general import xywh2xyxy, xyxy2xywh\r\nfrom asone.pose_estimators.yolov7_pose.utils.metrics import fitness\r\n\r\n# Settings\r\nmatplotlib.rc('font', **{'size': 11})\r\nmatplotlib.use('Agg') # for writing to files only\r\n\r\nclass Colors:\r\n # Ultralytics color palette https://ultralytics.com/\r\n def __init__(self):\r\n self.palette = [self.hex2rgb(c) for c in matplotlib.colors.TABLEAU_COLORS.values()]\r\n self.n = len(self.palette)\r\n\r\n def __call__(self, i, bgr=False):\r\n c = self.palette[int(i) % self.n]\r\n return (c[2], c[1], c[0]) if bgr else c\r\n\r\n @staticmethod\r\n def hex2rgb(h): # rgb order (PIL)\r\n return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4))\r\n\r\n\r\ndef plot_one_box_kpt(x, im, color=None, label=None, line_thickness=3, kpt_label=False, kpts=None, steps=2, orig_shape=None):\r\n # Plots one bounding box on image 'im' using OpenCV\r\n assert im.data.contiguous, 'Image not contiguous. Apply np.ascontiguousarray(im) to plot_on_box() input image.'\r\n tl = line_thickness or round(0.002 * (im.shape[0] + im.shape[1]) / 2) + 1 # line/font thickness\r\n color = color or [random.randint(0, 255) for _ in range(3)]\r\n c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\r\n # cv2.rectangle(im, c1, c2, (255,0,0), thickness=tl*1//3, lineType=cv2.LINE_AA)\r\n if label:\r\n if len(label.split(' ')) > 1:\r\n label = label.split(' ')[-1]\r\n tf = max(tl - 1, 1) # font thickness\r\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 6, thickness=tf)[0]\r\n c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\r\n cv2.rectangle(im, c1, c2, color, -1, cv2.LINE_AA) # filled\r\n cv2.putText(im, label, (c1[0], c1[1] - 2), 0, tl / 6, [225, 255, 255], thickness=tf//2, lineType=cv2.LINE_AA)\r\n if kpt_label:\r\n plot_skeleton_kpts(im, kpts, steps, orig_shape=orig_shape)\r\n\r\ncolors = Colors() \r\n\r\ndef color_list():\r\n def hex2rgb(h):\r\n return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4))\r\n\r\n return [hex2rgb(h) for h in matplotlib.colors.TABLEAU_COLORS.values()] # or BASE_ (8), CSS4_ (148), XKCD_ (949)\r\n\r\n\r\ndef hist2d(x, y, n=100):\r\n # 2d histogram used in labels.png and evolve.png\r\n xedges, yedges = np.linspace(x.min(), x.max(), n), np.linspace(y.min(), y.max(), n)\r\n hist, xedges, yedges = np.histogram2d(x, y, (xedges, yedges))\r\n xidx = np.clip(np.digitize(x, xedges) - 1, 0, hist.shape[0] - 1)\r\n yidx = np.clip(np.digitize(y, yedges) - 1, 0, hist.shape[1] - 1)\r\n return np.log(hist[xidx, yidx])\r\n\r\n\r\ndef butter_lowpass_filtfilt(data, cutoff=1500, fs=50000, order=5):\r\n # https://stackoverflow.com/questions/28536191/how-to-filter-smooth-with-scipy-numpy\r\n def butter_lowpass(cutoff, fs, order):\r\n nyq = 0.5 * fs\r\n normal_cutoff = cutoff / nyq\r\n return butter(order, normal_cutoff, btype='low', analog=False)\r\n\r\n b, a = butter_lowpass(cutoff, fs, order=order)\r\n return filtfilt(b, a, data) # forward-backward filter\r\n\r\n\r\ndef plot_one_box(x, img, color=None, label=None, line_thickness=1):\r\n # Plots one bounding box on image img\r\n tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 2 # line/font thickness\r\n color = color or [random.randint(0, 255) for _ in range(3)]\r\n c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\r\n cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\r\n if label:\r\n tf = max(tl - 1, 1) # font thickness\r\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\r\n c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\r\n cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled\r\n cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\r\n\r\n\r\ndef plot_one_box_PIL(box, img, color=None, label=None, line_thickness=None):\r\n img = Image.fromarray(img)\r\n draw = ImageDraw.Draw(img)\r\n line_thickness = line_thickness or max(int(min(img.size) / 200), 2)\r\n draw.rectangle(box, width=line_thickness, outline=tuple(color)) # plot\r\n if label:\r\n fontsize = max(round(max(img.size) / 40), 12)\r\n font = ImageFont.truetype(\"Arial.ttf\", fontsize)\r\n txt_width, txt_height = font.getsize(label)\r\n draw.rectangle([box[0], box[1] - txt_height + 4, box[0] + txt_width, box[1]], fill=tuple(color))\r\n draw.text((box[0], box[1] - txt_height + 1), label, fill=(255, 255, 255), font=font)\r\n return np.asarray(img)\r\n\r\n\r\ndef plot_wh_methods(): # from utils.plots import *; plot_wh_methods()\r\n # Compares the two methods for width-height anchor multiplication\r\n # https://github.com/ultralytics/yolov3/issues/168\r\n x = np.arange(-4.0, 4.0, .1)\r\n ya = np.exp(x)\r\n yb = torch.sigmoid(torch.from_numpy(x)).numpy() * 2\r\n\r\n fig = plt.figure(figsize=(6, 3), tight_layout=True)\r\n plt.plot(x, ya, '.-', label='YOLOv3')\r\n plt.plot(x, yb ** 2, '.-', label='YOLOR ^2')\r\n plt.plot(x, yb ** 1.6, '.-', label='YOLOR ^1.6')\r\n plt.xlim(left=-4, right=4)\r\n plt.ylim(bottom=0, top=6)\r\n plt.xlabel('input')\r\n plt.ylabel('output')\r\n plt.grid()\r\n plt.legend()\r\n fig.savefig('comparison.png', dpi=200)\r\n\r\n\r\ndef output_to_target(output):\r\n # Convert model output to target format [batch_id, class_id, x, y, w, h, conf]\r\n targets = []\r\n for i, o in enumerate(output):\r\n for *box, conf, cls in o.cpu().numpy():\r\n targets.append([i, cls, *list(*xyxy2xywh(np.array(box)[None])), conf])\r\n return np.array(targets)\r\n\r\n\r\ndef plot_images(images, targets, paths=None, fname='images.jpg', names=None, max_size=640, max_subplots=16):\r\n # Plot image grid with labels\r\n\r\n if isinstance(images, torch.Tensor):\r\n images = images.cpu().float().numpy()\r\n if isinstance(targets, torch.Tensor):\r\n targets = targets.cpu().numpy()\r\n\r\n # un-normalise\r\n if np.max(images[0]) <= 1:\r\n images *= 255\r\n\r\n tl = 3 # line thickness\r\n tf = max(tl - 1, 1) # font thickness\r\n bs, _, h, w = images.shape # batch size, _, height, width\r\n bs = min(bs, max_subplots) # limit plot images\r\n ns = np.ceil(bs ** 0.5) # number of subplots (square)\r\n\r\n # Check if we should resize\r\n scale_factor = max_size / max(h, w)\r\n if scale_factor < 1:\r\n h = math.ceil(scale_factor * h)\r\n w = math.ceil(scale_factor * w)\r\n\r\n colors = color_list() # list of colors\r\n mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init\r\n for i, img in enumerate(images):\r\n if i == max_subplots: # if last batch has fewer images than we expect\r\n break\r\n\r\n block_x = int(w * (i // ns))\r\n block_y = int(h * (i % ns))\r\n\r\n img = img.transpose(1, 2, 0)\r\n if scale_factor < 1:\r\n img = cv2.resize(img, (w, h))\r\n\r\n mosaic[block_y:block_y + h, block_x:block_x + w, :] = img\r\n if len(targets) > 0:\r\n image_targets = targets[targets[:, 0] == i]\r\n boxes = xywh2xyxy(image_targets[:, 2:6]).T\r\n classes = image_targets[:, 1].astype('int')\r\n labels = image_targets.shape[1] == 6 # labels if no conf column\r\n conf = None if labels else image_targets[:, 6] # check for confidence presence (label vs pred)\r\n\r\n if boxes.shape[1]:\r\n if boxes.max() <= 1.01: # if normalized with tolerance 0.01\r\n boxes[[0, 2]] *= w # scale to pixels\r\n boxes[[1, 3]] *= h\r\n elif scale_factor < 1: # absolute coords need scale if image scales\r\n boxes *= scale_factor\r\n boxes[[0, 2]] += block_x\r\n boxes[[1, 3]] += block_y\r\n for j, box in enumerate(boxes.T):\r\n cls = int(classes[j])\r\n color = colors[cls % len(colors)]\r\n cls = names[cls] if names else cls\r\n if labels or conf[j] > 0.25: # 0.25 conf thresh\r\n label = '%s' % cls if labels else '%s %.1f' % (cls, conf[j])\r\n plot_one_box(box, mosaic, label=label, color=color, line_thickness=tl)\r\n\r\n # Draw image filename labels\r\n if paths:\r\n label = Path(paths[i]).name[:40] # trim to 40 char\r\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\r\n cv2.putText(mosaic, label, (block_x + 5, block_y + t_size[1] + 5), 0, tl / 3, [220, 220, 220], thickness=tf,\r\n lineType=cv2.LINE_AA)\r\n\r\n # Image border\r\n cv2.rectangle(mosaic, (block_x, block_y), (block_x + w, block_y + h), (255, 255, 255), thickness=3)\r\n\r\n if fname:\r\n r = min(1280. / max(h, w) / ns, 1.0) # ratio to limit image size\r\n mosaic = cv2.resize(mosaic, (int(ns * w * r), int(ns * h * r)), interpolation=cv2.INTER_AREA)\r\n # cv2.imwrite(fname, cv2.cvtColor(mosaic, cv2.COLOR_BGR2RGB)) # cv2 save\r\n Image.fromarray(mosaic).save(fname) # PIL save\r\n return mosaic\r\n\r\n\r\ndef plot_lr_scheduler(optimizer, scheduler, epochs=300, save_dir=''):\r\n # Plot LR simulating training for full epochs\r\n optimizer, scheduler = copy(optimizer), copy(scheduler) # do not modify originals\r\n y = []\r\n for _ in range(epochs):\r\n scheduler.step()\r\n y.append(optimizer.param_groups[0]['lr'])\r\n plt.plot(y, '.-', label='LR')\r\n plt.xlabel('epoch')\r\n plt.ylabel('LR')\r\n plt.grid()\r\n plt.xlim(0, epochs)\r\n plt.ylim(0)\r\n plt.savefig(Path(save_dir) / 'LR.png', dpi=200)\r\n plt.close()\r\n\r\n\r\ndef plot_test_txt(): # from utils.plots import *; plot_test()\r\n # Plot test.txt histograms\r\n x = np.loadtxt('test.txt', dtype=np.float32)\r\n box = xyxy2xywh(x[:, :4])\r\n cx, cy = box[:, 0], box[:, 1]\r\n\r\n fig, ax = plt.subplots(1, 1, figsize=(6, 6), tight_layout=True)\r\n ax.hist2d(cx, cy, bins=600, cmax=10, cmin=0)\r\n ax.set_aspect('equal')\r\n plt.savefig('hist2d.png', dpi=300)\r\n\r\n fig, ax = plt.subplots(1, 2, figsize=(12, 6), tight_layout=True)\r\n ax[0].hist(cx, bins=600)\r\n ax[1].hist(cy, bins=600)\r\n plt.savefig('hist1d.png', dpi=200)\r\n\r\n\r\ndef plot_targets_txt(): # from utils.plots import *; plot_targets_txt()\r\n # Plot targets.txt histograms\r\n x = np.loadtxt('targets.txt', dtype=np.float32).T\r\n s = ['x targets', 'y targets', 'width targets', 'height targets']\r\n fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)\r\n ax = ax.ravel()\r\n for i in range(4):\r\n ax[i].hist(x[i], bins=100, label='%.3g +/- %.3g' % (x[i].mean(), x[i].std()))\r\n ax[i].legend()\r\n ax[i].set_title(s[i])\r\n plt.savefig('targets.jpg', dpi=200)\r\n\r\n\r\ndef plot_study_txt(path='', x=None): # from utils.plots import *; plot_study_txt()\r\n # Plot study.txt generated by test.py\r\n fig, ax = plt.subplots(2, 4, figsize=(10, 6), tight_layout=True)\r\n # ax = ax.ravel()\r\n\r\n fig2, ax2 = plt.subplots(1, 1, figsize=(8, 4), tight_layout=True)\r\n # for f in [Path(path) / f'study_coco_{x}.txt' for x in ['yolor-p6', 'yolor-w6', 'yolor-e6', 'yolor-d6']]:\r\n for f in sorted(Path(path).glob('study*.txt')):\r\n y = np.loadtxt(f, dtype=np.float32, usecols=[0, 1, 2, 3, 7, 8, 9], ndmin=2).T\r\n x = np.arange(y.shape[1]) if x is None else np.array(x)\r\n s = ['P', 'R', 'mAP@.5', 'mAP@.5:.95', 't_inference (ms/img)', 't_NMS (ms/img)', 't_total (ms/img)']\r\n # for i in range(7):\r\n # ax[i].plot(x, y[i], '.-', linewidth=2, markersize=8)\r\n # ax[i].set_title(s[i])\r\n\r\n j = y[3].argmax() + 1\r\n ax2.plot(y[6, 1:j], y[3, 1:j] * 1E2, '.-', linewidth=2, markersize=8,\r\n label=f.stem.replace('study_coco_', '').replace('yolo', 'YOLO'))\r\n\r\n ax2.plot(1E3 / np.array([209, 140, 97, 58, 35, 18]), [34.6, 40.5, 43.0, 47.5, 49.7, 51.5],\r\n 'k.-', linewidth=2, markersize=8, alpha=.25, label='EfficientDet')\r\n\r\n ax2.grid(alpha=0.2)\r\n ax2.set_yticks(np.arange(20, 60, 5))\r\n ax2.set_xlim(0, 57)\r\n ax2.set_ylim(30, 55)\r\n ax2.set_xlabel('GPU Speed (ms/img)')\r\n ax2.set_ylabel('COCO AP val')\r\n ax2.legend(loc='lower right')\r\n plt.savefig(str(Path(path).name) + '.png', dpi=300)\r\n\r\n\r\ndef plot_labels(labels, names=(), save_dir=Path(''), loggers=None):\r\n # plot dataset labels\r\n print('Plotting labels... ')\r\n c, b = labels[:, 0], labels[:, 1:].transpose() # classes, boxes\r\n nc = int(c.max() + 1) # number of classes\r\n colors = color_list()\r\n x = pd.DataFrame(b.transpose(), columns=['x', 'y', 'width', 'height'])\r\n\r\n # seaborn correlogram\r\n sns.pairplot(x, corner=True, diag_kind='auto', kind='hist', diag_kws=dict(bins=50), plot_kws=dict(pmax=0.9))\r\n plt.savefig(save_dir / 'labels_correlogram.jpg', dpi=200)\r\n plt.close()\r\n\r\n # matplotlib labels\r\n matplotlib.use('svg') # faster\r\n ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)[1].ravel()\r\n ax[0].hist(c, bins=np.linspace(0, nc, nc + 1) - 0.5, rwidth=0.8)\r\n ax[0].set_ylabel('instances')\r\n if 0 < len(names) < 30:\r\n ax[0].set_xticks(range(len(names)))\r\n ax[0].set_xticklabels(names, rotation=90, fontsize=10)\r\n else:\r\n ax[0].set_xlabel('classes')\r\n sns.histplot(x, x='x', y='y', ax=ax[2], bins=50, pmax=0.9)\r\n sns.histplot(x, x='width', y='height', ax=ax[3], bins=50, pmax=0.9)\r\n\r\n # rectangles\r\n labels[:, 1:3] = 0.5 # center\r\n labels[:, 1:] = xywh2xyxy(labels[:, 1:]) * 2000\r\n img = Image.fromarray(np.ones((2000, 2000, 3), dtype=np.uint8) * 255)\r\n for cls, *box in labels[:1000]:\r\n ImageDraw.Draw(img).rectangle(box, width=1, outline=colors[int(cls) % 10]) # plot\r\n ax[1].imshow(img)\r\n ax[1].axis('off')\r\n\r\n for a in [0, 1, 2, 3]:\r\n for s in ['top', 'right', 'left', 'bottom']:\r\n ax[a].spines[s].set_visible(False)\r\n\r\n plt.savefig(save_dir / 'labels.jpg', dpi=200)\r\n matplotlib.use('Agg')\r\n plt.close()\r\n\r\n # loggers\r\n for k, v in loggers.items() or {}:\r\n if k == 'wandb' and v:\r\n v.log({\"Labels\": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, commit=False)\r\n\r\n\r\ndef plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.plots import *; plot_evolution()\r\n # Plot hyperparameter evolution results in evolve.txt\r\n with open(yaml_file) as f:\r\n hyp = yaml.load(f, Loader=yaml.SafeLoader)\r\n x = np.loadtxt('evolve.txt', ndmin=2)\r\n f = fitness(x)\r\n # weights = (f - f.min()) ** 2 # for weighted results\r\n plt.figure(figsize=(10, 12), tight_layout=True)\r\n matplotlib.rc('font', **{'size': 8})\r\n for i, (k, v) in enumerate(hyp.items()):\r\n y = x[:, i + 7]\r\n # mu = (y * weights).sum() / weights.sum() # best weighted result\r\n mu = y[f.argmax()] # best single result\r\n plt.subplot(6, 5, i + 1)\r\n plt.scatter(y, f, c=hist2d(y, f, 20), cmap='viridis', alpha=.8, edgecolors='none')\r\n plt.plot(mu, f.max(), 'k+', markersize=15)\r\n plt.title('%s = %.3g' % (k, mu), fontdict={'size': 9}) # limit to 40 characters\r\n if i % 5 != 0:\r\n plt.yticks([])\r\n print('%15s: %.3g' % (k, mu))\r\n plt.savefig('evolve.png', dpi=200)\r\n print('\\nPlot saved as evolve.png')\r\n\r\n\r\ndef profile_idetection(start=0, stop=0, labels=(), save_dir=''):\r\n # Plot iDetection '*.txt' per-image logs. from utils.plots import *; profile_idetection()\r\n ax = plt.subplots(2, 4, figsize=(12, 6), tight_layout=True)[1].ravel()\r\n s = ['Images', 'Free Storage (GB)', 'RAM Usage (GB)', 'Battery', 'dt_raw (ms)', 'dt_smooth (ms)', 'real-world FPS']\r\n files = list(Path(save_dir).glob('frames*.txt'))\r\n for fi, f in enumerate(files):\r\n try:\r\n results = np.loadtxt(f, ndmin=2).T[:, 90:-30] # clip first and last rows\r\n n = results.shape[1] # number of rows\r\n x = np.arange(start, min(stop, n) if stop else n)\r\n results = results[:, x]\r\n t = (results[0] - results[0].min()) # set t0=0s\r\n results[0] = x\r\n for i, a in enumerate(ax):\r\n if i < len(results):\r\n label = labels[fi] if len(labels) else f.stem.replace('frames_', '')\r\n a.plot(t, results[i], marker='.', label=label, linewidth=1, markersize=5)\r\n a.set_title(s[i])\r\n a.set_xlabel('time (s)')\r\n # if fi == len(files) - 1:\r\n # a.set_ylim(bottom=0)\r\n for side in ['top', 'right']:\r\n a.spines[side].set_visible(False)\r\n else:\r\n a.remove()\r\n except Exception as e:\r\n print('Warning: Plotting error for %s; %s' % (f, e))\r\n\r\n ax[1].legend()\r\n plt.savefig(Path(save_dir) / 'idetection_profile.png', dpi=200)\r\n\r\n\r\ndef plot_results_overlay(start=0, stop=0): # from utils.plots import *; plot_results_overlay()\r\n # Plot training 'results*.txt', overlaying train and val losses\r\n s = ['train', 'train', 'train', 'Precision', 'mAP@0.5', 'val', 'val', 'val', 'Recall', 'mAP@0.5:0.95'] # legends\r\n t = ['Box', 'Objectness', 'Classification', 'P-R', 'mAP-F1'] # titles\r\n for f in sorted(glob.glob('results*.txt') + glob.glob('../../Downloads/results*.txt')):\r\n results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T\r\n n = results.shape[1] # number of rows\r\n x = range(start, min(stop, n) if stop else n)\r\n fig, ax = plt.subplots(1, 5, figsize=(14, 3.5), tight_layout=True)\r\n ax = ax.ravel()\r\n for i in range(5):\r\n for j in [i, i + 5]:\r\n y = results[j, x]\r\n ax[i].plot(x, y, marker='.', label=s[j])\r\n # y_smooth = butter_lowpass_filtfilt(y)\r\n # ax[i].plot(x, np.gradient(y_smooth), marker='.', label=s[j])\r\n\r\n ax[i].set_title(t[i])\r\n ax[i].legend()\r\n ax[i].set_ylabel(f) if i == 0 else None # add filename\r\n fig.savefig(f.replace('.txt', '.png'), dpi=200)\r\n\r\n\r\ndef plot_results(start=0, stop=0, bucket='', id=(), labels=(), save_dir=''):\r\n # Plot training 'results*.txt'. from utils.plots import *; plot_results(save_dir='runs/train/exp')\r\n fig, ax = plt.subplots(2, 5, figsize=(12, 6), tight_layout=True)\r\n ax = ax.ravel()\r\n s = ['Box', 'Objectness', 'Classification', 'Precision', 'Recall',\r\n 'val Box', 'val Objectness', 'val Classification', 'mAP@0.5', 'mAP@0.5:0.95']\r\n if bucket:\r\n # files = ['https://storage.googleapis.com/%s/results%g.txt' % (bucket, x) for x in id]\r\n files = ['results%g.txt' % x for x in id]\r\n c = ('gsutil cp ' + '%s ' * len(files) + '.') % tuple('gs://%s/results%g.txt' % (bucket, x) for x in id)\r\n os.system(c)\r\n else:\r\n files = list(Path(save_dir).glob('results*.txt'))\r\n assert len(files), 'No results.txt files found in %s, nothing to plot.' % os.path.abspath(save_dir)\r\n for fi, f in enumerate(files):\r\n try:\r\n results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T\r\n n = results.shape[1] # number of rows\r\n x = range(start, min(stop, n) if stop else n)\r\n for i in range(10):\r\n y = results[i, x]\r\n if i in [0, 1, 2, 5, 6, 7]:\r\n y[y == 0] = np.nan # don't show zero loss values\r\n # y /= y[0] # normalize\r\n label = labels[fi] if len(labels) else f.stem\r\n ax[i].plot(x, y, marker='.', label=label, linewidth=2, markersize=8)\r\n ax[i].set_title(s[i])\r\n # if i in [5, 6, 7]: # share train and val loss y axes\r\n # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5])\r\n except Exception as e:\r\n print('Warning: Plotting error for %s; %s' % (f, e))\r\n\r\n ax[1].legend()\r\n fig.savefig(Path(save_dir) / 'results.png', dpi=200)\r\n \r\n \r\ndef output_to_keypoint(output):\r\n # Convert model output to target format [batch_id, class_id, x, y, w, h, conf]\r\n targets = []\r\n for i, o in enumerate(output):\r\n kpts = o[:,6:]\r\n o = o[:,:6]\r\n for index, (*box, conf, cls) in enumerate(o.cpu().numpy()):\r\n targets.append([i, cls, *list(*xyxy2xywh(np.array(box)[None])), conf, *list(kpts.cpu().numpy()[index])])\r\n return np.array(targets)\r\n\r\n\r\ndef plot_skeleton_kpts(im, kpts, steps, orig_shape=None):\r\n #Plot the skeleton and keypointsfor coco datatset\r\n palette = np.array([[255, 128, 0], [255, 153, 51], [255, 178, 102],\r\n [230, 230, 0], [255, 153, 255], [153, 204, 255],\r\n [255, 102, 255], [255, 51, 255], [102, 178, 255],\r\n [51, 153, 255], [255, 153, 153], [255, 102, 102],\r\n [255, 51, 51], [153, 255, 153], [102, 255, 102],\r\n [51, 255, 51], [0, 255, 0], [0, 0, 255], [255, 0, 0],\r\n [255, 255, 255]])\r\n\r\n skeleton = [[16, 14], [14, 12], [17, 15], [15, 13], [12, 13], [6, 12],\r\n [7, 13], [6, 7], [6, 8], [7, 9], [8, 10], [9, 11], [2, 3],\r\n [1, 2], [1, 3], [2, 4], [3, 5], [4, 6], [5, 7]]\r\n\r\n pose_limb_color = palette[[9, 9, 9, 9, 7, 7, 7, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16]]\r\n pose_kpt_color = palette[[16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9]]\r\n radius = 5\r\n num_kpts = len(kpts) // steps\r\n\r\n for kid in range(num_kpts):\r\n r, g, b = pose_kpt_color[kid]\r\n x_coord, y_coord = kpts[steps * kid], kpts[steps * kid + 1]\r\n if not (x_coord % 640 == 0 or y_coord % 640 == 0):\r\n if steps == 3:\r\n conf = kpts[steps * kid + 2]\r\n if conf < 0.5:\r\n continue\r\n overlay = im.copy()\r\n alpha = 0.4\r\n cv2.circle(overlay, (int(x_coord), int(y_coord)), 8, (int(220), int(237), int(245)), 8)\r\n cv2.circle(im, (int(x_coord), int(y_coord)), 5, (int(255), int(255), int(255)), -1)\r\n # im = output\r\n cv2.addWeighted(overlay, alpha, im, 1 - alpha, 0, im)\r\n\r\n for sk_id, sk in enumerate(skeleton):\r\n r, g, b = pose_limb_color[sk_id]\r\n pos1 = (int(kpts[(sk[0]-1)*steps]), int(kpts[(sk[0]-1)*steps+1]))\r\n pos2 = (int(kpts[(sk[1]-1)*steps]), int(kpts[(sk[1]-1)*steps+1]))\r\n if steps == 3:\r\n conf1 = kpts[(sk[0]-1)*steps+2]\r\n conf2 = kpts[(sk[1]-1)*steps+2]\r\n if conf1<0.5 or conf2<0.5:\r\n continue\r\n if pos1[0]%640 == 0 or pos1[1]%640==0 or pos1[0]<0 or pos1[1]<0:\r\n continue\r\n if pos2[0] % 640 == 0 or pos2[1] % 640 == 0 or pos2[0]<0 or pos2[1]<0:\r\n continue\r\n cv2.line(im, pos1, pos2, (int(255), int(255), int(255)), thickness=2)\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/torch_utils.py", "content": "# YOLOR PyTorch utils\r\n\r\nimport datetime\r\nimport logging\r\nimport math\r\nimport os\r\nimport platform\r\nimport subprocess\r\nimport time\r\nfrom contextlib import contextmanager\r\nfrom copy import deepcopy\r\nfrom pathlib import Path\r\n\r\nimport torch\r\nimport torch.backends.cudnn as cudnn\r\nimport torch.nn as nn\r\nimport torch.nn.functional as F\r\nimport torchvision\r\n\r\ntry:\r\n import thop # for FLOPS computation\r\nexcept ImportError:\r\n thop = None\r\nlogger = logging.getLogger(__name__)\r\n\r\n\r\n@contextmanager\r\ndef torch_distributed_zero_first(local_rank: int):\r\n \"\"\"\r\n Decorator to make all processes in distributed training wait for each local_master to do something.\r\n \"\"\"\r\n if local_rank not in [-1, 0]:\r\n torch.distributed.barrier()\r\n yield\r\n if local_rank == 0:\r\n torch.distributed.barrier()\r\n\r\n\r\ndef init_torch_seeds(seed=0):\r\n # Speed-reproducibility tradeoff https://pytorch.org/docs/stable/notes/randomness.html\r\n torch.manual_seed(seed)\r\n if seed == 0: # slower, more reproducible\r\n cudnn.benchmark, cudnn.deterministic = False, True\r\n else: # faster, less reproducible\r\n cudnn.benchmark, cudnn.deterministic = True, False\r\n\r\n\r\ndef date_modified(path=__file__):\r\n # return human-readable file modification date, i.e. '2021-3-26'\r\n t = datetime.datetime.fromtimestamp(Path(path).stat().st_mtime)\r\n return f'{t.year}-{t.month}-{t.day}'\r\n\r\n\r\ndef git_describe(path=Path(__file__).parent): # path must be a directory\r\n # return human-readable git description, i.e. v5.0-5-g3e25f1e https://git-scm.com/docs/git-describe\r\n s = f'git -C {path} describe --tags --long --always'\r\n try:\r\n return subprocess.check_output(s, shell=True, stderr=subprocess.STDOUT).decode()[:-1]\r\n except subprocess.CalledProcessError as e:\r\n return '' # not a git repository\r\n\r\n\r\ndef select_device(device='', batch_size=None):\r\n # device = 'cpu' or '0' or '0,1,2,3'\r\n s = f'YOLOR 🚀 {git_describe() or date_modified()} torch {torch.__version__} ' # string\r\n cpu = device.lower() == 'cpu'\r\n if cpu:\r\n os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # force torch.cuda.is_available() = False\r\n elif device: # non-cpu device requested\r\n os.environ['CUDA_VISIBLE_DEVICES'] = device # set environment variable\r\n assert torch.cuda.is_available(), f'CUDA unavailable, invalid device {device} requested' # check availability\r\n\r\n cuda = not cpu and torch.cuda.is_available()\r\n if cuda:\r\n n = torch.cuda.device_count()\r\n if n > 1 and batch_size: # check that batch_size is compatible with device_count\r\n assert batch_size % n == 0, f'batch-size {batch_size} not multiple of GPU count {n}'\r\n space = ' ' * len(s)\r\n for i, d in enumerate(device.split(',') if device else range(n)):\r\n p = torch.cuda.get_device_properties(i)\r\n s += f\"{'' if i == 0 else space}CUDA:{d} ({p.name}, {p.total_memory / 1024 ** 2}MB)\\n\" # bytes to MB\r\n else:\r\n s += 'CPU\\n'\r\n\r\n logger.info(s.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else s) # emoji-safe\r\n return torch.device('cuda:0' if cuda else 'cpu')\r\n\r\n\r\ndef time_synchronized():\r\n # pytorch-accurate time\r\n if torch.cuda.is_available():\r\n torch.cuda.synchronize()\r\n return time.time()\r\n\r\n\r\ndef profile(x, ops, n=100, device=None):\r\n # profile a pytorch module or list of modules. Example usage:\r\n # x = torch.randn(16, 3, 640, 640) # input\r\n # m1 = lambda x: x * torch.sigmoid(x)\r\n # m2 = nn.SiLU()\r\n # profile(x, [m1, m2], n=100) # profile speed over 100 iterations\r\n\r\n device = device or torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\r\n x = x.to(device)\r\n x.requires_grad = True\r\n print(torch.__version__, device.type, torch.cuda.get_device_properties(0) if device.type == 'cuda' else '')\r\n print(f\"\\n{'Params':>12s}{'GFLOPS':>12s}{'forward (ms)':>16s}{'backward (ms)':>16s}{'input':>24s}{'output':>24s}\")\r\n for m in ops if isinstance(ops, list) else [ops]:\r\n m = m.to(device) if hasattr(m, 'to') else m # device\r\n m = m.half() if hasattr(m, 'half') and isinstance(x, torch.Tensor) and x.dtype is torch.float16 else m # type\r\n dtf, dtb, t = 0., 0., [0., 0., 0.] # dt forward, backward\r\n try:\r\n flops = thop.profile(m, inputs=(x,), verbose=False)[0] / 1E9 * 2 # GFLOPS\r\n except:\r\n flops = 0\r\n\r\n for _ in range(n):\r\n t[0] = time_synchronized()\r\n y = m(x)\r\n t[1] = time_synchronized()\r\n try:\r\n _ = y.sum().backward()\r\n t[2] = time_synchronized()\r\n except: # no backward method\r\n t[2] = float('nan')\r\n dtf += (t[1] - t[0]) * 1000 / n # ms per op forward\r\n dtb += (t[2] - t[1]) * 1000 / n # ms per op backward\r\n\r\n s_in = tuple(x.shape) if isinstance(x, torch.Tensor) else 'list'\r\n s_out = tuple(y.shape) if isinstance(y, torch.Tensor) else 'list'\r\n p = sum(list(x.numel() for x in m.parameters())) if isinstance(m, nn.Module) else 0 # parameters\r\n print(f'{p:12}{flops:12.4g}{dtf:16.4g}{dtb:16.4g}{str(s_in):>24s}{str(s_out):>24s}')\r\n\r\n\r\ndef is_parallel(model):\r\n return type(model) in (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)\r\n\r\n\r\ndef intersect_dicts(da, db, exclude=()):\r\n # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values\r\n return {k: v for k, v in da.items() if k in db and not any(x in k for x in exclude) and v.shape == db[k].shape}\r\n\r\n\r\ndef initialize_weights(model):\r\n for m in model.modules():\r\n t = type(m)\r\n if t is nn.Conv2d:\r\n pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\r\n elif t is nn.BatchNorm2d:\r\n m.eps = 1e-3\r\n m.momentum = 0.03\r\n elif t in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6]:\r\n m.inplace = True\r\n\r\n\r\ndef find_modules(model, mclass=nn.Conv2d):\r\n # Finds layer indices matching module class 'mclass'\r\n return [i for i, m in enumerate(model.module_list) if isinstance(m, mclass)]\r\n\r\n\r\ndef sparsity(model):\r\n # Return global model sparsity\r\n a, b = 0., 0.\r\n for p in model.parameters():\r\n a += p.numel()\r\n b += (p == 0).sum()\r\n return b / a\r\n\r\n\r\ndef prune(model, amount=0.3):\r\n # Prune model to requested global sparsity\r\n import torch.nn.utils.prune as prune\r\n print('Pruning model... ', end='')\r\n for name, m in model.named_modules():\r\n if isinstance(m, nn.Conv2d):\r\n prune.l1_unstructured(m, name='weight', amount=amount) # prune\r\n prune.remove(m, 'weight') # make permanent\r\n print(' %.3g global sparsity' % sparsity(model))\r\n\r\n\r\ndef fuse_conv_and_bn(conv, bn):\r\n # Fuse convolution and batchnorm layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/\r\n fusedconv = nn.Conv2d(conv.in_channels,\r\n conv.out_channels,\r\n kernel_size=conv.kernel_size,\r\n stride=conv.stride,\r\n padding=conv.padding,\r\n groups=conv.groups,\r\n bias=True).requires_grad_(False).to(conv.weight.device)\r\n\r\n # prepare filters\r\n w_conv = conv.weight.clone().view(conv.out_channels, -1)\r\n w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var)))\r\n fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape))\r\n\r\n # prepare spatial bias\r\n b_conv = torch.zeros(conv.weight.size(0), device=conv.weight.device) if conv.bias is None else conv.bias\r\n b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps))\r\n fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn)\r\n\r\n return fusedconv\r\n\r\n\r\ndef model_info(model, verbose=False, img_size=640):\r\n # Model information. img_size may be int or list, i.e. img_size=640 or img_size=[640, 320]\r\n n_p = sum(x.numel() for x in model.parameters()) # number parameters\r\n n_g = sum(x.numel() for x in model.parameters() if x.requires_grad) # number gradients\r\n if verbose:\r\n print('%5s %40s %9s %12s %20s %10s %10s' % ('layer', 'name', 'gradient', 'parameters', 'shape', 'mu', 'sigma'))\r\n for i, (name, p) in enumerate(model.named_parameters()):\r\n name = name.replace('module_list.', '')\r\n print('%5g %40s %9s %12g %20s %10.3g %10.3g' %\r\n (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std()))\r\n\r\n try: # FLOPS\r\n from thop import profile\r\n stride = max(int(model.stride.max()), 32) if hasattr(model, 'stride') else 32\r\n img = torch.zeros((1, model.yaml.get('ch', 3), stride, stride), device=next(model.parameters()).device) # input\r\n flops = profile(deepcopy(model), inputs=(img,), verbose=False)[0] / 1E9 * 2 # stride GFLOPS\r\n img_size = img_size if isinstance(img_size, list) else [img_size, img_size] # expand if int/float\r\n fs = ', %.1f GFLOPS' % (flops * img_size[0] / stride * img_size[1] / stride) # 640x640 GFLOPS\r\n except (ImportError, Exception):\r\n fs = ''\r\n\r\n logger.info(f\"Model Summary: {len(list(model.modules()))} layers, {n_p} parameters, {n_g} gradients{fs}\")\r\n\r\n\r\ndef load_classifier(name='resnet101', n=2):\r\n # Loads a pretrained model reshaped to n-class output\r\n model = torchvision.models.__dict__[name](pretrained=True)\r\n\r\n # ResNet model properties\r\n # input_size = [3, 224, 224]\r\n # input_space = 'RGB'\r\n # input_range = [0, 1]\r\n # mean = [0.485, 0.456, 0.406]\r\n # std = [0.229, 0.224, 0.225]\r\n\r\n # Reshape output to n classes\r\n filters = model.fc.weight.shape[1]\r\n model.fc.bias = nn.Parameter(torch.zeros(n), requires_grad=True)\r\n model.fc.weight = nn.Parameter(torch.zeros(n, filters), requires_grad=True)\r\n model.fc.out_features = n\r\n return model\r\n\r\n\r\ndef scale_img(img, ratio=1.0, same_shape=False, gs=32): # img(16,3,256,416)\r\n # scales img(bs,3,y,x) by ratio constrained to gs-multiple\r\n if ratio == 1.0:\r\n return img\r\n else:\r\n h, w = img.shape[2:]\r\n s = (int(h * ratio), int(w * ratio)) # new size\r\n img = F.interpolate(img, size=s, mode='bilinear', align_corners=False) # resize\r\n if not same_shape: # pad/crop img\r\n h, w = [math.ceil(x * ratio / gs) * gs for x in (h, w)]\r\n return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean\r\n\r\n\r\ndef copy_attr(a, b, include=(), exclude=()):\r\n # Copy attributes from b to a, options to only include [...] and to exclude [...]\r\n for k, v in b.__dict__.items():\r\n if (len(include) and k not in include) or k.startswith('_') or k in exclude:\r\n continue\r\n else:\r\n setattr(a, k, v)\r\n\r\n\r\nclass ModelEMA:\r\n \"\"\" Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models\r\n Keep a moving average of everything in the model state_dict (parameters and buffers).\r\n This is intended to allow functionality like\r\n https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage\r\n A smoothed version of the weights is necessary for some training schemes to perform well.\r\n This class is sensitive where it is initialized in the sequence of model init,\r\n GPU assignment and distributed training wrappers.\r\n \"\"\"\r\n\r\n def __init__(self, model, decay=0.9999, updates=0):\r\n # Create EMA\r\n self.ema = deepcopy(model.module if is_parallel(model) else model).eval() # FP32 EMA\r\n # if next(model.parameters()).device.type != 'cpu':\r\n # self.ema.half() # FP16 EMA\r\n self.updates = updates # number of EMA updates\r\n self.decay = lambda x: decay * (1 - math.exp(-x / 2000)) # decay exponential ramp (to help early epochs)\r\n for p in self.ema.parameters():\r\n p.requires_grad_(False)\r\n\r\n def update(self, model):\r\n # Update EMA parameters\r\n with torch.no_grad():\r\n self.updates += 1\r\n d = self.decay(self.updates)\r\n\r\n msd = model.module.state_dict() if is_parallel(model) else model.state_dict() # model state_dict\r\n for k, v in self.ema.state_dict().items():\r\n if v.dtype.is_floating_point:\r\n v *= d\r\n v += (1. - d) * msd[k].detach()\r\n\r\n def update_attr(self, model, include=(), exclude=('process_group', 'reducer')):\r\n # Update EMA attributes\r\n copy_attr(self.ema, model, include, exclude)\r\n\r\n\r\nclass BatchNormXd(torch.nn.modules.batchnorm._BatchNorm):\r\n def _check_input_dim(self, input):\r\n # The only difference between BatchNorm1d, BatchNorm2d, BatchNorm3d, etc\r\n # is this method that is overwritten by the sub-class\r\n # This original goal of this method was for tensor sanity checks\r\n # If you're ok bypassing those sanity checks (eg. if you trust your inference\r\n # to provide the right dimensional inputs), then you can just use this method\r\n # for easy conversion from SyncBatchNorm\r\n # (unfortunately, SyncBatchNorm does not store the original class - if it did\r\n # we could return the one that was originally created)\r\n return\r\n\r\ndef revert_sync_batchnorm(module):\r\n # this is very similar to the function that it is trying to revert:\r\n # https://github.com/pytorch/pytorch/blob/c8b3686a3e4ba63dc59e5dcfe5db3430df256833/torch/nn/modules/batchnorm.py#L679\r\n module_output = module\r\n if isinstance(module, torch.nn.modules.batchnorm.SyncBatchNorm):\r\n new_cls = BatchNormXd\r\n module_output = BatchNormXd(module.num_features,\r\n module.eps, module.momentum,\r\n module.affine,\r\n module.track_running_stats)\r\n if module.affine:\r\n with torch.no_grad():\r\n module_output.weight = module.weight\r\n module_output.bias = module.bias\r\n module_output.running_mean = module.running_mean\r\n module_output.running_var = module.running_var\r\n module_output.num_batches_tracked = module.num_batches_tracked\r\n if hasattr(module, \"qconfig\"):\r\n module_output.qconfig = module.qconfig\r\n for name, child in module.named_children():\r\n module_output.add_module(name, revert_sync_batchnorm(child))\r\n del module\r\n return module_output\r\n\r\n\r\nclass TracedModel(nn.Module):\r\n\r\n def __init__(self, model=None, device=None, img_size=(640,640)): \r\n super(TracedModel, self).__init__()\r\n \r\n print(\" Convert model to Traced-model... \") \r\n self.stride = model.stride\r\n self.names = model.names\r\n self.model = model\r\n\r\n self.model = revert_sync_batchnorm(self.model)\r\n self.model.to('cpu')\r\n self.model.eval()\r\n\r\n self.detect_layer = self.model.model[-1]\r\n self.model.traced = True\r\n \r\n rand_example = torch.rand(1, 3, img_size, img_size)\r\n \r\n traced_script_module = torch.jit.trace(self.model, rand_example, strict=False)\r\n #traced_script_module = torch.jit.script(self.model)\r\n traced_script_module.save(\"traced_model.pt\")\r\n print(\" traced_script_module saved! \")\r\n self.model = traced_script_module\r\n self.model.to(device)\r\n self.detect_layer.to(device)\r\n print(\" model is traced! \\n\") \r\n\r\n def forward(self, x, augment=False, profile=False):\r\n out = self.model(x)\r\n out = self.detect_layer(out)\r\n return out" }, { "path": "asone/pose_estimators/yolov7_pose/utils/wandb_logging/__init__.py", "content": "# init" }, { "path": "asone/pose_estimators/yolov7_pose/utils/wandb_logging/log_dataset.py", "content": "import argparse\r\n\r\nimport yaml\r\n\r\nfrom wandb_utils import WandbLogger\r\n\r\nWANDB_ARTIFACT_PREFIX = 'wandb-artifact://'\r\n\r\n\r\ndef create_dataset_artifact(opt):\r\n with open(opt.data) as f:\r\n data = yaml.load(f, Loader=yaml.SafeLoader) # data dict\r\n logger = WandbLogger(opt, '', None, data, job_type='Dataset Creation')\r\n\r\n\r\nif __name__ == '__main__':\r\n parser = argparse.ArgumentParser()\r\n parser.add_argument('--data', type=str, default='data/coco.yaml', help='data.yaml path')\r\n parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')\r\n parser.add_argument('--project', type=str, default='YOLOR', help='name of W&B Project')\r\n opt = parser.parse_args()\r\n opt.resume = False # Explicitly disallow resume check for dataset upload job\r\n\r\n create_dataset_artifact(opt)\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/wandb_logging/wandb_utils.py", "content": "import json\r\nimport sys\r\nfrom pathlib import Path\r\n\r\nimport torch\r\nimport yaml\r\nfrom tqdm import tqdm\r\n\r\n# sys.path.append(str(Path(__file__).parent.parent.parent)) # add utils/ to path\r\nfrom utils.datasets import LoadImagesAndLabels\r\nfrom utils.datasets import img2label_paths\r\nfrom utils.general import colorstr, xywh2xyxy, check_dataset\r\n\r\ntry:\r\n import wandb\r\n from wandb import init, finish\r\nexcept ImportError:\r\n wandb = None\r\n\r\nWANDB_ARTIFACT_PREFIX = 'wandb-artifact://'\r\n\r\n\r\ndef remove_prefix(from_string, prefix=WANDB_ARTIFACT_PREFIX):\r\n return from_string[len(prefix):]\r\n\r\n\r\ndef check_wandb_config_file(data_config_file):\r\n wandb_config = '_wandb.'.join(data_config_file.rsplit('.', 1)) # updated data.yaml path\r\n if Path(wandb_config).is_file():\r\n return wandb_config\r\n return data_config_file\r\n\r\n\r\ndef get_run_info(run_path):\r\n run_path = Path(remove_prefix(run_path, WANDB_ARTIFACT_PREFIX))\r\n run_id = run_path.stem\r\n project = run_path.parent.stem\r\n model_artifact_name = 'run_' + run_id + '_model'\r\n return run_id, project, model_artifact_name\r\n\r\n\r\ndef check_wandb_resume(opt):\r\n process_wandb_config_ddp_mode(opt) if opt.global_rank not in [-1, 0] else None\r\n if isinstance(opt.resume, str):\r\n if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):\r\n if opt.global_rank not in [-1, 0]: # For resuming DDP runs\r\n run_id, project, model_artifact_name = get_run_info(opt.resume)\r\n api = wandb.Api()\r\n artifact = api.artifact(project + '/' + model_artifact_name + ':latest')\r\n modeldir = artifact.download()\r\n opt.weights = str(Path(modeldir) / \"last.pt\")\r\n return True\r\n return None\r\n\r\n\r\ndef process_wandb_config_ddp_mode(opt):\r\n with open(opt.data) as f:\r\n data_dict = yaml.load(f, Loader=yaml.SafeLoader) # data dict\r\n train_dir, val_dir = None, None\r\n if isinstance(data_dict['train'], str) and data_dict['train'].startswith(WANDB_ARTIFACT_PREFIX):\r\n api = wandb.Api()\r\n train_artifact = api.artifact(remove_prefix(data_dict['train']) + ':' + opt.artifact_alias)\r\n train_dir = train_artifact.download()\r\n train_path = Path(train_dir) / 'data/images/'\r\n data_dict['train'] = str(train_path)\r\n\r\n if isinstance(data_dict['val'], str) and data_dict['val'].startswith(WANDB_ARTIFACT_PREFIX):\r\n api = wandb.Api()\r\n val_artifact = api.artifact(remove_prefix(data_dict['val']) + ':' + opt.artifact_alias)\r\n val_dir = val_artifact.download()\r\n val_path = Path(val_dir) / 'data/images/'\r\n data_dict['val'] = str(val_path)\r\n if train_dir or val_dir:\r\n ddp_data_path = str(Path(val_dir) / 'wandb_local_data.yaml')\r\n with open(ddp_data_path, 'w') as f:\r\n yaml.dump(data_dict, f)\r\n opt.data = ddp_data_path\r\n\r\n\r\nclass WandbLogger():\r\n def __init__(self, opt, name, run_id, data_dict, job_type='Training'):\r\n # Pre-training routine --\r\n self.job_type = job_type\r\n self.wandb, self.wandb_run, self.data_dict = wandb, None if not wandb else wandb.run, data_dict\r\n # It's more elegant to stick to 1 wandb.init call, but useful config data is overwritten in the WandbLogger's wandb.init call\r\n if isinstance(opt.resume, str): # checks resume from artifact\r\n if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):\r\n run_id, project, model_artifact_name = get_run_info(opt.resume)\r\n model_artifact_name = WANDB_ARTIFACT_PREFIX + model_artifact_name\r\n assert wandb, 'install wandb to resume wandb runs'\r\n # Resume wandb-artifact:// runs here| workaround for not overwriting wandb.config\r\n self.wandb_run = wandb.init(id=run_id, project=project, resume='allow')\r\n opt.resume = model_artifact_name\r\n elif self.wandb:\r\n self.wandb_run = wandb.init(config=opt,\r\n resume=\"allow\",\r\n project='YOLOR' if opt.project == 'runs/train' else Path(opt.project).stem,\r\n name=name,\r\n job_type=job_type,\r\n id=run_id) if not wandb.run else wandb.run\r\n if self.wandb_run:\r\n if self.job_type == 'Training':\r\n if not opt.resume:\r\n wandb_data_dict = self.check_and_upload_dataset(opt) if opt.upload_dataset else data_dict\r\n # Info useful for resuming from artifacts\r\n self.wandb_run.config.opt = vars(opt)\r\n self.wandb_run.config.data_dict = wandb_data_dict\r\n self.data_dict = self.setup_training(opt, data_dict)\r\n if self.job_type == 'Dataset Creation':\r\n self.data_dict = self.check_and_upload_dataset(opt)\r\n else:\r\n prefix = colorstr('wandb: ')\r\n print(f\"{prefix}Install Weights & Biases for YOLOR logging with 'pip install wandb' (recommended)\")\r\n\r\n def check_and_upload_dataset(self, opt):\r\n assert wandb, 'Install wandb to upload dataset'\r\n check_dataset(self.data_dict)\r\n config_path = self.log_dataset_artifact(opt.data,\r\n opt.single_cls,\r\n 'YOLOR' if opt.project == 'runs/train' else Path(opt.project).stem)\r\n print(\"Created dataset config file \", config_path)\r\n with open(config_path) as f:\r\n wandb_data_dict = yaml.load(f, Loader=yaml.SafeLoader)\r\n return wandb_data_dict\r\n\r\n def setup_training(self, opt, data_dict):\r\n self.log_dict, self.current_epoch, self.log_imgs = {}, 0, 16 # Logging Constants\r\n self.bbox_interval = opt.bbox_interval\r\n if isinstance(opt.resume, str):\r\n modeldir, _ = self.download_model_artifact(opt)\r\n if modeldir:\r\n self.weights = Path(modeldir) / \"last.pt\"\r\n config = self.wandb_run.config\r\n opt.weights, opt.save_period, opt.batch_size, opt.bbox_interval, opt.epochs, opt.hyp = str(\r\n self.weights), config.save_period, config.total_batch_size, config.bbox_interval, config.epochs, \\\r\n config.opt['hyp']\r\n data_dict = dict(self.wandb_run.config.data_dict) # eliminates the need for config file to resume\r\n if 'val_artifact' not in self.__dict__: # If --upload_dataset is set, use the existing artifact, don't download\r\n self.train_artifact_path, self.train_artifact = self.download_dataset_artifact(data_dict.get('train'),\r\n opt.artifact_alias)\r\n self.val_artifact_path, self.val_artifact = self.download_dataset_artifact(data_dict.get('val'),\r\n opt.artifact_alias)\r\n self.result_artifact, self.result_table, self.val_table, self.weights = None, None, None, None\r\n if self.train_artifact_path is not None:\r\n train_path = Path(self.train_artifact_path) / 'data/images/'\r\n data_dict['train'] = str(train_path)\r\n if self.val_artifact_path is not None:\r\n val_path = Path(self.val_artifact_path) / 'data/images/'\r\n data_dict['val'] = str(val_path)\r\n self.val_table = self.val_artifact.get(\"val\")\r\n self.map_val_table_path()\r\n if self.val_artifact is not None:\r\n self.result_artifact = wandb.Artifact(\"run_\" + wandb.run.id + \"_progress\", \"evaluation\")\r\n self.result_table = wandb.Table([\"epoch\", \"id\", \"prediction\", \"avg_confidence\"])\r\n if opt.bbox_interval == -1:\r\n self.bbox_interval = opt.bbox_interval = (opt.epochs // 10) if opt.epochs > 10 else 1\r\n return data_dict\r\n\r\n def download_dataset_artifact(self, path, alias):\r\n if isinstance(path, str) and path.startswith(WANDB_ARTIFACT_PREFIX):\r\n dataset_artifact = wandb.use_artifact(remove_prefix(path, WANDB_ARTIFACT_PREFIX) + \":\" + alias)\r\n assert dataset_artifact is not None, \"'Error: W&B dataset artifact doesn\\'t exist'\"\r\n datadir = dataset_artifact.download()\r\n return datadir, dataset_artifact\r\n return None, None\r\n\r\n def download_model_artifact(self, opt):\r\n if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):\r\n model_artifact = wandb.use_artifact(remove_prefix(opt.resume, WANDB_ARTIFACT_PREFIX) + \":latest\")\r\n assert model_artifact is not None, 'Error: W&B model artifact doesn\\'t exist'\r\n modeldir = model_artifact.download()\r\n epochs_trained = model_artifact.metadata.get('epochs_trained')\r\n total_epochs = model_artifact.metadata.get('total_epochs')\r\n assert epochs_trained < total_epochs, 'training to %g epochs is finished, nothing to resume.' % (\r\n total_epochs)\r\n return modeldir, model_artifact\r\n return None, None\r\n\r\n def log_model(self, path, opt, epoch, fitness_score, best_model=False):\r\n model_artifact = wandb.Artifact('run_' + wandb.run.id + '_model', type='model', metadata={\r\n 'original_url': str(path),\r\n 'epochs_trained': epoch + 1,\r\n 'save period': opt.save_period,\r\n 'project': opt.project,\r\n 'total_epochs': opt.epochs,\r\n 'fitness_score': fitness_score\r\n })\r\n model_artifact.add_file(str(path / 'last.pt'), name='last.pt')\r\n wandb.log_artifact(model_artifact,\r\n aliases=['latest', 'epoch ' + str(self.current_epoch), 'best' if best_model else ''])\r\n print(\"Saving model artifact on epoch \", epoch + 1)\r\n\r\n def log_dataset_artifact(self, data_file, single_cls, project, overwrite_config=False):\r\n with open(data_file) as f:\r\n data = yaml.load(f, Loader=yaml.SafeLoader) # data dict\r\n nc, names = (1, ['item']) if single_cls else (int(data['nc']), data['names'])\r\n names = {k: v for k, v in enumerate(names)} # to index dictionary\r\n self.train_artifact = self.create_dataset_table(LoadImagesAndLabels(\r\n data['train']), names, name='train') if data.get('train') else None\r\n self.val_artifact = self.create_dataset_table(LoadImagesAndLabels(\r\n data['val']), names, name='val') if data.get('val') else None\r\n if data.get('train'):\r\n data['train'] = WANDB_ARTIFACT_PREFIX + str(Path(project) / 'train')\r\n if data.get('val'):\r\n data['val'] = WANDB_ARTIFACT_PREFIX + str(Path(project) / 'val')\r\n path = data_file if overwrite_config else '_wandb.'.join(data_file.rsplit('.', 1)) # updated data.yaml path\r\n data.pop('download', None)\r\n with open(path, 'w') as f:\r\n yaml.dump(data, f)\r\n\r\n if self.job_type == 'Training': # builds correct artifact pipeline graph\r\n self.wandb_run.use_artifact(self.val_artifact)\r\n self.wandb_run.use_artifact(self.train_artifact)\r\n self.val_artifact.wait()\r\n self.val_table = self.val_artifact.get('val')\r\n self.map_val_table_path()\r\n else:\r\n self.wandb_run.log_artifact(self.train_artifact)\r\n self.wandb_run.log_artifact(self.val_artifact)\r\n return path\r\n\r\n def map_val_table_path(self):\r\n self.val_table_map = {}\r\n print(\"Mapping dataset\")\r\n for i, data in enumerate(tqdm(self.val_table.data)):\r\n self.val_table_map[data[3]] = data[0]\r\n\r\n def create_dataset_table(self, dataset, class_to_id, name='dataset'):\r\n # TODO: Explore multiprocessing to slpit this loop parallely| This is essential for speeding up the the logging\r\n artifact = wandb.Artifact(name=name, type=\"dataset\")\r\n img_files = tqdm([dataset.path]) if isinstance(dataset.path, str) and Path(dataset.path).is_dir() else None\r\n img_files = tqdm(dataset.img_files) if not img_files else img_files\r\n for img_file in img_files:\r\n if Path(img_file).is_dir():\r\n artifact.add_dir(img_file, name='data/images')\r\n labels_path = 'labels'.join(dataset.path.rsplit('images', 1))\r\n artifact.add_dir(labels_path, name='data/labels')\r\n else:\r\n artifact.add_file(img_file, name='data/images/' + Path(img_file).name)\r\n label_file = Path(img2label_paths([img_file])[0])\r\n artifact.add_file(str(label_file),\r\n name='data/labels/' + label_file.name) if label_file.exists() else None\r\n table = wandb.Table(columns=[\"id\", \"train_image\", \"Classes\", \"name\"])\r\n class_set = wandb.Classes([{'id': id, 'name': name} for id, name in class_to_id.items()])\r\n for si, (img, labels, paths, shapes) in enumerate(tqdm(dataset)):\r\n height, width = shapes[0]\r\n labels[:, 2:] = (xywh2xyxy(labels[:, 2:].view(-1, 4))) * torch.Tensor([width, height, width, height])\r\n box_data, img_classes = [], {}\r\n for cls, *xyxy in labels[:, 1:].tolist():\r\n cls = int(cls)\r\n box_data.append({\"position\": {\"minX\": xyxy[0], \"minY\": xyxy[1], \"maxX\": xyxy[2], \"maxY\": xyxy[3]},\r\n \"class_id\": cls,\r\n \"box_caption\": \"%s\" % (class_to_id[cls]),\r\n \"scores\": {\"acc\": 1},\r\n \"domain\": \"pixel\"})\r\n img_classes[cls] = class_to_id[cls]\r\n boxes = {\"ground_truth\": {\"box_data\": box_data, \"class_labels\": class_to_id}} # inference-space\r\n table.add_data(si, wandb.Image(paths, classes=class_set, boxes=boxes), json.dumps(img_classes),\r\n Path(paths).name)\r\n artifact.add(table, name)\r\n return artifact\r\n\r\n def log_training_progress(self, predn, path, names):\r\n if self.val_table and self.result_table:\r\n class_set = wandb.Classes([{'id': id, 'name': name} for id, name in names.items()])\r\n box_data = []\r\n total_conf = 0\r\n for *xyxy, conf, cls in predn.tolist():\r\n if conf >= 0.25:\r\n box_data.append(\r\n {\"position\": {\"minX\": xyxy[0], \"minY\": xyxy[1], \"maxX\": xyxy[2], \"maxY\": xyxy[3]},\r\n \"class_id\": int(cls),\r\n \"box_caption\": \"%s %.3f\" % (names[cls], conf),\r\n \"scores\": {\"class_score\": conf},\r\n \"domain\": \"pixel\"})\r\n total_conf = total_conf + conf\r\n boxes = {\"predictions\": {\"box_data\": box_data, \"class_labels\": names}} # inference-space\r\n id = self.val_table_map[Path(path).name]\r\n self.result_table.add_data(self.current_epoch,\r\n id,\r\n wandb.Image(self.val_table.data[id][1], boxes=boxes, classes=class_set),\r\n total_conf / max(1, len(box_data))\r\n )\r\n\r\n def log(self, log_dict):\r\n if self.wandb_run:\r\n for key, value in log_dict.items():\r\n self.log_dict[key] = value\r\n\r\n def end_epoch(self, best_result=False):\r\n if self.wandb_run:\r\n wandb.log(self.log_dict)\r\n self.log_dict = {}\r\n if self.result_artifact:\r\n train_results = wandb.JoinedTable(self.val_table, self.result_table, \"id\")\r\n self.result_artifact.add(train_results, 'result')\r\n wandb.log_artifact(self.result_artifact, aliases=['latest', 'epoch ' + str(self.current_epoch),\r\n ('best' if best_result else '')])\r\n self.result_table = wandb.Table([\"epoch\", \"id\", \"prediction\", \"avg_confidence\"])\r\n self.result_artifact = wandb.Artifact(\"run_\" + wandb.run.id + \"_progress\", \"evaluation\")\r\n\r\n def finish_run(self):\r\n if self.wandb_run:\r\n if self.log_dict:\r\n wandb.log(self.log_dict)\r\n wandb.run.finish()\r\n" }, { "path": "asone/pose_estimators/yolov7_pose/utils/yolov7_pose_utils.py", "content": "import contextlib\nfrom pathlib import Path\nimport sys\n\n@contextlib.contextmanager\ndef yolov7_in_syspath():\n \"\"\"\n Temporarily add yolov5 folder to `sys.path`.\n \n torch.hub handles it in the same way: https://github.com/pytorch/pytorch/blob/75024e228ca441290b6a1c2e564300ad507d7af6/torch/hub.py#L387\n \n Proper fix for: #22, #134, #353, #1155, #1389, #1680, #2531, #3071 \n No need for such workarounds: #869, #1052, #2949\n \"\"\"\n yolov7_folder_dir = str(Path(__file__).parents[1].absolute())\n try:\n sys.path.insert(0, yolov7_folder_dir)\n yield\n finally:\n sys.path.remove(yolov7_folder_dir)" }, { "path": "asone/pose_estimators/yolov7_pose/yolov7.py", "content": "import cv2\nimport time\nimport torch\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom torchvision import transforms\nfrom asone.pose_estimators.yolov7_pose.utils.datasets import letterbox\nfrom asone.pose_estimators.yolov7_pose.utils.torch_utils import select_device\nfrom asone.pose_estimators.yolov7_pose.models.experimental import attempt_load\nfrom asone.pose_estimators.yolov7_pose.utils.general import non_max_suppression_kpt,strip_optimizer,xyxy2xywh\nfrom asone.pose_estimators.yolov7_pose.utils.plots import output_to_keypoint, plot_skeleton_kpts,colors,plot_one_box_kpt\nimport os\nimport sys\n\n\nclass Yolov7PoseEstimator:\n def __init__(self, weights=\"yolov7-w6-pose.pt\", use_cuda=True): \n self.weights=weights\n # device = device\n device = '0' if use_cuda else 'cpu'\n self.device = select_device(device)\n half = self.device.type != 'cpu'\n self.model = attempt_load(self.weights, map_location=self.device)\n _ = self.model.eval()\n\n @torch.no_grad()\n def estimate(self, frame):\n\n frame_height, frame_width = frame.shape[:2]\n image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n image = letterbox(image, (frame_width), stride=64, auto=True)[0]\n \n image_ = image.copy()\n image = transforms.ToTensor()(image)\n image = torch.tensor(np.array([image.numpy()]))\n\n image = image.to(self.device)\n image = image.float()\n start_time = time.time()\n \n with torch.no_grad():\n output, _ = self.model(image)\n \n output = non_max_suppression_kpt(output, 0.25, 0.65, nc=self.model.yaml['nc'], \n nkpt=self.model.yaml['nkpt'], kpt_label=True)\n output = output_to_keypoint(output)\n \n # ............................................\n # converting output to the format as yolov8\n reformated_output = []\n steps = 3\n kpts = output\n new_size = frame.shape[:2]\n old_size = image.shape[2:]\n ratio = (new_size[0] / old_size[0], new_size[1] / old_size[1])\n for idx in range(output.shape[0]):\n single_person_kpts = kpts[idx, 7:].T\n num_kpts = len(single_person_kpts) // steps\n xyc = []\n for kid in range(num_kpts):\n x_coord, y_coord = single_person_kpts[steps * kid], single_person_kpts[steps * kid + 1]\n xyc.append([x_coord * ratio[1], y_coord* ratio[0], single_person_kpts[steps * kid + 2]])\n reformated_output.append(xyc)\n out = np.array(reformated_output)\n\n output = torch.from_numpy(out)\n \n # exit()\n return output\n " }, { "path": "asone/pose_estimators/yolov8_pose/__init__.py", "content": "from .yolov8 import Yolov8PoseEstimator\n__all__ = ['Yolov8PoseEstimator']" }, { "path": "asone/pose_estimators/yolov8_pose/plots.py", "content": "# Plotting utils\n\nimport glob\nimport math\nimport os\nimport random\nfrom copy import copy\nfrom pathlib import Path\n\nimport cv2\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport torch\nimport yaml\nfrom PIL import Image, ImageDraw, ImageFont\nfrom scipy.signal import butter, filtfilt\n\nfrom .general import xywh2xyxy, xyxy2xywh\nfrom .metrics import fitness\n\n# Settings\nmatplotlib.rc('font', **{'size': 11})\nmatplotlib.use('Agg') # for writing to files only\n\nclass Colors:\n # Ultralytics color palette https://ultralytics.com/\n def __init__(self):\n self.palette = [self.hex2rgb(c) for c in matplotlib.colors.TABLEAU_COLORS.values()]\n self.n = len(self.palette)\n\n def __call__(self, i, bgr=False):\n c = self.palette[int(i) % self.n]\n return (c[2], c[1], c[0]) if bgr else c\n\n @staticmethod\n def hex2rgb(h): # rgb order (PIL)\n return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4))\n\n\ndef plot_one_box_kpt(x, im, color=None, label=None, line_thickness=3, kpt_label=False, kpts=None, steps=2, orig_shape=None):\n # Plots one bounding box on image 'im' using OpenCV\n assert im.data.contiguous, 'Image not contiguous. Apply np.ascontiguousarray(im) to plot_on_box() input image.'\n tl = line_thickness or round(0.002 * (im.shape[0] + im.shape[1]) / 2) + 1 # line/font thickness\n color = color or [random.randint(0, 255) for _ in range(3)]\n c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n cv2.rectangle(im, c1, c2, (255,0,0), thickness=tl*1//3, lineType=cv2.LINE_AA)\n if label:\n if len(label.split(' ')) > 1:\n label = label.split(' ')[-1]\n tf = max(tl - 1, 1) # font thickness\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 6, thickness=tf)[0]\n c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n cv2.rectangle(im, c1, c2, color, -1, cv2.LINE_AA) # filled\n cv2.putText(im, label, (c1[0], c1[1] - 2), 0, tl / 6, [225, 255, 255], thickness=tf//2, lineType=cv2.LINE_AA)\n if kpt_label:\n plot_skeleton_kpts(im, kpts, steps, orig_shape=orig_shape)\n\ncolors = Colors() \n\ndef color_list():\n def hex2rgb(h):\n return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4))\n\n return [hex2rgb(h) for h in matplotlib.colors.TABLEAU_COLORS.values()] # or BASE_ (8), CSS4_ (148), XKCD_ (949)\n\n\ndef hist2d(x, y, n=100):\n # 2d histogram used in labels.png and evolve.png\n xedges, yedges = np.linspace(x.min(), x.max(), n), np.linspace(y.min(), y.max(), n)\n hist, xedges, yedges = np.histogram2d(x, y, (xedges, yedges))\n xidx = np.clip(np.digitize(x, xedges) - 1, 0, hist.shape[0] - 1)\n yidx = np.clip(np.digitize(y, yedges) - 1, 0, hist.shape[1] - 1)\n return np.log(hist[xidx, yidx])\n\n\ndef butter_lowpass_filtfilt(data, cutoff=1500, fs=50000, order=5):\n # https://stackoverflow.com/questions/28536191/how-to-filter-smooth-with-scipy-numpy\n def butter_lowpass(cutoff, fs, order):\n nyq = 0.5 * fs\n normal_cutoff = cutoff / nyq\n return butter(order, normal_cutoff, btype='low', analog=False)\n\n b, a = butter_lowpass(cutoff, fs, order=order)\n return filtfilt(b, a, data) # forward-backward filter\n\n\ndef plot_one_box(x, img, color=None, label=None, line_thickness=1):\n # Plots one bounding box on image img\n tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 2 # line/font thickness\n color = color or [random.randint(0, 255) for _ in range(3)]\n c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n if label:\n tf = max(tl - 1, 1) # font thickness\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled\n cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\n\n\ndef plot_one_box_PIL(box, img, color=None, label=None, line_thickness=None):\n img = Image.fromarray(img)\n draw = ImageDraw.Draw(img)\n line_thickness = line_thickness or max(int(min(img.size) / 200), 2)\n draw.rectangle(box, width=line_thickness, outline=tuple(color)) # plot\n if label:\n fontsize = max(round(max(img.size) / 40), 12)\n font = ImageFont.truetype(\"Arial.ttf\", fontsize)\n txt_width, txt_height = font.getsize(label)\n draw.rectangle([box[0], box[1] - txt_height + 4, box[0] + txt_width, box[1]], fill=tuple(color))\n draw.text((box[0], box[1] - txt_height + 1), label, fill=(255, 255, 255), font=font)\n return np.asarray(img)\n\n\ndef plot_wh_methods(): # from utils.plots import *; plot_wh_methods()\n # Compares the two methods for width-height anchor multiplication\n # https://github.com/ultralytics/yolov3/issues/168\n x = np.arange(-4.0, 4.0, .1)\n ya = np.exp(x)\n yb = torch.sigmoid(torch.from_numpy(x)).numpy() * 2\n\n fig = plt.figure(figsize=(6, 3), tight_layout=True)\n plt.plot(x, ya, '.-', label='YOLOv3')\n plt.plot(x, yb ** 2, '.-', label='YOLOR ^2')\n plt.plot(x, yb ** 1.6, '.-', label='YOLOR ^1.6')\n plt.xlim(left=-4, right=4)\n plt.ylim(bottom=0, top=6)\n plt.xlabel('input')\n plt.ylabel('output')\n plt.grid()\n plt.legend()\n fig.savefig('comparison.png', dpi=200)\n\n\ndef output_to_target(output):\n # Convert model output to target format [batch_id, class_id, x, y, w, h, conf]\n targets = []\n for i, o in enumerate(output):\n for *box, conf, cls in o.cpu().numpy():\n targets.append([i, cls, *list(*xyxy2xywh(np.array(box)[None])), conf])\n return np.array(targets)\n\n\ndef plot_images(images, targets, paths=None, fname='images.jpg', names=None, max_size=640, max_subplots=16):\n # Plot image grid with labels\n\n if isinstance(images, torch.Tensor):\n images = images.cpu().float().numpy()\n if isinstance(targets, torch.Tensor):\n targets = targets.cpu().numpy()\n\n # un-normalise\n if np.max(images[0]) <= 1:\n images *= 255\n\n tl = 3 # line thickness\n tf = max(tl - 1, 1) # font thickness\n bs, _, h, w = images.shape # batch size, _, height, width\n bs = min(bs, max_subplots) # limit plot images\n ns = np.ceil(bs ** 0.5) # number of subplots (square)\n\n # Check if we should resize\n scale_factor = max_size / max(h, w)\n if scale_factor < 1:\n h = math.ceil(scale_factor * h)\n w = math.ceil(scale_factor * w)\n\n colors = color_list() # list of colors\n mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init\n for i, img in enumerate(images):\n if i == max_subplots: # if last batch has fewer images than we expect\n break\n\n block_x = int(w * (i // ns))\n block_y = int(h * (i % ns))\n\n img = img.transpose(1, 2, 0)\n if scale_factor < 1:\n img = cv2.resize(img, (w, h))\n\n mosaic[block_y:block_y + h, block_x:block_x + w, :] = img\n if len(targets) > 0:\n image_targets = targets[targets[:, 0] == i]\n boxes = xywh2xyxy(image_targets[:, 2:6]).T\n classes = image_targets[:, 1].astype('int')\n labels = image_targets.shape[1] == 6 # labels if no conf column\n conf = None if labels else image_targets[:, 6] # check for confidence presence (label vs pred)\n\n if boxes.shape[1]:\n if boxes.max() <= 1.01: # if normalized with tolerance 0.01\n boxes[[0, 2]] *= w # scale to pixels\n boxes[[1, 3]] *= h\n elif scale_factor < 1: # absolute coords need scale if image scales\n boxes *= scale_factor\n boxes[[0, 2]] += block_x\n boxes[[1, 3]] += block_y\n for j, box in enumerate(boxes.T):\n cls = int(classes[j])\n color = colors[cls % len(colors)]\n cls = names[cls] if names else cls\n if labels or conf[j] > 0.25: # 0.25 conf thresh\n label = '%s' % cls if labels else '%s %.1f' % (cls, conf[j])\n plot_one_box(box, mosaic, label=label, color=color, line_thickness=tl)\n\n # Draw image filename labels\n if paths:\n label = Path(paths[i]).name[:40] # trim to 40 char\n t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]\n cv2.putText(mosaic, label, (block_x + 5, block_y + t_size[1] + 5), 0, tl / 3, [220, 220, 220], thickness=tf,\n lineType=cv2.LINE_AA)\n\n # Image border\n cv2.rectangle(mosaic, (block_x, block_y), (block_x + w, block_y + h), (255, 255, 255), thickness=3)\n\n if fname:\n r = min(1280. / max(h, w) / ns, 1.0) # ratio to limit image size\n mosaic = cv2.resize(mosaic, (int(ns * w * r), int(ns * h * r)), interpolation=cv2.INTER_AREA)\n # cv2.imwrite(fname, cv2.cvtColor(mosaic, cv2.COLOR_BGR2RGB)) # cv2 save\n Image.fromarray(mosaic).save(fname) # PIL save\n return mosaic\n\n\ndef plot_lr_scheduler(optimizer, scheduler, epochs=300, save_dir=''):\n # Plot LR simulating training for full epochs\n optimizer, scheduler = copy(optimizer), copy(scheduler) # do not modify originals\n y = []\n for _ in range(epochs):\n scheduler.step()\n y.append(optimizer.param_groups[0]['lr'])\n plt.plot(y, '.-', label='LR')\n plt.xlabel('epoch')\n plt.ylabel('LR')\n plt.grid()\n plt.xlim(0, epochs)\n plt.ylim(0)\n plt.savefig(Path(save_dir) / 'LR.png', dpi=200)\n plt.close()\n\n\ndef plot_test_txt(): # from utils.plots import *; plot_test()\n # Plot test.txt histograms\n x = np.loadtxt('test.txt', dtype=np.float32)\n box = xyxy2xywh(x[:, :4])\n cx, cy = box[:, 0], box[:, 1]\n\n fig, ax = plt.subplots(1, 1, figsize=(6, 6), tight_layout=True)\n ax.hist2d(cx, cy, bins=600, cmax=10, cmin=0)\n ax.set_aspect('equal')\n plt.savefig('hist2d.png', dpi=300)\n\n fig, ax = plt.subplots(1, 2, figsize=(12, 6), tight_layout=True)\n ax[0].hist(cx, bins=600)\n ax[1].hist(cy, bins=600)\n plt.savefig('hist1d.png', dpi=200)\n\n\ndef plot_targets_txt(): # from utils.plots import *; plot_targets_txt()\n # Plot targets.txt histograms\n x = np.loadtxt('targets.txt', dtype=np.float32).T\n s = ['x targets', 'y targets', 'width targets', 'height targets']\n fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)\n ax = ax.ravel()\n for i in range(4):\n ax[i].hist(x[i], bins=100, label='%.3g +/- %.3g' % (x[i].mean(), x[i].std()))\n ax[i].legend()\n ax[i].set_title(s[i])\n plt.savefig('targets.jpg', dpi=200)\n\n\ndef plot_study_txt(path='', x=None): # from utils.plots import *; plot_study_txt()\n # Plot study.txt generated by test.py\n fig, ax = plt.subplots(2, 4, figsize=(10, 6), tight_layout=True)\n # ax = ax.ravel()\n\n fig2, ax2 = plt.subplots(1, 1, figsize=(8, 4), tight_layout=True)\n # for f in [Path(path) / f'study_coco_{x}.txt' for x in ['yolor-p6', 'yolor-w6', 'yolor-e6', 'yolor-d6']]:\n for f in sorted(Path(path).glob('study*.txt')):\n y = np.loadtxt(f, dtype=np.float32, usecols=[0, 1, 2, 3, 7, 8, 9], ndmin=2).T\n x = np.arange(y.shape[1]) if x is None else np.array(x)\n s = ['P', 'R', 'mAP@.5', 'mAP@.5:.95', 't_inference (ms/img)', 't_NMS (ms/img)', 't_total (ms/img)']\n # for i in range(7):\n # ax[i].plot(x, y[i], '.-', linewidth=2, markersize=8)\n # ax[i].set_title(s[i])\n\n j = y[3].argmax() + 1\n ax2.plot(y[6, 1:j], y[3, 1:j] * 1E2, '.-', linewidth=2, markersize=8,\n label=f.stem.replace('study_coco_', '').replace('yolo', 'YOLO'))\n\n ax2.plot(1E3 / np.array([209, 140, 97, 58, 35, 18]), [34.6, 40.5, 43.0, 47.5, 49.7, 51.5],\n 'k.-', linewidth=2, markersize=8, alpha=.25, label='EfficientDet')\n\n ax2.grid(alpha=0.2)\n ax2.set_yticks(np.arange(20, 60, 5))\n ax2.set_xlim(0, 57)\n ax2.set_ylim(30, 55)\n ax2.set_xlabel('GPU Speed (ms/img)')\n ax2.set_ylabel('COCO AP val')\n ax2.legend(loc='lower right')\n plt.savefig(str(Path(path).name) + '.png', dpi=300)\n\n\ndef plot_labels(labels, names=(), save_dir=Path(''), loggers=None):\n # plot dataset labels\n print('Plotting labels... ')\n c, b = labels[:, 0], labels[:, 1:].transpose() # classes, boxes\n nc = int(c.max() + 1) # number of classes\n colors = color_list()\n x = pd.DataFrame(b.transpose(), columns=['x', 'y', 'width', 'height'])\n\n # seaborn correlogram\n sns.pairplot(x, corner=True, diag_kind='auto', kind='hist', diag_kws=dict(bins=50), plot_kws=dict(pmax=0.9))\n plt.savefig(save_dir / 'labels_correlogram.jpg', dpi=200)\n plt.close()\n\n # matplotlib labels\n matplotlib.use('svg') # faster\n ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)[1].ravel()\n ax[0].hist(c, bins=np.linspace(0, nc, nc + 1) - 0.5, rwidth=0.8)\n ax[0].set_ylabel('instances')\n if 0 < len(names) < 30:\n ax[0].set_xticks(range(len(names)))\n ax[0].set_xticklabels(names, rotation=90, fontsize=10)\n else:\n ax[0].set_xlabel('classes')\n sns.histplot(x, x='x', y='y', ax=ax[2], bins=50, pmax=0.9)\n sns.histplot(x, x='width', y='height', ax=ax[3], bins=50, pmax=0.9)\n\n # rectangles\n labels[:, 1:3] = 0.5 # center\n labels[:, 1:] = xywh2xyxy(labels[:, 1:]) * 2000\n img = Image.fromarray(np.ones((2000, 2000, 3), dtype=np.uint8) * 255)\n for cls, *box in labels[:1000]:\n ImageDraw.Draw(img).rectangle(box, width=1, outline=colors[int(cls) % 10]) # plot\n ax[1].imshow(img)\n ax[1].axis('off')\n\n for a in [0, 1, 2, 3]:\n for s in ['top', 'right', 'left', 'bottom']:\n ax[a].spines[s].set_visible(False)\n\n plt.savefig(save_dir / 'labels.jpg', dpi=200)\n matplotlib.use('Agg')\n plt.close()\n\n # loggers\n for k, v in loggers.items() or {}:\n if k == 'wandb' and v:\n v.log({\"Labels\": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, commit=False)\n\n\ndef plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.plots import *; plot_evolution()\n # Plot hyperparameter evolution results in evolve.txt\n with open(yaml_file) as f:\n hyp = yaml.load(f, Loader=yaml.SafeLoader)\n x = np.loadtxt('evolve.txt', ndmin=2)\n f = fitness(x)\n # weights = (f - f.min()) ** 2 # for weighted results\n plt.figure(figsize=(10, 12), tight_layout=True)\n matplotlib.rc('font', **{'size': 8})\n for i, (k, v) in enumerate(hyp.items()):\n y = x[:, i + 7]\n # mu = (y * weights).sum() / weights.sum() # best weighted result\n mu = y[f.argmax()] # best single result\n plt.subplot(6, 5, i + 1)\n plt.scatter(y, f, c=hist2d(y, f, 20), cmap='viridis', alpha=.8, edgecolors='none')\n plt.plot(mu, f.max(), 'k+', markersize=15)\n plt.title('%s = %.3g' % (k, mu), fontdict={'size': 9}) # limit to 40 characters\n if i % 5 != 0:\n plt.yticks([])\n print('%15s: %.3g' % (k, mu))\n plt.savefig('evolve.png', dpi=200)\n print('\\nPlot saved as evolve.png')\n\n\ndef profile_idetection(start=0, stop=0, labels=(), save_dir=''):\n # Plot iDetection '*.txt' per-image logs. from utils.plots import *; profile_idetection()\n ax = plt.subplots(2, 4, figsize=(12, 6), tight_layout=True)[1].ravel()\n s = ['Images', 'Free Storage (GB)', 'RAM Usage (GB)', 'Battery', 'dt_raw (ms)', 'dt_smooth (ms)', 'real-world FPS']\n files = list(Path(save_dir).glob('frames*.txt'))\n for fi, f in enumerate(files):\n try:\n results = np.loadtxt(f, ndmin=2).T[:, 90:-30] # clip first and last rows\n n = results.shape[1] # number of rows\n x = np.arange(start, min(stop, n) if stop else n)\n results = results[:, x]\n t = (results[0] - results[0].min()) # set t0=0s\n results[0] = x\n for i, a in enumerate(ax):\n if i < len(results):\n label = labels[fi] if len(labels) else f.stem.replace('frames_', '')\n a.plot(t, results[i], marker='.', label=label, linewidth=1, markersize=5)\n a.set_title(s[i])\n a.set_xlabel('time (s)')\n # if fi == len(files) - 1:\n # a.set_ylim(bottom=0)\n for side in ['top', 'right']:\n a.spines[side].set_visible(False)\n else:\n a.remove()\n except Exception as e:\n print('Warning: Plotting error for %s; %s' % (f, e))\n\n ax[1].legend()\n plt.savefig(Path(save_dir) / 'idetection_profile.png', dpi=200)\n\n\ndef plot_results_overlay(start=0, stop=0): # from utils.plots import *; plot_results_overlay()\n # Plot training 'results*.txt', overlaying train and val losses\n s = ['train', 'train', 'train', 'Precision', 'mAP@0.5', 'val', 'val', 'val', 'Recall', 'mAP@0.5:0.95'] # legends\n t = ['Box', 'Objectness', 'Classification', 'P-R', 'mAP-F1'] # titles\n for f in sorted(glob.glob('results*.txt') + glob.glob('../../Downloads/results*.txt')):\n results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T\n n = results.shape[1] # number of rows\n x = range(start, min(stop, n) if stop else n)\n fig, ax = plt.subplots(1, 5, figsize=(14, 3.5), tight_layout=True)\n ax = ax.ravel()\n for i in range(5):\n for j in [i, i + 5]:\n y = results[j, x]\n ax[i].plot(x, y, marker='.', label=s[j])\n # y_smooth = butter_lowpass_filtfilt(y)\n # ax[i].plot(x, np.gradient(y_smooth), marker='.', label=s[j])\n\n ax[i].set_title(t[i])\n ax[i].legend()\n ax[i].set_ylabel(f) if i == 0 else None # add filename\n fig.savefig(f.replace('.txt', '.png'), dpi=200)\n\n\ndef plot_results(start=0, stop=0, bucket='', id=(), labels=(), save_dir=''):\n # Plot training 'results*.txt'. from utils.plots import *; plot_results(save_dir='runs/train/exp')\n fig, ax = plt.subplots(2, 5, figsize=(12, 6), tight_layout=True)\n ax = ax.ravel()\n s = ['Box', 'Objectness', 'Classification', 'Precision', 'Recall',\n 'val Box', 'val Objectness', 'val Classification', 'mAP@0.5', 'mAP@0.5:0.95']\n if bucket:\n # files = ['https://storage.googleapis.com/%s/results%g.txt' % (bucket, x) for x in id]\n files = ['results%g.txt' % x for x in id]\n c = ('gsutil cp ' + '%s ' * len(files) + '.') % tuple('gs://%s/results%g.txt' % (bucket, x) for x in id)\n os.system(c)\n else:\n files = list(Path(save_dir).glob('results*.txt'))\n assert len(files), 'No results.txt files found in %s, nothing to plot.' % os.path.abspath(save_dir)\n for fi, f in enumerate(files):\n try:\n results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T\n n = results.shape[1] # number of rows\n x = range(start, min(stop, n) if stop else n)\n for i in range(10):\n y = results[i, x]\n if i in [0, 1, 2, 5, 6, 7]:\n y[y == 0] = np.nan # don't show zero loss values\n # y /= y[0] # normalize\n label = labels[fi] if len(labels) else f.stem\n ax[i].plot(x, y, marker='.', label=label, linewidth=2, markersize=8)\n ax[i].set_title(s[i])\n # if i in [5, 6, 7]: # share train and val loss y axes\n # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5])\n except Exception as e:\n print('Warning: Plotting error for %s; %s' % (f, e))\n\n ax[1].legend()\n fig.savefig(Path(save_dir) / 'results.png', dpi=200)\n \n \ndef output_to_keypoint(output):\n # Convert model output to target format [batch_id, class_id, x, y, w, h, conf]\n targets = []\n for i, o in enumerate(output):\n kpts = o[:,6:]\n o = o[:,:6]\n for index, (*box, conf, cls) in enumerate(o.cpu().numpy()):\n targets.append([i, cls, *list(*xyxy2xywh(np.array(box)[None])), conf, *list(kpts.cpu().numpy()[index])])\n return np.array(targets)\n\n\ndef plot_skeleton_kpts(im, kpts, steps, orig_shape=None):\n #Plot the skeleton and keypointsfor coco datatset\n palette = np.array([[255, 128, 0], [255, 153, 51], [255, 178, 102],\n [230, 230, 0], [255, 153, 255], [153, 204, 255],\n [255, 102, 255], [255, 51, 255], [102, 178, 255],\n [51, 153, 255], [255, 153, 153], [255, 102, 102],\n [255, 51, 51], [153, 255, 153], [102, 255, 102],\n [51, 255, 51], [0, 255, 0], [0, 0, 255], [255, 0, 0],\n [255, 255, 255]])\n\n skeleton = [[16, 14], [14, 12], [17, 15], [15, 13], [12, 13], [6, 12],\n [7, 13], [6, 7], [6, 8], [7, 9], [8, 10], [9, 11], [2, 3],\n [1, 2], [1, 3], [2, 4], [3, 5], [4, 6], [5, 7]]\n\n pose_limb_color = palette[[9, 9, 9, 9, 7, 7, 7, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16]]\n pose_kpt_color = palette[[16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9]]\n radius = 5\n num_kpts = len(kpts) // steps\n\n for kid in range(num_kpts):\n r, g, b = pose_kpt_color[kid]\n x_coord, y_coord = kpts[steps * kid], kpts[steps * kid + 1]\n if not (x_coord % 640 == 0 or y_coord % 640 == 0):\n if steps == 3:\n conf = kpts[steps * kid + 2]\n if conf < 0.5:\n continue\n cv2.circle(im, (int(x_coord), int(y_coord)), radius, (int(r), int(g), int(b)), -1)\n\n for sk_id, sk in enumerate(skeleton):\n r, g, b = pose_limb_color[sk_id]\n pos1 = (int(kpts[(sk[0]-1)*steps]), int(kpts[(sk[0]-1)*steps+1]))\n pos2 = (int(kpts[(sk[1]-1)*steps]), int(kpts[(sk[1]-1)*steps+1]))\n if steps == 3:\n conf1 = kpts[(sk[0]-1)*steps+2]\n conf2 = kpts[(sk[1]-1)*steps+2]\n if conf1<0.5 or conf2<0.5:\n continue\n if pos1[0]%640 == 0 or pos1[1]%640==0 or pos1[0]<0 or pos1[1]<0:\n continue\n if pos2[0] % 640 == 0 or pos2[1] % 640 == 0 or pos2[0]<0 or pos2[1]<0:\n continue\n cv2.line(im, pos1, pos2, (int(r), int(g), int(b)), thickness=2)\n" }, { "path": "asone/pose_estimators/yolov8_pose/yolov8.py", "content": "from ultralytics import YOLO\nimport torch\n\n\nclass Yolov8PoseEstimator:\n def __init__(self, weights, use_cuda=True):\n self.model = YOLO(weights)\n self.device = 0 if use_cuda and torch.cuda.is_available() else 'cpu'\n \n def estimate(self, source):\n results = self.model(source, device=self.device)\n return results[0].keypoints\n " }, { "path": "asone/recognizers/__init__.py", "content": "from asone.recognizers.easyocr_recognizer import EasyOCRRecognizer\nfrom asone.recognizers.recognizer import TextRecognizer\n__all__ = [\n 'TextRecognizer',\n 'EasyOCRRecognizer']\n" }, { "path": "asone/recognizers/easyocr_recognizer/__init__.py", "content": "from .easyocr_recognizer import EasyOCRRecognizer\n__all__ = ['EasyOCRRecognizer']" }, { "path": "asone/recognizers/easyocr_recognizer/easyocr_recognizer.py", "content": "import easyocr\nimport numpy as np\nfrom PIL import *\n\n\nclass EasyOCRRecognizer:\n \n def __init__ (self, languages: list = ['en'], detect_network=\"craft\", \n recog_network='standard', gpu=True): \n self.detect_network = detect_network\n self.gpu = gpu\n self.model = easyocr.Reader(languages, detect_network=self.detect_network, gpu=self.gpu) \n \n def detect(self, img):\n horizontal_list, free_list = self.model.detect(img) \n return horizontal_list, free_list\n \n def recognize(self, img, horizontal_list=None, free_list=None): \n \n horizontal_list = np.array(horizontal_list)\n horizontal_list = horizontal_list.astype(int)\n horizontal_list = horizontal_list.tolist()\n reformated_input = []\n for bbx in horizontal_list:\n bbx[1], bbx[2] = bbx[2], bbx[1]\n reformated_input.append(bbx[:4])\n horizontal_list = reformated_input\n \n free_list_format = []\n if horizontal_list!=[]:\n for text in horizontal_list:\n xmin, xmax, ymin, ymax = text\n free_list_format.append([[xmin,ymin], [xmax,ymin], [xmax,ymax] , [xmin,ymax]])\n \n free_list.extend(free_list_format)\n results = self.model.recognize(img, horizontal_list=[], free_list=free_list)\n\n formated_output = []\n for data in results:\n x_list = []\n y_list = []\n for bbx in data[0]:\n x_list.append(int(bbx[0]))\n y_list.append(int(bbx[1]))\n formated_output.append([min(x_list), min(y_list), max(x_list), max(y_list), data[1], data[2]])\n\n return formated_output\n" }, { "path": "asone/recognizers/recognizer.py", "content": "from asone.recognizers import EasyOCRRecognizer\nfrom asone.recognizers.utils.recognizer_name import get_recognizer_name\n\n\nclass TextRecognizer:\n \n def __init__(self,\n model_flag: int,\n languages: list=['en'],\n use_cuda: bool=True):\n\n self.model = self._select_recognizer(model_flag, use_cuda, languages)\n\n def _select_recognizer(self, model_flag, use_cuda, languages):\n recognizer_name = get_recognizer_name(model_flag)\n if recognizer_name == 'easyocr':\n _recognizer = EasyOCRRecognizer(gpu=use_cuda, languages=languages)\n return _recognizer\n\n def get_recognizer(self):\n return self.model\n\n def recognize(self,\n image: list,\n horizontal_list=None, free_list=None):\n print(horizontal_list)\n return self.model.recognize(image, horizontal_list=horizontal_list, free_list=free_list)" }, { "path": "asone/recognizers/utils/__init__.py", "content": "" }, { "path": "asone/recognizers/utils/recognizer_name.py", "content": "import os \n\nrecognizers = {\n '200': 'easyocr'\n}\n\ndef get_recognizer_name(model_flag):\n \n if model_flag == 200:\n recognizer = recognizers[str(model_flag)]\n \n return recognizer" }, { "path": "asone/schemas/output_schemas.py", "content": "class ModelOutput:\n def __init__(self):\n self.dets = Detections()\n self.info = ImageInfo()\n\nclass Detections:\n def __init__(self):\n self.bbox = None\n self.ids = []\n self.score = []\n self.class_ids = []\n\nclass ImageInfo:\n def __init__(self):\n self.image = None\n self.frame_no = None\n self.fps = None" }, { "path": "asone/segmentors/__init__.py", "content": "from asone.segmentors.segmentor import Segmentor\n\n__all__ = ['Segmentor']\n" }, { "path": "asone/segmentors/segment_anything/__init__.py", "content": "" }, { "path": "asone/segmentors/segment_anything/sam.py", "content": "import numpy as np\nimport cv2\nimport torch\n\nfrom segment_anything import sam_model_registry, SamPredictor\nfrom asone.utils.utils import PathResolver\n\n\nclass SamSegmentor:\n def __init__(self,\n weights: str=None,\n use_cuda: bool = True):\n self.device = \"cuda\" if use_cuda and torch.cuda.is_available() else \"cpu\"\n with PathResolver():\n self.model = self.load_models(weights)\n \n def load_models(self, ckpt: str) -> None:\n sam = sam_model_registry[\"vit_h\"](checkpoint=ckpt).to(device=self.device)\n model = SamPredictor(sam)\n \n return model\n \n def draw_masks_fromList(self, image, masks_generated, labels, colors=[0, 255, 0]):\n masked_image = image.copy()\n for i in range(len(masks_generated)):\n mask = masks_generated[i].squeeze() # Squeeze to remove singleton dimension\n color = np.asarray(colors, dtype='uint8')\n mask_color = np.expand_dims(mask, axis=-1) * color # Apply color to the mask\n\n # Apply the mask to the image\n masked_image = np.where(mask_color > 0, mask_color, masked_image)\n\n masked_image = masked_image.astype(np.uint8)\n return cv2.addWeighted(image, 0.5, masked_image, 0.5, 0)\n \n \n def create_mask(self, bbox_xyxy, image):\n self.model.set_image(image)\n \n input_boxes = torch.from_numpy(bbox_xyxy).to(self.device)\n transformed_boxes = self.model.transform.apply_boxes_torch(input_boxes, image.shape[:2])\n \n masks, _, _ = self.model.predict_torch(\n point_coords=None,\n point_labels=None,\n boxes=transformed_boxes,\n multimask_output=False,\n )\n \n # result_image = self.draw_masks_fromList(image, masks.cpu(), bbox_xyxy)\n \n return masks.cpu()\n" }, { "path": "asone/segmentors/segmentor.py", "content": "import os\n\nfrom asone import utils\nfrom asone.segmentors.utils.weights_path import get_weight_path\nfrom asone.segmentors.segment_anything.sam import SamSegmentor\n\n\nclass Segmentor:\n def __init__(self, \n model_flag,\n weights: str=None,\n use_cuda: bool=True):\n \n if weights is None:\n weight = get_weight_path(model_flag)\n \n if not os.path.exists(weight):\n utils.download_weights(weight)\n \n self.model = self._select_segmentor(model_flag, weight, use_cuda)\n \n def _select_segmentor(self, model_flag, weights, use_cuda):\n if model_flag == 171:\n model = SamSegmentor(weights, use_cuda=use_cuda)\n return model\n \n def create_mask(self, bbox_xyxy, image):\n return self.model.create_mask(bbox_xyxy, image)\n" }, { "path": "asone/segmentors/utils/__init__.py", "content": "" }, { "path": "asone/segmentors/utils/weights_path.py", "content": "import os \n\nweights = { \n # Segmentor\n '171':os.path.join('sam','weights','sam_vit_h_4b8939.pth'),\n}\n\ndef get_weight_path(model_flag):\n if model_flag == 171:\n weight = weights[str(model_flag)]\n \n return weight" }, { "path": "asone/trackers/__init__.py", "content": "from asone.trackers.byte_track.bytetracker import ByteTrack\nfrom asone.trackers.deep_sort.deepsort import DeepSort\nfrom asone.trackers.nor_fair.norfair import NorFair\nfrom asone.trackers.motpy.motpy import Motpy\nfrom asone.trackers.oc_sort.ocsort import OcSort\nfrom asone.trackers.strong_sort.strongsort import StrongSort\nfrom asone.trackers.tracker import Tracker\n\n\n__all__ = ['Tracker', 'ByteTrack', 'DeepSort', 'NorFair', 'Motpy', 'OcSort', 'StrongSort']\n" }, { "path": "asone/trackers/byte_track/__init__.py", "content": "" }, { "path": "asone/trackers/byte_track/bytetracker.py", "content": "from .tracker.byte_tracker import BYTETracker\nimport numpy as np\nfrom asone import utils\n\n\nclass ByteTrack(object):\n def __init__(self, detector, min_box_area: int = 10, aspect_ratio_thresh:float= 3.0) -> None:\n\n self.min_box_area = min_box_area\n self.aspect_ratio_thresh = aspect_ratio_thresh\n self.min_box_area = min_box_area\n self.rgb_means = (0.485, 0.456, 0.406)\n self.std = (0.229, 0.224, 0.225)\n\n self.detector = detector\n try:\n self.input_shape = tuple(detector.model.get_inputs()[0].shape[2:])\n except AttributeError as e:\n self.input_shape = (640, 640)\n\n self.tracker = BYTETracker(frame_rate=30)\n\n def detect_and_track(self, image: np.ndarray, config: dict) -> tuple:\n dets_xyxy, free_list = self.detector.detect(image, **config)\n image_info = {\"width\":image.shape[0],\"height\":image.shape[1]}\n class_ids = []\n ids = []\n bboxes_xyxy = []\n scores = []\n\n if isinstance(dets_xyxy, np.ndarray) and len(dets_xyxy) > 0:\n class_ids = [int(i) for i in dets_xyxy[:, -1].tolist()]\n bboxes_xyxy, ids, scores = self._tracker_update(\n dets_xyxy,\n image_info,\n )\n return bboxes_xyxy, ids, scores, class_ids\n\n def _tracker_update(self, dets: np.ndarray, image_info: dict):\n online_targets = []\n class_id = 0\n if dets is not None:\n online_targets = self.tracker.update(\n dets[:, :-1],\n [image_info['height'], image_info['width']],\n [image_info['height'], image_info['width']],\n )\n \n online_xyxys = []\n online_ids = []\n online_scores = []\n for online_target in online_targets:\n tlwh = online_target.tlwh\n track_id = online_target.track_id\n vertical = tlwh[2] / tlwh[3] > self.aspect_ratio_thresh\n if tlwh[2] * tlwh[3] > self.min_box_area and not vertical:\n online_xyxys.append(utils.tlwh_to_xyxy(tlwh))\n online_ids.append(track_id)\n online_scores.append(online_target.score)\n return online_xyxys, online_ids, online_scores\n" }, { "path": "asone/trackers/byte_track/tracker/__init__.py", "content": "" }, { "path": "asone/trackers/byte_track/tracker/basetrack.py", "content": "import numpy as np\nfrom collections import OrderedDict\n\n\nclass TrackState(object):\n New = 0\n Tracked = 1\n Lost = 2\n Removed = 3\n\n\nclass BaseTrack(object):\n _count = 0\n\n track_id = 0\n is_activated = False\n state = TrackState.New\n\n history = OrderedDict()\n features = []\n curr_feature = None\n score = 0\n start_frame = 0\n frame_id = 0\n time_since_update = 0\n\n # multi-camera\n location = (np.inf, np.inf)\n\n @property\n def end_frame(self):\n return self.frame_id\n\n @staticmethod\n def next_id():\n BaseTrack._count += 1\n return BaseTrack._count\n\n def activate(self, *args):\n raise NotImplementedError\n\n def predict(self):\n raise NotImplementedError\n\n def update(self, *args, **kwargs):\n raise NotImplementedError\n\n def mark_lost(self):\n self.state = TrackState.Lost\n\n def mark_removed(self):\n self.state = TrackState.Removed" }, { "path": "asone/trackers/byte_track/tracker/byte_tracker.py", "content": "import numpy as np\nfrom .kalman_filter import KalmanFilter\nfrom asone.trackers.byte_track.tracker import matching\nfrom .basetrack import BaseTrack, TrackState\n\nclass STrack(BaseTrack):\n shared_kalman = KalmanFilter()\n def __init__(self, tlwh, score):\n\n # wait activate\n self._tlwh = np.asarray(tlwh, dtype=np.float)\n self.kalman_filter = None\n self.mean, self.covariance = None, None\n self.is_activated = False\n\n self.score = score\n self.tracklet_len = 0\n\n def predict(self):\n mean_state = self.mean.copy()\n if self.state != TrackState.Tracked:\n mean_state[7] = 0\n self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance)\n\n @staticmethod\n def multi_predict(stracks):\n if len(stracks) > 0:\n multi_mean = np.asarray([st.mean.copy() for st in stracks])\n multi_covariance = np.asarray([st.covariance for st in stracks])\n for i, st in enumerate(stracks):\n if st.state != TrackState.Tracked:\n multi_mean[i][7] = 0\n multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance)\n for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):\n stracks[i].mean = mean\n stracks[i].covariance = cov\n\n def activate(self, kalman_filter, frame_id):\n \"\"\"Start a new tracklet\"\"\"\n self.kalman_filter = kalman_filter\n self.track_id = self.next_id()\n self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh))\n\n self.tracklet_len = 0\n self.state = TrackState.Tracked\n if frame_id == 1:\n self.is_activated = True\n # self.is_activated = True\n self.frame_id = frame_id\n self.start_frame = frame_id\n\n def re_activate(self, new_track, frame_id, new_id=False):\n self.mean, self.covariance = self.kalman_filter.update(\n self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh)\n )\n self.tracklet_len = 0\n self.state = TrackState.Tracked\n self.is_activated = True\n self.frame_id = frame_id\n if new_id:\n self.track_id = self.next_id()\n self.score = new_track.score\n\n def update(self, new_track, frame_id):\n \"\"\"\n Update a matched track\n :type new_track: STrack\n :type frame_id: int\n :type update_feature: bool\n :return:\n \"\"\"\n self.frame_id = frame_id\n self.tracklet_len += 1\n\n new_tlwh = new_track.tlwh\n self.mean, self.covariance = self.kalman_filter.update(\n self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh))\n self.state = TrackState.Tracked\n self.is_activated = True\n\n self.score = new_track.score\n\n @property\n # @jit(nopython=True)\n def tlwh(self):\n \"\"\"Get current position in bounding box format `(top left x, top left y,\n width, height)`.\n \"\"\"\n if self.mean is None:\n return self._tlwh.copy()\n ret = self.mean[:4].copy()\n ret[2] *= ret[3]\n ret[:2] -= ret[2:] / 2\n return ret\n\n @property\n # @jit(nopython=True)\n def tlbr(self):\n \"\"\"Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,\n `(top left, bottom right)`.\n \"\"\"\n ret = self.tlwh.copy()\n ret[2:] += ret[:2]\n return ret\n\n @staticmethod\n # @jit(nopython=True)\n def tlwh_to_xyah(tlwh):\n \"\"\"Convert bounding box to format `(center x, center y, aspect ratio,\n height)`, where the aspect ratio is `width / height`.\n \"\"\"\n ret = np.asarray(tlwh).copy()\n ret[:2] += ret[2:] / 2\n ret[2] /= ret[3]\n return ret\n\n def to_xyah(self):\n return self.tlwh_to_xyah(self.tlwh)\n\n @staticmethod\n # @jit(nopython=True)\n def tlbr_to_tlwh(tlbr):\n ret = np.asarray(tlbr).copy()\n ret[2:] -= ret[:2]\n return ret\n\n @staticmethod\n # @jit(nopython=True)\n def tlwh_to_tlbr(tlwh):\n ret = np.asarray(tlwh).copy()\n ret[2:] += ret[:2]\n return ret\n\n def __repr__(self):\n return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame)\n\n\nclass BYTETracker(object):\n def __init__(self, track_thresh=0.5,match_thresh=0.8, track_buffer=30, mot20=False, frame_rate=30):\n self.tracked_stracks = [] # type: list[STrack]\n self.lost_stracks = [] # type: list[STrack]\n self.removed_stracks = [] # type: list[STrack]\n\n self.track_thresh = track_thresh\n self.track_buffer = track_buffer\n self.mot20 = mot20\n self.match_thresh = match_thresh\n\n self.frame_id = 0\n self.det_thresh = track_thresh + 0.1\n self.buffer_size = int(frame_rate / 30.0 * self.track_buffer)\n self.max_time_lost = self.buffer_size\n self.kalman_filter = KalmanFilter()\n\n def update(self, output_results, img_info, img_size):\n self.frame_id += 1\n activated_starcks = []\n refind_stracks = []\n lost_stracks = []\n removed_stracks = []\n\n if output_results.shape[1] == 5:\n scores = output_results[:, 4]\n bboxes = output_results[:, :4]\n else:\n output_results = output_results.cpu().numpy()\n scores = output_results[:, 4] * output_results[:, 5]\n bboxes = output_results[:, :4] # x1y1x2y2\n img_h, img_w = img_info[0], img_info[1]\n scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w))\n bboxes /= scale\n\n remain_inds = scores > self.track_thresh\n inds_low = scores > 0.1\n inds_high = scores < self.track_thresh\n\n inds_second = np.logical_and(inds_low, inds_high)\n dets_second = bboxes[inds_second]\n dets = bboxes[remain_inds]\n scores_keep = scores[remain_inds]\n scores_second = scores[inds_second]\n\n if len(dets) > 0:\n '''Detections'''\n detections = [STrack(STrack.tlbr_to_tlwh(tlbr), s) for\n (tlbr, s) in zip(dets, scores_keep)]\n else:\n detections = []\n\n ''' Add newly detected tracklets to tracked_stracks'''\n unconfirmed = []\n tracked_stracks = [] # type: list[STrack]\n for track in self.tracked_stracks:\n if not track.is_activated:\n unconfirmed.append(track)\n else:\n tracked_stracks.append(track)\n\n ''' Step 2: First association, with high score detection boxes'''\n strack_pool = joint_stracks(tracked_stracks, self.lost_stracks)\n # Predict the current location with KF\n STrack.multi_predict(strack_pool)\n dists = matching.iou_distance(strack_pool, detections)\n if not self.mot20:\n dists = matching.fuse_score(dists, detections)\n matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.match_thresh)\n\n for itracked, idet in matches:\n track = strack_pool[itracked]\n det = detections[idet]\n if track.state == TrackState.Tracked:\n track.update(detections[idet], self.frame_id)\n activated_starcks.append(track)\n else:\n track.re_activate(det, self.frame_id, new_id=False)\n refind_stracks.append(track)\n\n ''' Step 3: Second association, with low score detection boxes'''\n # association the untrack to the low score detections\n if len(dets_second) > 0:\n '''Detections'''\n detections_second = [STrack(STrack.tlbr_to_tlwh(tlbr), s) for\n (tlbr, s) in zip(dets_second, scores_second)]\n else:\n detections_second = []\n r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked]\n dists = matching.iou_distance(r_tracked_stracks, detections_second)\n matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.5)\n for itracked, idet in matches:\n track = r_tracked_stracks[itracked]\n det = detections_second[idet]\n if track.state == TrackState.Tracked:\n track.update(det, self.frame_id)\n activated_starcks.append(track)\n else:\n track.re_activate(det, self.frame_id, new_id=False)\n refind_stracks.append(track)\n\n for it in u_track:\n track = r_tracked_stracks[it]\n if not track.state == TrackState.Lost:\n track.mark_lost()\n lost_stracks.append(track)\n\n '''Deal with unconfirmed tracks, usually tracks with only one beginning frame'''\n detections = [detections[i] for i in u_detection]\n dists = matching.iou_distance(unconfirmed, detections)\n if not self.mot20:\n dists = matching.fuse_score(dists, detections)\n matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7)\n for itracked, idet in matches:\n unconfirmed[itracked].update(detections[idet], self.frame_id)\n activated_starcks.append(unconfirmed[itracked])\n for it in u_unconfirmed:\n track = unconfirmed[it]\n track.mark_removed()\n removed_stracks.append(track)\n\n \"\"\" Step 4: Init new stracks\"\"\"\n for inew in u_detection:\n track = detections[inew]\n if track.score < self.det_thresh:\n continue\n track.activate(self.kalman_filter, self.frame_id)\n activated_starcks.append(track)\n \"\"\" Step 5: Update state\"\"\"\n for track in self.lost_stracks:\n if self.frame_id - track.end_frame > self.max_time_lost:\n track.mark_removed()\n removed_stracks.append(track)\n\n # print('Ramained match {} s'.format(t4-t3))\n\n self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked]\n self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks)\n self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks)\n self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks)\n self.lost_stracks.extend(lost_stracks)\n self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks)\n self.removed_stracks.extend(removed_stracks)\n self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks)\n # get scores of lost tracks\n output_stracks = [track for track in self.tracked_stracks if track.is_activated]\n\n return output_stracks\n\n\ndef joint_stracks(tlista, tlistb):\n exists = {}\n res = []\n for t in tlista:\n exists[t.track_id] = 1\n res.append(t)\n for t in tlistb:\n tid = t.track_id\n if not exists.get(tid, 0):\n exists[tid] = 1\n res.append(t)\n return res\n\n\ndef sub_stracks(tlista, tlistb):\n stracks = {}\n for t in tlista:\n stracks[t.track_id] = t\n for t in tlistb:\n tid = t.track_id\n if stracks.get(tid, 0):\n del stracks[tid]\n return list(stracks.values())\n\n\ndef remove_duplicate_stracks(stracksa, stracksb):\n pdist = matching.iou_distance(stracksa, stracksb)\n pairs = np.where(pdist < 0.15)\n dupa, dupb = list(), list()\n for p, q in zip(*pairs):\n timep = stracksa[p].frame_id - stracksa[p].start_frame\n timeq = stracksb[q].frame_id - stracksb[q].start_frame\n if timep > timeq:\n dupb.append(q)\n else:\n dupa.append(p)\n resa = [t for i, t in enumerate(stracksa) if not i in dupa]\n resb = [t for i, t in enumerate(stracksb) if not i in dupb]\n return resa, resb\n" }, { "path": "asone/trackers/byte_track/tracker/kalman_filter.py", "content": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport scipy.linalg\n\n\n\"\"\"\nTable for the 0.95 quantile of the chi-square distribution with N degrees of\nfreedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv\nfunction and used as Mahalanobis gating threshold.\n\"\"\"\nchi2inv95 = {\n 1: 3.8415,\n 2: 5.9915,\n 3: 7.8147,\n 4: 9.4877,\n 5: 11.070,\n 6: 12.592,\n 7: 14.067,\n 8: 15.507,\n 9: 16.919}\n\n\nclass KalmanFilter(object):\n \"\"\"\n A simple Kalman filter for tracking bounding boxes in image space.\n\n The 8-dimensional state space\n\n x, y, a, h, vx, vy, va, vh\n\n contains the bounding box center position (x, y), aspect ratio a, height h,\n and their respective velocities.\n\n Object motion follows a constant velocity model. The bounding box location\n (x, y, a, h) is taken as direct observation of the state space (linear\n observation model).\n\n \"\"\"\n\n def __init__(self):\n ndim, dt = 4, 1.\n\n # Create Kalman filter model matrices.\n self._motion_mat = np.eye(2 * ndim, 2 * ndim)\n for i in range(ndim):\n self._motion_mat[i, ndim + i] = dt\n self._update_mat = np.eye(ndim, 2 * ndim)\n\n # Motion and observation uncertainty are chosen relative to the current\n # state estimate. These weights control the amount of uncertainty in\n # the model. This is a bit hacky.\n self._std_weight_position = 1. / 20\n self._std_weight_velocity = 1. / 160\n\n def initiate(self, measurement):\n \"\"\"Create track from unassociated measurement.\n\n Parameters\n ----------\n measurement : ndarray\n Bounding box coordinates (x, y, a, h) with center position (x, y),\n aspect ratio a, and height h.\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the mean vector (8 dimensional) and covariance matrix (8x8\n dimensional) of the new track. Unobserved velocities are initialized\n to 0 mean.\n\n \"\"\"\n mean_pos = measurement\n mean_vel = np.zeros_like(mean_pos)\n mean = np.r_[mean_pos, mean_vel]\n\n std = [\n 2 * self._std_weight_position * measurement[3],\n 2 * self._std_weight_position * measurement[3],\n 1e-2,\n 2 * self._std_weight_position * measurement[3],\n 10 * self._std_weight_velocity * measurement[3],\n 10 * self._std_weight_velocity * measurement[3],\n 1e-5,\n 10 * self._std_weight_velocity * measurement[3]]\n covariance = np.diag(np.square(std))\n return mean, covariance\n\n def predict(self, mean, covariance):\n \"\"\"Run Kalman filter prediction step.\n\n Parameters\n ----------\n mean : ndarray\n The 8 dimensional mean vector of the object state at the previous\n time step.\n covariance : ndarray\n The 8x8 dimensional covariance matrix of the object state at the\n previous time step.\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the mean vector and covariance matrix of the predicted\n state. Unobserved velocities are initialized to 0 mean.\n\n \"\"\"\n std_pos = [\n self._std_weight_position * mean[3],\n self._std_weight_position * mean[3],\n 1e-2,\n self._std_weight_position * mean[3]]\n std_vel = [\n self._std_weight_velocity * mean[3],\n self._std_weight_velocity * mean[3],\n 1e-5,\n self._std_weight_velocity * mean[3]]\n motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))\n\n #mean = np.dot(self._motion_mat, mean)\n mean = np.dot(mean, self._motion_mat.T)\n covariance = np.linalg.multi_dot((\n self._motion_mat, covariance, self._motion_mat.T)) + motion_cov\n\n return mean, covariance\n\n def project(self, mean, covariance):\n \"\"\"Project state distribution to measurement space.\n\n Parameters\n ----------\n mean : ndarray\n The state's mean vector (8 dimensional array).\n covariance : ndarray\n The state's covariance matrix (8x8 dimensional).\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the projected mean and covariance matrix of the given state\n estimate.\n\n \"\"\"\n std = [\n self._std_weight_position * mean[3],\n self._std_weight_position * mean[3],\n 1e-1,\n self._std_weight_position * mean[3]]\n innovation_cov = np.diag(np.square(std))\n\n mean = np.dot(self._update_mat, mean)\n covariance = np.linalg.multi_dot((\n self._update_mat, covariance, self._update_mat.T))\n return mean, covariance + innovation_cov\n\n def multi_predict(self, mean, covariance):\n \"\"\"Run Kalman filter prediction step (Vectorized version).\n Parameters\n ----------\n mean : ndarray\n The Nx8 dimensional mean matrix of the object states at the previous\n time step.\n covariance : ndarray\n The Nx8x8 dimensional covariance matrics of the object states at the\n previous time step.\n Returns\n -------\n (ndarray, ndarray)\n Returns the mean vector and covariance matrix of the predicted\n state. Unobserved velocities are initialized to 0 mean.\n \"\"\"\n std_pos = [\n self._std_weight_position * mean[:, 3],\n self._std_weight_position * mean[:, 3],\n 1e-2 * np.ones_like(mean[:, 3]),\n self._std_weight_position * mean[:, 3]]\n std_vel = [\n self._std_weight_velocity * mean[:, 3],\n self._std_weight_velocity * mean[:, 3],\n 1e-5 * np.ones_like(mean[:, 3]),\n self._std_weight_velocity * mean[:, 3]]\n sqr = np.square(np.r_[std_pos, std_vel]).T\n\n motion_cov = []\n for i in range(len(mean)):\n motion_cov.append(np.diag(sqr[i]))\n motion_cov = np.asarray(motion_cov)\n\n mean = np.dot(mean, self._motion_mat.T)\n left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2))\n covariance = np.dot(left, self._motion_mat.T) + motion_cov\n\n return mean, covariance\n\n def update(self, mean, covariance, measurement):\n \"\"\"Run Kalman filter correction step.\n\n Parameters\n ----------\n mean : ndarray\n The predicted state's mean vector (8 dimensional).\n covariance : ndarray\n The state's covariance matrix (8x8 dimensional).\n measurement : ndarray\n The 4 dimensional measurement vector (x, y, a, h), where (x, y)\n is the center position, a the aspect ratio, and h the height of the\n bounding box.\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the measurement-corrected state distribution.\n\n \"\"\"\n projected_mean, projected_cov = self.project(mean, covariance)\n\n chol_factor, lower = scipy.linalg.cho_factor(\n projected_cov, lower=True, check_finite=False)\n kalman_gain = scipy.linalg.cho_solve(\n (chol_factor, lower), np.dot(covariance, self._update_mat.T).T,\n check_finite=False).T\n innovation = measurement - projected_mean\n\n new_mean = mean + np.dot(innovation, kalman_gain.T)\n new_covariance = covariance - np.linalg.multi_dot((\n kalman_gain, projected_cov, kalman_gain.T))\n return new_mean, new_covariance\n\n def gating_distance(self, mean, covariance, measurements,\n only_position=False, metric='maha'):\n \"\"\"Compute gating distance between state distribution and measurements.\n A suitable distance threshold can be obtained from `chi2inv95`. If\n `only_position` is False, the chi-square distribution has 4 degrees of\n freedom, otherwise 2.\n Parameters\n ----------\n mean : ndarray\n Mean vector over the state distribution (8 dimensional).\n covariance : ndarray\n Covariance of the state distribution (8x8 dimensional).\n measurements : ndarray\n An Nx4 dimensional matrix of N measurements, each in\n format (x, y, a, h) where (x, y) is the bounding box center\n position, a the aspect ratio, and h the height.\n only_position : Optional[bool]\n If True, distance computation is done with respect to the bounding\n box center position only.\n Returns\n -------\n ndarray\n Returns an array of length N, where the i-th element contains the\n squared Mahalanobis distance between (mean, covariance) and\n `measurements[i]`.\n \"\"\"\n mean, covariance = self.project(mean, covariance)\n if only_position:\n mean, covariance = mean[:2], covariance[:2, :2]\n measurements = measurements[:, :2]\n\n d = measurements - mean\n if metric == 'gaussian':\n return np.sum(d * d, axis=1)\n elif metric == 'maha':\n cholesky_factor = np.linalg.cholesky(covariance)\n z = scipy.linalg.solve_triangular(\n cholesky_factor, d.T, lower=True, check_finite=False,\n overwrite_b=True)\n squared_maha = np.sum(z * z, axis=0)\n return squared_maha\n else:\n raise ValueError('invalid distance metric')" }, { "path": "asone/trackers/byte_track/tracker/matching.py", "content": "import numpy as np\nimport scipy\nimport lap\nfrom scipy.spatial.distance import cdist\nfrom cython_bbox import bbox_overlaps as bbox_ious\nfrom asone.trackers.byte_track.tracker import kalman_filter\n\ndef merge_matches(m1, m2, shape):\n O,P,Q = shape\n m1 = np.asarray(m1)\n m2 = np.asarray(m2)\n\n M1 = scipy.sparse.coo_matrix((np.ones(len(m1)), (m1[:, 0], m1[:, 1])), shape=(O, P))\n M2 = scipy.sparse.coo_matrix((np.ones(len(m2)), (m2[:, 0], m2[:, 1])), shape=(P, Q))\n\n mask = M1*M2\n match = mask.nonzero()\n match = list(zip(match[0], match[1]))\n unmatched_O = tuple(set(range(O)) - set([i for i, j in match]))\n unmatched_Q = tuple(set(range(Q)) - set([j for i, j in match]))\n\n return match, unmatched_O, unmatched_Q\n\n\ndef _indices_to_matches(cost_matrix, indices, thresh):\n matched_cost = cost_matrix[tuple(zip(*indices))]\n matched_mask = (matched_cost <= thresh)\n\n matches = indices[matched_mask]\n unmatched_a = tuple(set(range(cost_matrix.shape[0])) - set(matches[:, 0]))\n unmatched_b = tuple(set(range(cost_matrix.shape[1])) - set(matches[:, 1]))\n\n return matches, unmatched_a, unmatched_b\n\n\ndef linear_assignment(cost_matrix, thresh):\n if cost_matrix.size == 0:\n return np.empty((0, 2), dtype=int), tuple(range(cost_matrix.shape[0])), tuple(range(cost_matrix.shape[1]))\n matches, unmatched_a, unmatched_b = [], [], []\n cost, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh)\n for ix, mx in enumerate(x):\n if mx >= 0:\n matches.append([ix, mx])\n unmatched_a = np.where(x < 0)[0]\n unmatched_b = np.where(y < 0)[0]\n matches = np.asarray(matches)\n return matches, unmatched_a, unmatched_b\n\n\ndef ious(atlbrs, btlbrs):\n \"\"\"\n Compute cost based on IoU\n :type atlbrs: list[tlbr] | np.ndarray\n :type atlbrs: list[tlbr] | np.ndarray\n\n :rtype ious np.ndarray\n \"\"\"\n ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float64)\n if ious.size == 0:\n return ious\n\n ious = bbox_ious(\n np.ascontiguousarray(atlbrs, dtype=np.float64),\n np.ascontiguousarray(btlbrs, dtype=np.float64)\n )\n\n return ious\n\n\ndef iou_distance(atracks, btracks):\n \"\"\"\n Compute cost based on IoU\n :type atracks: list[STrack]\n :type btracks: list[STrack]\n\n :rtype cost_matrix np.ndarray\n \"\"\"\n\n if (len(atracks)>0 and isinstance(atracks[0], np.ndarray)) or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)):\n atlbrs = atracks\n btlbrs = btracks\n else:\n atlbrs = [track.tlbr for track in atracks]\n btlbrs = [track.tlbr for track in btracks]\n _ious = ious(atlbrs, btlbrs)\n cost_matrix = 1 - _ious\n\n return cost_matrix\n\ndef v_iou_distance(atracks, btracks):\n \"\"\"\n Compute cost based on IoU\n :type atracks: list[STrack]\n :type btracks: list[STrack]\n\n :rtype cost_matrix np.ndarray\n \"\"\"\n\n if (len(atracks)>0 and isinstance(atracks[0], np.ndarray)) or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)):\n atlbrs = atracks\n btlbrs = btracks\n else:\n atlbrs = [track.tlwh_to_tlbr(track.pred_bbox) for track in atracks]\n btlbrs = [track.tlwh_to_tlbr(track.pred_bbox) for track in btracks]\n _ious = ious(atlbrs, btlbrs)\n cost_matrix = 1 - _ious\n\n return cost_matrix\n\ndef embedding_distance(tracks, detections, metric='cosine'):\n \"\"\"\n :param tracks: list[STrack]\n :param detections: list[BaseTrack]\n :param metric:\n :return: cost_matrix np.ndarray\n \"\"\"\n\n cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float64)\n if cost_matrix.size == 0:\n return cost_matrix\n det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float64)\n #for i, track in enumerate(tracks):\n #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric))\n track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float64)\n cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features\n return cost_matrix\n\n\ndef gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False):\n if cost_matrix.size == 0:\n return cost_matrix\n gating_dim = 2 if only_position else 4\n gating_threshold = kalman_filter.chi2inv95[gating_dim]\n measurements = np.asarray([det.to_xyah() for det in detections])\n for row, track in enumerate(tracks):\n gating_distance = kf.gating_distance(\n track.mean, track.covariance, measurements, only_position)\n cost_matrix[row, gating_distance > gating_threshold] = np.inf\n return cost_matrix\n\n\ndef fuse_motion(kf, cost_matrix, tracks, detections, only_position=False, lambda_=0.98):\n if cost_matrix.size == 0:\n return cost_matrix\n gating_dim = 2 if only_position else 4\n gating_threshold = kalman_filter.chi2inv95[gating_dim]\n measurements = np.asarray([det.to_xyah() for det in detections])\n for row, track in enumerate(tracks):\n gating_distance = kf.gating_distance(\n track.mean, track.covariance, measurements, only_position, metric='maha')\n cost_matrix[row, gating_distance > gating_threshold] = np.inf\n cost_matrix[row] = lambda_ * cost_matrix[row] + (1 - lambda_) * gating_distance\n return cost_matrix\n\n\ndef fuse_iou(cost_matrix, tracks, detections):\n if cost_matrix.size == 0:\n return cost_matrix\n reid_sim = 1 - cost_matrix\n iou_dist = iou_distance(tracks, detections)\n iou_sim = 1 - iou_dist\n fuse_sim = reid_sim * (1 + iou_sim) / 2\n det_scores = np.array([det.score for det in detections])\n det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0)\n #fuse_sim = fuse_sim * (1 + det_scores) / 2\n fuse_cost = 1 - fuse_sim\n return fuse_cost\n\n\ndef fuse_score(cost_matrix, detections):\n if cost_matrix.size == 0:\n return cost_matrix\n iou_sim = 1 - cost_matrix\n det_scores = np.array([det.score for det in detections])\n det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0)\n fuse_sim = iou_sim * det_scores\n fuse_cost = 1 - fuse_sim\n return fuse_cost" }, { "path": "asone/trackers/deep_sort/__init__.py", "content": "" }, { "path": "asone/trackers/deep_sort/deepsort.py", "content": "from .tracker import build_tracker\nimport numpy as np\nimport os\nfrom asone import utils\n\n\nclass DeepSort:\n def __init__(self, detector, weights=None, use_cuda=True):\n\n if weights is None:\n weights = os.path.join(os.path.dirname(\n os.path.abspath(__file__)), \"tracker/deep/checkpoint/ckpt.t7\")\n\n if not os.path.exists(weights):\n utils.download_weights(weights)\n\n cfg = {\n 'MAX_DIST': 0.2,\n 'MIN_CONFIDENCE': 0.3,\n 'NMS_MAX_OVERLAP': 0.5,\n 'MAX_IOU_DISTANCE': 0.7,\n 'MAX_AGE': 70,\n 'N_INIT': 3,\n 'NN_BUDGET': 100\n }\n\n self.tracker = build_tracker(weights, cfg, use_cuda=use_cuda)\n self.detector = detector\n try:\n self.input_shape = tuple(detector.model.get_inputs()[0].shape[2:])\n except AttributeError as e:\n self.input_shape = (640, 640)\n \n def detect_and_track(self, image: np.ndarray, config: dict) -> tuple:\n \n dets_xyxy, image_info = self.detector.detect(\n image, **config\n )\n\n image_info['im0'] = image\n\n class_ids = []\n ids = []\n bboxes_xyxy = []\n scores = []\n\n if isinstance(dets_xyxy, np.ndarray) and len(dets_xyxy) > 0:\n class_ids = dets_xyxy[:, -1].tolist()\n bboxes_xyxy, ids, class_ids = self._tracker_update(\n dets_xyxy,\n image_info,\n )\n\n return bboxes_xyxy, ids, [], class_ids\n\n def _tracker_update(self, dets_xyxy: np.ndarray, image_info: dict):\n\n bbox_xyxy = []\n ids = []\n object_id = []\n\n if dets_xyxy is not None:\n dets_xywh = np.array([np.array(utils.xyxy_to_xywh(det))\n for det in dets_xyxy[:, :4]])\n\n outputs = self.tracker.update(\n dets_xywh, dets_xyxy[:, -2].tolist(), dets_xyxy[:, -1].tolist(), image_info['im0'])\n\n if len(outputs) > 0:\n bbox_xyxy = outputs[:, :4]\n ids = outputs[:, -2]\n object_id = outputs[:, -1]\n\n return bbox_xyxy, ids, object_id\n" }, { "path": "asone/trackers/deep_sort/tracker/.gitignore", "content": "# Folders\n__pycache__/\nbuild/\n*.egg-info\n\n\n# Files\n*.weights\n*.t7\n*.mp4\n*.avi\n*.so\n*.txt\n" }, { "path": "asone/trackers/deep_sort/tracker/README.md", "content": "# Deep Sort with PyTorch\n\n![](demo/demo.gif)\n\n## Update(1-1-2020)\nChanges\n- fix bugs\n- refactor code\n- accerate detection by adding nms on gpu\n\n## Latest Update(07-22)\nChanges\n- bug fix (Thanks @JieChen91 and @yingsen1 for bug reporting). \n- using batch for feature extracting for each frame, which lead to a small speed up. \n- code improvement.\n\nFuther improvement direction \n- Train detector on specific dataset rather than the official one.\n- Retrain REID model on pedestrain dataset for better performance.\n- Replace YOLOv3 detector with advanced ones.\n\n**Any contributions to this repository is welcome!**\n\n\n## Introduction\nThis is an implement of MOT tracking algorithm deep sort. Deep sort is basicly the same with sort but added a CNN model to extract features in image of human part bounded by a detector. This CNN model is indeed a RE-ID model and the detector used in [PAPER](https://arxiv.org/abs/1703.07402) is FasterRCNN , and the original source code is [HERE](https://github.com/nwojke/deep_sort). \nHowever in original code, the CNN model is implemented with tensorflow, which I'm not familier with. SO I re-implemented the CNN feature extraction model with PyTorch, and changed the CNN model a little bit. Also, I use **YOLOv3** to generate bboxes instead of FasterRCNN.\n\n## Dependencies\n- python 3 (python2 not sure)\n- numpy\n- scipy\n- opencv-python\n- sklearn\n- torch >= 0.4\n- torchvision >= 0.1\n- pillow\n- vizer\n- edict\n\n## Quick Start\n0. Check all dependencies installed\n```bash\npip install -r requirements.txt\n```\nfor user in china, you can specify pypi source to accelerate install like:\n```bash\npip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple\n```\n\n1. Clone this repository\n```\ngit clone git@github.com:ZQPei/deep_sort_pytorch.git\n```\n\n2. Download YOLOv3 parameters\n```\ncd detector/YOLOv3/weight/\nwget https://pjreddie.com/media/files/yolov3.weights\nwget https://pjreddie.com/media/files/yolov3-tiny.weights\ncd ../../../\n```\n\n3. Download deepsort parameters ckpt.t7\n```\ncd deep_sort/deep/checkpoint\n# download ckpt.t7 from\nhttps://drive.google.com/drive/folders/1xhG0kRH1EX5B9_Iz8gQJb7UNnn_riXi6 to this folder\ncd ../../../\n``` \n\n4. Compile nms module\n```bash\ncd detector/YOLOv3/nms\nsh build.sh\ncd ../../..\n```\n\nNotice:\nIf compiling failed, the simplist way is to **Upgrade your pytorch >= 1.1 and torchvision >= 0.3\" and you can avoid the troublesome compiling problems which are most likely caused by either `gcc version too low` or `libraries missing`.\n\n5. Run demo\n```\nusage: python yolov3_deepsort.py VIDEO_PATH\n [--help]\n [--frame_interval FRAME_INTERVAL]\n [--config_detection CONFIG_DETECTION]\n [--config_deepsort CONFIG_DEEPSORT]\n [--display]\n [--display_width DISPLAY_WIDTH]\n [--display_height DISPLAY_HEIGHT]\n [--save_path SAVE_PATH] \n [--cpu] \n\n# yolov3 + deepsort\npython yolov3_deepsort.py [VIDEO_PATH]\n\n# yolov3_tiny + deepsort\npython yolov3_deepsort.py [VIDEO_PATH] --config_detection ./configs/yolov3_tiny.yaml\n\n# yolov3 + deepsort on webcam\npython3 yolov3_deepsort.py /dev/video0 --camera 0\n\n# yolov3_tiny + deepsort on webcam\npython3 yolov3_deepsort.py /dev/video0 --config_detection ./configs/yolov3_tiny.yaml --camera 0\n```\nUse `--display` to enable display. \nResults will be saved to `./output/results.avi` and `./output/results.txt`.\n\nAll files above can also be accessed from BaiduDisk! \nlinker：[BaiduDisk](https://pan.baidu.com/s/1YJ1iPpdFTlUyLFoonYvozg)\npasswd：fbuw\n\n## Training the RE-ID model\nThe original model used in paper is in original_model.py, and its parameter here [original_ckpt.t7](https://drive.google.com/drive/folders/1xhG0kRH1EX5B9_Iz8gQJb7UNnn_riXi6). \n\nTo train the model, first you need download [Market1501](http://www.liangzheng.com.cn/Project/project_reid.html) dataset or [Mars](http://www.liangzheng.com.cn/Project/project_mars.html) dataset. \n\nThen you can try [train.py](deep_sort/deep/train.py) to train your own parameter and evaluate it using [test.py](deep_sort/deep/test.py) and [evaluate.py](deep_sort/deep/evalute.py).\n![train.jpg](deep_sort/deep/train.jpg)\n\n## Demo videos and images\n[demo.avi](https://drive.google.com/drive/folders/1xhG0kRH1EX5B9_Iz8gQJb7UNnn_riXi6)\n[demo2.avi](https://drive.google.com/drive/folders/1xhG0kRH1EX5B9_Iz8gQJb7UNnn_riXi6)\n\n![1.jpg](demo/1.jpg)\n![2.jpg](demo/2.jpg)\n\n\n## References\n- paper: [Simple Online and Realtime Tracking with a Deep Association Metric](https://arxiv.org/abs/1703.07402)\n\n- code: [nwojke/deep_sort](https://github.com/nwojke/deep_sort)\n\n- paper: [YOLOv3](https://pjreddie.com/media/files/papers/YOLOv3.pdf)\n\n- code: [Joseph Redmon/yolov3](https://pjreddie.com/darknet/yolo/)\n" }, { "path": "asone/trackers/deep_sort/tracker/__init__.py", "content": "from .deep_sort import DeepSORT\nfrom .parser import get_config\n\n__all__ = ['DeepSORT', 'build_tracker']\n\n\ndef build_tracker(weights, cfg, use_cuda=True):\n # cfg = get_config()\n # cfg.merge_from_file(cfg_deep)\n\n return DeepSORT(weights,\n max_dist=cfg['MAX_DIST'], min_confidence=cfg['MIN_CONFIDENCE'],\n nms_max_overlap=cfg['NMS_MAX_OVERLAP'], max_iou_distance=cfg['MAX_IOU_DISTANCE'],\n max_age=cfg['MAX_AGE'], n_init=cfg['N_INIT'], nn_budget=cfg['NN_BUDGET'], use_cuda=use_cuda)\n" }, { "path": "asone/trackers/deep_sort/tracker/deep/__init__.py", "content": "" }, { "path": "asone/trackers/deep_sort/tracker/deep/evaluate.py", "content": "import torch\n\nfeatures = torch.load(\"features.pth\")\nqf = features[\"qf\"]\nql = features[\"ql\"]\ngf = features[\"gf\"]\ngl = features[\"gl\"]\n\nscores = qf.mm(gf.t())\nres = scores.topk(5, dim=1)[1][:, 0]\ntop1correct = gl[res].eq(ql).sum().item()\n\nprint(\"Acc top1:{:.3f}\".format(top1correct / ql.size(0)))\n" }, { "path": "asone/trackers/deep_sort/tracker/deep/feature_extractor.py", "content": "import torch\nimport torchvision.transforms as transforms\nimport numpy as np\nimport cv2\nimport logging\n\nfrom .model import Net\n\n\nclass Extractor(object):\n def __init__(self, model_path, use_cuda=True):\n self.net = Net(reid=True)\n self.device = \"cuda\" if torch.cuda.is_available() and use_cuda else \"cpu\"\n state_dict = torch.load(model_path, map_location=torch.device(self.device))[\n 'net_dict']\n self.net.load_state_dict(state_dict)\n logger = logging.getLogger(\"root.tracker\")\n logger.info(\"Loading weights from {}... Done!\".format(model_path))\n self.net.to(self.device)\n self.size = (64, 128)\n self.norm = transforms.Compose([\n transforms.ToTensor(),\n transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n ])\n\n def _preprocess(self, im_crops):\n \"\"\"\n TODO:\n 1. to float with scale from 0 to 1\n 2. resize to (64, 128) as Market1501 dataset did\n 3. concatenate to a numpy array\n 3. to torch Tensor\n 4. normalize\n \"\"\"\n def _resize(im, size):\n return cv2.resize(im.astype(np.float32)/255., size)\n\n im_batch = torch.cat([self.norm(_resize(im, self.size)).unsqueeze(\n 0) for im in im_crops], dim=0).float()\n return im_batch\n\n def __call__(self, im_crops):\n im_batch = self._preprocess(im_crops)\n with torch.no_grad():\n im_batch = im_batch.to(self.device)\n features = self.net(im_batch)\n return features.cpu().numpy()\n\n\nif __name__ == '__main__':\n img = cv2.imread(\"demo.jpg\")[:, :, (2, 1, 0)]\n extr = Extractor(\"checkpoint/ckpt.t7\")\n feature = extr(img)\n print(feature.shape)\n" }, { "path": "asone/trackers/deep_sort/tracker/deep/model.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass BasicBlock(nn.Module):\n def __init__(self, c_in, c_out, is_downsample=False):\n super(BasicBlock, self).__init__()\n self.is_downsample = is_downsample\n if is_downsample:\n self.conv1 = nn.Conv2d(\n c_in, c_out, 3, stride=2, padding=1, bias=False)\n else:\n self.conv1 = nn.Conv2d(\n c_in, c_out, 3, stride=1, padding=1, bias=False)\n self.bn1 = nn.BatchNorm2d(c_out)\n self.relu = nn.ReLU(True)\n self.conv2 = nn.Conv2d(c_out, c_out, 3, stride=1,\n padding=1, bias=False)\n self.bn2 = nn.BatchNorm2d(c_out)\n if is_downsample:\n self.downsample = nn.Sequential(\n nn.Conv2d(c_in, c_out, 1, stride=2, bias=False),\n nn.BatchNorm2d(c_out)\n )\n elif c_in != c_out:\n self.downsample = nn.Sequential(\n nn.Conv2d(c_in, c_out, 1, stride=1, bias=False),\n nn.BatchNorm2d(c_out)\n )\n self.is_downsample = True\n\n def forward(self, x):\n y = self.conv1(x)\n y = self.bn1(y)\n y = self.relu(y)\n y = self.conv2(y)\n y = self.bn2(y)\n if self.is_downsample:\n x = self.downsample(x)\n return F.relu(x.add(y), True)\n\n\ndef make_layers(c_in, c_out, repeat_times, is_downsample=False):\n blocks = []\n for i in range(repeat_times):\n if i == 0:\n blocks += [BasicBlock(c_in, c_out, is_downsample=is_downsample), ]\n else:\n blocks += [BasicBlock(c_out, c_out), ]\n return nn.Sequential(*blocks)\n\n\nclass Net(nn.Module):\n def __init__(self, num_classes=751, reid=False):\n super(Net, self).__init__()\n # 3 128 64\n self.conv = nn.Sequential(\n nn.Conv2d(3, 64, 3, stride=1, padding=1),\n nn.BatchNorm2d(64),\n nn.ReLU(inplace=True),\n # nn.Conv2d(32,32,3,stride=1,padding=1),\n # nn.BatchNorm2d(32),\n # nn.ReLU(inplace=True),\n nn.MaxPool2d(3, 2, padding=1),\n )\n # 32 64 32\n self.layer1 = make_layers(64, 64, 2, False)\n # 32 64 32\n self.layer2 = make_layers(64, 128, 2, True)\n # 64 32 16\n self.layer3 = make_layers(128, 256, 2, True)\n # 128 16 8\n self.layer4 = make_layers(256, 512, 2, True)\n # 256 8 4\n self.avgpool = nn.AvgPool2d((8, 4), 1)\n # 256 1 1\n self.reid = reid\n self.classifier = nn.Sequential(\n nn.Linear(512, 256),\n nn.BatchNorm1d(256),\n nn.ReLU(inplace=True),\n nn.Dropout(),\n nn.Linear(256, num_classes),\n )\n\n def forward(self, x):\n x = self.conv(x)\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x = self.layer4(x)\n x = self.avgpool(x)\n x = x.view(x.size(0), -1)\n # B x 128\n if self.reid:\n x = x.div(x.norm(p=2, dim=1, keepdim=True))\n return x\n # classifier\n x = self.classifier(x)\n return x\n\n\nif __name__ == '__main__':\n net = Net()\n x = torch.randn(4, 3, 128, 64)\n y = net(x)\n import ipdb\n ipdb.set_trace()\n" }, { "path": "asone/trackers/deep_sort/tracker/deep/original_model.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass BasicBlock(nn.Module):\n def __init__(self, c_in, c_out, is_downsample=False):\n super(BasicBlock, self).__init__()\n self.is_downsample = is_downsample\n if is_downsample:\n self.conv1 = nn.Conv2d(\n c_in, c_out, 3, stride=2, padding=1, bias=False)\n else:\n self.conv1 = nn.Conv2d(\n c_in, c_out, 3, stride=1, padding=1, bias=False)\n self.bn1 = nn.BatchNorm2d(c_out)\n self.relu = nn.ReLU(True)\n self.conv2 = nn.Conv2d(c_out, c_out, 3, stride=1,\n padding=1, bias=False)\n self.bn2 = nn.BatchNorm2d(c_out)\n if is_downsample:\n self.downsample = nn.Sequential(\n nn.Conv2d(c_in, c_out, 1, stride=2, bias=False),\n nn.BatchNorm2d(c_out)\n )\n elif c_in != c_out:\n self.downsample = nn.Sequential(\n nn.Conv2d(c_in, c_out, 1, stride=1, bias=False),\n nn.BatchNorm2d(c_out)\n )\n self.is_downsample = True\n\n def forward(self, x):\n y = self.conv1(x)\n y = self.bn1(y)\n y = self.relu(y)\n y = self.conv2(y)\n y = self.bn2(y)\n if self.is_downsample:\n x = self.downsample(x)\n return F.relu(x.add(y), True)\n\n\ndef make_layers(c_in, c_out, repeat_times, is_downsample=False):\n blocks = []\n for i in range(repeat_times):\n if i == 0:\n blocks += [BasicBlock(c_in, c_out, is_downsample=is_downsample), ]\n else:\n blocks += [BasicBlock(c_out, c_out), ]\n return nn.Sequential(*blocks)\n\n\nclass Net(nn.Module):\n def __init__(self, num_classes=625, reid=False):\n super(Net, self).__init__()\n # 3 128 64\n self.conv = nn.Sequential(\n nn.Conv2d(3, 32, 3, stride=1, padding=1),\n nn.BatchNorm2d(32),\n nn.ELU(inplace=True),\n nn.Conv2d(32, 32, 3, stride=1, padding=1),\n nn.BatchNorm2d(32),\n nn.ELU(inplace=True),\n nn.MaxPool2d(3, 2, padding=1),\n )\n # 32 64 32\n self.layer1 = make_layers(32, 32, 2, False)\n # 32 64 32\n self.layer2 = make_layers(32, 64, 2, True)\n # 64 32 16\n self.layer3 = make_layers(64, 128, 2, True)\n # 128 16 8\n self.dense = nn.Sequential(\n nn.Dropout(p=0.6),\n nn.Linear(128*16*8, 128),\n nn.BatchNorm1d(128),\n nn.ELU(inplace=True)\n )\n # 256 1 1\n self.reid = reid\n self.batch_norm = nn.BatchNorm1d(128)\n self.classifier = nn.Sequential(\n nn.Linear(128, num_classes),\n )\n\n def forward(self, x):\n x = self.conv(x)\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n\n x = x.view(x.size(0), -1)\n if self.reid:\n x = self.dense[0](x)\n x = self.dense[1](x)\n x = x.div(x.norm(p=2, dim=1, keepdim=True))\n return x\n x = self.dense(x)\n # B x 128\n # classifier\n x = self.classifier(x)\n return x\n\n\nif __name__ == '__main__':\n net = Net(reid=True)\n x = torch.randn(4, 3, 128, 64)\n y = net(x)\n import ipdb\n ipdb.set_trace()\n" }, { "path": "asone/trackers/deep_sort/tracker/deep/test.py", "content": "import torch\nimport torch.backends.cudnn as cudnn\nimport torchvision\n\nimport argparse\nimport os\n\nfrom model import Net\n\nparser = argparse.ArgumentParser(description=\"Train on market1501\")\nparser.add_argument(\"--data-dir\", default='data', type=str)\nparser.add_argument(\"--no-cuda\", action=\"store_true\")\nparser.add_argument(\"--gpu-id\", default=0, type=int)\nargs = parser.parse_args()\n\n# device\ndevice = \"cuda:{}\".format(\n args.gpu_id) if torch.cuda.is_available() and not args.no_cuda else \"cpu\"\nif torch.cuda.is_available() and not args.no_cuda:\n cudnn.benchmark = True\n\n# data loader\nroot = args.data_dir\nquery_dir = os.path.join(root, \"query\")\ngallery_dir = os.path.join(root, \"gallery\")\ntransform = torchvision.transforms.Compose([\n torchvision.transforms.Resize((128, 64)),\n torchvision.transforms.ToTensor(),\n torchvision.transforms.Normalize(\n [0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\nqueryloader = torch.utils.data.DataLoader(\n torchvision.datasets.ImageFolder(query_dir, transform=transform),\n batch_size=64, shuffle=False\n)\ngalleryloader = torch.utils.data.DataLoader(\n torchvision.datasets.ImageFolder(gallery_dir, transform=transform),\n batch_size=64, shuffle=False\n)\n\n# net definition\nnet = Net(reid=True)\nassert os.path.isfile(\n \"./checkpoint/ckpt.t7\"), \"Error: no checkpoint file found!\"\nprint('Loading from checkpoint/ckpt.t7')\ncheckpoint = torch.load(\"./checkpoint/ckpt.t7\")\nnet_dict = checkpoint['net_dict']\nnet.load_state_dict(net_dict, strict=False)\nnet.eval()\nnet.to(device)\n\n# compute features\nquery_features = torch.tensor([]).float()\nquery_labels = torch.tensor([]).long()\ngallery_features = torch.tensor([]).float()\ngallery_labels = torch.tensor([]).long()\n\nwith torch.no_grad():\n for idx, (inputs, labels) in enumerate(queryloader):\n inputs = inputs.to(device)\n features = net(inputs).cpu()\n query_features = torch.cat((query_features, features), dim=0)\n query_labels = torch.cat((query_labels, labels))\n\n for idx, (inputs, labels) in enumerate(galleryloader):\n inputs = inputs.to(device)\n features = net(inputs).cpu()\n gallery_features = torch.cat((gallery_features, features), dim=0)\n gallery_labels = torch.cat((gallery_labels, labels))\n\ngallery_labels -= 2\n\n# save features\nfeatures = {\n \"qf\": query_features,\n \"ql\": query_labels,\n \"gf\": gallery_features,\n \"gl\": gallery_labels\n}\ntorch.save(features, \"features.pth\")\n" }, { "path": "asone/trackers/deep_sort/tracker/deep/train.py", "content": "import argparse\nimport os\nimport time\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.backends.cudnn as cudnn\nimport torchvision\n\nfrom model import Net\n\nparser = argparse.ArgumentParser(description=\"Train on market1501\")\nparser.add_argument(\"--data-dir\", default='data', type=str)\nparser.add_argument(\"--no-cuda\", action=\"store_true\")\nparser.add_argument(\"--gpu-id\", default=0, type=int)\nparser.add_argument(\"--lr\", default=0.1, type=float)\nparser.add_argument(\"--interval\", '-i', default=20, type=int)\nparser.add_argument('--resume', '-r', action='store_true')\nargs = parser.parse_args()\n\n# device\ndevice = \"cuda:{}\".format(\n args.gpu_id) if torch.cuda.is_available() and not args.no_cuda else \"cpu\"\nif torch.cuda.is_available() and not args.no_cuda:\n cudnn.benchmark = True\n\n# data loading\nroot = args.data_dir\ntrain_dir = os.path.join(root, \"train\")\ntest_dir = os.path.join(root, \"test\")\ntransform_train = torchvision.transforms.Compose([\n torchvision.transforms.RandomCrop((128, 64), padding=4),\n torchvision.transforms.RandomHorizontalFlip(),\n torchvision.transforms.ToTensor(),\n torchvision.transforms.Normalize(\n [0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\ntransform_test = torchvision.transforms.Compose([\n torchvision.transforms.Resize((128, 64)),\n torchvision.transforms.ToTensor(),\n torchvision.transforms.Normalize(\n [0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\ntrainloader = torch.utils.data.DataLoader(\n torchvision.datasets.ImageFolder(train_dir, transform=transform_train),\n batch_size=64, shuffle=True\n)\ntestloader = torch.utils.data.DataLoader(\n torchvision.datasets.ImageFolder(test_dir, transform=transform_test),\n batch_size=64, shuffle=True\n)\nnum_classes = max(len(trainloader.dataset.classes),\n len(testloader.dataset.classes))\n\n# net definition\nstart_epoch = 0\nnet = Net(num_classes=num_classes)\nif args.resume:\n assert os.path.isfile(\n \"./checkpoint/ckpt.t7\"), \"Error: no checkpoint file found!\"\n print('Loading from checkpoint/ckpt.t7')\n checkpoint = torch.load(\"./checkpoint/ckpt.t7\")\n # import ipdb; ipdb.set_trace()\n net_dict = checkpoint['net_dict']\n net.load_state_dict(net_dict)\n best_acc = checkpoint['acc']\n start_epoch = checkpoint['epoch']\nnet.to(device)\n\n# loss and optimizer\ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(\n net.parameters(), args.lr, momentum=0.9, weight_decay=5e-4)\nbest_acc = 0.\n\n# train function for each epoch\n\n\ndef train(epoch):\n print(\"\\nEpoch : %d\" % (epoch+1))\n net.train()\n training_loss = 0.\n train_loss = 0.\n correct = 0\n total = 0\n interval = args.interval\n start = time.time()\n for idx, (inputs, labels) in enumerate(trainloader):\n # forward\n inputs, labels = inputs.to(device), labels.to(device)\n outputs = net(inputs)\n loss = criterion(outputs, labels)\n\n # backward\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n # accumurating\n training_loss += loss.item()\n train_loss += loss.item()\n correct += outputs.max(dim=1)[1].eq(labels).sum().item()\n total += labels.size(0)\n\n # print\n if (idx+1) % interval == 0:\n end = time.time()\n print(\"[progress:{:.1f}%]time:{:.2f}s Loss:{:.5f} Correct:{}/{} Acc:{:.3f}%\".format(\n 100.*(idx+1)/len(trainloader), end-start, training_loss /\n interval, correct, total, 100.*correct/total\n ))\n training_loss = 0.\n start = time.time()\n\n return train_loss/len(trainloader), 1. - correct/total\n\n\ndef test(epoch):\n global best_acc\n net.eval()\n test_loss = 0.\n correct = 0\n total = 0\n start = time.time()\n with torch.no_grad():\n for idx, (inputs, labels) in enumerate(testloader):\n inputs, labels = inputs.to(device), labels.to(device)\n outputs = net(inputs)\n loss = criterion(outputs, labels)\n\n test_loss += loss.item()\n correct += outputs.max(dim=1)[1].eq(labels).sum().item()\n total += labels.size(0)\n\n print(\"Testing ...\")\n end = time.time()\n print(\"[progress:{:.1f}%]time:{:.2f}s Loss:{:.5f} Correct:{}/{} Acc:{:.3f}%\".format(\n 100.*(idx+1)/len(testloader), end-start, test_loss /\n len(testloader), correct, total, 100.*correct/total\n ))\n\n # saving checkpoint\n acc = 100.*correct/total\n if acc > best_acc:\n best_acc = acc\n print(\"Saving parameters to checkpoint/ckpt.t7\")\n checkpoint = {\n 'net_dict': net.state_dict(),\n 'acc': acc,\n 'epoch': epoch,\n }\n if not os.path.isdir('checkpoint'):\n os.mkdir('checkpoint')\n torch.save(checkpoint, './checkpoint/ckpt.t7')\n\n return test_loss/len(testloader), 1. - correct/total\n\n\n# plot figure\nx_epoch = []\nrecord = {'train_loss': [], 'train_err': [], 'test_loss': [], 'test_err': []}\nfig = plt.figure()\nax0 = fig.add_subplot(121, title=\"loss\")\nax1 = fig.add_subplot(122, title=\"top1err\")\n\n\ndef draw_curve(epoch, train_loss, train_err, test_loss, test_err):\n global record\n record['train_loss'].append(train_loss)\n record['train_err'].append(train_err)\n record['test_loss'].append(test_loss)\n record['test_err'].append(test_err)\n\n x_epoch.append(epoch)\n ax0.plot(x_epoch, record['train_loss'], 'bo-', label='train')\n ax0.plot(x_epoch, record['test_loss'], 'ro-', label='val')\n ax1.plot(x_epoch, record['train_err'], 'bo-', label='train')\n ax1.plot(x_epoch, record['test_err'], 'ro-', label='val')\n if epoch == 0:\n ax0.legend()\n ax1.legend()\n fig.savefig(\"train.jpg\")\n\n# lr decay\n\n\ndef lr_decay():\n global optimizer\n for params in optimizer.param_groups:\n params['lr'] *= 0.1\n lr = params['lr']\n print(\"Learning rate adjusted to {}\".format(lr))\n\n\ndef main():\n for epoch in range(start_epoch, start_epoch+40):\n train_loss, train_err = train(epoch)\n test_loss, test_err = test(epoch)\n draw_curve(epoch, train_loss, train_err, test_loss, test_err)\n if (epoch+1) % 20 == 0:\n lr_decay()\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "asone/trackers/deep_sort/tracker/deep_sort.py", "content": "import numpy as np\nimport torch\n\nfrom .deep.feature_extractor import Extractor\nfrom .sort.nn_matching import NearestNeighborDistanceMetric\nfrom .sort.detection import Detection\nfrom .sort.tracker import Tracker\n\n\n__all__ = ['DeepSORT']\n\n\nclass DeepSORT(object):\n def __init__(self, model_path, max_dist=0.2, min_confidence=0.3, nms_max_overlap=1.0, max_iou_distance=0.7, max_age=70, n_init=3, nn_budget=100, use_cuda=True):\n self.min_confidence = min_confidence\n self.nms_max_overlap = nms_max_overlap\n\n self.extractor = Extractor(model_path, use_cuda=use_cuda)\n\n max_cosine_distance = max_dist\n metric = NearestNeighborDistanceMetric(\n \"cosine\", max_cosine_distance, nn_budget)\n self.tracker = Tracker(\n metric, max_iou_distance=max_iou_distance, max_age=max_age, n_init=n_init)\n\n def update(self, bbox_xywh, confidences, oids, ori_img):\n self.height, self.width = ori_img.shape[:2]\n # generate detections\n features = self._get_features(bbox_xywh, ori_img)\n bbox_tlwh = self._xywh_to_tlwh(bbox_xywh)\n detections = [Detection(bbox_tlwh[i], conf, features[i], oid) for i, (conf,oid) in enumerate(zip(confidences,oids)) if conf > self.min_confidence]\n\n # print(detections)\n # run on non-maximum supression\n boxes = np.array([d.tlwh for d in detections])\n scores = np.array([d.confidence for d in detections])\n\n # update tracker\n self.tracker.predict()\n self.tracker.update(detections)\n # print(\"len(scores):\", len(scores))\n # print(\"self.tracker.tracks\",len(self.tracker.tracks))\n # output bbox identities\n outputs = []\n for track in self.tracker.tracks:\n if not track.is_confirmed() or track.time_since_update > 1:\n continue\n box = track.to_tlwh()\n x1, y1, x2, y2 = self._tlwh_to_xyxy(box)\n track_id = track.track_id\n track_oid = track.oid\n outputs.append(np.array([x1, y1, x2, y2, track_id, track_oid], dtype=np.int))\n if len(outputs) > 0:\n outputs = np.stack(outputs, axis=0)\n return outputs\n\n \"\"\"\n TODO:\n Convert bbox from xc_yc_w_h to xtl_ytl_w_h\n Thanks JieChen91@github.com for reporting this bug!\n \"\"\"\n @staticmethod\n def _xywh_to_tlwh(bbox_xywh):\n if isinstance(bbox_xywh, np.ndarray):\n bbox_tlwh = bbox_xywh.copy()\n elif isinstance(bbox_xywh, torch.Tensor):\n bbox_tlwh = bbox_xywh.clone()\n bbox_tlwh[:, 0] = bbox_xywh[:, 0] - bbox_xywh[:, 2] / 2.\n bbox_tlwh[:, 1] = bbox_xywh[:, 1] - bbox_xywh[:, 3] / 2.\n return bbox_tlwh\n\n def _xywh_to_xyxy(self, bbox_xywh):\n x, y, w, h = bbox_xywh\n x1 = max(int(x - w / 2), 0)\n x2 = min(int(x + w / 2), self.width - 1)\n y1 = max(int(y - h / 2), 0)\n y2 = min(int(y + h / 2), self.height - 1)\n return x1, y1, x2, y2\n\n def _tlwh_to_xyxy(self, bbox_tlwh):\n \"\"\"\n TODO:\n Convert bbox from xtl_ytl_w_h to xc_yc_w_h\n Thanks JieChen91@github.com for reporting this bug!\n \"\"\"\n x, y, w, h = bbox_tlwh\n x1 = max(int(x), 0)\n x2 = min(int(x+w), self.width - 1)\n y1 = max(int(y), 0)\n y2 = min(int(y+h), self.height - 1)\n return x1, y1, x2, y2\n\n def increment_ages(self):\n self.tracker.increment_ages()\n\n def _xyxy_to_tlwh(self, bbox_xyxy):\n x1, y1, x2, y2 = bbox_xyxy\n\n t = x1\n l = y1\n w = int(x2 - x1)\n h = int(y2 - y1)\n return t, l, w, h\n\n def _get_features(self, bbox_xywh, ori_img):\n im_crops = []\n for box in bbox_xywh:\n x1, y1, x2, y2 = self._xywh_to_xyxy(box)\n im = ori_img[y1:y2, x1:x2]\n im_crops.append(im)\n if im_crops:\n features = self.extractor(im_crops)\n else:\n features = np.array([])\n return features\n" }, { "path": "asone/trackers/deep_sort/tracker/parser.py", "content": "import os\nimport yaml\nfrom easydict import EasyDict as edict\n\n\nclass YamlParser(edict):\n \"\"\"\n This is yaml parser based on EasyDict.\n \"\"\"\n\n def __init__(self, cfg_dict=None, config_file=None):\n if cfg_dict is None:\n cfg_dict = {}\n\n if config_file is not None:\n assert(os.path.isfile(config_file))\n with open(config_file, 'r') as fo:\n yaml_ = yaml.load(fo.read(), Loader=yaml.FullLoader)\n cfg_dict.update(yaml_)\n\n super(YamlParser, self).__init__(cfg_dict)\n\n def merge_from_file(self, config_file):\n with open(config_file, 'r') as fo:\n yaml_ = yaml.load(fo.read(), Loader=yaml.FullLoader)\n self.update(yaml_)\n\n def merge_from_dict(self, config_dict):\n self.update(config_dict)\n\n\ndef get_config(config_file=None):\n return YamlParser(config_file=config_file)\n\n\nif __name__ == \"__main__\":\n cfg = YamlParser(config_file=\"../configs/yolov3.yaml\")\n cfg.merge_from_file(\"../configs/deep_sort.yaml\")\n\n import ipdb\n ipdb.set_trace()\n" }, { "path": "asone/trackers/deep_sort/tracker/sort/__init__.py", "content": "" }, { "path": "asone/trackers/deep_sort/tracker/sort/detection.py", "content": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\n\n\nclass Detection(object):\n \"\"\"\n This class represents a bounding box detection in a single image.\n\n Parameters\n ----------\n tlwh : array_like\n Bounding box in format `(x, y, w, h)`.\n confidence : float\n Detector confidence score.\n feature : array_like\n A feature vector that describes the object contained in this image.\n\n Attributes\n ----------\n tlwh : ndarray\n Bounding box in format `(top left x, top left y, width, height)`.\n confidence : ndarray\n Detector confidence score.\n feature : ndarray | NoneType\n A feature vector that describes the object contained in this image.\n\n \"\"\"\n\n def __init__(self, tlwh, confidence, feature, oid):\n self.tlwh = np.asarray(tlwh, dtype=np.float)\n self.confidence = float(confidence)\n self.feature = np.asarray(feature, dtype=np.float32)\n self.oid = oid\n\n def to_tlbr(self):\n \"\"\"Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,\n `(top left, bottom right)`.\n \"\"\"\n ret = self.tlwh.copy()\n ret[2:] += ret[:2]\n return ret\n\n def to_xyah(self):\n \"\"\"Convert bounding box to format `(center x, center y, aspect ratio,\n height)`, where the aspect ratio is `width / height`.\n \"\"\"\n ret = self.tlwh.copy()\n ret[:2] += ret[2:] / 2\n ret[2] /= ret[3]\n return ret\n" }, { "path": "asone/trackers/deep_sort/tracker/sort/iou_matching.py", "content": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom . import linear_assignment\n\n\ndef iou(bbox, candidates):\n \"\"\"Computer intersection over union.\n\n Parameters\n ----------\n bbox : ndarray\n A bounding box in format `(top left x, top left y, width, height)`.\n candidates : ndarray\n A matrix of candidate bounding boxes (one per row) in the same format\n as `bbox`.\n\n Returns\n -------\n ndarray\n The intersection over union in [0, 1] between the `bbox` and each\n candidate. A higher score means a larger fraction of the `bbox` is\n occluded by the candidate.\n\n \"\"\"\n bbox_tl, bbox_br = bbox[:2], bbox[:2] + bbox[2:]\n candidates_tl = candidates[:, :2]\n candidates_br = candidates[:, :2] + candidates[:, 2:]\n\n tl = np.c_[np.maximum(bbox_tl[0], candidates_tl[:, 0])[:, np.newaxis],\n np.maximum(bbox_tl[1], candidates_tl[:, 1])[:, np.newaxis]]\n br = np.c_[np.minimum(bbox_br[0], candidates_br[:, 0])[:, np.newaxis],\n np.minimum(bbox_br[1], candidates_br[:, 1])[:, np.newaxis]]\n wh = np.maximum(0., br - tl)\n\n area_intersection = wh.prod(axis=1)\n area_bbox = bbox[2:].prod()\n area_candidates = candidates[:, 2:].prod(axis=1)\n return area_intersection / (area_bbox + area_candidates - area_intersection)\n\n\ndef iou_cost(tracks, detections, track_indices=None,\n detection_indices=None):\n \"\"\"An intersection over union distance metric.\n\n Parameters\n ----------\n tracks : List[deep_sort.track.Track]\n A list of tracks.\n detections : List[deep_sort.detection.Detection]\n A list of detections.\n track_indices : Optional[List[int]]\n A list of indices to tracks that should be matched. Defaults to\n all `tracks`.\n detection_indices : Optional[List[int]]\n A list of indices to detections that should be matched. Defaults\n to all `detections`.\n\n Returns\n -------\n ndarray\n Returns a cost matrix of shape\n len(track_indices), len(detection_indices) where entry (i, j) is\n `1 - iou(tracks[track_indices[i]], detections[detection_indices[j]])`.\n\n \"\"\"\n if track_indices is None:\n track_indices = np.arange(len(tracks))\n if detection_indices is None:\n detection_indices = np.arange(len(detections))\n\n cost_matrix = np.zeros((len(track_indices), len(detection_indices)))\n for row, track_idx in enumerate(track_indices):\n if tracks[track_idx].time_since_update > 1:\n cost_matrix[row, :] = linear_assignment.INFTY_COST\n continue\n\n bbox = tracks[track_idx].to_tlwh()\n candidates = np.asarray(\n [detections[i].tlwh for i in detection_indices])\n cost_matrix[row, :] = 1. - iou(bbox, candidates)\n return cost_matrix\n" }, { "path": "asone/trackers/deep_sort/tracker/sort/kalman_filter.py", "content": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport scipy.linalg\n\n\n\"\"\"\nTable for the 0.95 quantile of the chi-square distribution with N degrees of\nfreedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv\nfunction and used as Mahalanobis gating threshold.\n\"\"\"\nchi2inv95 = {\n 1: 3.8415,\n 2: 5.9915,\n 3: 7.8147,\n 4: 9.4877,\n 5: 11.070,\n 6: 12.592,\n 7: 14.067,\n 8: 15.507,\n 9: 16.919}\n\n\nclass KalmanFilter(object):\n \"\"\"\n A simple Kalman filter for tracking bounding boxes in image space.\n\n The 8-dimensional state space\n\n x, y, a, h, vx, vy, va, vh\n\n contains the bounding box center position (x, y), aspect ratio a, height h,\n and their respective velocities.\n\n Object motion follows a constant velocity model. The bounding box location\n (x, y, a, h) is taken as direct observation of the state space (linear\n observation model).\n\n \"\"\"\n\n def __init__(self):\n ndim, dt = 4, 1.\n\n # Create Kalman filter model matrices.\n self._motion_mat = np.eye(2 * ndim, 2 * ndim)\n for i in range(ndim):\n self._motion_mat[i, ndim + i] = dt\n self._update_mat = np.eye(ndim, 2 * ndim)\n\n # Motion and observation uncertainty are chosen relative to the current\n # state estimate. These weights control the amount of uncertainty in\n # the model. This is a bit hacky.\n self._std_weight_position = 1. / 20\n self._std_weight_velocity = 1. / 160\n\n def initiate(self, measurement):\n \"\"\"Create track from unassociated measurement.\n\n Parameters\n ----------\n measurement : ndarray\n Bounding box coordinates (x, y, a, h) with center position (x, y),\n aspect ratio a, and height h.\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the mean vector (8 dimensional) and covariance matrix (8x8\n dimensional) of the new track. Unobserved velocities are initialized\n to 0 mean.\n\n \"\"\"\n mean_pos = measurement\n mean_vel = np.zeros_like(mean_pos)\n mean = np.r_[mean_pos, mean_vel]\n\n std = [\n 2 * self._std_weight_position * measurement[3],\n 2 * self._std_weight_position * measurement[3],\n 1e-2,\n 2 * self._std_weight_position * measurement[3],\n 10 * self._std_weight_velocity * measurement[3],\n 10 * self._std_weight_velocity * measurement[3],\n 1e-5,\n 10 * self._std_weight_velocity * measurement[3]]\n covariance = np.diag(np.square(std))\n return mean, covariance\n\n def predict(self, mean, covariance):\n \"\"\"Run Kalman filter prediction step.\n\n Parameters\n ----------\n mean : ndarray\n The 8 dimensional mean vector of the object state at the previous\n time step.\n covariance : ndarray\n The 8x8 dimensional covariance matrix of the object state at the\n previous time step.\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the mean vector and covariance matrix of the predicted\n state. Unobserved velocities are initialized to 0 mean.\n\n \"\"\"\n std_pos = [\n self._std_weight_position * mean[3],\n self._std_weight_position * mean[3],\n 1e-2,\n self._std_weight_position * mean[3]]\n std_vel = [\n self._std_weight_velocity * mean[3],\n self._std_weight_velocity * mean[3],\n 1e-5,\n self._std_weight_velocity * mean[3]]\n motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))\n\n mean = np.dot(self._motion_mat, mean)\n covariance = np.linalg.multi_dot((\n self._motion_mat, covariance, self._motion_mat.T)) + motion_cov\n\n return mean, covariance\n\n def project(self, mean, covariance):\n \"\"\"Project state distribution to measurement space.\n\n Parameters\n ----------\n mean : ndarray\n The state's mean vector (8 dimensional array).\n covariance : ndarray\n The state's covariance matrix (8x8 dimensional).\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the projected mean and covariance matrix of the given state\n estimate.\n\n \"\"\"\n std = [\n self._std_weight_position * mean[3],\n self._std_weight_position * mean[3],\n 1e-1,\n self._std_weight_position * mean[3]]\n innovation_cov = np.diag(np.square(std))\n\n mean = np.dot(self._update_mat, mean)\n covariance = np.linalg.multi_dot((\n self._update_mat, covariance, self._update_mat.T))\n return mean, covariance + innovation_cov\n\n def update(self, mean, covariance, measurement):\n \"\"\"Run Kalman filter correction step.\n\n Parameters\n ----------\n mean : ndarray\n The predicted state's mean vector (8 dimensional).\n covariance : ndarray\n The state's covariance matrix (8x8 dimensional).\n measurement : ndarray\n The 4 dimensional measurement vector (x, y, a, h), where (x, y)\n is the center position, a the aspect ratio, and h the height of the\n bounding box.\n\n Returns\n -------\n (ndarray, ndarray)\n Returns the measurement-corrected state distribution.\n\n \"\"\"\n projected_mean, projected_cov = self.project(mean, covariance)\n\n chol_factor, lower = scipy.linalg.cho_factor(\n projected_cov, lower=True, check_finite=False)\n kalman_gain = scipy.linalg.cho_solve(\n (chol_factor, lower), np.dot(covariance, self._update_mat.T).T,\n check_finite=False).T\n innovation = measurement - projected_mean\n\n new_mean = mean + np.dot(innovation, kalman_gain.T)\n new_covariance = covariance - np.linalg.multi_dot((\n kalman_gain, projected_cov, kalman_gain.T))\n return new_mean, new_covariance\n\n def gating_distance(self, mean, covariance, measurements,\n only_position=False):\n \"\"\"Compute gating distance between state distribution and measurements.\n\n A suitable distance threshold can be obtained from `chi2inv95`. If\n `only_position` is False, the chi-square distribution has 4 degrees of\n freedom, otherwise 2.\n\n Parameters\n ----------\n mean : ndarray\n Mean vector over the state distribution (8 dimensional).\n covariance : ndarray\n Covariance of the state distribution (8x8 dimensional).\n measurements : ndarray\n An Nx4 dimensional matrix of N measurements, each in\n format (x, y, a, h) where (x, y) is the bounding box center\n position, a the aspect ratio, and h the height.\n only_position : Optional[bool]\n If True, distance computation is done with respect to the bounding\n box center position only.\n\n Returns\n -------\n ndarray\n Returns an array of length N, where the i-th element contains the\n squared Mahalanobis distance between (mean, covariance) and\n `measurements[i]`.\n\n \"\"\"\n mean, covariance = self.project(mean, covariance)\n if only_position:\n mean, covariance = mean[:2], covariance[:2, :2]\n measurements = measurements[:, :2]\n\n cholesky_factor = np.linalg.cholesky(covariance)\n d = measurements - mean\n z = scipy.linalg.solve_triangular(\n cholesky_factor, d.T, lower=True, check_finite=False,\n overwrite_b=True)\n squared_maha = np.sum(z * z, axis=0)\n return squared_maha\n" }, { "path": "asone/trackers/deep_sort/tracker/sort/linear_assignment.py", "content": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\n# from sklearn.utils.linear_assignment_ import linear_assignment\nfrom scipy.optimize import linear_sum_assignment as linear_assignment\nfrom . import kalman_filter\n\n\nINFTY_COST = 1e+5\n\n\ndef min_cost_matching(\n distance_metric, max_distance, tracks, detections, track_indices=None,\n detection_indices=None):\n \"\"\"Solve linear assignment problem.\n\n Parameters\n ----------\n distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray\n The distance metric is given a list of tracks and detections as well as\n a list of N track indices and M detection indices. The metric should\n return the NxM dimensional cost matrix, where element (i, j) is the\n association cost between the i-th track in the given track indices and\n the j-th detection in the given detection_indices.\n max_distance : float\n Gating threshold. Associations with cost larger than this value are\n disregarded.\n tracks : List[track.Track]\n A list of predicted tracks at the current time step.\n detections : List[detection.Detection]\n A list of detections at the current time step.\n track_indices : List[int]\n List of track indices that maps rows in `cost_matrix` to tracks in\n `tracks` (see description above).\n detection_indices : List[int]\n List of detection indices that maps columns in `cost_matrix` to\n detections in `detections` (see description above).\n\n Returns\n -------\n (List[(int, int)], List[int], List[int])\n Returns a tuple with the following three entries:\n * A list of matched track and detection indices.\n * A list of unmatched track indices.\n * A list of unmatched detection indices.\n\n \"\"\"\n if track_indices is None:\n track_indices = np.arange(len(tracks))\n if detection_indices is None:\n detection_indices = np.arange(len(detections))\n\n if len(detection_indices) == 0 or len(track_indices) == 0:\n return [], track_indices, detection_indices # Nothing to match.\n\n cost_matrix = distance_metric(\n tracks, detections, track_indices, detection_indices)\n cost_matrix[cost_matrix > max_distance] = max_distance + 1e-5\n\n row_indices, col_indices = linear_assignment(cost_matrix)\n\n matches, unmatched_tracks, unmatched_detections = [], [], []\n for col, detection_idx in enumerate(detection_indices):\n if col not in col_indices:\n unmatched_detections.append(detection_idx)\n for row, track_idx in enumerate(track_indices):\n if row not in row_indices:\n unmatched_tracks.append(track_idx)\n for row, col in zip(row_indices, col_indices):\n track_idx = track_indices[row]\n detection_idx = detection_indices[col]\n if cost_matrix[row, col] > max_distance:\n unmatched_tracks.append(track_idx)\n unmatched_detections.append(detection_idx)\n else:\n matches.append((track_idx, detection_idx))\n return matches, unmatched_tracks, unmatched_detections\n\n\ndef matching_cascade(\n distance_metric, max_distance, cascade_depth, tracks, detections,\n track_indices=None, detection_indices=None):\n \"\"\"Run matching cascade.\n\n Parameters\n ----------\n distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray\n The distance metric is given a list of tracks and detections as well as\n a list of N track indices and M detection indices. The metric should\n return the NxM dimensional cost matrix, where element (i, j) is the\n association cost between the i-th track in the given track indices and\n the j-th detection in the given detection indices.\n max_distance : float\n Gating threshold. Associations with cost larger than this value are\n disregarded.\n cascade_depth: int\n The cascade depth, should be se to the maximum track age.\n tracks : List[track.Track]\n A list of predicted tracks at the current time step.\n detections : List[detection.Detection]\n A list of detections at the current time step.\n track_indices : Optional[List[int]]\n List of track indices that maps rows in `cost_matrix` to tracks in\n `tracks` (see description above). Defaults to all tracks.\n detection_indices : Optional[List[int]]\n List of detection indices that maps columns in `cost_matrix` to\n detections in `detections` (see description above). Defaults to all\n detections.\n\n Returns\n -------\n (List[(int, int)], List[int], List[int])\n Returns a tuple with the following three entries:\n * A list of matched track and detection indices.\n * A list of unmatched track indices.\n * A list of unmatched detection indices.\n\n \"\"\"\n if track_indices is None:\n track_indices = list(range(len(tracks)))\n if detection_indices is None:\n detection_indices = list(range(len(detections)))\n\n unmatched_detections = detection_indices\n matches = []\n for level in range(cascade_depth):\n if len(unmatched_detections) == 0: # No detections left\n break\n\n track_indices_l = [\n k for k in track_indices\n if tracks[k].time_since_update == 1 + level\n ]\n if len(track_indices_l) == 0: # Nothing to match at this level\n continue\n\n matches_l, _, unmatched_detections = \\\n min_cost_matching(\n distance_metric, max_distance, tracks, detections,\n track_indices_l, unmatched_detections)\n matches += matches_l\n unmatched_tracks = list(set(track_indices) - set(k for k, _ in matches))\n return matches, unmatched_tracks, unmatched_detections\n\n\ndef gate_cost_matrix(\n kf, cost_matrix, tracks, detections, track_indices, detection_indices,\n gated_cost=INFTY_COST, only_position=False):\n \"\"\"Invalidate infeasible entries in cost matrix based on the state\n distributions obtained by Kalman filtering.\n\n Parameters\n ----------\n kf : The Kalman filter.\n cost_matrix : ndarray\n The NxM dimensional cost matrix, where N is the number of track indices\n and M is the number of detection indices, such that entry (i, j) is the\n association cost between `tracks[track_indices[i]]` and\n `detections[detection_indices[j]]`.\n tracks : List[track.Track]\n A list of predicted tracks at the current time step.\n detections : List[detection.Detection]\n A list of detections at the current time step.\n track_indices : List[int]\n List of track indices that maps rows in `cost_matrix` to tracks in\n `tracks` (see description above).\n detection_indices : List[int]\n List of detection indices that maps columns in `cost_matrix` to\n detections in `detections` (see description above).\n gated_cost : Optional[float]\n Entries in the cost matrix corresponding to infeasible associations are\n set this value. Defaults to a very large value.\n only_position : Optional[bool]\n If True, only the x, y position of the state distribution is considered\n during gating. Defaults to False.\n\n Returns\n -------\n ndarray\n Returns the modified cost matrix.\n\n \"\"\"\n gating_dim = 2 if only_position else 4\n gating_threshold = kalman_filter.chi2inv95[gating_dim]\n measurements = np.asarray(\n [detections[i].to_xyah() for i in detection_indices])\n for row, track_idx in enumerate(track_indices):\n track = tracks[track_idx]\n gating_distance = kf.gating_distance(\n track.mean, track.covariance, measurements, only_position)\n cost_matrix[row, gating_distance > gating_threshold] = gated_cost\n return cost_matrix\n" }, { "path": "asone/trackers/deep_sort/tracker/sort/nn_matching.py", "content": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\n\n\ndef _pdist(a, b):\n \"\"\"Compute pair-wise squared distance between points in `a` and `b`.\n\n Parameters\n ----------\n a : array_like\n An NxM matrix of N samples of dimensionality M.\n b : array_like\n An LxM matrix of L samples of dimensionality M.\n\n Returns\n -------\n ndarray\n Returns a matrix of size len(a), len(b) such that eleement (i, j)\n contains the squared distance between `a[i]` and `b[j]`.\n\n \"\"\"\n a, b = np.asarray(a), np.asarray(b)\n if len(a) == 0 or len(b) == 0:\n return np.zeros((len(a), len(b)))\n a2, b2 = np.square(a).sum(axis=1), np.square(b).sum(axis=1)\n r2 = -2. * np.dot(a, b.T) + a2[:, None] + b2[None, :]\n r2 = np.clip(r2, 0., float(np.inf))\n return r2\n\n\ndef _cosine_distance(a, b, data_is_normalized=False):\n \"\"\"Compute pair-wise cosine distance between points in `a` and `b`.\n\n Parameters\n ----------\n a : array_like\n An NxM matrix of N samples of dimensionality M.\n b : array_like\n An LxM matrix of L samples of dimensionality M.\n data_is_normalized : Optional[bool]\n If True, assumes rows in a and b are unit length vectors.\n Otherwise, a and b are explicitly normalized to lenght 1.\n\n Returns\n -------\n ndarray\n Returns a matrix of size len(a), len(b) such that eleement (i, j)\n contains the squared distance between `a[i]` and `b[j]`.\n\n \"\"\"\n if not data_is_normalized:\n a = np.asarray(a) / np.linalg.norm(a, axis=1, keepdims=True)\n b = np.asarray(b) / np.linalg.norm(b, axis=1, keepdims=True)\n return 1. - np.dot(a, b.T)\n\n\ndef _nn_euclidean_distance(x, y):\n \"\"\" Helper function for nearest neighbor distance metric (Euclidean).\n\n Parameters\n ----------\n x : ndarray\n A matrix of N row-vectors (sample points).\n y : ndarray\n A matrix of M row-vectors (query points).\n\n Returns\n -------\n ndarray\n A vector of length M that contains for each entry in `y` the\n smallest Euclidean distance to a sample in `x`.\n\n \"\"\"\n distances = _pdist(x, y)\n return np.maximum(0.0, distances.min(axis=0))\n\n\ndef _nn_cosine_distance(x, y):\n \"\"\" Helper function for nearest neighbor distance metric (cosine).\n\n Parameters\n ----------\n x : ndarray\n A matrix of N row-vectors (sample points).\n y : ndarray\n A matrix of M row-vectors (query points).\n\n Returns\n -------\n ndarray\n A vector of length M that contains for each entry in `y` the\n smallest cosine distance to a sample in `x`.\n\n \"\"\"\n distances = _cosine_distance(x, y)\n return distances.min(axis=0)\n\n\nclass NearestNeighborDistanceMetric(object):\n \"\"\"\n A nearest neighbor distance metric that, for each target, returns\n the closest distance to any sample that has been observed so far.\n\n Parameters\n ----------\n metric : str\n Either \"euclidean\" or \"cosine\".\n matching_threshold: float\n The matching threshold. Samples with larger distance are considered an\n invalid match.\n budget : Optional[int]\n If not None, fix samples per class to at most this number. Removes\n the oldest samples when the budget is reached.\n\n Attributes\n ----------\n samples : Dict[int -> List[ndarray]]\n A dictionary that maps from target identities to the list of samples\n that have been observed so far.\n\n \"\"\"\n\n def __init__(self, metric, matching_threshold, budget=None):\n\n if metric == \"euclidean\":\n self._metric = _nn_euclidean_distance\n elif metric == \"cosine\":\n self._metric = _nn_cosine_distance\n else:\n raise ValueError(\n \"Invalid metric; must be either 'euclidean' or 'cosine'\")\n self.matching_threshold = matching_threshold\n self.budget = budget\n self.samples = {}\n\n def partial_fit(self, features, targets, active_targets):\n \"\"\"Update the distance metric with new data.\n\n Parameters\n ----------\n features : ndarray\n An NxM matrix of N features of dimensionality M.\n targets : ndarray\n An integer array of associated target identities.\n active_targets : List[int]\n A list of targets that are currently present in the scene.\n\n \"\"\"\n for feature, target in zip(features, targets):\n self.samples.setdefault(target, []).append(feature)\n if self.budget is not None:\n self.samples[target] = self.samples[target][-self.budget:]\n self.samples = {k: self.samples[k] for k in active_targets}\n\n def distance(self, features, targets):\n \"\"\"Compute distance between features and targets.\n\n Parameters\n ----------\n features : ndarray\n An NxM matrix of N features of dimensionality M.\n targets : List[int]\n A list of targets to match the given `features` against.\n\n Returns\n -------\n ndarray\n Returns a cost matrix of shape len(targets), len(features), where\n element (i, j) contains the closest squared distance between\n `targets[i]` and `features[j]`.\n\n \"\"\"\n cost_matrix = np.zeros((len(targets), len(features)))\n for i, target in enumerate(targets):\n cost_matrix[i, :] = self._metric(self.samples[target], features)\n return cost_matrix\n" }, { "path": "asone/trackers/deep_sort/tracker/sort/preprocessing.py", "content": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport cv2\n\n\ndef non_max_suppression(boxes, max_bbox_overlap, scores=None):\n \"\"\"Suppress overlapping detections.\n\n Original code from [1]_ has been adapted to include confidence score.\n\n .. [1] http://www.pyimagesearch.com/2015/02/16/\n faster-non-maximum-suppression-python/\n\n Examples\n --------\n\n >>> boxes = [d.roi for d in detections]\n >>> scores = [d.confidence for d in detections]\n >>> indices = non_max_suppression(boxes, max_bbox_overlap, scores)\n >>> detections = [detections[i] for i in indices]\n\n Parameters\n ----------\n boxes : ndarray\n Array of ROIs (x, y, width, height).\n max_bbox_overlap : float\n ROIs that overlap more than this values are suppressed.\n scores : Optional[array_like]\n Detector confidence score.\n\n Returns\n -------\n List[int]\n Returns indices of detections that have survived non-maxima suppression.\n\n \"\"\"\n if len(boxes) == 0:\n return []\n\n boxes = boxes.astype(np.float)\n pick = []\n\n x1 = boxes[:, 0]\n y1 = boxes[:, 1]\n x2 = boxes[:, 2] + boxes[:, 0]\n y2 = boxes[:, 3] + boxes[:, 1]\n\n area = (x2 - x1 + 1) * (y2 - y1 + 1)\n if scores is not None:\n idxs = np.argsort(scores)\n else:\n idxs = np.argsort(y2)\n\n while len(idxs) > 0:\n last = len(idxs) - 1\n i = idxs[last]\n pick.append(i)\n\n xx1 = np.maximum(x1[i], x1[idxs[:last]])\n yy1 = np.maximum(y1[i], y1[idxs[:last]])\n xx2 = np.minimum(x2[i], x2[idxs[:last]])\n yy2 = np.minimum(y2[i], y2[idxs[:last]])\n\n w = np.maximum(0, xx2 - xx1 + 1)\n h = np.maximum(0, yy2 - yy1 + 1)\n\n overlap = (w * h) / area[idxs[:last]]\n\n idxs = np.delete(\n idxs, np.concatenate(\n ([last], np.where(overlap > max_bbox_overlap)[0])))\n\n return pick\n" }, { "path": "asone/trackers/deep_sort/tracker/sort/track.py", "content": "# vim: expandtab:ts=4:sw=4\n\n\nclass TrackState:\n \"\"\"\n Enumeration type for the single target track state. Newly created tracks are\n classified as `tentative` until enough evidence has been collected. Then,\n the track state is changed to `confirmed`. Tracks that are no longer alive\n are classified as `deleted` to mark them for removal from the set of active\n tracks.\n\n \"\"\"\n\n Tentative = 1\n Confirmed = 2\n Deleted = 3\n\n\nclass Track:\n \"\"\"\n A single target track with state space `(x, y, a, h)` and associated\n velocities, where `(x, y)` is the center of the bounding box, `a` is the\n aspect ratio and `h` is the height.\n\n Parameters\n ----------\n mean : ndarray\n Mean vector of the initial state distribution.\n covariance : ndarray\n Covariance matrix of the initial state distribution.\n track_id : int\n A unique track identifier.\n n_init : int\n Number of consecutive detections before the track is confirmed. The\n track state is set to `Deleted` if a miss occurs within the first\n `n_init` frames.\n max_age : int\n The maximum number of consecutive misses before the track state is\n set to `Deleted`.\n feature : Optional[ndarray]\n Feature vector of the detection this track originates from. If not None,\n this feature is added to the `features` cache.\n\n Attributes\n ----------\n mean : ndarray\n Mean vector of the initial state distribution.\n covariance : ndarray\n Covariance matrix of the initial state distribution.\n track_id : int\n A unique track identifier.\n hits : int\n Total number of measurement updates.\n age : int\n Total number of frames since first occurance.\n time_since_update : int\n Total number of frames since last measurement update.\n state : TrackState\n The current track state.\n features : List[ndarray]\n A cache of features. On each measurement update, the associated feature\n vector is added to this list.\n\n \"\"\"\n\n def __init__(self, mean, covariance, track_id, n_init, max_age, oid,\n feature=None):\n self.mean = mean\n self.covariance = covariance\n self.track_id = track_id\n self.oid = oid\n self.hits = 1\n self.age = 1\n self.time_since_update = 0\n\n self.state = TrackState.Tentative\n self.features = []\n if feature is not None:\n self.features.append(feature)\n\n self._n_init = n_init\n self._max_age = max_age\n\n def to_tlwh(self):\n \"\"\"Get current position in bounding box format `(top left x, top left y,\n width, height)`.\n\n Returns\n -------\n ndarray\n The bounding box.\n\n \"\"\"\n ret = self.mean[:4].copy()\n ret[2] *= ret[3]\n ret[:2] -= ret[2:] / 2\n return ret\n\n def to_tlbr(self):\n \"\"\"Get current position in bounding box format `(min x, miny, max x,\n max y)`.\n\n Returns\n -------\n ndarray\n The bounding box.\n\n \"\"\"\n ret = self.to_tlwh()\n ret[2:] = ret[:2] + ret[2:]\n return ret\n\n def increment_age(self):\n self.age += 1\n self.time_since_update += 1\n\n def predict(self, kf):\n \"\"\"Propagate the state distribution to the current time step using a\n Kalman filter prediction step.\n\n Parameters\n ----------\n kf : kalman_filter.KalmanFilter\n The Kalman filter.\n\n \"\"\"\n self.mean, self.covariance = kf.predict(self.mean, self.covariance)\n self.increment_age()\n\n def update(self, kf, detection):\n \"\"\"Perform Kalman filter measurement update step and update the feature\n cache.\n\n Parameters\n ----------\n kf : kalman_filter.KalmanFilter\n The Kalman filter.\n detection : Detection\n The associated detection.\n\n \"\"\"\n self.mean, self.covariance = kf.update(\n self.mean, self.covariance, detection.to_xyah())\n self.features.append(detection.feature)\n\n self.hits += 1\n self.time_since_update = 0\n if self.state == TrackState.Tentative and self.hits >= self._n_init:\n self.state = TrackState.Confirmed\n\n def mark_missed(self):\n \"\"\"Mark this track as missed (no association at the current time step).\n \"\"\"\n if self.state == TrackState.Tentative:\n self.state = TrackState.Deleted\n elif self.time_since_update > self._max_age:\n self.state = TrackState.Deleted\n\n def is_tentative(self):\n \"\"\"Returns True if this track is tentative (unconfirmed).\n \"\"\"\n return self.state == TrackState.Tentative\n\n def is_confirmed(self):\n \"\"\"Returns True if this track is confirmed.\"\"\"\n return self.state == TrackState.Confirmed\n\n def is_deleted(self):\n \"\"\"Returns True if this track is dead and should be deleted.\"\"\"\n return self.state == TrackState.Deleted\n" }, { "path": "asone/trackers/deep_sort/tracker/sort/tracker.py", "content": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom . import kalman_filter\nfrom . import linear_assignment\nfrom . import iou_matching\nfrom .track import Track\n\n\nclass Tracker:\n \"\"\"\n This is the multi-target tracker.\n\n Parameters\n ----------\n metric : nn_matching.NearestNeighborDistanceMetric\n A distance metric for measurement-to-track association.\n max_age : int\n Maximum number of missed misses before a track is deleted.\n n_init : int\n Number of consecutive detections before the track is confirmed. The\n track state is set to `Deleted` if a miss occurs within the first\n `n_init` frames.\n\n Attributes\n ----------\n metric : nn_matching.NearestNeighborDistanceMetric\n The distance metric used for measurement to track association.\n max_age : int\n Maximum number of missed misses before a track is deleted.\n n_init : int\n Number of frames that a track remains in initialization phase.\n kf : kalman_filter.KalmanFilter\n A Kalman filter to filter target trajectories in image space.\n tracks : List[Track]\n The list of active tracks at the current time step.\n\n \"\"\"\n\n def __init__(self, metric, max_iou_distance=0.7, max_age=70, n_init=3):\n self.metric = metric\n self.max_iou_distance = max_iou_distance\n self.max_age = max_age\n self.n_init = n_init\n\n self.kf = kalman_filter.KalmanFilter()\n self.tracks = []\n self._next_id = 1\n\n def predict(self):\n \"\"\"Propagate track state distributions one time step forward.\n\n This function should be called once every time step, before `update`.\n \"\"\"\n for track in self.tracks:\n track.predict(self.kf)\n\n def increment_ages(self):\n for track in self.tracks:\n track.increment_age()\n track.mark_missed()\n\n def update(self, detections):\n \"\"\"Perform measurement update and track management.\n\n Parameters\n ----------\n detections : List[deep_sort.detection.Detection]\n A list of detections at the current time step.\n\n \"\"\"\n # Run matching cascade.\n matches, unmatched_tracks, unmatched_detections = \\\n self._match(detections)\n\n # Update track set.\n for track_idx, detection_idx in matches:\n self.tracks[track_idx].update(self.kf, detections[detection_idx])\n for track_idx in unmatched_tracks:\n self.tracks[track_idx].mark_missed()\n for detection_idx in unmatched_detections:\n self._initiate_track(detections[detection_idx])\n self.tracks = [t for t in self.tracks if not t.is_deleted()]\n # print(\"LEN self.tracks\", len(self.tracks))\n # for t in self.tracks:\n # if not t.is_deleted():\n # print(t.__dict__)\n\n # Update distance metric.\n active_targets = [t.track_id for t in self.tracks if t.is_confirmed()]\n # print(active_targets)\n features, targets, oids = [], [], []\n for track in self.tracks:\n if not track.is_confirmed():\n continue\n features += track.features\n targets += [track.track_id for _ in track.features]\n # oids += track.oid\n track.features = []\n self.metric.partial_fit(\n np.asarray(features), np.asarray(targets), active_targets)\n\n def _match(self, detections):\n\n def gated_metric(tracks, dets, track_indices, detection_indices):\n features = np.array([dets[i].feature for i in detection_indices])\n targets = np.array([tracks[i].track_id for i in track_indices])\n cost_matrix = self.metric.distance(features, targets)\n cost_matrix = linear_assignment.gate_cost_matrix(\n self.kf, cost_matrix, tracks, dets, track_indices,\n detection_indices)\n\n return cost_matrix\n\n # Split track set into confirmed and unconfirmed tracks.\n confirmed_tracks = [\n i for i, t in enumerate(self.tracks) if t.is_confirmed()]\n unconfirmed_tracks = [\n i for i, t in enumerate(self.tracks) if not t.is_confirmed()]\n\n # Associate confirmed tracks using appearance features.\n matches_a, unmatched_tracks_a, unmatched_detections = linear_assignment.matching_cascade(gated_metric, self.metric.matching_threshold, self.max_age, self.tracks, detections, confirmed_tracks)\n\n # Associate remaining tracks together with unconfirmed tracks using IOU.\n iou_track_candidates = unconfirmed_tracks + [\n k for k in unmatched_tracks_a if\n self.tracks[k].time_since_update == 1]\n unmatched_tracks_a = [\n k for k in unmatched_tracks_a if\n self.tracks[k].time_since_update != 1]\n matches_b, unmatched_tracks_b, unmatched_detections = \\\n linear_assignment.min_cost_matching(\n iou_matching.iou_cost, self.max_iou_distance, self.tracks,\n detections, iou_track_candidates, unmatched_detections)\n\n matches = matches_a + matches_b\n unmatched_tracks = list(set(unmatched_tracks_a + unmatched_tracks_b))\n return matches, unmatched_tracks, unmatched_detections\n\n def _initiate_track(self, detection):\n mean, covariance = self.kf.initiate(detection.to_xyah())\n self.tracks.append(Track(\n mean, covariance, self._next_id, self.n_init, self.max_age, detection.oid,\n detection.feature))\n self._next_id += 1\n" }, { "path": "asone/trackers/motpy/__init__.py", "content": "" }, { "path": "asone/trackers/motpy/motpy.py", "content": "from motpy import Detection, MultiObjectTracker\nimport numpy as np\n\n\nclass Motpy:\n \n def __init__(self, detector, dt=0.1) -> None:\n self.tracker = MultiObjectTracker(dt=dt)\n self.detector = detector\n try:\n self.input_shape = tuple(detector.model.get_inputs()[0].shape[2:])\n except AttributeError as e:\n self.input_shape = (640, 640)\n self.obj_count = 0\n self.uuids = {}\n \n def detect_and_track(self, image: np.ndarray, config: dict) -> tuple:\n _dets_xyxy, image_info = self.detector.detect(\n image, **config\n )\n class_ids = []\n ids = []\n bboxes_xyxy = []\n scores = []\n if isinstance(_dets_xyxy, np.ndarray) and len(_dets_xyxy) > 0:\n self.tracker.step(detections=[\n Detection(\n box=box[:4],\n score= box[4],\n class_id=box[5]\n )\n for box in _dets_xyxy\n ])\n bboxes_xyxy, ids, scores, class_ids = self._tracker_update()\n return bboxes_xyxy, ids, scores, class_ids\n\n def _tracker_update(self):\n\n bboxes_xyxy = []\n class_ids = []\n scores = []\n ids = []\n\n tracked_objects = self.tracker.active_tracks()\n for obj in tracked_objects:\n \n if obj[0] in self.uuids:\n obj_id = self.uuids[obj[0]] \n else:\n self.obj_count += 1\n self.uuids[obj[0]] = self.obj_count \n obj_id = self.uuids[obj[0]]\n \n bboxes_xyxy.append(obj[1:2][0].tolist())\n class_ids.append(obj[3])\n scores.append(obj[2])\n ids.append(obj_id)\n return np.array(bboxes_xyxy), ids, scores, class_ids\n" }, { "path": "asone/trackers/nor_fair/__init__.py", "content": "" }, { "path": "asone/trackers/nor_fair/norfair.py", "content": "from norfair import Detection, Tracker\nimport numpy as np\n\n\nclass NorFair:\n def __init__(self, detector, max_distance_between_points=30) -> None:\n\n self.tracker = Tracker(\n distance_function=self._euclidean_distance,\n distance_threshold=max_distance_between_points,\n )\n self.detector = detector\n try:\n self.input_shape = tuple(detector.model.get_inputs()[0].shape[2:])\n except AttributeError as e:\n self.input_shape = (640, 640)\n\n def _euclidean_distance(self, detection, tracked_object):\n return np.linalg.norm(detection.points - tracked_object.estimate)\n\n def detect_and_track(self, image: np.ndarray, config: dict) -> tuple:\n \n _dets_xyxy, image_info = self.detector.detect(\n image, **config\n )\n\n class_ids = []\n ids = []\n bboxes_xyxy = []\n scores = []\n\n if isinstance(_dets_xyxy, np.ndarray) and len(_dets_xyxy) > 0:\n\n dets_xyxy = [\n Detection(\n np.array([(box[2] + box[0])/2, (box[3] + box[1])/2]), data=box)\n for box in _dets_xyxy\n # if box[-1] == 2\n ]\n\n bboxes_xyxy, ids, scores, class_ids = self._tracker_update(\n dets_xyxy, image_info)\n\n return bboxes_xyxy, ids, scores, class_ids\n\n def _tracker_update(self, dets_xyxy: list, image_info: dict):\n\n bboxes_xyxy = []\n class_ids = []\n scores = []\n ids = []\n\n tracked_objects = self.tracker.update(detections=dets_xyxy)\n\n for obj in tracked_objects:\n det = obj.last_detection.data\n bboxes_xyxy.append(det[:4])\n class_ids.append(int(det[-1]))\n scores.append(int(det[-2]))\n ids.append(obj.id)\n return np.array(bboxes_xyxy), ids, scores, class_ids\n" }, { "path": "asone/trackers/oc_sort/__init__.py", "content": "from asone.trackers.oc_sort.tracker.ocsort import OCSort" }, { "path": "asone/trackers/oc_sort/ocsort.py", "content": "from motpy import Detection, MultiObjectTracker\nimport numpy as np\nfrom .tracker.ocsort import OCSort\n\n\nclass OcSort:\n def __init__(self, detector) -> None:\n\n self.tracker = OCSort(det_thresh=0.2)\n self.detector = detector\n try:\n self.input_shape = tuple(detector.model.get_inputs()[0].shape[2:])\n except AttributeError as e:\n self.input_shape = (640, 640)\n \n def detect_and_track(self, image: np.ndarray, config: dict) -> tuple:\n \n _dets_xyxy, image_info = self.detector.detect(\n image, **config\n )\n image_info = [image_info['height'], image_info['width']]\n if isinstance(_dets_xyxy, np.ndarray) and len(_dets_xyxy) > 0:\n dets = self.tracker.update(_dets_xyxy, image_info)\n bbox_xyxy = dets[:, :4]\n ids = dets[:, 4]\n class_ids = dets[:, 5]\n scores = dets[:, 6]\n\n return bbox_xyxy, ids, scores, class_ids\n return [],[],[],[]\n" }, { "path": "asone/trackers/oc_sort/tracker/__init__.py", "content": "" }, { "path": "asone/trackers/oc_sort/tracker/association.py", "content": "import os\nimport numpy as np\n\n\ndef iou_batch(bboxes1, bboxes2):\n \"\"\"\n From SORT: Computes IOU between two bboxes in the form [x1,y1,x2,y2]\n \"\"\"\n bboxes2 = np.expand_dims(bboxes2, 0)\n bboxes1 = np.expand_dims(bboxes1, 1)\n \n xx1 = np.maximum(bboxes1[..., 0], bboxes2[..., 0])\n yy1 = np.maximum(bboxes1[..., 1], bboxes2[..., 1])\n xx2 = np.minimum(bboxes1[..., 2], bboxes2[..., 2])\n yy2 = np.minimum(bboxes1[..., 3], bboxes2[..., 3])\n w = np.maximum(0., xx2 - xx1)\n h = np.maximum(0., yy2 - yy1)\n wh = w * h\n o = wh / ((bboxes1[..., 2] - bboxes1[..., 0]) * (bboxes1[..., 3] - bboxes1[..., 1]) \n + (bboxes2[..., 2] - bboxes2[..., 0]) * (bboxes2[..., 3] - bboxes2[..., 1]) - wh) \n return(o) \n\n\ndef giou_batch(bboxes1, bboxes2):\n \"\"\"\n :param bbox_p: predict of bbox(N,4)(x1,y1,x2,y2)\n :param bbox_g: groundtruth of bbox(N,4)(x1,y1,x2,y2)\n :return:\n \"\"\"\n # for details should go to https://arxiv.org/pdf/1902.09630.pdf\n # ensure predict's bbox form\n bboxes2 = np.expand_dims(bboxes2, 0)\n bboxes1 = np.expand_dims(bboxes1, 1)\n\n xx1 = np.maximum(bboxes1[..., 0], bboxes2[..., 0])\n yy1 = np.maximum(bboxes1[..., 1], bboxes2[..., 1])\n xx2 = np.minimum(bboxes1[..., 2], bboxes2[..., 2])\n yy2 = np.minimum(bboxes1[..., 3], bboxes2[..., 3])\n w = np.maximum(0., xx2 - xx1)\n h = np.maximum(0., yy2 - yy1)\n wh = w * h\n union = ((bboxes1[..., 2] - bboxes1[..., 0]) * (bboxes1[..., 3] - bboxes1[..., 1]) \n + (bboxes2[..., 2] - bboxes2[..., 0]) * (bboxes2[..., 3] - bboxes2[..., 1]) - wh) \n iou = wh / union\n\n xxc1 = np.minimum(bboxes1[..., 0], bboxes2[..., 0])\n yyc1 = np.minimum(bboxes1[..., 1], bboxes2[..., 1])\n xxc2 = np.maximum(bboxes1[..., 2], bboxes2[..., 2])\n yyc2 = np.maximum(bboxes1[..., 3], bboxes2[..., 3])\n wc = xxc2 - xxc1 \n hc = yyc2 - yyc1 \n assert((wc > 0).all() and (hc > 0).all())\n area_enclose = wc * hc \n giou = iou - (area_enclose - union) / area_enclose\n giou = (giou + 1.)/2.0 # resize from (-1,1) to (0,1)\n return giou\n\n\ndef diou_batch(bboxes1, bboxes2):\n \"\"\"\n :param bbox_p: predict of bbox(N,4)(x1,y1,x2,y2)\n :param bbox_g: groundtruth of bbox(N,4)(x1,y1,x2,y2)\n :return:\n \"\"\"\n # for details should go to https://arxiv.org/pdf/1902.09630.pdf\n # ensure predict's bbox form\n bboxes2 = np.expand_dims(bboxes2, 0)\n bboxes1 = np.expand_dims(bboxes1, 1)\n\n # calculate the intersection box\n xx1 = np.maximum(bboxes1[..., 0], bboxes2[..., 0])\n yy1 = np.maximum(bboxes1[..., 1], bboxes2[..., 1])\n xx2 = np.minimum(bboxes1[..., 2], bboxes2[..., 2])\n yy2 = np.minimum(bboxes1[..., 3], bboxes2[..., 3])\n w = np.maximum(0., xx2 - xx1)\n h = np.maximum(0., yy2 - yy1)\n wh = w * h\n union = ((bboxes1[..., 2] - bboxes1[..., 0]) * (bboxes1[..., 3] - bboxes1[..., 1]) \n + (bboxes2[..., 2] - bboxes2[..., 0]) * (bboxes2[..., 3] - bboxes2[..., 1]) - wh) \n iou = wh / union\n centerx1 = (bboxes1[..., 0] + bboxes1[..., 2]) / 2.0\n centery1 = (bboxes1[..., 1] + bboxes1[..., 3]) / 2.0\n centerx2 = (bboxes2[..., 0] + bboxes2[..., 2]) / 2.0\n centery2 = (bboxes2[..., 1] + bboxes2[..., 3]) / 2.0\n\n inner_diag = (centerx1 - centerx2) ** 2 + (centery1 - centery2) ** 2\n\n xxc1 = np.minimum(bboxes1[..., 0], bboxes2[..., 0])\n yyc1 = np.minimum(bboxes1[..., 1], bboxes2[..., 1])\n xxc2 = np.maximum(bboxes1[..., 2], bboxes2[..., 2])\n yyc2 = np.maximum(bboxes1[..., 3], bboxes2[..., 3])\n\n outer_diag = (xxc2 - xxc1) ** 2 + (yyc2 - yyc1) ** 2\n diou = iou - inner_diag / outer_diag\n\n return (diou + 1) / 2.0 # resize from (-1,1) to (0,1)\n\ndef ciou_batch(bboxes1, bboxes2):\n \"\"\"\n :param bbox_p: predict of bbox(N,4)(x1,y1,x2,y2)\n :param bbox_g: groundtruth of bbox(N,4)(x1,y1,x2,y2)\n :return:\n \"\"\"\n # for details should go to https://arxiv.org/pdf/1902.09630.pdf\n # ensure predict's bbox form\n bboxes2 = np.expand_dims(bboxes2, 0)\n bboxes1 = np.expand_dims(bboxes1, 1)\n\n # calculate the intersection box\n xx1 = np.maximum(bboxes1[..., 0], bboxes2[..., 0])\n yy1 = np.maximum(bboxes1[..., 1], bboxes2[..., 1])\n xx2 = np.minimum(bboxes1[..., 2], bboxes2[..., 2])\n yy2 = np.minimum(bboxes1[..., 3], bboxes2[..., 3])\n w = np.maximum(0., xx2 - xx1)\n h = np.maximum(0., yy2 - yy1)\n wh = w * h\n union = ((bboxes1[..., 2] - bboxes1[..., 0]) * (bboxes1[..., 3] - bboxes1[..., 1]) \n + (bboxes2[..., 2] - bboxes2[..., 0]) * (bboxes2[..., 3] - bboxes2[..., 1]) - wh) \n iou = wh / union\n\n centerx1 = (bboxes1[..., 0] + bboxes1[..., 2]) / 2.0\n centery1 = (bboxes1[..., 1] + bboxes1[..., 3]) / 2.0\n centerx2 = (bboxes2[..., 0] + bboxes2[..., 2]) / 2.0\n centery2 = (bboxes2[..., 1] + bboxes2[..., 3]) / 2.0\n\n inner_diag = (centerx1 - centerx2) ** 2 + (centery1 - centery2) ** 2\n\n xxc1 = np.minimum(bboxes1[..., 0], bboxes2[..., 0])\n yyc1 = np.minimum(bboxes1[..., 1], bboxes2[..., 1])\n xxc2 = np.maximum(bboxes1[..., 2], bboxes2[..., 2])\n yyc2 = np.maximum(bboxes1[..., 3], bboxes2[..., 3])\n\n outer_diag = (xxc2 - xxc1) ** 2 + (yyc2 - yyc1) ** 2\n \n w1 = bboxes1[..., 2] - bboxes1[..., 0]\n h1 = bboxes1[..., 3] - bboxes1[..., 1]\n w2 = bboxes2[..., 2] - bboxes2[..., 0]\n h2 = bboxes2[..., 3] - bboxes2[..., 1]\n\n # prevent dividing over zero. add one pixel shift\n h2 = h2 + 1.\n h1 = h1 + 1.\n arctan = np.arctan(w2/h2) - np.arctan(w1/h1)\n v = (4 / (np.pi ** 2)) * (arctan ** 2)\n S = 1 - iou \n alpha = v / (S+v)\n ciou = iou - inner_diag / outer_diag - alpha * v\n \n return (ciou + 1) / 2.0 # resize from (-1,1) to (0,1)\n\n\ndef ct_dist(bboxes1, bboxes2):\n \"\"\"\n Measure the center distance between two sets of bounding boxes,\n this is a coarse implementation, we don't recommend using it only\n for association, which can be unstable and sensitive to frame rate\n and object speed.\n \"\"\"\n bboxes2 = np.expand_dims(bboxes2, 0)\n bboxes1 = np.expand_dims(bboxes1, 1)\n\n centerx1 = (bboxes1[..., 0] + bboxes1[..., 2]) / 2.0\n centery1 = (bboxes1[..., 1] + bboxes1[..., 3]) / 2.0\n centerx2 = (bboxes2[..., 0] + bboxes2[..., 2]) / 2.0\n centery2 = (bboxes2[..., 1] + bboxes2[..., 3]) / 2.0\n\n ct_dist2 = (centerx1 - centerx2) ** 2 + (centery1 - centery2) ** 2\n\n ct_dist = np.sqrt(ct_dist2)\n\n # The linear rescaling is a naive version and needs more study\n ct_dist = ct_dist / ct_dist.max()\n return ct_dist.max() - ct_dist # resize to (0,1)\n\n\n\ndef speed_direction_batch(dets, tracks):\n tracks = tracks[..., np.newaxis]\n CX1, CY1 = (dets[:,0] + dets[:,2])/2.0, (dets[:,1]+dets[:,3])/2.0\n CX2, CY2 = (tracks[:,0] + tracks[:,2]) /2.0, (tracks[:,1]+tracks[:,3])/2.0\n dx = CX1 - CX2 \n dy = CY1 - CY2 \n norm = np.sqrt(dx**2 + dy**2) + 1e-6\n dx = dx / norm \n dy = dy / norm\n return dy, dx # size: num_track x num_det\n\n\ndef linear_assignment(cost_matrix):\n try:\n import lap\n _, x, y = lap.lapjv(cost_matrix, extend_cost=True)\n return np.array([[y[i],i] for i in x if i >= 0]) #\n except ImportError:\n from scipy.optimize import linear_sum_assignment\n x, y = linear_sum_assignment(cost_matrix)\n return np.array(list(zip(x, y)))\n\n\ndef associate_detections_to_trackers(detections,trackers,iou_threshold = 0.3):\n \"\"\"\n Assigns detections to tracked object (both represented as bounding boxes)\n Returns 3 lists of matches, unmatched_detections and unmatched_trackers\n \"\"\"\n if(len(trackers)==0):\n return np.empty((0,2),dtype=int), np.arange(len(detections)), np.empty((0,5),dtype=int)\n\n iou_matrix = iou_batch(detections, trackers)\n\n if min(iou_matrix.shape) > 0:\n a = (iou_matrix > iou_threshold).astype(np.int32)\n if a.sum(1).max() == 1 and a.sum(0).max() == 1:\n matched_indices = np.stack(np.where(a), axis=1)\n else:\n matched_indices = linear_assignment(-iou_matrix)\n else:\n matched_indices = np.empty(shape=(0,2))\n\n unmatched_detections = []\n for d, det in enumerate(detections):\n if(d not in matched_indices[:,0]):\n unmatched_detections.append(d)\n unmatched_trackers = []\n for t, trk in enumerate(trackers):\n if(t not in matched_indices[:,1]):\n unmatched_trackers.append(t)\n\n #filter out matched with low IOU\n matches = []\n for m in matched_indices:\n if(iou_matrix[m[0], m[1]] 0:\n a = (iou_matrix > iou_threshold).astype(np.int32)\n if a.sum(1).max() == 1 and a.sum(0).max() == 1:\n matched_indices = np.stack(np.where(a), axis=1)\n else:\n matched_indices = linear_assignment(-(iou_matrix+angle_diff_cost))\n else:\n matched_indices = np.empty(shape=(0,2))\n\n unmatched_detections = []\n for d, det in enumerate(detections):\n if(d not in matched_indices[:,0]):\n unmatched_detections.append(d)\n unmatched_trackers = []\n for t, trk in enumerate(trackers):\n if(t not in matched_indices[:,1]):\n unmatched_trackers.append(t)\n\n # filter out matched with low IOU\n matches = []\n for m in matched_indices:\n if(iou_matrix[m[0], m[1]] 0:\n a = (iou_matrix > iou_threshold).astype(np.int32)\n if a.sum(1).max() == 1 and a.sum(0).max() == 1:\n matched_indices = np.stack(np.where(a), axis=1)\n else:\n matched_indices = linear_assignment(cost_matrix)\n else:\n matched_indices = np.empty(shape=(0,2))\n\n unmatched_detections = []\n for d, det in enumerate(detections):\n if(d not in matched_indices[:,0]):\n unmatched_detections.append(d)\n unmatched_trackers = []\n for t, trk in enumerate(trackers):\n if(t not in matched_indices[:,1]):\n unmatched_trackers.append(t)\n\n #filter out matched with low IOU\n matches = []\n for m in matched_indices:\n if(iou_matrix[m[0], m[1]]update cycle. The\npredict step, implemented with the method or function predict(),\nuses the state transition matrix F to predict the state in the next\ntime period (epoch). The state is stored as a gaussian (x, P), where\nx is the state (column) vector, and P is its covariance. Covariance\nmatrix Q specifies the process covariance. In Bayesian terms, this\nprediction is called the *prior*, which you can think of colloquially\nas the estimate prior to incorporating the measurement.\nThe update step, implemented with the method or function `update()`,\nincorporates the measurement z with covariance R, into the state\nestimate (x, P). The class stores the system uncertainty in S,\nthe innovation (residual between prediction and measurement in\nmeasurement space) in y, and the Kalman gain in k. The procedural\nform returns these variables to you. In Bayesian terms this computes\nthe *posterior* - the estimate after the information from the\nmeasurement is incorporated.\nWhether you use the OO form or procedural form is up to you. If\nmatrices such as H, R, and F are changing each epoch, you'll probably\nopt to use the procedural form. If they are unchanging, the OO\nform is perhaps easier to use since you won't need to keep track\nof these matrices. This is especially useful if you are implementing\nbanks of filters or comparing various KF designs for performance;\na trivial coding bug could lead to using the wrong sets of matrices.\nThis module also offers an implementation of the RTS smoother, and\nother helper functions, such as log likelihood computations.\nThe Saver class allows you to easily save the state of the\nKalmanFilter class after every update\nThis module expects NumPy arrays for all values that expect\narrays, although in a few cases, particularly method parameters,\nit will accept types that convert to NumPy arrays, such as lists\nof lists. These exceptions are documented in the method or function.\nExamples\n--------\nThe following example constructs a constant velocity kinematic\nfilter, filters noisy data, and plots the results. It also demonstrates\nusing the Saver class to save the state of the filter at each epoch.\n.. code-block:: Python\n import matplotlib.pyplot as plt\n import numpy as np\n from filterpy.kalman import KalmanFilter\n from filterpy.common import Q_discrete_white_noise, Saver\n r_std, q_std = 2., 0.003\n cv = KalmanFilter(dim_x=2, dim_z=1)\n cv.x = np.array([[0., 1.]]) # position, velocity\n cv.F = np.array([[1, dt],[ [0, 1]])\n cv.R = np.array([[r_std^^2]])\n f.H = np.array([[1., 0.]])\n f.P = np.diag([.1^^2, .03^^2)\n f.Q = Q_discrete_white_noise(2, dt, q_std**2)\n saver = Saver(cv)\n for z in range(100):\n cv.predict()\n cv.update([z + randn() * r_std])\n saver.save() # save the filter's state\n saver.to_array()\n plt.plot(saver.x[:, 0])\n # plot all of the priors\n plt.plot(saver.x_prior[:, 0])\n # plot mahalanobis distance\n plt.figure()\n plt.plot(saver.mahalanobis)\nThis code implements the same filter using the procedural form\n x = np.array([[0., 1.]]) # position, velocity\n F = np.array([[1, dt],[ [0, 1]])\n R = np.array([[r_std^^2]])\n H = np.array([[1., 0.]])\n P = np.diag([.1^^2, .03^^2)\n Q = Q_discrete_white_noise(2, dt, q_std**2)\n for z in range(100):\n x, P = predict(x, P, F=F, Q=Q)\n x, P = update(x, P, z=[z + randn() * r_std], R=R, H=H)\n xs.append(x[0, 0])\n plt.plot(xs)\nFor more examples see the test subdirectory, or refer to the\nbook cited below. In it I both teach Kalman filtering from basic\nprinciples, and teach the use of this library in great detail.\nFilterPy library.\nhttp://github.com/rlabbe/filterpy\nDocumentation at:\nhttps://filterpy.readthedocs.org\nSupporting book at:\nhttps://github.com/rlabbe/Kalman-and-Bayesian-Filters-in-Python\nThis is licensed under an MIT license. See the readme.MD file\nfor more information.\nCopyright 2014-2018 Roger R Labbe Jr.\n\"\"\"\n\nfrom __future__ import absolute_import, division\n\nfrom copy import deepcopy\nfrom math import log, exp, sqrt\nimport sys\nimport numpy as np\nfrom numpy import dot, zeros, eye, isscalar, shape\nimport numpy.linalg as linalg\nfrom filterpy.stats import logpdf\nfrom filterpy.common import pretty_str, reshape_z\n\n\nclass KalmanFilterNew(object):\n \"\"\" Implements a Kalman filter. You are responsible for setting the\n various state variables to reasonable values; the defaults will\n not give you a functional filter.\n For now the best documentation is my free book Kalman and Bayesian\n Filters in Python [2]_. The test files in this directory also give you a\n basic idea of use, albeit without much description.\n In brief, you will first construct this object, specifying the size of\n the state vector with dim_x and the size of the measurement vector that\n you will be using with dim_z. These are mostly used to perform size checks\n when you assign values to the various matrices. For example, if you\n specified dim_z=2 and then try to assign a 3x3 matrix to R (the\n measurement noise matrix you will get an assert exception because R\n should be 2x2. (If for whatever reason you need to alter the size of\n things midstream just use the underscore version of the matrices to\n assign directly: your_filter._R = a_3x3_matrix.)\n After construction the filter will have default matrices created for you,\n but you must specify the values for each. It’s usually easiest to just\n overwrite them rather than assign to each element yourself. This will be\n clearer in the example below. All are of type numpy.array.\n Examples\n --------\n Here is a filter that tracks position and velocity using a sensor that only\n reads position.\n First construct the object with the required dimensionality. Here the state\n (`dim_x`) has 2 coefficients (position and velocity), and the measurement\n (`dim_z`) has one. In FilterPy `x` is the state, `z` is the measurement.\n .. code::\n from filterpy.kalman import KalmanFilter\n f = KalmanFilter (dim_x=2, dim_z=1)\n Assign the initial value for the state (position and velocity). You can do this\n with a two dimensional array like so:\n .. code::\n f.x = np.array([[2.], # position\n [0.]]) # velocity\n or just use a one dimensional array, which I prefer doing.\n .. code::\n f.x = np.array([2., 0.])\n Define the state transition matrix:\n .. code::\n f.F = np.array([[1.,1.],\n [0.,1.]])\n Define the measurement function. Here we need to convert a position-velocity\n vector into just a position vector, so we use:\n .. code::\n f.H = np.array([[1., 0.]])\n Define the state's covariance matrix P. \n .. code::\n f.P = np.array([[1000., 0.],\n [ 0., 1000.] ])\n Now assign the measurement noise. Here the dimension is 1x1, so I can\n use a scalar\n .. code::\n f.R = 5\n I could have done this instead:\n .. code::\n f.R = np.array([[5.]])\n Note that this must be a 2 dimensional array.\n Finally, I will assign the process noise. Here I will take advantage of\n another FilterPy library function:\n .. code::\n from filterpy.common import Q_discrete_white_noise\n f.Q = Q_discrete_white_noise(dim=2, dt=0.1, var=0.13)\n Now just perform the standard predict/update loop:\n .. code::\n while some_condition_is_true:\n z = get_sensor_reading()\n f.predict()\n f.update(z)\n do_something_with_estimate (f.x)\n **Procedural Form**\n This module also contains stand alone functions to perform Kalman filtering.\n Use these if you are not a fan of objects.\n **Example**\n .. code::\n while True:\n z, R = read_sensor()\n x, P = predict(x, P, F, Q)\n x, P = update(x, P, z, R, H)\n See my book Kalman and Bayesian Filters in Python [2]_.\n You will have to set the following attributes after constructing this\n object for the filter to perform properly. Please note that there are\n various checks in place to ensure that you have made everything the\n 'correct' size. However, it is possible to provide incorrectly sized\n arrays such that the linear algebra can not perform an operation.\n It can also fail silently - you can end up with matrices of a size that\n allows the linear algebra to work, but are the wrong shape for the problem\n you are trying to solve.\n Parameters\n ----------\n dim_x : int\n Number of state variables for the Kalman filter. For example, if\n you are tracking the position and velocity of an object in two\n dimensions, dim_x would be 4.\n This is used to set the default size of P, Q, and u\n dim_z : int\n Number of of measurement inputs. For example, if the sensor\n provides you with position in (x,y), dim_z would be 2.\n dim_u : int (optional)\n size of the control input, if it is being used.\n Default value of 0 indicates it is not used.\n compute_log_likelihood : bool (default = True)\n Computes log likelihood by default, but this can be a slow\n computation, so if you never use it you can turn this computation\n off.\n Attributes\n ----------\n x : numpy.array(dim_x, 1)\n Current state estimate. Any call to update() or predict() updates\n this variable.\n P : numpy.array(dim_x, dim_x)\n Current state covariance matrix. Any call to update() or predict()\n updates this variable.\n x_prior : numpy.array(dim_x, 1)\n Prior (predicted) state estimate. The *_prior and *_post attributes\n are for convenience; they store the prior and posterior of the\n current epoch. Read Only.\n P_prior : numpy.array(dim_x, dim_x)\n Prior (predicted) state covariance matrix. Read Only.\n x_post : numpy.array(dim_x, 1)\n Posterior (updated) state estimate. Read Only.\n P_post : numpy.array(dim_x, dim_x)\n Posterior (updated) state covariance matrix. Read Only.\n z : numpy.array\n Last measurement used in update(). Read only.\n R : numpy.array(dim_z, dim_z)\n Measurement noise covariance matrix. Also known as the\n observation covariance.\n Q : numpy.array(dim_x, dim_x)\n Process noise covariance matrix. Also known as the transition\n covariance.\n F : numpy.array()\n State Transition matrix. Also known as `A` in some formulation.\n H : numpy.array(dim_z, dim_x)\n Measurement function. Also known as the observation matrix, or as `C`.\n y : numpy.array\n Residual of the update step. Read only.\n K : numpy.array(dim_x, dim_z)\n Kalman gain of the update step. Read only.\n S : numpy.array\n System uncertainty (P projected to measurement space). Read only.\n SI : numpy.array\n Inverse system uncertainty. Read only.\n log_likelihood : float\n log-likelihood of the last measurement. Read only.\n likelihood : float\n likelihood of last measurement. Read only.\n Computed from the log-likelihood. The log-likelihood can be very\n small, meaning a large negative value such as -28000. Taking the\n exp() of that results in 0.0, which can break typical algorithms\n which multiply by this value, so by default we always return a\n number >= sys.float_info.min.\n mahalanobis : float\n mahalanobis distance of the innovation. Read only.\n inv : function, default numpy.linalg.inv\n If you prefer another inverse function, such as the Moore-Penrose\n pseudo inverse, set it to that instead: kf.inv = np.linalg.pinv\n This is only used to invert self.S. If you know it is diagonal, you\n might choose to set it to filterpy.common.inv_diagonal, which is\n several times faster than numpy.linalg.inv for diagonal matrices.\n alpha : float\n Fading memory setting. 1.0 gives the normal Kalman filter, and\n values slightly larger than 1.0 (such as 1.02) give a fading\n memory effect - previous measurements have less influence on the\n filter's estimates. This formulation of the Fading memory filter\n (there are many) is due to Dan Simon [1]_.\n References\n ----------\n .. [1] Dan Simon. \"Optimal State Estimation.\" John Wiley & Sons.\n p. 208-212. (2006)\n .. [2] Roger Labbe. \"Kalman and Bayesian Filters in Python\"\n https://github.com/rlabbe/Kalman-and-Bayesian-Filters-in-Python\n \"\"\"\n\n def __init__(self, dim_x, dim_z, dim_u=0):\n if dim_x < 1:\n raise ValueError('dim_x must be 1 or greater')\n if dim_z < 1:\n raise ValueError('dim_z must be 1 or greater')\n if dim_u < 0:\n raise ValueError('dim_u must be 0 or greater')\n\n self.dim_x = dim_x\n self.dim_z = dim_z\n self.dim_u = dim_u\n\n self.x = zeros((dim_x, 1)) # state\n self.P = eye(dim_x) # uncertainty covariance\n self.Q = eye(dim_x) # process uncertainty\n self.B = None # control transition matrix\n self.F = eye(dim_x) # state transition matrix\n self.H = zeros((dim_z, dim_x)) # measurement function\n self.R = eye(dim_z) # measurement uncertainty\n self._alpha_sq = 1. # fading memory control\n self.M = np.zeros((dim_x, dim_z)) # process-measurement cross correlation\n self.z = np.array([[None]*self.dim_z]).T\n\n # gain and residual are computed during the innovation step. We\n # save them so that in case you want to inspect them for various\n # purposes\n self.K = np.zeros((dim_x, dim_z)) # kalman gain\n self.y = zeros((dim_z, 1))\n self.S = np.zeros((dim_z, dim_z)) # system uncertainty\n self.SI = np.zeros((dim_z, dim_z)) # inverse system uncertainty\n\n # identity matrix. Do not alter this.\n self._I = np.eye(dim_x)\n\n # these will always be a copy of x,P after predict() is called\n self.x_prior = self.x.copy()\n self.P_prior = self.P.copy()\n\n # these will always be a copy of x,P after update() is called\n self.x_post = self.x.copy() \n self.P_post = self.P.copy()\n\n # Only computed only if requested via property\n self._log_likelihood = log(sys.float_info.min)\n self._likelihood = sys.float_info.min\n self._mahalanobis = None\n\n # keep all observations \n self.history_obs = []\n\n self.inv = np.linalg.inv\n\n self.attr_saved = None\n self.observed = False \n\n\n def predict(self, u=None, B=None, F=None, Q=None):\n \"\"\"\n Predict next state (prior) using the Kalman filter state propagation\n equations.\n Parameters\n ----------\n u : np.array, default 0\n Optional control vector.\n B : np.array(dim_x, dim_u), or None\n Optional control transition matrix; a value of None\n will cause the filter to use `self.B`.\n F : np.array(dim_x, dim_x), or None\n Optional state transition matrix; a value of None\n will cause the filter to use `self.F`.\n Q : np.array(dim_x, dim_x), scalar, or None\n Optional process noise matrix; a value of None will cause the\n filter to use `self.Q`.\n \"\"\"\n\n if B is None:\n B = self.B\n if F is None:\n F = self.F\n if Q is None:\n Q = self.Q\n elif isscalar(Q):\n Q = eye(self.dim_x) * Q\n\n\n # x = Fx + Bu\n if B is not None and u is not None:\n self.x = dot(F, self.x) + dot(B, u)\n else:\n self.x = dot(F, self.x)\n\n # P = FPF' + Q\n self.P = self._alpha_sq * dot(dot(F, self.P), F.T) + Q\n\n # save prior\n self.x_prior = self.x.copy()\n self.P_prior = self.P.copy()\n\n\n\n def freeze(self):\n \"\"\"\n Save the parameters before non-observation forward\n \"\"\"\n self.attr_saved = deepcopy(self.__dict__)\n\n\n def unfreeze(self):\n if self.attr_saved is not None:\n new_history = deepcopy(self.history_obs)\n self.__dict__ = self.attr_saved\n # self.history_obs = new_history \n self.history_obs = self.history_obs[:-1]\n occur = [int(d is None) for d in new_history]\n indices = np.where(np.array(occur)==0)[0]\n index1 = indices[-2]\n index2 = indices[-1]\n box1 = new_history[index1]\n x1, y1, s1, r1 = box1 \n w1 = np.sqrt(s1 * r1)\n h1 = np.sqrt(s1 / r1)\n box2 = new_history[index2]\n x2, y2, s2, r2 = box2 \n w2 = np.sqrt(s2 * r2)\n h2 = np.sqrt(s2 / r2)\n time_gap = index2 - index1\n dx = (x2-x1)/time_gap\n dy = (y2-y1)/time_gap \n dw = (w2-w1)/time_gap \n dh = (h2-h1)/time_gap\n for i in range(index2 - index1):\n \"\"\"\n The default virtual trajectory generation is by linear\n motion (constant speed hypothesis), you could modify this \n part to implement your own. \n \"\"\"\n x = x1 + (i+1) * dx \n y = y1 + (i+1) * dy \n w = w1 + (i+1) * dw \n h = h1 + (i+1) * dh\n s = w * h \n r = w / float(h)\n new_box = np.array([x, y, s, r]).reshape((4, 1))\n \"\"\"\n I still use predict-update loop here to refresh the parameters,\n but this can be faster by directly modifying the internal parameters\n as suggested in the paper. I keep this naive but slow way for \n easy read and understanding\n \"\"\"\n self.update(new_box)\n if not i == (index2-index1-1):\n self.predict()\n\n\n def update(self, z, R=None, H=None):\n \"\"\"\n Add a new measurement (z) to the Kalman filter.\n If z is None, nothing is computed. However, x_post and P_post are\n updated with the prior (x_prior, P_prior), and self.z is set to None.\n Parameters\n ----------\n z : (dim_z, 1): array_like\n measurement for this update. z can be a scalar if dim_z is 1,\n otherwise it must be convertible to a column vector.\n If you pass in a value of H, z must be a column vector the\n of the correct size.\n R : np.array, scalar, or None\n Optionally provide R to override the measurement noise for this\n one call, otherwise self.R will be used.\n H : np.array, or None\n Optionally provide H to override the measurement function for this\n one call, otherwise self.H will be used.\n \"\"\"\n\n # set to None to force recompute\n self._log_likelihood = None\n self._likelihood = None\n self._mahalanobis = None\n\n # append the observation\n self.history_obs.append(z)\n \n if z is None:\n if self.observed:\n \"\"\"\n Got no observation so freeze the current parameters for future\n potential online smoothing.\n \"\"\"\n self.freeze()\n self.observed = False \n self.z = np.array([[None]*self.dim_z]).T\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n self.y = zeros((self.dim_z, 1))\n return\n \n # self.observed = True\n if not self.observed:\n \"\"\"\n Get observation, use online smoothing to re-update parameters\n \"\"\"\n self.unfreeze()\n self.observed = True\n\n if R is None:\n R = self.R\n elif isscalar(R):\n R = eye(self.dim_z) * R\n\n if H is None:\n z = reshape_z(z, self.dim_z, self.x.ndim)\n H = self.H\n\n # y = z - Hx\n # error (residual) between measurement and prediction\n self.y = z - dot(H, self.x)\n\n # common subexpression for speed\n PHT = dot(self.P, H.T)\n\n # S = HPH' + R\n # project system uncertainty into measurement space\n self.S = dot(H, PHT) + R\n self.SI = self.inv(self.S)\n # K = PH'inv(S)\n # map system uncertainty into kalman gain\n self.K = dot(PHT, self.SI)\n\n # x = x + Ky\n # predict new x with residual scaled by the kalman gain\n self.x = self.x + dot(self.K, self.y)\n\n # P = (I-KH)P(I-KH)' + KRK'\n # This is more numerically stable\n # and works for non-optimal K vs the equation\n # P = (I-KH)P usually seen in the literature.\n\n I_KH = self._I - dot(self.K, H)\n self.P = dot(dot(I_KH, self.P), I_KH.T) + dot(dot(self.K, R), self.K.T)\n\n # save measurement and posterior state\n self.z = deepcopy(z)\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n\n def predict_steadystate(self, u=0, B=None):\n \"\"\"\n Predict state (prior) using the Kalman filter state propagation\n equations. Only x is updated, P is left unchanged. See\n update_steadstate() for a longer explanation of when to use this\n method.\n Parameters\n ----------\n u : np.array\n Optional control vector. If non-zero, it is multiplied by B\n to create the control input into the system.\n B : np.array(dim_x, dim_u), or None\n Optional control transition matrix; a value of None\n will cause the filter to use `self.B`.\n \"\"\"\n\n if B is None:\n B = self.B\n\n # x = Fx + Bu\n if B is not None:\n self.x = dot(self.F, self.x) + dot(B, u)\n else:\n self.x = dot(self.F, self.x)\n\n # save prior\n self.x_prior = self.x.copy()\n self.P_prior = self.P.copy()\n\n def update_steadystate(self, z):\n \"\"\"\n Add a new measurement (z) to the Kalman filter without recomputing\n the Kalman gain K, the state covariance P, or the system\n uncertainty S.\n You can use this for LTI systems since the Kalman gain and covariance\n converge to a fixed value. Precompute these and assign them explicitly,\n or run the Kalman filter using the normal predict()/update(0 cycle\n until they converge.\n The main advantage of this call is speed. We do significantly less\n computation, notably avoiding a costly matrix inversion.\n Use in conjunction with predict_steadystate(), otherwise P will grow\n without bound.\n Parameters\n ----------\n z : (dim_z, 1): array_like\n measurement for this update. z can be a scalar if dim_z is 1,\n otherwise it must be convertible to a column vector.\n Examples\n --------\n >>> cv = kinematic_kf(dim=3, order=2) # 3D const velocity filter\n >>> # let filter converge on representative data, then save k and P\n >>> for i in range(100):\n >>> cv.predict()\n >>> cv.update([i, i, i])\n >>> saved_k = np.copy(cv.K)\n >>> saved_P = np.copy(cv.P)\n later on:\n >>> cv = kinematic_kf(dim=3, order=2) # 3D const velocity filter\n >>> cv.K = np.copy(saved_K)\n >>> cv.P = np.copy(saved_P)\n >>> for i in range(100):\n >>> cv.predict_steadystate()\n >>> cv.update_steadystate([i, i, i])\n \"\"\"\n\n # set to None to force recompute\n self._log_likelihood = None\n self._likelihood = None\n self._mahalanobis = None\n\n if z is None:\n self.z = np.array([[None]*self.dim_z]).T\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n self.y = zeros((self.dim_z, 1))\n return\n\n z = reshape_z(z, self.dim_z, self.x.ndim)\n\n # y = z - Hx\n # error (residual) between measurement and prediction\n self.y = z - dot(self.H, self.x)\n\n # x = x + Ky\n # predict new x with residual scaled by the kalman gain\n self.x = self.x + dot(self.K, self.y)\n\n self.z = deepcopy(z)\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n\n # set to None to force recompute\n self._log_likelihood = None\n self._likelihood = None\n self._mahalanobis = None\n\n def update_correlated(self, z, R=None, H=None):\n \"\"\" Add a new measurement (z) to the Kalman filter assuming that\n process noise and measurement noise are correlated as defined in\n the `self.M` matrix.\n A partial derivation can be found in [1]\n If z is None, nothing is changed.\n Parameters\n ----------\n z : (dim_z, 1): array_like\n measurement for this update. z can be a scalar if dim_z is 1,\n otherwise it must be convertible to a column vector.\n R : np.array, scalar, or None\n Optionally provide R to override the measurement noise for this\n one call, otherwise self.R will be used.\n H : np.array, or None\n Optionally provide H to override the measurement function for this\n one call, otherwise self.H will be used.\n References\n ----------\n .. [1] Bulut, Y. (2011). Applied Kalman filter theory (Doctoral dissertation, Northeastern University).\n http://people.duke.edu/~hpgavin/SystemID/References/Balut-KalmanFilter-PhD-NEU-2011.pdf\n \"\"\"\n\n # set to None to force recompute\n self._log_likelihood = None\n self._likelihood = None\n self._mahalanobis = None\n\n if z is None:\n self.z = np.array([[None]*self.dim_z]).T\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n self.y = zeros((self.dim_z, 1))\n return\n\n if R is None:\n R = self.R\n elif isscalar(R):\n R = eye(self.dim_z) * R\n\n # rename for readability and a tiny extra bit of speed\n if H is None:\n z = reshape_z(z, self.dim_z, self.x.ndim)\n H = self.H\n\n # handle special case: if z is in form [[z]] but x is not a column\n # vector dimensions will not match\n if self.x.ndim == 1 and shape(z) == (1, 1):\n z = z[0]\n\n if shape(z) == (): # is it scalar, e.g. z=3 or z=np.array(3)\n z = np.asarray([z])\n\n # y = z - Hx\n # error (residual) between measurement and prediction\n self.y = z - dot(H, self.x)\n\n # common subexpression for speed\n PHT = dot(self.P, H.T)\n\n # project system uncertainty into measurement space\n self.S = dot(H, PHT) + dot(H, self.M) + dot(self.M.T, H.T) + R\n self.SI = self.inv(self.S)\n\n # K = PH'inv(S)\n # map system uncertainty into kalman gain\n self.K = dot(PHT + self.M, self.SI)\n\n # x = x + Ky\n # predict new x with residual scaled by the kalman gain\n self.x = self.x + dot(self.K, self.y)\n self.P = self.P - dot(self.K, dot(H, self.P) + self.M.T)\n\n self.z = deepcopy(z)\n self.x_post = self.x.copy()\n self.P_post = self.P.copy()\n\n def batch_filter(self, zs, Fs=None, Qs=None, Hs=None,\n Rs=None, Bs=None, us=None, update_first=False,\n saver=None):\n \"\"\" Batch processes a sequences of measurements.\n Parameters\n ----------\n zs : list-like\n list of measurements at each time step `self.dt`. Missing\n measurements must be represented by `None`.\n Fs : None, list-like, default=None\n optional value or list of values to use for the state transition\n matrix F.\n If Fs is None then self.F is used for all epochs.\n Otherwise it must contain a list-like list of F's, one for\n each epoch. This allows you to have varying F per epoch.\n Qs : None, np.array or list-like, default=None\n optional value or list of values to use for the process error\n covariance Q.\n If Qs is None then self.Q is used for all epochs.\n Otherwise it must contain a list-like list of Q's, one for\n each epoch. This allows you to have varying Q per epoch.\n Hs : None, np.array or list-like, default=None\n optional list of values to use for the measurement matrix H.\n If Hs is None then self.H is used for all epochs.\n If Hs contains a single matrix, then it is used as H for all\n epochs.\n Otherwise it must contain a list-like list of H's, one for\n each epoch. This allows you to have varying H per epoch.\n Rs : None, np.array or list-like, default=None\n optional list of values to use for the measurement error\n covariance R.\n If Rs is None then self.R is used for all epochs.\n Otherwise it must contain a list-like list of R's, one for\n each epoch. This allows you to have varying R per epoch.\n Bs : None, np.array or list-like, default=None\n optional list of values to use for the control transition matrix B.\n If Bs is None then self.B is used for all epochs.\n Otherwise it must contain a list-like list of B's, one for\n each epoch. This allows you to have varying B per epoch.\n us : None, np.array or list-like, default=None\n optional list of values to use for the control input vector;\n If us is None then None is used for all epochs (equivalent to 0,\n or no control input).\n Otherwise it must contain a list-like list of u's, one for\n each epoch.\n update_first : bool, optional, default=False\n controls whether the order of operations is update followed by\n predict, or predict followed by update. Default is predict->update.\n saver : filterpy.common.Saver, optional\n filterpy.common.Saver object. If provided, saver.save() will be\n called after every epoch\n Returns\n -------\n means : np.array((n,dim_x,1))\n array of the state for each time step after the update. Each entry\n is an np.array. In other words `means[k,:]` is the state at step\n `k`.\n covariance : np.array((n,dim_x,dim_x))\n array of the covariances for each time step after the update.\n In other words `covariance[k,:,:]` is the covariance at step `k`.\n means_predictions : np.array((n,dim_x,1))\n array of the state for each time step after the predictions. Each\n entry is an np.array. In other words `means[k,:]` is the state at\n step `k`.\n covariance_predictions : np.array((n,dim_x,dim_x))\n array of the covariances for each time step after the prediction.\n In other words `covariance[k,:,:]` is the covariance at step `k`.\n Examples\n --------\n .. code-block:: Python\n # this example demonstrates tracking a measurement where the time\n # between measurement varies, as stored in dts. This requires\n # that F be recomputed for each epoch. The output is then smoothed\n # with an RTS smoother.\n zs = [t + random.randn()*4 for t in range (40)]\n Fs = [np.array([[1., dt], [0, 1]] for dt in dts]\n (mu, cov, _, _) = kf.batch_filter(zs, Fs=Fs)\n (xs, Ps, Ks, Pps) = kf.rts_smoother(mu, cov, Fs=Fs)\n \"\"\"\n\n #pylint: disable=too-many-statements\n n = np.size(zs, 0)\n if Fs is None:\n Fs = [self.F] * n\n if Qs is None:\n Qs = [self.Q] * n\n if Hs is None:\n Hs = [self.H] * n\n if Rs is None:\n Rs = [self.R] * n\n if Bs is None:\n Bs = [self.B] * n\n if us is None:\n us = [0] * n\n\n # mean estimates from Kalman Filter\n if self.x.ndim == 1:\n means = zeros((n, self.dim_x))\n means_p = zeros((n, self.dim_x))\n else:\n means = zeros((n, self.dim_x, 1))\n means_p = zeros((n, self.dim_x, 1))\n\n # state covariances from Kalman Filter\n covariances = zeros((n, self.dim_x, self.dim_x))\n covariances_p = zeros((n, self.dim_x, self.dim_x))\n\n if update_first:\n for i, (z, F, Q, H, R, B, u) in enumerate(zip(zs, Fs, Qs, Hs, Rs, Bs, us)):\n\n self.update(z, R=R, H=H)\n means[i, :] = self.x\n covariances[i, :, :] = self.P\n\n self.predict(u=u, B=B, F=F, Q=Q)\n means_p[i, :] = self.x\n covariances_p[i, :, :] = self.P\n\n if saver is not None:\n saver.save()\n else:\n for i, (z, F, Q, H, R, B, u) in enumerate(zip(zs, Fs, Qs, Hs, Rs, Bs, us)):\n\n self.predict(u=u, B=B, F=F, Q=Q)\n means_p[i, :] = self.x\n covariances_p[i, :, :] = self.P\n\n self.update(z, R=R, H=H)\n means[i, :] = self.x\n covariances[i, :, :] = self.P\n\n if saver is not None:\n saver.save()\n\n return (means, covariances, means_p, covariances_p)\n\n def rts_smoother(self, Xs, Ps, Fs=None, Qs=None, inv=np.linalg.inv):\n \"\"\"\n Runs the Rauch-Tung-Striebel Kalman smoother on a set of\n means and covariances computed by a Kalman filter. The usual input\n would come from the output of `KalmanFilter.batch_filter()`.\n Parameters\n ----------\n Xs : numpy.array\n array of the means (state variable x) of the output of a Kalman\n filter.\n Ps : numpy.array\n array of the covariances of the output of a kalman filter.\n Fs : list-like collection of numpy.array, optional\n State transition matrix of the Kalman filter at each time step.\n Optional, if not provided the filter's self.F will be used\n Qs : list-like collection of numpy.array, optional\n Process noise of the Kalman filter at each time step. Optional,\n if not provided the filter's self.Q will be used\n inv : function, default numpy.linalg.inv\n If you prefer another inverse function, such as the Moore-Penrose\n pseudo inverse, set it to that instead: kf.inv = np.linalg.pinv\n Returns\n -------\n x : numpy.ndarray\n smoothed means\n P : numpy.ndarray\n smoothed state covariances\n K : numpy.ndarray\n smoother gain at each step\n Pp : numpy.ndarray\n Predicted state covariances\n Examples\n --------\n .. code-block:: Python\n zs = [t + random.randn()*4 for t in range (40)]\n (mu, cov, _, _) = kalman.batch_filter(zs)\n (x, P, K, Pp) = rts_smoother(mu, cov, kf.F, kf.Q)\n \"\"\"\n\n if len(Xs) != len(Ps):\n raise ValueError('length of Xs and Ps must be the same')\n\n n = Xs.shape[0]\n dim_x = Xs.shape[1]\n\n if Fs is None:\n Fs = [self.F] * n\n if Qs is None:\n Qs = [self.Q] * n\n\n # smoother gain\n K = zeros((n, dim_x, dim_x))\n\n x, P, Pp = Xs.copy(), Ps.copy(), Ps.copy()\n for k in range(n-2, -1, -1):\n Pp[k] = dot(dot(Fs[k+1], P[k]), Fs[k+1].T) + Qs[k+1]\n\n #pylint: disable=bad-whitespace\n K[k] = dot(dot(P[k], Fs[k+1].T), inv(Pp[k]))\n x[k] += dot(K[k], x[k+1] - dot(Fs[k+1], x[k]))\n P[k] += dot(dot(K[k], P[k+1] - Pp[k]), K[k].T)\n\n return (x, P, K, Pp)\n\n def get_prediction(self, u=None, B=None, F=None, Q=None):\n \"\"\"\n Predict next state (prior) using the Kalman filter state propagation\n equations and returns it without modifying the object.\n Parameters\n ----------\n u : np.array, default 0\n Optional control vector.\n B : np.array(dim_x, dim_u), or None\n Optional control transition matrix; a value of None\n will cause the filter to use `self.B`.\n F : np.array(dim_x, dim_x), or None\n Optional state transition matrix; a value of None\n will cause the filter to use `self.F`.\n Q : np.array(dim_x, dim_x), scalar, or None\n Optional process noise matrix; a value of None will cause the\n filter to use `self.Q`.\n Returns\n -------\n (x, P) : tuple\n State vector and covariance array of the prediction.\n \"\"\"\n\n if B is None:\n B = self.B\n if F is None:\n F = self.F\n if Q is None:\n Q = self.Q\n elif isscalar(Q):\n Q = eye(self.dim_x) * Q\n\n # x = Fx + Bu\n if B is not None and u is not None:\n x = dot(F, self.x) + dot(B, u)\n else:\n x = dot(F, self.x)\n\n # P = FPF' + Q\n P = self._alpha_sq * dot(dot(F, self.P), F.T) + Q\n\n return x, P\n\n def get_update(self, z=None):\n \"\"\"\n Computes the new estimate based on measurement `z` and returns it\n without altering the state of the filter.\n Parameters\n ----------\n z : (dim_z, 1): array_like\n measurement for this update. z can be a scalar if dim_z is 1,\n otherwise it must be convertible to a column vector.\n Returns\n -------\n (x, P) : tuple\n State vector and covariance array of the update.\n \"\"\"\n\n if z is None:\n return self.x, self.P\n z = reshape_z(z, self.dim_z, self.x.ndim)\n\n R = self.R\n H = self.H\n P = self.P\n x = self.x\n\n # error (residual) between measurement and prediction\n y = z - dot(H, x)\n\n # common subexpression for speed\n PHT = dot(P, H.T)\n\n # project system uncertainty into measurement space\n S = dot(H, PHT) + R\n\n # map system uncertainty into kalman gain\n K = dot(PHT, self.inv(S))\n\n # predict new x with residual scaled by the kalman gain\n x = x + dot(K, y)\n\n # P = (I-KH)P(I-KH)' + KRK'\n I_KH = self._I - dot(K, H)\n P = dot(dot(I_KH, P), I_KH.T) + dot(dot(K, R), K.T)\n\n return x, P\n\n def residual_of(self, z):\n \"\"\"\n Returns the residual for the given measurement (z). Does not alter\n the state of the filter.\n \"\"\"\n z = reshape_z(z, self.dim_z, self.x.ndim)\n return z - dot(self.H, self.x_prior)\n\n def measurement_of_state(self, x):\n \"\"\"\n Helper function that converts a state into a measurement.\n Parameters\n ----------\n x : np.array\n kalman state vector\n Returns\n -------\n z : (dim_z, 1): array_like\n measurement for this update. z can be a scalar if dim_z is 1,\n otherwise it must be convertible to a column vector.\n \"\"\"\n\n return dot(self.H, x)\n\n @property\n def log_likelihood(self):\n \"\"\"\n log-likelihood of the last measurement.\n \"\"\"\n if self._log_likelihood is None:\n self._log_likelihood = logpdf(x=self.y, cov=self.S)\n return self._log_likelihood\n\n @property\n def likelihood(self):\n \"\"\"\n Computed from the log-likelihood. The log-likelihood can be very\n small, meaning a large negative value such as -28000. Taking the\n exp() of that results in 0.0, which can break typical algorithms\n which multiply by this value, so by default we always return a\n number >= sys.float_info.min.\n \"\"\"\n if self._likelihood is None:\n self._likelihood = exp(self.log_likelihood)\n if self._likelihood == 0:\n self._likelihood = sys.float_info.min\n return self._likelihood\n\n @property\n def mahalanobis(self):\n \"\"\"\"\n Mahalanobis distance of measurement. E.g. 3 means measurement\n was 3 standard deviations away from the predicted value.\n Returns\n -------\n mahalanobis : float\n \"\"\"\n if self._mahalanobis is None:\n self._mahalanobis = sqrt(float(dot(dot(self.y.T, self.SI), self.y)))\n return self._mahalanobis\n\n @property\n def alpha(self):\n \"\"\"\n Fading memory setting. 1.0 gives the normal Kalman filter, and\n values slightly larger than 1.0 (such as 1.02) give a fading\n memory effect - previous measurements have less influence on the\n filter's estimates. This formulation of the Fading memory filter\n (there are many) is due to Dan Simon [1]_.\n \"\"\"\n return self._alpha_sq**.5\n\n def log_likelihood_of(self, z):\n \"\"\"\n log likelihood of the measurement `z`. This should only be called\n after a call to update(). Calling after predict() will yield an\n incorrect result.\"\"\"\n\n if z is None:\n return log(sys.float_info.min)\n return logpdf(z, dot(self.H, self.x), self.S)\n\n @alpha.setter\n def alpha(self, value):\n if not np.isscalar(value) or value < 1:\n raise ValueError('alpha must be a float greater than 1')\n\n self._alpha_sq = value**2\n\n def __repr__(self):\n return '\\n'.join([\n 'KalmanFilter object',\n pretty_str('dim_x', self.dim_x),\n pretty_str('dim_z', self.dim_z),\n pretty_str('dim_u', self.dim_u),\n pretty_str('x', self.x),\n pretty_str('P', self.P),\n pretty_str('x_prior', self.x_prior),\n pretty_str('P_prior', self.P_prior),\n pretty_str('x_post', self.x_post),\n pretty_str('P_post', self.P_post),\n pretty_str('F', self.F),\n pretty_str('Q', self.Q),\n pretty_str('R', self.R),\n pretty_str('H', self.H),\n pretty_str('K', self.K),\n pretty_str('y', self.y),\n pretty_str('S', self.S),\n pretty_str('SI', self.SI),\n pretty_str('M', self.M),\n pretty_str('B', self.B),\n pretty_str('z', self.z),\n pretty_str('log-likelihood', self.log_likelihood),\n pretty_str('likelihood', self.likelihood),\n pretty_str('mahalanobis', self.mahalanobis),\n pretty_str('alpha', self.alpha),\n pretty_str('inv', self.inv)\n ])\n\n def test_matrix_dimensions(self, z=None, H=None, R=None, F=None, Q=None):\n \"\"\"\n Performs a series of asserts to check that the size of everything\n is what it should be. This can help you debug problems in your design.\n If you pass in H, R, F, Q those will be used instead of this object's\n value for those matrices.\n Testing `z` (the measurement) is problamatic. x is a vector, and can be\n implemented as either a 1D array or as a nx1 column vector. Thus Hx\n can be of different shapes. Then, if Hx is a single value, it can\n be either a 1D array or 2D vector. If either is true, z can reasonably\n be a scalar (either '3' or np.array('3') are scalars under this\n definition), a 1D, 1 element array, or a 2D, 1 element array. You are\n allowed to pass in any combination that works.\n \"\"\"\n\n if H is None:\n H = self.H\n if R is None:\n R = self.R\n if F is None:\n F = self.F\n if Q is None:\n Q = self.Q\n x = self.x\n P = self.P\n\n assert x.ndim == 1 or x.ndim == 2, \\\n \"x must have one or two dimensions, but has {}\".format(x.ndim)\n\n if x.ndim == 1:\n assert x.shape[0] == self.dim_x, \\\n \"Shape of x must be ({},{}), but is {}\".format(\n self.dim_x, 1, x.shape)\n else:\n assert x.shape == (self.dim_x, 1), \\\n \"Shape of x must be ({},{}), but is {}\".format(\n self.dim_x, 1, x.shape)\n\n assert P.shape == (self.dim_x, self.dim_x), \\\n \"Shape of P must be ({},{}), but is {}\".format(\n self.dim_x, self.dim_x, P.shape)\n\n assert Q.shape == (self.dim_x, self.dim_x), \\\n \"Shape of Q must be ({},{}), but is {}\".format(\n self.dim_x, self.dim_x, P.shape)\n\n assert F.shape == (self.dim_x, self.dim_x), \\\n \"Shape of F must be ({},{}), but is {}\".format(\n self.dim_x, self.dim_x, F.shape)\n\n assert np.ndim(H) == 2, \\\n \"Shape of H must be (dim_z, {}), but is {}\".format(\n P.shape[0], shape(H))\n\n assert H.shape[1] == P.shape[0], \\\n \"Shape of H must be (dim_z, {}), but is {}\".format(\n P.shape[0], H.shape)\n\n # shape of R must be the same as HPH'\n hph_shape = (H.shape[0], H.shape[0])\n r_shape = shape(R)\n\n if H.shape[0] == 1:\n # r can be scalar, 1D, or 2D in this case\n assert r_shape in [(), (1,), (1, 1)], \\\n \"R must be scalar or one element array, but is shaped {}\".format(\n r_shape)\n else:\n assert r_shape == hph_shape, \\\n \"shape of R should be {} but it is {}\".format(hph_shape, r_shape)\n\n\n if z is not None:\n z_shape = shape(z)\n else:\n z_shape = (self.dim_z, 1)\n\n # H@x must have shape of z\n Hx = dot(H, x)\n\n if z_shape == (): # scalar or np.array(scalar)\n assert Hx.ndim == 1 or shape(Hx) == (1, 1), \\\n \"shape of z should be {}, not {} for the given H\".format(\n shape(Hx), z_shape)\n\n elif shape(Hx) == (1,):\n assert z_shape[0] == 1, 'Shape of z must be {} for the given H'.format(shape(Hx))\n\n else:\n assert (z_shape == shape(Hx) or\n (len(z_shape) == 1 and shape(Hx) == (z_shape[0], 1))), \\\n \"shape of z should be {}, not {} for the given H\".format(\n shape(Hx), z_shape)\n\n if np.ndim(Hx) > 1 and shape(Hx) != (1, 1):\n assert shape(Hx) == z_shape, \\\n 'shape of z should be {} for the given H, but it is {}'.format(\n shape(Hx), z_shape)\n\n\ndef update(x, P, z, R, H=None, return_all=False):\n \"\"\"\n Add a new measurement (z) to the Kalman filter. If z is None, nothing\n is changed.\n This can handle either the multidimensional or unidimensional case. If\n all parameters are floats instead of arrays the filter will still work,\n and return floats for x, P as the result.\n update(1, 2, 1, 1, 1) # univariate\n update(x, P, 1\n Parameters\n ----------\n x : numpy.array(dim_x, 1), or float\n State estimate vector\n P : numpy.array(dim_x, dim_x), or float\n Covariance matrix\n z : (dim_z, 1): array_like\n measurement for this update. z can be a scalar if dim_z is 1,\n otherwise it must be convertible to a column vector.\n R : numpy.array(dim_z, dim_z), or float\n Measurement noise matrix\n H : numpy.array(dim_x, dim_x), or float, optional\n Measurement function. If not provided, a value of 1 is assumed.\n return_all : bool, default False\n If true, y, K, S, and log_likelihood are returned, otherwise\n only x and P are returned.\n Returns\n -------\n x : numpy.array\n Posterior state estimate vector\n P : numpy.array\n Posterior covariance matrix\n y : numpy.array or scalar\n Residua. Difference between measurement and state in measurement space\n K : numpy.array\n Kalman gain\n S : numpy.array\n System uncertainty in measurement space\n log_likelihood : float\n log likelihood of the measurement\n \"\"\"\n\n #pylint: disable=bare-except\n\n if z is None:\n if return_all:\n return x, P, None, None, None, None\n return x, P\n\n if H is None:\n H = np.array([1])\n\n if np.isscalar(H):\n H = np.array([H])\n\n Hx = np.atleast_1d(dot(H, x))\n z = reshape_z(z, Hx.shape[0], x.ndim)\n\n # error (residual) between measurement and prediction\n y = z - Hx\n\n # project system uncertainty into measurement space\n S = dot(dot(H, P), H.T) + R\n\n\n # map system uncertainty into kalman gain\n try:\n K = dot(dot(P, H.T), linalg.inv(S))\n except:\n # can't invert a 1D array, annoyingly\n K = dot(dot(P, H.T), 1./S)\n\n\n # predict new x with residual scaled by the kalman gain\n x = x + dot(K, y)\n\n # P = (I-KH)P(I-KH)' + KRK'\n KH = dot(K, H)\n\n try:\n I_KH = np.eye(KH.shape[0]) - KH\n except:\n I_KH = np.array([1 - KH])\n P = dot(dot(I_KH, P), I_KH.T) + dot(dot(K, R), K.T)\n\n\n if return_all:\n # compute log likelihood\n log_likelihood = logpdf(z, dot(H, x), S)\n return x, P, y, K, S, log_likelihood\n return x, P\n\n\ndef update_steadystate(x, z, K, H=None):\n \"\"\"\n Add a new measurement (z) to the Kalman filter. If z is None, nothing\n is changed.\n Parameters\n ----------\n x : numpy.array(dim_x, 1), or float\n State estimate vector\n z : (dim_z, 1): array_like\n measurement for this update. z can be a scalar if dim_z is 1,\n otherwise it must be convertible to a column vector.\n K : numpy.array, or float\n Kalman gain matrix\n H : numpy.array(dim_x, dim_x), or float, optional\n Measurement function. If not provided, a value of 1 is assumed.\n Returns\n -------\n x : numpy.array\n Posterior state estimate vector\n Examples\n --------\n This can handle either the multidimensional or unidimensional case. If\n all parameters are floats instead of arrays the filter will still work,\n and return floats for x, P as the result.\n >>> update_steadystate(1, 2, 1) # univariate\n >>> update_steadystate(x, P, z, H)\n \"\"\"\n\n\n if z is None:\n return x\n\n if H is None:\n H = np.array([1])\n\n if np.isscalar(H):\n H = np.array([H])\n\n Hx = np.atleast_1d(dot(H, x))\n z = reshape_z(z, Hx.shape[0], x.ndim)\n\n # error (residual) between measurement and prediction\n y = z - Hx\n\n # estimate new x with residual scaled by the kalman gain\n return x + dot(K, y)\n\n\ndef predict(x, P, F=1, Q=0, u=0, B=1, alpha=1.):\n \"\"\"\n Predict next state (prior) using the Kalman filter state propagation\n equations.\n Parameters\n ----------\n x : numpy.array\n State estimate vector\n P : numpy.array\n Covariance matrix\n F : numpy.array()\n State Transition matrix\n Q : numpy.array, Optional\n Process noise matrix\n u : numpy.array, Optional, default 0.\n Control vector. If non-zero, it is multiplied by B\n to create the control input into the system.\n B : numpy.array, optional, default 0.\n Control transition matrix.\n alpha : float, Optional, default=1.0\n Fading memory setting. 1.0 gives the normal Kalman filter, and\n values slightly larger than 1.0 (such as 1.02) give a fading\n memory effect - previous measurements have less influence on the\n filter's estimates. This formulation of the Fading memory filter\n (there are many) is due to Dan Simon\n Returns\n -------\n x : numpy.array\n Prior state estimate vector\n P : numpy.array\n Prior covariance matrix\n \"\"\"\n\n if np.isscalar(F):\n F = np.array(F)\n x = dot(F, x) + dot(B, u)\n P = (alpha * alpha) * dot(dot(F, P), F.T) + Q\n\n return x, P\n\n\ndef predict_steadystate(x, F=1, u=0, B=1):\n \"\"\"\n Predict next state (prior) using the Kalman filter state propagation\n equations. This steady state form only computes x, assuming that the\n covariance is constant.\n Parameters\n ----------\n x : numpy.array\n State estimate vector\n P : numpy.array\n Covariance matrix\n F : numpy.array()\n State Transition matrix\n u : numpy.array, Optional, default 0.\n Control vector. If non-zero, it is multiplied by B\n to create the control input into the system.\n B : numpy.array, optional, default 0.\n Control transition matrix.\n Returns\n -------\n x : numpy.array\n Prior state estimate vector\n \"\"\"\n\n if np.isscalar(F):\n F = np.array(F)\n x = dot(F, x) + dot(B, u)\n\n return x\n\n\n\ndef batch_filter(x, P, zs, Fs, Qs, Hs, Rs, Bs=None, us=None,\n update_first=False, saver=None):\n \"\"\"\n Batch processes a sequences of measurements.\n Parameters\n ----------\n zs : list-like\n list of measurements at each time step. Missing measurements must be\n represented by None.\n Fs : list-like\n list of values to use for the state transition matrix matrix.\n Qs : list-like\n list of values to use for the process error\n covariance.\n Hs : list-like\n list of values to use for the measurement matrix.\n Rs : list-like\n list of values to use for the measurement error\n covariance.\n Bs : list-like, optional\n list of values to use for the control transition matrix;\n a value of None in any position will cause the filter\n to use `self.B` for that time step.\n us : list-like, optional\n list of values to use for the control input vector;\n a value of None in any position will cause the filter to use\n 0 for that time step.\n update_first : bool, optional\n controls whether the order of operations is update followed by\n predict, or predict followed by update. Default is predict->update.\n saver : filterpy.common.Saver, optional\n filterpy.common.Saver object. If provided, saver.save() will be\n called after every epoch\n Returns\n -------\n means : np.array((n,dim_x,1))\n array of the state for each time step after the update. Each entry\n is an np.array. In other words `means[k,:]` is the state at step\n `k`.\n covariance : np.array((n,dim_x,dim_x))\n array of the covariances for each time step after the update.\n In other words `covariance[k,:,:]` is the covariance at step `k`.\n means_predictions : np.array((n,dim_x,1))\n array of the state for each time step after the predictions. Each\n entry is an np.array. In other words `means[k,:]` is the state at\n step `k`.\n covariance_predictions : np.array((n,dim_x,dim_x))\n array of the covariances for each time step after the prediction.\n In other words `covariance[k,:,:]` is the covariance at step `k`.\n Examples\n --------\n .. code-block:: Python\n zs = [t + random.randn()*4 for t in range (40)]\n Fs = [kf.F for t in range (40)]\n Hs = [kf.H for t in range (40)]\n (mu, cov, _, _) = kf.batch_filter(zs, Rs=R_list, Fs=Fs, Hs=Hs, Qs=None,\n Bs=None, us=None, update_first=False)\n (xs, Ps, Ks, Pps) = kf.rts_smoother(mu, cov, Fs=Fs, Qs=None)\n \"\"\"\n\n n = np.size(zs, 0)\n dim_x = x.shape[0]\n\n # mean estimates from Kalman Filter\n if x.ndim == 1:\n means = zeros((n, dim_x))\n means_p = zeros((n, dim_x))\n else:\n means = zeros((n, dim_x, 1))\n means_p = zeros((n, dim_x, 1))\n\n # state covariances from Kalman Filter\n covariances = zeros((n, dim_x, dim_x))\n covariances_p = zeros((n, dim_x, dim_x))\n\n if us is None:\n us = [0.] * n\n Bs = [0.] * n\n\n if update_first:\n for i, (z, F, Q, H, R, B, u) in enumerate(zip(zs, Fs, Qs, Hs, Rs, Bs, us)):\n\n x, P = update(x, P, z, R=R, H=H)\n means[i, :] = x\n covariances[i, :, :] = P\n\n x, P = predict(x, P, u=u, B=B, F=F, Q=Q)\n means_p[i, :] = x\n covariances_p[i, :, :] = P\n if saver is not None:\n saver.save()\n else:\n for i, (z, F, Q, H, R, B, u) in enumerate(zip(zs, Fs, Qs, Hs, Rs, Bs, us)):\n\n x, P = predict(x, P, u=u, B=B, F=F, Q=Q)\n means_p[i, :] = x\n covariances_p[i, :, :] = P\n\n x, P = update(x, P, z, R=R, H=H)\n means[i, :] = x\n covariances[i, :, :] = P\n if saver is not None:\n saver.save()\n\n return (means, covariances, means_p, covariances_p)\n\n\n\ndef rts_smoother(Xs, Ps, Fs, Qs):\n \"\"\"\n Runs the Rauch-Tung-Striebel Kalman smoother on a set of\n means and covariances computed by a Kalman filter. The usual input\n would come from the output of `KalmanFilter.batch_filter()`.\n Parameters\n ----------\n Xs : numpy.array\n array of the means (state variable x) of the output of a Kalman\n filter.\n Ps : numpy.array\n array of the covariances of the output of a kalman filter.\n Fs : list-like collection of numpy.array\n State transition matrix of the Kalman filter at each time step.\n Qs : list-like collection of numpy.array, optional\n Process noise of the Kalman filter at each time step.\n Returns\n -------\n x : numpy.ndarray\n smoothed means\n P : numpy.ndarray\n smoothed state covariances\n K : numpy.ndarray\n smoother gain at each step\n pP : numpy.ndarray\n predicted state covariances\n Examples\n --------\n .. code-block:: Python\n zs = [t + random.randn()*4 for t in range (40)]\n (mu, cov, _, _) = kalman.batch_filter(zs)\n (x, P, K, pP) = rts_smoother(mu, cov, kf.F, kf.Q)\n \"\"\"\n\n if len(Xs) != len(Ps):\n raise ValueError('length of Xs and Ps must be the same')\n\n n = Xs.shape[0]\n dim_x = Xs.shape[1]\n\n # smoother gain\n K = zeros((n, dim_x, dim_x))\n x, P, pP = Xs.copy(), Ps.copy(), Ps.copy()\n\n for k in range(n-2, -1, -1):\n pP[k] = dot(dot(Fs[k], P[k]), Fs[k].T) + Qs[k]\n\n #pylint: disable=bad-whitespace\n K[k] = dot(dot(P[k], Fs[k].T), linalg.inv(pP[k]))\n x[k] += dot(K[k], x[k+1] - dot(Fs[k], x[k]))\n P[k] += dot(dot(K[k], P[k+1] - pP[k]), K[k].T)\n\n return (x, P, K, pP)" }, { "path": "asone/trackers/oc_sort/tracker/ocsort.py", "content": "\"\"\"\n This script is adopted from the SORT script by Alex Bewley alex@bewley.ai\n\"\"\"\nfrom __future__ import print_function\n\nimport numpy as np\n\nfrom .association import *\n\n\ndef k_previous_obs(observations, cur_age, k):\n if len(observations) == 0:\n return [-1, -1, -1, -1, -1]\n for i in range(k):\n dt = k - i\n if cur_age - dt in observations:\n return observations[cur_age - dt]\n max_age = max(observations.keys())\n return observations[max_age]\n\n\ndef convert_bbox_to_z(bbox):\n \"\"\"\n Takes a bounding box in the form [x1,y1,x2,y2] and returns z in the form\n [x,y,s,r] where x,y is the centre of the box and s is the scale/area and r is\n the aspect ratio\n \"\"\"\n w = bbox[2] - bbox[0]\n h = bbox[3] - bbox[1]\n x = bbox[0] + w / 2.0\n y = bbox[1] + h / 2.0\n s = w * h # scale is just area\n r = w / float(h + 1e-6)\n return np.array([x, y, s, r]).reshape((4, 1))\n\n\ndef convert_x_to_bbox(x, score=None):\n \"\"\"\n Takes a bounding box in the centre form [x,y,s,r] and returns it in the form\n [x1,y1,x2,y2] where x1,y1 is the top left and x2,y2 is the bottom right\n \"\"\"\n w = np.sqrt(x[2] * x[3])\n h = x[2] / w\n if score == None:\n return np.array([x[0] - w / 2.0, x[1] - h / 2.0, x[0] + w / 2.0, x[1] + h / 2.0]).reshape((1, 4))\n else:\n return np.array([x[0] - w / 2.0, x[1] - h / 2.0, x[0] + w / 2.0, x[1] + h / 2.0, score]).reshape((1, 5))\n\n\ndef speed_direction(bbox1, bbox2):\n cx1, cy1 = (bbox1[0] + bbox1[2]) / 2.0, (bbox1[1] + bbox1[3]) / 2.0\n cx2, cy2 = (bbox2[0] + bbox2[2]) / 2.0, (bbox2[1] + bbox2[3]) / 2.0\n speed = np.array([cy2 - cy1, cx2 - cx1])\n norm = np.sqrt((cy2 - cy1) ** 2 + (cx2 - cx1) ** 2) + 1e-6\n return speed / norm\n\n\nclass KalmanBoxTracker(object):\n \"\"\"\n This class represents the internal state of individual tracked objects observed as bbox.\n \"\"\"\n\n count = 0\n\n def __init__(self, bbox, conf, cls, delta_t=3, orig=False):\n \"\"\"\n Initialises a tracker using initial bounding box.\n\n \"\"\"\n # define constant velocity model\n if not orig:\n from .kalmanfilter import KalmanFilterNew as KalmanFilter\n\n self.kf = KalmanFilter(dim_x=7, dim_z=4)\n else:\n from filterpy.kalman import KalmanFilter\n\n self.kf = KalmanFilter(dim_x=7, dim_z=4)\n self.kf.F = np.array(\n [\n [1, 0, 0, 0, 1, 0, 0],\n [0, 1, 0, 0, 0, 1, 0],\n [0, 0, 1, 0, 0, 0, 1],\n [0, 0, 0, 1, 0, 0, 0],\n [0, 0, 0, 0, 1, 0, 0],\n [0, 0, 0, 0, 0, 1, 0],\n [0, 0, 0, 0, 0, 0, 1],\n ]\n )\n self.kf.H = np.array(\n [[1, 0, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0]]\n )\n\n self.kf.R[2:, 2:] *= 10.0\n self.kf.P[4:, 4:] *= 1000.0 # give high uncertainty to the unobservable initial velocities\n self.kf.P *= 10.0\n self.kf.Q[-1, -1] *= 0.01\n self.kf.Q[4:, 4:] *= 0.01\n\n self.kf.x[:4] = convert_bbox_to_z(bbox)\n self.time_since_update = 0\n self.id = KalmanBoxTracker.count\n KalmanBoxTracker.count += 1\n self.history = []\n self.hits = 0\n self.hit_streak = 0\n self.age = 0\n self.conf = conf\n self.cls = cls\n \"\"\"\n NOTE: [-1,-1,-1,-1,-1] is a compromising placeholder for non-observation status, the same for the return of \n function k_previous_obs. It is ugly and I do not like it. But to support generate observation array in a \n fast and unified way, which you would see below k_observations = np.array([k_previous_obs(...]]), let's bear it for now.\n \"\"\"\n self.last_observation = np.array([-1, -1, -1, -1, -1]) # placeholder\n self.observations = dict()\n self.history_observations = []\n self.velocity = None\n self.delta_t = delta_t\n\n def update(self, bbox, cls):\n \"\"\"\n Updates the state vector with observed bbox.\n \"\"\"\n\n if bbox is not None:\n self.cls = cls\n if self.last_observation.sum() >= 0: # no previous observation\n previous_box = None\n for i in range(self.delta_t):\n dt = self.delta_t - i\n if self.age - dt in self.observations:\n previous_box = self.observations[self.age - dt]\n break\n if previous_box is None:\n previous_box = self.last_observation\n \"\"\"\n Estimate the track speed direction with observations \\Delta t steps away\n \"\"\"\n self.velocity = speed_direction(previous_box, bbox)\n\n \"\"\"\n Insert new observations. This is a ugly way to maintain both self.observations\n and self.history_observations. Bear it for the moment.\n \"\"\"\n self.last_observation = bbox\n self.observations[self.age] = bbox\n self.history_observations.append(bbox)\n\n self.time_since_update = 0\n self.history = []\n self.hits += 1\n self.hit_streak += 1\n self.kf.update(convert_bbox_to_z(bbox))\n else:\n self.kf.update(bbox)\n\n def predict(self):\n \"\"\"\n Advances the state vector and returns the predicted bounding box estimate.\n \"\"\"\n if (self.kf.x[6] + self.kf.x[2]) <= 0:\n self.kf.x[6] *= 0.0\n\n self.kf.predict()\n self.age += 1\n if self.time_since_update > 0:\n self.hit_streak = 0\n self.time_since_update += 1\n self.history.append(convert_x_to_bbox(self.kf.x))\n return self.history[-1]\n\n def get_state(self):\n \"\"\"\n Returns the current bounding box estimate.\n \"\"\"\n return convert_x_to_bbox(self.kf.x)\n\n\n\"\"\"\n We support multiple ways for association cost calculation, by default\n we use IoU. GIoU may have better performance in some situations. We note \n that we hardly normalize the cost by all methods to (0,1) which may not be \n the best practice.\n\"\"\"\nASSO_FUNCS = {\"iou\": iou_batch, \"giou\": giou_batch, \"ciou\": ciou_batch, \"diou\": diou_batch, \"ct_dist\": ct_dist}\n\n\nclass OCSort(object):\n def __init__(\n self,\n det_thresh=0.2,\n max_age=100,\n min_hits=3,\n iou_threshold=0.3,\n delta_t=3,\n asso_func=\"iou\",\n inertia=0.2,\n use_byte=False,\n ):\n \"\"\"\n Sets key parameters for SORT\n \"\"\"\n self.max_age = max_age\n self.min_hits = min_hits\n self.iou_threshold = iou_threshold\n self.trackers = []\n self.frame_count = 0\n self.det_thresh = det_thresh\n self.delta_t = delta_t\n self.asso_func = ASSO_FUNCS[asso_func]\n self.inertia = inertia\n self.use_byte = use_byte\n KalmanBoxTracker.count = 0\n\n def update(self, dets, _):\n \"\"\"\n Params:\n dets - a numpy array of detections in the format [[x1,y1,x2,y2,score],[x1,y1,x2,y2,score],...]\n Requires: this method must be called once for each frame even with empty detections (use np.empty((0, 5)) for frames without detections).\n Returns the a similar array, where the last column is the object ID.\n NOTE: The number of objects returned may differ from the number of detections provided.\n \"\"\"\n\n self.frame_count += 1\n\n xyxys = dets[:, 0:4]\n confs = dets[:, 4]\n clss = dets[:, 5]\n\n classes = clss\n xyxys = xyxys\n confs = confs\n\n output_results = np.column_stack((xyxys, confs, classes))\n\n inds_low = confs > 0.1\n inds_high = confs < self.det_thresh\n inds_second = np.logical_and(inds_low, inds_high) # self.det_thresh > score > 0.1, for second matching\n dets_second = output_results[inds_second] # detections for second matching\n remain_inds = confs > self.det_thresh\n dets = output_results[remain_inds]\n\n # get predicted locations from existing trackers.\n trks = np.zeros((len(self.trackers), 5))\n to_del = []\n ret = []\n for t, trk in enumerate(trks):\n pos = self.trackers[t].predict()[0]\n trk[:] = [pos[0], pos[1], pos[2], pos[3], 0]\n if np.any(np.isnan(pos)):\n to_del.append(t)\n trks = np.ma.compress_rows(np.ma.masked_invalid(trks))\n for t in reversed(to_del):\n self.trackers.pop(t)\n\n velocities = np.array([trk.velocity if trk.velocity is not None else np.array((0, 0)) for trk in self.trackers])\n last_boxes = np.array([trk.last_observation for trk in self.trackers])\n k_observations = np.array([k_previous_obs(trk.observations, trk.age, self.delta_t) for trk in self.trackers])\n\n \"\"\"\n First round of association\n \"\"\"\n matched, unmatched_dets, unmatched_trks = associate(\n dets, trks, self.iou_threshold, velocities, k_observations, self.inertia\n )\n for m in matched:\n self.trackers[m[1]].update(dets[m[0], :5], dets[m[0], 5])\n\n \"\"\"\n Second round of associaton by OCR\n \"\"\"\n # BYTE association\n if self.use_byte and len(dets_second) > 0 and unmatched_trks.shape[0] > 0:\n u_trks = trks[unmatched_trks]\n iou_left = self.asso_func(dets_second, u_trks) # iou between low score detections and unmatched tracks\n iou_left = np.array(iou_left)\n if iou_left.max() > self.iou_threshold:\n \"\"\"\n NOTE: by using a lower threshold, e.g., self.iou_threshold - 0.1, you may\n get a higher performance especially on MOT17/MOT20 datasets. But we keep it\n uniform here for simplicity\n \"\"\"\n matched_indices = linear_assignment(-iou_left)\n to_remove_trk_indices = []\n for m in matched_indices:\n det_ind, trk_ind = m[0], unmatched_trks[m[1]]\n if iou_left[m[0], m[1]] < self.iou_threshold:\n continue\n self.trackers[trk_ind].update(dets_second[det_ind, :5], dets_second[det_ind, 5])\n to_remove_trk_indices.append(trk_ind)\n unmatched_trks = np.setdiff1d(unmatched_trks, np.array(to_remove_trk_indices))\n\n if unmatched_dets.shape[0] > 0 and unmatched_trks.shape[0] > 0:\n left_dets = dets[unmatched_dets]\n left_trks = last_boxes[unmatched_trks]\n iou_left = self.asso_func(left_dets, left_trks)\n iou_left = np.array(iou_left)\n if iou_left.max() > self.iou_threshold:\n \"\"\"\n NOTE: by using a lower threshold, e.g., self.iou_threshold - 0.1, you may\n get a higher performance especially on MOT17/MOT20 datasets. But we keep it\n uniform here for simplicity\n \"\"\"\n rematched_indices = linear_assignment(-iou_left)\n to_remove_det_indices = []\n to_remove_trk_indices = []\n for m in rematched_indices:\n det_ind, trk_ind = unmatched_dets[m[0]], unmatched_trks[m[1]]\n if iou_left[m[0], m[1]] < self.iou_threshold:\n continue\n self.trackers[trk_ind].update(dets[det_ind, :5], dets[det_ind, 5])\n to_remove_det_indices.append(det_ind)\n to_remove_trk_indices.append(trk_ind)\n unmatched_dets = np.setdiff1d(unmatched_dets, np.array(to_remove_det_indices))\n unmatched_trks = np.setdiff1d(unmatched_trks, np.array(to_remove_trk_indices))\n\n for m in unmatched_trks:\n self.trackers[m].update(None, None)\n\n # create and initialise new trackers for unmatched detections\n for i in unmatched_dets:\n trk = KalmanBoxTracker(dets[i, :4], dets[i, 4], dets[i, 5], delta_t=self.delta_t)\n self.trackers.append(trk)\n i = len(self.trackers)\n for trk in reversed(self.trackers):\n if trk.last_observation.sum() < 0:\n d = trk.get_state()[0]\n else:\n \"\"\"\n this is optional to use the recent observation or the kalman filter prediction,\n we didn't notice significant difference here\n \"\"\"\n d = trk.last_observation[:4]\n if (trk.time_since_update < 1) and (trk.hit_streak >= self.min_hits or self.frame_count <= self.min_hits):\n # +1 as MOT benchmark requires positive\n ret.append(np.concatenate((d, [trk.id + 1], [trk.cls], [trk.conf])).reshape(1, -1))\n i -= 1\n # remove dead tracklet\n if trk.time_since_update > self.max_age:\n self.trackers.pop(i)\n if len(ret) > 0:\n return np.concatenate(ret)\n return np.empty((0, 7))\n " }, { "path": "asone/trackers/strong_sort/__init__.py", "content": "" }, { "path": "asone/trackers/strong_sort/strongsort.py", "content": "import numpy as np\nfrom .tracker.strong_sort import StrongSORT\n\n\nclass StrongSort:\n def __init__(self, detector) -> None:\n \n self.tracker = StrongSORT(model_weights='osnet_x0_25_msmt17.pt', device='cpu')\n self.detector = detector\n try:\n self.input_shape = tuple(detector.model.get_inputs()[0].shape[2:])\n except AttributeError as e:\n self.input_shape = (640, 640)\n \n def detect_and_track(self, image: np.ndarray, config: dict) -> tuple:\n \n _dets_xyxy, img = self.detector.detect(\n image, **config, return_image=True\n )\n \n bbox_xyxy = _dets_xyxy[:, :4]\n conf = _dets_xyxy[:, 4]\n classes = _dets_xyxy[:, 5]\n \n # if isinstance(_dets_xyxy, np.ndarray) and len(_dets_xyxy) > 0:\n dets = self.tracker.update(bbox_xyxy, conf, classes, img)\n dets = np.array(dets)\n \n if dets != []:\n bbox_xyxy = dets[:, :4]\n ids = dets[:, 4]\n class_ids = dets[:, 5]\n scores = dets[:, 6]\n return bbox_xyxy, ids, scores, class_ids\n else:\n return [], [], [], []\n" }, { "path": "asone/trackers/strong_sort/tracker/__init__.py", "content": "from .strong_sort import StrongSORT\n\n\n__all__ = ['StrongSORT', 'build_tracker']\n\n\ndef build_tracker(cfg, use_cuda):\n return StrongSORT(cfg.STRONGSORT.REID_CKPT, \n max_dist=cfg.STRONGSORT.MAX_DIST, min_confidence=cfg.STRONGSORT.MIN_CONFIDENCE, \n nms_max_overlap=cfg.STRONGSORT.NMS_MAX_OVERLAP, max_iou_distance=cfg.STRONGSORT.MAX_IOU_DISTANCE, \n max_age=cfg.STRONGSORT.MAX_AGE, n_init=cfg.STRONGSORT.N_INIT, nn_budget=cfg.STRONGSORT.NN_BUDGET, use_cuda=use_cuda)\n" }, { "path": "asone/trackers/strong_sort/tracker/configs/strong_sort.yaml", "content": "STRONGSORT:\n ECC: True # activate camera motion compensation\n MC_LAMBDA: 0.995 # matching with both appearance (1 - MC_LAMBDA) and motion cost\n EMA_ALPHA: 0.9 # updates appearance state in an exponential moving average manner\n MAX_DIST: 0.2 # The matching threshold. Samples with larger distance are considered an invalid match\n MAX_IOU_DISTANCE: 0.7 # Gating threshold. Associations with cost larger than this value are disregarded.\n MAX_AGE: 30 # Maximum number of missed misses before a track is deleted\n N_INIT: 3 # Number of frames that a track remains in initialization phase\n NN_BUDGET: 100 # Maximum size of the appearance descriptors gallery\n \n" }, { "path": "asone/trackers/strong_sort/tracker/deep/__init__.py", "content": "" }, { "path": "asone/trackers/strong_sort/tracker/deep/checkpoint/.gitkeep", "content": "" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/.flake8", "content": "[flake8]\nignore =\n # At least two spaces before inline comment\n E261,\n # Line lengths are recommended to be no greater than 79 characters\n E501,\n # Missing whitespace around arithmetic operator \n E226,\n # Blank line contains whitespace\n W293,\n # Do not use bare 'except'\n E722,\n # Line break after binary operator\n W504,\n # isort found an import in the wrong position\n I001\nmax-line-length = 79\nexclude = __init__.py, build, torchreid/metrics/rank_cylib/" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/.gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# Cython eval code\n*.c\n*.html\n\n# OS X\n.DS_Store\n.Spotlight-V100\n.Trashes\n._*\n\n# ReID\nreid-data/\nlog/\nsaved-models/\nmodel-zoo/\ndebug*\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/.isort.cfg", "content": "[isort]\nline_length=79\nmulti_line_output=3\nlength_sort=true\nknown_standard_library=numpy,setuptools\nknown_myself=torchreid\nknown_third_party=matplotlib,cv2,torch,torchvision,PIL,yacs\nno_lines_before=STDLIB,THIRDPARTY\nsections=FUTURE,STDLIB,THIRDPARTY,myself,FIRSTPARTY,LOCALFOLDER\ndefault_section=FIRSTPARTY" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/.style.yapf", "content": "[style]\nBASED_ON_STYLE = pep8\nBLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = true\nSPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN = true\nDEDENT_CLOSING_BRACKETS = true\nSPACES_BEFORE_COMMENT = 1\nARITHMETIC_PRECEDENCE_INDICATION = true" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/LICENSE", "content": "MIT License\n\nCopyright (c) 2018 Kaiyang Zhou\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/README.rst", "content": "Torchreid\n===========\nTorchreid is a library for deep-learning person re-identification, written in `PyTorch `_ and developed for our ICCV'19 project, `Omni-Scale Feature Learning for Person Re-Identification `_.\n\nIt features:\n\n- multi-GPU training\n- support both image- and video-reid\n- end-to-end training and evaluation\n- incredibly easy preparation of reid datasets\n- multi-dataset training\n- cross-dataset evaluation\n- standard protocol used by most research papers\n- highly extensible (easy to add models, datasets, training methods, etc.)\n- implementations of state-of-the-art deep reid models\n- access to pretrained reid models\n- advanced training techniques\n- visualization tools (tensorboard, ranks, etc.)\n\n\nCode: https://github.com/KaiyangZhou/deep-person-reid.\n\nDocumentation: https://kaiyangzhou.github.io/deep-person-reid/.\n\nHow-to instructions: https://kaiyangzhou.github.io/deep-person-reid/user_guide.\n\nModel zoo: https://kaiyangzhou.github.io/deep-person-reid/MODEL_ZOO.\n\nTech report: https://arxiv.org/abs/1910.10093.\n\nYou can find some research projects that are built on top of Torchreid `here `_.\n\n\nWhat's new\n------------\n- [Aug 2021] We have released the ImageNet-pretrained models of ``osnet_ain_x0_75``, ``osnet_ain_x0_5`` and ``osnet_ain_x0_25``. The pretraining setup follows `pycls `_.\n- [Apr 2021] We have updated the appendix in the `TPAMI version of OSNet `_ to include results in the multi-source domain generalization setting. The trained models can be found in the `Model Zoo `_.\n- [Apr 2021] We have added a script to automate the process of calculating average results over multiple splits. For more details please see ``tools/parse_test_res.py``.\n- [Apr 2021] ``v1.4.0``: We added the person search dataset, `CUHK-SYSU `_. Please see the `documentation `_ regarding how to download the dataset (it contains cropped person images).\n- [Apr 2021] All models in the model zoo have been moved to google drive. Please raise an issue if any model's performance is inconsistent with the numbers shown in the model zoo page (could be caused by wrong links).\n- [Mar 2021] `OSNet `_ will appear in the TPAMI journal! Compared with the conference version, which focuses on discriminative feature learning using the omni-scale building block, this journal extension further considers generalizable feature learning by integrating `instance normalization layers `_ with the OSNet architecture. We hope this journal paper can motivate more future work to taclke the generalization issue in cross-dataset re-ID.\n- [Mar 2021] Generalization across domains (datasets) in person re-ID is crucial in real-world applications, which is closely related to the topic of *domain generalization*. Interested in learning how the field of domain generalization has developed over the last decade? Check our recent survey in this topic at https://arxiv.org/abs/2103.02503, with coverage on the history, datasets, related problems, methodologies, potential directions, and so on (*methods designed for generalizable re-ID are also covered*!).\n- [Feb 2021] ``v1.3.6`` Added `University-1652 `_, a new dataset for multi-view multi-source geo-localization (credit to `Zhedong Zheng `_).\n- [Feb 2021] ``v1.3.5``: Now the `cython code `_ works on Windows (credit to `lablabla `_).\n- [Jan 2021] Our recent work, `MixStyle `_ (mixing instance-level feature statistics of samples of different domains for improving domain generalization), has been accepted to ICLR'21. The code has been released at https://github.com/KaiyangZhou/mixstyle-release where the person re-ID part is based on Torchreid.\n- [Jan 2021] A new evaluation metric called `mean Inverse Negative Penalty (mINP)` for person re-ID has been introduced in `Deep Learning for Person Re-identification: A Survey and Outlook (TPAMI 2021) `_. Their code can be accessed at ``_.\n- [Aug 2020] ``v1.3.3``: Fixed bug in ``visrank`` (caused by not unpacking ``dsetid``).\n- [Aug 2020] ``v1.3.2``: Added ``_junk_pids`` to ``grid`` and ``prid``. This avoids using mislabeled gallery images for training when setting ``combineall=True``.\n- [Aug 2020] ``v1.3.0``: (1) Added ``dsetid`` to the existing 3-tuple data source, resulting in ``(impath, pid, camid, dsetid)``. This variable denotes the dataset ID and is useful when combining multiple datasets for training (as a dataset indicator). E.g., when combining ``market1501`` and ``cuhk03``, the former will be assigned ``dsetid=0`` while the latter will be assigned ``dsetid=1``. (2) Added ``RandomDatasetSampler``. Analogous to ``RandomDomainSampler``, ``RandomDatasetSampler`` samples a certain number of images (``batch_size // num_datasets``) from each of specified datasets (the amount is determined by ``num_datasets``).\n- [Aug 2020] ``v1.2.6``: Added ``RandomDomainSampler`` (it samples ``num_cams`` cameras each with ``batch_size // num_cams`` images to form a mini-batch).\n- [Jun 2020] ``v1.2.5``: (1) Dataloader's output from ``__getitem__`` has been changed from ``list`` to ``dict``. Previously, an element, e.g. image tensor, was fetched with ``imgs=data[0]``. Now it should be obtained by ``imgs=data['img']``. See this `commit `_ for detailed changes. (2) Added ``k_tfm`` as an option to image data loader, which allows data augmentation to be applied ``k_tfm`` times *independently* to an image. If ``k_tfm > 1``, ``imgs=data['img']`` returns a list with ``k_tfm`` image tensors.\n- [May 2020] Added the person attribute recognition code used in `Omni-Scale Feature Learning for Person Re-Identification (ICCV'19) `_. See ``projects/attribute_recognition/``.\n- [May 2020] ``v1.2.1``: Added a simple API for feature extraction (``torchreid/utils/feature_extractor.py``). See the `documentation `_ for the instruction.\n- [Apr 2020] Code for reproducing the experiments of `deep mutual learning `_ in the `OSNet paper `__ (Supp. B) has been released at ``projects/DML``.\n- [Apr 2020] Upgraded to ``v1.2.0``. The engine class has been made more model-agnostic to improve extensibility. See `Engine `_ and `ImageSoftmaxEngine `_ for more details. Credit to `Dassl.pytorch `_.\n- [Dec 2019] Our `OSNet paper `_ has been updated, with additional experiments (in section B of the supplementary) showing some useful techniques for improving OSNet's performance in practice.\n- [Nov 2019] ``ImageDataManager`` can load training data from target datasets by setting ``load_train_targets=True``, and the train-loader can be accessed with ``train_loader_t = datamanager.train_loader_t``. This feature is useful for domain adaptation research.\n\n\nInstallation\n---------------\n\nMake sure `conda `_ is installed.\n\n\n.. code-block:: bash\n\n # cd to your preferred directory and clone this repo\n git clone https://github.com/KaiyangZhou/deep-person-reid.git\n\n # create environment\n cd deep-person-reid/\n conda create --name torchreid python=3.7\n conda activate torchreid\n\n # install dependencies\n # make sure `which python` and `which pip` point to the correct path\n pip install -r requirements.txt\n\n # install torch and torchvision (select the proper cuda version to suit your machine)\n conda install pytorch torchvision cudatoolkit=9.0 -c pytorch\n\n # install torchreid (don't need to re-build it if you modify the source code)\n python setup.py develop\n\n\nGet started: 30 seconds to Torchreid\n-------------------------------------\n1. Import ``torchreid``\n\n.. code-block:: python\n \n import torchreid\n\n2. Load data manager\n\n.. code-block:: python\n \n datamanager = torchreid.data.ImageDataManager(\n root=\"reid-data\",\n sources=\"market1501\",\n targets=\"market1501\",\n height=256,\n width=128,\n batch_size_train=32,\n batch_size_test=100,\n transforms=[\"random_flip\", \"random_crop\"]\n )\n\n3 Build model, optimizer and lr_scheduler\n\n.. code-block:: python\n \n model = torchreid.models.build_model(\n name=\"resnet50\",\n num_classes=datamanager.num_train_pids,\n loss=\"softmax\",\n pretrained=True\n )\n\n model = model.cuda()\n\n optimizer = torchreid.optim.build_optimizer(\n model,\n optim=\"adam\",\n lr=0.0003\n )\n\n scheduler = torchreid.optim.build_lr_scheduler(\n optimizer,\n lr_scheduler=\"single_step\",\n stepsize=20\n )\n\n4. Build engine\n\n.. code-block:: python\n \n engine = torchreid.engine.ImageSoftmaxEngine(\n datamanager,\n model,\n optimizer=optimizer,\n scheduler=scheduler,\n label_smooth=True\n )\n\n5. Run training and test\n\n.. code-block:: python\n \n engine.run(\n save_dir=\"log/resnet50\",\n max_epoch=60,\n eval_freq=10,\n print_freq=10,\n test_only=False\n )\n\n\nA unified interface\n-----------------------\nIn \"deep-person-reid/scripts/\", we provide a unified interface to train and test a model. See \"scripts/main.py\" and \"scripts/default_config.py\" for more details. The folder \"configs/\" contains some predefined configs which you can use as a starting point.\n\nBelow we provide an example to train and test `OSNet (Zhou et al. ICCV'19) `_. Assume :code:`PATH_TO_DATA` is the directory containing reid datasets. The environmental variable :code:`CUDA_VISIBLE_DEVICES` is omitted, which you need to specify if you have a pool of gpus and want to use a specific set of them.\n\nConventional setting\n^^^^^^^^^^^^^^^^^^^^^\n\nTo train OSNet on Market1501, do\n\n.. code-block:: bash\n\n python scripts/main.py \\\n --config-file configs/im_osnet_x1_0_softmax_256x128_amsgrad_cosine.yaml \\\n --transforms random_flip random_erase \\\n --root $PATH_TO_DATA\n\n\nThe config file sets Market1501 as the default dataset. If you wanna use DukeMTMC-reID, do\n\n.. code-block:: bash\n\n python scripts/main.py \\\n --config-file configs/im_osnet_x1_0_softmax_256x128_amsgrad_cosine.yaml \\\n -s dukemtmcreid \\\n -t dukemtmcreid \\\n --transforms random_flip random_erase \\\n --root $PATH_TO_DATA \\\n data.save_dir log/osnet_x1_0_dukemtmcreid_softmax_cosinelr\n\nThe code will automatically (download and) load the ImageNet pretrained weights. After the training is done, the model will be saved as \"log/osnet_x1_0_market1501_softmax_cosinelr/model.pth.tar-250\". Under the same folder, you can find the `tensorboard `_ file. To visualize the learning curves using tensorboard, you can run :code:`tensorboard --logdir=log/osnet_x1_0_market1501_softmax_cosinelr` in the terminal and visit :code:`http://localhost:6006/` in your web browser.\n\nEvaluation is automatically performed at the end of training. To run the test again using the trained model, do\n\n.. code-block:: bash\n\n python scripts/main.py \\\n --config-file configs/im_osnet_x1_0_softmax_256x128_amsgrad_cosine.yaml \\\n --root $PATH_TO_DATA \\\n model.load_weights log/osnet_x1_0_market1501_softmax_cosinelr/model.pth.tar-250 \\\n test.evaluate True\n\n\nCross-domain setting\n^^^^^^^^^^^^^^^^^^^^^\n\nSuppose you wanna train OSNet on DukeMTMC-reID and test its performance on Market1501, you can do\n\n.. code-block:: bash\n\n python scripts/main.py \\\n --config-file configs/im_osnet_x1_0_softmax_256x128_amsgrad.yaml \\\n -s dukemtmcreid \\\n -t market1501 \\\n --transforms random_flip color_jitter \\\n --root $PATH_TO_DATA\n\nHere we only test the cross-domain performance. However, if you also want to test the performance on the source dataset, i.e. DukeMTMC-reID, you can set :code:`-t dukemtmcreid market1501`, which will evaluate the model on the two datasets separately.\n\nDifferent from the same-domain setting, here we replace :code:`random_erase` with :code:`color_jitter`. This can improve the generalization performance on the unseen target dataset.\n\nPretrained models are available in the `Model Zoo `_.\n\n\nDatasets\n--------\n\nImage-reid datasets\n^^^^^^^^^^^^^^^^^^^^^\n- `Market1501 `_\n- `CUHK03 `_\n- `DukeMTMC-reID `_\n- `MSMT17 `_\n- `VIPeR `_\n- `GRID `_\n- `CUHK01 `_\n- `SenseReID `_\n- `QMUL-iLIDS `_\n- `PRID `_\n\nGeo-localization datasets\n^^^^^^^^^^^^^^^^^^^^^^^^^^^\n- `University-1652 `_\n\nVideo-reid datasets\n^^^^^^^^^^^^^^^^^^^^^^^\n- `MARS `_\n- `iLIDS-VID `_\n- `PRID2011 `_\n- `DukeMTMC-VideoReID `_\n\n\nModels\n-------\n\nImageNet classification models\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n- `ResNet `_\n- `ResNeXt `_\n- `SENet `_\n- `DenseNet `_\n- `Inception-ResNet-V2 `_\n- `Inception-V4 `_\n- `Xception `_\n- `IBN-Net `_\n\nLightweight models\n^^^^^^^^^^^^^^^^^^^\n- `NASNet `_\n- `MobileNetV2 `_\n- `ShuffleNet `_\n- `ShuffleNetV2 `_\n- `SqueezeNet `_\n\nReID-specific models\n^^^^^^^^^^^^^^^^^^^^^^\n- `MuDeep `_\n- `ResNet-mid `_\n- `HACNN `_\n- `PCB `_\n- `MLFN `_\n- `OSNet `_\n- `OSNet-AIN `_\n\n\nUseful links\n-------------\n- `OSNet-IBN1-Lite (test-only code with lite docker container) `_\n- `Deep Learning for Person Re-identification: A Survey and Outlook `_\n\n\nCitation\n---------\nIf you use this code or the models in your research, please give credit to the following papers:\n\n.. code-block:: bash\n\n @article{torchreid,\n title={Torchreid: A Library for Deep Learning Person Re-Identification in Pytorch},\n author={Zhou, Kaiyang and Xiang, Tao},\n journal={arXiv preprint arXiv:1910.10093},\n year={2019}\n }\n \n @inproceedings{zhou2019osnet,\n title={Omni-Scale Feature Learning for Person Re-Identification},\n author={Zhou, Kaiyang and Yang, Yongxin and Cavallaro, Andrea and Xiang, Tao},\n booktitle={ICCV},\n year={2019}\n }\n\n @article{zhou2021osnet,\n title={Learning Generalisable Omni-Scale Representations for Person Re-Identification},\n author={Zhou, Kaiyang and Yang, Yongxin and Cavallaro, Andrea and Xiang, Tao},\n journal={TPAMI},\n year={2021}\n }\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/configs/im_osnet_ain_x1_0_softmax_256x128_amsgrad_cosine.yaml", "content": "model:\n name: 'osnet_ain_x1_0'\n pretrained: True\n\ndata:\n type: 'image'\n sources: ['market1501']\n targets: ['market1501', 'dukemtmcreid']\n height: 256\n width: 128\n combineall: False\n transforms: ['random_flip', 'color_jitter']\n save_dir: 'log/osnet_ain_x1_0_market1501_softmax_cosinelr'\n\nloss:\n name: 'softmax'\n softmax:\n label_smooth: True\n\ntrain:\n optim: 'amsgrad'\n lr: 0.0015\n max_epoch: 100\n batch_size: 64\n fixbase_epoch: 10\n open_layers: ['classifier']\n lr_scheduler: 'cosine'\n\ntest:\n batch_size: 300\n dist_metric: 'cosine'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/configs/im_osnet_ibn_x1_0_softmax_256x128_amsgrad.yaml", "content": "model:\n name: 'osnet_ibn_x1_0'\n pretrained: True\n\ndata:\n type: 'image'\n sources: ['market1501']\n targets: ['dukemtmcreid']\n height: 256\n width: 128\n combineall: False\n transforms: ['random_flip', 'color_jitter']\n save_dir: 'log/osnet_ibn_x1_0_market2duke_softmax'\n\nloss:\n name: 'softmax'\n softmax:\n label_smooth: True\n\ntrain:\n optim: 'amsgrad'\n lr: 0.0015\n max_epoch: 150\n batch_size: 64\n fixbase_epoch: 10\n open_layers: ['classifier']\n lr_scheduler: 'single_step'\n stepsize: [60]\n\ntest:\n batch_size: 300\n dist_metric: 'euclidean'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/configs/im_osnet_x0_25_softmax_256x128_amsgrad.yaml", "content": "model:\n name: 'osnet_x0_25'\n pretrained: True\n\ndata:\n type: 'image'\n sources: ['market1501']\n targets: ['market1501']\n height: 256\n width: 128\n combineall: False\n transforms: ['random_flip']\n save_dir: 'log/osnet_x0_25_market1501_softmax'\n\nloss:\n name: 'softmax'\n softmax:\n label_smooth: True\n\ntrain:\n optim: 'amsgrad'\n lr: 0.003\n max_epoch: 180\n batch_size: 128\n fixbase_epoch: 10\n open_layers: ['classifier']\n lr_scheduler: 'single_step'\n stepsize: [80]\n\ntest:\n batch_size: 300\n dist_metric: 'euclidean'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/configs/im_osnet_x0_5_softmax_256x128_amsgrad.yaml", "content": "model:\n name: 'osnet_x0_5'\n pretrained: True\n\ndata:\n type: 'image'\n sources: ['market1501']\n targets: ['market1501']\n height: 256\n width: 128\n combineall: False\n transforms: ['random_flip']\n save_dir: 'log/osnet_x0_5_market1501_softmax'\n\nloss:\n name: 'softmax'\n softmax:\n label_smooth: True\n\ntrain:\n optim: 'amsgrad'\n lr: 0.003\n max_epoch: 180\n batch_size: 128\n fixbase_epoch: 10\n open_layers: ['classifier']\n lr_scheduler: 'single_step'\n stepsize: [80]\n\ntest:\n batch_size: 300\n dist_metric: 'euclidean'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/configs/im_osnet_x0_75_softmax_256x128_amsgrad.yaml", "content": "model:\n name: 'osnet_x0_75'\n pretrained: True\n\ndata:\n type: 'image'\n sources: ['market1501']\n targets: ['market1501']\n height: 256\n width: 128\n combineall: False\n transforms: ['random_flip']\n save_dir: 'log/osnet_x0_75_market1501_softmax'\n\nloss:\n name: 'softmax'\n softmax:\n label_smooth: True\n\ntrain:\n optim: 'amsgrad'\n lr: 0.0015\n max_epoch: 150\n batch_size: 64\n fixbase_epoch: 10\n open_layers: ['classifier']\n lr_scheduler: 'single_step'\n stepsize: [60]\n\ntest:\n batch_size: 300\n dist_metric: 'euclidean'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/configs/im_osnet_x1_0_softmax_256x128_amsgrad.yaml", "content": "model:\n name: 'osnet_x1_0'\n pretrained: True\n\ndata:\n type: 'image'\n sources: ['market1501']\n targets: ['market1501']\n height: 256\n width: 128\n combineall: False\n transforms: ['random_flip']\n save_dir: 'log/osnet_x1_0_market1501_softmax'\n\nloss:\n name: 'softmax'\n softmax:\n label_smooth: True\n\ntrain:\n optim: 'amsgrad'\n lr: 0.0015\n max_epoch: 150\n batch_size: 64\n fixbase_epoch: 10\n open_layers: ['classifier']\n lr_scheduler: 'single_step'\n stepsize: [60]\n\ntest:\n batch_size: 300\n dist_metric: 'euclidean'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/configs/im_osnet_x1_0_softmax_256x128_amsgrad_cosine.yaml", "content": "model:\n name: 'osnet_x1_0'\n pretrained: True\n\ndata:\n type: 'image'\n sources: ['market1501']\n targets: ['market1501']\n height: 256\n width: 128\n combineall: False\n transforms: ['random_flip']\n save_dir: 'log/osnet_x1_0_market1501_softmax_cosinelr'\n\nloss:\n name: 'softmax'\n softmax:\n label_smooth: True\n\ntrain:\n optim: 'amsgrad'\n lr: 0.0015\n max_epoch: 250\n batch_size: 64\n fixbase_epoch: 10\n open_layers: ['classifier']\n lr_scheduler: 'cosine'\n\ntest:\n batch_size: 300\n dist_metric: 'euclidean'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/configs/im_r50_softmax_256x128_amsgrad.yaml", "content": "model:\n name: 'resnet50_fc512'\n pretrained: True\n\ndata:\n type: 'image'\n sources: ['market1501']\n targets: ['market1501']\n height: 256\n width: 128\n combineall: False\n transforms: ['random_flip']\n save_dir: 'log/resnet50_market1501_softmax'\n\nloss:\n name: 'softmax'\n softmax:\n label_smooth: True\n\ntrain:\n optim: 'amsgrad'\n lr: 0.0003\n max_epoch: 60\n batch_size: 32\n fixbase_epoch: 5\n open_layers: ['classifier']\n lr_scheduler: 'single_step'\n stepsize: [20]\n\ntest:\n batch_size: 100\n dist_metric: 'euclidean'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/configs/im_r50fc512_softmax_256x128_amsgrad.yaml", "content": "model:\n name: 'resnet50_fc512'\n pretrained: True\n\ndata:\n type: 'image'\n sources: ['market1501']\n targets: ['market1501']\n height: 256\n width: 128\n combineall: False\n transforms: ['random_flip']\n save_dir: 'log/resnet50_fc512_market1501_softmax'\n\nloss:\n name: 'softmax'\n softmax:\n label_smooth: True\n\ntrain:\n optim: 'amsgrad'\n lr: 0.0003\n max_epoch: 60\n batch_size: 32\n fixbase_epoch: 5\n open_layers: ['fc', 'classifier']\n lr_scheduler: 'single_step'\n stepsize: [20]\n\ntest:\n batch_size: 100\n dist_metric: 'euclidean'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/AWESOME_REID.md", "content": "# Awesome-ReID\nHere is a collection of ReID-related research with links to papers and code. You are welcome to submit [PR](https://help.github.com/articles/creating-a-pull-request/)s if you find something missing.\n\n\n- [TPAMI21] Learning Generalisable Omni-Scale Representations for Person Re-Identification [[paper](https://arxiv.org/abs/1910.06827)][[code](https://github.com/KaiyangZhou/deep-person-reid)]\n\n- [TPAMI21] Deep Learning for Person Re-identification: A Survey and Outlook [[paper](https://arxiv.org/abs/2001.04193)] [[code](https://github.com/mangye16/ReID-Survey)]\n\n- [ICCV19] RGB-Infrared Cross-Modality Person Re-Identification via Joint Pixel and Feature Alignment. [[paper](http://openaccess.thecvf.com/content_ICCV_2019/papers/Wang_RGB-Infrared_Cross-Modality_Person_Re-Identification_via_Joint_Pixel_and_Feature_Alignment_ICCV_2019_paper.pdf)] [[code](https://github.com/wangguanan/AlignGAN)]\n\n- [ICCV19] Unsupervised Graph Association for Person Re-identification. [[paper](https://github.com/yichuan9527/Unsupervised-Graph-Association-for-Person-Re-identification)] [[code](https://github.com/yichuan9527/Unsupervised-Graph-Association-for-Person-Re-identification)]\n\n- [ICCV19] Self-similarity Grouping: A Simple Unsupervised Cross Domain Adaptation Approach for Person Re-identification. [[paper](http://openaccess.thecvf.com/content_ICCV_2019/papers/Fu_Self-Similarity_Grouping_A_Simple_Unsupervised_Cross_Domain_Adaptation_Approach_for_ICCV_2019_paper.pdf)] [[code](https://github.com/OasisYang/SSG)]\n\n- [ICCV19] Spectral Feature Transformation for Person Re-Identification. [[paper](http://openaccess.thecvf.com/content_ICCV_2019/papers/Luo_Spectral_Feature_Transformation_for_Person_Re-Identification_ICCV_2019_paper.pdf)] [[code](https://github.com/LuckyDC/SFT_REID)]\n\n- [ICCV19] Beyond Human Parts: Dual Part-Aligned Representations for Person Re-Identification. [[paper](http://openaccess.thecvf.com/content_ICCV_2019/papers/Guo_Beyond_Human_Parts_Dual_Part-Aligned_Representations_for_Person_Re-Identification_ICCV_2019_paper.pdf)] [[code](https://github.com/ggjy/P2Net.pytorch)]\n\n- [ICCV19] Co-segmentation Inspired Attention Networks for Video-based Person Re-identification. [[paper](http://openaccess.thecvf.com/content_ICCV_2019/papers/Subramaniam_Co-Segmentation_Inspired_Attention_Networks_for_Video-Based_Person_Re-Identification_ICCV_2019_paper.pdf)][[code](https://github.com/InnovArul/vidreid_cosegmentation)]\n\n- [ICCV19] Mixed High-Order Attention Network for Person Re-Identification. [[paper](https://arxiv.org/abs/1908.05819)][[code](https://github.com/chenbinghui1/MHN)]\n\n- [ICCV19] ABD-Net: Attentive but Diverse Person Re-Identification. [[paper](https://arxiv.org/abs/1908.01114)] [[code](https://github.com/TAMU-VITA/ABD-Net)]\n\n- [ICCV19] Omni-Scale Feature Learning for Person Re-Identification. [[paper](https://arxiv.org/abs/1905.00953)] [[code](https://github.com/KaiyangZhou/deep-person-reid)]\n\n- [CVPR19] Joint Discriminative and Generative Learning for Person Re-identification. [[paper](https://arxiv.org/abs/1904.07223)][[code](https://github.com/NVlabs/DG-Net)]\n- [CVPR19] Invariance Matters: Exemplar Memory for Domain Adaptive Person Re-identification. [[paper](https://arxiv.org/abs/1904.01990)][[code](https://github.com/zhunzhong07/ECN)]\n- [CVPR19] Dissecting Person Re-identification from the Viewpoint of Viewpoint. [[paper](https://arxiv.org/abs/1812.02162)][[code](https://github.com/sxzrt/Dissecting-Person-Re-ID-from-the-Viewpoint-of-Viewpoint)]\n- [CVPR19] Unsupervised Person Re-identification by Soft Multilabel Learning. [[paper](https://arxiv.org/abs/1903.06325)][[code](https://github.com/KovenYu/MAR)]\n- [CVPR19] Patch-based Discriminative Feature Learning for Unsupervised Person Re-identification. [[paper](https://kovenyu.com/publication/2019-cvpr-pedal/)][[code](https://github.com/QizeYang/PAUL)]\n\n- [AAAI19] Spatial and Temporal Mutual Promotion for Video-based Person Re-identification. [[paper](https://arxiv.org/abs/1812.10305)][[code](https://github.com/yolomax/person-reid-lib)]\n\n- [AAAI19] Spatial-Temporal Person Re-identification. [[paper](https://arxiv.org/abs/1812.03282)][[code](https://github.com/Wanggcong/Spatial-Temporal-Re-identification)]\n\n- [AAAI19] Horizontal Pyramid Matching for Person Re-identification. [[paper](https://arxiv.org/abs/1804.05275)][[code](https://github.com/OasisYang/HPM)]\n\n- [AAAI19] Backbone Can Not be Trained at Once: Rolling Back to Pre-trained Network for Person Re-identification. [[paper](https://arxiv.org/abs/1901.06140)][[code](https://github.com/youngminPIL/rollback)]\n\n- [AAAI19] A Bottom-Up Clustering Approach to Unsupervised Person Re-identification. [[paper](https://vana77.github.io/vana77.github.io/images/AAAI19.pdf)][[code](https://github.com/vana77/Bottom-up-Clustering-Person-Re-identification)]\n\n- [NIPS18] FD-GAN: Pose-guided Feature Distilling GAN for Robust Person Re-identification. [[paper](https://arxiv.org/abs/1810.02936)][[code](https://github.com/yxgeee/FD-GAN)]\n\n- [ECCV18] Generalizing A Person Retrieval Model Hetero- and Homogeneously. [[paper](http://openaccess.thecvf.com/content_ECCV_2018/papers/Zhun_Zhong_Generalizing_A_Person_ECCV_2018_paper.pdf)][[code](https://github.com/zhunzhong07/HHL)]\n\n- [ECCV18] Pose-Normalized Image Generation for Person Re-identification. [[paper](https://arxiv.org/abs/1712.02225)][[code](https://github.com/naiq/PN_GAN)]\n\n- [CVPR18] Camera Style Adaptation for Person Re-Identification. [[paper](https://arxiv.org/abs/1711.10295)][[code](https://github.com/zhunzhong07/CamStyle)]\n\n- [CVPR18] Deep Group-Shuffling Random Walk for Person Re-Identification. [[paper](https://arxiv.org/abs/1807.11178)][[code](https://github.com/YantaoShen/kpm_rw_person_reid)]\n\n- [CVPR18] End-to-End Deep Kronecker-Product Matching for Person Re-identification. [[paper](https://arxiv.org/abs/1807.11182)][[code](https://github.com/YantaoShen/kpm_rw_person_reid)]\n\n- [CVPR18] Features for Multi-Target Multi-Camera Tracking and Re-Identification. [[paper](https://arxiv.org/abs/1803.10859)][[code](https://github.com/ergysr/DeepCC)]\n\n- [CVPR18] Group Consistent Similarity Learning via Deep CRF for Person Re-Identification. [[paper](http://openaccess.thecvf.com/content_cvpr_2018/papers/Chen_Group_Consistent_Similarity_CVPR_2018_paper.pdf)][[code](https://github.com/dapengchen123/crf_affinity)]\n\n- [CVPR18] Harmonious Attention Network for Person Re-Identification. [[paper](https://arxiv.org/abs/1802.08122)][[code](https://github.com/KaiyangZhou/deep-person-reid)]\n\n- [CVPR18] Human Semantic Parsing for Person Re-Identification. [[paper](https://arxiv.org/abs/1804.00216)][[code](https://github.com/emrahbasaran/SPReID)]\n\n- [CVPR18] Multi-Level Factorisation Net for Person Re-Identification. [[paper](https://arxiv.org/abs/1803.09132)][[code](https://github.com/KaiyangZhou/deep-person-reid)]\n\n- [CVPR18] Resource Aware Person Re-identification across Multiple Resolutions. [[paper](https://arxiv.org/abs/1805.08805)][[code](https://github.com/mileyan/DARENet)]\n\n- [CVPR18] Exploit the Unknown Gradually: One-Shot Video-Based Person Re-Identification by Stepwise Learning. [[paper](https://yu-wu.net/pdf/CVPR2018_Exploit-Unknown-Gradually.pdf)][[code](https://github.com/Yu-Wu/Exploit-Unknown-Gradually)]\n\n- [ArXiv18] Revisiting Temporal Modeling for Video-based Person ReID. [[paper](https://arxiv.org/abs/1805.02104)][[code](https://github.com/jiyanggao/Video-Person-ReID)]\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/MODEL_ZOO.md", "content": "# Model Zoo\n\n- Results are presented in the format of **.\n- When computing model size and FLOPs, only layers that are used at test time are considered (see `torchreid.utils.compute_model_complexity`).\n- Asterisk (\\*) means the model is trained from scratch.\n- `combineall=True` means all images in the dataset are used for model training.\n- Why not use heavy data augmentation like [random erasing](https://arxiv.org/abs/1708.04896) for model training? It's because heavy data augmentation might harm the cross-dataset generalization performance (see [this paper](https://arxiv.org/abs/1708.04896)).\n\n\n## ImageNet pretrained models\n\n\n| Model | Download |\n| :--- | :---: |\n| shufflenet | [model](https://drive.google.com/file/d/1RFnYcHK1TM-yt3yLsNecaKCoFO4Yb6a-/view?usp=sharing) |\n| mobilenetv2_x1_0 | [model](https://drive.google.com/file/d/1K7_CZE_L_Tf-BRY6_vVm0G-0ZKjVWh3R/view?usp=sharing) |\n| mobilenetv2_x1_4 | [model](https://drive.google.com/file/d/10c0ToIGIVI0QZTx284nJe8QfSJl5bIta/view?usp=sharing) |\n| mlfn | [model](https://drive.google.com/file/d/1PP8Eygct5OF4YItYRfA3qypYY9xiqHuV/view?usp=sharing) |\n| osnet_x1_0 | [model](https://drive.google.com/file/d/1LaG1EJpHrxdAxKnSCJ_i0u-nbxSAeiFY/view?usp=sharing) |\n| osnet_x0_75 | [model](https://drive.google.com/file/d/1uwA9fElHOk3ZogwbeY5GkLI6QPTX70Hq/view?usp=sharing) |\n| osnet_x0_5 | [model](https://drive.google.com/file/d/16DGLbZukvVYgINws8u8deSaOqjybZ83i/view?usp=sharing) |\n| osnet_x0_25 | [model](https://drive.google.com/file/d/1rb8UN5ZzPKRc_xvtHlyDh-cSz88YX9hs/view?usp=sharing) |\n| osnet_ibn_x1_0 | [model](https://drive.google.com/file/d/1sr90V6irlYYDd4_4ISU2iruoRG8J__6l/view?usp=sharing) |\n| osnet_ain_x1_0 | [model](https://drive.google.com/file/d/1-CaioD9NaqbHK_kzSMW8VE4_3KcsRjEo/view?usp=sharing) |\n| osnet_ain_x0_75 | [model](https://drive.google.com/file/d/1apy0hpsMypqstfencdH-jKIUEFOW4xoM/view?usp=sharing) |\n| osnet_ain_x0_5 | [model](https://drive.google.com/file/d/1KusKvEYyKGDTUBVRxRiz55G31wkihB6l/view?usp=sharing) |\n| osnet_ain_x0_25 | [model](https://drive.google.com/file/d/1SxQt2AvmEcgWNhaRb2xC4rP6ZwVDP0Wt/view?usp=sharing) |\n\n\n## Same-domain ReID\n\n\n| Model | # Param (10^6) | GFLOPs | Loss | Input | Transforms | Distance | market1501 | dukemtmcreid | msmt17 |\n| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n| resnet50 | 23.5 | 2.7 | softmax | (256, 128) | `random_flip`, `random_crop` | `euclidean` | [87.9 (70.4)](https://drive.google.com/file/d/1dUUZ4rHDWohmsQXCRe2C_HbYkzz94iBV/view?usp=sharing) | [78.3 (58.9)](https://drive.google.com/file/d/17ymnLglnc64NRvGOitY3BqMRS9UWd1wg/view?usp=sharing) | [63.2 (33.9)](https://drive.google.com/file/d/1ep7RypVDOthCRIAqDnn4_N-UhkkFHJsj/view?usp=sharing) |\n| resnet50_fc512 | 24.6 | 4.1 | softmax | (256, 128) | `random_flip`, `random_crop` | `euclidean` | [90.8 (75.3)](https://drive.google.com/file/d/1kv8l5laX_YCdIGVCetjlNdzKIA3NvsSt/view?usp=sharing) | [81.0 (64.0)](https://drive.google.com/file/d/13QN8Mp3XH81GK4BPGXobKHKyTGH50Rtx/view?usp=sharing) | [69.6 (38.4)](https://drive.google.com/file/d/1fDJLcz4O5wxNSUvImIIjoaIF9u1Rwaud/view?usp=sharing) |\n| mlfn | 32.5 | 2.8 | softmax | (256, 128) | `random_flip`, `random_crop` | `euclidean` | [90.1 (74.3)](https://drive.google.com/file/d/1wXcvhA_b1kpDfrt9s2Pma-MHxtj9pmvS/view?usp=sharing) | [81.1 (63.2)](https://drive.google.com/file/d/1rExgrTNb0VCIcOnXfMsbwSUW1h2L1Bum/view?usp=sharing) | [66.4 (37.2)](https://drive.google.com/file/d/18JzsZlJb3Wm7irCbZbZ07TN4IFKvR6p-/view?usp=sharing) |\n| hacnn* | 4.5 | 0.5 | softmax | (160, 64) | `random_flip`, `random_crop` | `euclidean` | [90.9 (75.6)](https://drive.google.com/file/d/1LRKIQduThwGxMDQMiVkTScBwR7WidmYF/view?usp=sharing) | [80.1 (63.2)](https://drive.google.com/file/d/1zNm6tP4ozFUCUQ7Sv1Z98EAJWXJEhtYH/view?usp=sharing) | [64.7 (37.2)](https://drive.google.com/file/d/1MsKRtPM5WJ3_Tk2xC0aGOO7pM3VaFDNZ/view?usp=sharing) |\n| mobilenetv2_x1_0 | 2.2 | 0.2 | softmax | (256, 128) | `random_flip`, `random_crop` | `euclidean` | [85.6 (67.3)](https://drive.google.com/file/d/18DgHC2ZJkjekVoqBWszD8_Xiikz-fewp/view?usp=sharing) | [74.2 (54.7)](https://drive.google.com/file/d/1q1WU2FETRJ3BXcpVtfJUuqq4z3psetds/view?usp=sharing) | [57.4 (29.3)](https://drive.google.com/file/d/1j50Hv14NOUAg7ZeB3frzfX-WYLi7SrhZ/view?usp=sharing) |\n| mobilenetv2_x1_4 | 4.3 | 0.4 | softmax | (256, 128) | `random_flip`, `random_crop` | `euclidean` | [87.0 (68.5)](https://drive.google.com/file/d/1t6JCqphJG-fwwPVkRLmGGyEBhGOf2GO5/view?usp=sharing) | [76.2 (55.8)](https://drive.google.com/file/d/12uD5FeVqLg9-AFDju2L7SQxjmPb4zpBN/view?usp=sharing) | [60.1 (31.5)](https://drive.google.com/file/d/1ZY5P2Zgm-3RbDpbXM0kIBMPvspeNIbXz/view?usp=sharing) |\n| osnet_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip` | `euclidean` | [94.2 (82.6)](https://drive.google.com/file/d/1vduhq5DpN2q1g4fYEZfPI17MJeh9qyrA/view?usp=sharing) | [87.0 (70.2)](https://drive.google.com/file/d/1QZO_4sNf4hdOKKKzKc-TZU9WW1v6zQbq/view?usp=sharing) | [74.9 (43.8)](https://drive.google.com/file/d/112EMUfBPYeYg70w-syK6V6Mx8-Qb9Q1M/view?usp=sharing) |\n| osnet_x0_75 | 1.3 | 0.57 | softmax | (256, 128) | `random_flip` | `euclidean` | [93.7 (81.2)](https://drive.google.com/file/d/1ozRaDSQw_EQ8_93OUmjDbvLXw9TnfPer/view?usp=sharing) | [85.8 (69.8)](https://drive.google.com/file/d/1IE3KRaTPp4OUa6PGTFL_d5_KQSJbP0Or/view?usp=sharing) | [72.8 (41.4)](https://drive.google.com/file/d/1QEGO6WnJ-BmUzVPd3q9NoaO_GsPNlmWc/view?usp=sharing) |\n| osnet_x0_5 | 0.6 | 0.27 | softmax | (256, 128) | `random_flip` | `euclidean` | [92.5 (79.8)](https://drive.google.com/file/d/1PLB9rgqrUM7blWrg4QlprCuPT7ILYGKT/view?usp=sharing) | [85.1 (67.4)](https://drive.google.com/file/d/1KoUVqmiST175hnkALg9XuTi1oYpqcyTu/view?usp=sharing) | [69.7 (37.5)](https://drive.google.com/file/d/1UT3AxIaDvS2PdxzZmbkLmjtiqq7AIKCv/view?usp=sharing) |\n| osnet_x0_25 | 0.2 | 0.08 | softmax | (256, 128) | `random_flip` | `euclidean` | [91.2 (75.0)](https://drive.google.com/file/d/1z1UghYvOTtjx7kEoRfmqSMu-z62J6MAj/view?usp=sharing) | [82.0 (61.4)](https://drive.google.com/file/d/1eumrtiXT4NOspjyEV4j8cHmlOaaCGk5l/view?usp=sharing) | [61.4 (29.5)](https://drive.google.com/file/d/1sSwXSUlj4_tHZequ_iZ8w_Jh0VaRQMqF/view?usp=sharing) |\n\n\n## Cross-domain ReID\n\n#### Market1501 -> DukeMTMC-reID\n\n\n| Model | # Param (10^6) | GFLOPs | Loss | Input | Transforms | Distance | Rank-1 | Rank-5 | Rank-10 | mAP | Download |\n| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n| osnet_ibn_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip`, `color_jitter` | `euclidean` | 48.5 | 62.3 | 67.4 | 26.7 | [model](https://drive.google.com/file/d/1uWW7_z_IcUmRNPqQOrEBdsvic94fWH37/view?usp=sharing) |\n| osnet_ain_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip`, `color_jitter` | `cosine` | 52.4 | 66.1 | 71.2 | 30.5 | [model](https://drive.google.com/file/d/14bNFGm0FhwHEkEpYKqKiDWjLNhXywFAd/view?usp=sharing) |\n\n\n#### DukeMTMC-reID -> Market1501\n\n\n| Model | # Param (10^6) | GFLOPs | Loss | Input | Transforms | Distance | Rank-1 | Rank-5 | Rank-10 | mAP | Download |\n| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n| osnet_ibn_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip`, `color_jitter` | `euclidean` | 57.7 | 73.7 | 80.0 | 26.1 | [model](https://drive.google.com/file/d/1CNxL1IP0BjcE1TSttiVOID1VNipAjiF3/view?usp=sharing) |\n| osnet_ain_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip`, `color_jitter` | `cosine` | 61.0 | 77.0 | 82.5 | 30.6 | [model](https://drive.google.com/file/d/1hypJvq8G04SOby6jvF337GEkg5K_bmCw/view?usp=sharing) |\n\n\n#### MSMT17 (`combineall=True`) -> Market1501 & DukeMTMC-reID\n\n\n| Model | # Param (10^6) | GFLOPs | Loss | Input | Transforms | Distance | msmt17 -> market1501 | msmt17 -> dukemtmcreid | Download |\n| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n| resnet50 | 23.5 | 2.7 | softmax | (256, 128) | `random_flip`, `color_jitter` | `euclidean` | 46.3 (22.8) | 52.3 (32.1) | [model](https://drive.google.com/file/d/1yiBteqgIZoOeywE8AhGmEQl7FTVwrQmf/view?usp=sharing) |\n| osnet_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip`, `color_jitter` | `euclidean` | 66.6 (37.5) | 66.0 (45.3) | [model](https://drive.google.com/file/d/1IosIFlLiulGIjwW3H8uMRmx3MzPwf86x/view?usp=sharing) |\n| osnet_x0_75 | 1.3 | 0.57 | softmax | (256, 128) | `random_flip`, `color_jitter` | `euclidean` | 63.6 (35.5) | 65.3 (44.5) | [model](https://drive.google.com/file/d/1fhjSS_7SUGCioIf2SWXaRGPqIY9j7-uw/view?usp=sharing) |\n| osnet_x0_5 | 0.6 | 0.27 | softmax | (256, 128) | `random_flip`, `color_jitter` | `euclidean` | 64.3 (34.9) | 65.2 (43.3) | [model](https://drive.google.com/file/d/1DHgmb6XV4fwG3n-CnCM0zdL9nMsZ9_RF/view?usp=sharing) |\n| osnet_x0_25 | 0.2 | 0.08 | softmax | (256, 128) | `random_flip`, `color_jitter` | `euclidean` | 59.9 (31.0) | 61.5 (39.6) | [model](https://drive.google.com/file/d/1Kkx2zW89jq_NETu4u42CFZTMVD5Hwm6e/view?usp=sharing) |\n| osnet_ibn_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip`, `color_jitter` | `euclidean` | 66.5 (37.2) | 67.4 (45.6) | [model](https://drive.google.com/file/d/1q3Sj2ii34NlfxA4LvmHdWO_75NDRmECJ/view?usp=sharing) |\n| osnet_ain_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip`, `color_jitter` | `cosine` | 70.1 (43.3) | 71.1 (52.7) | [model](https://drive.google.com/file/d/1SigwBE6mPdqiJMqhuIY4aqC7--5CsMal/view?usp=sharing) |\n\n\n#### Multi-source domain generalization\n\nThe models below are trained using multiple source datasets, as described in [Zhou et al. TPAMI'21](https://arxiv.org/abs/1910.06827).\n\nRegarding the abbreviations, MS is MSMT17; M is Market1501; D is DukeMTMC-reID; and C is CUHK03.\n\nAll models were trained with [im_osnet_ain_x1_0_softmax_256x128_amsgrad_cosine.yaml](https://github.com/KaiyangZhou/deep-person-reid/blob/master/configs/im_osnet_ain_x1_0_softmax_256x128_amsgrad_cosine.yaml) and `max_epoch=50`.\n\n| Model | # Param (10^6) | GFLOPs | Loss | Input | Transforms | Distance | MS+D+C->M | MS+M+C->D | MS+D+M->C |D+M+C->MS |\n| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n| osnet_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip`, `color_jitter` | `cosine` | [72.5 (44.2)](https://drive.google.com/file/d/1tuYY1vQXReEd8N8_npUkc7npPDDmjNCV/view?usp=sharing) | [65.2 (47.0)](https://drive.google.com/file/d/1UxUI4NsE108UCvcy3O1Ufe73nIVPKCiu/view?usp=sharing) | [23.9 (23.3)](https://drive.google.com/file/d/1kAA6qHJvbaJtyh1b39ZyEqWROwUgWIhl/view?usp=sharing) | [33.2 (12.6)](https://drive.google.com/file/d/1wAHuYVTzj8suOwqCNcEmu6YdbVnHDvA2/view?usp=sharing) |\n| osnet_ibn_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip`, `color_jitter` | `cosine` | [73.0 (44.9)](https://drive.google.com/file/d/14sH6yZwuNHPTElVoEZ26zozOOZIej5Mf/view?usp=sharing) | [64.6 (45.7)](https://drive.google.com/file/d/1Sk-2SSwKAF8n1Z4p_Lm_pl0E6v2WlIBn/view?usp=sharing) | [25.7 (25.4)](https://drive.google.com/file/d/1actHP7byqWcK4eBE1ojnspSMdo7k2W4G/view?usp=sharing) | [39.8 (16.2)](https://drive.google.com/file/d/1BGOSdLdZgqHe2qFafatb-5sPY40JlYfp/view?usp=sharing) |\n| osnet_ain_x1_0 | 2.2 | 0.98 | softmax | (256, 128) | `random_flip`, `color_jitter` | `cosine` | [73.3 (45.8)](https://drive.google.com/file/d/1nIrszJVYSHf3Ej8-j6DTFdWz8EnO42PB/view?usp=sharing) | [65.6 (47.2)](https://drive.google.com/file/d/1YjJ1ZprCmaKG6MH2P9nScB9FL_Utf9t1/view?usp=sharing) | [27.4 (27.1)](https://drive.google.com/file/d/1IxIg5P0cei3KPOJQ9ZRWDE_Mdrz01ha2/view?usp=sharing) | [40.2 (16.2)](https://drive.google.com/file/d/1KcoUKzLmsUoGHI7B6as_Z2fXL50gzexS/view?usp=sharing) |\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nSOURCEDIR = .\nBUILDDIR = _build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/conf.py", "content": "# -*- coding: utf-8 -*-\n#\n# Configuration file for the Sphinx documentation builder.\n#\n# This file does only contain a selection of the most common options. For a\n# full list see the documentation:\n# http://www.sphinx-doc.org/en/master/config\n\n# -- Path setup --------------------------------------------------------------\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\nimport os\nimport sys\n\nsys.path.insert(0, os.path.abspath('..'))\n\n# -- Project information -----------------------------------------------------\n\nproject = u'torchreid'\ncopyright = u'2019, Kaiyang Zhou'\nauthor = u'Kaiyang Zhou'\n\nversion_file = '../torchreid/__init__.py'\nwith open(version_file, 'r') as f:\n exec(compile(f.read(), version_file, 'exec'))\n__version__ = locals()['__version__']\n\n# The short X.Y version\nversion = __version__\n# The full version, including alpha/beta/rc tags\nrelease = __version__\n\n# -- General configuration ---------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n 'sphinx.ext.autodoc',\n 'sphinxcontrib.napoleon',\n 'sphinx.ext.viewcode',\n 'sphinx.ext.githubpages',\n 'sphinx_markdown_tables',\n]\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n#\nsource_suffix = ['.rst', '.md']\n# source_suffix = '.rst'\nsource_parsers = {'.md': 'recommonmark.parser.CommonMarkParser'}\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = None\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This pattern also affects html_static_path and html_extra_path.\nexclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = None\n\n# -- Options for HTML output -------------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = 'sphinx_rtd_theme'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#\n# html_theme_options = {}\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = ['_static']\n\n# Custom sidebar templates, must be a dictionary that maps document names\n# to template names.\n#\n# The default sidebars (for documents that don't match any pattern) are\n# defined by theme itself. Builtin themes are using these templates by\n# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',\n# 'searchbox.html']``.\n#\n# html_sidebars = {}\n\n# -- Options for HTMLHelp output ---------------------------------------------\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'torchreiddoc'\n\n# -- Options for LaTeX output ------------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n #\n # 'papersize': 'letterpaper',\n\n # The font size ('10pt', '11pt' or '12pt').\n #\n # 'pointsize': '10pt',\n\n # Additional stuff for the LaTeX preamble.\n #\n # 'preamble': '',\n\n # Latex figure (float) alignment\n #\n # 'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (\n master_doc, 'torchreid.tex', u'torchreid Documentation',\n u'Kaiyang Zhou', 'manual'\n ),\n]\n\n# -- Options for manual page output ------------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n (master_doc, 'torchreid', u'torchreid Documentation', [author], 1)\n]\n\n# -- Options for Texinfo output ----------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (\n master_doc, 'torchreid', u'torchreid Documentation', author,\n 'torchreid', 'One line description of project.', 'Miscellaneous'\n ),\n]\n\n# -- Options for Epub output -------------------------------------------------\n\n# Bibliographic Dublin Core info.\nepub_title = project\n\n# The unique identifier of the text. This can be a ISBN number\n# or the project homepage.\n#\n# epub_identifier = ''\n\n# A unique identification for the text.\n#\n# epub_uid = ''\n\n# A list of files that should not be packed into the epub file.\nepub_exclude_files = ['search.html']\n\n# -- Extension configuration -------------------------------------------------\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/datasets.rst", "content": ".. _datasets:\n\nDatasets\n=========\n\nHere we provide a comprehensive guide on how to prepare the datasets.\n\nSuppose you want to store the reid data in a directory called \"path/to/reid-data/\", you need to specify the ``root`` as *root='path/to/reid-data/'* when initializing ``DataManager``. Below we use ``$REID`` to denote \"path/to/reid-data\".\n\nPlease refer to :ref:`torchreid_data` for details regarding the arguments.\n\n\n.. note::\n Dataset with a :math:`\\dagger` symbol means that the process is automated, so you can directly call the dataset in ``DataManager`` (which automatically downloads the dataset and organizes the data structure). However, we also provide a way below to help the manual setup in case the automation fails.\n\n\n.. note::\n The keys to use specific datasets are enclosed in the parantheses beside the datasets' names.\n\n\n.. note::\n You are suggested to use the provided names for dataset folders such as \"market1501\" for Market1501 and \"dukemtmcreid\" for DukeMTMC-reID when doing the manual setup, otherwise you need to modify the source code accordingly (i.e. the ``dataset_dir`` attribute).\n\n.. note::\n Some download links provided by the original authors might not work. You can email `Kaiyang Zhou `_ to reqeust new links. Please do provide your full name, institution, and purpose of using the data in the email (best use your work email address).\n\n.. contents::\n :local:\n\n\nImage Datasets\n--------------\n\nMarket1501 :math:`^\\dagger` (``market1501``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n- Create a directory named \"market1501\" under ``$REID``.\n- Download the dataset to \"market1501\" from http://www.liangzheng.org/Project/project_reid.html and extract the files.\n- The data structure should look like\n\n.. code-block:: none\n \n market1501/\n Market-1501-v15.09.15/\n query/\n bounding_box_train/\n bounding_box_test/\n\n- To use the extra 500K distractors (i.e. Market1501 + 500K), go to the **Market-1501+500k Dataset** section at http://www.liangzheng.org/Project/project_reid.html, download the zip file \"distractors_500k.zip\" and extract it under \"market1501/Market-1501-v15.09.15\". The argument to use these 500K distrctors is ``market1501_500k`` in ``ImageDataManager``.\n\n\nCUHK03 (``cuhk03``)\n^^^^^^^^^^^^^^^^^^^^^\n- Create a folder named \"cuhk03\" under ``$REID``.\n- Download the dataset to \"cuhk03/\" from http://www.ee.cuhk.edu.hk/~xgwang/CUHK_identification.html and extract \"cuhk03_release.zip\", resulting in \"cuhk03/cuhk03_release/\".\n- Download the new split (767/700) from `person-re-ranking `_. What you need are \"cuhk03_new_protocol_config_detected.mat\" and \"cuhk03_new_protocol_config_labeled.mat\". Put these two mat files under \"cuhk03/\".\n- The data structure should look like\n\n.. code-block:: none\n \n cuhk03/\n cuhk03_release/\n cuhk03_new_protocol_config_detected.mat\n cuhk03_new_protocol_config_labeled.mat\n\n\n- In the default mode, we load data using the new split (767/700). If you wanna use the original (20) splits (1367/100), please set ``cuhk03_classic_split`` to True in ``ImageDataManager``. As the CMC is computed differently from Market1501 for the 1367/100 split (see `here `_), you need to enable ``use_metric_cuhk03`` in ``ImageDataManager`` to activate the *single-gallery-shot* metric for fair comparison with some methods that adopt the old splits (*do not need to report mAP*). In addition, we support both *labeled* and *detected* modes. The default mode loads *detected* images. Enable ``cuhk03_labeled`` in ``ImageDataManager`` if you wanna train and test on *labeled* images.\n\n.. note::\n The code will extract images in \"cuhk-03.mat\" and save them under \"cuhk03/images_detected\" and \"cuhk03/images_labeled\". Also, four json files will be automatically generated, i.e. \"splits_classic_detected.json\", \"splits_classic_labeled.json\", \"splits_new_detected.json\" and \"splits_new_labeled.json\". If the parent path of ``$REID`` is changed, these json files should be manually deleted. The code can automatically generate new json files to match the new path.\n \n\nDukeMTMC-reID :math:`^\\dagger` (``dukemtmcreid``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n- Create a directory called \"dukemtmc-reid\" under ``$REID``.\n- Download \"DukeMTMC-reID\" from http://vision.cs.duke.edu/DukeMTMC/ and extract it under \"dukemtmc-reid\".\n- The data structure should look like\n\n.. code-block:: none\n \n dukemtmc-reid/\n DukeMTMC-reID/\n query/\n bounding_box_train/\n bounding_box_test/\n ...\n\nMSMT17 (``msmt17``)\n^^^^^^^^^^^^^^^^^^^^^\n- Create a directory called \"msmt17\" under ``$REID``.\n- Download the dataset from http://www.pkuvmc.com/publications/msmt17.html to \"msmt17\" and extract the files.\n- The data structure should look like\n\n.. code-block:: none\n \n msmt17/\n MSMT17_V1/ # or MSMT17_V2\n train/\n test/\n list_train.txt\n list_query.txt\n list_gallery.txt\n list_val.txt\n\nVIPeR :math:`^\\dagger` (``viper``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n- The download link is http://users.soe.ucsc.edu/~manduchi/VIPeR.v1.0.zip.\n- Organize the dataset in a folder named \"viper\" as follows\n\n.. code-block:: none\n \n viper/\n VIPeR/\n cam_a/\n cam_b/\n\nGRID :math:`^\\dagger` (``grid``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n- The download link is http://personal.ie.cuhk.edu.hk/~ccloy/files/datasets/underground_reid.zip.\n- Organize the dataset in a folder named \"grid\" as follows\n\n.. code-block:: none\n \n grid/\n underground_reid/\n probe/\n gallery/\n ...\n\nCUHK01 (``cuhk01``)\n^^^^^^^^^^^^^^^^^^^^^^^^\n- Create a folder named \"cuhk01\" under ``$REID``.\n- Download \"CUHK01.zip\" from http://www.ee.cuhk.edu.hk/~xgwang/CUHK_identification.html and place it under \"cuhk01/\".\n- The code can automatically extract the files, or you can do it yourself.\n- The data structure should look like\n\n.. code-block:: none\n \n cuhk01/\n campus/\n\nSenseReID (``sensereid``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^\n- Create \"sensereid\" under ``$REID``.\n- Download the dataset from this `link `_ and extract it to \"sensereid\".\n- Organize the data to be like\n\n.. code-block:: none\n\n sensereid/\n SenseReID/\n test_probe/\n test_gallery/\n\nQMUL-iLIDS :math:`^\\dagger` (``ilids``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n- Create a folder named \"ilids\" under ``$REID``.\n- Download the dataset from http://www.eecs.qmul.ac.uk/~jason/data/i-LIDS_Pedestrian.tgz and organize it to look like\n\n.. code-block:: none\n \n ilids/\n i-LIDS_Pedestrian/\n Persons/\n\nPRID (``prid``)\n^^^^^^^^^^^^^^^^^^^\n- Create a directory named \"prid2011\" under ``$REID``.\n- Download the dataset from https://www.tugraz.at/institute/icg/research/team-bischof/lrs/downloads/PRID11/ and extract it under \"prid2011\".\n- The data structure should end up with\n\n.. code-block:: none\n\n prid2011/\n prid_2011/\n single_shot/\n multi_shot/\n\nCUHK02 (``cuhk02``)\n^^^^^^^^^^^^^^^^^^^^^\n- Create a folder named \"cuhk02\" under ``$REID``.\n- Download the data from http://www.ee.cuhk.edu.hk/~xgwang/CUHK_identification.html and put it under \"cuhk02/\".\n- Extract the file so the data structure looks like\n\n.. code-block:: none\n \n cuhk02/\n Dataset/\n P1/\n P2/\n P3/\n P4/\n P5/\n\nCUHKSYSU (``cuhksysu``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^\n- Create a folder named \"cuhksysu\" under ``$REID``.\n- Download the data to \"cuhksysu/\" from this `google drive link `_.\n- Extract the zip file under \"cuhksysu/\".\n- The data structure should look like\n\n.. code-block:: none\n \n cuhksysu/\n cropped_images\n\n\nVideo Datasets\n--------------\n\nMARS (``mars``)\n^^^^^^^^^^^^^^^^^\n- Create \"mars/\" under ``$REID``.\n- Download the dataset from http://www.liangzheng.com.cn/Project/project_mars.html and place it in \"mars/\".\n- Extract \"bbox_train.zip\" and \"bbox_test.zip\".\n- Download the split metadata from https://github.com/liangzheng06/MARS-evaluation/tree/master/info and put \"info/\" in \"mars/\".\n- The data structure should end up with\n\n.. code-block:: none\n \n mars/\n bbox_test/\n bbox_train/\n info/\n\niLIDS-VID :math:`^\\dagger` (``ilidsvid``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n- Create \"ilids-vid\" under ``$REID``.\n- Download the dataset from https://xiatian-zhu.github.io/downloads_qmul_iLIDS-VID_ReID_dataset.html to \"ilids-vid\".\n- Organize the data structure to match\n\n.. code-block:: none\n \n ilids-vid/\n i-LIDS-VID/\n train-test people splits/\n\nPRID2011 (``prid2011``)\n^^^^^^^^^^^^^^^^^^^^^^^^^\n- Create a directory named \"prid2011\" under ``$REID``.\n- Download the dataset from https://www.tugraz.at/institute/icg/research/team-bischof/lrs/downloads/PRID11/ and extract it under \"prid2011\".\n- Download the split created by *iLIDS-VID* from `this google drive `_ and put it under \"prid2011/\". Following the standard protocol, only 178 persons whose sequences are more than a threshold are used.\n- The data structure should end up with\n\n.. code-block:: none\n\n prid2011/\n splits_prid2011.json\n prid_2011/\n single_shot/\n multi_shot/\n\nDukeMTMC-VideoReID :math:`^\\dagger` (``dukemtmcvidreid``)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n- Create \"dukemtmc-vidreid\" under ``$REID``.\n- Download \"DukeMTMC-VideoReID\" from http://vision.cs.duke.edu/DukeMTMC/ and unzip the file to \"dukemtmc-vidreid/\".\n- The data structure should look like\n\n.. code-block:: none\n \n dukemtmc-vidreid/\n DukeMTMC-VideoReID/\n train/\n query/\n gallery/\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/evaluation.rst", "content": "Evaluation\n==========\n\nImage ReID\n-----------\n- **Market1501**, **DukeMTMC-reID**, **CUHK03 (767/700 split)** and **MSMT17** have fixed split so keeping ``split_id=0`` is fine.\n- **CUHK03 (classic split)** has 20 fixed splits, so do ``split_id=0~19``.\n- **VIPeR** contains 632 identities each with 2 images under two camera views. Evaluation should be done for 10 random splits. Each split randomly divides 632 identities to 316 train ids (632 images) and the other 316 test ids (632 images). Note that, in each random split, there are two sub-splits, one using camera-A as query and camera-B as gallery while the other one using camera-B as query and camera-A as gallery. Thus, there are totally 20 splits generated with ``split_id`` starting from 0 to 19. Models can be trained on ``split_id=[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]`` (because ``split_id=0`` and ``split_id=1`` share the same train set, and so on and so forth.). At test time, models trained on ``split_id=0`` can be directly evaluated on ``split_id=1``, models trained on ``split_id=2`` can be directly evaluated on ``split_id=3``, and so on and so forth.\n- **CUHK01** is similar to VIPeR in the split generation.\n- **GRID** , **iLIDS** and **PRID** have 10 random splits, so evaluation should be done by varying ``split_id`` from 0 to 9.\n- **SenseReID** has no training images and is used for evaluation only.\n\n\n.. note::\n The ``split_id`` argument is defined in ``ImageDataManager`` and ``VideoDataManager``. Please refer to :ref:`torchreid_data`.\n\n\nVideo ReID\n-----------\n- **MARS** and **DukeMTMC-VideoReID** have fixed single split so using ``split_id=0`` is ok.\n- **iLIDS-VID** and **PRID2011** have 10 predefined splits so evaluation should be done by varying ``split_id`` from 0 to 9." }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/index.rst", "content": ".. include:: ../README.rst\n\n\n.. toctree::\n :hidden:\n\n user_guide\n datasets\n evaluation\n\n.. toctree::\n :caption: Package Reference\n :hidden:\n\n pkg/data\n pkg/engine\n pkg/losses\n pkg/metrics\n pkg/models\n pkg/optim\n pkg/utils\n\n.. toctree::\n :caption: Resources\n :hidden:\n\n AWESOME_REID.md\n MODEL_ZOO.md\n\n\nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/pkg/data.rst", "content": ".. _torchreid_data:\n\ntorchreid.data\n==============\n\n\nData Manager\n---------------------------\n\n.. automodule:: torchreid.data.datamanager\n :members:\n\n\nSampler\n-----------------------\n\n.. automodule:: torchreid.data.sampler\n :members:\n\n\nTransforms\n---------------------------\n\n.. automodule:: torchreid.data.transforms\n :members:\n\n\nDataset\n---------------------------\n\n.. automodule:: torchreid.data.datasets.dataset\n :members:\n\n\n.. automodule:: torchreid.data.datasets.__init__\n :members:\n\n\nImage Datasets\n------------------------------\n\n.. automodule:: torchreid.data.datasets.image.market1501\n :members:\n\n.. automodule:: torchreid.data.datasets.image.cuhk03\n :members:\n\n.. automodule:: torchreid.data.datasets.image.dukemtmcreid\n :members:\n\n.. automodule:: torchreid.data.datasets.image.msmt17\n :members:\n\n.. automodule:: torchreid.data.datasets.image.viper\n :members:\n\n.. automodule:: torchreid.data.datasets.image.grid\n :members:\n\n.. automodule:: torchreid.data.datasets.image.cuhk01\n :members:\n\n.. automodule:: torchreid.data.datasets.image.ilids\n :members:\n\n.. automodule:: torchreid.data.datasets.image.sensereid\n :members:\n\n.. automodule:: torchreid.data.datasets.image.prid\n :members:\n\n\nVideo Datasets\n------------------------------\n\n.. automodule:: torchreid.data.datasets.video.mars\n :members:\n\n.. automodule:: torchreid.data.datasets.video.ilidsvid\n :members:\n\n.. automodule:: torchreid.data.datasets.video.prid2011\n :members:\n\n.. automodule:: torchreid.data.datasets.video.dukemtmcvidreid\n :members:" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/pkg/engine.rst", "content": ".. _torchreid_engine:\n\ntorchreid.engine\n==================\n\n\nBase Engine\n------------\n\n.. autoclass:: torchreid.engine.engine.Engine\n :members:\n\n\nImage Engines\n-------------\n\n.. autoclass:: torchreid.engine.image.softmax.ImageSoftmaxEngine\n :members:\n\n\n.. autoclass:: torchreid.engine.image.triplet.ImageTripletEngine\n :members:\n\n\nVideo Engines\n-------------\n\n.. autoclass:: torchreid.engine.video.softmax.VideoSoftmaxEngine\n\n\n.. autoclass:: torchreid.engine.video.triplet.VideoTripletEngine" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/pkg/losses.rst", "content": ".. _torchreid_losses:\n\ntorchreid.losses\n=================\n\n\nSoftmax\n--------\n\n.. automodule:: torchreid.losses.cross_entropy_loss\n :members:\n\n\nTriplet\n-------\n\n.. automodule:: torchreid.losses.hard_mine_triplet_loss\n :members:" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/pkg/metrics.rst", "content": ".. _torchreid_metrics:\n\ntorchreid.metrics\n=================\n\n\nDistance\n---------\n\n.. automodule:: torchreid.metrics.distance\n :members:\n\n\nAccuracy\n--------\n\n.. automodule:: torchreid.metrics.accuracy\n :members:\n\n\nRank\n-----\n\n.. automodule:: torchreid.metrics.rank\n :members: evaluate_rank" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/pkg/models.rst", "content": ".. _torchreid_models:\n\ntorchreid.models\n=================\n\nInterface\n---------\n\n.. automodule:: torchreid.models.__init__\n :members:\n\n\nImageNet Classification Models\n-------------------------------\n\n.. autoclass:: torchreid.models.resnet.ResNet\n.. autoclass:: torchreid.models.senet.SENet\n.. autoclass:: torchreid.models.densenet.DenseNet\n.. autoclass:: torchreid.models.inceptionresnetv2.InceptionResNetV2\n.. autoclass:: torchreid.models.inceptionv4.InceptionV4\n.. autoclass:: torchreid.models.xception.Xception\n\n\nLightweight Models\n------------------\n\n.. autoclass:: torchreid.models.nasnet.NASNetAMobile\n.. autoclass:: torchreid.models.mobilenetv2.MobileNetV2\n.. autoclass:: torchreid.models.shufflenet.ShuffleNet\n.. autoclass:: torchreid.models.squeezenet.SqueezeNet\n.. autoclass:: torchreid.models.shufflenetv2.ShuffleNetV2\n\n\nReID-specific Models\n--------------------\n\n.. autoclass:: torchreid.models.mudeep.MuDeep\n.. autoclass:: torchreid.models.resnetmid.ResNetMid\n.. autoclass:: torchreid.models.hacnn.HACNN\n.. autoclass:: torchreid.models.pcb.PCB\n.. autoclass:: torchreid.models.mlfn.MLFN\n.. autoclass:: torchreid.models.osnet.OSNet\n.. autoclass:: torchreid.models.osnet_ain.OSNet" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/pkg/optim.rst", "content": ".. _torchreid_optim:\n\ntorchreid.optim\n=================\n\n\nOptimizer\n----------\n\n.. automodule:: torchreid.optim.optimizer\n :members: build_optimizer\n\n\nLR Scheduler\n-------------\n\n.. automodule:: torchreid.optim.lr_scheduler\n :members: build_lr_scheduler" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/pkg/utils.rst", "content": ".. _torchreid_utils:\n\ntorchreid.utils\n=================\n\nAverage Meter\n--------------\n\n.. automodule:: torchreid.utils.avgmeter\n :members:\n\n\nLoggers\n-------\n\n.. automodule:: torchreid.utils.loggers\n :members:\n\n\nGeneric Tools\n---------------\n.. automodule:: torchreid.utils.tools\n :members:\n\n\nReID Tools\n----------\n\n.. automodule:: torchreid.utils.reidtools\n :members:\n\n\nTorch Tools\n------------\n\n.. automodule:: torchreid.utils.torchtools\n :members:\n\n\n.. automodule:: torchreid.utils.model_complexity\n :members:\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/docs/user_guide.rst", "content": "How-to\n============\n\n.. contents::\n :local:\n\n\nPrepare datasets\n-----------------\nSee :ref:`datasets`.\n\n\nFind model keys\n-----------------\nKeys are listed under the *Public keys* section within each model class in :ref:`torchreid_models`.\n\n\nShow available models\n----------------------\n\n.. code-block:: python\n \n import torchreid\n torchreid.models.show_avai_models()\n\n\nChange the training sampler\n-----------------------------\nThe default ``train_sampler`` is \"RandomSampler\". You can give the specific sampler name as input to ``train_sampler``, e.g. ``train_sampler='RandomIdentitySampler'`` for triplet loss.\n\n\nChoose an optimizer/lr_scheduler\n----------------------------------\nPlease refer to the source code of ``build_optimizer``/``build_lr_scheduler`` in :ref:`torchreid_optim` for details.\n\n\nResume training\n----------------\nSuppose the checkpoint is saved in \"log/resnet50/model.pth.tar-30\", you can do\n\n.. code-block:: python\n \n start_epoch = torchreid.utils.resume_from_checkpoint(\n 'log/resnet50/model.pth.tar-30',\n model,\n optimizer\n )\n\n engine.run(\n save_dir='log/resnet50',\n max_epoch=60,\n start_epoch=start_epoch\n )\n\n\nCompute model complexity\n--------------------------\nWe provide a tool in ``torchreid.utils.model_complexity.py`` to automatically compute the model complexity, i.e. number of parameters and FLOPs.\n\n.. code-block:: python\n \n from torchreid import models, utils\n \n model = models.build_model(name='resnet50', num_classes=1000)\n num_params, flops = utils.compute_model_complexity(model, (1, 3, 256, 128))\n\n # show detailed complexity for each module\n utils.compute_model_complexity(model, (1, 3, 256, 128), verbose=True)\n\n # count flops for all layers including ReLU and BatchNorm\n utils.compute_model_complexity(model, (1, 3, 256, 128), verbose=True, only_conv_linear=False)\n\nNote that (1) this function only provides an estimate of the theoretical time complexity rather than the actual running time which depends on implementations and hardware; (2) the FLOPs is only counted for layers that are used at test time. This means that redundant layers such as person ID classification layer will be ignored. The inference graph depends on how you define the computations in ``forward()``.\n\n\nCombine multiple datasets\n---------------------------\nEasy. Just give whatever datasets (keys) you want to the ``sources`` argument when instantiating a data manager. For example,\n\n.. code-block:: python\n \n datamanager = torchreid.data.ImageDataManager(\n root='reid-data',\n sources=['market1501', 'dukemtmcreid', 'cuhk03', 'msmt17'],\n height=256,\n width=128,\n batch_size=32\n )\n\nIn this example, the target datasets are Market1501, DukeMTMC-reID, CUHK03 and MSMT17 as the ``targets`` argument is not specified. Please refer to ``Engine.test()`` in :ref:`torchreid_engine` for details regarding how evaluation is performed.\n\n\nDo cross-dataset evaluation\n-----------------------------\nEasy. Just give whatever datasets (keys) you want to the argument ``targets``, like\n\n.. code-block:: python\n \n datamanager = torchreid.data.ImageDataManager(\n root='reid-data',\n sources='market1501',\n targets='dukemtmcreid', # or targets='cuhk03' or targets=['dukemtmcreid', 'cuhk03']\n height=256,\n width=128,\n batch_size=32\n )\n\n\nCombine train, query and gallery\n---------------------------------\nThis can be easily done by setting ``combineall=True`` when instantiating a data manager. Below is an example of using Market1501,\n\n.. code-block:: python\n \n datamanager = torchreid.data.ImageDataManager(\n root='reid-data',\n sources='market1501',\n height=256,\n width=128,\n batch_size=32,\n market1501_500k=False,\n combineall=True # it's me, here\n )\n\nMore specifically, with ``combineall=False``, you will get\n\n.. code-block:: none\n \n => Loaded Market1501\n ----------------------------------------\n subset | # ids | # images | # cameras\n ----------------------------------------\n train | 751 | 12936 | 6\n query | 750 | 3368 | 6\n gallery | 751 | 15913 | 6\n ---------------------------------------\n\nwith ``combineall=True``, you will get\n\n.. code-block:: none\n \n => Loaded Market1501\n ----------------------------------------\n subset | # ids | # images | # cameras\n ----------------------------------------\n train | 1501 | 29419 | 6\n query | 750 | 3368 | 6\n gallery | 751 | 15913 | 6\n ---------------------------------------\n\n\nOptimize layers with different learning rates\n-----------------------------------------------\nA common practice for fine-tuning pretrained models is to use a smaller learning rate for base layers and a large learning rate for randomly initialized layers (referred to as ``new_layers``). ``torchreid.optim.optimizer`` has implemented such feature. What you need to do is to set ``staged_lr=True`` and give the names of ``new_layers`` such as \"classifier\".\n\nBelow is an example of setting different learning rates for base layers and new layers in ResNet50,\n\n.. code-block:: python\n \n # New layer \"classifier\" has a learning rate of 0.01\n # The base layers have a learning rate of 0.001\n optimizer = torchreid.optim.build_optimizer(\n model,\n optim='sgd',\n lr=0.01,\n staged_lr=True,\n new_layers='classifier',\n base_lr_mult=0.1\n )\n\nPlease refer to :ref:`torchreid_optim` for more details.\n\n\nDo two-stepped transfer learning\n-------------------------------------\nTo prevent the pretrained layers from being damaged by harmful gradients back-propagated from randomly initialized layers, one can adopt the *two-stepped transfer learning strategy* presented in `Deep Transfer Learning for Person Re-identification `_. The basic idea is to pretrain the randomly initialized layers for few epochs while keeping the base layers frozen before training all layers end-to-end.\n\nThis has been implemented in ``Engine.train()`` (see :ref:`torchreid_engine`). The arguments related to this feature are ``fixbase_epoch`` and ``open_layers``. Intuitively, ``fixbase_epoch`` denotes the number of epochs to keep the base layers frozen; ``open_layers`` means which layer is open for training.\n\nFor example, say you want to pretrain the classification layer named \"classifier\" in ResNet50 for 5 epochs before training all layers, you can do\n\n.. code-block:: python\n \n engine.run(\n save_dir='log/resnet50',\n max_epoch=60,\n eval_freq=10,\n print_freq=10,\n test_only=False,\n fixbase_epoch=5,\n open_layers='classifier'\n )\n # or open_layers=['fc', 'classifier'] if there is another fc layer that\n # is randomly initialized, like resnet50_fc512\n\nNote that ``fixbase_epoch`` is counted into ``max_epoch``. In the above example, the base network will be fixed for 5 epochs and then open for training for 55 epochs. Thus, if you want to freeze some layers throughout the training, what you can do is to set ``fixbase_epoch`` equal to ``max_epoch`` and put the layer names in ``open_layers`` which you want to train.\n\n\nTest a trained model\n----------------------\nYou can load a trained model using :code:`torchreid.utils.load_pretrained_weights(model, weight_path)` and set ``test_only=True`` in ``engine.run()``.\n\n\nFine-tune a model pre-trained on reid datasets\n-----------------------------------------------\nUse :code:`torchreid.utils.load_pretrained_weights(model, weight_path)` to load the pre-trained weights and then fine-tune on the dataset you want.\n\n\nVisualize learning curves with tensorboard\n--------------------------------------------\nThe ``SummaryWriter()`` for tensorboard will be automatically initialized in ``engine.run()`` when you are training your model. Therefore, you do not need to do extra jobs. After the training is done, the ``*tf.events*`` file will be saved in ``save_dir``. Then, you just call ``tensorboard --logdir=your_save_dir`` in your terminal and visit ``http://localhost:6006/`` in a web browser. See `pytorch tensorboard `_ for further information.\n\n\nVisualize ranking results\n---------------------------\nThis can be achieved by setting ``visrank`` to true in ``engine.run()``. ``visrank_topk`` determines the top-k images to be visualized (Default is ``visrank_topk=10``). Note that ``visrank`` can only be used in test mode, i.e. ``test_only=True`` in ``engine.run()``. The output will be saved under ``save_dir/visrank_DATASETNAME`` where each plot contains the top-k similar gallery images given a query. An example is shown below where red and green denote incorrect and correct matches respectively.\n\n.. image:: figures/ranking_results.jpg\n :width: 800px\n :align: center\n\n\nVisualize activation maps\n--------------------------\nTo understand where the CNN focuses on to extract features for ReID, you can visualize the activation maps as in `OSNet `_. This is implemented in ``tools/visualize_actmap.py`` (check the code for more details). An example running command is\n\n.. code-block:: shell\n \n python tools/visualize_actmap.py \\\n --root $DATA/reid \\\n -d market1501 \\\n -m osnet_x1_0 \\\n --weights PATH_TO_PRETRAINED_WEIGHTS \\\n --save-dir log/visactmap_osnet_x1_0_market1501\n\nThe output will look like (from left to right: image, activation map, overlapped image)\n\n.. image:: figures/actmap.jpg\n :width: 300px\n :align: center\n\n\n.. note::\n In order to visualize activation maps, the CNN needs to output the last convolutional feature maps at eval mode. See ``torchreid/models/osnet.py`` for example.\n\n\nUse your own dataset\n----------------------\n1. Write your own dataset class. Below is a template for image dataset. However, it can also be applied to a video dataset class, for which you simply change ``ImageDataset`` to ``VideoDataset``.\n\n.. code-block:: python\n \n from __future__ import absolute_import\n from __future__ import print_function\n from __future__ import division\n\n import sys\n import os\n import os.path as osp\n\n from torchreid.data import ImageDataset\n\n\n class NewDataset(ImageDataset):\n dataset_dir = 'new_dataset'\n\n def __init__(self, root='', **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n\n # All you need to do here is to generate three lists,\n # which are train, query and gallery.\n # Each list contains tuples of (img_path, pid, camid),\n # where\n # - img_path (str): absolute path to an image.\n # - pid (int): person ID, e.g. 0, 1.\n # - camid (int): camera ID, e.g. 0, 1.\n # Note that\n # - pid and camid should be 0-based.\n # - query and gallery should share the same pid scope (e.g.\n # pid=0 in query refers to the same person as pid=0 in gallery).\n # - train, query and gallery share the same camid scope (e.g.\n # camid=0 in train refers to the same camera as camid=0\n # in query/gallery).\n train = ...\n query = ...\n gallery = ...\n\n super(NewDataset, self).__init__(train, query, gallery, **kwargs)\n\n\n2. Register your dataset.\n\n.. code-block:: python\n \n import torchreid\n torchreid.data.register_image_dataset('new_dataset', NewDataset)\n\n\n3. Initialize a data manager with your dataset.\n\n.. code-block:: python\n \n # use your own dataset only\n datamanager = torchreid.data.ImageDataManager(\n root='reid-data',\n sources='new_dataset'\n )\n # combine with other datasets\n datamanager = torchreid.data.ImageDataManager(\n root='reid-data',\n sources=['new_dataset', 'dukemtmcreid']\n )\n # cross-dataset evaluation\n datamanager = torchreid.data.ImageDataManager(\n root='reid-data',\n sources=['new_dataset', 'dukemtmcreid'],\n targets='market1501' # or targets=['market1501', 'cuhk03']\n )\n\n\n\nDesign your own Engine\n------------------------\nA new Engine should be designed if you have your own loss function. The base Engine class ``torchreid.engine.Engine`` has implemented some generic methods which you can inherit to avoid re-writing. Please refer to the source code for more details. You are suggested to see how ``ImageSoftmaxEngine`` and ``ImageTripletEngine`` are constructed (also ``VideoSoftmaxEngine`` and ``VideoTripletEngine``). All you need to implement might be just a ``forward_backward()`` function.\n\n\nUse Torchreid as a feature extractor in your projects\n-------------------------------------------------------\nWe have provided a simple API for feature extraction, which accepts input of various types such as a list of image paths or numpy arrays. More details can be found in the code at ``torchreid/utils/feature_extractor.py``. Here we show a simple example of how to extract features given a list of image paths.\n\n.. code-block:: python\n\n from torchreid.utils import FeatureExtractor\n\n extractor = FeatureExtractor(\n model_name='osnet_x1_0',\n model_path='a/b/c/model.pth.tar',\n device='cuda'\n )\n\n image_list = [\n 'a/b/c/image001.jpg',\n 'a/b/c/image002.jpg',\n 'a/b/c/image003.jpg',\n 'a/b/c/image004.jpg',\n 'a/b/c/image005.jpg'\n ]\n\n features = extractor(image_list)\n print(features.shape) # output (5, 512)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/linter.sh", "content": "echo \"Running isort\"\nisort -y -sp .\necho \"Done\"\n\necho \"Running yapf\"\nyapf -i -r -vv -e build .\necho \"Done\"\n\necho \"Running flake8\"\nflake8 .\necho \"Done\"" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/DML/README.md", "content": "# Deep mutual learning\n\nThis repo implements [Deep Mutual Learning (CVPR'18)](https://zpascal.net/cvpr2018/Zhang_Deep_Mutual_Learning_CVPR_2018_paper.pdf) (DML) for person re-id.\n\nWe used this code in our [OSNet](https://arxiv.org/pdf/1905.00953.pdf) paper (see Supp. B). The training command to reproduce the result of \"triplet + DML\" (Table 12f in the paper) is\n```bash\npython main.py \\\n--config-file im_osnet_x1_0_dml_256x128_amsgrad_cosine.yaml \\\n--root $DATA\n```\n\n`$DATA` corresponds to the path to your dataset folder.\n\nChange `model.deploy` to `both` if you wanna enable model ensembling.\n\nIf you have any questions, please raise an issue in the Issues area." }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/DML/default_config.py", "content": "from yacs.config import CfgNode as CN\n\n\ndef get_default_config():\n cfg = CN()\n\n # model\n cfg.model = CN()\n cfg.model.name = 'resnet50'\n cfg.model.pretrained = True # automatically load pretrained model weights if available\n cfg.model.load_weights1 = '' # path to model-1 weights\n cfg.model.load_weights2 = '' # path to model-2 weights\n cfg.model.resume1 = '' # path to checkpoint for resume training\n cfg.model.resume2 = '' # path to checkpoint for resume training\n cfg.model.deploy = 'model1' # model1, model2 or both\n\n # data\n cfg.data = CN()\n cfg.data.type = 'image'\n cfg.data.root = 'reid-data'\n cfg.data.sources = ['market1501']\n cfg.data.targets = ['market1501']\n cfg.data.workers = 4 # number of data loading workers\n cfg.data.split_id = 0 # split index\n cfg.data.height = 256 # image height\n cfg.data.width = 128 # image width\n cfg.data.combineall = False # combine train, query and gallery for training\n cfg.data.transforms = ['random_flip'] # data augmentation\n cfg.data.norm_mean = [0.485, 0.456, 0.406] # default is imagenet mean\n cfg.data.norm_std = [0.229, 0.224, 0.225] # default is imagenet std\n cfg.data.save_dir = 'log' # path to save log\n cfg.data.load_train_targets = False\n\n # specific datasets\n cfg.market1501 = CN()\n cfg.market1501.use_500k_distractors = False # add 500k distractors to the gallery set for market1501\n cfg.cuhk03 = CN()\n cfg.cuhk03.labeled_images = False # use labeled images, if False, use detected images\n cfg.cuhk03.classic_split = False # use classic split by Li et al. CVPR14\n cfg.cuhk03.use_metric_cuhk03 = False # use cuhk03's metric for evaluation\n\n # sampler\n cfg.sampler = CN()\n cfg.sampler.train_sampler = 'RandomSampler'\n cfg.sampler.num_instances = 4 # number of instances per identity for RandomIdentitySampler\n\n # video reid setting\n cfg.video = CN()\n cfg.video.seq_len = 15 # number of images to sample in a tracklet\n cfg.video.sample_method = 'evenly' # how to sample images from a tracklet\n cfg.video.pooling_method = 'avg' # how to pool features over a tracklet\n\n # train\n cfg.train = CN()\n cfg.train.optim = 'adam'\n cfg.train.lr = 0.0003\n cfg.train.weight_decay = 5e-4\n cfg.train.max_epoch = 60\n cfg.train.start_epoch = 0\n cfg.train.batch_size = 32\n cfg.train.fixbase_epoch = 0 # number of epochs to fix base layers\n cfg.train.open_layers = [\n 'classifier'\n ] # layers for training while keeping others frozen\n cfg.train.staged_lr = False # set different lr to different layers\n cfg.train.new_layers = ['classifier'] # newly added layers with default lr\n cfg.train.base_lr_mult = 0.1 # learning rate multiplier for base layers\n cfg.train.lr_scheduler = 'single_step'\n cfg.train.stepsize = [20] # stepsize to decay learning rate\n cfg.train.gamma = 0.1 # learning rate decay multiplier\n cfg.train.print_freq = 20 # print frequency\n cfg.train.seed = 1 # random seed\n\n # optimizer\n cfg.sgd = CN()\n cfg.sgd.momentum = 0.9 # momentum factor for sgd and rmsprop\n cfg.sgd.dampening = 0. # dampening for momentum\n cfg.sgd.nesterov = False # Nesterov momentum\n cfg.rmsprop = CN()\n cfg.rmsprop.alpha = 0.99 # smoothing constant\n cfg.adam = CN()\n cfg.adam.beta1 = 0.9 # exponential decay rate for first moment\n cfg.adam.beta2 = 0.999 # exponential decay rate for second moment\n\n # loss\n cfg.loss = CN()\n cfg.loss.name = 'triplet'\n cfg.loss.softmax = CN()\n cfg.loss.softmax.label_smooth = True # use label smoothing regularizer\n cfg.loss.triplet = CN()\n cfg.loss.triplet.margin = 0.3 # distance margin\n cfg.loss.triplet.weight_t = 1. # weight to balance hard triplet loss\n cfg.loss.triplet.weight_x = 0. # weight to balance cross entropy loss\n cfg.loss.dml = CN()\n cfg.loss.dml.weight_ml = 1. # weight for mutual learning loss\n\n # test\n cfg.test = CN()\n cfg.test.batch_size = 100\n cfg.test.dist_metric = 'euclidean' # distance metric, ['euclidean', 'cosine']\n cfg.test.normalize_feature = False # normalize feature vectors before computing distance\n cfg.test.ranks = [1, 5, 10, 20] # cmc ranks\n cfg.test.evaluate = False # test only\n cfg.test.eval_freq = -1 # evaluation frequency (-1 means to only test after training)\n cfg.test.start_eval = 0 # start to evaluate after a specific epoch\n cfg.test.rerank = False # use person re-ranking\n cfg.test.visrank = False # visualize ranked results (only available when cfg.test.evaluate=True)\n cfg.test.visrank_topk = 10 # top-k ranks to visualize\n\n return cfg\n\n\ndef imagedata_kwargs(cfg):\n return {\n 'root': cfg.data.root,\n 'sources': cfg.data.sources,\n 'targets': cfg.data.targets,\n 'height': cfg.data.height,\n 'width': cfg.data.width,\n 'transforms': cfg.data.transforms,\n 'norm_mean': cfg.data.norm_mean,\n 'norm_std': cfg.data.norm_std,\n 'use_gpu': cfg.use_gpu,\n 'split_id': cfg.data.split_id,\n 'combineall': cfg.data.combineall,\n 'load_train_targets': cfg.data.load_train_targets,\n 'batch_size_train': cfg.train.batch_size,\n 'batch_size_test': cfg.test.batch_size,\n 'workers': cfg.data.workers,\n 'num_instances': cfg.sampler.num_instances,\n 'train_sampler': cfg.sampler.train_sampler,\n # image\n 'cuhk03_labeled': cfg.cuhk03.labeled_images,\n 'cuhk03_classic_split': cfg.cuhk03.classic_split,\n 'market1501_500k': cfg.market1501.use_500k_distractors,\n }\n\n\ndef videodata_kwargs(cfg):\n return {\n 'root': cfg.data.root,\n 'sources': cfg.data.sources,\n 'targets': cfg.data.targets,\n 'height': cfg.data.height,\n 'width': cfg.data.width,\n 'transforms': cfg.data.transforms,\n 'norm_mean': cfg.data.norm_mean,\n 'norm_std': cfg.data.norm_std,\n 'use_gpu': cfg.use_gpu,\n 'split_id': cfg.data.split_id,\n 'combineall': cfg.data.combineall,\n 'batch_size_train': cfg.train.batch_size,\n 'batch_size_test': cfg.test.batch_size,\n 'workers': cfg.data.workers,\n 'num_instances': cfg.sampler.num_instances,\n 'train_sampler': cfg.sampler.train_sampler,\n # video\n 'seq_len': cfg.video.seq_len,\n 'sample_method': cfg.video.sample_method\n }\n\n\ndef optimizer_kwargs(cfg):\n return {\n 'optim': cfg.train.optim,\n 'lr': cfg.train.lr,\n 'weight_decay': cfg.train.weight_decay,\n 'momentum': cfg.sgd.momentum,\n 'sgd_dampening': cfg.sgd.dampening,\n 'sgd_nesterov': cfg.sgd.nesterov,\n 'rmsprop_alpha': cfg.rmsprop.alpha,\n 'adam_beta1': cfg.adam.beta1,\n 'adam_beta2': cfg.adam.beta2,\n 'staged_lr': cfg.train.staged_lr,\n 'new_layers': cfg.train.new_layers,\n 'base_lr_mult': cfg.train.base_lr_mult\n }\n\n\ndef lr_scheduler_kwargs(cfg):\n return {\n 'lr_scheduler': cfg.train.lr_scheduler,\n 'stepsize': cfg.train.stepsize,\n 'gamma': cfg.train.gamma,\n 'max_epoch': cfg.train.max_epoch\n }\n\n\ndef engine_run_kwargs(cfg):\n return {\n 'save_dir': cfg.data.save_dir,\n 'max_epoch': cfg.train.max_epoch,\n 'start_epoch': cfg.train.start_epoch,\n 'fixbase_epoch': cfg.train.fixbase_epoch,\n 'open_layers': cfg.train.open_layers,\n 'start_eval': cfg.test.start_eval,\n 'eval_freq': cfg.test.eval_freq,\n 'test_only': cfg.test.evaluate,\n 'print_freq': cfg.train.print_freq,\n 'dist_metric': cfg.test.dist_metric,\n 'normalize_feature': cfg.test.normalize_feature,\n 'visrank': cfg.test.visrank,\n 'visrank_topk': cfg.test.visrank_topk,\n 'use_metric_cuhk03': cfg.cuhk03.use_metric_cuhk03,\n 'ranks': cfg.test.ranks,\n 'rerank': cfg.test.rerank\n }\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/DML/dml.py", "content": "from __future__ import division, print_function, absolute_import\nimport torch\nfrom torch.nn import functional as F\n\nfrom torchreid.utils import open_all_layers, open_specified_layers\nfrom torchreid.engine import Engine\nfrom torchreid.losses import TripletLoss, CrossEntropyLoss\n\n\nclass ImageDMLEngine(Engine):\n\n def __init__(\n self,\n datamanager,\n model1,\n optimizer1,\n scheduler1,\n model2,\n optimizer2,\n scheduler2,\n margin=0.3,\n weight_t=0.5,\n weight_x=1.,\n weight_ml=1.,\n use_gpu=True,\n label_smooth=True,\n deploy='model1'\n ):\n super(ImageDMLEngine, self).__init__(datamanager, use_gpu)\n\n self.model1 = model1\n self.optimizer1 = optimizer1\n self.scheduler1 = scheduler1\n self.register_model('model1', model1, optimizer1, scheduler1)\n\n self.model2 = model2\n self.optimizer2 = optimizer2\n self.scheduler2 = scheduler2\n self.register_model('model2', model2, optimizer2, scheduler2)\n\n self.weight_t = weight_t\n self.weight_x = weight_x\n self.weight_ml = weight_ml\n\n assert deploy in ['model1', 'model2', 'both']\n self.deploy = deploy\n\n self.criterion_t = TripletLoss(margin=margin)\n self.criterion_x = CrossEntropyLoss(\n num_classes=self.datamanager.num_train_pids,\n use_gpu=self.use_gpu,\n label_smooth=label_smooth\n )\n\n def forward_backward(self, data):\n imgs, pids = self.parse_data_for_train(data)\n\n if self.use_gpu:\n imgs = imgs.cuda()\n pids = pids.cuda()\n\n outputs1, features1 = self.model1(imgs)\n loss1_x = self.compute_loss(self.criterion_x, outputs1, pids)\n loss1_t = self.compute_loss(self.criterion_t, features1, pids)\n\n outputs2, features2 = self.model2(imgs)\n loss2_x = self.compute_loss(self.criterion_x, outputs2, pids)\n loss2_t = self.compute_loss(self.criterion_t, features2, pids)\n\n loss1_ml = self.compute_kl_div(\n outputs2.detach(), outputs1, is_logit=True\n )\n loss2_ml = self.compute_kl_div(\n outputs1.detach(), outputs2, is_logit=True\n )\n\n loss1 = 0\n loss1 += loss1_x * self.weight_x\n loss1 += loss1_t * self.weight_t\n loss1 += loss1_ml * self.weight_ml\n\n loss2 = 0\n loss2 += loss2_x * self.weight_x\n loss2 += loss2_t * self.weight_t\n loss2 += loss2_ml * self.weight_ml\n\n self.optimizer1.zero_grad()\n loss1.backward()\n self.optimizer1.step()\n\n self.optimizer2.zero_grad()\n loss2.backward()\n self.optimizer2.step()\n\n loss_dict = {\n 'loss1_x': loss1_x.item(),\n 'loss1_t': loss1_t.item(),\n 'loss1_ml': loss1_ml.item(),\n 'loss2_x': loss1_x.item(),\n 'loss2_t': loss1_t.item(),\n 'loss2_ml': loss1_ml.item()\n }\n\n return loss_dict\n\n @staticmethod\n def compute_kl_div(p, q, is_logit=True):\n if is_logit:\n p = F.softmax(p, dim=1)\n q = F.softmax(q, dim=1)\n return -(p * torch.log(q + 1e-8)).sum(1).mean()\n\n def two_stepped_transfer_learning(\n self, epoch, fixbase_epoch, open_layers, model=None\n ):\n \"\"\"Two stepped transfer learning.\n\n The idea is to freeze base layers for a certain number of epochs\n and then open all layers for training.\n\n Reference: https://arxiv.org/abs/1611.05244\n \"\"\"\n model1 = self.model1\n model2 = self.model2\n\n if (epoch + 1) <= fixbase_epoch and open_layers is not None:\n print(\n '* Only train {} (epoch: {}/{})'.format(\n open_layers, epoch + 1, fixbase_epoch\n )\n )\n open_specified_layers(model1, open_layers)\n open_specified_layers(model2, open_layers)\n else:\n open_all_layers(model1)\n open_all_layers(model2)\n\n def extract_features(self, input):\n if self.deploy == 'model1':\n return self.model1(input)\n\n elif self.deploy == 'model2':\n return self.model2(input)\n\n else:\n features = []\n features.append(self.model1(input))\n features.append(self.model2(input))\n return torch.cat(features, 1)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/DML/im_osnet_x1_0_dml_256x128_amsgrad_cosine.yaml", "content": "model:\n name: 'osnet_x1_0'\n pretrained: True\n deploy: 'model1'\n\ndata:\n type: 'image'\n sources: ['market1501']\n targets: ['market1501']\n height: 256\n width: 128\n combineall: False\n transforms: ['random_flip', 'random_erase']\n save_dir: 'log/osnet_x1_0_market1501_dml_cosinelr'\n\nloss:\n name: 'triplet'\n softmax:\n label_smooth: True\n triplet:\n margin: 0.3\n weight_t: 0.5\n weight_x: 1.\n dml:\n weight_ml: 1.\n\ntrain:\n optim: 'amsgrad'\n lr: 0.0015\n max_epoch: 250\n batch_size: 64\n fixbase_epoch: 10\n open_layers: ['classifier']\n lr_scheduler: 'cosine'\n\ntest:\n batch_size: 300\n dist_metric: 'cosine'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/DML/main.py", "content": "import sys\nimport copy\nimport time\nimport os.path as osp\nimport argparse\nimport torch\nimport torch.nn as nn\n\nimport torchreid\nfrom torchreid.utils import (\n Logger, check_isfile, set_random_seed, collect_env_info,\n resume_from_checkpoint, load_pretrained_weights, compute_model_complexity\n)\n\nfrom dml import ImageDMLEngine\nfrom default_config import (\n imagedata_kwargs, optimizer_kwargs, engine_run_kwargs, get_default_config,\n lr_scheduler_kwargs\n)\n\n\ndef reset_config(cfg, args):\n if args.root:\n cfg.data.root = args.root\n if args.sources:\n cfg.data.sources = args.sources\n if args.targets:\n cfg.data.targets = args.targets\n if args.transforms:\n cfg.data.transforms = args.transforms\n\n\ndef main():\n parser = argparse.ArgumentParser(\n formatter_class=argparse.ArgumentDefaultsHelpFormatter\n )\n parser.add_argument(\n '--config-file', type=str, default='', help='path to config file'\n )\n parser.add_argument(\n '-s',\n '--sources',\n type=str,\n nargs='+',\n help='source datasets (delimited by space)'\n )\n parser.add_argument(\n '-t',\n '--targets',\n type=str,\n nargs='+',\n help='target datasets (delimited by space)'\n )\n parser.add_argument(\n '--transforms', type=str, nargs='+', help='data augmentation'\n )\n parser.add_argument(\n '--root', type=str, default='', help='path to data root'\n )\n parser.add_argument(\n 'opts',\n default=None,\n nargs=argparse.REMAINDER,\n help='Modify config options using the command-line'\n )\n args = parser.parse_args()\n\n cfg = get_default_config()\n cfg.use_gpu = torch.cuda.is_available()\n if args.config_file:\n cfg.merge_from_file(args.config_file)\n reset_config(cfg, args)\n cfg.merge_from_list(args.opts)\n set_random_seed(cfg.train.seed)\n\n log_name = 'test.log' if cfg.test.evaluate else 'train.log'\n log_name += time.strftime('-%Y-%m-%d-%H-%M-%S')\n sys.stdout = Logger(osp.join(cfg.data.save_dir, log_name))\n\n print('Show configuration\\n{}\\n'.format(cfg))\n print('Collecting env info ...')\n print('** System info **\\n{}\\n'.format(collect_env_info()))\n\n if cfg.use_gpu:\n torch.backends.cudnn.benchmark = True\n\n datamanager = torchreid.data.ImageDataManager(**imagedata_kwargs(cfg))\n\n print('Building model-1: {}'.format(cfg.model.name))\n model1 = torchreid.models.build_model(\n name=cfg.model.name,\n num_classes=datamanager.num_train_pids,\n loss=cfg.loss.name,\n pretrained=cfg.model.pretrained,\n use_gpu=cfg.use_gpu\n )\n num_params, flops = compute_model_complexity(\n model1, (1, 3, cfg.data.height, cfg.data.width)\n )\n print('Model complexity: params={:,} flops={:,}'.format(num_params, flops))\n\n print('Copying model-1 to model-2')\n model2 = copy.deepcopy(model1)\n\n if cfg.model.load_weights1 and check_isfile(cfg.model.load_weights1):\n load_pretrained_weights(model1, cfg.model.load_weights1)\n\n if cfg.model.load_weights2 and check_isfile(cfg.model.load_weights2):\n load_pretrained_weights(model2, cfg.model.load_weights2)\n\n if cfg.use_gpu:\n model1 = nn.DataParallel(model1).cuda()\n model2 = nn.DataParallel(model2).cuda()\n\n optimizer1 = torchreid.optim.build_optimizer(\n model1, **optimizer_kwargs(cfg)\n )\n scheduler1 = torchreid.optim.build_lr_scheduler(\n optimizer1, **lr_scheduler_kwargs(cfg)\n )\n\n optimizer2 = torchreid.optim.build_optimizer(\n model2, **optimizer_kwargs(cfg)\n )\n scheduler2 = torchreid.optim.build_lr_scheduler(\n optimizer2, **lr_scheduler_kwargs(cfg)\n )\n\n if cfg.model.resume1 and check_isfile(cfg.model.resume1):\n cfg.train.start_epoch = resume_from_checkpoint(\n cfg.model.resume1,\n model1,\n optimizer=optimizer1,\n scheduler=scheduler1\n )\n\n if cfg.model.resume2 and check_isfile(cfg.model.resume2):\n resume_from_checkpoint(\n cfg.model.resume2,\n model2,\n optimizer=optimizer2,\n scheduler=scheduler2\n )\n\n print('Building DML-engine for image-reid')\n engine = ImageDMLEngine(\n datamanager,\n model1,\n optimizer1,\n scheduler1,\n model2,\n optimizer2,\n scheduler2,\n margin=cfg.loss.triplet.margin,\n weight_t=cfg.loss.triplet.weight_t,\n weight_x=cfg.loss.triplet.weight_x,\n weight_ml=cfg.loss.dml.weight_ml,\n use_gpu=cfg.use_gpu,\n label_smooth=cfg.loss.softmax.label_smooth,\n deploy=cfg.model.deploy\n )\n engine.run(**engine_run_kwargs(cfg))\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/OSNet_AIN/README.md", "content": "# Differentiable NAS for OSNet-AIN\n\n## Introduction\nThis repository contains the neural architecture search (NAS) code (based on [Torchreid](https://arxiv.org/abs/1910.10093)) for [OSNet-AIN](https://arxiv.org/abs/1910.06827), an extension of [OSNet](https://arxiv.org/abs/1905.00953) that achieves strong performance on cross-domain person re-identification (re-ID) benchmarks (*without using any target data*). OSNet-AIN builds on the idea of using [instance normalisation](https://arxiv.org/abs/1607.08022) (IN) layers to eliminate instance-specific contrast in images for domain-generalisable representation learning. This is inspired by the [neural style transfer](https://arxiv.org/abs/1703.06868) works that use IN to remove image styles. Though IN naturally suits the cross-domain person re-ID task, it still remains unclear that where to insert IN to a re-ID CNN can maximise the performance gain. To avoid exhaustively evaluating all possible designs, OSNet-AIN learns to search for the optimal OSNet+IN design from data using a differentiable NAS algorithm. For technical details, please refer to our paper at https://arxiv.org/abs/1910.06827.\n\n
\n \n
\n\n## Training\nAssume the reid data is stored at `$DATA`. Run\n```\npython main.py --config-file nas.yaml --root $DATA\n```\n\nThe structure of the found architecture will be shown at the end of training.\n\nThe default config was designed for 8 Tesla V100 32GB GPUs. You can modify the batch size based on your device memory.\n\n**Note** that the test result obtained at the end of architecture search is not meaningful (due to the stochastic sampling layers). Therefore, do not rely on the result to judge the model performance. Instead, you should construct the found architecture in `osnet_child.py` and re-train and evaluate the model on the reid datasets.\n\n## Citation\nIf you find this code useful to your research, please consider citing the following papers.\n```\n@article{zhou2021osnet,\n title={Learning Generalisable Omni-Scale Representations for Person Re-Identification},\n author={Zhou, Kaiyang and Yang, Yongxin and Cavallaro, Andrea and Xiang, Tao},\n journal={TPAMI},\n year={2021}\n}\n\n@inproceedings{zhou2019osnet,\n title={Omni-Scale Feature Learning for Person Re-Identification},\n author={Zhou, Kaiyang and Yang, Yongxin and Cavallaro, Andrea and Xiang, Tao},\n booktitle={ICCV},\n year={2019}\n}\n```" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/OSNet_AIN/default_config.py", "content": "from yacs.config import CfgNode as CN\n\n\ndef get_default_config():\n cfg = CN()\n\n # model\n cfg.model = CN()\n cfg.model.name = 'resnet50'\n cfg.model.pretrained = True # automatically load pretrained model weights if available\n cfg.model.load_weights = '' # path to model weights\n cfg.model.resume = '' # path to checkpoint for resume training\n\n # NAS\n cfg.nas = CN()\n cfg.nas.mc_iter = 1 # Monte Carlo sampling\n cfg.nas.init_lmda = 10. # initial lambda value\n cfg.nas.min_lmda = 1. # minimum lambda value\n cfg.nas.lmda_decay_step = 20 # decay step for lambda\n cfg.nas.lmda_decay_rate = 0.5 # decay rate for lambda\n cfg.nas.fixed_lmda = False # keep lambda unchanged\n\n # data\n cfg.data = CN()\n cfg.data.type = 'image'\n cfg.data.root = 'reid-data'\n cfg.data.sources = ['market1501']\n cfg.data.targets = ['market1501']\n cfg.data.workers = 4 # number of data loading workers\n cfg.data.split_id = 0 # split index\n cfg.data.height = 256 # image height\n cfg.data.width = 128 # image width\n cfg.data.combineall = False # combine train, query and gallery for training\n cfg.data.transforms = ['random_flip'] # data augmentation\n cfg.data.norm_mean = [0.485, 0.456, 0.406] # default is imagenet mean\n cfg.data.norm_std = [0.229, 0.224, 0.225] # default is imagenet std\n cfg.data.save_dir = 'log' # path to save log\n\n # specific datasets\n cfg.market1501 = CN()\n cfg.market1501.use_500k_distractors = False # add 500k distractors to the gallery set for market1501\n cfg.cuhk03 = CN()\n cfg.cuhk03.labeled_images = False # use labeled images, if False, use detected images\n cfg.cuhk03.classic_split = False # use classic split by Li et al. CVPR14\n cfg.cuhk03.use_metric_cuhk03 = False # use cuhk03's metric for evaluation\n\n # sampler\n cfg.sampler = CN()\n cfg.sampler.train_sampler = 'RandomSampler'\n cfg.sampler.num_instances = 4 # number of instances per identity for RandomIdentitySampler\n\n # video reid setting\n cfg.video = CN()\n cfg.video.seq_len = 15 # number of images to sample in a tracklet\n cfg.video.sample_method = 'evenly' # how to sample images from a tracklet\n cfg.video.pooling_method = 'avg' # how to pool features over a tracklet\n\n # train\n cfg.train = CN()\n cfg.train.optim = 'adam'\n cfg.train.lr = 0.0003\n cfg.train.weight_decay = 5e-4\n cfg.train.max_epoch = 60\n cfg.train.start_epoch = 0\n cfg.train.batch_size = 32\n cfg.train.fixbase_epoch = 0 # number of epochs to fix base layers\n cfg.train.open_layers = [\n 'classifier'\n ] # layers for training while keeping others frozen\n cfg.train.staged_lr = False # set different lr to different layers\n cfg.train.new_layers = ['classifier'] # newly added layers with default lr\n cfg.train.base_lr_mult = 0.1 # learning rate multiplier for base layers\n cfg.train.lr_scheduler = 'single_step'\n cfg.train.stepsize = [20] # stepsize to decay learning rate\n cfg.train.gamma = 0.1 # learning rate decay multiplier\n cfg.train.print_freq = 20 # print frequency\n cfg.train.seed = 1 # random seed\n\n # optimizer\n cfg.sgd = CN()\n cfg.sgd.momentum = 0.9 # momentum factor for sgd and rmsprop\n cfg.sgd.dampening = 0. # dampening for momentum\n cfg.sgd.nesterov = False # Nesterov momentum\n cfg.rmsprop = CN()\n cfg.rmsprop.alpha = 0.99 # smoothing constant\n cfg.adam = CN()\n cfg.adam.beta1 = 0.9 # exponential decay rate for first moment\n cfg.adam.beta2 = 0.999 # exponential decay rate for second moment\n\n # loss\n cfg.loss = CN()\n cfg.loss.name = 'softmax'\n cfg.loss.softmax = CN()\n cfg.loss.softmax.label_smooth = True # use label smoothing regularizer\n cfg.loss.triplet = CN()\n cfg.loss.triplet.margin = 0.3 # distance margin\n cfg.loss.triplet.weight_t = 1. # weight to balance hard triplet loss\n cfg.loss.triplet.weight_x = 0. # weight to balance cross entropy loss\n\n # test\n cfg.test = CN()\n cfg.test.batch_size = 100\n cfg.test.dist_metric = 'euclidean' # distance metric, ['euclidean', 'cosine']\n cfg.test.normalize_feature = False # normalize feature vectors before computing distance\n cfg.test.ranks = [1, 5, 10, 20] # cmc ranks\n cfg.test.evaluate = False # test only\n cfg.test.eval_freq = -1 # evaluation frequency (-1 means to only test after training)\n cfg.test.start_eval = 0 # start to evaluate after a specific epoch\n cfg.test.rerank = False # use person re-ranking\n cfg.test.visrank = False # visualize ranked results (only available when cfg.test.evaluate=True)\n cfg.test.visrank_topk = 10 # top-k ranks to visualize\n cfg.test.visactmap = False # visualize CNN activation maps\n\n return cfg\n\n\ndef imagedata_kwargs(cfg):\n return {\n 'root': cfg.data.root,\n 'sources': cfg.data.sources,\n 'targets': cfg.data.targets,\n 'height': cfg.data.height,\n 'width': cfg.data.width,\n 'transforms': cfg.data.transforms,\n 'norm_mean': cfg.data.norm_mean,\n 'norm_std': cfg.data.norm_std,\n 'use_gpu': cfg.use_gpu,\n 'split_id': cfg.data.split_id,\n 'combineall': cfg.data.combineall,\n 'batch_size_train': cfg.train.batch_size,\n 'batch_size_test': cfg.test.batch_size,\n 'workers': cfg.data.workers,\n 'num_instances': cfg.sampler.num_instances,\n 'train_sampler': cfg.sampler.train_sampler,\n # image\n 'cuhk03_labeled': cfg.cuhk03.labeled_images,\n 'cuhk03_classic_split': cfg.cuhk03.classic_split,\n 'market1501_500k': cfg.market1501.use_500k_distractors,\n }\n\n\ndef videodata_kwargs(cfg):\n return {\n 'root': cfg.data.root,\n 'sources': cfg.data.sources,\n 'targets': cfg.data.targets,\n 'height': cfg.data.height,\n 'width': cfg.data.width,\n 'transforms': cfg.data.transforms,\n 'norm_mean': cfg.data.norm_mean,\n 'norm_std': cfg.data.norm_std,\n 'use_gpu': cfg.use_gpu,\n 'split_id': cfg.data.split_id,\n 'combineall': cfg.data.combineall,\n 'batch_size_train': cfg.train.batch_size,\n 'batch_size_test': cfg.test.batch_size,\n 'workers': cfg.data.workers,\n 'num_instances': cfg.sampler.num_instances,\n 'train_sampler': cfg.sampler.train_sampler,\n # video\n 'seq_len': cfg.video.seq_len,\n 'sample_method': cfg.video.sample_method\n }\n\n\ndef optimizer_kwargs(cfg):\n return {\n 'optim': cfg.train.optim,\n 'lr': cfg.train.lr,\n 'weight_decay': cfg.train.weight_decay,\n 'momentum': cfg.sgd.momentum,\n 'sgd_dampening': cfg.sgd.dampening,\n 'sgd_nesterov': cfg.sgd.nesterov,\n 'rmsprop_alpha': cfg.rmsprop.alpha,\n 'adam_beta1': cfg.adam.beta1,\n 'adam_beta2': cfg.adam.beta2,\n 'staged_lr': cfg.train.staged_lr,\n 'new_layers': cfg.train.new_layers,\n 'base_lr_mult': cfg.train.base_lr_mult\n }\n\n\ndef lr_scheduler_kwargs(cfg):\n return {\n 'lr_scheduler': cfg.train.lr_scheduler,\n 'stepsize': cfg.train.stepsize,\n 'gamma': cfg.train.gamma,\n 'max_epoch': cfg.train.max_epoch\n }\n\n\ndef engine_run_kwargs(cfg):\n return {\n 'save_dir': cfg.data.save_dir,\n 'max_epoch': cfg.train.max_epoch,\n 'start_epoch': cfg.train.start_epoch,\n 'fixbase_epoch': cfg.train.fixbase_epoch,\n 'open_layers': cfg.train.open_layers,\n 'start_eval': cfg.test.start_eval,\n 'eval_freq': cfg.test.eval_freq,\n 'test_only': cfg.test.evaluate,\n 'print_freq': cfg.train.print_freq,\n 'dist_metric': cfg.test.dist_metric,\n 'normalize_feature': cfg.test.normalize_feature,\n 'visrank': cfg.test.visrank,\n 'visrank_topk': cfg.test.visrank_topk,\n 'use_metric_cuhk03': cfg.cuhk03.use_metric_cuhk03,\n 'ranks': cfg.test.ranks,\n 'rerank': cfg.test.rerank\n }\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/OSNet_AIN/main.py", "content": "import os\nimport sys\nimport time\nimport os.path as osp\nimport argparse\nimport torch\nimport torch.nn as nn\n\nimport torchreid\nfrom torchreid.utils import (\n Logger, check_isfile, set_random_seed, collect_env_info,\n resume_from_checkpoint, compute_model_complexity\n)\n\nimport osnet_search as osnet_models\nfrom softmax_nas import ImageSoftmaxNASEngine\nfrom default_config import (\n imagedata_kwargs, optimizer_kwargs, engine_run_kwargs, get_default_config,\n lr_scheduler_kwargs\n)\n\n\ndef reset_config(cfg, args):\n if args.root:\n cfg.data.root = args.root\n if args.sources:\n cfg.data.sources = args.sources\n if args.targets:\n cfg.data.targets = args.targets\n if args.transforms:\n cfg.data.transforms = args.transforms\n\n\ndef main():\n parser = argparse.ArgumentParser(\n formatter_class=argparse.ArgumentDefaultsHelpFormatter\n )\n parser.add_argument(\n '--config-file', type=str, default='', help='path to config file'\n )\n parser.add_argument(\n '-s',\n '--sources',\n type=str,\n nargs='+',\n help='source datasets (delimited by space)'\n )\n parser.add_argument(\n '-t',\n '--targets',\n type=str,\n nargs='+',\n help='target datasets (delimited by space)'\n )\n parser.add_argument(\n '--transforms', type=str, nargs='+', help='data augmentation'\n )\n parser.add_argument(\n '--root', type=str, default='', help='path to data root'\n )\n parser.add_argument(\n '--gpu-devices',\n type=str,\n default='',\n )\n parser.add_argument(\n 'opts',\n default=None,\n nargs=argparse.REMAINDER,\n help='Modify config options using the command-line'\n )\n args = parser.parse_args()\n\n cfg = get_default_config()\n cfg.use_gpu = torch.cuda.is_available()\n if args.config_file:\n cfg.merge_from_file(args.config_file)\n reset_config(cfg, args)\n cfg.merge_from_list(args.opts)\n set_random_seed(cfg.train.seed)\n\n if cfg.use_gpu and args.gpu_devices:\n # if gpu_devices is not specified, all available gpus will be used\n os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices\n log_name = 'test.log' if cfg.test.evaluate else 'train.log'\n log_name += time.strftime('-%Y-%m-%d-%H-%M-%S')\n sys.stdout = Logger(osp.join(cfg.data.save_dir, log_name))\n\n print('Show configuration\\n{}\\n'.format(cfg))\n print('Collecting env info ...')\n print('** System info **\\n{}\\n'.format(collect_env_info()))\n\n if cfg.use_gpu:\n torch.backends.cudnn.benchmark = True\n\n datamanager = torchreid.data.ImageDataManager(**imagedata_kwargs(cfg))\n\n print('Building model: {}'.format(cfg.model.name))\n model = osnet_models.build_model(\n cfg.model.name, num_classes=datamanager.num_train_pids\n )\n num_params, flops = compute_model_complexity(\n model, (1, 3, cfg.data.height, cfg.data.width)\n )\n print('Model complexity: params={:,} flops={:,}'.format(num_params, flops))\n\n if cfg.use_gpu:\n model = nn.DataParallel(model).cuda()\n\n optimizer = torchreid.optim.build_optimizer(model, **optimizer_kwargs(cfg))\n scheduler = torchreid.optim.build_lr_scheduler(\n optimizer, **lr_scheduler_kwargs(cfg)\n )\n\n if cfg.model.resume and check_isfile(cfg.model.resume):\n cfg.train.start_epoch = resume_from_checkpoint(\n cfg.model.resume, model, optimizer=optimizer\n )\n\n print('Building NAS engine')\n engine = ImageSoftmaxNASEngine(\n datamanager,\n model,\n optimizer,\n scheduler=scheduler,\n use_gpu=cfg.use_gpu,\n label_smooth=cfg.loss.softmax.label_smooth,\n mc_iter=cfg.nas.mc_iter,\n init_lmda=cfg.nas.init_lmda,\n min_lmda=cfg.nas.min_lmda,\n lmda_decay_step=cfg.nas.lmda_decay_step,\n lmda_decay_rate=cfg.nas.lmda_decay_rate,\n fixed_lmda=cfg.nas.fixed_lmda\n )\n engine.run(**engine_run_kwargs(cfg))\n\n print('*** Display the found architecture ***')\n if cfg.use_gpu:\n model.module.build_child_graph()\n else:\n model.build_child_graph()\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/OSNet_AIN/nas.yaml", "content": "model:\n name: 'osnet_nas'\n pretrained: False\n\nnas:\n mc_iter: 1\n init_lmda: 10.\n min_lmda: 1.\n lmda_decay_step: 20\n lmda_decay_rate: 0.5\n fixed_lmda: False\n\ndata:\n type: 'image'\n sources: ['msmt17']\n targets: ['market1501']\n height: 256\n width: 128\n combineall: True\n transforms: ['random_flip', 'color_jitter']\n save_dir: 'log/osnet_nas'\n\nloss:\n name: 'softmax'\n softmax:\n label_smooth: True\n\ntrain:\n optim: 'sgd'\n lr: 0.1\n max_epoch: 120\n batch_size: 512\n fixbase_epoch: 0\n open_layers: ['classifier']\n lr_scheduler: 'cosine'\n\ntest:\n batch_size: 300\n dist_metric: 'cosine'\n normalize_feature: False\n evaluate: False\n eval_freq: -1\n rerank: False\n visactmap: False" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/OSNet_AIN/osnet_child.py", "content": "from __future__ import division, absolute_import\nfrom torch import nn\nfrom torch.nn import functional as F\n\n\n##########\n# Basic layers\n##########\nclass ConvLayer(nn.Module):\n \"\"\"Convolution layer (conv + bn + relu).\"\"\"\n\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n stride=1,\n padding=0,\n groups=1,\n IN=False\n ):\n super(ConvLayer, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n kernel_size,\n stride=stride,\n padding=padding,\n bias=False,\n groups=groups\n )\n if IN:\n self.bn = nn.InstanceNorm2d(out_channels, affine=True)\n else:\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass Conv1x1(nn.Module):\n \"\"\"1x1 convolution + bn + relu.\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, groups=1):\n super(Conv1x1, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n 1,\n stride=stride,\n padding=0,\n bias=False,\n groups=groups\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass Conv1x1Linear(nn.Module):\n \"\"\"1x1 convolution + bn (w/o non-linearity).\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, bn=True):\n super(Conv1x1Linear, self).__init__()\n self.conv = nn.Conv2d(\n in_channels, out_channels, 1, stride=stride, padding=0, bias=False\n )\n self.bn = None\n if bn:\n self.bn = nn.BatchNorm2d(out_channels)\n\n def forward(self, x):\n x = self.conv(x)\n if self.bn is not None:\n x = self.bn(x)\n return x\n\n\nclass Conv3x3(nn.Module):\n \"\"\"3x3 convolution + bn + relu.\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, groups=1):\n super(Conv3x3, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n 3,\n stride=stride,\n padding=1,\n bias=False,\n groups=groups\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass LightConv3x3(nn.Module):\n \"\"\"Lightweight 3x3 convolution.\n\n 1x1 (linear) + dw 3x3 (nonlinear).\n \"\"\"\n\n def __init__(self, in_channels, out_channels):\n super(LightConv3x3, self).__init__()\n self.conv1 = nn.Conv2d(\n in_channels, out_channels, 1, stride=1, padding=0, bias=False\n )\n self.conv2 = nn.Conv2d(\n out_channels,\n out_channels,\n 3,\n stride=1,\n padding=1,\n bias=False,\n groups=out_channels\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.conv2(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass LightConvStream(nn.Module):\n \"\"\"Lightweight convolution stream.\"\"\"\n\n def __init__(self, in_channels, out_channels, depth):\n super(LightConvStream, self).__init__()\n assert depth >= 1, 'depth must be equal to or larger than 1, but got {}'.format(\n depth\n )\n layers = []\n layers += [LightConv3x3(in_channels, out_channels)]\n for i in range(depth - 1):\n layers += [LightConv3x3(out_channels, out_channels)]\n self.layers = nn.Sequential(*layers)\n\n def forward(self, x):\n return self.layers(x)\n\n\n##########\n# Building blocks for omni-scale feature learning\n##########\nclass ChannelGate(nn.Module):\n \"\"\"A mini-network that generates channel-wise gates conditioned on input tensor.\"\"\"\n\n def __init__(\n self,\n in_channels,\n num_gates=None,\n return_gates=False,\n gate_activation='sigmoid',\n reduction=16,\n layer_norm=False\n ):\n super(ChannelGate, self).__init__()\n if num_gates is None:\n num_gates = in_channels\n self.return_gates = return_gates\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.fc1 = nn.Conv2d(\n in_channels,\n in_channels // reduction,\n kernel_size=1,\n bias=True,\n padding=0\n )\n self.norm1 = None\n if layer_norm:\n self.norm1 = nn.LayerNorm((in_channels // reduction, 1, 1))\n self.relu = nn.ReLU(inplace=True)\n self.fc2 = nn.Conv2d(\n in_channels // reduction,\n num_gates,\n kernel_size=1,\n bias=True,\n padding=0\n )\n if gate_activation == 'sigmoid':\n self.gate_activation = nn.Sigmoid()\n elif gate_activation == 'relu':\n self.gate_activation = nn.ReLU(inplace=True)\n elif gate_activation == 'linear':\n self.gate_activation = None\n else:\n raise RuntimeError(\n \"Unknown gate activation: {}\".format(gate_activation)\n )\n\n def forward(self, x):\n input = x\n x = self.global_avgpool(x)\n x = self.fc1(x)\n if self.norm1 is not None:\n x = self.norm1(x)\n x = self.relu(x)\n x = self.fc2(x)\n if self.gate_activation is not None:\n x = self.gate_activation(x)\n if self.return_gates:\n return x\n return input * x\n\n\nclass OSBlock(nn.Module):\n \"\"\"Omni-scale feature learning block.\"\"\"\n\n def __init__(self, in_channels, out_channels, reduction=4, T=4, **kwargs):\n super(OSBlock, self).__init__()\n assert T >= 1\n assert out_channels >= reduction and out_channels % reduction == 0\n mid_channels = out_channels // reduction\n\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2 = nn.ModuleList()\n for t in range(1, T + 1):\n self.conv2 += [LightConvStream(mid_channels, mid_channels, t)]\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2 = 0\n for conv2_t in self.conv2:\n x2_t = conv2_t(x1)\n x2 = x2 + self.gate(x2_t)\n x3 = self.conv3(x2)\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n return F.relu(out)\n\n\nclass OSBlockINv1(nn.Module):\n \"\"\"Omni-scale feature learning block with instance normalization.\"\"\"\n\n def __init__(self, in_channels, out_channels, reduction=4, T=4, **kwargs):\n super(OSBlockINv1, self).__init__()\n assert T >= 1\n assert out_channels >= reduction and out_channels % reduction == 0\n mid_channels = out_channels // reduction\n\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2 = nn.ModuleList()\n for t in range(1, T + 1):\n self.conv2 += [LightConvStream(mid_channels, mid_channels, t)]\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels, bn=False)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n self.IN = nn.InstanceNorm2d(out_channels, affine=True)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2 = 0\n for conv2_t in self.conv2:\n x2_t = conv2_t(x1)\n x2 = x2 + self.gate(x2_t)\n x3 = self.conv3(x2)\n x3 = self.IN(x3) # IN inside residual\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n return F.relu(out)\n\n\nclass OSBlockINv2(nn.Module):\n \"\"\"Omni-scale feature learning block with instance normalization.\"\"\"\n\n def __init__(self, in_channels, out_channels, reduction=4, T=4, **kwargs):\n super(OSBlockINv2, self).__init__()\n assert T >= 1\n assert out_channels >= reduction and out_channels % reduction == 0\n mid_channels = out_channels // reduction\n\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2 = nn.ModuleList()\n for t in range(1, T + 1):\n self.conv2 += [LightConvStream(mid_channels, mid_channels, t)]\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n self.IN = nn.InstanceNorm2d(out_channels, affine=True)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2 = 0\n for conv2_t in self.conv2:\n x2_t = conv2_t(x1)\n x2 = x2 + self.gate(x2_t)\n x3 = self.conv3(x2)\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n out = self.IN(out) # IN outside residual\n return F.relu(out)\n\n\nclass OSBlockINv3(nn.Module):\n \"\"\"Omni-scale feature learning block with instance normalization.\"\"\"\n\n def __init__(self, in_channels, out_channels, reduction=4, T=4, **kwargs):\n super(OSBlockINv3, self).__init__()\n assert T >= 1\n assert out_channels >= reduction and out_channels % reduction == 0\n mid_channels = out_channels // reduction\n\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2 = nn.ModuleList()\n for t in range(1, T + 1):\n self.conv2 += [LightConvStream(mid_channels, mid_channels, t)]\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels, bn=False)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n self.IN_in = nn.InstanceNorm2d(out_channels, affine=True)\n self.IN_out = nn.InstanceNorm2d(out_channels, affine=True)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2 = 0\n for conv2_t in self.conv2:\n x2_t = conv2_t(x1)\n x2 = x2 + self.gate(x2_t)\n x3 = self.conv3(x2)\n x3 = self.IN_in(x3) # IN inside residual\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n out = self.IN_out(out) # IN outside residual\n return F.relu(out)\n\n\n##########\n# Network architecture\n##########\nclass OSNet(nn.Module):\n \"\"\"Omni-Scale Network.\n \n Reference:\n - Zhou et al. Omni-Scale Feature Learning for Person Re-Identification. ICCV, 2019.\n - Zhou et al. Learning Generalisable Omni-Scale Representations\n for Person Re-Identification. TPAMI, 2021.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n blocks,\n layers,\n channels,\n feature_dim=512,\n loss='softmax',\n conv1_IN=True,\n **kwargs\n ):\n super(OSNet, self).__init__()\n num_blocks = len(blocks)\n assert num_blocks == len(layers)\n assert num_blocks == len(channels) - 1\n self.loss = loss\n self.feature_dim = feature_dim\n\n # convolutional backbone\n self.conv1 = ConvLayer(\n 3, channels[0], 7, stride=2, padding=3, IN=conv1_IN\n )\n self.maxpool = nn.MaxPool2d(3, stride=2, padding=1)\n self.conv2 = self._make_layer(\n blocks[0], layers[0], channels[0], channels[1]\n )\n self.pool2 = nn.Sequential(\n Conv1x1(channels[1], channels[1]), nn.AvgPool2d(2, stride=2)\n )\n self.conv3 = self._make_layer(\n blocks[1], layers[1], channels[1], channels[2]\n )\n self.pool3 = nn.Sequential(\n Conv1x1(channels[2], channels[2]), nn.AvgPool2d(2, stride=2)\n )\n self.conv4 = self._make_layer(\n blocks[2], layers[2], channels[2], channels[3]\n )\n self.conv5 = Conv1x1(channels[3], channels[3])\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n # fully connected layer\n self.fc = self._construct_fc_layer(\n self.feature_dim, channels[3], dropout_p=None\n )\n # identity classification layer\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n self._init_params()\n\n def _make_layer(self, blocks, layer, in_channels, out_channels):\n layers = []\n layers += [blocks[0](in_channels, out_channels)]\n for i in range(1, len(blocks)):\n layers += [blocks[i](out_channels, out_channels)]\n return nn.Sequential(*layers)\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n if fc_dims is None or fc_dims < 0:\n self.feature_dim = input_dim\n return None\n\n if isinstance(fc_dims, int):\n fc_dims = [fc_dims]\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.InstanceNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.maxpool(x)\n x = self.conv2(x)\n x = self.pool2(x)\n x = self.conv3(x)\n x = self.pool3(x)\n x = self.conv4(x)\n return self.conv5(x)\n\n def forward(self, x, return_featuremaps=False, **kwargs):\n x = self.featuremaps(x)\n if return_featuremaps:\n return x\n v = self.global_avgpool(x)\n v = v.view(v.size(0), -1)\n if self.fc is not None:\n v = self.fc(v)\n if not self.training:\n return v\n y = self.classifier(v)\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError(\"Unsupported loss: {}\".format(self.loss))\n\n\n##########\n# Instantiation\n##########\ndef osnet_ain_x1_0(\n num_classes=1000, pretrained=True, loss='softmax', **kwargs\n):\n model = OSNet(\n num_classes,\n blocks=[\n [OSBlockINv1, OSBlockINv1], [OSBlock, OSBlockINv1],\n [OSBlockINv1, OSBlock]\n ],\n layers=[2, 2, 2],\n channels=[64, 256, 384, 512],\n loss=loss,\n conv1_IN=True,\n **kwargs\n )\n return model\n\n\n__models = {'osnet_ain_x1_0': osnet_ain_x1_0}\n\n\ndef build_model(name, num_classes=100):\n avai_models = list(__models.keys())\n if name not in avai_models:\n raise KeyError(\n 'Unknown model: {}. Must be one of {}'.format(name, avai_models)\n )\n return __models[name](num_classes=num_classes)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/OSNet_AIN/osnet_search.py", "content": "from __future__ import division, absolute_import\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\nEPS = 1e-12\nNORM_AFFINE = False # enable affine transformations for normalization layer\n\n\n##########\n# Basic layers\n##########\nclass IBN(nn.Module):\n \"\"\"Instance + Batch Normalization.\"\"\"\n\n def __init__(self, num_channels):\n super(IBN, self).__init__()\n half1 = int(num_channels / 2)\n self.half = half1\n half2 = num_channels - half1\n self.IN = nn.InstanceNorm2d(half1, affine=NORM_AFFINE)\n self.BN = nn.BatchNorm2d(half2, affine=NORM_AFFINE)\n\n def forward(self, x):\n split = torch.split(x, self.half, 1)\n out1 = self.IN(split[0].contiguous())\n out2 = self.BN(split[1].contiguous())\n return torch.cat((out1, out2), 1)\n\n\nclass ConvLayer(nn.Module):\n \"\"\"Convolution layer (conv + bn + relu).\"\"\"\n\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n stride=1,\n padding=0,\n groups=1,\n IN=False\n ):\n super(ConvLayer, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n kernel_size,\n stride=stride,\n padding=padding,\n bias=False,\n groups=groups\n )\n if IN:\n self.bn = nn.InstanceNorm2d(out_channels, affine=NORM_AFFINE)\n else:\n self.bn = nn.BatchNorm2d(out_channels, affine=NORM_AFFINE)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass Conv1x1(nn.Module):\n \"\"\"1x1 convolution + bn + relu.\"\"\"\n\n def __init__(\n self, in_channels, out_channels, stride=1, groups=1, ibn=False\n ):\n super(Conv1x1, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n 1,\n stride=stride,\n padding=0,\n bias=False,\n groups=groups\n )\n if ibn:\n self.bn = IBN(out_channels)\n else:\n self.bn = nn.BatchNorm2d(out_channels, affine=NORM_AFFINE)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass Conv1x1Linear(nn.Module):\n \"\"\"1x1 convolution + bn (w/o non-linearity).\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, bn=True):\n super(Conv1x1Linear, self).__init__()\n self.conv = nn.Conv2d(\n in_channels, out_channels, 1, stride=stride, padding=0, bias=False\n )\n self.bn = None\n if bn:\n self.bn = nn.BatchNorm2d(out_channels, affine=NORM_AFFINE)\n\n def forward(self, x):\n x = self.conv(x)\n if self.bn is not None:\n x = self.bn(x)\n return x\n\n\nclass Conv3x3(nn.Module):\n \"\"\"3x3 convolution + bn + relu.\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, groups=1):\n super(Conv3x3, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n 3,\n stride=stride,\n padding=1,\n bias=False,\n groups=groups\n )\n self.bn = nn.BatchNorm2d(out_channels, affine=NORM_AFFINE)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass LightConv3x3(nn.Module):\n \"\"\"Lightweight 3x3 convolution.\n\n 1x1 (linear) + dw 3x3 (nonlinear).\n \"\"\"\n\n def __init__(self, in_channels, out_channels):\n super(LightConv3x3, self).__init__()\n self.conv1 = nn.Conv2d(\n in_channels, out_channels, 1, stride=1, padding=0, bias=False\n )\n self.conv2 = nn.Conv2d(\n out_channels,\n out_channels,\n 3,\n stride=1,\n padding=1,\n bias=False,\n groups=out_channels\n )\n self.bn = nn.BatchNorm2d(out_channels, affine=NORM_AFFINE)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.conv2(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass LightConvStream(nn.Module):\n \"\"\"Lightweight convolution stream.\"\"\"\n\n def __init__(self, in_channels, out_channels, depth):\n super(LightConvStream, self).__init__()\n assert depth >= 1, 'depth must be equal to or larger than 1, but got {}'.format(\n depth\n )\n layers = []\n layers += [LightConv3x3(in_channels, out_channels)]\n for i in range(depth - 1):\n layers += [LightConv3x3(out_channels, out_channels)]\n self.layers = nn.Sequential(*layers)\n\n def forward(self, x):\n return self.layers(x)\n\n\n##########\n# Building blocks for omni-scale feature learning\n##########\nclass ChannelGate(nn.Module):\n \"\"\"A mini-network that generates channel-wise gates conditioned on input tensor.\"\"\"\n\n def __init__(\n self,\n in_channels,\n num_gates=None,\n return_gates=False,\n gate_activation='sigmoid',\n reduction=16,\n layer_norm=False\n ):\n super(ChannelGate, self).__init__()\n if num_gates is None:\n num_gates = in_channels\n self.return_gates = return_gates\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.fc1 = nn.Conv2d(\n in_channels,\n in_channels // reduction,\n kernel_size=1,\n bias=True,\n padding=0\n )\n self.norm1 = None\n if layer_norm:\n self.norm1 = nn.LayerNorm((in_channels // reduction, 1, 1))\n self.relu = nn.ReLU(inplace=True)\n self.fc2 = nn.Conv2d(\n in_channels // reduction,\n num_gates,\n kernel_size=1,\n bias=True,\n padding=0\n )\n if gate_activation == 'sigmoid':\n self.gate_activation = nn.Sigmoid()\n elif gate_activation == 'relu':\n self.gate_activation = nn.ReLU(inplace=True)\n elif gate_activation == 'linear':\n self.gate_activation = None\n else:\n raise RuntimeError(\n \"Unknown gate activation: {}\".format(gate_activation)\n )\n\n def forward(self, x):\n input = x\n x = self.global_avgpool(x)\n x = self.fc1(x)\n if self.norm1 is not None:\n x = self.norm1(x)\n x = self.relu(x)\n x = self.fc2(x)\n if self.gate_activation is not None:\n x = self.gate_activation(x)\n if self.return_gates:\n return x\n return input * x\n\n\nclass OSBlock(nn.Module):\n \"\"\"Omni-scale feature learning block.\"\"\"\n\n def __init__(self, in_channels, out_channels, reduction=4, T=4, **kwargs):\n super(OSBlock, self).__init__()\n assert T >= 1\n assert out_channels >= reduction and out_channels % reduction == 0\n mid_channels = out_channels // reduction\n\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2 = nn.ModuleList()\n for t in range(1, T + 1):\n self.conv2 += [LightConvStream(mid_channels, mid_channels, t)]\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2 = 0\n for conv2_t in self.conv2:\n x2_t = conv2_t(x1)\n x2 = x2 + self.gate(x2_t)\n x3 = self.conv3(x2)\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n return F.relu(out)\n\n\nclass OSBlockINv1(nn.Module):\n \"\"\"Omni-scale feature learning block with instance normalization.\"\"\"\n\n def __init__(self, in_channels, out_channels, reduction=4, T=4, **kwargs):\n super(OSBlockINv1, self).__init__()\n assert T >= 1\n assert out_channels >= reduction and out_channels % reduction == 0\n mid_channels = out_channels // reduction\n\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2 = nn.ModuleList()\n for t in range(1, T + 1):\n self.conv2 += [LightConvStream(mid_channels, mid_channels, t)]\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels, bn=False)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n self.IN = nn.InstanceNorm2d(out_channels, affine=NORM_AFFINE)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2 = 0\n for conv2_t in self.conv2:\n x2_t = conv2_t(x1)\n x2 = x2 + self.gate(x2_t)\n x3 = self.conv3(x2)\n x3 = self.IN(x3) # IN inside residual\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n return F.relu(out)\n\n\nclass OSBlockINv2(nn.Module):\n \"\"\"Omni-scale feature learning block with instance normalization.\"\"\"\n\n def __init__(self, in_channels, out_channels, reduction=4, T=4, **kwargs):\n super(OSBlockINv2, self).__init__()\n assert T >= 1\n assert out_channels >= reduction and out_channels % reduction == 0\n mid_channels = out_channels // reduction\n\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2 = nn.ModuleList()\n for t in range(1, T + 1):\n self.conv2 += [LightConvStream(mid_channels, mid_channels, t)]\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n self.IN = nn.InstanceNorm2d(out_channels, affine=NORM_AFFINE)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2 = 0\n for conv2_t in self.conv2:\n x2_t = conv2_t(x1)\n x2 = x2 + self.gate(x2_t)\n x3 = self.conv3(x2)\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n out = self.IN(out) # IN outside residual\n return F.relu(out)\n\n\nclass OSBlockINv3(nn.Module):\n \"\"\"Omni-scale feature learning block with instance normalization.\"\"\"\n\n def __init__(self, in_channels, out_channels, reduction=4, T=4, **kwargs):\n super(OSBlockINv3, self).__init__()\n assert T >= 1\n assert out_channels >= reduction and out_channels % reduction == 0\n mid_channels = out_channels // reduction\n\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2 = nn.ModuleList()\n for t in range(1, T + 1):\n self.conv2 += [LightConvStream(mid_channels, mid_channels, t)]\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels, bn=False)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n self.IN_in = nn.InstanceNorm2d(out_channels, affine=NORM_AFFINE)\n self.IN_out = nn.InstanceNorm2d(out_channels, affine=NORM_AFFINE)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2 = 0\n for conv2_t in self.conv2:\n x2_t = conv2_t(x1)\n x2 = x2 + self.gate(x2_t)\n x3 = self.conv3(x2)\n x3 = self.IN_in(x3) # inside residual\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n out = self.IN_out(out) # IN outside residual\n return F.relu(out)\n\n\nclass NASBlock(nn.Module):\n \"\"\"Neural architecture search layer.\"\"\"\n\n def __init__(self, in_channels, out_channels, search_space=None):\n super(NASBlock, self).__init__()\n self._is_child_graph = False\n self.search_space = search_space\n if self.search_space is None:\n raise ValueError('search_space is None')\n\n self.os_block = nn.ModuleList()\n for block in self.search_space:\n self.os_block += [block(in_channels, out_channels)]\n self.weights = nn.Parameter(torch.ones(len(self.search_space)))\n\n def build_child_graph(self):\n if self._is_child_graph:\n raise RuntimeError('build_child_graph() can only be called once')\n\n idx = self.weights.data.max(dim=0)[1].item()\n self.os_block = self.os_block[idx]\n self.weights = None\n self._is_child_graph = True\n return self.search_space[idx]\n\n def forward(self, x, lmda=1.):\n if self._is_child_graph:\n return self.os_block(x)\n\n uniform = torch.rand_like(self.weights)\n gumbel = -torch.log(-torch.log(uniform + EPS))\n nonneg_weights = F.relu(self.weights)\n logits = torch.log(nonneg_weights + EPS) + gumbel\n exp = torch.exp(logits / lmda)\n weights_softmax = exp / (exp.sum() + EPS)\n\n output = 0\n for i, weight in enumerate(weights_softmax):\n output = output + weight * self.os_block[i](x)\n return output\n\n\n##########\n# Network architecture\n##########\nclass OSNet(nn.Module):\n \"\"\"Omni-Scale Network.\n \n Reference:\n - Zhou et al. Omni-Scale Feature Learning for Person Re-Identification. ICCV, 2019.\n - Zhou et al. Learning Generalisable Omni-Scale Representations\n for Person Re-Identification. TPAMI, 2021.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n blocks,\n layers,\n channels,\n feature_dim=512,\n loss='softmax',\n search_space=None,\n **kwargs\n ):\n super(OSNet, self).__init__()\n num_blocks = len(blocks)\n assert num_blocks == len(layers)\n assert num_blocks == len(channels) - 1\n # no matter what loss is specified, the model only returns the ID predictions\n self.loss = loss\n self.feature_dim = feature_dim\n\n # convolutional backbone\n self.conv1 = ConvLayer(3, channels[0], 7, stride=2, padding=3, IN=True)\n self.maxpool = nn.MaxPool2d(3, stride=2, padding=1)\n self.conv2 = self._make_layer(\n blocks[0], layers[0], channels[0], channels[1], search_space\n )\n self.pool2 = nn.Sequential(\n Conv1x1(channels[1], channels[1]), nn.AvgPool2d(2, stride=2)\n )\n self.conv3 = self._make_layer(\n blocks[1], layers[1], channels[1], channels[2], search_space\n )\n self.pool3 = nn.Sequential(\n Conv1x1(channels[2], channels[2]), nn.AvgPool2d(2, stride=2)\n )\n self.conv4 = self._make_layer(\n blocks[2], layers[2], channels[2], channels[3], search_space\n )\n self.conv5 = Conv1x1(channels[3], channels[3])\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n # fully connected layer\n self.fc = self._construct_fc_layer(\n self.feature_dim, channels[3], dropout_p=None\n )\n # identity classification layer\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n def _make_layer(\n self, block, layer, in_channels, out_channels, search_space\n ):\n layers = nn.ModuleList()\n layers += [block(in_channels, out_channels, search_space=search_space)]\n for i in range(1, layer):\n layers += [\n block(out_channels, out_channels, search_space=search_space)\n ]\n return layers\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n if fc_dims is None or fc_dims < 0:\n self.feature_dim = input_dim\n return None\n\n if isinstance(fc_dims, int):\n fc_dims = [fc_dims]\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim, affine=NORM_AFFINE))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def build_child_graph(self):\n print('Building child graph')\n for i, conv in enumerate(self.conv2):\n block = conv.build_child_graph()\n print('- conv2-{} Block={}'.format(i + 1, block.__name__))\n for i, conv in enumerate(self.conv3):\n block = conv.build_child_graph()\n print('- conv3-{} Block={}'.format(i + 1, block.__name__))\n for i, conv in enumerate(self.conv4):\n block = conv.build_child_graph()\n print('- conv4-{} Block={}'.format(i + 1, block.__name__))\n\n def featuremaps(self, x, lmda):\n x = self.conv1(x)\n x = self.maxpool(x)\n for conv in self.conv2:\n x = conv(x, lmda)\n x = self.pool2(x)\n for conv in self.conv3:\n x = conv(x, lmda)\n x = self.pool3(x)\n for conv in self.conv4:\n x = conv(x, lmda)\n return self.conv5(x)\n\n def forward(self, x, lmda=1., return_featuremaps=False):\n # lmda (float): temperature parameter for concrete distribution\n x = self.featuremaps(x, lmda)\n if return_featuremaps:\n return x\n v = self.global_avgpool(x)\n v = v.view(v.size(0), -1)\n if self.fc is not None:\n v = self.fc(v)\n if not self.training:\n return v\n return self.classifier(v)\n\n\n##########\n# Instantiation\n##########\ndef osnet_nas(num_classes=1000, loss='softmax', **kwargs):\n # standard size (width x1.0)\n return OSNet(\n num_classes,\n blocks=[NASBlock, NASBlock, NASBlock],\n layers=[2, 2, 2],\n channels=[64, 256, 384, 512],\n loss=loss,\n search_space=[OSBlock, OSBlockINv1, OSBlockINv2, OSBlockINv3],\n **kwargs\n )\n\n\n__NAS_models = {'osnet_nas': osnet_nas}\n\n\ndef build_model(name, num_classes=100):\n avai_models = list(__NAS_models.keys())\n if name not in avai_models:\n raise KeyError(\n 'Unknown model: {}. Must be one of {}'.format(name, avai_models)\n )\n return __NAS_models[name](num_classes=num_classes)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/OSNet_AIN/softmax_nas.py", "content": "from __future__ import division, print_function, absolute_import\n\nfrom torchreid import metrics\nfrom torchreid.engine import Engine\nfrom torchreid.losses import CrossEntropyLoss\n\n\nclass ImageSoftmaxNASEngine(Engine):\n\n def __init__(\n self,\n datamanager,\n model,\n optimizer,\n scheduler=None,\n use_gpu=False,\n label_smooth=True,\n mc_iter=1,\n init_lmda=1.,\n min_lmda=1.,\n lmda_decay_step=20,\n lmda_decay_rate=0.5,\n fixed_lmda=False\n ):\n super(ImageSoftmaxNASEngine, self).__init__(datamanager, use_gpu)\n self.mc_iter = mc_iter\n self.init_lmda = init_lmda\n self.min_lmda = min_lmda\n self.lmda_decay_step = lmda_decay_step\n self.lmda_decay_rate = lmda_decay_rate\n self.fixed_lmda = fixed_lmda\n\n self.model = model\n self.optimizer = optimizer\n self.scheduler = scheduler\n self.register_model('model', model, optimizer, scheduler)\n\n self.criterion = CrossEntropyLoss(\n num_classes=self.datamanager.num_train_pids,\n use_gpu=self.use_gpu,\n label_smooth=label_smooth\n )\n\n def forward_backward(self, data):\n imgs, pids = self.parse_data_for_train(data)\n\n if self.use_gpu:\n imgs = imgs.cuda()\n pids = pids.cuda()\n\n # softmax temporature\n if self.fixed_lmda or self.lmda_decay_step == -1:\n lmda = self.init_lmda\n else:\n lmda = self.init_lmda * self.lmda_decay_rate**(\n self.epoch // self.lmda_decay_step\n )\n if lmda < self.min_lmda:\n lmda = self.min_lmda\n\n for k in range(self.mc_iter):\n outputs = self.model(imgs, lmda=lmda)\n loss = self.compute_loss(self.criterion, outputs, pids)\n self.optimizer.zero_grad()\n loss.backward()\n self.optimizer.step()\n\n loss_dict = {\n 'loss': loss.item(),\n 'acc': metrics.accuracy(outputs, pids)[0].item()\n }\n\n return loss_dict\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/README.md", "content": "Here are some research projects built on [Torchreid](https://arxiv.org/abs/1910.10093).\n\n+ `OSNet_AIN`: [Learning Generalisable Omni-Scale Representations for Person Re-Identification](https://arxiv.org/abs/1910.06827)\n+ `DML`: [Deep Mutual Learning (CVPR'18)](https://arxiv.org/abs/1706.00384)\n+ `attribute_recognition`: [Omni-Scale Feature Learning for Person Re-Identification (ICCV'19)](https://arxiv.org/abs/1905.00953)" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/attribute_recognition/README.md", "content": "# Person Attribute Recognition\nThis code was developed for the experiment of person attribute recognition in [Omni-Scale Feature Learning for Person Re-Identification (ICCV'19)](https://arxiv.org/abs/1905.00953).\n\n## Download data\nDownload the PA-100K dataset from [https://github.com/xh-liu/HydraPlus-Net](https://github.com/xh-liu/HydraPlus-Net), and extract the file under the folder where you store your data (say $DATASET). The folder structure should look like\n```bash\n$DATASET/\n pa100k/\n data/ # images\n annotation/\n annotation.mat\n```\n\n## Train\nThe training command is provided in `train.sh`. Run `bash train.sh $DATASET` to start training.\n\n## Test\nTo test a pretrained model, add the following two arguments to `train.sh`: `--load-weights $PATH_TO_WEIGHTS --evaluate`." }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/attribute_recognition/datasets/__init__.py", "content": "from __future__ import division, print_function, absolute_import\n\nfrom .pa100k import PA100K\n\n__datasets = {'pa100k': PA100K}\n\n\ndef init_dataset(name, **kwargs):\n avai_datasets = list(__datasets.keys())\n if name not in avai_datasets:\n raise ValueError(\n 'Invalid dataset name. Received \"{}\", '\n 'but expected to be one of {}'.format(name, avai_datasets)\n )\n return __datasets[name](**kwargs)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/attribute_recognition/datasets/dataset.py", "content": "from __future__ import division, print_function, absolute_import\nimport os.path as osp\n\nfrom torchreid.utils import read_image\n\n\nclass Dataset(object):\n\n def __init__(\n self,\n train,\n val,\n test,\n attr_dict,\n transform=None,\n mode='train',\n verbose=True,\n **kwargs\n ):\n self.train = train\n self.val = val\n self.test = test\n self._attr_dict = attr_dict\n self._num_attrs = len(self.attr_dict)\n self.transform = transform\n\n if mode == 'train':\n self.data = self.train\n elif mode == 'val':\n self.data = self.val\n else:\n self.data = self.test\n\n if verbose:\n self.show_summary()\n\n @property\n def num_attrs(self):\n return self._num_attrs\n\n @property\n def attr_dict(self):\n return self._attr_dict\n\n def __len__(self):\n return len(self.data)\n\n def __getitem__(self, index):\n img_path, attrs = self.data[index]\n img = read_image(img_path)\n if self.transform is not None:\n img = self.transform(img)\n return img, attrs, img_path\n\n def check_before_run(self, required_files):\n \"\"\"Checks if required files exist before going deeper.\n Args:\n required_files (str or list): string file name(s).\n \"\"\"\n if isinstance(required_files, str):\n required_files = [required_files]\n\n for fpath in required_files:\n if not osp.exists(fpath):\n raise RuntimeError('\"{}\" is not found'.format(fpath))\n\n def show_summary(self):\n num_train = len(self.train)\n num_val = len(self.val)\n num_test = len(self.test)\n num_total = num_train + num_val + num_test\n\n print('=> Loaded {}'.format(self.__class__.__name__))\n print(\" ------------------------------\")\n print(\" subset | # images\")\n print(\" ------------------------------\")\n print(\" train | {:8d}\".format(num_train))\n print(\" val | {:8d}\".format(num_val))\n print(\" test | {:8d}\".format(num_test))\n print(\" ------------------------------\")\n print(\" total | {:8d}\".format(num_total))\n print(\" ------------------------------\")\n print(\" # attributes: {}\".format(len(self.attr_dict)))\n print(\" attributes:\")\n for label, attr in self.attr_dict.items():\n print(' {:3d}: {}'.format(label, attr))\n print(\" ------------------------------\")\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/attribute_recognition/datasets/pa100k.py", "content": "from __future__ import division, print_function, absolute_import\nimport numpy as np\nimport os.path as osp\nfrom scipy.io import loadmat\n\nfrom .dataset import Dataset\n\n\nclass PA100K(Dataset):\n \"\"\"Pedestrian attribute dataset.\n\n 80k training images + 20k test images.\n\n The folder structure should be:\n pa100k/\n data/ # images\n annotation/\n annotation.mat\n \"\"\"\n dataset_dir = 'pa100k'\n\n def __init__(self, root='', **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.data_dir = osp.join(self.dataset_dir, 'data')\n self.anno_mat_path = osp.join(\n self.dataset_dir, 'annotation', 'annotation.mat'\n )\n\n required_files = [self.data_dir, self.anno_mat_path]\n self.check_before_run(required_files)\n\n train, val, test, attr_dict = self.extract_data()\n super(PA100K, self).__init__(train, val, test, attr_dict, **kwargs)\n\n def extract_data(self):\n # anno_mat is a dictionary with keys: ['test_images_name', 'val_images_name',\n # 'train_images_name', 'val_label', 'attributes', 'test_label', 'train_label']\n anno_mat = loadmat(self.anno_mat_path)\n\n def _extract(key_name, key_label):\n names = anno_mat[key_name]\n labels = anno_mat[key_label]\n num_imgs = names.shape[0]\n data = []\n for i in range(num_imgs):\n name = names[i, 0][0]\n attrs = labels[i, :].astype(np.float32)\n img_path = osp.join(self.data_dir, name)\n data.append((img_path, attrs))\n return data\n\n train = _extract('train_images_name', 'train_label')\n val = _extract('val_images_name', 'val_label')\n test = _extract('test_images_name', 'test_label')\n attrs = anno_mat['attributes']\n attr_dict = {i: str(attr[0][0]) for i, attr in enumerate(attrs)}\n\n return train, val, test, attr_dict\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/attribute_recognition/default_parser.py", "content": "from __future__ import print_function, absolute_import\nimport argparse\n\n\ndef init_parser():\n parser = argparse.ArgumentParser(\n formatter_class=argparse.ArgumentDefaultsHelpFormatter\n )\n\n # ************************************************************\n # Datasets\n # ************************************************************\n parser.add_argument(\n '--root',\n type=str,\n default='',\n required=True,\n help='root path to data directory'\n )\n parser.add_argument(\n '-d',\n '--dataset',\n type=str,\n required=True,\n help='which dataset to choose'\n )\n parser.add_argument(\n '-j',\n '--workers',\n type=int,\n default=4,\n help='number of data loading workers (tips: 4 or 8 times number of gpus)'\n )\n parser.add_argument(\n '--height', type=int, default=256, help='height of an image'\n )\n parser.add_argument(\n '--width', type=int, default=128, help='width of an image'\n )\n\n # ************************************************************\n # Optimization options\n # ************************************************************\n parser.add_argument(\n '--optim',\n type=str,\n default='adam',\n help='optimization algorithm (see optimizers.py)'\n )\n parser.add_argument(\n '--lr', type=float, default=0.0003, help='initial learning rate'\n )\n parser.add_argument(\n '--weight-decay', type=float, default=5e-04, help='weight decay'\n )\n # sgd\n parser.add_argument(\n '--momentum',\n type=float,\n default=0.9,\n help='momentum factor for sgd and rmsprop'\n )\n parser.add_argument(\n '--sgd-dampening',\n type=float,\n default=0,\n help='sgd\\'s dampening for momentum'\n )\n parser.add_argument(\n '--sgd-nesterov',\n action='store_true',\n help='whether to enable sgd\\'s Nesterov momentum'\n )\n # rmsprop\n parser.add_argument(\n '--rmsprop-alpha',\n type=float,\n default=0.99,\n help='rmsprop\\'s smoothing constant'\n )\n # adam/amsgrad\n parser.add_argument(\n '--adam-beta1',\n type=float,\n default=0.9,\n help='exponential decay rate for adam\\'s first moment'\n )\n parser.add_argument(\n '--adam-beta2',\n type=float,\n default=0.999,\n help='exponential decay rate for adam\\'s second moment'\n )\n\n # ************************************************************\n # Training hyperparameters\n # ************************************************************\n parser.add_argument(\n '--max-epoch', type=int, default=60, help='maximum epochs to run'\n )\n parser.add_argument(\n '--start-epoch',\n type=int,\n default=0,\n help='manual epoch number (useful when restart)'\n )\n parser.add_argument(\n '--batch-size', type=int, default=32, help='batch size'\n )\n\n parser.add_argument(\n '--fixbase-epoch',\n type=int,\n default=0,\n help='number of epochs to fix base layers'\n )\n parser.add_argument(\n '--open-layers',\n type=str,\n nargs='+',\n default=['classifier'],\n help='open specified layers for training while keeping others frozen'\n )\n\n parser.add_argument(\n '--staged-lr',\n action='store_true',\n help='set different lr to different layers'\n )\n parser.add_argument(\n '--new-layers',\n type=str,\n nargs='+',\n default=['classifier'],\n help='newly added layers with default lr'\n )\n parser.add_argument(\n '--base-lr-mult',\n type=float,\n default=0.1,\n help='learning rate multiplier for base layers'\n )\n\n # ************************************************************\n # Learning rate scheduler options\n # ************************************************************\n parser.add_argument(\n '--lr-scheduler',\n type=str,\n default='multi_step',\n help='learning rate scheduler (see lr_schedulers.py)'\n )\n parser.add_argument(\n '--stepsize',\n type=int,\n default=[20, 40],\n nargs='+',\n help='stepsize to decay learning rate'\n )\n parser.add_argument(\n '--gamma', type=float, default=0.1, help='learning rate decay'\n )\n\n # ************************************************************\n # Architecture\n # ************************************************************\n parser.add_argument(\n '-a', '--arch', type=str, default='', help='model architecture'\n )\n parser.add_argument(\n '--no-pretrained',\n action='store_true',\n help='do not load pretrained weights'\n )\n\n # ************************************************************\n # Loss\n # ************************************************************\n parser.add_argument(\n '--weighted-bce', action='store_true', help='use weighted BCELoss'\n )\n\n # ************************************************************\n # Test settings\n # ************************************************************\n parser.add_argument(\n '--load-weights', type=str, default='', help='load pretrained weights'\n )\n parser.add_argument(\n '--evaluate', action='store_true', help='evaluate only'\n )\n parser.add_argument(\n '--save-prediction', action='store_true', help='save prediction'\n )\n\n # ************************************************************\n # Miscs\n # ************************************************************\n parser.add_argument(\n '--print-freq', type=int, default=20, help='print frequency'\n )\n parser.add_argument('--seed', type=int, default=1, help='manual seed')\n parser.add_argument(\n '--resume',\n type=str,\n default='',\n metavar='PATH',\n help='resume from a checkpoint'\n )\n parser.add_argument(\n '--save-dir',\n type=str,\n default='log',\n help='path to save log and model weights'\n )\n parser.add_argument('--use-cpu', action='store_true', help='use cpu')\n\n return parser\n\n\ndef optimizer_kwargs(parsed_args):\n return {\n 'optim': parsed_args.optim,\n 'lr': parsed_args.lr,\n 'weight_decay': parsed_args.weight_decay,\n 'momentum': parsed_args.momentum,\n 'sgd_dampening': parsed_args.sgd_dampening,\n 'sgd_nesterov': parsed_args.sgd_nesterov,\n 'rmsprop_alpha': parsed_args.rmsprop_alpha,\n 'adam_beta1': parsed_args.adam_beta1,\n 'adam_beta2': parsed_args.adam_beta2,\n 'staged_lr': parsed_args.staged_lr,\n 'new_layers': parsed_args.new_layers,\n 'base_lr_mult': parsed_args.base_lr_mult\n }\n\n\ndef lr_scheduler_kwargs(parsed_args):\n return {\n 'lr_scheduler': parsed_args.lr_scheduler,\n 'stepsize': parsed_args.stepsize,\n 'gamma': parsed_args.gamma\n }\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/attribute_recognition/main.py", "content": "from __future__ import division, print_function\nimport sys\nimport copy\nimport time\nimport numpy as np\nimport os.path as osp\nimport datetime\nimport warnings\nimport torch\nimport torch.nn as nn\n\nimport torchreid\nfrom torchreid.utils import (\n Logger, AverageMeter, check_isfile, open_all_layers, save_checkpoint,\n set_random_seed, collect_env_info, open_specified_layers,\n load_pretrained_weights, compute_model_complexity\n)\nfrom torchreid.data.transforms import (\n Resize, Compose, ToTensor, Normalize, Random2DTranslation,\n RandomHorizontalFlip\n)\n\nimport models\nimport datasets\nfrom default_parser import init_parser, optimizer_kwargs, lr_scheduler_kwargs\n\nparser = init_parser()\nargs = parser.parse_args()\n\n\ndef init_dataset(use_gpu):\n normalize = Normalize(\n mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]\n )\n\n transform_tr = Compose(\n [\n Random2DTranslation(args.height, args.width, p=0.5),\n RandomHorizontalFlip(),\n ToTensor(), normalize\n ]\n )\n\n transform_te = Compose(\n [Resize([args.height, args.width]),\n ToTensor(), normalize]\n )\n\n trainset = datasets.init_dataset(\n args.dataset,\n root=args.root,\n transform=transform_tr,\n mode='train',\n verbose=True\n )\n\n valset = datasets.init_dataset(\n args.dataset,\n root=args.root,\n transform=transform_te,\n mode='val',\n verbose=False\n )\n\n testset = datasets.init_dataset(\n args.dataset,\n root=args.root,\n transform=transform_te,\n mode='test',\n verbose=False\n )\n\n num_attrs = trainset.num_attrs\n attr_dict = trainset.attr_dict\n\n trainloader = torch.utils.data.DataLoader(\n trainset,\n batch_size=args.batch_size,\n shuffle=True,\n num_workers=args.workers,\n pin_memory=use_gpu,\n drop_last=True\n )\n\n valloader = torch.utils.data.DataLoader(\n valset,\n batch_size=args.batch_size,\n shuffle=False,\n num_workers=args.workers,\n pin_memory=use_gpu,\n drop_last=False\n )\n\n testloader = torch.utils.data.DataLoader(\n testset,\n batch_size=args.batch_size,\n shuffle=False,\n num_workers=args.workers,\n pin_memory=use_gpu,\n drop_last=False\n )\n\n return trainloader, valloader, testloader, num_attrs, attr_dict\n\n\ndef main():\n global args\n\n set_random_seed(args.seed)\n use_gpu = torch.cuda.is_available() and not args.use_cpu\n log_name = 'test.log' if args.evaluate else 'train.log'\n sys.stdout = Logger(osp.join(args.save_dir, log_name))\n\n print('** Arguments **')\n arg_keys = list(args.__dict__.keys())\n arg_keys.sort()\n for key in arg_keys:\n print('{}: {}'.format(key, args.__dict__[key]))\n print('\\n')\n print('Collecting env info ...')\n print('** System info **\\n{}\\n'.format(collect_env_info()))\n\n if use_gpu:\n torch.backends.cudnn.benchmark = True\n else:\n warnings.warn(\n 'Currently using CPU, however, GPU is highly recommended'\n )\n\n dataset_vars = init_dataset(use_gpu)\n trainloader, valloader, testloader, num_attrs, attr_dict = dataset_vars\n\n if args.weighted_bce:\n print('Use weighted binary cross entropy')\n print('Computing the weights ...')\n bce_weights = torch.zeros(num_attrs, dtype=torch.float)\n for _, attrs, _ in trainloader:\n bce_weights += attrs.sum(0) # sum along the batch dim\n bce_weights /= len(trainloader) * args.batch_size\n print('Sample ratio for each attribute: {}'.format(bce_weights))\n bce_weights = torch.exp(-1 * bce_weights)\n print('BCE weights: {}'.format(bce_weights))\n bce_weights = bce_weights.expand(args.batch_size, num_attrs)\n criterion = nn.BCEWithLogitsLoss(weight=bce_weights)\n\n else:\n print('Use plain binary cross entropy')\n criterion = nn.BCEWithLogitsLoss()\n\n print('Building model: {}'.format(args.arch))\n model = models.build_model(\n args.arch,\n num_attrs,\n pretrained=not args.no_pretrained,\n use_gpu=use_gpu\n )\n num_params, flops = compute_model_complexity(\n model, (1, 3, args.height, args.width)\n )\n print('Model complexity: params={:,} flops={:,}'.format(num_params, flops))\n\n if args.load_weights and check_isfile(args.load_weights):\n load_pretrained_weights(model, args.load_weights)\n\n if use_gpu:\n model = nn.DataParallel(model).cuda()\n criterion = criterion.cuda()\n\n if args.evaluate:\n test(model, testloader, attr_dict, use_gpu)\n return\n\n optimizer = torchreid.optim.build_optimizer(\n model, **optimizer_kwargs(args)\n )\n scheduler = torchreid.optim.build_lr_scheduler(\n optimizer, **lr_scheduler_kwargs(args)\n )\n\n start_epoch = args.start_epoch\n best_result = -np.inf\n if args.resume and check_isfile(args.resume):\n checkpoint = torch.load(args.resume)\n model.load_state_dict(checkpoint['state_dict'])\n optimizer.load_state_dict(checkpoint['optimizer'])\n start_epoch = checkpoint['epoch']\n best_result = checkpoint['label_mA']\n print('Loaded checkpoint from \"{}\"'.format(args.resume))\n print('- start epoch: {}'.format(start_epoch))\n print('- label_mA: {}'.format(best_result))\n\n time_start = time.time()\n\n for epoch in range(start_epoch, args.max_epoch):\n train(\n epoch, model, criterion, optimizer, scheduler, trainloader, use_gpu\n )\n test_outputs = test(model, testloader, attr_dict, use_gpu)\n label_mA = test_outputs[0]\n is_best = label_mA > best_result\n if is_best:\n best_result = label_mA\n\n save_checkpoint(\n {\n 'state_dict': model.state_dict(),\n 'epoch': epoch + 1,\n 'label_mA': label_mA,\n 'optimizer': optimizer.state_dict(),\n },\n args.save_dir,\n is_best=is_best\n )\n\n elapsed = round(time.time() - time_start)\n elapsed = str(datetime.timedelta(seconds=elapsed))\n print('Elapsed {}'.format(elapsed))\n\n\ndef train(epoch, model, criterion, optimizer, scheduler, trainloader, use_gpu):\n losses = AverageMeter()\n batch_time = AverageMeter()\n data_time = AverageMeter()\n model.train()\n\n if (epoch + 1) <= args.fixbase_epoch and args.open_layers is not None:\n print(\n '* Only train {} (epoch: {}/{})'.format(\n args.open_layers, epoch + 1, args.fixbase_epoch\n )\n )\n open_specified_layers(model, args.open_layers)\n else:\n open_all_layers(model)\n\n end = time.time()\n for batch_idx, data in enumerate(trainloader):\n data_time.update(time.time() - end)\n\n imgs, attrs = data[0], data[1]\n if use_gpu:\n imgs = imgs.cuda()\n attrs = attrs.cuda()\n\n optimizer.zero_grad()\n outputs = model(imgs)\n loss = criterion(outputs, attrs)\n loss.backward()\n optimizer.step()\n\n batch_time.update(time.time() - end)\n\n losses.update(loss.item(), imgs.size(0))\n\n if (batch_idx+1) % args.print_freq == 0:\n # estimate remaining time\n num_batches = len(trainloader)\n eta_seconds = batch_time.avg * (\n num_batches - (batch_idx+1) + (args.max_epoch -\n (epoch+1)) * num_batches\n )\n eta_str = str(datetime.timedelta(seconds=int(eta_seconds)))\n print(\n 'Epoch: [{0}/{1}][{2}/{3}]\\t'\n 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n 'Data {data_time.val:.3f} ({data_time.avg:.3f})\\t'\n 'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n 'Lr {lr:.6f}\\t'\n 'Eta {eta}'.format(\n epoch + 1,\n args.max_epoch,\n batch_idx + 1,\n len(trainloader),\n batch_time=batch_time,\n data_time=data_time,\n loss=losses,\n lr=optimizer.param_groups[0]['lr'],\n eta=eta_str\n )\n )\n\n end = time.time()\n\n scheduler.step()\n\n\n@torch.no_grad()\ndef test(model, testloader, attr_dict, use_gpu):\n batch_time = AverageMeter()\n model.eval()\n\n num_persons = 0\n prob_thre = 0.5\n ins_acc = 0\n ins_prec = 0\n ins_rec = 0\n mA_history = {\n 'correct_pos': 0,\n 'real_pos': 0,\n 'correct_neg': 0,\n 'real_neg': 0\n }\n\n print('Testing ...')\n\n for batch_idx, data in enumerate(testloader):\n imgs, attrs, img_paths = data\n if use_gpu:\n imgs = imgs.cuda()\n\n end = time.time()\n orig_outputs = model(imgs)\n batch_time.update(time.time() - end)\n\n orig_outputs = orig_outputs.data.cpu().numpy()\n attrs = attrs.data.numpy()\n\n # transform raw outputs to attributes (binary codes)\n outputs = copy.deepcopy(orig_outputs)\n outputs[outputs < prob_thre] = 0\n outputs[outputs >= prob_thre] = 1\n\n # compute label-based metric\n overlaps = outputs * attrs\n mA_history['correct_pos'] += overlaps.sum(0)\n mA_history['real_pos'] += attrs.sum(0)\n inv_overlaps = (1-outputs) * (1-attrs)\n mA_history['correct_neg'] += inv_overlaps.sum(0)\n mA_history['real_neg'] += (1 - attrs).sum(0)\n\n outputs = outputs.astype(bool)\n attrs = attrs.astype(bool)\n\n # compute instabce-based accuracy\n intersect = (outputs & attrs).astype(float)\n union = (outputs | attrs).astype(float)\n ins_acc += (intersect.sum(1) / union.sum(1)).sum()\n ins_prec += (intersect.sum(1) / outputs.astype(float).sum(1)).sum()\n ins_rec += (intersect.sum(1) / attrs.astype(float).sum(1)).sum()\n\n num_persons += imgs.size(0)\n\n if (batch_idx+1) % args.print_freq == 0:\n print(\n 'Processed batch {}/{}'.format(batch_idx + 1, len(testloader))\n )\n\n if args.save_prediction:\n txtfile = open(osp.join(args.save_dir, 'prediction.txt'), 'a')\n for idx in range(imgs.size(0)):\n img_path = img_paths[idx]\n probs = orig_outputs[idx, :]\n labels = attrs[idx, :]\n txtfile.write('{}\\n'.format(img_path))\n txtfile.write('*** Correct prediction ***\\n')\n for attr_idx, (label, prob) in enumerate(zip(labels, probs)):\n if label:\n attr_name = attr_dict[attr_idx]\n info = '{}: {:.1%} '.format(attr_name, prob)\n txtfile.write(info)\n txtfile.write('\\n*** Incorrect prediction ***\\n')\n for attr_idx, (label, prob) in enumerate(zip(labels, probs)):\n if not label and prob > 0.5:\n attr_name = attr_dict[attr_idx]\n info = '{}: {:.1%} '.format(attr_name, prob)\n txtfile.write(info)\n txtfile.write('\\n\\n')\n txtfile.close()\n\n print(\n '=> BatchTime(s)/BatchSize(img): {:.4f}/{}'.format(\n batch_time.avg, args.batch_size\n )\n )\n\n ins_acc /= num_persons\n ins_prec /= num_persons\n ins_rec /= num_persons\n ins_f1 = (2*ins_prec*ins_rec) / (ins_prec+ins_rec)\n\n term1 = mA_history['correct_pos'] / mA_history['real_pos']\n term2 = mA_history['correct_neg'] / mA_history['real_neg']\n label_mA_verbose = (term1+term2) * 0.5\n label_mA = label_mA_verbose.mean()\n\n print('* Results *')\n print(' # test persons: {}'.format(num_persons))\n print(' (instance-based) accuracy: {:.1%}'.format(ins_acc))\n print(' (instance-based) precition: {:.1%}'.format(ins_prec))\n print(' (instance-based) recall: {:.1%}'.format(ins_rec))\n print(' (instance-based) f1-score: {:.1%}'.format(ins_f1))\n print(' (label-based) mean accuracy: {:.1%}'.format(label_mA))\n print(' mA for each attribute: {}'.format(label_mA_verbose))\n\n return label_mA, ins_acc, ins_prec, ins_rec, ins_f1\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/attribute_recognition/models/__init__.py", "content": "from __future__ import absolute_import\n\nfrom .osnet import *\n\n__model_factory = {\n 'osnet_avgpool': osnet_avgpool,\n 'osnet_maxpool': osnet_maxpool\n}\n\n\ndef build_model(name, num_classes, pretrained=True, use_gpu=True):\n avai_models = list(__model_factory.keys())\n if name not in avai_models:\n raise KeyError\n return __model_factory[name](\n num_classes=num_classes, pretrained=pretrained, use_gpu=use_gpu\n )\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/attribute_recognition/models/osnet.py", "content": "from __future__ import division, absolute_import\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\n__all__ = ['osnet_avgpool', 'osnet_maxpool']\n\n\n##########\n# Basic layers\n##########\nclass ConvLayer(nn.Module):\n \"\"\"Convolution layer.\"\"\"\n\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n stride=1,\n padding=0,\n groups=1\n ):\n super(ConvLayer, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n kernel_size,\n stride=stride,\n padding=padding,\n bias=False,\n groups=groups\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n x = self.relu(x)\n return x\n\n\nclass Conv1x1(nn.Module):\n \"\"\"1x1 convolution.\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, groups=1):\n super(Conv1x1, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n 1,\n stride=stride,\n padding=0,\n bias=False,\n groups=groups\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n x = self.relu(x)\n return x\n\n\nclass Conv1x1Linear(nn.Module):\n \"\"\"1x1 convolution without non-linearity.\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1):\n super(Conv1x1Linear, self).__init__()\n self.conv = nn.Conv2d(\n in_channels, out_channels, 1, stride=stride, padding=0, bias=False\n )\n self.bn = nn.BatchNorm2d(out_channels)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return x\n\n\nclass Conv3x3(nn.Module):\n \"\"\"3x3 convolution.\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, groups=1):\n super(Conv3x3, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n 3,\n stride=stride,\n padding=1,\n bias=False,\n groups=groups\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n x = self.relu(x)\n return x\n\n\nclass LightConv3x3(nn.Module):\n \"\"\"Lightweight 3x3 convolution.\n\n 1x1 (linear) + dw 3x3 (nonlinear).\n \"\"\"\n\n def __init__(self, in_channels, out_channels):\n super(LightConv3x3, self).__init__()\n self.conv1 = nn.Conv2d(\n in_channels, out_channels, 1, stride=1, padding=0, bias=False\n )\n self.conv2 = nn.Conv2d(\n out_channels,\n out_channels,\n 3,\n stride=1,\n padding=1,\n bias=False,\n groups=out_channels\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.conv2(x)\n x = self.bn(x)\n x = self.relu(x)\n return x\n\n\n##########\n# Building blocks for omni-scale feature learning\n##########\nclass ChannelGate(nn.Module):\n \"\"\"A mini-network that generates channel-wise gates conditioned on input.\"\"\"\n\n def __init__(\n self,\n in_channels,\n num_gates=None,\n return_gates=False,\n gate_activation='sigmoid',\n reduction=16,\n layer_norm=False\n ):\n super(ChannelGate, self).__init__()\n if num_gates is None:\n num_gates = in_channels\n self.return_gates = return_gates\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.fc1 = nn.Conv2d(\n in_channels,\n in_channels // reduction,\n kernel_size=1,\n bias=True,\n padding=0\n )\n self.norm1 = None\n if layer_norm:\n self.norm1 = nn.LayerNorm((in_channels // reduction, 1, 1))\n self.relu = nn.ReLU(inplace=True)\n self.fc2 = nn.Conv2d(\n in_channels // reduction,\n num_gates,\n kernel_size=1,\n bias=True,\n padding=0\n )\n if gate_activation == 'sigmoid':\n self.gate_activation = nn.Sigmoid()\n elif gate_activation == 'relu':\n self.gate_activation = nn.ReLU(inplace=True)\n elif gate_activation == 'linear':\n self.gate_activation = None\n else:\n raise RuntimeError(\n \"Unknown gate activation: {}\".format(gate_activation)\n )\n\n def forward(self, x):\n input = x\n x = self.global_avgpool(x)\n x = self.fc1(x)\n if self.norm1 is not None:\n x = self.norm1(x)\n x = self.relu(x)\n x = self.fc2(x)\n if self.gate_activation is not None:\n x = self.gate_activation(x)\n if self.return_gates:\n return x\n return input * x\n\n\nclass OSBlock(nn.Module):\n \"\"\"Omni-scale feature learning block.\"\"\"\n\n def __init__(self, in_channels, out_channels, **kwargs):\n super(OSBlock, self).__init__()\n mid_channels = out_channels // 4\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2a = LightConv3x3(mid_channels, mid_channels)\n self.conv2b = nn.Sequential(\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n )\n self.conv2c = nn.Sequential(\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n )\n self.conv2d = nn.Sequential(\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n )\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n\n def forward(self, x):\n residual = x\n x1 = self.conv1(x)\n x2a = self.conv2a(x1)\n x2b = self.conv2b(x1)\n x2c = self.conv2c(x1)\n x2d = self.conv2d(x1)\n x2 = self.gate(x2a) + self.gate(x2b) + self.gate(x2c) + self.gate(x2d)\n x3 = self.conv3(x2)\n if self.downsample is not None:\n residual = self.downsample(residual)\n out = x3 + residual\n return F.relu(out)\n\n\n##########\n# Network architecture\n##########\nclass BaseNet(nn.Module):\n\n def _make_layer(\n self, block, layer, in_channels, out_channels, reduce_spatial_size\n ):\n layers = []\n\n layers.append(block(in_channels, out_channels))\n for i in range(1, layer):\n layers.append(block(out_channels, out_channels))\n\n if reduce_spatial_size:\n layers.append(\n nn.Sequential(\n Conv1x1(out_channels, out_channels),\n nn.AvgPool2d(2, stride=2)\n )\n )\n\n return nn.Sequential(*layers)\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n if fc_dims is None or fc_dims < 0:\n self.feature_dim = input_dim\n return None\n\n if isinstance(fc_dims, int):\n fc_dims = [fc_dims]\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n\nclass OSNet(BaseNet):\n\n def __init__(\n self,\n num_classes,\n blocks,\n layers,\n channels,\n feature_dim=512,\n loss='softmax',\n pool='avg',\n **kwargs\n ):\n super(OSNet, self).__init__()\n num_blocks = len(blocks)\n assert num_blocks == len(layers)\n assert num_blocks == len(channels) - 1\n self.loss = loss\n\n # convolutional backbone\n self.conv1 = ConvLayer(3, channels[0], 7, stride=2, padding=3)\n self.maxpool = nn.MaxPool2d(3, stride=2, padding=1)\n self.conv2 = self._make_layer(\n blocks[0],\n layers[0],\n channels[0],\n channels[1],\n reduce_spatial_size=True\n )\n self.conv3 = self._make_layer(\n blocks[1],\n layers[1],\n channels[1],\n channels[2],\n reduce_spatial_size=True\n )\n self.conv4 = self._make_layer(\n blocks[2],\n layers[2],\n channels[2],\n channels[3],\n reduce_spatial_size=False\n )\n self.conv5 = Conv1x1(channels[3], channels[3])\n if pool == 'avg':\n self.global_pool = nn.AdaptiveAvgPool2d(1)\n else:\n self.global_pool = nn.AdaptiveMaxPool2d(1)\n # fully connected layer\n self.fc = self._construct_fc_layer(\n feature_dim, channels[3], dropout_p=None\n )\n # classification layer\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n self.init_params()\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.maxpool(x)\n x = self.conv2(x)\n x = self.conv3(x)\n x = self.conv4(x)\n x = self.conv5(x)\n return x\n\n def forward(self, x):\n x = self.featuremaps(x)\n v = self.global_pool(x)\n v = v.view(v.size(0), -1)\n if self.fc is not None:\n v = self.fc(v)\n y = self.classifier(v)\n if not self.training:\n y = torch.sigmoid(y)\n return y\n\n\ndef osnet_avgpool(num_classes=1000, loss='softmax', **kwargs):\n return OSNet(\n num_classes,\n blocks=[OSBlock, OSBlock, OSBlock],\n layers=[2, 2, 2],\n channels=[64, 256, 384, 512],\n loss=loss,\n pool='avg',\n **kwargs\n )\n\n\ndef osnet_maxpool(num_classes=1000, loss='softmax', **kwargs):\n return OSNet(\n num_classes,\n blocks=[OSBlock, OSBlock, OSBlock],\n layers=[2, 2, 2],\n channels=[64, 256, 384, 512],\n loss=loss,\n pool='max',\n **kwargs\n )\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/projects/attribute_recognition/train.sh", "content": "#!/bin/bash\n\n# DATASET points to the directory containing pa100k/\nDATASET=$1\n\npython main.py \\\n--root ${DATASET} \\\n-d pa100k \\\n-a osnet_maxpool \\\n--max-epoch 50 \\\n--stepsize 30 40 \\\n--batch-size 32 \\\n--lr 0.065 \\\n--optim sgd \\\n--weighted-bce \\\n--save-dir log/pa100k-osnet_maxpool" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/scripts/default_config.py", "content": "from yacs.config import CfgNode as CN\n\n\ndef get_default_config():\n cfg = CN()\n\n # model\n cfg.model = CN()\n cfg.model.name = 'resnet50'\n cfg.model.pretrained = True # automatically load pretrained model weights if available\n cfg.model.load_weights = '' # path to model weights\n cfg.model.resume = '' # path to checkpoint for resume training\n\n # data\n cfg.data = CN()\n cfg.data.type = 'image'\n cfg.data.root = 'reid-data'\n cfg.data.sources = ['market1501']\n cfg.data.targets = ['market1501']\n cfg.data.workers = 4 # number of data loading workers\n cfg.data.split_id = 0 # split index\n cfg.data.height = 256 # image height\n cfg.data.width = 128 # image width\n cfg.data.combineall = False # combine train, query and gallery for training\n cfg.data.transforms = ['random_flip'] # data augmentation\n cfg.data.k_tfm = 1 # number of times to apply augmentation to an image independently\n cfg.data.norm_mean = [0.485, 0.456, 0.406] # default is imagenet mean\n cfg.data.norm_std = [0.229, 0.224, 0.225] # default is imagenet std\n cfg.data.save_dir = 'log' # path to save log\n cfg.data.load_train_targets = False # load training set from target dataset\n\n # specific datasets\n cfg.market1501 = CN()\n cfg.market1501.use_500k_distractors = False # add 500k distractors to the gallery set for market1501\n cfg.cuhk03 = CN()\n cfg.cuhk03.labeled_images = False # use labeled images, if False, use detected images\n cfg.cuhk03.classic_split = False # use classic split by Li et al. CVPR14\n cfg.cuhk03.use_metric_cuhk03 = False # use cuhk03's metric for evaluation\n\n # sampler\n cfg.sampler = CN()\n cfg.sampler.train_sampler = 'RandomSampler' # sampler for source train loader\n cfg.sampler.train_sampler_t = 'RandomSampler' # sampler for target train loader\n cfg.sampler.num_instances = 4 # number of instances per identity for RandomIdentitySampler\n cfg.sampler.num_cams = 1 # number of cameras to sample in a batch (for RandomDomainSampler)\n cfg.sampler.num_datasets = 1 # number of datasets to sample in a batch (for RandomDatasetSampler)\n\n # video reid setting\n cfg.video = CN()\n cfg.video.seq_len = 15 # number of images to sample in a tracklet\n cfg.video.sample_method = 'evenly' # how to sample images from a tracklet\n cfg.video.pooling_method = 'avg' # how to pool features over a tracklet\n\n # train\n cfg.train = CN()\n cfg.train.optim = 'adam'\n cfg.train.lr = 0.0003\n cfg.train.weight_decay = 5e-4\n cfg.train.max_epoch = 60\n cfg.train.start_epoch = 0\n cfg.train.batch_size = 32\n cfg.train.fixbase_epoch = 0 # number of epochs to fix base layers\n cfg.train.open_layers = [\n 'classifier'\n ] # layers for training while keeping others frozen\n cfg.train.staged_lr = False # set different lr to different layers\n cfg.train.new_layers = ['classifier'] # newly added layers with default lr\n cfg.train.base_lr_mult = 0.1 # learning rate multiplier for base layers\n cfg.train.lr_scheduler = 'single_step'\n cfg.train.stepsize = [20] # stepsize to decay learning rate\n cfg.train.gamma = 0.1 # learning rate decay multiplier\n cfg.train.print_freq = 20 # print frequency\n cfg.train.seed = 1 # random seed\n\n # optimizer\n cfg.sgd = CN()\n cfg.sgd.momentum = 0.9 # momentum factor for sgd and rmsprop\n cfg.sgd.dampening = 0. # dampening for momentum\n cfg.sgd.nesterov = False # Nesterov momentum\n cfg.rmsprop = CN()\n cfg.rmsprop.alpha = 0.99 # smoothing constant\n cfg.adam = CN()\n cfg.adam.beta1 = 0.9 # exponential decay rate for first moment\n cfg.adam.beta2 = 0.999 # exponential decay rate for second moment\n\n # loss\n cfg.loss = CN()\n cfg.loss.name = 'softmax'\n cfg.loss.softmax = CN()\n cfg.loss.softmax.label_smooth = True # use label smoothing regularizer\n cfg.loss.triplet = CN()\n cfg.loss.triplet.margin = 0.3 # distance margin\n cfg.loss.triplet.weight_t = 1. # weight to balance hard triplet loss\n cfg.loss.triplet.weight_x = 0. # weight to balance cross entropy loss\n\n # test\n cfg.test = CN()\n cfg.test.batch_size = 100\n cfg.test.dist_metric = 'euclidean' # distance metric, ['euclidean', 'cosine']\n cfg.test.normalize_feature = False # normalize feature vectors before computing distance\n cfg.test.ranks = [1, 5, 10, 20] # cmc ranks\n cfg.test.evaluate = False # test only\n cfg.test.eval_freq = -1 # evaluation frequency (-1 means to only test after training)\n cfg.test.start_eval = 0 # start to evaluate after a specific epoch\n cfg.test.rerank = False # use person re-ranking\n cfg.test.visrank = False # visualize ranked results (only available when cfg.test.evaluate=True)\n cfg.test.visrank_topk = 10 # top-k ranks to visualize\n\n return cfg\n\n\ndef imagedata_kwargs(cfg):\n return {\n 'root': cfg.data.root,\n 'sources': cfg.data.sources,\n 'targets': cfg.data.targets,\n 'height': cfg.data.height,\n 'width': cfg.data.width,\n 'transforms': cfg.data.transforms,\n 'k_tfm': cfg.data.k_tfm,\n 'norm_mean': cfg.data.norm_mean,\n 'norm_std': cfg.data.norm_std,\n 'use_gpu': cfg.use_gpu,\n 'split_id': cfg.data.split_id,\n 'combineall': cfg.data.combineall,\n 'load_train_targets': cfg.data.load_train_targets,\n 'batch_size_train': cfg.train.batch_size,\n 'batch_size_test': cfg.test.batch_size,\n 'workers': cfg.data.workers,\n 'num_instances': cfg.sampler.num_instances,\n 'num_cams': cfg.sampler.num_cams,\n 'num_datasets': cfg.sampler.num_datasets,\n 'train_sampler': cfg.sampler.train_sampler,\n 'train_sampler_t': cfg.sampler.train_sampler_t,\n # image dataset specific\n 'cuhk03_labeled': cfg.cuhk03.labeled_images,\n 'cuhk03_classic_split': cfg.cuhk03.classic_split,\n 'market1501_500k': cfg.market1501.use_500k_distractors,\n }\n\n\ndef videodata_kwargs(cfg):\n return {\n 'root': cfg.data.root,\n 'sources': cfg.data.sources,\n 'targets': cfg.data.targets,\n 'height': cfg.data.height,\n 'width': cfg.data.width,\n 'transforms': cfg.data.transforms,\n 'norm_mean': cfg.data.norm_mean,\n 'norm_std': cfg.data.norm_std,\n 'use_gpu': cfg.use_gpu,\n 'split_id': cfg.data.split_id,\n 'combineall': cfg.data.combineall,\n 'batch_size_train': cfg.train.batch_size,\n 'batch_size_test': cfg.test.batch_size,\n 'workers': cfg.data.workers,\n 'num_instances': cfg.sampler.num_instances,\n 'num_cams': cfg.sampler.num_cams,\n 'num_datasets': cfg.sampler.num_datasets,\n 'train_sampler': cfg.sampler.train_sampler,\n # video dataset specific\n 'seq_len': cfg.video.seq_len,\n 'sample_method': cfg.video.sample_method\n }\n\n\ndef optimizer_kwargs(cfg):\n return {\n 'optim': cfg.train.optim,\n 'lr': cfg.train.lr,\n 'weight_decay': cfg.train.weight_decay,\n 'momentum': cfg.sgd.momentum,\n 'sgd_dampening': cfg.sgd.dampening,\n 'sgd_nesterov': cfg.sgd.nesterov,\n 'rmsprop_alpha': cfg.rmsprop.alpha,\n 'adam_beta1': cfg.adam.beta1,\n 'adam_beta2': cfg.adam.beta2,\n 'staged_lr': cfg.train.staged_lr,\n 'new_layers': cfg.train.new_layers,\n 'base_lr_mult': cfg.train.base_lr_mult\n }\n\n\ndef lr_scheduler_kwargs(cfg):\n return {\n 'lr_scheduler': cfg.train.lr_scheduler,\n 'stepsize': cfg.train.stepsize,\n 'gamma': cfg.train.gamma,\n 'max_epoch': cfg.train.max_epoch\n }\n\n\ndef engine_run_kwargs(cfg):\n return {\n 'save_dir': cfg.data.save_dir,\n 'max_epoch': cfg.train.max_epoch,\n 'start_epoch': cfg.train.start_epoch,\n 'fixbase_epoch': cfg.train.fixbase_epoch,\n 'open_layers': cfg.train.open_layers,\n 'start_eval': cfg.test.start_eval,\n 'eval_freq': cfg.test.eval_freq,\n 'test_only': cfg.test.evaluate,\n 'print_freq': cfg.train.print_freq,\n 'dist_metric': cfg.test.dist_metric,\n 'normalize_feature': cfg.test.normalize_feature,\n 'visrank': cfg.test.visrank,\n 'visrank_topk': cfg.test.visrank_topk,\n 'use_metric_cuhk03': cfg.cuhk03.use_metric_cuhk03,\n 'ranks': cfg.test.ranks,\n 'rerank': cfg.test.rerank\n }\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/scripts/main.py", "content": "import sys\nimport time\nimport os.path as osp\nimport argparse\nimport torch\nimport torch.nn as nn\n\nimport torchreid\nfrom torchreid.utils import (\n Logger, check_isfile, set_random_seed, collect_env_info,\n resume_from_checkpoint, load_pretrained_weights, compute_model_complexity\n)\n\nfrom default_config import (\n imagedata_kwargs, optimizer_kwargs, videodata_kwargs, engine_run_kwargs,\n get_default_config, lr_scheduler_kwargs\n)\n\n\ndef build_datamanager(cfg):\n if cfg.data.type == 'image':\n return torchreid.data.ImageDataManager(**imagedata_kwargs(cfg))\n else:\n return torchreid.data.VideoDataManager(**videodata_kwargs(cfg))\n\n\ndef build_engine(cfg, datamanager, model, optimizer, scheduler):\n if cfg.data.type == 'image':\n if cfg.loss.name == 'softmax':\n engine = torchreid.engine.ImageSoftmaxEngine(\n datamanager,\n model,\n optimizer=optimizer,\n scheduler=scheduler,\n use_gpu=cfg.use_gpu,\n label_smooth=cfg.loss.softmax.label_smooth\n )\n\n else:\n engine = torchreid.engine.ImageTripletEngine(\n datamanager,\n model,\n optimizer=optimizer,\n margin=cfg.loss.triplet.margin,\n weight_t=cfg.loss.triplet.weight_t,\n weight_x=cfg.loss.triplet.weight_x,\n scheduler=scheduler,\n use_gpu=cfg.use_gpu,\n label_smooth=cfg.loss.softmax.label_smooth\n )\n\n else:\n if cfg.loss.name == 'softmax':\n engine = torchreid.engine.VideoSoftmaxEngine(\n datamanager,\n model,\n optimizer=optimizer,\n scheduler=scheduler,\n use_gpu=cfg.use_gpu,\n label_smooth=cfg.loss.softmax.label_smooth,\n pooling_method=cfg.video.pooling_method\n )\n\n else:\n engine = torchreid.engine.VideoTripletEngine(\n datamanager,\n model,\n optimizer=optimizer,\n margin=cfg.loss.triplet.margin,\n weight_t=cfg.loss.triplet.weight_t,\n weight_x=cfg.loss.triplet.weight_x,\n scheduler=scheduler,\n use_gpu=cfg.use_gpu,\n label_smooth=cfg.loss.softmax.label_smooth\n )\n\n return engine\n\n\ndef reset_config(cfg, args):\n if args.root:\n cfg.data.root = args.root\n if args.sources:\n cfg.data.sources = args.sources\n if args.targets:\n cfg.data.targets = args.targets\n if args.transforms:\n cfg.data.transforms = args.transforms\n\n\ndef check_cfg(cfg):\n if cfg.loss.name == 'triplet' and cfg.loss.triplet.weight_x == 0:\n assert cfg.train.fixbase_epoch == 0, \\\n 'The output of classifier is not included in the computational graph'\n\n\ndef main():\n parser = argparse.ArgumentParser(\n formatter_class=argparse.ArgumentDefaultsHelpFormatter\n )\n parser.add_argument(\n '--config-file', type=str, default='', help='path to config file'\n )\n parser.add_argument(\n '-s',\n '--sources',\n type=str,\n nargs='+',\n help='source datasets (delimited by space)'\n )\n parser.add_argument(\n '-t',\n '--targets',\n type=str,\n nargs='+',\n help='target datasets (delimited by space)'\n )\n parser.add_argument(\n '--transforms', type=str, nargs='+', help='data augmentation'\n )\n parser.add_argument(\n '--root', type=str, default='', help='path to data root'\n )\n parser.add_argument(\n 'opts',\n default=None,\n nargs=argparse.REMAINDER,\n help='Modify config options using the command-line'\n )\n args = parser.parse_args()\n\n cfg = get_default_config()\n cfg.use_gpu = torch.cuda.is_available()\n if args.config_file:\n cfg.merge_from_file(args.config_file)\n reset_config(cfg, args)\n cfg.merge_from_list(args.opts)\n set_random_seed(cfg.train.seed)\n check_cfg(cfg)\n\n log_name = 'test.log' if cfg.test.evaluate else 'train.log'\n log_name += time.strftime('-%Y-%m-%d-%H-%M-%S')\n sys.stdout = Logger(osp.join(cfg.data.save_dir, log_name))\n\n print('Show configuration\\n{}\\n'.format(cfg))\n print('Collecting env info ...')\n print('** System info **\\n{}\\n'.format(collect_env_info()))\n\n if cfg.use_gpu:\n torch.backends.cudnn.benchmark = True\n\n datamanager = build_datamanager(cfg)\n\n print('Building model: {}'.format(cfg.model.name))\n model = torchreid.models.build_model(\n name=cfg.model.name,\n num_classes=datamanager.num_train_pids,\n loss=cfg.loss.name,\n pretrained=cfg.model.pretrained,\n use_gpu=cfg.use_gpu\n )\n num_params, flops = compute_model_complexity(\n model, (1, 3, cfg.data.height, cfg.data.width)\n )\n print('Model complexity: params={:,} flops={:,}'.format(num_params, flops))\n\n if cfg.model.load_weights and check_isfile(cfg.model.load_weights):\n load_pretrained_weights(model, cfg.model.load_weights)\n\n if cfg.use_gpu:\n model = nn.DataParallel(model).cuda()\n\n optimizer = torchreid.optim.build_optimizer(model, **optimizer_kwargs(cfg))\n scheduler = torchreid.optim.build_lr_scheduler(\n optimizer, **lr_scheduler_kwargs(cfg)\n )\n\n if cfg.model.resume and check_isfile(cfg.model.resume):\n cfg.train.start_epoch = resume_from_checkpoint(\n cfg.model.resume, model, optimizer=optimizer, scheduler=scheduler\n )\n\n print(\n 'Building {}-engine for {}-reid'.format(cfg.loss.name, cfg.data.type)\n )\n engine = build_engine(cfg, datamanager, model, optimizer, scheduler)\n engine.run(**engine_run_kwargs(cfg))\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/setup.py", "content": "import numpy as np\nimport os.path as osp\nfrom setuptools import setup, find_packages\nfrom distutils.extension import Extension\nfrom Cython.Build import cythonize\n\n\ndef readme():\n with open('README.rst') as f:\n content = f.read()\n return content\n\n\ndef find_version():\n version_file = 'torchreid/__init__.py'\n with open(version_file, 'r') as f:\n exec(compile(f.read(), version_file, 'exec'))\n return locals()['__version__']\n\n\ndef numpy_include():\n try:\n numpy_include = np.get_include()\n except AttributeError:\n numpy_include = np.get_numpy_include()\n return numpy_include\n\n\next_modules = [\n Extension(\n 'torchreid.metrics.rank_cylib.rank_cy',\n ['torchreid/metrics/rank_cylib/rank_cy.pyx'],\n include_dirs=[numpy_include()],\n )\n]\n\n\ndef get_requirements(filename='requirements.txt'):\n here = osp.dirname(osp.realpath(__file__))\n with open(osp.join(here, filename), 'r') as f:\n requires = [line.replace('\\n', '') for line in f.readlines()]\n return requires\n\n\nsetup(\n name='torchreid',\n version=find_version(),\n description='A library for deep learning person re-ID in PyTorch',\n author='Kaiyang Zhou',\n license='MIT',\n long_description=readme(),\n url='https://github.com/KaiyangZhou/deep-person-reid',\n packages=find_packages(),\n install_requires=get_requirements(),\n keywords=['Person Re-Identification', 'Deep Learning', 'Computer Vision'],\n ext_modules=cythonize(ext_modules)\n)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/tools/compute_mean_std.py", "content": "\"\"\"\nCompute channel-wise mean and standard deviation of a dataset.\n\nUsage:\n$ python compute_mean_std.py DATASET_ROOT DATASET_KEY\n\n- The first argument points to the root path where you put the datasets.\n- The second argument means the specific dataset key.\n\nFor instance, your datasets are put under $DATA and you wanna\ncompute the statistics of Market1501, do\n$ python compute_mean_std.py $DATA market1501\n\"\"\"\nimport argparse\n\nimport torchreid\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument('root', type=str)\n parser.add_argument('sources', type=str)\n args = parser.parse_args()\n\n datamanager = torchreid.data.ImageDataManager(\n root=args.root,\n sources=args.sources,\n targets=None,\n height=256,\n width=128,\n batch_size_train=100,\n batch_size_test=100,\n transforms=None,\n norm_mean=[0., 0., 0.],\n norm_std=[1., 1., 1.],\n train_sampler='SequentialSampler'\n )\n train_loader = datamanager.train_loader\n\n print('Computing mean and std ...')\n mean = 0.\n std = 0.\n n_samples = 0.\n for data in train_loader:\n data = data['img']\n batch_size = data.size(0)\n data = data.view(batch_size, data.size(1), -1)\n mean += data.mean(2).sum(0)\n std += data.std(2).sum(0)\n n_samples += batch_size\n\n mean /= n_samples\n std /= n_samples\n print('Mean: {}'.format(mean))\n print('Std: {}'.format(std))\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/tools/parse_test_res.py", "content": "\"\"\"\nThis script aims to automate the process of calculating average results\nstored in the test.log files over multiple splits.\n\nHow to use:\nFor example, you have done evaluation over 20 splits on VIPeR, leading to\nthe following file structure\n\nlog/\n eval_viper/\n split_0/\n test.log-xxxx\n split_1/\n test.log-xxxx\n split_2/\n test.log-xxxx\n ...\n\nYou can run the following command in your terminal to get the average performance:\n$ python tools/parse_test_res.py log/eval_viper\n\"\"\"\nimport os\nimport re\nimport glob\nimport numpy as np\nimport argparse\nfrom collections import defaultdict\n\nfrom torchreid.utils import check_isfile, listdir_nohidden\n\n\ndef parse_file(filepath, regex_mAP, regex_r1, regex_r5, regex_r10, regex_r20):\n results = {}\n\n with open(filepath, 'r') as f:\n lines = f.readlines()\n\n for line in lines:\n line = line.strip()\n\n match_mAP = regex_mAP.search(line)\n if match_mAP:\n mAP = float(match_mAP.group(1))\n results['mAP'] = mAP\n\n match_r1 = regex_r1.search(line)\n if match_r1:\n r1 = float(match_r1.group(1))\n results['r1'] = r1\n\n match_r5 = regex_r5.search(line)\n if match_r5:\n r5 = float(match_r5.group(1))\n results['r5'] = r5\n\n match_r10 = regex_r10.search(line)\n if match_r10:\n r10 = float(match_r10.group(1))\n results['r10'] = r10\n\n match_r20 = regex_r20.search(line)\n if match_r20:\n r20 = float(match_r20.group(1))\n results['r20'] = r20\n\n return results\n\n\ndef main(args):\n regex_mAP = re.compile(r'mAP: ([\\.\\deE+-]+)%')\n regex_r1 = re.compile(r'Rank-1 : ([\\.\\deE+-]+)%')\n regex_r5 = re.compile(r'Rank-5 : ([\\.\\deE+-]+)%')\n regex_r10 = re.compile(r'Rank-10 : ([\\.\\deE+-]+)%')\n regex_r20 = re.compile(r'Rank-20 : ([\\.\\deE+-]+)%')\n\n final_res = defaultdict(list)\n\n directories = listdir_nohidden(args.directory, sort=True)\n num_dirs = len(directories)\n for directory in directories:\n fullpath = os.path.join(args.directory, directory)\n filepath = glob.glob(os.path.join(fullpath, 'test.log*'))[0]\n check_isfile(filepath)\n print(f'Parsing {filepath}')\n res = parse_file(\n filepath, regex_mAP, regex_r1, regex_r5, regex_r10, regex_r20\n )\n for key, value in res.items():\n final_res[key].append(value)\n\n print('Finished parsing')\n print(f'The average results over {num_dirs} splits are shown below')\n\n for key, values in final_res.items():\n mean_val = np.mean(values)\n print(f'{key}: {mean_val:.1f}')\n\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument('directory', type=str, help='Path to directory')\n args = parser.parse_args()\n main(args)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/tools/visualize_actmap.py", "content": "\"\"\"Visualizes CNN activation maps to see where the CNN focuses on to extract features.\n\nReference:\n - Zagoruyko and Komodakis. Paying more attention to attention: Improving the\n performance of convolutional neural networks via attention transfer. ICLR, 2017\n - Zhou et al. Omni-Scale Feature Learning for Person Re-Identification. ICCV, 2019.\n\"\"\"\nimport numpy as np\nimport os.path as osp\nimport argparse\nimport cv2\nimport torch\nfrom torch.nn import functional as F\n\nimport torchreid\nfrom torchreid.utils import (\n check_isfile, mkdir_if_missing, load_pretrained_weights\n)\n\nIMAGENET_MEAN = [0.485, 0.456, 0.406]\nIMAGENET_STD = [0.229, 0.224, 0.225]\nGRID_SPACING = 10\n\n\n@torch.no_grad()\ndef visactmap(\n model,\n test_loader,\n save_dir,\n width,\n height,\n use_gpu,\n img_mean=None,\n img_std=None\n):\n if img_mean is None or img_std is None:\n # use imagenet mean and std\n img_mean = IMAGENET_MEAN\n img_std = IMAGENET_STD\n\n model.eval()\n\n for target in list(test_loader.keys()):\n data_loader = test_loader[target]['query'] # only process query images\n # original images and activation maps are saved individually\n actmap_dir = osp.join(save_dir, 'actmap_' + target)\n mkdir_if_missing(actmap_dir)\n print('Visualizing activation maps for {} ...'.format(target))\n\n for batch_idx, data in enumerate(data_loader):\n imgs, paths = data['img'], data['impath']\n if use_gpu:\n imgs = imgs.cuda()\n\n # forward to get convolutional feature maps\n try:\n outputs = model(imgs, return_featuremaps=True)\n except TypeError:\n raise TypeError(\n 'forward() got unexpected keyword argument \"return_featuremaps\". '\n 'Please add return_featuremaps as an input argument to forward(). When '\n 'return_featuremaps=True, return feature maps only.'\n )\n\n if outputs.dim() != 4:\n raise ValueError(\n 'The model output is supposed to have '\n 'shape of (b, c, h, w), i.e. 4 dimensions, but got {} dimensions. '\n 'Please make sure you set the model output at eval mode '\n 'to be the last convolutional feature maps'.format(\n outputs.dim()\n )\n )\n\n # compute activation maps\n outputs = (outputs**2).sum(1)\n b, h, w = outputs.size()\n outputs = outputs.view(b, h * w)\n outputs = F.normalize(outputs, p=2, dim=1)\n outputs = outputs.view(b, h, w)\n\n if use_gpu:\n imgs, outputs = imgs.cpu(), outputs.cpu()\n\n for j in range(outputs.size(0)):\n # get image name\n path = paths[j]\n imname = osp.basename(osp.splitext(path)[0])\n\n # RGB image\n img = imgs[j, ...]\n for t, m, s in zip(img, img_mean, img_std):\n t.mul_(s).add_(m).clamp_(0, 1)\n img_np = np.uint8(np.floor(img.numpy() * 255))\n img_np = img_np.transpose((1, 2, 0)) # (c, h, w) -> (h, w, c)\n\n # activation map\n am = outputs[j, ...].numpy()\n am = cv2.resize(am, (width, height))\n am = 255 * (am - np.min(am)) / (\n np.max(am) - np.min(am) + 1e-12\n )\n am = np.uint8(np.floor(am))\n am = cv2.applyColorMap(am, cv2.COLORMAP_JET)\n\n # overlapped\n overlapped = img_np*0.3 + am*0.7\n overlapped[overlapped > 255] = 255\n overlapped = overlapped.astype(np.uint8)\n\n # save images in a single figure (add white spacing between images)\n # from left to right: original image, activation map, overlapped image\n grid_img = 255 * np.ones(\n (height, 3*width + 2*GRID_SPACING, 3), dtype=np.uint8\n )\n grid_img[:, :width, :] = img_np[:, :, ::-1]\n grid_img[:,\n width + GRID_SPACING:2*width + GRID_SPACING, :] = am\n grid_img[:, 2*width + 2*GRID_SPACING:, :] = overlapped\n cv2.imwrite(osp.join(actmap_dir, imname + '.jpg'), grid_img)\n\n if (batch_idx+1) % 10 == 0:\n print(\n '- done batch {}/{}'.format(\n batch_idx + 1, len(data_loader)\n )\n )\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument('--root', type=str)\n parser.add_argument('-d', '--dataset', type=str, default='market1501')\n parser.add_argument('-m', '--model', type=str, default='osnet_x1_0')\n parser.add_argument('--weights', type=str)\n parser.add_argument('--save-dir', type=str, default='log')\n parser.add_argument('--height', type=int, default=256)\n parser.add_argument('--width', type=int, default=128)\n args = parser.parse_args()\n\n use_gpu = torch.cuda.is_available()\n\n datamanager = torchreid.data.ImageDataManager(\n root=args.root,\n sources=args.dataset,\n height=args.height,\n width=args.width,\n batch_size_train=100,\n batch_size_test=100,\n transforms=None,\n train_sampler='SequentialSampler'\n )\n test_loader = datamanager.test_loader\n\n model = torchreid.models.build_model(\n name=args.model,\n num_classes=datamanager.num_train_pids,\n use_gpu=use_gpu\n )\n\n if use_gpu:\n model = model.cuda()\n\n if args.weights and check_isfile(args.weights):\n load_pretrained_weights(model, args.weights)\n\n visactmap(\n model, test_loader, args.save_dir, args.width, args.height, use_gpu\n )\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/__init__.py", "content": "from __future__ import print_function, absolute_import\n\nfrom torchreid import data, optim, utils, engine, losses, models, metrics\n\n__version__ = '1.4.0'\n__author__ = 'Kaiyang Zhou'\n__homepage__ = 'https://kaiyangzhou.github.io/'\n__description__ = 'Deep learning person re-identification in PyTorch'\n__url__ = 'https://github.com/KaiyangZhou/deep-person-reid'\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/__init__.py", "content": "from __future__ import print_function, absolute_import\n\nfrom .datasets import (\n Dataset, ImageDataset, VideoDataset, register_image_dataset,\n register_video_dataset\n)\nfrom .datamanager import ImageDataManager, VideoDataManager\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datamanager.py", "content": "from __future__ import division, print_function, absolute_import\nimport torch\n\nfrom torchreid.data.sampler import build_train_sampler\nfrom torchreid.data.datasets import init_image_dataset, init_video_dataset\nfrom torchreid.data.transforms import build_transforms\n\n\nclass DataManager(object):\n r\"\"\"Base data manager.\n\n Args:\n sources (str or list): source dataset(s).\n targets (str or list, optional): target dataset(s). If not given,\n it equals to ``sources``.\n height (int, optional): target image height. Default is 256.\n width (int, optional): target image width. Default is 128.\n transforms (str or list of str, optional): transformations applied to model training.\n Default is 'random_flip'.\n norm_mean (list or None, optional): data mean. Default is None (use imagenet mean).\n norm_std (list or None, optional): data std. Default is None (use imagenet std).\n use_gpu (bool, optional): use gpu. Default is True.\n \"\"\"\n\n def __init__(\n self,\n sources=None,\n targets=None,\n height=256,\n width=128,\n transforms='random_flip',\n norm_mean=None,\n norm_std=None,\n use_gpu=False\n ):\n self.sources = sources\n self.targets = targets\n self.height = height\n self.width = width\n\n if self.sources is None:\n raise ValueError('sources must not be None')\n\n if isinstance(self.sources, str):\n self.sources = [self.sources]\n\n if self.targets is None:\n self.targets = self.sources\n\n if isinstance(self.targets, str):\n self.targets = [self.targets]\n\n self.transform_tr, self.transform_te = build_transforms(\n self.height,\n self.width,\n transforms=transforms,\n norm_mean=norm_mean,\n norm_std=norm_std\n )\n\n self.use_gpu = (torch.cuda.is_available() and use_gpu)\n\n @property\n def num_train_pids(self):\n \"\"\"Returns the number of training person identities.\"\"\"\n return self._num_train_pids\n\n @property\n def num_train_cams(self):\n \"\"\"Returns the number of training cameras.\"\"\"\n return self._num_train_cams\n\n def fetch_test_loaders(self, name):\n \"\"\"Returns query and gallery of a test dataset, each containing\n tuples of (img_path(s), pid, camid).\n\n Args:\n name (str): dataset name.\n \"\"\"\n query_loader = self.test_dataset[name]['query']\n gallery_loader = self.test_dataset[name]['gallery']\n return query_loader, gallery_loader\n\n def preprocess_pil_img(self, img):\n \"\"\"Transforms a PIL image to torch tensor for testing.\"\"\"\n return self.transform_te(img)\n\n\nclass ImageDataManager(DataManager):\n r\"\"\"Image data manager.\n\n Args:\n root (str): root path to datasets.\n sources (str or list): source dataset(s).\n targets (str or list, optional): target dataset(s). If not given,\n it equals to ``sources``.\n height (int, optional): target image height. Default is 256.\n width (int, optional): target image width. Default is 128.\n transforms (str or list of str, optional): transformations applied to model training.\n Default is 'random_flip'.\n k_tfm (int): number of times to apply augmentation to an image\n independently. If k_tfm > 1, the transform function will be\n applied k_tfm times to an image. This variable will only be\n useful for training and is currently valid for image datasets only.\n norm_mean (list or None, optional): data mean. Default is None (use imagenet mean).\n norm_std (list or None, optional): data std. Default is None (use imagenet std).\n use_gpu (bool, optional): use gpu. Default is True.\n split_id (int, optional): split id (*0-based*). Default is 0.\n combineall (bool, optional): combine train, query and gallery in a dataset for\n training. Default is False.\n load_train_targets (bool, optional): construct train-loader for target datasets.\n Default is False. This is useful for domain adaptation research.\n batch_size_train (int, optional): number of images in a training batch. Default is 32.\n batch_size_test (int, optional): number of images in a test batch. Default is 32.\n workers (int, optional): number of workers. Default is 4.\n num_instances (int, optional): number of instances per identity in a batch.\n Default is 4.\n num_cams (int, optional): number of cameras to sample in a batch (when using\n ``RandomDomainSampler``). Default is 1.\n num_datasets (int, optional): number of datasets to sample in a batch (when\n using ``RandomDatasetSampler``). Default is 1.\n train_sampler (str, optional): sampler. Default is RandomSampler.\n train_sampler_t (str, optional): sampler for target train loader. Default is RandomSampler.\n cuhk03_labeled (bool, optional): use cuhk03 labeled images.\n Default is False (defaul is to use detected images).\n cuhk03_classic_split (bool, optional): use the classic split in cuhk03.\n Default is False.\n market1501_500k (bool, optional): add 500K distractors to the gallery\n set in market1501. Default is False.\n\n Examples::\n\n datamanager = torchreid.data.ImageDataManager(\n root='path/to/reid-data',\n sources='market1501',\n height=256,\n width=128,\n batch_size_train=32,\n batch_size_test=100\n )\n\n # return train loader of source data\n train_loader = datamanager.train_loader\n\n # return test loader of target data\n test_loader = datamanager.test_loader\n\n # return train loader of target data\n train_loader_t = datamanager.train_loader_t\n \"\"\"\n data_type = 'image'\n\n def __init__(\n self,\n root='',\n sources=None,\n targets=None,\n height=256,\n width=128,\n transforms='random_flip',\n k_tfm=1,\n norm_mean=None,\n norm_std=None,\n use_gpu=True,\n split_id=0,\n combineall=False,\n load_train_targets=False,\n batch_size_train=32,\n batch_size_test=32,\n workers=4,\n num_instances=4,\n num_cams=1,\n num_datasets=1,\n train_sampler='RandomSampler',\n train_sampler_t='RandomSampler',\n cuhk03_labeled=False,\n cuhk03_classic_split=False,\n market1501_500k=False\n ):\n\n super(ImageDataManager, self).__init__(\n sources=sources,\n targets=targets,\n height=height,\n width=width,\n transforms=transforms,\n norm_mean=norm_mean,\n norm_std=norm_std,\n use_gpu=use_gpu\n )\n\n print('=> Loading train (source) dataset')\n trainset = []\n for name in self.sources:\n trainset_ = init_image_dataset(\n name,\n transform=self.transform_tr,\n k_tfm=k_tfm,\n mode='train',\n combineall=combineall,\n root=root,\n split_id=split_id,\n cuhk03_labeled=cuhk03_labeled,\n cuhk03_classic_split=cuhk03_classic_split,\n market1501_500k=market1501_500k\n )\n trainset.append(trainset_)\n trainset = sum(trainset)\n\n self._num_train_pids = trainset.num_train_pids\n self._num_train_cams = trainset.num_train_cams\n\n self.train_loader = torch.utils.data.DataLoader(\n trainset,\n sampler=build_train_sampler(\n trainset.train,\n train_sampler,\n batch_size=batch_size_train,\n num_instances=num_instances,\n num_cams=num_cams,\n num_datasets=num_datasets\n ),\n batch_size=batch_size_train,\n shuffle=False,\n num_workers=workers,\n pin_memory=self.use_gpu,\n drop_last=True\n )\n\n self.train_loader_t = None\n if load_train_targets:\n # check if sources and targets are identical\n assert len(set(self.sources) & set(self.targets)) == 0, \\\n 'sources={} and targets={} must not have overlap'.format(self.sources, self.targets)\n\n print('=> Loading train (target) dataset')\n trainset_t = []\n for name in self.targets:\n trainset_t_ = init_image_dataset(\n name,\n transform=self.transform_tr,\n k_tfm=k_tfm,\n mode='train',\n combineall=False, # only use the training data\n root=root,\n split_id=split_id,\n cuhk03_labeled=cuhk03_labeled,\n cuhk03_classic_split=cuhk03_classic_split,\n market1501_500k=market1501_500k\n )\n trainset_t.append(trainset_t_)\n trainset_t = sum(trainset_t)\n\n self.train_loader_t = torch.utils.data.DataLoader(\n trainset_t,\n sampler=build_train_sampler(\n trainset_t.train,\n train_sampler_t,\n batch_size=batch_size_train,\n num_instances=num_instances,\n num_cams=num_cams,\n num_datasets=num_datasets\n ),\n batch_size=batch_size_train,\n shuffle=False,\n num_workers=workers,\n pin_memory=self.use_gpu,\n drop_last=True\n )\n\n print('=> Loading test (target) dataset')\n self.test_loader = {\n name: {\n 'query': None,\n 'gallery': None\n }\n for name in self.targets\n }\n self.test_dataset = {\n name: {\n 'query': None,\n 'gallery': None\n }\n for name in self.targets\n }\n\n for name in self.targets:\n # build query loader\n queryset = init_image_dataset(\n name,\n transform=self.transform_te,\n mode='query',\n combineall=combineall,\n root=root,\n split_id=split_id,\n cuhk03_labeled=cuhk03_labeled,\n cuhk03_classic_split=cuhk03_classic_split,\n market1501_500k=market1501_500k\n )\n self.test_loader[name]['query'] = torch.utils.data.DataLoader(\n queryset,\n batch_size=batch_size_test,\n shuffle=False,\n num_workers=workers,\n pin_memory=self.use_gpu,\n drop_last=False\n )\n\n # build gallery loader\n galleryset = init_image_dataset(\n name,\n transform=self.transform_te,\n mode='gallery',\n combineall=combineall,\n verbose=False,\n root=root,\n split_id=split_id,\n cuhk03_labeled=cuhk03_labeled,\n cuhk03_classic_split=cuhk03_classic_split,\n market1501_500k=market1501_500k\n )\n self.test_loader[name]['gallery'] = torch.utils.data.DataLoader(\n galleryset,\n batch_size=batch_size_test,\n shuffle=False,\n num_workers=workers,\n pin_memory=self.use_gpu,\n drop_last=False\n )\n\n self.test_dataset[name]['query'] = queryset.query\n self.test_dataset[name]['gallery'] = galleryset.gallery\n\n print('\\n')\n print(' **************** Summary ****************')\n print(' source : {}'.format(self.sources))\n print(' # source datasets : {}'.format(len(self.sources)))\n print(' # source ids : {}'.format(self.num_train_pids))\n print(' # source images : {}'.format(len(trainset)))\n print(' # source cameras : {}'.format(self.num_train_cams))\n if load_train_targets:\n print(\n ' # target images : {} (unlabeled)'.format(len(trainset_t))\n )\n print(' target : {}'.format(self.targets))\n print(' *****************************************')\n print('\\n')\n\n\nclass VideoDataManager(DataManager):\n r\"\"\"Video data manager.\n\n Args:\n root (str): root path to datasets.\n sources (str or list): source dataset(s).\n targets (str or list, optional): target dataset(s). If not given,\n it equals to ``sources``.\n height (int, optional): target image height. Default is 256.\n width (int, optional): target image width. Default is 128.\n transforms (str or list of str, optional): transformations applied to model training.\n Default is 'random_flip'.\n norm_mean (list or None, optional): data mean. Default is None (use imagenet mean).\n norm_std (list or None, optional): data std. Default is None (use imagenet std).\n use_gpu (bool, optional): use gpu. Default is True.\n split_id (int, optional): split id (*0-based*). Default is 0.\n combineall (bool, optional): combine train, query and gallery in a dataset for\n training. Default is False.\n batch_size_train (int, optional): number of tracklets in a training batch. Default is 3.\n batch_size_test (int, optional): number of tracklets in a test batch. Default is 3.\n workers (int, optional): number of workers. Default is 4.\n num_instances (int, optional): number of instances per identity in a batch.\n Default is 4.\n num_cams (int, optional): number of cameras to sample in a batch (when using\n ``RandomDomainSampler``). Default is 1.\n num_datasets (int, optional): number of datasets to sample in a batch (when\n using ``RandomDatasetSampler``). Default is 1.\n train_sampler (str, optional): sampler. Default is RandomSampler.\n seq_len (int, optional): how many images to sample in a tracklet. Default is 15.\n sample_method (str, optional): how to sample images in a tracklet. Default is \"evenly\".\n Choices are [\"evenly\", \"random\", \"all\"]. \"evenly\" and \"random\" will sample ``seq_len``\n images in a tracklet while \"all\" samples all images in a tracklet, where the batch size\n needs to be set to 1.\n\n Examples::\n\n datamanager = torchreid.data.VideoDataManager(\n root='path/to/reid-data',\n sources='mars',\n height=256,\n width=128,\n batch_size_train=3,\n batch_size_test=3,\n seq_len=15,\n sample_method='evenly'\n )\n\n # return train loader of source data\n train_loader = datamanager.train_loader\n\n # return test loader of target data\n test_loader = datamanager.test_loader\n\n .. note::\n The current implementation only supports image-like training. Therefore, each image in a\n sampled tracklet will undergo independent transformation functions. To achieve tracklet-aware\n training, you need to modify the transformation functions for video reid such that each function\n applies the same operation to all images in a tracklet to keep consistency.\n \"\"\"\n data_type = 'video'\n\n def __init__(\n self,\n root='',\n sources=None,\n targets=None,\n height=256,\n width=128,\n transforms='random_flip',\n norm_mean=None,\n norm_std=None,\n use_gpu=True,\n split_id=0,\n combineall=False,\n batch_size_train=3,\n batch_size_test=3,\n workers=4,\n num_instances=4,\n num_cams=1,\n num_datasets=1,\n train_sampler='RandomSampler',\n seq_len=15,\n sample_method='evenly'\n ):\n\n super(VideoDataManager, self).__init__(\n sources=sources,\n targets=targets,\n height=height,\n width=width,\n transforms=transforms,\n norm_mean=norm_mean,\n norm_std=norm_std,\n use_gpu=use_gpu\n )\n\n print('=> Loading train (source) dataset')\n trainset = []\n for name in self.sources:\n trainset_ = init_video_dataset(\n name,\n transform=self.transform_tr,\n mode='train',\n combineall=combineall,\n root=root,\n split_id=split_id,\n seq_len=seq_len,\n sample_method=sample_method\n )\n trainset.append(trainset_)\n trainset = sum(trainset)\n\n self._num_train_pids = trainset.num_train_pids\n self._num_train_cams = trainset.num_train_cams\n\n train_sampler = build_train_sampler(\n trainset.train,\n train_sampler,\n batch_size=batch_size_train,\n num_instances=num_instances,\n num_cams=num_cams,\n num_datasets=num_datasets\n )\n\n self.train_loader = torch.utils.data.DataLoader(\n trainset,\n sampler=train_sampler,\n batch_size=batch_size_train,\n shuffle=False,\n num_workers=workers,\n pin_memory=self.use_gpu,\n drop_last=True\n )\n\n print('=> Loading test (target) dataset')\n self.test_loader = {\n name: {\n 'query': None,\n 'gallery': None\n }\n for name in self.targets\n }\n self.test_dataset = {\n name: {\n 'query': None,\n 'gallery': None\n }\n for name in self.targets\n }\n\n for name in self.targets:\n # build query loader\n queryset = init_video_dataset(\n name,\n transform=self.transform_te,\n mode='query',\n combineall=combineall,\n root=root,\n split_id=split_id,\n seq_len=seq_len,\n sample_method=sample_method\n )\n self.test_loader[name]['query'] = torch.utils.data.DataLoader(\n queryset,\n batch_size=batch_size_test,\n shuffle=False,\n num_workers=workers,\n pin_memory=self.use_gpu,\n drop_last=False\n )\n\n # build gallery loader\n galleryset = init_video_dataset(\n name,\n transform=self.transform_te,\n mode='gallery',\n combineall=combineall,\n verbose=False,\n root=root,\n split_id=split_id,\n seq_len=seq_len,\n sample_method=sample_method\n )\n self.test_loader[name]['gallery'] = torch.utils.data.DataLoader(\n galleryset,\n batch_size=batch_size_test,\n shuffle=False,\n num_workers=workers,\n pin_memory=self.use_gpu,\n drop_last=False\n )\n\n self.test_dataset[name]['query'] = queryset.query\n self.test_dataset[name]['gallery'] = galleryset.gallery\n\n print('\\n')\n print(' **************** Summary ****************')\n print(' source : {}'.format(self.sources))\n print(' # source datasets : {}'.format(len(self.sources)))\n print(' # source ids : {}'.format(self.num_train_pids))\n print(' # source tracklets : {}'.format(len(trainset)))\n print(' # source cameras : {}'.format(self.num_train_cams))\n print(' target : {}'.format(self.targets))\n print(' *****************************************')\n print('\\n')\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/__init__.py", "content": "from __future__ import print_function, absolute_import\n\nfrom .image import (\n GRID, PRID, CUHK01, CUHK02, CUHK03, MSMT17, CUHKSYSU, VIPeR, SenseReID,\n Market1501, DukeMTMCreID, University1652, iLIDS\n)\nfrom .video import PRID2011, Mars, DukeMTMCVidReID, iLIDSVID\nfrom .dataset import Dataset, ImageDataset, VideoDataset\n\n__image_datasets = {\n 'market1501': Market1501,\n 'cuhk03': CUHK03,\n 'dukemtmcreid': DukeMTMCreID,\n 'msmt17': MSMT17,\n 'viper': VIPeR,\n 'grid': GRID,\n 'cuhk01': CUHK01,\n 'ilids': iLIDS,\n 'sensereid': SenseReID,\n 'prid': PRID,\n 'cuhk02': CUHK02,\n 'university1652': University1652,\n 'cuhksysu': CUHKSYSU\n}\n\n__video_datasets = {\n 'mars': Mars,\n 'ilidsvid': iLIDSVID,\n 'prid2011': PRID2011,\n 'dukemtmcvidreid': DukeMTMCVidReID\n}\n\n\ndef init_image_dataset(name, **kwargs):\n \"\"\"Initializes an image dataset.\"\"\"\n avai_datasets = list(__image_datasets.keys())\n if name not in avai_datasets:\n raise ValueError(\n 'Invalid dataset name. Received \"{}\", '\n 'but expected to be one of {}'.format(name, avai_datasets)\n )\n return __image_datasets[name](**kwargs)\n\n\ndef init_video_dataset(name, **kwargs):\n \"\"\"Initializes a video dataset.\"\"\"\n avai_datasets = list(__video_datasets.keys())\n if name not in avai_datasets:\n raise ValueError(\n 'Invalid dataset name. Received \"{}\", '\n 'but expected to be one of {}'.format(name, avai_datasets)\n )\n return __video_datasets[name](**kwargs)\n\n\ndef register_image_dataset(name, dataset):\n \"\"\"Registers a new image dataset.\n\n Args:\n name (str): key corresponding to the new dataset.\n dataset (Dataset): the new dataset class.\n\n Examples::\n \n import torchreid\n import NewDataset\n torchreid.data.register_image_dataset('new_dataset', NewDataset)\n # single dataset case\n datamanager = torchreid.data.ImageDataManager(\n root='reid-data',\n sources='new_dataset'\n )\n # multiple dataset case\n datamanager = torchreid.data.ImageDataManager(\n root='reid-data',\n sources=['new_dataset', 'dukemtmcreid']\n )\n \"\"\"\n global __image_datasets\n curr_datasets = list(__image_datasets.keys())\n if name in curr_datasets:\n raise ValueError(\n 'The given name already exists, please choose '\n 'another name excluding {}'.format(curr_datasets)\n )\n __image_datasets[name] = dataset\n\n\ndef register_video_dataset(name, dataset):\n \"\"\"Registers a new video dataset.\n\n Args:\n name (str): key corresponding to the new dataset.\n dataset (Dataset): the new dataset class.\n\n Examples::\n \n import torchreid\n import NewDataset\n torchreid.data.register_video_dataset('new_dataset', NewDataset)\n # single dataset case\n datamanager = torchreid.data.VideoDataManager(\n root='reid-data',\n sources='new_dataset'\n )\n # multiple dataset case\n datamanager = torchreid.data.VideoDataManager(\n root='reid-data',\n sources=['new_dataset', 'ilidsvid']\n )\n \"\"\"\n global __video_datasets\n curr_datasets = list(__video_datasets.keys())\n if name in curr_datasets:\n raise ValueError(\n 'The given name already exists, please choose '\n 'another name excluding {}'.format(curr_datasets)\n )\n __video_datasets[name] = dataset\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/dataset.py", "content": "from __future__ import division, print_function, absolute_import\nimport copy\nimport numpy as np\nimport os.path as osp\nimport tarfile\nimport zipfile\nimport torch\n\nfrom torchreid.utils import read_image, download_url, mkdir_if_missing\n\n\nclass Dataset(object):\n \"\"\"An abstract class representing a Dataset.\n\n This is the base class for ``ImageDataset`` and ``VideoDataset``.\n\n Args:\n train (list): contains tuples of (img_path(s), pid, camid).\n query (list): contains tuples of (img_path(s), pid, camid).\n gallery (list): contains tuples of (img_path(s), pid, camid).\n transform: transform function.\n k_tfm (int): number of times to apply augmentation to an image\n independently. If k_tfm > 1, the transform function will be\n applied k_tfm times to an image. This variable will only be\n useful for training and is currently valid for image datasets only.\n mode (str): 'train', 'query' or 'gallery'.\n combineall (bool): combines train, query and gallery in a\n dataset for training.\n verbose (bool): show information.\n \"\"\"\n\n # junk_pids contains useless person IDs, e.g. background,\n # false detections, distractors. These IDs will be ignored\n # when combining all images in a dataset for training, i.e.\n # combineall=True\n _junk_pids = []\n\n # Some datasets are only used for training, like CUHK-SYSU\n # In this case, \"combineall=True\" is not used for them\n _train_only = False\n\n def __init__(\n self,\n train,\n query,\n gallery,\n transform=None,\n k_tfm=1,\n mode='train',\n combineall=False,\n verbose=True,\n **kwargs\n ):\n # extend 3-tuple (img_path(s), pid, camid) to\n # 4-tuple (img_path(s), pid, camid, dsetid) by\n # adding a dataset indicator \"dsetid\"\n if len(train[0]) == 3:\n train = [(*items, 0) for items in train]\n if len(query[0]) == 3:\n query = [(*items, 0) for items in query]\n if len(gallery[0]) == 3:\n gallery = [(*items, 0) for items in gallery]\n\n self.train = train\n self.query = query\n self.gallery = gallery\n self.transform = transform\n self.k_tfm = k_tfm\n self.mode = mode\n self.combineall = combineall\n self.verbose = verbose\n\n self.num_train_pids = self.get_num_pids(self.train)\n self.num_train_cams = self.get_num_cams(self.train)\n self.num_datasets = self.get_num_datasets(self.train)\n\n if self.combineall:\n self.combine_all()\n\n if self.mode == 'train':\n self.data = self.train\n elif self.mode == 'query':\n self.data = self.query\n elif self.mode == 'gallery':\n self.data = self.gallery\n else:\n raise ValueError(\n 'Invalid mode. Got {}, but expected to be '\n 'one of [train | query | gallery]'.format(self.mode)\n )\n\n if self.verbose:\n self.show_summary()\n\n def __getitem__(self, index):\n raise NotImplementedError\n\n def __len__(self):\n return len(self.data)\n\n def __add__(self, other):\n \"\"\"Adds two datasets together (only the train set).\"\"\"\n train = copy.deepcopy(self.train)\n\n for img_path, pid, camid, dsetid in other.train:\n pid += self.num_train_pids\n camid += self.num_train_cams\n dsetid += self.num_datasets\n train.append((img_path, pid, camid, dsetid))\n\n ###################################\n # Note that\n # 1. set verbose=False to avoid unnecessary print\n # 2. set combineall=False because combineall would have been applied\n # if it was True for a specific dataset; setting it to True will\n # create new IDs that should have already been included\n ###################################\n if isinstance(train[0][0], str):\n return ImageDataset(\n train,\n self.query,\n self.gallery,\n transform=self.transform,\n mode=self.mode,\n combineall=False,\n verbose=False\n )\n else:\n return VideoDataset(\n train,\n self.query,\n self.gallery,\n transform=self.transform,\n mode=self.mode,\n combineall=False,\n verbose=False,\n seq_len=self.seq_len,\n sample_method=self.sample_method\n )\n\n def __radd__(self, other):\n \"\"\"Supports sum([dataset1, dataset2, dataset3]).\"\"\"\n if other == 0:\n return self\n else:\n return self.__add__(other)\n\n def get_num_pids(self, data):\n \"\"\"Returns the number of training person identities.\n\n Each tuple in data contains (img_path(s), pid, camid, dsetid).\n \"\"\"\n pids = set()\n for items in data:\n pid = items[1]\n pids.add(pid)\n return len(pids)\n\n def get_num_cams(self, data):\n \"\"\"Returns the number of training cameras.\n\n Each tuple in data contains (img_path(s), pid, camid, dsetid).\n \"\"\"\n cams = set()\n for items in data:\n camid = items[2]\n cams.add(camid)\n return len(cams)\n\n def get_num_datasets(self, data):\n \"\"\"Returns the number of datasets included.\n\n Each tuple in data contains (img_path(s), pid, camid, dsetid).\n \"\"\"\n dsets = set()\n for items in data:\n dsetid = items[3]\n dsets.add(dsetid)\n return len(dsets)\n\n def show_summary(self):\n \"\"\"Shows dataset statistics.\"\"\"\n pass\n\n def combine_all(self):\n \"\"\"Combines train, query and gallery in a dataset for training.\"\"\"\n if self._train_only:\n return\n\n combined = copy.deepcopy(self.train)\n\n # relabel pids in gallery (query shares the same scope)\n g_pids = set()\n for items in self.gallery:\n pid = items[1]\n if pid in self._junk_pids:\n continue\n g_pids.add(pid)\n pid2label = {pid: i for i, pid in enumerate(g_pids)}\n\n def _combine_data(data):\n for img_path, pid, camid, dsetid in data:\n if pid in self._junk_pids:\n continue\n pid = pid2label[pid] + self.num_train_pids\n combined.append((img_path, pid, camid, dsetid))\n\n _combine_data(self.query)\n _combine_data(self.gallery)\n\n self.train = combined\n self.num_train_pids = self.get_num_pids(self.train)\n\n def download_dataset(self, dataset_dir, dataset_url):\n \"\"\"Downloads and extracts dataset.\n\n Args:\n dataset_dir (str): dataset directory.\n dataset_url (str): url to download dataset.\n \"\"\"\n if osp.exists(dataset_dir):\n return\n\n if dataset_url is None:\n raise RuntimeError(\n '{} dataset needs to be manually '\n 'prepared, please follow the '\n 'document to prepare this dataset'.format(\n self.__class__.__name__\n )\n )\n\n print('Creating directory \"{}\"'.format(dataset_dir))\n mkdir_if_missing(dataset_dir)\n fpath = osp.join(dataset_dir, osp.basename(dataset_url))\n\n print(\n 'Downloading {} dataset to \"{}\"'.format(\n self.__class__.__name__, dataset_dir\n )\n )\n download_url(dataset_url, fpath)\n\n print('Extracting \"{}\"'.format(fpath))\n try:\n tar = tarfile.open(fpath)\n tar.extractall(path=dataset_dir)\n tar.close()\n except:\n zip_ref = zipfile.ZipFile(fpath, 'r')\n zip_ref.extractall(dataset_dir)\n zip_ref.close()\n\n print('{} dataset is ready'.format(self.__class__.__name__))\n\n def check_before_run(self, required_files):\n \"\"\"Checks if required files exist before going deeper.\n\n Args:\n required_files (str or list): string file name(s).\n \"\"\"\n if isinstance(required_files, str):\n required_files = [required_files]\n\n for fpath in required_files:\n if not osp.exists(fpath):\n raise RuntimeError('\"{}\" is not found'.format(fpath))\n\n def __repr__(self):\n num_train_pids = self.get_num_pids(self.train)\n num_train_cams = self.get_num_cams(self.train)\n\n num_query_pids = self.get_num_pids(self.query)\n num_query_cams = self.get_num_cams(self.query)\n\n num_gallery_pids = self.get_num_pids(self.gallery)\n num_gallery_cams = self.get_num_cams(self.gallery)\n\n msg = ' ----------------------------------------\\n' \\\n ' subset | # ids | # items | # cameras\\n' \\\n ' ----------------------------------------\\n' \\\n ' train | {:5d} | {:7d} | {:9d}\\n' \\\n ' query | {:5d} | {:7d} | {:9d}\\n' \\\n ' gallery | {:5d} | {:7d} | {:9d}\\n' \\\n ' ----------------------------------------\\n' \\\n ' items: images/tracklets for image/video dataset\\n'.format(\n num_train_pids, len(self.train), num_train_cams,\n num_query_pids, len(self.query), num_query_cams,\n num_gallery_pids, len(self.gallery), num_gallery_cams\n )\n\n return msg\n\n def _transform_image(self, tfm, k_tfm, img0):\n \"\"\"Transforms a raw image (img0) k_tfm times with\n the transform function tfm.\n \"\"\"\n img_list = []\n\n for k in range(k_tfm):\n img_list.append(tfm(img0))\n\n img = img_list\n if len(img) == 1:\n img = img[0]\n\n return img\n\n\nclass ImageDataset(Dataset):\n \"\"\"A base class representing ImageDataset.\n\n All other image datasets should subclass it.\n\n ``__getitem__`` returns an image given index.\n It will return ``img``, ``pid``, ``camid`` and ``img_path``\n where ``img`` has shape (channel, height, width). As a result,\n data in each batch has shape (batch_size, channel, height, width).\n \"\"\"\n\n def __init__(self, train, query, gallery, **kwargs):\n super(ImageDataset, self).__init__(train, query, gallery, **kwargs)\n\n def __getitem__(self, index):\n img_path, pid, camid, dsetid = self.data[index]\n img = read_image(img_path)\n if self.transform is not None:\n img = self._transform_image(self.transform, self.k_tfm, img)\n item = {\n 'img': img,\n 'pid': pid,\n 'camid': camid,\n 'impath': img_path,\n 'dsetid': dsetid\n }\n return item\n\n def show_summary(self):\n num_train_pids = self.get_num_pids(self.train)\n num_train_cams = self.get_num_cams(self.train)\n\n num_query_pids = self.get_num_pids(self.query)\n num_query_cams = self.get_num_cams(self.query)\n\n num_gallery_pids = self.get_num_pids(self.gallery)\n num_gallery_cams = self.get_num_cams(self.gallery)\n\n print('=> Loaded {}'.format(self.__class__.__name__))\n print(' ----------------------------------------')\n print(' subset | # ids | # images | # cameras')\n print(' ----------------------------------------')\n print(\n ' train | {:5d} | {:8d} | {:9d}'.format(\n num_train_pids, len(self.train), num_train_cams\n )\n )\n print(\n ' query | {:5d} | {:8d} | {:9d}'.format(\n num_query_pids, len(self.query), num_query_cams\n )\n )\n print(\n ' gallery | {:5d} | {:8d} | {:9d}'.format(\n num_gallery_pids, len(self.gallery), num_gallery_cams\n )\n )\n print(' ----------------------------------------')\n\n\nclass VideoDataset(Dataset):\n \"\"\"A base class representing VideoDataset.\n\n All other video datasets should subclass it.\n\n ``__getitem__`` returns an image given index.\n It will return ``imgs``, ``pid`` and ``camid``\n where ``imgs`` has shape (seq_len, channel, height, width). As a result,\n data in each batch has shape (batch_size, seq_len, channel, height, width).\n \"\"\"\n\n def __init__(\n self,\n train,\n query,\n gallery,\n seq_len=15,\n sample_method='evenly',\n **kwargs\n ):\n super(VideoDataset, self).__init__(train, query, gallery, **kwargs)\n self.seq_len = seq_len\n self.sample_method = sample_method\n\n if self.transform is None:\n raise RuntimeError('transform must not be None')\n\n def __getitem__(self, index):\n img_paths, pid, camid, dsetid = self.data[index]\n num_imgs = len(img_paths)\n\n if self.sample_method == 'random':\n # Randomly samples seq_len images from a tracklet of length num_imgs,\n # if num_imgs is smaller than seq_len, then replicates images\n indices = np.arange(num_imgs)\n replace = False if num_imgs >= self.seq_len else True\n indices = np.random.choice(\n indices, size=self.seq_len, replace=replace\n )\n # sort indices to keep temporal order (comment it to be order-agnostic)\n indices = np.sort(indices)\n\n elif self.sample_method == 'evenly':\n # Evenly samples seq_len images from a tracklet\n if num_imgs >= self.seq_len:\n num_imgs -= num_imgs % self.seq_len\n indices = np.arange(0, num_imgs, num_imgs / self.seq_len)\n else:\n # if num_imgs is smaller than seq_len, simply replicate the last image\n # until the seq_len requirement is satisfied\n indices = np.arange(0, num_imgs)\n num_pads = self.seq_len - num_imgs\n indices = np.concatenate(\n [\n indices,\n np.ones(num_pads).astype(np.int32) * (num_imgs-1)\n ]\n )\n assert len(indices) == self.seq_len\n\n elif self.sample_method == 'all':\n # Samples all images in a tracklet. batch_size must be set to 1\n indices = np.arange(num_imgs)\n\n else:\n raise ValueError(\n 'Unknown sample method: {}'.format(self.sample_method)\n )\n\n imgs = []\n for index in indices:\n img_path = img_paths[int(index)]\n img = read_image(img_path)\n if self.transform is not None:\n img = self.transform(img)\n img = img.unsqueeze(0) # img must be torch.Tensor\n imgs.append(img)\n imgs = torch.cat(imgs, dim=0)\n\n item = {'img': imgs, 'pid': pid, 'camid': camid, 'dsetid': dsetid}\n\n return item\n\n def show_summary(self):\n num_train_pids = self.get_num_pids(self.train)\n num_train_cams = self.get_num_cams(self.train)\n\n num_query_pids = self.get_num_pids(self.query)\n num_query_cams = self.get_num_cams(self.query)\n\n num_gallery_pids = self.get_num_pids(self.gallery)\n num_gallery_cams = self.get_num_cams(self.gallery)\n\n print('=> Loaded {}'.format(self.__class__.__name__))\n print(' -------------------------------------------')\n print(' subset | # ids | # tracklets | # cameras')\n print(' -------------------------------------------')\n print(\n ' train | {:5d} | {:11d} | {:9d}'.format(\n num_train_pids, len(self.train), num_train_cams\n )\n )\n print(\n ' query | {:5d} | {:11d} | {:9d}'.format(\n num_query_pids, len(self.query), num_query_cams\n )\n )\n print(\n ' gallery | {:5d} | {:11d} | {:9d}'.format(\n num_gallery_pids, len(self.gallery), num_gallery_cams\n )\n )\n print(' -------------------------------------------')\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/__init__.py", "content": "from __future__ import print_function, absolute_import\n\nfrom .grid import GRID\nfrom .prid import PRID\nfrom .ilids import iLIDS\nfrom .viper import VIPeR\nfrom .cuhk01 import CUHK01\nfrom .cuhk02 import CUHK02\nfrom .cuhk03 import CUHK03\nfrom .msmt17 import MSMT17\nfrom .cuhksysu import CUHKSYSU\nfrom .sensereid import SenseReID\nfrom .market1501 import Market1501\nfrom .dukemtmcreid import DukeMTMCreID\nfrom .university1652 import University1652\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/cuhk01.py", "content": "from __future__ import division, print_function, absolute_import\nimport glob\nimport numpy as np\nimport os.path as osp\nimport zipfile\n\nfrom torchreid.utils import read_json, write_json\n\nfrom ..dataset import ImageDataset\n\n\nclass CUHK01(ImageDataset):\n \"\"\"CUHK01.\n\n Reference:\n Li et al. Human Reidentification with Transferred Metric Learning. ACCV 2012.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 971.\n - images: 3884.\n - cameras: 4.\n\n Note: CUHK01 and CUHK02 overlap.\n \"\"\"\n dataset_dir = 'cuhk01'\n dataset_url = None\n\n def __init__(self, root='', split_id=0, **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.zip_path = osp.join(self.dataset_dir, 'CUHK01.zip')\n self.campus_dir = osp.join(self.dataset_dir, 'campus')\n self.split_path = osp.join(self.dataset_dir, 'splits.json')\n\n self.extract_file()\n\n required_files = [self.dataset_dir, self.campus_dir]\n self.check_before_run(required_files)\n\n self.prepare_split()\n splits = read_json(self.split_path)\n if split_id >= len(splits):\n raise ValueError(\n 'split_id exceeds range, received {}, but expected between 0 and {}'\n .format(split_id,\n len(splits) - 1)\n )\n split = splits[split_id]\n\n train = split['train']\n query = split['query']\n gallery = split['gallery']\n\n train = [tuple(item) for item in train]\n query = [tuple(item) for item in query]\n gallery = [tuple(item) for item in gallery]\n\n super(CUHK01, self).__init__(train, query, gallery, **kwargs)\n\n def extract_file(self):\n if not osp.exists(self.campus_dir):\n print('Extracting files')\n zip_ref = zipfile.ZipFile(self.zip_path, 'r')\n zip_ref.extractall(self.dataset_dir)\n zip_ref.close()\n\n def prepare_split(self):\n \"\"\"\n Image name format: 0001001.png, where first four digits represent identity\n and last four digits represent cameras. Camera 1&2 are considered the same\n view and camera 3&4 are considered the same view.\n \"\"\"\n if not osp.exists(self.split_path):\n print('Creating 10 random splits of train ids and test ids')\n img_paths = sorted(glob.glob(osp.join(self.campus_dir, '*.png')))\n img_list = []\n pid_container = set()\n for img_path in img_paths:\n img_name = osp.basename(img_path)\n pid = int(img_name[:4]) - 1\n camid = (int(img_name[4:7]) - 1) // 2 # result is either 0 or 1\n img_list.append((img_path, pid, camid))\n pid_container.add(pid)\n\n num_pids = len(pid_container)\n num_train_pids = num_pids // 2\n\n splits = []\n for _ in range(10):\n order = np.arange(num_pids)\n np.random.shuffle(order)\n train_idxs = order[:num_train_pids]\n train_idxs = np.sort(train_idxs)\n idx2label = {\n idx: label\n for label, idx in enumerate(train_idxs)\n }\n\n train, test_a, test_b = [], [], []\n for img_path, pid, camid in img_list:\n if pid in train_idxs:\n train.append((img_path, idx2label[pid], camid))\n else:\n if camid == 0:\n test_a.append((img_path, pid, camid))\n else:\n test_b.append((img_path, pid, camid))\n\n # use cameraA as query and cameraB as gallery\n split = {\n 'train': train,\n 'query': test_a,\n 'gallery': test_b,\n 'num_train_pids': num_train_pids,\n 'num_query_pids': num_pids - num_train_pids,\n 'num_gallery_pids': num_pids - num_train_pids\n }\n splits.append(split)\n\n # use cameraB as query and cameraA as gallery\n split = {\n 'train': train,\n 'query': test_b,\n 'gallery': test_a,\n 'num_train_pids': num_train_pids,\n 'num_query_pids': num_pids - num_train_pids,\n 'num_gallery_pids': num_pids - num_train_pids\n }\n splits.append(split)\n\n print('Totally {} splits are created'.format(len(splits)))\n write_json(splits, self.split_path)\n print('Split file saved to {}'.format(self.split_path))\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/cuhk02.py", "content": "from __future__ import division, print_function, absolute_import\nimport glob\nimport os.path as osp\n\nfrom ..dataset import ImageDataset\n\n\nclass CUHK02(ImageDataset):\n \"\"\"CUHK02.\n\n Reference:\n Li and Wang. Locally Aligned Feature Transforms across Views. CVPR 2013.\n\n URL: ``_\n \n Dataset statistics:\n - 5 camera view pairs each with two cameras\n - 971, 306, 107, 193 and 239 identities from P1 - P5\n - totally 1,816 identities\n - image format is png\n\n Protocol: Use P1 - P4 for training and P5 for evaluation.\n\n Note: CUHK01 and CUHK02 overlap.\n \"\"\"\n dataset_dir = 'cuhk02'\n cam_pairs = ['P1', 'P2', 'P3', 'P4', 'P5']\n test_cam_pair = 'P5'\n\n def __init__(self, root='', **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir, 'Dataset')\n\n required_files = [self.dataset_dir]\n self.check_before_run(required_files)\n\n train, query, gallery = self.get_data_list()\n\n super(CUHK02, self).__init__(train, query, gallery, **kwargs)\n\n def get_data_list(self):\n num_train_pids, camid = 0, 0\n train, query, gallery = [], [], []\n\n for cam_pair in self.cam_pairs:\n cam_pair_dir = osp.join(self.dataset_dir, cam_pair)\n\n cam1_dir = osp.join(cam_pair_dir, 'cam1')\n cam2_dir = osp.join(cam_pair_dir, 'cam2')\n\n impaths1 = glob.glob(osp.join(cam1_dir, '*.png'))\n impaths2 = glob.glob(osp.join(cam2_dir, '*.png'))\n\n if cam_pair == self.test_cam_pair:\n # add images to query\n for impath in impaths1:\n pid = osp.basename(impath).split('_')[0]\n pid = int(pid)\n query.append((impath, pid, camid))\n camid += 1\n\n # add images to gallery\n for impath in impaths2:\n pid = osp.basename(impath).split('_')[0]\n pid = int(pid)\n gallery.append((impath, pid, camid))\n camid += 1\n\n else:\n pids1 = [\n osp.basename(impath).split('_')[0] for impath in impaths1\n ]\n pids2 = [\n osp.basename(impath).split('_')[0] for impath in impaths2\n ]\n pids = set(pids1 + pids2)\n pid2label = {\n pid: label + num_train_pids\n for label, pid in enumerate(pids)\n }\n\n # add images to train from cam1\n for impath in impaths1:\n pid = osp.basename(impath).split('_')[0]\n pid = pid2label[pid]\n train.append((impath, pid, camid))\n camid += 1\n\n # add images to train from cam2\n for impath in impaths2:\n pid = osp.basename(impath).split('_')[0]\n pid = pid2label[pid]\n train.append((impath, pid, camid))\n camid += 1\n num_train_pids += len(pids)\n\n return train, query, gallery\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/cuhk03.py", "content": "from __future__ import division, print_function, absolute_import\nimport os.path as osp\n\nfrom torchreid.utils import read_json, write_json, mkdir_if_missing\n\nfrom ..dataset import ImageDataset\n\n\nclass CUHK03(ImageDataset):\n \"\"\"CUHK03.\n\n Reference:\n Li et al. DeepReID: Deep Filter Pairing Neural Network for Person Re-identification. CVPR 2014.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 1360.\n - images: 13164.\n - cameras: 6.\n - splits: 20 (classic).\n \"\"\"\n dataset_dir = 'cuhk03'\n dataset_url = None\n\n def __init__(\n self,\n root='',\n split_id=0,\n cuhk03_labeled=False,\n cuhk03_classic_split=False,\n **kwargs\n ):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.data_dir = osp.join(self.dataset_dir, 'cuhk03_release')\n self.raw_mat_path = osp.join(self.data_dir, 'cuhk-03.mat')\n\n self.imgs_detected_dir = osp.join(self.dataset_dir, 'images_detected')\n self.imgs_labeled_dir = osp.join(self.dataset_dir, 'images_labeled')\n\n self.split_classic_det_json_path = osp.join(\n self.dataset_dir, 'splits_classic_detected.json'\n )\n self.split_classic_lab_json_path = osp.join(\n self.dataset_dir, 'splits_classic_labeled.json'\n )\n\n self.split_new_det_json_path = osp.join(\n self.dataset_dir, 'splits_new_detected.json'\n )\n self.split_new_lab_json_path = osp.join(\n self.dataset_dir, 'splits_new_labeled.json'\n )\n\n self.split_new_det_mat_path = osp.join(\n self.dataset_dir, 'cuhk03_new_protocol_config_detected.mat'\n )\n self.split_new_lab_mat_path = osp.join(\n self.dataset_dir, 'cuhk03_new_protocol_config_labeled.mat'\n )\n\n required_files = [\n self.dataset_dir, self.data_dir, self.raw_mat_path,\n self.split_new_det_mat_path, self.split_new_lab_mat_path\n ]\n self.check_before_run(required_files)\n\n self.preprocess_split()\n\n if cuhk03_labeled:\n split_path = self.split_classic_lab_json_path if cuhk03_classic_split else self.split_new_lab_json_path\n else:\n split_path = self.split_classic_det_json_path if cuhk03_classic_split else self.split_new_det_json_path\n\n splits = read_json(split_path)\n assert split_id < len(\n splits\n ), 'Condition split_id ({}) < len(splits) ({}) is false'.format(\n split_id, len(splits)\n )\n split = splits[split_id]\n\n train = split['train']\n query = split['query']\n gallery = split['gallery']\n\n super(CUHK03, self).__init__(train, query, gallery, **kwargs)\n\n def preprocess_split(self):\n # This function is a bit complex and ugly, what it does is\n # 1. extract data from cuhk-03.mat and save as png images\n # 2. create 20 classic splits (Li et al. CVPR'14)\n # 3. create new split (Zhong et al. CVPR'17)\n if osp.exists(self.imgs_labeled_dir) \\\n and osp.exists(self.imgs_detected_dir) \\\n and osp.exists(self.split_classic_det_json_path) \\\n and osp.exists(self.split_classic_lab_json_path) \\\n and osp.exists(self.split_new_det_json_path) \\\n and osp.exists(self.split_new_lab_json_path):\n return\n\n import h5py\n import imageio\n from scipy.io import loadmat\n\n mkdir_if_missing(self.imgs_detected_dir)\n mkdir_if_missing(self.imgs_labeled_dir)\n\n print(\n 'Extract image data from \"{}\" and save as png'.format(\n self.raw_mat_path\n )\n )\n mat = h5py.File(self.raw_mat_path, 'r')\n\n def _deref(ref):\n return mat[ref][:].T\n\n def _process_images(img_refs, campid, pid, save_dir):\n img_paths = [] # Note: some persons only have images for one view\n for imgid, img_ref in enumerate(img_refs):\n img = _deref(img_ref)\n if img.size == 0 or img.ndim < 3:\n continue # skip empty cell\n # images are saved with the following format, index-1 (ensure uniqueness)\n # campid: index of camera pair (1-5)\n # pid: index of person in 'campid'-th camera pair\n # viewid: index of view, {1, 2}\n # imgid: index of image, (1-10)\n viewid = 1 if imgid < 5 else 2\n img_name = '{:01d}_{:03d}_{:01d}_{:02d}.png'.format(\n campid + 1, pid + 1, viewid, imgid + 1\n )\n img_path = osp.join(save_dir, img_name)\n if not osp.isfile(img_path):\n imageio.imwrite(img_path, img)\n img_paths.append(img_path)\n return img_paths\n\n def _extract_img(image_type):\n print('Processing {} images ...'.format(image_type))\n meta_data = []\n imgs_dir = self.imgs_detected_dir if image_type == 'detected' else self.imgs_labeled_dir\n for campid, camp_ref in enumerate(mat[image_type][0]):\n camp = _deref(camp_ref)\n num_pids = camp.shape[0]\n for pid in range(num_pids):\n img_paths = _process_images(\n camp[pid, :], campid, pid, imgs_dir\n )\n assert len(img_paths) > 0, \\\n 'campid{}-pid{} has no images'.format(campid, pid)\n meta_data.append((campid + 1, pid + 1, img_paths))\n print(\n '- done camera pair {} with {} identities'.format(\n campid + 1, num_pids\n )\n )\n return meta_data\n\n meta_detected = _extract_img('detected')\n meta_labeled = _extract_img('labeled')\n\n def _extract_classic_split(meta_data, test_split):\n train, test = [], []\n num_train_pids, num_test_pids = 0, 0\n num_train_imgs, num_test_imgs = 0, 0\n for i, (campid, pid, img_paths) in enumerate(meta_data):\n\n if [campid, pid] in test_split:\n for img_path in img_paths:\n camid = int(\n osp.basename(img_path).split('_')[2]\n ) - 1 # make it 0-based\n test.append((img_path, num_test_pids, camid))\n num_test_pids += 1\n num_test_imgs += len(img_paths)\n else:\n for img_path in img_paths:\n camid = int(\n osp.basename(img_path).split('_')[2]\n ) - 1 # make it 0-based\n train.append((img_path, num_train_pids, camid))\n num_train_pids += 1\n num_train_imgs += len(img_paths)\n return train, num_train_pids, num_train_imgs, test, num_test_pids, num_test_imgs\n\n print('Creating classic splits (# = 20) ...')\n splits_classic_det, splits_classic_lab = [], []\n for split_ref in mat['testsets'][0]:\n test_split = _deref(split_ref).tolist()\n\n # create split for detected images\n train, num_train_pids, num_train_imgs, test, num_test_pids, num_test_imgs = \\\n _extract_classic_split(meta_detected, test_split)\n splits_classic_det.append(\n {\n 'train': train,\n 'query': test,\n 'gallery': test,\n 'num_train_pids': num_train_pids,\n 'num_train_imgs': num_train_imgs,\n 'num_query_pids': num_test_pids,\n 'num_query_imgs': num_test_imgs,\n 'num_gallery_pids': num_test_pids,\n 'num_gallery_imgs': num_test_imgs\n }\n )\n\n # create split for labeled images\n train, num_train_pids, num_train_imgs, test, num_test_pids, num_test_imgs = \\\n _extract_classic_split(meta_labeled, test_split)\n splits_classic_lab.append(\n {\n 'train': train,\n 'query': test,\n 'gallery': test,\n 'num_train_pids': num_train_pids,\n 'num_train_imgs': num_train_imgs,\n 'num_query_pids': num_test_pids,\n 'num_query_imgs': num_test_imgs,\n 'num_gallery_pids': num_test_pids,\n 'num_gallery_imgs': num_test_imgs\n }\n )\n\n write_json(splits_classic_det, self.split_classic_det_json_path)\n write_json(splits_classic_lab, self.split_classic_lab_json_path)\n\n def _extract_set(filelist, pids, pid2label, idxs, img_dir, relabel):\n tmp_set = []\n unique_pids = set()\n for idx in idxs:\n img_name = filelist[idx][0]\n camid = int(img_name.split('_')[2]) - 1 # make it 0-based\n pid = pids[idx]\n if relabel:\n pid = pid2label[pid]\n img_path = osp.join(img_dir, img_name)\n tmp_set.append((img_path, int(pid), camid))\n unique_pids.add(pid)\n return tmp_set, len(unique_pids), len(idxs)\n\n def _extract_new_split(split_dict, img_dir):\n train_idxs = split_dict['train_idx'].flatten() - 1 # index-0\n pids = split_dict['labels'].flatten()\n train_pids = set(pids[train_idxs])\n pid2label = {pid: label for label, pid in enumerate(train_pids)}\n query_idxs = split_dict['query_idx'].flatten() - 1\n gallery_idxs = split_dict['gallery_idx'].flatten() - 1\n filelist = split_dict['filelist'].flatten()\n train_info = _extract_set(\n filelist, pids, pid2label, train_idxs, img_dir, relabel=True\n )\n query_info = _extract_set(\n filelist, pids, pid2label, query_idxs, img_dir, relabel=False\n )\n gallery_info = _extract_set(\n filelist,\n pids,\n pid2label,\n gallery_idxs,\n img_dir,\n relabel=False\n )\n return train_info, query_info, gallery_info\n\n print('Creating new split for detected images (767/700) ...')\n train_info, query_info, gallery_info = _extract_new_split(\n loadmat(self.split_new_det_mat_path), self.imgs_detected_dir\n )\n split = [\n {\n 'train': train_info[0],\n 'query': query_info[0],\n 'gallery': gallery_info[0],\n 'num_train_pids': train_info[1],\n 'num_train_imgs': train_info[2],\n 'num_query_pids': query_info[1],\n 'num_query_imgs': query_info[2],\n 'num_gallery_pids': gallery_info[1],\n 'num_gallery_imgs': gallery_info[2]\n }\n ]\n write_json(split, self.split_new_det_json_path)\n\n print('Creating new split for labeled images (767/700) ...')\n train_info, query_info, gallery_info = _extract_new_split(\n loadmat(self.split_new_lab_mat_path), self.imgs_labeled_dir\n )\n split = [\n {\n 'train': train_info[0],\n 'query': query_info[0],\n 'gallery': gallery_info[0],\n 'num_train_pids': train_info[1],\n 'num_train_imgs': train_info[2],\n 'num_query_pids': query_info[1],\n 'num_query_imgs': query_info[2],\n 'num_gallery_pids': gallery_info[1],\n 'num_gallery_imgs': gallery_info[2]\n }\n ]\n write_json(split, self.split_new_lab_json_path)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/cuhksysu.py", "content": "from __future__ import division, print_function, absolute_import\nimport copy\nimport glob\nimport os.path as osp\n\nfrom ..dataset import ImageDataset\n\n\nclass CUHKSYSU(ImageDataset):\n \"\"\"CUHKSYSU.\n\n This dataset can only be used for model training.\n\n Reference:\n Xiao et al. End-to-end deep learning for person search.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 11,934\n - images: 34,574\n \"\"\"\n _train_only = True\n dataset_dir = 'cuhksysu'\n\n def __init__(self, root='', **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.data_dir = osp.join(self.dataset_dir, 'cropped_images')\n\n # image name format: p11422_s16929_1.jpg\n train = self.process_dir(self.data_dir)\n query = [copy.deepcopy(train[0])]\n gallery = [copy.deepcopy(train[0])]\n\n super(CUHKSYSU, self).__init__(train, query, gallery, **kwargs)\n\n def process_dir(self, dirname):\n img_paths = glob.glob(osp.join(dirname, '*.jpg'))\n # num_imgs = len(img_paths)\n\n # get all identities:\n pid_container = set()\n for img_path in img_paths:\n img_name = osp.basename(img_path)\n pid = img_name.split('_')[0]\n pid_container.add(pid)\n pid2label = {pid: label for label, pid in enumerate(pid_container)}\n\n # num_pids = len(pid_container)\n\n # extract data\n data = []\n for img_path in img_paths:\n img_name = osp.basename(img_path)\n pid = img_name.split('_')[0]\n label = pid2label[pid]\n data.append((img_path, label, 0)) # dummy camera id\n\n return data\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/dukemtmcreid.py", "content": "from __future__ import division, print_function, absolute_import\nimport re\nimport glob\nimport os.path as osp\n\nfrom ..dataset import ImageDataset\n\n\nclass DukeMTMCreID(ImageDataset):\n \"\"\"DukeMTMC-reID.\n\n Reference:\n - Ristani et al. Performance Measures and a Data Set for Multi-Target, Multi-Camera Tracking. ECCVW 2016.\n - Zheng et al. Unlabeled Samples Generated by GAN Improve the Person Re-identification Baseline in vitro. ICCV 2017.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 1404 (train + query).\n - images:16522 (train) + 2228 (query) + 17661 (gallery).\n - cameras: 8.\n \"\"\"\n dataset_dir = 'dukemtmc-reid'\n dataset_url = 'http://vision.cs.duke.edu/DukeMTMC/data/misc/DukeMTMC-reID.zip'\n\n def __init__(self, root='', **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n self.train_dir = osp.join(\n self.dataset_dir, 'DukeMTMC-reID/bounding_box_train'\n )\n self.query_dir = osp.join(self.dataset_dir, 'DukeMTMC-reID/query')\n self.gallery_dir = osp.join(\n self.dataset_dir, 'DukeMTMC-reID/bounding_box_test'\n )\n\n required_files = [\n self.dataset_dir, self.train_dir, self.query_dir, self.gallery_dir\n ]\n self.check_before_run(required_files)\n\n train = self.process_dir(self.train_dir, relabel=True)\n query = self.process_dir(self.query_dir, relabel=False)\n gallery = self.process_dir(self.gallery_dir, relabel=False)\n\n super(DukeMTMCreID, self).__init__(train, query, gallery, **kwargs)\n\n def process_dir(self, dir_path, relabel=False):\n img_paths = glob.glob(osp.join(dir_path, '*.jpg'))\n pattern = re.compile(r'([-\\d]+)_c(\\d)')\n\n pid_container = set()\n for img_path in img_paths:\n pid, _ = map(int, pattern.search(img_path).groups())\n pid_container.add(pid)\n pid2label = {pid: label for label, pid in enumerate(pid_container)}\n\n data = []\n for img_path in img_paths:\n pid, camid = map(int, pattern.search(img_path).groups())\n assert 1 <= camid <= 8\n camid -= 1 # index starts from 0\n if relabel:\n pid = pid2label[pid]\n data.append((img_path, pid, camid))\n\n return data\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/grid.py", "content": "from __future__ import division, print_function, absolute_import\nimport glob\nimport os.path as osp\nfrom scipy.io import loadmat\n\nfrom torchreid.utils import read_json, write_json\n\nfrom ..dataset import ImageDataset\n\n\nclass GRID(ImageDataset):\n \"\"\"GRID.\n\n Reference:\n Loy et al. Multi-camera activity correlation analysis. CVPR 2009.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 250.\n - images: 1275.\n - cameras: 8.\n \"\"\"\n dataset_dir = 'grid'\n dataset_url = 'http://personal.ie.cuhk.edu.hk/~ccloy/files/datasets/underground_reid.zip'\n _junk_pids = [0]\n\n def __init__(self, root='', split_id=0, **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.probe_path = osp.join(\n self.dataset_dir, 'underground_reid', 'probe'\n )\n self.gallery_path = osp.join(\n self.dataset_dir, 'underground_reid', 'gallery'\n )\n self.split_mat_path = osp.join(\n self.dataset_dir, 'underground_reid', 'features_and_partitions.mat'\n )\n self.split_path = osp.join(self.dataset_dir, 'splits.json')\n\n required_files = [\n self.dataset_dir, self.probe_path, self.gallery_path,\n self.split_mat_path\n ]\n self.check_before_run(required_files)\n\n self.prepare_split()\n splits = read_json(self.split_path)\n if split_id >= len(splits):\n raise ValueError(\n 'split_id exceeds range, received {}, '\n 'but expected between 0 and {}'.format(\n split_id,\n len(splits) - 1\n )\n )\n split = splits[split_id]\n\n train = split['train']\n query = split['query']\n gallery = split['gallery']\n\n train = [tuple(item) for item in train]\n query = [tuple(item) for item in query]\n gallery = [tuple(item) for item in gallery]\n\n super(GRID, self).__init__(train, query, gallery, **kwargs)\n\n def prepare_split(self):\n if not osp.exists(self.split_path):\n print('Creating 10 random splits')\n split_mat = loadmat(self.split_mat_path)\n trainIdxAll = split_mat['trainIdxAll'][0] # length = 10\n probe_img_paths = sorted(\n glob.glob(osp.join(self.probe_path, '*.jpeg'))\n )\n gallery_img_paths = sorted(\n glob.glob(osp.join(self.gallery_path, '*.jpeg'))\n )\n\n splits = []\n for split_idx in range(10):\n train_idxs = trainIdxAll[split_idx][0][0][2][0].tolist()\n assert len(train_idxs) == 125\n idx2label = {\n idx: label\n for label, idx in enumerate(train_idxs)\n }\n\n train, query, gallery = [], [], []\n\n # processing probe folder\n for img_path in probe_img_paths:\n img_name = osp.basename(img_path)\n img_idx = int(img_name.split('_')[0])\n camid = int(\n img_name.split('_')[1]\n ) - 1 # index starts from 0\n if img_idx in train_idxs:\n train.append((img_path, idx2label[img_idx], camid))\n else:\n query.append((img_path, img_idx, camid))\n\n # process gallery folder\n for img_path in gallery_img_paths:\n img_name = osp.basename(img_path)\n img_idx = int(img_name.split('_')[0])\n camid = int(\n img_name.split('_')[1]\n ) - 1 # index starts from 0\n if img_idx in train_idxs:\n train.append((img_path, idx2label[img_idx], camid))\n else:\n gallery.append((img_path, img_idx, camid))\n\n split = {\n 'train': train,\n 'query': query,\n 'gallery': gallery,\n 'num_train_pids': 125,\n 'num_query_pids': 125,\n 'num_gallery_pids': 900\n }\n splits.append(split)\n\n print('Totally {} splits are created'.format(len(splits)))\n write_json(splits, self.split_path)\n print('Split file saved to {}'.format(self.split_path))\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/ilids.py", "content": "from __future__ import division, print_function, absolute_import\nimport copy\nimport glob\nimport random\nimport os.path as osp\nfrom collections import defaultdict\n\nfrom torchreid.utils import read_json, write_json\n\nfrom ..dataset import ImageDataset\n\n\nclass iLIDS(ImageDataset):\n \"\"\"QMUL-iLIDS.\n\n Reference:\n Zheng et al. Associating Groups of People. BMVC 2009.\n \n Dataset statistics:\n - identities: 119.\n - images: 476.\n - cameras: 8 (not explicitly provided).\n \"\"\"\n dataset_dir = 'ilids'\n dataset_url = 'http://www.eecs.qmul.ac.uk/~jason/data/i-LIDS_Pedestrian.tgz'\n\n def __init__(self, root='', split_id=0, **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.data_dir = osp.join(self.dataset_dir, 'i-LIDS_Pedestrian/Persons')\n self.split_path = osp.join(self.dataset_dir, 'splits.json')\n\n required_files = [self.dataset_dir, self.data_dir]\n self.check_before_run(required_files)\n\n self.prepare_split()\n splits = read_json(self.split_path)\n if split_id >= len(splits):\n raise ValueError(\n 'split_id exceeds range, received {}, but '\n 'expected between 0 and {}'.format(split_id,\n len(splits) - 1)\n )\n split = splits[split_id]\n\n train, query, gallery = self.process_split(split)\n\n super(iLIDS, self).__init__(train, query, gallery, **kwargs)\n\n def prepare_split(self):\n if not osp.exists(self.split_path):\n print('Creating splits ...')\n\n paths = glob.glob(osp.join(self.data_dir, '*.jpg'))\n img_names = [osp.basename(path) for path in paths]\n num_imgs = len(img_names)\n assert num_imgs == 476, 'There should be 476 images, but ' \\\n 'got {}, please check the data'.format(num_imgs)\n\n # store image names\n # image naming format:\n # the first four digits denote the person ID\n # the last four digits denote the sequence index\n pid_dict = defaultdict(list)\n for img_name in img_names:\n pid = int(img_name[:4])\n pid_dict[pid].append(img_name)\n pids = list(pid_dict.keys())\n num_pids = len(pids)\n assert num_pids == 119, 'There should be 119 identities, ' \\\n 'but got {}, please check the data'.format(num_pids)\n\n num_train_pids = int(num_pids * 0.5)\n\n splits = []\n for _ in range(10):\n # randomly choose num_train_pids train IDs and the rest for test IDs\n pids_copy = copy.deepcopy(pids)\n random.shuffle(pids_copy)\n train_pids = pids_copy[:num_train_pids]\n test_pids = pids_copy[num_train_pids:]\n\n train = []\n query = []\n gallery = []\n\n # for train IDs, all images are used in the train set.\n for pid in train_pids:\n img_names = pid_dict[pid]\n train.extend(img_names)\n\n # for each test ID, randomly choose two images, one for\n # query and the other one for gallery.\n for pid in test_pids:\n img_names = pid_dict[pid]\n samples = random.sample(img_names, 2)\n query.append(samples[0])\n gallery.append(samples[1])\n\n split = {'train': train, 'query': query, 'gallery': gallery}\n splits.append(split)\n\n print('Totally {} splits are created'.format(len(splits)))\n write_json(splits, self.split_path)\n print('Split file is saved to {}'.format(self.split_path))\n\n def get_pid2label(self, img_names):\n pid_container = set()\n for img_name in img_names:\n pid = int(img_name[:4])\n pid_container.add(pid)\n pid2label = {pid: label for label, pid in enumerate(pid_container)}\n return pid2label\n\n def parse_img_names(self, img_names, pid2label=None):\n data = []\n\n for img_name in img_names:\n pid = int(img_name[:4])\n if pid2label is not None:\n pid = pid2label[pid]\n camid = int(img_name[4:7]) - 1 # 0-based\n img_path = osp.join(self.data_dir, img_name)\n data.append((img_path, pid, camid))\n\n return data\n\n def process_split(self, split):\n train_pid2label = self.get_pid2label(split['train'])\n train = self.parse_img_names(split['train'], train_pid2label)\n query = self.parse_img_names(split['query'])\n gallery = self.parse_img_names(split['gallery'])\n return train, query, gallery\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/market1501.py", "content": "from __future__ import division, print_function, absolute_import\nimport re\nimport glob\nimport os.path as osp\nimport warnings\n\nfrom ..dataset import ImageDataset\n\n\nclass Market1501(ImageDataset):\n \"\"\"Market1501.\n\n Reference:\n Zheng et al. Scalable Person Re-identification: A Benchmark. ICCV 2015.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 1501 (+1 for background).\n - images: 12936 (train) + 3368 (query) + 15913 (gallery).\n \"\"\"\n _junk_pids = [0, -1]\n dataset_dir = 'market1501'\n dataset_url = 'http://188.138.127.15:81/Datasets/Market-1501-v15.09.15.zip'\n\n def __init__(self, root='', market1501_500k=False, **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n # allow alternative directory structure\n self.data_dir = self.dataset_dir\n data_dir = osp.join(self.data_dir, 'Market-1501-v15.09.15')\n if osp.isdir(data_dir):\n self.data_dir = data_dir\n else:\n warnings.warn(\n 'The current data structure is deprecated. Please '\n 'put data folders such as \"bounding_box_train\" under '\n '\"Market-1501-v15.09.15\".'\n )\n\n self.train_dir = osp.join(self.data_dir, 'bounding_box_train')\n self.query_dir = osp.join(self.data_dir, 'query')\n self.gallery_dir = osp.join(self.data_dir, 'bounding_box_test')\n self.extra_gallery_dir = osp.join(self.data_dir, 'images')\n self.market1501_500k = market1501_500k\n\n required_files = [\n self.data_dir, self.train_dir, self.query_dir, self.gallery_dir\n ]\n if self.market1501_500k:\n required_files.append(self.extra_gallery_dir)\n self.check_before_run(required_files)\n\n train = self.process_dir(self.train_dir, relabel=True)\n query = self.process_dir(self.query_dir, relabel=False)\n gallery = self.process_dir(self.gallery_dir, relabel=False)\n if self.market1501_500k:\n gallery += self.process_dir(self.extra_gallery_dir, relabel=False)\n\n super(Market1501, self).__init__(train, query, gallery, **kwargs)\n\n def process_dir(self, dir_path, relabel=False):\n img_paths = glob.glob(osp.join(dir_path, '*.jpg'))\n pattern = re.compile(r'([-\\d]+)_c(\\d)')\n\n pid_container = set()\n for img_path in img_paths:\n pid, _ = map(int, pattern.search(img_path).groups())\n if pid == -1:\n continue # junk images are just ignored\n pid_container.add(pid)\n pid2label = {pid: label for label, pid in enumerate(pid_container)}\n\n data = []\n for img_path in img_paths:\n pid, camid = map(int, pattern.search(img_path).groups())\n if pid == -1:\n continue # junk images are just ignored\n assert 0 <= pid <= 1501 # pid == 0 means background\n assert 1 <= camid <= 6\n camid -= 1 # index starts from 0\n if relabel:\n pid = pid2label[pid]\n data.append((img_path, pid, camid))\n\n return data\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/msmt17.py", "content": "from __future__ import division, print_function, absolute_import\nimport os.path as osp\n\nfrom ..dataset import ImageDataset\n\n# Log\n# 22.01.2019\n# - add v2\n# - v1 and v2 differ in dir names\n# - note that faces in v2 are blurred\nTRAIN_DIR_KEY = 'train_dir'\nTEST_DIR_KEY = 'test_dir'\nVERSION_DICT = {\n 'MSMT17_V1': {\n TRAIN_DIR_KEY: 'train',\n TEST_DIR_KEY: 'test',\n },\n 'MSMT17_V2': {\n TRAIN_DIR_KEY: 'mask_train_v2',\n TEST_DIR_KEY: 'mask_test_v2',\n }\n}\n\n\nclass MSMT17(ImageDataset):\n \"\"\"MSMT17.\n\n Reference:\n Wei et al. Person Transfer GAN to Bridge Domain Gap for Person Re-Identification. CVPR 2018.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 4101.\n - images: 32621 (train) + 11659 (query) + 82161 (gallery).\n - cameras: 15.\n \"\"\"\n dataset_dir = 'msmt17'\n dataset_url = None\n\n def __init__(self, root='', **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n has_main_dir = False\n for main_dir in VERSION_DICT:\n if osp.exists(osp.join(self.dataset_dir, main_dir)):\n train_dir = VERSION_DICT[main_dir][TRAIN_DIR_KEY]\n test_dir = VERSION_DICT[main_dir][TEST_DIR_KEY]\n has_main_dir = True\n break\n assert has_main_dir, 'Dataset folder not found'\n\n self.train_dir = osp.join(self.dataset_dir, main_dir, train_dir)\n self.test_dir = osp.join(self.dataset_dir, main_dir, test_dir)\n self.list_train_path = osp.join(\n self.dataset_dir, main_dir, 'list_train.txt'\n )\n self.list_val_path = osp.join(\n self.dataset_dir, main_dir, 'list_val.txt'\n )\n self.list_query_path = osp.join(\n self.dataset_dir, main_dir, 'list_query.txt'\n )\n self.list_gallery_path = osp.join(\n self.dataset_dir, main_dir, 'list_gallery.txt'\n )\n\n required_files = [self.dataset_dir, self.train_dir, self.test_dir]\n self.check_before_run(required_files)\n\n train = self.process_dir(self.train_dir, self.list_train_path)\n val = self.process_dir(self.train_dir, self.list_val_path)\n query = self.process_dir(self.test_dir, self.list_query_path)\n gallery = self.process_dir(self.test_dir, self.list_gallery_path)\n\n # Note: to fairly compare with published methods on the conventional ReID setting,\n # do not add val images to the training set.\n if 'combineall' in kwargs and kwargs['combineall']:\n train += val\n\n super(MSMT17, self).__init__(train, query, gallery, **kwargs)\n\n def process_dir(self, dir_path, list_path):\n with open(list_path, 'r') as txt:\n lines = txt.readlines()\n\n data = []\n\n for img_idx, img_info in enumerate(lines):\n img_path, pid = img_info.split(' ')\n pid = int(pid) # no need to relabel\n camid = int(img_path.split('_')[2]) - 1 # index starts from 0\n img_path = osp.join(dir_path, img_path)\n data.append((img_path, pid, camid))\n\n return data\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/prid.py", "content": "from __future__ import division, print_function, absolute_import\nimport random\nimport os.path as osp\n\nfrom torchreid.utils import read_json, write_json\n\nfrom ..dataset import ImageDataset\n\n\nclass PRID(ImageDataset):\n \"\"\"PRID (single-shot version of prid-2011)\n\n Reference:\n Hirzer et al. Person Re-Identification by Descriptive and Discriminative\n Classification. SCIA 2011.\n\n URL: ``_\n \n Dataset statistics:\n - Two views.\n - View A captures 385 identities.\n - View B captures 749 identities.\n - 200 identities appear in both views (index starts from 1 to 200).\n \"\"\"\n dataset_dir = 'prid2011'\n dataset_url = None\n _junk_pids = list(range(201, 750))\n\n def __init__(self, root='', split_id=0, **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.cam_a_dir = osp.join(\n self.dataset_dir, 'prid_2011', 'single_shot', 'cam_a'\n )\n self.cam_b_dir = osp.join(\n self.dataset_dir, 'prid_2011', 'single_shot', 'cam_b'\n )\n self.split_path = osp.join(self.dataset_dir, 'splits_single_shot.json')\n\n required_files = [self.dataset_dir, self.cam_a_dir, self.cam_b_dir]\n self.check_before_run(required_files)\n\n self.prepare_split()\n splits = read_json(self.split_path)\n if split_id >= len(splits):\n raise ValueError(\n 'split_id exceeds range, received {}, but expected between 0 and {}'\n .format(split_id,\n len(splits) - 1)\n )\n split = splits[split_id]\n\n train, query, gallery = self.process_split(split)\n\n super(PRID, self).__init__(train, query, gallery, **kwargs)\n\n def prepare_split(self):\n if not osp.exists(self.split_path):\n print('Creating splits ...')\n\n splits = []\n for _ in range(10):\n # randomly sample 100 IDs for train and use the rest 100 IDs for test\n # (note: there are only 200 IDs appearing in both views)\n pids = [i for i in range(1, 201)]\n train_pids = random.sample(pids, 100)\n train_pids.sort()\n test_pids = [i for i in pids if i not in train_pids]\n split = {'train': train_pids, 'test': test_pids}\n splits.append(split)\n\n print('Totally {} splits are created'.format(len(splits)))\n write_json(splits, self.split_path)\n print('Split file is saved to {}'.format(self.split_path))\n\n def process_split(self, split):\n train_pids = split['train']\n test_pids = split['test']\n\n train_pid2label = {pid: label for label, pid in enumerate(train_pids)}\n\n # train\n train = []\n for pid in train_pids:\n img_name = 'person_' + str(pid).zfill(4) + '.png'\n pid = train_pid2label[pid]\n img_a_path = osp.join(self.cam_a_dir, img_name)\n train.append((img_a_path, pid, 0))\n img_b_path = osp.join(self.cam_b_dir, img_name)\n train.append((img_b_path, pid, 1))\n\n # query and gallery\n query, gallery = [], []\n for pid in test_pids:\n img_name = 'person_' + str(pid).zfill(4) + '.png'\n img_a_path = osp.join(self.cam_a_dir, img_name)\n query.append((img_a_path, pid, 0))\n img_b_path = osp.join(self.cam_b_dir, img_name)\n gallery.append((img_b_path, pid, 1))\n for pid in range(201, 750):\n img_name = 'person_' + str(pid).zfill(4) + '.png'\n img_b_path = osp.join(self.cam_b_dir, img_name)\n gallery.append((img_b_path, pid, 1))\n\n return train, query, gallery\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/sensereid.py", "content": "from __future__ import division, print_function, absolute_import\nimport copy\nimport glob\nimport os.path as osp\n\nfrom ..dataset import ImageDataset\n\n\nclass SenseReID(ImageDataset):\n \"\"\"SenseReID.\n\n This dataset is used for test purpose only.\n\n Reference:\n Zhao et al. Spindle Net: Person Re-identification with Human Body\n Region Guided Feature Decomposition and Fusion. CVPR 2017.\n\n URL: ``_\n\n Dataset statistics:\n - query: 522 ids, 1040 images.\n - gallery: 1717 ids, 3388 images.\n \"\"\"\n dataset_dir = 'sensereid'\n dataset_url = None\n\n def __init__(self, root='', **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.query_dir = osp.join(self.dataset_dir, 'SenseReID', 'test_probe')\n self.gallery_dir = osp.join(\n self.dataset_dir, 'SenseReID', 'test_gallery'\n )\n\n required_files = [self.dataset_dir, self.query_dir, self.gallery_dir]\n self.check_before_run(required_files)\n\n query = self.process_dir(self.query_dir)\n gallery = self.process_dir(self.gallery_dir)\n\n # relabel\n g_pids = set()\n for _, pid, _ in gallery:\n g_pids.add(pid)\n pid2label = {pid: i for i, pid in enumerate(g_pids)}\n\n query = [\n (img_path, pid2label[pid], camid) for img_path, pid, camid in query\n ]\n gallery = [\n (img_path, pid2label[pid], camid)\n for img_path, pid, camid in gallery\n ]\n train = copy.deepcopy(query) + copy.deepcopy(gallery) # dummy variable\n\n super(SenseReID, self).__init__(train, query, gallery, **kwargs)\n\n def process_dir(self, dir_path):\n img_paths = glob.glob(osp.join(dir_path, '*.jpg'))\n data = []\n\n for img_path in img_paths:\n img_name = osp.splitext(osp.basename(img_path))[0]\n pid, camid = img_name.split('_')\n pid, camid = int(pid), int(camid)\n data.append((img_path, pid, camid))\n\n return data\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/university1652.py", "content": "from __future__ import division, print_function, absolute_import\nimport os\nimport glob\nimport os.path as osp\nimport gdown\n\nfrom ..dataset import ImageDataset\n\n\nclass University1652(ImageDataset):\n \"\"\"University-1652.\n\n Reference:\n - Zheng et al. University-1652: A Multi-view Multi-source Benchmark for Drone-based Geo-localization. ACM MM 2020.\n\n URL: ``_\n OneDrive:\n https://studentutsedu-my.sharepoint.com/:u:/g/personal/12639605_student_uts_edu_au/Ecrz6xK-PcdCjFdpNb0T0s8B_9J5ynaUy3q63_XumjJyrA?e=z4hpcz\n [Backup] GoogleDrive:\n https://drive.google.com/file/d/1iVnP4gjw-iHXa0KerZQ1IfIO0i1jADsR/view?usp=sharing\n [Backup] Baidu Yun:\n https://pan.baidu.com/s/1H_wBnWwikKbaBY1pMPjoqQ password: hrqp\n \n Dataset statistics:\n - buildings: 1652 (train + query).\n - The dataset split is as follows: \n | Split | #imgs | #buildings | #universities|\n | -------- | ----- | ----| ----|\n | Training | 50,218 | 701 | 33 |\n | Query_drone | 37,855 | 701 | 39 |\n | Query_satellite | 701 | 701 | 39|\n | Query_ground | 2,579 | 701 | 39|\n | Gallery_drone | 51,355 | 951 | 39|\n | Gallery_satellite | 951 | 951 | 39|\n | Gallery_ground | 2,921 | 793 | 39|\n - cameras: None.\n \n datamanager = torchreid.data.ImageDataManager(\n root='reid-data',\n sources='university1652',\n targets='university1652',\n height=256,\n width=256,\n batch_size_train=32,\n batch_size_test=100,\n transforms=['random_flip', 'random_crop']\n )\n \"\"\"\n dataset_dir = 'university1652'\n dataset_url = 'https://drive.google.com/uc?id=1iVnP4gjw-iHXa0KerZQ1IfIO0i1jADsR'\n\n def __init__(self, root='', **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n print(self.dataset_dir)\n if not os.path.isdir(self.dataset_dir):\n os.mkdir(self.dataset_dir)\n gdown.download(\n self.dataset_url, self.dataset_dir + 'data.zip', quiet=False\n )\n os.system('unzip %s' % (self.dataset_dir + 'data.zip'))\n self.train_dir = osp.join(\n self.dataset_dir, 'University-Release/train/'\n )\n self.query_dir = osp.join(\n self.dataset_dir, 'University-Release/test/query_drone'\n )\n self.gallery_dir = osp.join(\n self.dataset_dir, 'University-Release/test/gallery_satellite'\n )\n\n required_files = [\n self.dataset_dir, self.train_dir, self.query_dir, self.gallery_dir\n ]\n self.check_before_run(required_files)\n\n self.fake_camid = 0\n train = self.process_dir(self.train_dir, relabel=True, train=True)\n query = self.process_dir(self.query_dir, relabel=False)\n gallery = self.process_dir(self.gallery_dir, relabel=False)\n\n super(University1652, self).__init__(train, query, gallery, **kwargs)\n\n def process_dir(self, dir_path, relabel=False, train=False):\n IMG_EXTENSIONS = (\n '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff',\n '.webp'\n )\n if train:\n img_paths = glob.glob(osp.join(dir_path, '*/*/*'))\n else:\n img_paths = glob.glob(osp.join(dir_path, '*/*'))\n pid_container = set()\n for img_path in img_paths:\n if not img_path.lower().endswith(IMG_EXTENSIONS):\n continue\n pid = int(os.path.basename(os.path.dirname(img_path)))\n pid_container.add(pid)\n pid2label = {pid: label for label, pid in enumerate(pid_container)}\n data = []\n # no camera for university\n for img_path in img_paths:\n if not img_path.lower().endswith(IMG_EXTENSIONS):\n continue\n pid = int(os.path.basename(os.path.dirname(img_path)))\n if relabel:\n pid = pid2label[pid]\n data.append((img_path, pid, self.fake_camid))\n self.fake_camid += 1\n return data\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/image/viper.py", "content": "from __future__ import division, print_function, absolute_import\nimport glob\nimport numpy as np\nimport os.path as osp\n\nfrom torchreid.utils import read_json, write_json\n\nfrom ..dataset import ImageDataset\n\n\nclass VIPeR(ImageDataset):\n \"\"\"VIPeR.\n\n Reference:\n Gray et al. Evaluating appearance models for recognition, reacquisition, and tracking. PETS 2007.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 632.\n - images: 632 x 2 = 1264.\n - cameras: 2.\n \"\"\"\n dataset_dir = 'viper'\n dataset_url = 'http://users.soe.ucsc.edu/~manduchi/VIPeR.v1.0.zip'\n\n def __init__(self, root='', split_id=0, **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.cam_a_dir = osp.join(self.dataset_dir, 'VIPeR', 'cam_a')\n self.cam_b_dir = osp.join(self.dataset_dir, 'VIPeR', 'cam_b')\n self.split_path = osp.join(self.dataset_dir, 'splits.json')\n\n required_files = [self.dataset_dir, self.cam_a_dir, self.cam_b_dir]\n self.check_before_run(required_files)\n\n self.prepare_split()\n splits = read_json(self.split_path)\n if split_id >= len(splits):\n raise ValueError(\n 'split_id exceeds range, received {}, '\n 'but expected between 0 and {}'.format(\n split_id,\n len(splits) - 1\n )\n )\n split = splits[split_id]\n\n train = split['train']\n query = split['query'] # query and gallery share the same images\n gallery = split['gallery']\n\n train = [tuple(item) for item in train]\n query = [tuple(item) for item in query]\n gallery = [tuple(item) for item in gallery]\n\n super(VIPeR, self).__init__(train, query, gallery, **kwargs)\n\n def prepare_split(self):\n if not osp.exists(self.split_path):\n print('Creating 10 random splits of train ids and test ids')\n\n cam_a_imgs = sorted(glob.glob(osp.join(self.cam_a_dir, '*.bmp')))\n cam_b_imgs = sorted(glob.glob(osp.join(self.cam_b_dir, '*.bmp')))\n assert len(cam_a_imgs) == len(cam_b_imgs)\n num_pids = len(cam_a_imgs)\n print('Number of identities: {}'.format(num_pids))\n num_train_pids = num_pids // 2\n \"\"\"\n In total, there will be 20 splits because each random split creates two\n sub-splits, one using cameraA as query and cameraB as gallery\n while the other using cameraB as query and cameraA as gallery.\n Therefore, results should be averaged over 20 splits (split_id=0~19).\n \n In practice, a model trained on split_id=0 can be applied to split_id=0&1\n as split_id=0&1 share the same training data (so on and so forth).\n \"\"\"\n splits = []\n for _ in range(10):\n order = np.arange(num_pids)\n np.random.shuffle(order)\n train_idxs = order[:num_train_pids]\n test_idxs = order[num_train_pids:]\n assert not bool(set(train_idxs) & set(test_idxs)), \\\n 'Error: train and test overlap'\n\n train = []\n for pid, idx in enumerate(train_idxs):\n cam_a_img = cam_a_imgs[idx]\n cam_b_img = cam_b_imgs[idx]\n train.append((cam_a_img, pid, 0))\n train.append((cam_b_img, pid, 1))\n\n test_a = []\n test_b = []\n for pid, idx in enumerate(test_idxs):\n cam_a_img = cam_a_imgs[idx]\n cam_b_img = cam_b_imgs[idx]\n test_a.append((cam_a_img, pid, 0))\n test_b.append((cam_b_img, pid, 1))\n\n # use cameraA as query and cameraB as gallery\n split = {\n 'train': train,\n 'query': test_a,\n 'gallery': test_b,\n 'num_train_pids': num_train_pids,\n 'num_query_pids': num_pids - num_train_pids,\n 'num_gallery_pids': num_pids - num_train_pids\n }\n splits.append(split)\n\n # use cameraB as query and cameraA as gallery\n split = {\n 'train': train,\n 'query': test_b,\n 'gallery': test_a,\n 'num_train_pids': num_train_pids,\n 'num_query_pids': num_pids - num_train_pids,\n 'num_gallery_pids': num_pids - num_train_pids\n }\n splits.append(split)\n\n print('Totally {} splits are created'.format(len(splits)))\n write_json(splits, self.split_path)\n print('Split file saved to {}'.format(self.split_path))\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/video/__init__.py", "content": "from __future__ import print_function, absolute_import\n\nfrom .mars import Mars\nfrom .ilidsvid import iLIDSVID\nfrom .prid2011 import PRID2011\nfrom .dukemtmcvidreid import DukeMTMCVidReID\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/video/dukemtmcvidreid.py", "content": "from __future__ import division, print_function, absolute_import\nimport glob\nimport os.path as osp\nimport warnings\n\nfrom torchreid.utils import read_json, write_json\n\nfrom ..dataset import VideoDataset\n\n\nclass DukeMTMCVidReID(VideoDataset):\n \"\"\"DukeMTMCVidReID.\n\n Reference:\n - Ristani et al. Performance Measures and a Data Set for Multi-Target,\n Multi-Camera Tracking. ECCVW 2016.\n - Wu et al. Exploit the Unknown Gradually: One-Shot Video-Based Person\n Re-Identification by Stepwise Learning. CVPR 2018.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 702 (train) + 702 (test).\n - tracklets: 2196 (train) + 2636 (test).\n \"\"\"\n dataset_dir = 'dukemtmc-vidreid'\n dataset_url = 'http://vision.cs.duke.edu/DukeMTMC/data/misc/DukeMTMC-VideoReID.zip'\n\n def __init__(self, root='', min_seq_len=0, **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.train_dir = osp.join(self.dataset_dir, 'DukeMTMC-VideoReID/train')\n self.query_dir = osp.join(self.dataset_dir, 'DukeMTMC-VideoReID/query')\n self.gallery_dir = osp.join(\n self.dataset_dir, 'DukeMTMC-VideoReID/gallery'\n )\n self.split_train_json_path = osp.join(\n self.dataset_dir, 'split_train.json'\n )\n self.split_query_json_path = osp.join(\n self.dataset_dir, 'split_query.json'\n )\n self.split_gallery_json_path = osp.join(\n self.dataset_dir, 'split_gallery.json'\n )\n self.min_seq_len = min_seq_len\n\n required_files = [\n self.dataset_dir, self.train_dir, self.query_dir, self.gallery_dir\n ]\n self.check_before_run(required_files)\n\n train = self.process_dir(\n self.train_dir, self.split_train_json_path, relabel=True\n )\n query = self.process_dir(\n self.query_dir, self.split_query_json_path, relabel=False\n )\n gallery = self.process_dir(\n self.gallery_dir, self.split_gallery_json_path, relabel=False\n )\n\n super(DukeMTMCVidReID, self).__init__(train, query, gallery, **kwargs)\n\n def process_dir(self, dir_path, json_path, relabel):\n if osp.exists(json_path):\n split = read_json(json_path)\n return split['tracklets']\n\n print('=> Generating split json file (** this might take a while **)')\n pdirs = glob.glob(osp.join(dir_path, '*')) # avoid .DS_Store\n print(\n 'Processing \"{}\" with {} person identities'.format(\n dir_path, len(pdirs)\n )\n )\n\n pid_container = set()\n for pdir in pdirs:\n pid = int(osp.basename(pdir))\n pid_container.add(pid)\n pid2label = {pid: label for label, pid in enumerate(pid_container)}\n\n tracklets = []\n for pdir in pdirs:\n pid = int(osp.basename(pdir))\n if relabel:\n pid = pid2label[pid]\n tdirs = glob.glob(osp.join(pdir, '*'))\n for tdir in tdirs:\n raw_img_paths = glob.glob(osp.join(tdir, '*.jpg'))\n num_imgs = len(raw_img_paths)\n\n if num_imgs < self.min_seq_len:\n continue\n\n img_paths = []\n for img_idx in range(num_imgs):\n # some tracklet starts from 0002 instead of 0001\n img_idx_name = 'F' + str(img_idx + 1).zfill(4)\n res = glob.glob(\n osp.join(tdir, '*' + img_idx_name + '*.jpg')\n )\n if len(res) == 0:\n warnings.warn(\n 'Index name {} in {} is missing, skip'.format(\n img_idx_name, tdir\n )\n )\n continue\n img_paths.append(res[0])\n img_name = osp.basename(img_paths[0])\n if img_name.find('_') == -1:\n # old naming format: 0001C6F0099X30823.jpg\n camid = int(img_name[5]) - 1\n else:\n # new naming format: 0001_C6_F0099_X30823.jpg\n camid = int(img_name[6]) - 1\n img_paths = tuple(img_paths)\n tracklets.append((img_paths, pid, camid))\n\n print('Saving split to {}'.format(json_path))\n split_dict = {'tracklets': tracklets}\n write_json(split_dict, json_path)\n\n return tracklets\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/video/ilidsvid.py", "content": "from __future__ import division, print_function, absolute_import\nimport glob\nimport os.path as osp\nfrom scipy.io import loadmat\n\nfrom torchreid.utils import read_json, write_json\n\nfrom ..dataset import VideoDataset\n\n\nclass iLIDSVID(VideoDataset):\n \"\"\"iLIDS-VID.\n\n Reference:\n Wang et al. Person Re-Identification by Video Ranking. ECCV 2014.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 300.\n - tracklets: 600.\n - cameras: 2.\n \"\"\"\n dataset_dir = 'ilids-vid'\n dataset_url = 'http://www.eecs.qmul.ac.uk/~xiatian/iLIDS-VID/iLIDS-VID.tar'\n\n def __init__(self, root='', split_id=0, **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.data_dir = osp.join(self.dataset_dir, 'i-LIDS-VID')\n self.split_dir = osp.join(self.dataset_dir, 'train-test people splits')\n self.split_mat_path = osp.join(\n self.split_dir, 'train_test_splits_ilidsvid.mat'\n )\n self.split_path = osp.join(self.dataset_dir, 'splits.json')\n self.cam_1_path = osp.join(\n self.dataset_dir, 'i-LIDS-VID/sequences/cam1'\n )\n self.cam_2_path = osp.join(\n self.dataset_dir, 'i-LIDS-VID/sequences/cam2'\n )\n\n required_files = [self.dataset_dir, self.data_dir, self.split_dir]\n self.check_before_run(required_files)\n\n self.prepare_split()\n splits = read_json(self.split_path)\n if split_id >= len(splits):\n raise ValueError(\n 'split_id exceeds range, received {}, but expected between 0 and {}'\n .format(split_id,\n len(splits) - 1)\n )\n split = splits[split_id]\n train_dirs, test_dirs = split['train'], split['test']\n\n train = self.process_data(train_dirs, cam1=True, cam2=True)\n query = self.process_data(test_dirs, cam1=True, cam2=False)\n gallery = self.process_data(test_dirs, cam1=False, cam2=True)\n\n super(iLIDSVID, self).__init__(train, query, gallery, **kwargs)\n\n def prepare_split(self):\n if not osp.exists(self.split_path):\n print('Creating splits ...')\n mat_split_data = loadmat(self.split_mat_path)['ls_set']\n\n num_splits = mat_split_data.shape[0]\n num_total_ids = mat_split_data.shape[1]\n assert num_splits == 10\n assert num_total_ids == 300\n num_ids_each = num_total_ids // 2\n\n # pids in mat_split_data are indices, so we need to transform them\n # to real pids\n person_cam1_dirs = sorted(\n glob.glob(osp.join(self.cam_1_path, '*'))\n )\n person_cam2_dirs = sorted(\n glob.glob(osp.join(self.cam_2_path, '*'))\n )\n\n person_cam1_dirs = [\n osp.basename(item) for item in person_cam1_dirs\n ]\n person_cam2_dirs = [\n osp.basename(item) for item in person_cam2_dirs\n ]\n\n # make sure persons in one camera view can be found in the other camera view\n assert set(person_cam1_dirs) == set(person_cam2_dirs)\n\n splits = []\n for i_split in range(num_splits):\n # first 50% for testing and the remaining for training, following Wang et al. ECCV'14.\n train_idxs = sorted(\n list(mat_split_data[i_split, num_ids_each:])\n )\n test_idxs = sorted(\n list(mat_split_data[i_split, :num_ids_each])\n )\n\n train_idxs = [int(i) - 1 for i in train_idxs]\n test_idxs = [int(i) - 1 for i in test_idxs]\n\n # transform pids to person dir names\n train_dirs = [person_cam1_dirs[i] for i in train_idxs]\n test_dirs = [person_cam1_dirs[i] for i in test_idxs]\n\n split = {'train': train_dirs, 'test': test_dirs}\n splits.append(split)\n\n print(\n 'Totally {} splits are created, following Wang et al. ECCV\\'14'\n .format(len(splits))\n )\n print('Split file is saved to {}'.format(self.split_path))\n write_json(splits, self.split_path)\n\n def process_data(self, dirnames, cam1=True, cam2=True):\n tracklets = []\n dirname2pid = {dirname: i for i, dirname in enumerate(dirnames)}\n\n for dirname in dirnames:\n if cam1:\n person_dir = osp.join(self.cam_1_path, dirname)\n img_names = glob.glob(osp.join(person_dir, '*.png'))\n assert len(img_names) > 0\n img_names = tuple(img_names)\n pid = dirname2pid[dirname]\n tracklets.append((img_names, pid, 0))\n\n if cam2:\n person_dir = osp.join(self.cam_2_path, dirname)\n img_names = glob.glob(osp.join(person_dir, '*.png'))\n assert len(img_names) > 0\n img_names = tuple(img_names)\n pid = dirname2pid[dirname]\n tracklets.append((img_names, pid, 1))\n\n return tracklets\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/video/mars.py", "content": "from __future__ import division, print_function, absolute_import\nimport os.path as osp\nimport warnings\nfrom scipy.io import loadmat\n\nfrom ..dataset import VideoDataset\n\n\nclass Mars(VideoDataset):\n \"\"\"MARS.\n\n Reference:\n Zheng et al. MARS: A Video Benchmark for Large-Scale Person Re-identification. ECCV 2016.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 1261.\n - tracklets: 8298 (train) + 1980 (query) + 9330 (gallery).\n - cameras: 6.\n \"\"\"\n dataset_dir = 'mars'\n dataset_url = None\n\n def __init__(self, root='', **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.train_name_path = osp.join(\n self.dataset_dir, 'info/train_name.txt'\n )\n self.test_name_path = osp.join(self.dataset_dir, 'info/test_name.txt')\n self.track_train_info_path = osp.join(\n self.dataset_dir, 'info/tracks_train_info.mat'\n )\n self.track_test_info_path = osp.join(\n self.dataset_dir, 'info/tracks_test_info.mat'\n )\n self.query_IDX_path = osp.join(self.dataset_dir, 'info/query_IDX.mat')\n\n required_files = [\n self.dataset_dir, self.train_name_path, self.test_name_path,\n self.track_train_info_path, self.track_test_info_path,\n self.query_IDX_path\n ]\n self.check_before_run(required_files)\n\n train_names = self.get_names(self.train_name_path)\n test_names = self.get_names(self.test_name_path)\n track_train = loadmat(self.track_train_info_path\n )['track_train_info'] # numpy.ndarray (8298, 4)\n track_test = loadmat(self.track_test_info_path\n )['track_test_info'] # numpy.ndarray (12180, 4)\n query_IDX = loadmat(self.query_IDX_path\n )['query_IDX'].squeeze() # numpy.ndarray (1980,)\n query_IDX -= 1 # index from 0\n track_query = track_test[query_IDX, :]\n gallery_IDX = [\n i for i in range(track_test.shape[0]) if i not in query_IDX\n ]\n track_gallery = track_test[gallery_IDX, :]\n\n train = self.process_data(\n train_names, track_train, home_dir='bbox_train', relabel=True\n )\n query = self.process_data(\n test_names, track_query, home_dir='bbox_test', relabel=False\n )\n gallery = self.process_data(\n test_names, track_gallery, home_dir='bbox_test', relabel=False\n )\n\n super(Mars, self).__init__(train, query, gallery, **kwargs)\n\n def get_names(self, fpath):\n names = []\n with open(fpath, 'r') as f:\n for line in f:\n new_line = line.rstrip()\n names.append(new_line)\n return names\n\n def process_data(\n self, names, meta_data, home_dir=None, relabel=False, min_seq_len=0\n ):\n assert home_dir in ['bbox_train', 'bbox_test']\n num_tracklets = meta_data.shape[0]\n pid_list = list(set(meta_data[:, 2].tolist()))\n\n if relabel:\n pid2label = {pid: label for label, pid in enumerate(pid_list)}\n tracklets = []\n\n for tracklet_idx in range(num_tracklets):\n data = meta_data[tracklet_idx, ...]\n start_index, end_index, pid, camid = data\n if pid == -1:\n continue # junk images are just ignored\n assert 1 <= camid <= 6\n if relabel:\n pid = pid2label[pid]\n camid -= 1 # index starts from 0\n img_names = names[start_index - 1:end_index]\n\n # make sure image names correspond to the same person\n pnames = [img_name[:4] for img_name in img_names]\n assert len(\n set(pnames)\n ) == 1, 'Error: a single tracklet contains different person images'\n\n # make sure all images are captured under the same camera\n camnames = [img_name[5] for img_name in img_names]\n assert len(\n set(camnames)\n ) == 1, 'Error: images are captured under different cameras!'\n\n # append image names with directory information\n img_paths = [\n osp.join(self.dataset_dir, home_dir, img_name[:4], img_name)\n for img_name in img_names\n ]\n if len(img_paths) >= min_seq_len:\n img_paths = tuple(img_paths)\n tracklets.append((img_paths, pid, camid))\n\n return tracklets\n\n def combine_all(self):\n warnings.warn(\n 'Some query IDs do not appear in gallery. Therefore, combineall '\n 'does not make any difference to Mars'\n )\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/datasets/video/prid2011.py", "content": "from __future__ import division, print_function, absolute_import\nimport glob\nimport os.path as osp\n\nfrom torchreid.utils import read_json\n\nfrom ..dataset import VideoDataset\n\n\nclass PRID2011(VideoDataset):\n \"\"\"PRID2011.\n\n Reference:\n Hirzer et al. Person Re-Identification by Descriptive and\n Discriminative Classification. SCIA 2011.\n\n URL: ``_\n \n Dataset statistics:\n - identities: 200.\n - tracklets: 400.\n - cameras: 2.\n \"\"\"\n dataset_dir = 'prid2011'\n dataset_url = None\n\n def __init__(self, root='', split_id=0, **kwargs):\n self.root = osp.abspath(osp.expanduser(root))\n self.dataset_dir = osp.join(self.root, self.dataset_dir)\n self.download_dataset(self.dataset_dir, self.dataset_url)\n\n self.split_path = osp.join(self.dataset_dir, 'splits_prid2011.json')\n self.cam_a_dir = osp.join(\n self.dataset_dir, 'prid_2011', 'multi_shot', 'cam_a'\n )\n self.cam_b_dir = osp.join(\n self.dataset_dir, 'prid_2011', 'multi_shot', 'cam_b'\n )\n\n required_files = [self.dataset_dir, self.cam_a_dir, self.cam_b_dir]\n self.check_before_run(required_files)\n\n splits = read_json(self.split_path)\n if split_id >= len(splits):\n raise ValueError(\n 'split_id exceeds range, received {}, but expected between 0 and {}'\n .format(split_id,\n len(splits) - 1)\n )\n split = splits[split_id]\n train_dirs, test_dirs = split['train'], split['test']\n\n train = self.process_dir(train_dirs, cam1=True, cam2=True)\n query = self.process_dir(test_dirs, cam1=True, cam2=False)\n gallery = self.process_dir(test_dirs, cam1=False, cam2=True)\n\n super(PRID2011, self).__init__(train, query, gallery, **kwargs)\n\n def process_dir(self, dirnames, cam1=True, cam2=True):\n tracklets = []\n dirname2pid = {dirname: i for i, dirname in enumerate(dirnames)}\n\n for dirname in dirnames:\n if cam1:\n person_dir = osp.join(self.cam_a_dir, dirname)\n img_names = glob.glob(osp.join(person_dir, '*.png'))\n assert len(img_names) > 0\n img_names = tuple(img_names)\n pid = dirname2pid[dirname]\n tracklets.append((img_names, pid, 0))\n\n if cam2:\n person_dir = osp.join(self.cam_b_dir, dirname)\n img_names = glob.glob(osp.join(person_dir, '*.png'))\n assert len(img_names) > 0\n img_names = tuple(img_names)\n pid = dirname2pid[dirname]\n tracklets.append((img_names, pid, 1))\n\n return tracklets\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/sampler.py", "content": "from __future__ import division, absolute_import\nimport copy\nimport numpy as np\nimport random\nfrom collections import defaultdict\nfrom torch.utils.data.sampler import Sampler, RandomSampler, SequentialSampler\n\nAVAI_SAMPLERS = [\n 'RandomIdentitySampler', 'SequentialSampler', 'RandomSampler',\n 'RandomDomainSampler', 'RandomDatasetSampler'\n]\n\n\nclass RandomIdentitySampler(Sampler):\n \"\"\"Randomly samples N identities each with K instances.\n\n Args:\n data_source (list): contains tuples of (img_path(s), pid, camid, dsetid).\n batch_size (int): batch size.\n num_instances (int): number of instances per identity in a batch.\n \"\"\"\n\n def __init__(self, data_source, batch_size, num_instances):\n if batch_size < num_instances:\n raise ValueError(\n 'batch_size={} must be no less '\n 'than num_instances={}'.format(batch_size, num_instances)\n )\n\n self.data_source = data_source\n self.batch_size = batch_size\n self.num_instances = num_instances\n self.num_pids_per_batch = self.batch_size // self.num_instances\n self.index_dic = defaultdict(list)\n for index, items in enumerate(data_source):\n pid = items[1]\n self.index_dic[pid].append(index)\n self.pids = list(self.index_dic.keys())\n assert len(self.pids) >= self.num_pids_per_batch\n\n # estimate number of examples in an epoch\n # TODO: improve precision\n self.length = 0\n for pid in self.pids:\n idxs = self.index_dic[pid]\n num = len(idxs)\n if num < self.num_instances:\n num = self.num_instances\n self.length += num - num % self.num_instances\n\n def __iter__(self):\n batch_idxs_dict = defaultdict(list)\n\n for pid in self.pids:\n idxs = copy.deepcopy(self.index_dic[pid])\n if len(idxs) < self.num_instances:\n idxs = np.random.choice(\n idxs, size=self.num_instances, replace=True\n )\n random.shuffle(idxs)\n batch_idxs = []\n for idx in idxs:\n batch_idxs.append(idx)\n if len(batch_idxs) == self.num_instances:\n batch_idxs_dict[pid].append(batch_idxs)\n batch_idxs = []\n\n avai_pids = copy.deepcopy(self.pids)\n final_idxs = []\n\n while len(avai_pids) >= self.num_pids_per_batch:\n selected_pids = random.sample(avai_pids, self.num_pids_per_batch)\n for pid in selected_pids:\n batch_idxs = batch_idxs_dict[pid].pop(0)\n final_idxs.extend(batch_idxs)\n if len(batch_idxs_dict[pid]) == 0:\n avai_pids.remove(pid)\n\n return iter(final_idxs)\n\n def __len__(self):\n return self.length\n\n\nclass RandomDomainSampler(Sampler):\n \"\"\"Random domain sampler.\n\n We consider each camera as a visual domain.\n\n How does the sampling work:\n 1. Randomly sample N cameras (based on the \"camid\" label).\n 2. From each camera, randomly sample K images.\n\n Args:\n data_source (list): contains tuples of (img_path(s), pid, camid, dsetid).\n batch_size (int): batch size.\n n_domain (int): number of cameras to sample in a batch.\n \"\"\"\n\n def __init__(self, data_source, batch_size, n_domain):\n self.data_source = data_source\n\n # Keep track of image indices for each domain\n self.domain_dict = defaultdict(list)\n for i, items in enumerate(data_source):\n camid = items[2]\n self.domain_dict[camid].append(i)\n self.domains = list(self.domain_dict.keys())\n\n # Make sure each domain can be assigned an equal number of images\n if n_domain is None or n_domain <= 0:\n n_domain = len(self.domains)\n assert batch_size % n_domain == 0\n self.n_img_per_domain = batch_size // n_domain\n\n self.batch_size = batch_size\n self.n_domain = n_domain\n self.length = len(list(self.__iter__()))\n\n def __iter__(self):\n domain_dict = copy.deepcopy(self.domain_dict)\n final_idxs = []\n stop_sampling = False\n\n while not stop_sampling:\n selected_domains = random.sample(self.domains, self.n_domain)\n\n for domain in selected_domains:\n idxs = domain_dict[domain]\n selected_idxs = random.sample(idxs, self.n_img_per_domain)\n final_idxs.extend(selected_idxs)\n\n for idx in selected_idxs:\n domain_dict[domain].remove(idx)\n\n remaining = len(domain_dict[domain])\n if remaining < self.n_img_per_domain:\n stop_sampling = True\n\n return iter(final_idxs)\n\n def __len__(self):\n return self.length\n\n\nclass RandomDatasetSampler(Sampler):\n \"\"\"Random dataset sampler.\n\n How does the sampling work:\n 1. Randomly sample N datasets (based on the \"dsetid\" label).\n 2. From each dataset, randomly sample K images.\n\n Args:\n data_source (list): contains tuples of (img_path(s), pid, camid, dsetid).\n batch_size (int): batch size.\n n_dataset (int): number of datasets to sample in a batch.\n \"\"\"\n\n def __init__(self, data_source, batch_size, n_dataset):\n self.data_source = data_source\n\n # Keep track of image indices for each dataset\n self.dataset_dict = defaultdict(list)\n for i, items in enumerate(data_source):\n dsetid = items[3]\n self.dataset_dict[dsetid].append(i)\n self.datasets = list(self.dataset_dict.keys())\n\n # Make sure each dataset can be assigned an equal number of images\n if n_dataset is None or n_dataset <= 0:\n n_dataset = len(self.datasets)\n assert batch_size % n_dataset == 0\n self.n_img_per_dset = batch_size // n_dataset\n\n self.batch_size = batch_size\n self.n_dataset = n_dataset\n self.length = len(list(self.__iter__()))\n\n def __iter__(self):\n dataset_dict = copy.deepcopy(self.dataset_dict)\n final_idxs = []\n stop_sampling = False\n\n while not stop_sampling:\n selected_datasets = random.sample(self.datasets, self.n_dataset)\n\n for dset in selected_datasets:\n idxs = dataset_dict[dset]\n selected_idxs = random.sample(idxs, self.n_img_per_dset)\n final_idxs.extend(selected_idxs)\n\n for idx in selected_idxs:\n dataset_dict[dset].remove(idx)\n\n remaining = len(dataset_dict[dset])\n if remaining < self.n_img_per_dset:\n stop_sampling = True\n\n return iter(final_idxs)\n\n def __len__(self):\n return self.length\n\n\ndef build_train_sampler(\n data_source,\n train_sampler,\n batch_size=32,\n num_instances=4,\n num_cams=1,\n num_datasets=1,\n **kwargs\n):\n \"\"\"Builds a training sampler.\n\n Args:\n data_source (list): contains tuples of (img_path(s), pid, camid).\n train_sampler (str): sampler name (default: ``RandomSampler``).\n batch_size (int, optional): batch size. Default is 32.\n num_instances (int, optional): number of instances per identity in a\n batch (when using ``RandomIdentitySampler``). Default is 4.\n num_cams (int, optional): number of cameras to sample in a batch (when using\n ``RandomDomainSampler``). Default is 1.\n num_datasets (int, optional): number of datasets to sample in a batch (when\n using ``RandomDatasetSampler``). Default is 1.\n \"\"\"\n assert train_sampler in AVAI_SAMPLERS, \\\n 'train_sampler must be one of {}, but got {}'.format(AVAI_SAMPLERS, train_sampler)\n\n if train_sampler == 'RandomIdentitySampler':\n sampler = RandomIdentitySampler(data_source, batch_size, num_instances)\n\n elif train_sampler == 'RandomDomainSampler':\n sampler = RandomDomainSampler(data_source, batch_size, num_cams)\n\n elif train_sampler == 'RandomDatasetSampler':\n sampler = RandomDatasetSampler(data_source, batch_size, num_datasets)\n\n elif train_sampler == 'SequentialSampler':\n sampler = SequentialSampler(data_source)\n\n elif train_sampler == 'RandomSampler':\n sampler = RandomSampler(data_source)\n\n return sampler\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/data/transforms.py", "content": "from __future__ import division, print_function, absolute_import\nimport math\nimport random\nfrom collections import deque\nimport torch\nfrom PIL import Image\nfrom torchvision.transforms import (\n Resize, Compose, ToTensor, Normalize, ColorJitter, RandomHorizontalFlip\n)\n\n\nclass Random2DTranslation(object):\n \"\"\"Randomly translates the input image with a probability.\n\n Specifically, given a predefined shape (height, width), the input is first\n resized with a factor of 1.125, leading to (height*1.125, width*1.125), then\n a random crop is performed. Such operation is done with a probability.\n\n Args:\n height (int): target image height.\n width (int): target image width.\n p (float, optional): probability that this operation takes place.\n Default is 0.5.\n interpolation (int, optional): desired interpolation. Default is\n ``PIL.Image.BILINEAR``\n \"\"\"\n\n def __init__(self, height, width, p=0.5, interpolation=Image.BILINEAR):\n self.height = height\n self.width = width\n self.p = p\n self.interpolation = interpolation\n\n def __call__(self, img):\n if random.uniform(0, 1) > self.p:\n return img.resize((self.width, self.height), self.interpolation)\n\n new_width, new_height = int(round(self.width * 1.125)\n ), int(round(self.height * 1.125))\n resized_img = img.resize((new_width, new_height), self.interpolation)\n x_maxrange = new_width - self.width\n y_maxrange = new_height - self.height\n x1 = int(round(random.uniform(0, x_maxrange)))\n y1 = int(round(random.uniform(0, y_maxrange)))\n croped_img = resized_img.crop(\n (x1, y1, x1 + self.width, y1 + self.height)\n )\n return croped_img\n\n\nclass RandomErasing(object):\n \"\"\"Randomly erases an image patch.\n\n Origin: ``_\n\n Reference:\n Zhong et al. Random Erasing Data Augmentation.\n\n Args:\n probability (float, optional): probability that this operation takes place.\n Default is 0.5.\n sl (float, optional): min erasing area.\n sh (float, optional): max erasing area.\n r1 (float, optional): min aspect ratio.\n mean (list, optional): erasing value.\n \"\"\"\n\n def __init__(\n self,\n probability=0.5,\n sl=0.02,\n sh=0.4,\n r1=0.3,\n mean=[0.4914, 0.4822, 0.4465]\n ):\n self.probability = probability\n self.mean = mean\n self.sl = sl\n self.sh = sh\n self.r1 = r1\n\n def __call__(self, img):\n if random.uniform(0, 1) > self.probability:\n return img\n\n for attempt in range(100):\n area = img.size()[1] * img.size()[2]\n\n target_area = random.uniform(self.sl, self.sh) * area\n aspect_ratio = random.uniform(self.r1, 1 / self.r1)\n\n h = int(round(math.sqrt(target_area * aspect_ratio)))\n w = int(round(math.sqrt(target_area / aspect_ratio)))\n\n if w < img.size()[2] and h < img.size()[1]:\n x1 = random.randint(0, img.size()[1] - h)\n y1 = random.randint(0, img.size()[2] - w)\n if img.size()[0] == 3:\n img[0, x1:x1 + h, y1:y1 + w] = self.mean[0]\n img[1, x1:x1 + h, y1:y1 + w] = self.mean[1]\n img[2, x1:x1 + h, y1:y1 + w] = self.mean[2]\n else:\n img[0, x1:x1 + h, y1:y1 + w] = self.mean[0]\n return img\n\n return img\n\n\nclass ColorAugmentation(object):\n \"\"\"Randomly alters the intensities of RGB channels.\n\n Reference:\n Krizhevsky et al. ImageNet Classification with Deep ConvolutionalNeural\n Networks. NIPS 2012.\n\n Args:\n p (float, optional): probability that this operation takes place.\n Default is 0.5.\n \"\"\"\n\n def __init__(self, p=0.5):\n self.p = p\n self.eig_vec = torch.Tensor(\n [\n [0.4009, 0.7192, -0.5675],\n [-0.8140, -0.0045, -0.5808],\n [0.4203, -0.6948, -0.5836],\n ]\n )\n self.eig_val = torch.Tensor([[0.2175, 0.0188, 0.0045]])\n\n def _check_input(self, tensor):\n assert tensor.dim() == 3 and tensor.size(0) == 3\n\n def __call__(self, tensor):\n if random.uniform(0, 1) > self.p:\n return tensor\n alpha = torch.normal(mean=torch.zeros_like(self.eig_val)) * 0.1\n quatity = torch.mm(self.eig_val * alpha, self.eig_vec)\n tensor = tensor + quatity.view(3, 1, 1)\n return tensor\n\n\nclass RandomPatch(object):\n \"\"\"Random patch data augmentation.\n\n There is a patch pool that stores randomly extracted pathces from person images.\n \n For each input image, RandomPatch\n 1) extracts a random patch and stores the patch in the patch pool;\n 2) randomly selects a patch from the patch pool and pastes it on the\n input (at random position) to simulate occlusion.\n\n Reference:\n - Zhou et al. Omni-Scale Feature Learning for Person Re-Identification. ICCV, 2019.\n - Zhou et al. Learning Generalisable Omni-Scale Representations\n for Person Re-Identification. TPAMI, 2021.\n \"\"\"\n\n def __init__(\n self,\n prob_happen=0.5,\n pool_capacity=50000,\n min_sample_size=100,\n patch_min_area=0.01,\n patch_max_area=0.5,\n patch_min_ratio=0.1,\n prob_rotate=0.5,\n prob_flip_leftright=0.5,\n ):\n self.prob_happen = prob_happen\n\n self.patch_min_area = patch_min_area\n self.patch_max_area = patch_max_area\n self.patch_min_ratio = patch_min_ratio\n\n self.prob_rotate = prob_rotate\n self.prob_flip_leftright = prob_flip_leftright\n\n self.patchpool = deque(maxlen=pool_capacity)\n self.min_sample_size = min_sample_size\n\n def generate_wh(self, W, H):\n area = W * H\n for attempt in range(100):\n target_area = random.uniform(\n self.patch_min_area, self.patch_max_area\n ) * area\n aspect_ratio = random.uniform(\n self.patch_min_ratio, 1. / self.patch_min_ratio\n )\n h = int(round(math.sqrt(target_area * aspect_ratio)))\n w = int(round(math.sqrt(target_area / aspect_ratio)))\n if w < W and h < H:\n return w, h\n return None, None\n\n def transform_patch(self, patch):\n if random.uniform(0, 1) > self.prob_flip_leftright:\n patch = patch.transpose(Image.FLIP_LEFT_RIGHT)\n if random.uniform(0, 1) > self.prob_rotate:\n patch = patch.rotate(random.randint(-10, 10))\n return patch\n\n def __call__(self, img):\n W, H = img.size # original image size\n\n # collect new patch\n w, h = self.generate_wh(W, H)\n if w is not None and h is not None:\n x1 = random.randint(0, W - w)\n y1 = random.randint(0, H - h)\n new_patch = img.crop((x1, y1, x1 + w, y1 + h))\n self.patchpool.append(new_patch)\n\n if len(self.patchpool) < self.min_sample_size:\n return img\n\n if random.uniform(0, 1) > self.prob_happen:\n return img\n\n # paste a randomly selected patch on a random position\n patch = random.sample(self.patchpool, 1)[0]\n patchW, patchH = patch.size\n x1 = random.randint(0, W - patchW)\n y1 = random.randint(0, H - patchH)\n patch = self.transform_patch(patch)\n img.paste(patch, (x1, y1))\n\n return img\n\n\ndef build_transforms(\n height,\n width,\n transforms='random_flip',\n norm_mean=[0.485, 0.456, 0.406],\n norm_std=[0.229, 0.224, 0.225],\n **kwargs\n):\n \"\"\"Builds train and test transform functions.\n\n Args:\n height (int): target image height.\n width (int): target image width.\n transforms (str or list of str, optional): transformations applied to model training.\n Default is 'random_flip'.\n norm_mean (list or None, optional): normalization mean values. Default is ImageNet means.\n norm_std (list or None, optional): normalization standard deviation values. Default is\n ImageNet standard deviation values.\n \"\"\"\n if transforms is None:\n transforms = []\n\n if isinstance(transforms, str):\n transforms = [transforms]\n\n if not isinstance(transforms, list):\n raise ValueError(\n 'transforms must be a list of strings, but found to be {}'.format(\n type(transforms)\n )\n )\n\n if len(transforms) > 0:\n transforms = [t.lower() for t in transforms]\n\n if norm_mean is None or norm_std is None:\n norm_mean = [0.485, 0.456, 0.406] # imagenet mean\n norm_std = [0.229, 0.224, 0.225] # imagenet std\n normalize = Normalize(mean=norm_mean, std=norm_std)\n\n print('Building train transforms ...')\n transform_tr = []\n\n print('+ resize to {}x{}'.format(height, width))\n transform_tr += [Resize((height, width))]\n\n if 'random_flip' in transforms:\n print('+ random flip')\n transform_tr += [RandomHorizontalFlip()]\n\n if 'random_crop' in transforms:\n print(\n '+ random crop (enlarge to {}x{} and '\n 'crop {}x{})'.format(\n int(round(height * 1.125)), int(round(width * 1.125)), height,\n width\n )\n )\n transform_tr += [Random2DTranslation(height, width)]\n\n if 'random_patch' in transforms:\n print('+ random patch')\n transform_tr += [RandomPatch()]\n\n if 'color_jitter' in transforms:\n print('+ color jitter')\n transform_tr += [\n ColorJitter(brightness=0.2, contrast=0.15, saturation=0, hue=0)\n ]\n\n print('+ to torch tensor of range [0, 1]')\n transform_tr += [ToTensor()]\n\n print('+ normalization (mean={}, std={})'.format(norm_mean, norm_std))\n transform_tr += [normalize]\n\n if 'random_erase' in transforms:\n print('+ random erase')\n transform_tr += [RandomErasing(mean=norm_mean)]\n\n transform_tr = Compose(transform_tr)\n\n print('Building test transforms ...')\n print('+ resize to {}x{}'.format(height, width))\n print('+ to torch tensor of range [0, 1]')\n print('+ normalization (mean={}, std={})'.format(norm_mean, norm_std))\n\n transform_te = Compose([\n Resize((height, width)),\n ToTensor(),\n normalize,\n ])\n\n return transform_tr, transform_te\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/engine/__init__.py", "content": "from __future__ import print_function, absolute_import\n\nfrom .image import ImageSoftmaxEngine, ImageTripletEngine\nfrom .video import VideoSoftmaxEngine, VideoTripletEngine\nfrom .engine import Engine\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/engine/engine.py", "content": "from __future__ import division, print_function, absolute_import\nimport time\nimport numpy as np\nimport os.path as osp\nimport datetime\nfrom collections import OrderedDict\nimport torch\nfrom torch.nn import functional as F\nfrom torch.utils.tensorboard import SummaryWriter\n\nfrom torchreid import metrics\nfrom torchreid.utils import (\n MetricMeter, AverageMeter, re_ranking, open_all_layers, save_checkpoint,\n open_specified_layers, visualize_ranked_results\n)\nfrom torchreid.losses import DeepSupervision\n\n\nclass Engine(object):\n r\"\"\"A generic base Engine class for both image- and video-reid.\n\n Args:\n datamanager (DataManager): an instance of ``torchreid.data.ImageDataManager``\n or ``torchreid.data.VideoDataManager``.\n use_gpu (bool, optional): use gpu. Default is True.\n \"\"\"\n\n def __init__(self, datamanager, use_gpu=True):\n self.datamanager = datamanager\n self.train_loader = self.datamanager.train_loader\n self.test_loader = self.datamanager.test_loader\n self.use_gpu = (torch.cuda.is_available() and use_gpu)\n self.writer = None\n self.epoch = 0\n\n self.model = None\n self.optimizer = None\n self.scheduler = None\n\n self._models = OrderedDict()\n self._optims = OrderedDict()\n self._scheds = OrderedDict()\n\n def register_model(self, name='model', model=None, optim=None, sched=None):\n if self.__dict__.get('_models') is None:\n raise AttributeError(\n 'Cannot assign model before super().__init__() call'\n )\n\n if self.__dict__.get('_optims') is None:\n raise AttributeError(\n 'Cannot assign optim before super().__init__() call'\n )\n\n if self.__dict__.get('_scheds') is None:\n raise AttributeError(\n 'Cannot assign sched before super().__init__() call'\n )\n\n self._models[name] = model\n self._optims[name] = optim\n self._scheds[name] = sched\n\n def get_model_names(self, names=None):\n names_real = list(self._models.keys())\n if names is not None:\n if not isinstance(names, list):\n names = [names]\n for name in names:\n assert name in names_real\n return names\n else:\n return names_real\n\n def save_model(self, epoch, rank1, save_dir, is_best=False):\n names = self.get_model_names()\n\n for name in names:\n save_checkpoint(\n {\n 'state_dict': self._models[name].state_dict(),\n 'epoch': epoch + 1,\n 'rank1': rank1,\n 'optimizer': self._optims[name].state_dict(),\n 'scheduler': self._scheds[name].state_dict()\n },\n osp.join(save_dir, name),\n is_best=is_best\n )\n\n def set_model_mode(self, mode='train', names=None):\n assert mode in ['train', 'eval', 'test']\n names = self.get_model_names(names)\n\n for name in names:\n if mode == 'train':\n self._models[name].train()\n else:\n self._models[name].eval()\n\n def get_current_lr(self, names=None):\n names = self.get_model_names(names)\n name = names[0]\n return self._optims[name].param_groups[-1]['lr']\n\n def update_lr(self, names=None):\n names = self.get_model_names(names)\n\n for name in names:\n if self._scheds[name] is not None:\n self._scheds[name].step()\n\n def run(\n self,\n save_dir='log',\n max_epoch=0,\n start_epoch=0,\n print_freq=10,\n fixbase_epoch=0,\n open_layers=None,\n start_eval=0,\n eval_freq=-1,\n test_only=False,\n dist_metric='euclidean',\n normalize_feature=False,\n visrank=False,\n visrank_topk=10,\n use_metric_cuhk03=False,\n ranks=[1, 5, 10, 20],\n rerank=False\n ):\n r\"\"\"A unified pipeline for training and evaluating a model.\n\n Args:\n save_dir (str): directory to save model.\n max_epoch (int): maximum epoch.\n start_epoch (int, optional): starting epoch. Default is 0.\n print_freq (int, optional): print_frequency. Default is 10.\n fixbase_epoch (int, optional): number of epochs to train ``open_layers`` (new layers)\n while keeping base layers frozen. Default is 0. ``fixbase_epoch`` is counted\n in ``max_epoch``.\n open_layers (str or list, optional): layers (attribute names) open for training.\n start_eval (int, optional): from which epoch to start evaluation. Default is 0.\n eval_freq (int, optional): evaluation frequency. Default is -1 (meaning evaluation\n is only performed at the end of training).\n test_only (bool, optional): if True, only runs evaluation on test datasets.\n Default is False.\n dist_metric (str, optional): distance metric used to compute distance matrix\n between query and gallery. Default is \"euclidean\".\n normalize_feature (bool, optional): performs L2 normalization on feature vectors before\n computing feature distance. Default is False.\n visrank (bool, optional): visualizes ranked results. Default is False. It is recommended to\n enable ``visrank`` when ``test_only`` is True. The ranked images will be saved to\n \"save_dir/visrank_dataset\", e.g. \"save_dir/visrank_market1501\".\n visrank_topk (int, optional): top-k ranked images to be visualized. Default is 10.\n use_metric_cuhk03 (bool, optional): use single-gallery-shot setting for cuhk03.\n Default is False. This should be enabled when using cuhk03 classic split.\n ranks (list, optional): cmc ranks to be computed. Default is [1, 5, 10, 20].\n rerank (bool, optional): uses person re-ranking (by Zhong et al. CVPR'17).\n Default is False. This is only enabled when test_only=True.\n \"\"\"\n\n if visrank and not test_only:\n raise ValueError(\n 'visrank can be set to True only if test_only=True'\n )\n\n if test_only:\n self.test(\n dist_metric=dist_metric,\n normalize_feature=normalize_feature,\n visrank=visrank,\n visrank_topk=visrank_topk,\n save_dir=save_dir,\n use_metric_cuhk03=use_metric_cuhk03,\n ranks=ranks,\n rerank=rerank\n )\n return\n\n if self.writer is None:\n self.writer = SummaryWriter(log_dir=save_dir)\n\n time_start = time.time()\n self.start_epoch = start_epoch\n self.max_epoch = max_epoch\n print('=> Start training')\n\n for self.epoch in range(self.start_epoch, self.max_epoch):\n self.train(\n print_freq=print_freq,\n fixbase_epoch=fixbase_epoch,\n open_layers=open_layers\n )\n\n if (self.epoch + 1) >= start_eval \\\n and eval_freq > 0 \\\n and (self.epoch+1) % eval_freq == 0 \\\n and (self.epoch + 1) != self.max_epoch:\n rank1 = self.test(\n dist_metric=dist_metric,\n normalize_feature=normalize_feature,\n visrank=visrank,\n visrank_topk=visrank_topk,\n save_dir=save_dir,\n use_metric_cuhk03=use_metric_cuhk03,\n ranks=ranks\n )\n self.save_model(self.epoch, rank1, save_dir)\n\n if self.max_epoch > 0:\n print('=> Final test')\n rank1 = self.test(\n dist_metric=dist_metric,\n normalize_feature=normalize_feature,\n visrank=visrank,\n visrank_topk=visrank_topk,\n save_dir=save_dir,\n use_metric_cuhk03=use_metric_cuhk03,\n ranks=ranks\n )\n self.save_model(self.epoch, rank1, save_dir)\n\n elapsed = round(time.time() - time_start)\n elapsed = str(datetime.timedelta(seconds=elapsed))\n print('Elapsed {}'.format(elapsed))\n if self.writer is not None:\n self.writer.close()\n\n def train(self, print_freq=10, fixbase_epoch=0, open_layers=None):\n losses = MetricMeter()\n batch_time = AverageMeter()\n data_time = AverageMeter()\n\n self.set_model_mode('train')\n\n self.two_stepped_transfer_learning(\n self.epoch, fixbase_epoch, open_layers\n )\n\n self.num_batches = len(self.train_loader)\n end = time.time()\n for self.batch_idx, data in enumerate(self.train_loader):\n data_time.update(time.time() - end)\n loss_summary = self.forward_backward(data)\n batch_time.update(time.time() - end)\n losses.update(loss_summary)\n\n if (self.batch_idx + 1) % print_freq == 0:\n nb_this_epoch = self.num_batches - (self.batch_idx + 1)\n nb_future_epochs = (\n self.max_epoch - (self.epoch + 1)\n ) * self.num_batches\n eta_seconds = batch_time.avg * (nb_this_epoch+nb_future_epochs)\n eta_str = str(datetime.timedelta(seconds=int(eta_seconds)))\n print(\n 'epoch: [{0}/{1}][{2}/{3}]\\t'\n 'time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n 'data {data_time.val:.3f} ({data_time.avg:.3f})\\t'\n 'eta {eta}\\t'\n '{losses}\\t'\n 'lr {lr:.6f}'.format(\n self.epoch + 1,\n self.max_epoch,\n self.batch_idx + 1,\n self.num_batches,\n batch_time=batch_time,\n data_time=data_time,\n eta=eta_str,\n losses=losses,\n lr=self.get_current_lr()\n )\n )\n\n if self.writer is not None:\n n_iter = self.epoch * self.num_batches + self.batch_idx\n self.writer.add_scalar('Train/time', batch_time.avg, n_iter)\n self.writer.add_scalar('Train/data', data_time.avg, n_iter)\n for name, meter in losses.meters.items():\n self.writer.add_scalar('Train/' + name, meter.avg, n_iter)\n self.writer.add_scalar(\n 'Train/lr', self.get_current_lr(), n_iter\n )\n\n end = time.time()\n\n self.update_lr()\n\n def forward_backward(self, data):\n raise NotImplementedError\n\n def test(\n self,\n dist_metric='euclidean',\n normalize_feature=False,\n visrank=False,\n visrank_topk=10,\n save_dir='',\n use_metric_cuhk03=False,\n ranks=[1, 5, 10, 20],\n rerank=False\n ):\n r\"\"\"Tests model on target datasets.\n\n .. note::\n\n This function has been called in ``run()``.\n\n .. note::\n\n The test pipeline implemented in this function suits both image- and\n video-reid. In general, a subclass of Engine only needs to re-implement\n ``extract_features()`` and ``parse_data_for_eval()`` (most of the time),\n but not a must. Please refer to the source code for more details.\n \"\"\"\n self.set_model_mode('eval')\n targets = list(self.test_loader.keys())\n\n for name in targets:\n domain = 'source' if name in self.datamanager.sources else 'target'\n print('##### Evaluating {} ({}) #####'.format(name, domain))\n query_loader = self.test_loader[name]['query']\n gallery_loader = self.test_loader[name]['gallery']\n rank1, mAP = self._evaluate(\n dataset_name=name,\n query_loader=query_loader,\n gallery_loader=gallery_loader,\n dist_metric=dist_metric,\n normalize_feature=normalize_feature,\n visrank=visrank,\n visrank_topk=visrank_topk,\n save_dir=save_dir,\n use_metric_cuhk03=use_metric_cuhk03,\n ranks=ranks,\n rerank=rerank\n )\n\n if self.writer is not None:\n self.writer.add_scalar(f'Test/{name}/rank1', rank1, self.epoch)\n self.writer.add_scalar(f'Test/{name}/mAP', mAP, self.epoch)\n\n return rank1\n\n @torch.no_grad()\n def _evaluate(\n self,\n dataset_name='',\n query_loader=None,\n gallery_loader=None,\n dist_metric='euclidean',\n normalize_feature=False,\n visrank=False,\n visrank_topk=10,\n save_dir='',\n use_metric_cuhk03=False,\n ranks=[1, 5, 10, 20],\n rerank=False\n ):\n batch_time = AverageMeter()\n\n def _feature_extraction(data_loader):\n f_, pids_, camids_ = [], [], []\n for batch_idx, data in enumerate(data_loader):\n imgs, pids, camids = self.parse_data_for_eval(data)\n if self.use_gpu:\n imgs = imgs.cuda()\n end = time.time()\n features = self.extract_features(imgs)\n batch_time.update(time.time() - end)\n features = features.cpu()\n f_.append(features)\n pids_.extend(pids.tolist())\n camids_.extend(camids.tolist())\n f_ = torch.cat(f_, 0)\n pids_ = np.asarray(pids_)\n camids_ = np.asarray(camids_)\n return f_, pids_, camids_\n\n print('Extracting features from query set ...')\n qf, q_pids, q_camids = _feature_extraction(query_loader)\n print('Done, obtained {}-by-{} matrix'.format(qf.size(0), qf.size(1)))\n\n print('Extracting features from gallery set ...')\n gf, g_pids, g_camids = _feature_extraction(gallery_loader)\n print('Done, obtained {}-by-{} matrix'.format(gf.size(0), gf.size(1)))\n\n print('Speed: {:.4f} sec/batch'.format(batch_time.avg))\n\n if normalize_feature:\n print('Normalzing features with L2 norm ...')\n qf = F.normalize(qf, p=2, dim=1)\n gf = F.normalize(gf, p=2, dim=1)\n\n print(\n 'Computing distance matrix with metric={} ...'.format(dist_metric)\n )\n distmat = metrics.compute_distance_matrix(qf, gf, dist_metric)\n distmat = distmat.numpy()\n\n if rerank:\n print('Applying person re-ranking ...')\n distmat_qq = metrics.compute_distance_matrix(qf, qf, dist_metric)\n distmat_gg = metrics.compute_distance_matrix(gf, gf, dist_metric)\n distmat = re_ranking(distmat, distmat_qq, distmat_gg)\n\n print('Computing CMC and mAP ...')\n cmc, mAP = metrics.evaluate_rank(\n distmat,\n q_pids,\n g_pids,\n q_camids,\n g_camids,\n use_metric_cuhk03=use_metric_cuhk03\n )\n\n print('** Results **')\n print('mAP: {:.1%}'.format(mAP))\n print('CMC curve')\n for r in ranks:\n print('Rank-{:<3}: {:.1%}'.format(r, cmc[r - 1]))\n\n if visrank:\n visualize_ranked_results(\n distmat,\n self.datamanager.fetch_test_loaders(dataset_name),\n self.datamanager.data_type,\n width=self.datamanager.width,\n height=self.datamanager.height,\n save_dir=osp.join(save_dir, 'visrank_' + dataset_name),\n topk=visrank_topk\n )\n\n return cmc[0], mAP\n\n def compute_loss(self, criterion, outputs, targets):\n if isinstance(outputs, (tuple, list)):\n loss = DeepSupervision(criterion, outputs, targets)\n else:\n loss = criterion(outputs, targets)\n return loss\n\n def extract_features(self, input):\n return self.model(input)\n\n def parse_data_for_train(self, data):\n imgs = data['img']\n pids = data['pid']\n return imgs, pids\n\n def parse_data_for_eval(self, data):\n imgs = data['img']\n pids = data['pid']\n camids = data['camid']\n return imgs, pids, camids\n\n def two_stepped_transfer_learning(\n self, epoch, fixbase_epoch, open_layers, model=None\n ):\n \"\"\"Two-stepped transfer learning.\n\n The idea is to freeze base layers for a certain number of epochs\n and then open all layers for training.\n\n Reference: https://arxiv.org/abs/1611.05244\n \"\"\"\n model = self.model if model is None else model\n if model is None:\n return\n\n if (epoch + 1) <= fixbase_epoch and open_layers is not None:\n print(\n '* Only train {} (epoch: {}/{})'.format(\n open_layers, epoch + 1, fixbase_epoch\n )\n )\n open_specified_layers(model, open_layers)\n else:\n open_all_layers(model)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/engine/image/__init__.py", "content": "from __future__ import absolute_import\n\nfrom .softmax import ImageSoftmaxEngine\nfrom .triplet import ImageTripletEngine\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/engine/image/softmax.py", "content": "from __future__ import division, print_function, absolute_import\n\nfrom torchreid import metrics\nfrom torchreid.losses import CrossEntropyLoss\n\nfrom ..engine import Engine\n\n\nclass ImageSoftmaxEngine(Engine):\n r\"\"\"Softmax-loss engine for image-reid.\n\n Args:\n datamanager (DataManager): an instance of ``torchreid.data.ImageDataManager``\n or ``torchreid.data.VideoDataManager``.\n model (nn.Module): model instance.\n optimizer (Optimizer): an Optimizer.\n scheduler (LRScheduler, optional): if None, no learning rate decay will be performed.\n use_gpu (bool, optional): use gpu. Default is True.\n label_smooth (bool, optional): use label smoothing regularizer. Default is True.\n\n Examples::\n \n import torchreid\n datamanager = torchreid.data.ImageDataManager(\n root='path/to/reid-data',\n sources='market1501',\n height=256,\n width=128,\n combineall=False,\n batch_size=32\n )\n model = torchreid.models.build_model(\n name='resnet50',\n num_classes=datamanager.num_train_pids,\n loss='softmax'\n )\n model = model.cuda()\n optimizer = torchreid.optim.build_optimizer(\n model, optim='adam', lr=0.0003\n )\n scheduler = torchreid.optim.build_lr_scheduler(\n optimizer,\n lr_scheduler='single_step',\n stepsize=20\n )\n engine = torchreid.engine.ImageSoftmaxEngine(\n datamanager, model, optimizer, scheduler=scheduler\n )\n engine.run(\n max_epoch=60,\n save_dir='log/resnet50-softmax-market1501',\n print_freq=10\n )\n \"\"\"\n\n def __init__(\n self,\n datamanager,\n model,\n optimizer,\n scheduler=None,\n use_gpu=True,\n label_smooth=True\n ):\n super(ImageSoftmaxEngine, self).__init__(datamanager, use_gpu)\n\n self.model = model\n self.optimizer = optimizer\n self.scheduler = scheduler\n self.register_model('model', model, optimizer, scheduler)\n\n self.criterion = CrossEntropyLoss(\n num_classes=self.datamanager.num_train_pids,\n use_gpu=self.use_gpu,\n label_smooth=label_smooth\n )\n\n def forward_backward(self, data):\n imgs, pids = self.parse_data_for_train(data)\n\n if self.use_gpu:\n imgs = imgs.cuda()\n pids = pids.cuda()\n\n outputs = self.model(imgs)\n loss = self.compute_loss(self.criterion, outputs, pids)\n\n self.optimizer.zero_grad()\n loss.backward()\n self.optimizer.step()\n\n loss_summary = {\n 'loss': loss.item(),\n 'acc': metrics.accuracy(outputs, pids)[0].item()\n }\n\n return loss_summary\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/engine/image/triplet.py", "content": "from __future__ import division, print_function, absolute_import\n\nfrom torchreid import metrics\nfrom torchreid.losses import TripletLoss, CrossEntropyLoss\n\nfrom ..engine import Engine\n\n\nclass ImageTripletEngine(Engine):\n r\"\"\"Triplet-loss engine for image-reid.\n\n Args:\n datamanager (DataManager): an instance of ``torchreid.data.ImageDataManager``\n or ``torchreid.data.VideoDataManager``.\n model (nn.Module): model instance.\n optimizer (Optimizer): an Optimizer.\n margin (float, optional): margin for triplet loss. Default is 0.3.\n weight_t (float, optional): weight for triplet loss. Default is 1.\n weight_x (float, optional): weight for softmax loss. Default is 1.\n scheduler (LRScheduler, optional): if None, no learning rate decay will be performed.\n use_gpu (bool, optional): use gpu. Default is True.\n label_smooth (bool, optional): use label smoothing regularizer. Default is True.\n\n Examples::\n \n import torchreid\n datamanager = torchreid.data.ImageDataManager(\n root='path/to/reid-data',\n sources='market1501',\n height=256,\n width=128,\n combineall=False,\n batch_size=32,\n num_instances=4,\n train_sampler='RandomIdentitySampler' # this is important\n )\n model = torchreid.models.build_model(\n name='resnet50',\n num_classes=datamanager.num_train_pids,\n loss='triplet'\n )\n model = model.cuda()\n optimizer = torchreid.optim.build_optimizer(\n model, optim='adam', lr=0.0003\n )\n scheduler = torchreid.optim.build_lr_scheduler(\n optimizer,\n lr_scheduler='single_step',\n stepsize=20\n )\n engine = torchreid.engine.ImageTripletEngine(\n datamanager, model, optimizer, margin=0.3,\n weight_t=0.7, weight_x=1, scheduler=scheduler\n )\n engine.run(\n max_epoch=60,\n save_dir='log/resnet50-triplet-market1501',\n print_freq=10\n )\n \"\"\"\n\n def __init__(\n self,\n datamanager,\n model,\n optimizer,\n margin=0.3,\n weight_t=1,\n weight_x=1,\n scheduler=None,\n use_gpu=True,\n label_smooth=True\n ):\n super(ImageTripletEngine, self).__init__(datamanager, use_gpu)\n\n self.model = model\n self.optimizer = optimizer\n self.scheduler = scheduler\n self.register_model('model', model, optimizer, scheduler)\n\n assert weight_t >= 0 and weight_x >= 0\n assert weight_t + weight_x > 0\n self.weight_t = weight_t\n self.weight_x = weight_x\n\n self.criterion_t = TripletLoss(margin=margin)\n self.criterion_x = CrossEntropyLoss(\n num_classes=self.datamanager.num_train_pids,\n use_gpu=self.use_gpu,\n label_smooth=label_smooth\n )\n\n def forward_backward(self, data):\n imgs, pids = self.parse_data_for_train(data)\n\n if self.use_gpu:\n imgs = imgs.cuda()\n pids = pids.cuda()\n\n outputs, features = self.model(imgs)\n\n loss = 0\n loss_summary = {}\n\n if self.weight_t > 0:\n loss_t = self.compute_loss(self.criterion_t, features, pids)\n loss += self.weight_t * loss_t\n loss_summary['loss_t'] = loss_t.item()\n\n if self.weight_x > 0:\n loss_x = self.compute_loss(self.criterion_x, outputs, pids)\n loss += self.weight_x * loss_x\n loss_summary['loss_x'] = loss_x.item()\n loss_summary['acc'] = metrics.accuracy(outputs, pids)[0].item()\n\n assert loss_summary\n\n self.optimizer.zero_grad()\n loss.backward()\n self.optimizer.step()\n\n return loss_summary\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/engine/video/__init__.py", "content": "from __future__ import absolute_import\n\nfrom .softmax import VideoSoftmaxEngine\nfrom .triplet import VideoTripletEngine\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/engine/video/softmax.py", "content": "from __future__ import division, print_function, absolute_import\nimport torch\n\nfrom torchreid.engine.image import ImageSoftmaxEngine\n\n\nclass VideoSoftmaxEngine(ImageSoftmaxEngine):\n \"\"\"Softmax-loss engine for video-reid.\n\n Args:\n datamanager (DataManager): an instance of ``torchreid.data.ImageDataManager``\n or ``torchreid.data.VideoDataManager``.\n model (nn.Module): model instance.\n optimizer (Optimizer): an Optimizer.\n scheduler (LRScheduler, optional): if None, no learning rate decay will be performed.\n use_gpu (bool, optional): use gpu. Default is True.\n label_smooth (bool, optional): use label smoothing regularizer. Default is True.\n pooling_method (str, optional): how to pool features for a tracklet.\n Default is \"avg\" (average). Choices are [\"avg\", \"max\"].\n\n Examples::\n \n import torch\n import torchreid\n # Each batch contains batch_size*seq_len images\n datamanager = torchreid.data.VideoDataManager(\n root='path/to/reid-data',\n sources='mars',\n height=256,\n width=128,\n combineall=False,\n batch_size=8, # number of tracklets\n seq_len=15 # number of images in each tracklet\n )\n model = torchreid.models.build_model(\n name='resnet50',\n num_classes=datamanager.num_train_pids,\n loss='softmax'\n )\n model = model.cuda()\n optimizer = torchreid.optim.build_optimizer(\n model, optim='adam', lr=0.0003\n )\n scheduler = torchreid.optim.build_lr_scheduler(\n optimizer,\n lr_scheduler='single_step',\n stepsize=20\n )\n engine = torchreid.engine.VideoSoftmaxEngine(\n datamanager, model, optimizer, scheduler=scheduler,\n pooling_method='avg'\n )\n engine.run(\n max_epoch=60,\n save_dir='log/resnet50-softmax-mars',\n print_freq=10\n )\n \"\"\"\n\n def __init__(\n self,\n datamanager,\n model,\n optimizer,\n scheduler=None,\n use_gpu=True,\n label_smooth=True,\n pooling_method='avg'\n ):\n super(VideoSoftmaxEngine, self).__init__(\n datamanager,\n model,\n optimizer,\n scheduler=scheduler,\n use_gpu=use_gpu,\n label_smooth=label_smooth\n )\n self.pooling_method = pooling_method\n\n def parse_data_for_train(self, data):\n imgs = data['img']\n pids = data['pid']\n if imgs.dim() == 5:\n # b: batch size\n # s: sqeuence length\n # c: channel depth\n # h: height\n # w: width\n b, s, c, h, w = imgs.size()\n imgs = imgs.view(b * s, c, h, w)\n pids = pids.view(b, 1).expand(b, s)\n pids = pids.contiguous().view(b * s)\n return imgs, pids\n\n def extract_features(self, input):\n # b: batch size\n # s: sqeuence length\n # c: channel depth\n # h: height\n # w: width\n b, s, c, h, w = input.size()\n input = input.view(b * s, c, h, w)\n features = self.model(input)\n features = features.view(b, s, -1)\n if self.pooling_method == 'avg':\n features = torch.mean(features, 1)\n else:\n features = torch.max(features, 1)[0]\n return features\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/engine/video/triplet.py", "content": "from __future__ import division, print_function, absolute_import\nimport torch\n\nfrom torchreid.engine.image import ImageTripletEngine\n\n\nclass VideoTripletEngine(ImageTripletEngine):\n \"\"\"Triplet-loss engine for video-reid.\n\n Args:\n datamanager (DataManager): an instance of ``torchreid.data.ImageDataManager``\n or ``torchreid.data.VideoDataManager``.\n model (nn.Module): model instance.\n optimizer (Optimizer): an Optimizer.\n margin (float, optional): margin for triplet loss. Default is 0.3.\n weight_t (float, optional): weight for triplet loss. Default is 1.\n weight_x (float, optional): weight for softmax loss. Default is 1.\n scheduler (LRScheduler, optional): if None, no learning rate decay will be performed.\n use_gpu (bool, optional): use gpu. Default is True.\n label_smooth (bool, optional): use label smoothing regularizer. Default is True.\n pooling_method (str, optional): how to pool features for a tracklet.\n Default is \"avg\" (average). Choices are [\"avg\", \"max\"].\n\n Examples::\n\n import torch\n import torchreid\n # Each batch contains batch_size*seq_len images\n # Each identity is sampled with num_instances tracklets\n datamanager = torchreid.data.VideoDataManager(\n root='path/to/reid-data',\n sources='mars',\n height=256,\n width=128,\n combineall=False,\n num_instances=4,\n train_sampler='RandomIdentitySampler'\n batch_size=8, # number of tracklets\n seq_len=15 # number of images in each tracklet\n )\n model = torchreid.models.build_model(\n name='resnet50',\n num_classes=datamanager.num_train_pids,\n loss='triplet'\n )\n model = model.cuda()\n optimizer = torchreid.optim.build_optimizer(\n model, optim='adam', lr=0.0003\n )\n scheduler = torchreid.optim.build_lr_scheduler(\n optimizer,\n lr_scheduler='single_step',\n stepsize=20\n )\n engine = torchreid.engine.VideoTripletEngine(\n datamanager, model, optimizer, margin=0.3,\n weight_t=0.7, weight_x=1, scheduler=scheduler,\n pooling_method='avg'\n )\n engine.run(\n max_epoch=60,\n save_dir='log/resnet50-triplet-mars',\n print_freq=10\n )\n \"\"\"\n\n def __init__(\n self,\n datamanager,\n model,\n optimizer,\n margin=0.3,\n weight_t=1,\n weight_x=1,\n scheduler=None,\n use_gpu=True,\n label_smooth=True,\n pooling_method='avg'\n ):\n super(VideoTripletEngine, self).__init__(\n datamanager,\n model,\n optimizer,\n margin=margin,\n weight_t=weight_t,\n weight_x=weight_x,\n scheduler=scheduler,\n use_gpu=use_gpu,\n label_smooth=label_smooth\n )\n self.pooling_method = pooling_method\n\n def parse_data_for_train(self, data):\n imgs = data['img']\n pids = data['pid']\n if imgs.dim() == 5:\n # b: batch size\n # s: sqeuence length\n # c: channel depth\n # h: height\n # w: width\n b, s, c, h, w = imgs.size()\n imgs = imgs.view(b * s, c, h, w)\n pids = pids.view(b, 1).expand(b, s)\n pids = pids.contiguous().view(b * s)\n return imgs, pids\n\n def extract_features(self, input):\n # b: batch size\n # s: sqeuence length\n # c: channel depth\n # h: height\n # w: width\n b, s, c, h, w = input.size()\n input = input.view(b * s, c, h, w)\n features = self.model(input)\n features = features.view(b, s, -1)\n if self.pooling_method == 'avg':\n features = torch.mean(features, 1)\n else:\n features = torch.max(features, 1)[0]\n return features\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/losses/__init__.py", "content": "from __future__ import division, print_function, absolute_import\n\nfrom .cross_entropy_loss import CrossEntropyLoss\nfrom .hard_mine_triplet_loss import TripletLoss\n\n\ndef DeepSupervision(criterion, xs, y):\n \"\"\"DeepSupervision\n\n Applies criterion to each element in a list.\n\n Args:\n criterion: loss function\n xs: tuple of inputs\n y: ground truth\n \"\"\"\n loss = 0.\n for x in xs:\n loss += criterion(x, y)\n loss /= len(xs)\n return loss\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/losses/cross_entropy_loss.py", "content": "from __future__ import division, absolute_import\nimport torch\nimport torch.nn as nn\n\n\nclass CrossEntropyLoss(nn.Module):\n r\"\"\"Cross entropy loss with label smoothing regularizer.\n \n Reference:\n Szegedy et al. Rethinking the Inception Architecture for Computer Vision. CVPR 2016.\n\n With label smoothing, the label :math:`y` for a class is computed by\n \n .. math::\n \\begin{equation}\n (1 - \\eps) \\times y + \\frac{\\eps}{K},\n \\end{equation}\n\n where :math:`K` denotes the number of classes and :math:`\\eps` is a weight. When\n :math:`\\eps = 0`, the loss function reduces to the normal cross entropy.\n \n Args:\n num_classes (int): number of classes.\n eps (float, optional): weight. Default is 0.1.\n use_gpu (bool, optional): whether to use gpu devices. Default is True.\n label_smooth (bool, optional): whether to apply label smoothing. Default is True.\n \"\"\"\n\n def __init__(self, num_classes, eps=0.1, use_gpu=True, label_smooth=True):\n super(CrossEntropyLoss, self).__init__()\n self.num_classes = num_classes\n self.eps = eps if label_smooth else 0\n self.use_gpu = use_gpu\n self.logsoftmax = nn.LogSoftmax(dim=1)\n\n def forward(self, inputs, targets):\n \"\"\"\n Args:\n inputs (torch.Tensor): prediction matrix (before softmax) with\n shape (batch_size, num_classes).\n targets (torch.LongTensor): ground truth labels with shape (batch_size).\n Each position contains the label index.\n \"\"\"\n log_probs = self.logsoftmax(inputs)\n zeros = torch.zeros(log_probs.size())\n targets = zeros.scatter_(1, targets.unsqueeze(1).data.cpu(), 1)\n if self.use_gpu:\n targets = targets.cuda()\n targets = (1 - self.eps) * targets + self.eps / self.num_classes\n return (-targets * log_probs).mean(0).sum()\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/losses/hard_mine_triplet_loss.py", "content": "from __future__ import division, absolute_import\nimport torch\nimport torch.nn as nn\n\n\nclass TripletLoss(nn.Module):\n \"\"\"Triplet loss with hard positive/negative mining.\n \n Reference:\n Hermans et al. In Defense of the Triplet Loss for Person Re-Identification. arXiv:1703.07737.\n \n Imported from ``_.\n \n Args:\n margin (float, optional): margin for triplet. Default is 0.3.\n \"\"\"\n\n def __init__(self, margin=0.3):\n super(TripletLoss, self).__init__()\n self.margin = margin\n self.ranking_loss = nn.MarginRankingLoss(margin=margin)\n\n def forward(self, inputs, targets):\n \"\"\"\n Args:\n inputs (torch.Tensor): feature matrix with shape (batch_size, feat_dim).\n targets (torch.LongTensor): ground truth labels with shape (num_classes).\n \"\"\"\n n = inputs.size(0)\n\n # Compute pairwise distance, replace by the official when merged\n dist = torch.pow(inputs, 2).sum(dim=1, keepdim=True).expand(n, n)\n dist = dist + dist.t()\n dist.addmm_(inputs, inputs.t(), beta=1, alpha=-2)\n dist = dist.clamp(min=1e-12).sqrt() # for numerical stability\n\n # For each anchor, find the hardest positive and negative\n mask = targets.expand(n, n).eq(targets.expand(n, n).t())\n dist_ap, dist_an = [], []\n for i in range(n):\n dist_ap.append(dist[i][mask[i]].max().unsqueeze(0))\n dist_an.append(dist[i][mask[i] == 0].min().unsqueeze(0))\n dist_ap = torch.cat(dist_ap)\n dist_an = torch.cat(dist_an)\n\n # Compute ranking hinge loss\n y = torch.ones_like(dist_an)\n return self.ranking_loss(dist_an, dist_ap, y)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/metrics/__init__.py", "content": "from __future__ import absolute_import\n\nfrom .rank import evaluate_rank\nfrom .accuracy import accuracy\nfrom .distance import compute_distance_matrix\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/metrics/accuracy.py", "content": "from __future__ import division, print_function, absolute_import\n\n\ndef accuracy(output, target, topk=(1, )):\n \"\"\"Computes the accuracy over the k top predictions for\n the specified values of k.\n\n Args:\n output (torch.Tensor): prediction matrix with shape (batch_size, num_classes).\n target (torch.LongTensor): ground truth labels with shape (batch_size).\n topk (tuple, optional): accuracy at top-k will be computed. For example,\n topk=(1, 5) means accuracy at top-1 and top-5 will be computed.\n\n Returns:\n list: accuracy at top-k.\n\n Examples::\n >>> from torchreid import metrics\n >>> metrics.accuracy(output, target)\n \"\"\"\n maxk = max(topk)\n batch_size = target.size(0)\n\n if isinstance(output, (tuple, list)):\n output = output[0]\n\n _, pred = output.topk(maxk, 1, True, True)\n pred = pred.t()\n correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n res = []\n for k in topk:\n correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n acc = correct_k.mul_(100.0 / batch_size)\n res.append(acc)\n\n return res\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/metrics/distance.py", "content": "from __future__ import division, print_function, absolute_import\nimport torch\nfrom torch.nn import functional as F\n\n\ndef compute_distance_matrix(input1, input2, metric='euclidean'):\n \"\"\"A wrapper function for computing distance matrix.\n\n Args:\n input1 (torch.Tensor): 2-D feature matrix.\n input2 (torch.Tensor): 2-D feature matrix.\n metric (str, optional): \"euclidean\" or \"cosine\".\n Default is \"euclidean\".\n\n Returns:\n torch.Tensor: distance matrix.\n\n Examples::\n >>> from torchreid import metrics\n >>> input1 = torch.rand(10, 2048)\n >>> input2 = torch.rand(100, 2048)\n >>> distmat = metrics.compute_distance_matrix(input1, input2)\n >>> distmat.size() # (10, 100)\n \"\"\"\n # check input\n assert isinstance(input1, torch.Tensor)\n assert isinstance(input2, torch.Tensor)\n assert input1.dim() == 2, 'Expected 2-D tensor, but got {}-D'.format(\n input1.dim()\n )\n assert input2.dim() == 2, 'Expected 2-D tensor, but got {}-D'.format(\n input2.dim()\n )\n assert input1.size(1) == input2.size(1)\n\n if metric == 'euclidean':\n distmat = euclidean_squared_distance(input1, input2)\n elif metric == 'cosine':\n distmat = cosine_distance(input1, input2)\n else:\n raise ValueError(\n 'Unknown distance metric: {}. '\n 'Please choose either \"euclidean\" or \"cosine\"'.format(metric)\n )\n\n return distmat\n\n\ndef euclidean_squared_distance(input1, input2):\n \"\"\"Computes euclidean squared distance.\n\n Args:\n input1 (torch.Tensor): 2-D feature matrix.\n input2 (torch.Tensor): 2-D feature matrix.\n\n Returns:\n torch.Tensor: distance matrix.\n \"\"\"\n m, n = input1.size(0), input2.size(0)\n mat1 = torch.pow(input1, 2).sum(dim=1, keepdim=True).expand(m, n)\n mat2 = torch.pow(input2, 2).sum(dim=1, keepdim=True).expand(n, m).t()\n distmat = mat1 + mat2\n distmat.addmm_(input1, input2.t(), beta=1, alpha=-2)\n return distmat\n\n\ndef cosine_distance(input1, input2):\n \"\"\"Computes cosine distance.\n\n Args:\n input1 (torch.Tensor): 2-D feature matrix.\n input2 (torch.Tensor): 2-D feature matrix.\n\n Returns:\n torch.Tensor: distance matrix.\n \"\"\"\n input1_normed = F.normalize(input1, p=2, dim=1)\n input2_normed = F.normalize(input2, p=2, dim=1)\n distmat = 1 - torch.mm(input1_normed, input2_normed.t())\n return distmat\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/metrics/rank.py", "content": "from __future__ import division, print_function, absolute_import\nimport numpy as np\nimport warnings\nfrom collections import defaultdict\n\ntry:\n from torchreid.metrics.rank_cylib.rank_cy import evaluate_cy\n IS_CYTHON_AVAI = True\nexcept ImportError:\n IS_CYTHON_AVAI = False\n warnings.warn(\n 'Cython evaluation (very fast so highly recommended) is '\n 'unavailable, now use python evaluation.'\n )\n\n\ndef eval_cuhk03(distmat, q_pids, g_pids, q_camids, g_camids, max_rank):\n \"\"\"Evaluation with cuhk03 metric\n Key: one image for each gallery identity is randomly sampled for each query identity.\n Random sampling is performed num_repeats times.\n \"\"\"\n num_repeats = 10\n num_q, num_g = distmat.shape\n\n if num_g < max_rank:\n max_rank = num_g\n print(\n 'Note: number of gallery samples is quite small, got {}'.\n format(num_g)\n )\n\n indices = np.argsort(distmat, axis=1)\n matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32)\n\n # compute cmc curve for each query\n all_cmc = []\n all_AP = []\n num_valid_q = 0. # number of valid query\n\n for q_idx in range(num_q):\n # get query pid and camid\n q_pid = q_pids[q_idx]\n q_camid = q_camids[q_idx]\n\n # remove gallery samples that have the same pid and camid with query\n order = indices[q_idx]\n remove = (g_pids[order] == q_pid) & (g_camids[order] == q_camid)\n keep = np.invert(remove)\n\n # compute cmc curve\n raw_cmc = matches[q_idx][\n keep] # binary vector, positions with value 1 are correct matches\n if not np.any(raw_cmc):\n # this condition is true when query identity does not appear in gallery\n continue\n\n kept_g_pids = g_pids[order][keep]\n g_pids_dict = defaultdict(list)\n for idx, pid in enumerate(kept_g_pids):\n g_pids_dict[pid].append(idx)\n\n cmc = 0.\n for repeat_idx in range(num_repeats):\n mask = np.zeros(len(raw_cmc), dtype=np.bool)\n for _, idxs in g_pids_dict.items():\n # randomly sample one image for each gallery person\n rnd_idx = np.random.choice(idxs)\n mask[rnd_idx] = True\n masked_raw_cmc = raw_cmc[mask]\n _cmc = masked_raw_cmc.cumsum()\n _cmc[_cmc > 1] = 1\n cmc += _cmc[:max_rank].astype(np.float32)\n\n cmc /= num_repeats\n all_cmc.append(cmc)\n # compute AP\n num_rel = raw_cmc.sum()\n tmp_cmc = raw_cmc.cumsum()\n tmp_cmc = [x / (i+1.) for i, x in enumerate(tmp_cmc)]\n tmp_cmc = np.asarray(tmp_cmc) * raw_cmc\n AP = tmp_cmc.sum() / num_rel\n all_AP.append(AP)\n num_valid_q += 1.\n\n assert num_valid_q > 0, 'Error: all query identities do not appear in gallery'\n\n all_cmc = np.asarray(all_cmc).astype(np.float32)\n all_cmc = all_cmc.sum(0) / num_valid_q\n mAP = np.mean(all_AP)\n\n return all_cmc, mAP\n\n\ndef eval_market1501(distmat, q_pids, g_pids, q_camids, g_camids, max_rank):\n \"\"\"Evaluation with market1501 metric\n Key: for each query identity, its gallery images from the same camera view are discarded.\n \"\"\"\n num_q, num_g = distmat.shape\n\n if num_g < max_rank:\n max_rank = num_g\n print(\n 'Note: number of gallery samples is quite small, got {}'.\n format(num_g)\n )\n\n indices = np.argsort(distmat, axis=1)\n matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32)\n\n # compute cmc curve for each query\n all_cmc = []\n all_AP = []\n num_valid_q = 0. # number of valid query\n\n for q_idx in range(num_q):\n # get query pid and camid\n q_pid = q_pids[q_idx]\n q_camid = q_camids[q_idx]\n\n # remove gallery samples that have the same pid and camid with query\n order = indices[q_idx]\n remove = (g_pids[order] == q_pid) & (g_camids[order] == q_camid)\n keep = np.invert(remove)\n\n # compute cmc curve\n raw_cmc = matches[q_idx][\n keep] # binary vector, positions with value 1 are correct matches\n if not np.any(raw_cmc):\n # this condition is true when query identity does not appear in gallery\n continue\n\n cmc = raw_cmc.cumsum()\n cmc[cmc > 1] = 1\n\n all_cmc.append(cmc[:max_rank])\n num_valid_q += 1.\n\n # compute average precision\n # reference: https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision\n num_rel = raw_cmc.sum()\n tmp_cmc = raw_cmc.cumsum()\n tmp_cmc = [x / (i+1.) for i, x in enumerate(tmp_cmc)]\n tmp_cmc = np.asarray(tmp_cmc) * raw_cmc\n AP = tmp_cmc.sum() / num_rel\n all_AP.append(AP)\n\n assert num_valid_q > 0, 'Error: all query identities do not appear in gallery'\n\n all_cmc = np.asarray(all_cmc).astype(np.float32)\n all_cmc = all_cmc.sum(0) / num_valid_q\n mAP = np.mean(all_AP)\n\n return all_cmc, mAP\n\n\ndef evaluate_py(\n distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_metric_cuhk03\n):\n if use_metric_cuhk03:\n return eval_cuhk03(\n distmat, q_pids, g_pids, q_camids, g_camids, max_rank\n )\n else:\n return eval_market1501(\n distmat, q_pids, g_pids, q_camids, g_camids, max_rank\n )\n\n\ndef evaluate_rank(\n distmat,\n q_pids,\n g_pids,\n q_camids,\n g_camids,\n max_rank=50,\n use_metric_cuhk03=False,\n use_cython=True\n):\n \"\"\"Evaluates CMC rank.\n\n Args:\n distmat (numpy.ndarray): distance matrix of shape (num_query, num_gallery).\n q_pids (numpy.ndarray): 1-D array containing person identities\n of each query instance.\n g_pids (numpy.ndarray): 1-D array containing person identities\n of each gallery instance.\n q_camids (numpy.ndarray): 1-D array containing camera views under\n which each query instance is captured.\n g_camids (numpy.ndarray): 1-D array containing camera views under\n which each gallery instance is captured.\n max_rank (int, optional): maximum CMC rank to be computed. Default is 50.\n use_metric_cuhk03 (bool, optional): use single-gallery-shot setting for cuhk03.\n Default is False. This should be enabled when using cuhk03 classic split.\n use_cython (bool, optional): use cython code for evaluation. Default is True.\n This is highly recommended as the cython code can speed up the cmc computation\n by more than 10x. This requires Cython to be installed.\n \"\"\"\n if use_cython and IS_CYTHON_AVAI:\n return evaluate_cy(\n distmat, q_pids, g_pids, q_camids, g_camids, max_rank,\n use_metric_cuhk03\n )\n else:\n return evaluate_py(\n distmat, q_pids, g_pids, q_camids, g_camids, max_rank,\n use_metric_cuhk03\n )\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/metrics/rank_cylib/Makefile", "content": "all:\n\t$(PYTHON) setup.py build_ext --inplace\n\trm -rf build\nclean:\n\trm -rf build\n\trm -f rank_cy.c *.so" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/metrics/rank_cylib/__init__.py", "content": "" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/metrics/rank_cylib/rank_cy.pyx", "content": "# cython: boundscheck=False, wraparound=False, nonecheck=False, cdivision=True\n\nfrom __future__ import print_function\nimport numpy as np\nfrom libc.stdint cimport int64_t, uint64_t\n\nimport cython\n\ncimport numpy as np\n\nimport random\nfrom collections import defaultdict\n\n\"\"\"\nCompiler directives:\nhttps://github.com/cython/cython/wiki/enhancements-compilerdirectives\n\nCython tutorial:\nhttps://cython.readthedocs.io/en/latest/src/userguide/numpy_tutorial.html\n\nCredit to https://github.com/luzai\n\"\"\"\n\n\n# Main interface\ncpdef evaluate_cy(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_metric_cuhk03=False):\n distmat = np.asarray(distmat, dtype=np.float32)\n q_pids = np.asarray(q_pids, dtype=np.int64)\n g_pids = np.asarray(g_pids, dtype=np.int64)\n q_camids = np.asarray(q_camids, dtype=np.int64)\n g_camids = np.asarray(g_camids, dtype=np.int64)\n if use_metric_cuhk03:\n return eval_cuhk03_cy(distmat, q_pids, g_pids, q_camids, g_camids, max_rank)\n return eval_market1501_cy(distmat, q_pids, g_pids, q_camids, g_camids, max_rank)\n\n\ncpdef eval_cuhk03_cy(float[:,:] distmat, int64_t[:] q_pids, int64_t[:]g_pids,\n int64_t[:]q_camids, int64_t[:]g_camids, int64_t max_rank):\n\n cdef int64_t num_q = distmat.shape[0]\n cdef int64_t num_g = distmat.shape[1]\n\n if num_g < max_rank:\n max_rank = num_g\n print('Note: number of gallery samples is quite small, got {}'.format(num_g))\n\n cdef:\n int64_t num_repeats = 10\n int64_t[:,:] indices = np.argsort(distmat, axis=1)\n int64_t[:,:] matches = (np.asarray(g_pids)[np.asarray(indices)] == np.asarray(q_pids)[:, np.newaxis]).astype(np.int64)\n\n float[:,:] all_cmc = np.zeros((num_q, max_rank), dtype=np.float32)\n float[:] all_AP = np.zeros(num_q, dtype=np.float32)\n float num_valid_q = 0. # number of valid query\n\n int64_t q_idx, q_pid, q_camid, g_idx\n int64_t[:] order = np.zeros(num_g, dtype=np.int64)\n int64_t keep\n\n float[:] raw_cmc = np.zeros(num_g, dtype=np.float32) # binary vector, positions with value 1 are correct matches\n float[:] masked_raw_cmc = np.zeros(num_g, dtype=np.float32)\n float[:] cmc, masked_cmc\n int64_t num_g_real, num_g_real_masked, rank_idx, rnd_idx\n uint64_t meet_condition\n float AP\n int64_t[:] kept_g_pids, mask\n\n float num_rel\n float[:] tmp_cmc = np.zeros(num_g, dtype=np.float32)\n float tmp_cmc_sum\n\n for q_idx in range(num_q):\n # get query pid and camid\n q_pid = q_pids[q_idx]\n q_camid = q_camids[q_idx]\n\n # remove gallery samples that have the same pid and camid with query\n for g_idx in range(num_g):\n order[g_idx] = indices[q_idx, g_idx]\n num_g_real = 0\n meet_condition = 0\n kept_g_pids = np.zeros(num_g, dtype=np.int64)\n\n for g_idx in range(num_g):\n if (g_pids[order[g_idx]] != q_pid) or (g_camids[order[g_idx]] != q_camid):\n raw_cmc[num_g_real] = matches[q_idx][g_idx]\n kept_g_pids[num_g_real] = g_pids[order[g_idx]]\n num_g_real += 1\n if matches[q_idx][g_idx] > 1e-31:\n meet_condition = 1\n\n if not meet_condition:\n # this condition is true when query identity does not appear in gallery\n continue\n\n # cuhk03-specific setting\n g_pids_dict = defaultdict(list) # overhead!\n for g_idx in range(num_g_real):\n g_pids_dict[kept_g_pids[g_idx]].append(g_idx)\n\n cmc = np.zeros(max_rank, dtype=np.float32)\n for _ in range(num_repeats):\n mask = np.zeros(num_g_real, dtype=np.int64)\n\n for _, idxs in g_pids_dict.items():\n # randomly sample one image for each gallery person\n rnd_idx = np.random.choice(idxs)\n #rnd_idx = idxs[0] # use deterministic for debugging\n mask[rnd_idx] = 1\n\n num_g_real_masked = 0\n for g_idx in range(num_g_real):\n if mask[g_idx] == 1:\n masked_raw_cmc[num_g_real_masked] = raw_cmc[g_idx]\n num_g_real_masked += 1\n\n masked_cmc = np.zeros(num_g, dtype=np.float32)\n function_cumsum(masked_raw_cmc, masked_cmc, num_g_real_masked)\n for g_idx in range(num_g_real_masked):\n if masked_cmc[g_idx] > 1:\n masked_cmc[g_idx] = 1\n\n for rank_idx in range(max_rank):\n cmc[rank_idx] += masked_cmc[rank_idx] / num_repeats\n\n for rank_idx in range(max_rank):\n all_cmc[q_idx, rank_idx] = cmc[rank_idx]\n # compute average precision\n # reference: https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision\n function_cumsum(raw_cmc, tmp_cmc, num_g_real)\n num_rel = 0\n tmp_cmc_sum = 0\n for g_idx in range(num_g_real):\n tmp_cmc_sum += (tmp_cmc[g_idx] / (g_idx + 1.)) * raw_cmc[g_idx]\n num_rel += raw_cmc[g_idx]\n all_AP[q_idx] = tmp_cmc_sum / num_rel\n num_valid_q += 1.\n\n assert num_valid_q > 0, 'Error: all query identities do not appear in gallery'\n\n # compute averaged cmc\n cdef float[:] avg_cmc = np.zeros(max_rank, dtype=np.float32)\n for rank_idx in range(max_rank):\n for q_idx in range(num_q):\n avg_cmc[rank_idx] += all_cmc[q_idx, rank_idx]\n avg_cmc[rank_idx] /= num_valid_q\n\n cdef float mAP = 0\n for q_idx in range(num_q):\n mAP += all_AP[q_idx]\n mAP /= num_valid_q\n\n return np.asarray(avg_cmc).astype(np.float32), mAP\n\n\ncpdef eval_market1501_cy(float[:,:] distmat, int64_t[:] q_pids, int64_t[:]g_pids,\n int64_t[:]q_camids, int64_t[:]g_camids, int64_t max_rank):\n\n cdef int64_t num_q = distmat.shape[0]\n cdef int64_t num_g = distmat.shape[1]\n\n if num_g < max_rank:\n max_rank = num_g\n print('Note: number of gallery samples is quite small, got {}'.format(num_g))\n\n cdef:\n int64_t[:,:] indices = np.argsort(distmat, axis=1)\n int64_t[:,:] matches = (np.asarray(g_pids)[np.asarray(indices)] == np.asarray(q_pids)[:, np.newaxis]).astype(np.int64)\n\n float[:,:] all_cmc = np.zeros((num_q, max_rank), dtype=np.float32)\n float[:] all_AP = np.zeros(num_q, dtype=np.float32)\n float num_valid_q = 0. # number of valid query\n\n int64_t q_idx, q_pid, q_camid, g_idx\n int64_t[:] order = np.zeros(num_g, dtype=np.int64)\n int64_t keep\n\n float[:] raw_cmc = np.zeros(num_g, dtype=np.float32) # binary vector, positions with value 1 are correct matches\n float[:] cmc = np.zeros(num_g, dtype=np.float32)\n int64_t num_g_real, rank_idx\n uint64_t meet_condition\n\n float num_rel\n float[:] tmp_cmc = np.zeros(num_g, dtype=np.float32)\n float tmp_cmc_sum\n\n for q_idx in range(num_q):\n # get query pid and camid\n q_pid = q_pids[q_idx]\n q_camid = q_camids[q_idx]\n\n # remove gallery samples that have the same pid and camid with query\n for g_idx in range(num_g):\n order[g_idx] = indices[q_idx, g_idx]\n num_g_real = 0\n meet_condition = 0\n\n for g_idx in range(num_g):\n if (g_pids[order[g_idx]] != q_pid) or (g_camids[order[g_idx]] != q_camid):\n raw_cmc[num_g_real] = matches[q_idx][g_idx]\n num_g_real += 1\n if matches[q_idx][g_idx] > 1e-31:\n meet_condition = 1\n\n if not meet_condition:\n # this condition is true when query identity does not appear in gallery\n continue\n\n # compute cmc\n function_cumsum(raw_cmc, cmc, num_g_real)\n for g_idx in range(num_g_real):\n if cmc[g_idx] > 1:\n cmc[g_idx] = 1\n\n for rank_idx in range(max_rank):\n all_cmc[q_idx, rank_idx] = cmc[rank_idx]\n num_valid_q += 1.\n\n # compute average precision\n # reference: https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision\n function_cumsum(raw_cmc, tmp_cmc, num_g_real)\n num_rel = 0\n tmp_cmc_sum = 0\n for g_idx in range(num_g_real):\n tmp_cmc_sum += (tmp_cmc[g_idx] / (g_idx + 1.)) * raw_cmc[g_idx]\n num_rel += raw_cmc[g_idx]\n all_AP[q_idx] = tmp_cmc_sum / num_rel\n\n assert num_valid_q > 0, 'Error: all query identities do not appear in gallery'\n\n # compute averaged cmc\n cdef float[:] avg_cmc = np.zeros(max_rank, dtype=np.float32)\n for rank_idx in range(max_rank):\n for q_idx in range(num_q):\n avg_cmc[rank_idx] += all_cmc[q_idx, rank_idx]\n avg_cmc[rank_idx] /= num_valid_q\n\n cdef float mAP = 0\n for q_idx in range(num_q):\n mAP += all_AP[q_idx]\n mAP /= num_valid_q\n\n return np.asarray(avg_cmc).astype(np.float32), mAP\n\n\n# Compute the cumulative sum\ncdef void function_cumsum(cython.numeric[:] src, cython.numeric[:] dst, int64_t n):\n cdef int64_t i\n dst[0] = src[0]\n for i in range(1, n):\n dst[i] = src[i] + dst[i - 1]" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/metrics/rank_cylib/setup.py", "content": "import numpy as np\nfrom distutils.core import setup\nfrom distutils.extension import Extension\nfrom Cython.Build import cythonize\n\n\ndef numpy_include():\n try:\n numpy_include = np.get_include()\n except AttributeError:\n numpy_include = np.get_numpy_include()\n return numpy_include\n\n\next_modules = [\n Extension(\n 'rank_cy',\n ['rank_cy.pyx'],\n include_dirs=[numpy_include()],\n )\n]\n\nsetup(\n name='Cython-based reid evaluation code',\n ext_modules=cythonize(ext_modules)\n)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/metrics/rank_cylib/test_cython.py", "content": "from __future__ import print_function\nimport sys\nimport numpy as np\nimport timeit\nimport os.path as osp\n\nfrom torchreid import metrics\n\nsys.path.insert(0, osp.dirname(osp.abspath(__file__)) + '/../../..')\n\"\"\"\nTest the speed of cython-based evaluation code. The speed improvements\ncan be much bigger when using the real reid data, which contains a larger\namount of query and gallery images.\n\nNote: you might encounter the following error:\n 'AssertionError: Error: all query identities do not appear in gallery'.\nThis is normal because the inputs are random numbers. Just try again.\n\"\"\"\n\nprint('*** Compare running time ***')\n\nsetup = '''\nimport sys\nimport os.path as osp\nimport numpy as np\nsys.path.insert(0, osp.dirname(osp.abspath(__file__)) + '/../../..')\nfrom torchreid import metrics\nnum_q = 30\nnum_g = 300\nmax_rank = 5\ndistmat = np.random.rand(num_q, num_g) * 20\nq_pids = np.random.randint(0, num_q, size=num_q)\ng_pids = np.random.randint(0, num_g, size=num_g)\nq_camids = np.random.randint(0, 5, size=num_q)\ng_camids = np.random.randint(0, 5, size=num_g)\n'''\n\nprint('=> Using market1501\\'s metric')\npytime = timeit.timeit(\n 'metrics.evaluate_rank(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_cython=False)',\n setup=setup,\n number=20\n)\ncytime = timeit.timeit(\n 'metrics.evaluate_rank(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_cython=True)',\n setup=setup,\n number=20\n)\nprint('Python time: {} s'.format(pytime))\nprint('Cython time: {} s'.format(cytime))\nprint('Cython is {} times faster than python\\n'.format(pytime / cytime))\n\nprint('=> Using cuhk03\\'s metric')\npytime = timeit.timeit(\n 'metrics.evaluate_rank(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_metric_cuhk03=True, use_cython=False)',\n setup=setup,\n number=20\n)\ncytime = timeit.timeit(\n 'metrics.evaluate_rank(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_metric_cuhk03=True, use_cython=True)',\n setup=setup,\n number=20\n)\nprint('Python time: {} s'.format(pytime))\nprint('Cython time: {} s'.format(cytime))\nprint('Cython is {} times faster than python\\n'.format(pytime / cytime))\n\"\"\"\nprint(\"=> Check precision\")\n\nnum_q = 30\nnum_g = 300\nmax_rank = 5\ndistmat = np.random.rand(num_q, num_g) * 20\nq_pids = np.random.randint(0, num_q, size=num_q)\ng_pids = np.random.randint(0, num_g, size=num_g)\nq_camids = np.random.randint(0, 5, size=num_q)\ng_camids = np.random.randint(0, 5, size=num_g)\n\ncmc, mAP = evaluate(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_cython=False)\nprint(\"Python:\\nmAP = {} \\ncmc = {}\\n\".format(mAP, cmc))\ncmc, mAP = evaluate(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, use_cython=True)\nprint(\"Cython:\\nmAP = {} \\ncmc = {}\\n\".format(mAP, cmc))\n\"\"\"\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/__init__.py", "content": "from __future__ import absolute_import\nimport torch\n\nfrom .pcb import *\nfrom .mlfn import *\nfrom .hacnn import *\nfrom .osnet import *\nfrom .senet import *\nfrom .mudeep import *\nfrom .nasnet import *\nfrom .resnet import *\nfrom .densenet import *\nfrom .xception import *\nfrom .osnet_ain import *\nfrom .resnetmid import *\nfrom .shufflenet import *\nfrom .squeezenet import *\nfrom .inceptionv4 import *\nfrom .mobilenetv2 import *\nfrom .resnet_ibn_a import *\nfrom .resnet_ibn_b import *\nfrom .shufflenetv2 import *\nfrom .inceptionresnetv2 import *\n\n__model_factory = {\n # image classification models\n 'resnet18': resnet18,\n 'resnet34': resnet34,\n 'resnet50': resnet50,\n 'resnet101': resnet101,\n 'resnet152': resnet152,\n 'resnext50_32x4d': resnext50_32x4d,\n 'resnext101_32x8d': resnext101_32x8d,\n 'resnet50_fc512': resnet50_fc512,\n 'se_resnet50': se_resnet50,\n 'se_resnet50_fc512': se_resnet50_fc512,\n 'se_resnet101': se_resnet101,\n 'se_resnext50_32x4d': se_resnext50_32x4d,\n 'se_resnext101_32x4d': se_resnext101_32x4d,\n 'densenet121': densenet121,\n 'densenet169': densenet169,\n 'densenet201': densenet201,\n 'densenet161': densenet161,\n 'densenet121_fc512': densenet121_fc512,\n 'inceptionresnetv2': inceptionresnetv2,\n 'inceptionv4': inceptionv4,\n 'xception': xception,\n 'resnet50_ibn_a': resnet50_ibn_a,\n 'resnet50_ibn_b': resnet50_ibn_b,\n # lightweight models\n 'nasnsetmobile': nasnetamobile,\n 'mobilenetv2_x1_0': mobilenetv2_x1_0,\n 'mobilenetv2_x1_4': mobilenetv2_x1_4,\n 'shufflenet': shufflenet,\n 'squeezenet1_0': squeezenet1_0,\n 'squeezenet1_0_fc512': squeezenet1_0_fc512,\n 'squeezenet1_1': squeezenet1_1,\n 'shufflenet_v2_x0_5': shufflenet_v2_x0_5,\n 'shufflenet_v2_x1_0': shufflenet_v2_x1_0,\n 'shufflenet_v2_x1_5': shufflenet_v2_x1_5,\n 'shufflenet_v2_x2_0': shufflenet_v2_x2_0,\n # reid-specific models\n 'mudeep': MuDeep,\n 'resnet50mid': resnet50mid,\n 'hacnn': HACNN,\n 'pcb_p6': pcb_p6,\n 'pcb_p4': pcb_p4,\n 'mlfn': mlfn,\n 'osnet_x1_0': osnet_x1_0,\n 'osnet_x0_75': osnet_x0_75,\n 'osnet_x0_5': osnet_x0_5,\n 'osnet_x0_25': osnet_x0_25,\n 'osnet_ibn_x1_0': osnet_ibn_x1_0,\n 'osnet_ain_x1_0': osnet_ain_x1_0,\n 'osnet_ain_x0_75': osnet_ain_x0_75,\n 'osnet_ain_x0_5': osnet_ain_x0_5,\n 'osnet_ain_x0_25': osnet_ain_x0_25\n}\n\n\ndef show_avai_models():\n \"\"\"Displays available models.\n\n Examples::\n >>> from torchreid import models\n >>> models.show_avai_models()\n \"\"\"\n print(list(__model_factory.keys()))\n\n\ndef build_model(\n name, num_classes, loss='softmax', pretrained=True, use_gpu=True\n):\n \"\"\"A function wrapper for building a model.\n\n Args:\n name (str): model name.\n num_classes (int): number of training identities.\n loss (str, optional): loss function to optimize the model. Currently\n supports \"softmax\" and \"triplet\". Default is \"softmax\".\n pretrained (bool, optional): whether to load ImageNet-pretrained weights.\n Default is True.\n use_gpu (bool, optional): whether to use gpu. Default is True.\n\n Returns:\n nn.Module\n\n Examples::\n >>> from torchreid import models\n >>> model = models.build_model('resnet50', 751, loss='softmax')\n \"\"\"\n avai_models = list(__model_factory.keys())\n if name not in avai_models:\n raise KeyError(\n 'Unknown model: {}. Must be one of {}'.format(name, avai_models)\n )\n return __model_factory[name](\n num_classes=num_classes,\n loss=loss,\n pretrained=pretrained,\n use_gpu=use_gpu\n )\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/densenet.py", "content": "\"\"\"\nCode source: https://github.com/pytorch/vision\n\"\"\"\nfrom __future__ import division, absolute_import\nimport re\nfrom collections import OrderedDict\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.utils import model_zoo\n\n__all__ = [\n 'densenet121', 'densenet169', 'densenet201', 'densenet161',\n 'densenet121_fc512'\n]\n\nmodel_urls = {\n 'densenet121':\n 'https://download.pytorch.org/models/densenet121-a639ec97.pth',\n 'densenet169':\n 'https://download.pytorch.org/models/densenet169-b2777c0a.pth',\n 'densenet201':\n 'https://download.pytorch.org/models/densenet201-c1103571.pth',\n 'densenet161':\n 'https://download.pytorch.org/models/densenet161-8d451a50.pth',\n}\n\n\nclass _DenseLayer(nn.Sequential):\n\n def __init__(self, num_input_features, growth_rate, bn_size, drop_rate):\n super(_DenseLayer, self).__init__()\n self.add_module('norm1', nn.BatchNorm2d(num_input_features)),\n self.add_module('relu1', nn.ReLU(inplace=True)),\n self.add_module(\n 'conv1',\n nn.Conv2d(\n num_input_features,\n bn_size * growth_rate,\n kernel_size=1,\n stride=1,\n bias=False\n )\n ),\n self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)),\n self.add_module('relu2', nn.ReLU(inplace=True)),\n self.add_module(\n 'conv2',\n nn.Conv2d(\n bn_size * growth_rate,\n growth_rate,\n kernel_size=3,\n stride=1,\n padding=1,\n bias=False\n )\n ),\n self.drop_rate = drop_rate\n\n def forward(self, x):\n new_features = super(_DenseLayer, self).forward(x)\n if self.drop_rate > 0:\n new_features = F.dropout(\n new_features, p=self.drop_rate, training=self.training\n )\n return torch.cat([x, new_features], 1)\n\n\nclass _DenseBlock(nn.Sequential):\n\n def __init__(\n self, num_layers, num_input_features, bn_size, growth_rate, drop_rate\n ):\n super(_DenseBlock, self).__init__()\n for i in range(num_layers):\n layer = _DenseLayer(\n num_input_features + i*growth_rate, growth_rate, bn_size,\n drop_rate\n )\n self.add_module('denselayer%d' % (i+1), layer)\n\n\nclass _Transition(nn.Sequential):\n\n def __init__(self, num_input_features, num_output_features):\n super(_Transition, self).__init__()\n self.add_module('norm', nn.BatchNorm2d(num_input_features))\n self.add_module('relu', nn.ReLU(inplace=True))\n self.add_module(\n 'conv',\n nn.Conv2d(\n num_input_features,\n num_output_features,\n kernel_size=1,\n stride=1,\n bias=False\n )\n )\n self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2))\n\n\nclass DenseNet(nn.Module):\n \"\"\"Densely connected network.\n \n Reference:\n Huang et al. Densely Connected Convolutional Networks. CVPR 2017.\n\n Public keys:\n - ``densenet121``: DenseNet121.\n - ``densenet169``: DenseNet169.\n - ``densenet201``: DenseNet201.\n - ``densenet161``: DenseNet161.\n - ``densenet121_fc512``: DenseNet121 + FC.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n loss,\n growth_rate=32,\n block_config=(6, 12, 24, 16),\n num_init_features=64,\n bn_size=4,\n drop_rate=0,\n fc_dims=None,\n dropout_p=None,\n **kwargs\n ):\n\n super(DenseNet, self).__init__()\n self.loss = loss\n\n # First convolution\n self.features = nn.Sequential(\n OrderedDict(\n [\n (\n 'conv0',\n nn.Conv2d(\n 3,\n num_init_features,\n kernel_size=7,\n stride=2,\n padding=3,\n bias=False\n )\n ),\n ('norm0', nn.BatchNorm2d(num_init_features)),\n ('relu0', nn.ReLU(inplace=True)),\n (\n 'pool0',\n nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n ),\n ]\n )\n )\n\n # Each denseblock\n num_features = num_init_features\n for i, num_layers in enumerate(block_config):\n block = _DenseBlock(\n num_layers=num_layers,\n num_input_features=num_features,\n bn_size=bn_size,\n growth_rate=growth_rate,\n drop_rate=drop_rate\n )\n self.features.add_module('denseblock%d' % (i+1), block)\n num_features = num_features + num_layers*growth_rate\n if i != len(block_config) - 1:\n trans = _Transition(\n num_input_features=num_features,\n num_output_features=num_features // 2\n )\n self.features.add_module('transition%d' % (i+1), trans)\n num_features = num_features // 2\n\n # Final batch norm\n self.features.add_module('norm5', nn.BatchNorm2d(num_features))\n\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.feature_dim = num_features\n self.fc = self._construct_fc_layer(fc_dims, num_features, dropout_p)\n\n # Linear layer\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n self._init_params()\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n \"\"\"Constructs fully connected layer.\n\n Args:\n fc_dims (list or tuple): dimensions of fc layers, if None, no fc layers are constructed\n input_dim (int): input dimension\n dropout_p (float): dropout probability, if None, dropout is unused\n \"\"\"\n if fc_dims is None:\n self.feature_dim = input_dim\n return None\n\n assert isinstance(\n fc_dims, (list, tuple)\n ), 'fc_dims must be either list or tuple, but got {}'.format(\n type(fc_dims)\n )\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def forward(self, x):\n f = self.features(x)\n f = F.relu(f, inplace=True)\n v = self.global_avgpool(f)\n v = v.view(v.size(0), -1)\n\n if self.fc is not None:\n v = self.fc(v)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n\n # '.'s are no longer allowed in module names, but pervious _DenseLayer\n # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'.\n # They are also in the checkpoints in model_urls. This pattern is used\n # to find such keys.\n pattern = re.compile(\n r'^(.*denselayer\\d+\\.(?:norm|relu|conv))\\.((?:[12])\\.(?:weight|bias|running_mean|running_var))$'\n )\n for key in list(pretrain_dict.keys()):\n res = pattern.match(key)\n if res:\n new_key = res.group(1) + res.group(2)\n pretrain_dict[new_key] = pretrain_dict[key]\n del pretrain_dict[key]\n\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\n\"\"\"\nDense network configurations:\n--\ndensenet121: num_init_features=64, growth_rate=32, block_config=(6, 12, 24, 16)\ndensenet169: num_init_features=64, growth_rate=32, block_config=(6, 12, 32, 32)\ndensenet201: num_init_features=64, growth_rate=32, block_config=(6, 12, 48, 32)\ndensenet161: num_init_features=96, growth_rate=48, block_config=(6, 12, 36, 24)\n\"\"\"\n\n\ndef densenet121(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = DenseNet(\n num_classes=num_classes,\n loss=loss,\n num_init_features=64,\n growth_rate=32,\n block_config=(6, 12, 24, 16),\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['densenet121'])\n return model\n\n\ndef densenet169(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = DenseNet(\n num_classes=num_classes,\n loss=loss,\n num_init_features=64,\n growth_rate=32,\n block_config=(6, 12, 32, 32),\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['densenet169'])\n return model\n\n\ndef densenet201(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = DenseNet(\n num_classes=num_classes,\n loss=loss,\n num_init_features=64,\n growth_rate=32,\n block_config=(6, 12, 48, 32),\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['densenet201'])\n return model\n\n\ndef densenet161(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = DenseNet(\n num_classes=num_classes,\n loss=loss,\n num_init_features=96,\n growth_rate=48,\n block_config=(6, 12, 36, 24),\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['densenet161'])\n return model\n\n\ndef densenet121_fc512(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = DenseNet(\n num_classes=num_classes,\n loss=loss,\n num_init_features=64,\n growth_rate=32,\n block_config=(6, 12, 24, 16),\n fc_dims=[512],\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['densenet121'])\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/hacnn.py", "content": "from __future__ import division, absolute_import\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\n__all__ = ['HACNN']\n\n\nclass ConvBlock(nn.Module):\n \"\"\"Basic convolutional block.\n \n convolution + batch normalization + relu.\n\n Args:\n in_c (int): number of input channels.\n out_c (int): number of output channels.\n k (int or tuple): kernel size.\n s (int or tuple): stride.\n p (int or tuple): padding.\n \"\"\"\n\n def __init__(self, in_c, out_c, k, s=1, p=0):\n super(ConvBlock, self).__init__()\n self.conv = nn.Conv2d(in_c, out_c, k, stride=s, padding=p)\n self.bn = nn.BatchNorm2d(out_c)\n\n def forward(self, x):\n return F.relu(self.bn(self.conv(x)))\n\n\nclass InceptionA(nn.Module):\n\n def __init__(self, in_channels, out_channels):\n super(InceptionA, self).__init__()\n mid_channels = out_channels // 4\n\n self.stream1 = nn.Sequential(\n ConvBlock(in_channels, mid_channels, 1),\n ConvBlock(mid_channels, mid_channels, 3, p=1),\n )\n self.stream2 = nn.Sequential(\n ConvBlock(in_channels, mid_channels, 1),\n ConvBlock(mid_channels, mid_channels, 3, p=1),\n )\n self.stream3 = nn.Sequential(\n ConvBlock(in_channels, mid_channels, 1),\n ConvBlock(mid_channels, mid_channels, 3, p=1),\n )\n self.stream4 = nn.Sequential(\n nn.AvgPool2d(3, stride=1, padding=1),\n ConvBlock(in_channels, mid_channels, 1),\n )\n\n def forward(self, x):\n s1 = self.stream1(x)\n s2 = self.stream2(x)\n s3 = self.stream3(x)\n s4 = self.stream4(x)\n y = torch.cat([s1, s2, s3, s4], dim=1)\n return y\n\n\nclass InceptionB(nn.Module):\n\n def __init__(self, in_channels, out_channels):\n super(InceptionB, self).__init__()\n mid_channels = out_channels // 4\n\n self.stream1 = nn.Sequential(\n ConvBlock(in_channels, mid_channels, 1),\n ConvBlock(mid_channels, mid_channels, 3, s=2, p=1),\n )\n self.stream2 = nn.Sequential(\n ConvBlock(in_channels, mid_channels, 1),\n ConvBlock(mid_channels, mid_channels, 3, p=1),\n ConvBlock(mid_channels, mid_channels, 3, s=2, p=1),\n )\n self.stream3 = nn.Sequential(\n nn.MaxPool2d(3, stride=2, padding=1),\n ConvBlock(in_channels, mid_channels * 2, 1),\n )\n\n def forward(self, x):\n s1 = self.stream1(x)\n s2 = self.stream2(x)\n s3 = self.stream3(x)\n y = torch.cat([s1, s2, s3], dim=1)\n return y\n\n\nclass SpatialAttn(nn.Module):\n \"\"\"Spatial Attention (Sec. 3.1.I.1)\"\"\"\n\n def __init__(self):\n super(SpatialAttn, self).__init__()\n self.conv1 = ConvBlock(1, 1, 3, s=2, p=1)\n self.conv2 = ConvBlock(1, 1, 1)\n\n def forward(self, x):\n # global cross-channel averaging\n x = x.mean(1, keepdim=True)\n # 3-by-3 conv\n x = self.conv1(x)\n # bilinear resizing\n x = F.upsample(\n x, (x.size(2) * 2, x.size(3) * 2),\n mode='bilinear',\n align_corners=True\n )\n # scaling conv\n x = self.conv2(x)\n return x\n\n\nclass ChannelAttn(nn.Module):\n \"\"\"Channel Attention (Sec. 3.1.I.2)\"\"\"\n\n def __init__(self, in_channels, reduction_rate=16):\n super(ChannelAttn, self).__init__()\n assert in_channels % reduction_rate == 0\n self.conv1 = ConvBlock(in_channels, in_channels // reduction_rate, 1)\n self.conv2 = ConvBlock(in_channels // reduction_rate, in_channels, 1)\n\n def forward(self, x):\n # squeeze operation (global average pooling)\n x = F.avg_pool2d(x, x.size()[2:])\n # excitation operation (2 conv layers)\n x = self.conv1(x)\n x = self.conv2(x)\n return x\n\n\nclass SoftAttn(nn.Module):\n \"\"\"Soft Attention (Sec. 3.1.I)\n \n Aim: Spatial Attention + Channel Attention\n \n Output: attention maps with shape identical to input.\n \"\"\"\n\n def __init__(self, in_channels):\n super(SoftAttn, self).__init__()\n self.spatial_attn = SpatialAttn()\n self.channel_attn = ChannelAttn(in_channels)\n self.conv = ConvBlock(in_channels, in_channels, 1)\n\n def forward(self, x):\n y_spatial = self.spatial_attn(x)\n y_channel = self.channel_attn(x)\n y = y_spatial * y_channel\n y = torch.sigmoid(self.conv(y))\n return y\n\n\nclass HardAttn(nn.Module):\n \"\"\"Hard Attention (Sec. 3.1.II)\"\"\"\n\n def __init__(self, in_channels):\n super(HardAttn, self).__init__()\n self.fc = nn.Linear(in_channels, 4 * 2)\n self.init_params()\n\n def init_params(self):\n self.fc.weight.data.zero_()\n self.fc.bias.data.copy_(\n torch.tensor(\n [0, -0.75, 0, -0.25, 0, 0.25, 0, 0.75], dtype=torch.float\n )\n )\n\n def forward(self, x):\n # squeeze operation (global average pooling)\n x = F.avg_pool2d(x, x.size()[2:]).view(x.size(0), x.size(1))\n # predict transformation parameters\n theta = torch.tanh(self.fc(x))\n theta = theta.view(-1, 4, 2)\n return theta\n\n\nclass HarmAttn(nn.Module):\n \"\"\"Harmonious Attention (Sec. 3.1)\"\"\"\n\n def __init__(self, in_channels):\n super(HarmAttn, self).__init__()\n self.soft_attn = SoftAttn(in_channels)\n self.hard_attn = HardAttn(in_channels)\n\n def forward(self, x):\n y_soft_attn = self.soft_attn(x)\n theta = self.hard_attn(x)\n return y_soft_attn, theta\n\n\nclass HACNN(nn.Module):\n \"\"\"Harmonious Attention Convolutional Neural Network.\n\n Reference:\n Li et al. Harmonious Attention Network for Person Re-identification. CVPR 2018.\n\n Public keys:\n - ``hacnn``: HACNN.\n \"\"\"\n\n # Args:\n # num_classes (int): number of classes to predict\n # nchannels (list): number of channels AFTER concatenation\n # feat_dim (int): feature dimension for a single stream\n # learn_region (bool): whether to learn region features (i.e. local branch)\n\n def __init__(\n self,\n num_classes,\n loss='softmax',\n nchannels=[128, 256, 384],\n feat_dim=512,\n learn_region=True,\n use_gpu=True,\n **kwargs\n ):\n super(HACNN, self).__init__()\n self.loss = loss\n self.learn_region = learn_region\n self.use_gpu = use_gpu\n\n self.conv = ConvBlock(3, 32, 3, s=2, p=1)\n\n # Construct Inception + HarmAttn blocks\n # ============== Block 1 ==============\n self.inception1 = nn.Sequential(\n InceptionA(32, nchannels[0]),\n InceptionB(nchannels[0], nchannels[0]),\n )\n self.ha1 = HarmAttn(nchannels[0])\n\n # ============== Block 2 ==============\n self.inception2 = nn.Sequential(\n InceptionA(nchannels[0], nchannels[1]),\n InceptionB(nchannels[1], nchannels[1]),\n )\n self.ha2 = HarmAttn(nchannels[1])\n\n # ============== Block 3 ==============\n self.inception3 = nn.Sequential(\n InceptionA(nchannels[1], nchannels[2]),\n InceptionB(nchannels[2], nchannels[2]),\n )\n self.ha3 = HarmAttn(nchannels[2])\n\n self.fc_global = nn.Sequential(\n nn.Linear(nchannels[2], feat_dim),\n nn.BatchNorm1d(feat_dim),\n nn.ReLU(),\n )\n self.classifier_global = nn.Linear(feat_dim, num_classes)\n\n if self.learn_region:\n self.init_scale_factors()\n self.local_conv1 = InceptionB(32, nchannels[0])\n self.local_conv2 = InceptionB(nchannels[0], nchannels[1])\n self.local_conv3 = InceptionB(nchannels[1], nchannels[2])\n self.fc_local = nn.Sequential(\n nn.Linear(nchannels[2] * 4, feat_dim),\n nn.BatchNorm1d(feat_dim),\n nn.ReLU(),\n )\n self.classifier_local = nn.Linear(feat_dim, num_classes)\n self.feat_dim = feat_dim * 2\n else:\n self.feat_dim = feat_dim\n\n def init_scale_factors(self):\n # initialize scale factors (s_w, s_h) for four regions\n self.scale_factors = []\n self.scale_factors.append(\n torch.tensor([[1, 0], [0, 0.25]], dtype=torch.float)\n )\n self.scale_factors.append(\n torch.tensor([[1, 0], [0, 0.25]], dtype=torch.float)\n )\n self.scale_factors.append(\n torch.tensor([[1, 0], [0, 0.25]], dtype=torch.float)\n )\n self.scale_factors.append(\n torch.tensor([[1, 0], [0, 0.25]], dtype=torch.float)\n )\n\n def stn(self, x, theta):\n \"\"\"Performs spatial transform\n \n x: (batch, channel, height, width)\n theta: (batch, 2, 3)\n \"\"\"\n grid = F.affine_grid(theta, x.size())\n x = F.grid_sample(x, grid)\n return x\n\n def transform_theta(self, theta_i, region_idx):\n \"\"\"Transforms theta to include (s_w, s_h), resulting in (batch, 2, 3)\"\"\"\n scale_factors = self.scale_factors[region_idx]\n theta = torch.zeros(theta_i.size(0), 2, 3)\n theta[:, :, :2] = scale_factors\n theta[:, :, -1] = theta_i\n if self.use_gpu:\n theta = theta.cuda()\n return theta\n\n def forward(self, x):\n assert x.size(2) == 160 and x.size(3) == 64, \\\n 'Input size does not match, expected (160, 64) but got ({}, {})'.format(x.size(2), x.size(3))\n x = self.conv(x)\n\n # ============== Block 1 ==============\n # global branch\n x1 = self.inception1(x)\n x1_attn, x1_theta = self.ha1(x1)\n x1_out = x1 * x1_attn\n # local branch\n if self.learn_region:\n x1_local_list = []\n for region_idx in range(4):\n x1_theta_i = x1_theta[:, region_idx, :]\n x1_theta_i = self.transform_theta(x1_theta_i, region_idx)\n x1_trans_i = self.stn(x, x1_theta_i)\n x1_trans_i = F.upsample(\n x1_trans_i, (24, 28), mode='bilinear', align_corners=True\n )\n x1_local_i = self.local_conv1(x1_trans_i)\n x1_local_list.append(x1_local_i)\n\n # ============== Block 2 ==============\n # Block 2\n # global branch\n x2 = self.inception2(x1_out)\n x2_attn, x2_theta = self.ha2(x2)\n x2_out = x2 * x2_attn\n # local branch\n if self.learn_region:\n x2_local_list = []\n for region_idx in range(4):\n x2_theta_i = x2_theta[:, region_idx, :]\n x2_theta_i = self.transform_theta(x2_theta_i, region_idx)\n x2_trans_i = self.stn(x1_out, x2_theta_i)\n x2_trans_i = F.upsample(\n x2_trans_i, (12, 14), mode='bilinear', align_corners=True\n )\n x2_local_i = x2_trans_i + x1_local_list[region_idx]\n x2_local_i = self.local_conv2(x2_local_i)\n x2_local_list.append(x2_local_i)\n\n # ============== Block 3 ==============\n # Block 3\n # global branch\n x3 = self.inception3(x2_out)\n x3_attn, x3_theta = self.ha3(x3)\n x3_out = x3 * x3_attn\n # local branch\n if self.learn_region:\n x3_local_list = []\n for region_idx in range(4):\n x3_theta_i = x3_theta[:, region_idx, :]\n x3_theta_i = self.transform_theta(x3_theta_i, region_idx)\n x3_trans_i = self.stn(x2_out, x3_theta_i)\n x3_trans_i = F.upsample(\n x3_trans_i, (6, 7), mode='bilinear', align_corners=True\n )\n x3_local_i = x3_trans_i + x2_local_list[region_idx]\n x3_local_i = self.local_conv3(x3_local_i)\n x3_local_list.append(x3_local_i)\n\n # ============== Feature generation ==============\n # global branch\n x_global = F.avg_pool2d(x3_out,\n x3_out.size()[2:]\n ).view(x3_out.size(0), x3_out.size(1))\n x_global = self.fc_global(x_global)\n # local branch\n if self.learn_region:\n x_local_list = []\n for region_idx in range(4):\n x_local_i = x3_local_list[region_idx]\n x_local_i = F.avg_pool2d(x_local_i,\n x_local_i.size()[2:]\n ).view(x_local_i.size(0), -1)\n x_local_list.append(x_local_i)\n x_local = torch.cat(x_local_list, 1)\n x_local = self.fc_local(x_local)\n\n if not self.training:\n # l2 normalization before concatenation\n if self.learn_region:\n x_global = x_global / x_global.norm(p=2, dim=1, keepdim=True)\n x_local = x_local / x_local.norm(p=2, dim=1, keepdim=True)\n return torch.cat([x_global, x_local], 1)\n else:\n return x_global\n\n prelogits_global = self.classifier_global(x_global)\n if self.learn_region:\n prelogits_local = self.classifier_local(x_local)\n\n if self.loss == 'softmax':\n if self.learn_region:\n return (prelogits_global, prelogits_local)\n else:\n return prelogits_global\n\n elif self.loss == 'triplet':\n if self.learn_region:\n return (prelogits_global, prelogits_local), (x_global, x_local)\n else:\n return prelogits_global, x_global\n\n else:\n raise KeyError(\"Unsupported loss: {}\".format(self.loss))\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/inceptionresnetv2.py", "content": "\"\"\"\nCode imported from https://github.com/Cadene/pretrained-models.pytorch\n\"\"\"\nfrom __future__ import division, absolute_import\nimport torch\nimport torch.nn as nn\nimport torch.utils.model_zoo as model_zoo\n\n__all__ = ['inceptionresnetv2']\n\npretrained_settings = {\n 'inceptionresnetv2': {\n 'imagenet': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionresnetv2-520b38e4.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 299, 299],\n 'input_range': [0, 1],\n 'mean': [0.5, 0.5, 0.5],\n 'std': [0.5, 0.5, 0.5],\n 'num_classes': 1000\n },\n 'imagenet+background': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionresnetv2-520b38e4.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 299, 299],\n 'input_range': [0, 1],\n 'mean': [0.5, 0.5, 0.5],\n 'std': [0.5, 0.5, 0.5],\n 'num_classes': 1001\n }\n }\n}\n\n\nclass BasicConv2d(nn.Module):\n\n def __init__(self, in_planes, out_planes, kernel_size, stride, padding=0):\n super(BasicConv2d, self).__init__()\n self.conv = nn.Conv2d(\n in_planes,\n out_planes,\n kernel_size=kernel_size,\n stride=stride,\n padding=padding,\n bias=False\n ) # verify bias false\n self.bn = nn.BatchNorm2d(\n out_planes,\n eps=0.001, # value found in tensorflow\n momentum=0.1, # default pytorch value\n affine=True\n )\n self.relu = nn.ReLU(inplace=False)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n x = self.relu(x)\n return x\n\n\nclass Mixed_5b(nn.Module):\n\n def __init__(self):\n super(Mixed_5b, self).__init__()\n\n self.branch0 = BasicConv2d(192, 96, kernel_size=1, stride=1)\n\n self.branch1 = nn.Sequential(\n BasicConv2d(192, 48, kernel_size=1, stride=1),\n BasicConv2d(48, 64, kernel_size=5, stride=1, padding=2)\n )\n\n self.branch2 = nn.Sequential(\n BasicConv2d(192, 64, kernel_size=1, stride=1),\n BasicConv2d(64, 96, kernel_size=3, stride=1, padding=1),\n BasicConv2d(96, 96, kernel_size=3, stride=1, padding=1)\n )\n\n self.branch3 = nn.Sequential(\n nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),\n BasicConv2d(192, 64, kernel_size=1, stride=1)\n )\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n x2 = self.branch2(x)\n x3 = self.branch3(x)\n out = torch.cat((x0, x1, x2, x3), 1)\n return out\n\n\nclass Block35(nn.Module):\n\n def __init__(self, scale=1.0):\n super(Block35, self).__init__()\n\n self.scale = scale\n\n self.branch0 = BasicConv2d(320, 32, kernel_size=1, stride=1)\n\n self.branch1 = nn.Sequential(\n BasicConv2d(320, 32, kernel_size=1, stride=1),\n BasicConv2d(32, 32, kernel_size=3, stride=1, padding=1)\n )\n\n self.branch2 = nn.Sequential(\n BasicConv2d(320, 32, kernel_size=1, stride=1),\n BasicConv2d(32, 48, kernel_size=3, stride=1, padding=1),\n BasicConv2d(48, 64, kernel_size=3, stride=1, padding=1)\n )\n\n self.conv2d = nn.Conv2d(128, 320, kernel_size=1, stride=1)\n self.relu = nn.ReLU(inplace=False)\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n x2 = self.branch2(x)\n out = torch.cat((x0, x1, x2), 1)\n out = self.conv2d(out)\n out = out * self.scale + x\n out = self.relu(out)\n return out\n\n\nclass Mixed_6a(nn.Module):\n\n def __init__(self):\n super(Mixed_6a, self).__init__()\n\n self.branch0 = BasicConv2d(320, 384, kernel_size=3, stride=2)\n\n self.branch1 = nn.Sequential(\n BasicConv2d(320, 256, kernel_size=1, stride=1),\n BasicConv2d(256, 256, kernel_size=3, stride=1, padding=1),\n BasicConv2d(256, 384, kernel_size=3, stride=2)\n )\n\n self.branch2 = nn.MaxPool2d(3, stride=2)\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n x2 = self.branch2(x)\n out = torch.cat((x0, x1, x2), 1)\n return out\n\n\nclass Block17(nn.Module):\n\n def __init__(self, scale=1.0):\n super(Block17, self).__init__()\n\n self.scale = scale\n\n self.branch0 = BasicConv2d(1088, 192, kernel_size=1, stride=1)\n\n self.branch1 = nn.Sequential(\n BasicConv2d(1088, 128, kernel_size=1, stride=1),\n BasicConv2d(\n 128, 160, kernel_size=(1, 7), stride=1, padding=(0, 3)\n ),\n BasicConv2d(\n 160, 192, kernel_size=(7, 1), stride=1, padding=(3, 0)\n )\n )\n\n self.conv2d = nn.Conv2d(384, 1088, kernel_size=1, stride=1)\n self.relu = nn.ReLU(inplace=False)\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n out = torch.cat((x0, x1), 1)\n out = self.conv2d(out)\n out = out * self.scale + x\n out = self.relu(out)\n return out\n\n\nclass Mixed_7a(nn.Module):\n\n def __init__(self):\n super(Mixed_7a, self).__init__()\n\n self.branch0 = nn.Sequential(\n BasicConv2d(1088, 256, kernel_size=1, stride=1),\n BasicConv2d(256, 384, kernel_size=3, stride=2)\n )\n\n self.branch1 = nn.Sequential(\n BasicConv2d(1088, 256, kernel_size=1, stride=1),\n BasicConv2d(256, 288, kernel_size=3, stride=2)\n )\n\n self.branch2 = nn.Sequential(\n BasicConv2d(1088, 256, kernel_size=1, stride=1),\n BasicConv2d(256, 288, kernel_size=3, stride=1, padding=1),\n BasicConv2d(288, 320, kernel_size=3, stride=2)\n )\n\n self.branch3 = nn.MaxPool2d(3, stride=2)\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n x2 = self.branch2(x)\n x3 = self.branch3(x)\n out = torch.cat((x0, x1, x2, x3), 1)\n return out\n\n\nclass Block8(nn.Module):\n\n def __init__(self, scale=1.0, noReLU=False):\n super(Block8, self).__init__()\n\n self.scale = scale\n self.noReLU = noReLU\n\n self.branch0 = BasicConv2d(2080, 192, kernel_size=1, stride=1)\n\n self.branch1 = nn.Sequential(\n BasicConv2d(2080, 192, kernel_size=1, stride=1),\n BasicConv2d(\n 192, 224, kernel_size=(1, 3), stride=1, padding=(0, 1)\n ),\n BasicConv2d(\n 224, 256, kernel_size=(3, 1), stride=1, padding=(1, 0)\n )\n )\n\n self.conv2d = nn.Conv2d(448, 2080, kernel_size=1, stride=1)\n if not self.noReLU:\n self.relu = nn.ReLU(inplace=False)\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n out = torch.cat((x0, x1), 1)\n out = self.conv2d(out)\n out = out * self.scale + x\n if not self.noReLU:\n out = self.relu(out)\n return out\n\n\n# ----------------\n# Model Definition\n# ----------------\nclass InceptionResNetV2(nn.Module):\n \"\"\"Inception-ResNet-V2.\n\n Reference:\n Szegedy et al. Inception-v4, Inception-ResNet and the Impact of Residual\n Connections on Learning. AAAI 2017.\n\n Public keys:\n - ``inceptionresnetv2``: Inception-ResNet-V2.\n \"\"\"\n\n def __init__(self, num_classes, loss='softmax', **kwargs):\n super(InceptionResNetV2, self).__init__()\n self.loss = loss\n\n # Modules\n self.conv2d_1a = BasicConv2d(3, 32, kernel_size=3, stride=2)\n self.conv2d_2a = BasicConv2d(32, 32, kernel_size=3, stride=1)\n self.conv2d_2b = BasicConv2d(\n 32, 64, kernel_size=3, stride=1, padding=1\n )\n self.maxpool_3a = nn.MaxPool2d(3, stride=2)\n self.conv2d_3b = BasicConv2d(64, 80, kernel_size=1, stride=1)\n self.conv2d_4a = BasicConv2d(80, 192, kernel_size=3, stride=1)\n self.maxpool_5a = nn.MaxPool2d(3, stride=2)\n self.mixed_5b = Mixed_5b()\n self.repeat = nn.Sequential(\n Block35(scale=0.17), Block35(scale=0.17), Block35(scale=0.17),\n Block35(scale=0.17), Block35(scale=0.17), Block35(scale=0.17),\n Block35(scale=0.17), Block35(scale=0.17), Block35(scale=0.17),\n Block35(scale=0.17)\n )\n self.mixed_6a = Mixed_6a()\n self.repeat_1 = nn.Sequential(\n Block17(scale=0.10), Block17(scale=0.10), Block17(scale=0.10),\n Block17(scale=0.10), Block17(scale=0.10), Block17(scale=0.10),\n Block17(scale=0.10), Block17(scale=0.10), Block17(scale=0.10),\n Block17(scale=0.10), Block17(scale=0.10), Block17(scale=0.10),\n Block17(scale=0.10), Block17(scale=0.10), Block17(scale=0.10),\n Block17(scale=0.10), Block17(scale=0.10), Block17(scale=0.10),\n Block17(scale=0.10), Block17(scale=0.10)\n )\n self.mixed_7a = Mixed_7a()\n self.repeat_2 = nn.Sequential(\n Block8(scale=0.20), Block8(scale=0.20), Block8(scale=0.20),\n Block8(scale=0.20), Block8(scale=0.20), Block8(scale=0.20),\n Block8(scale=0.20), Block8(scale=0.20), Block8(scale=0.20)\n )\n\n self.block8 = Block8(noReLU=True)\n self.conv2d_7b = BasicConv2d(2080, 1536, kernel_size=1, stride=1)\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.classifier = nn.Linear(1536, num_classes)\n\n def load_imagenet_weights(self):\n settings = pretrained_settings['inceptionresnetv2']['imagenet']\n pretrain_dict = model_zoo.load_url(settings['url'])\n model_dict = self.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n self.load_state_dict(model_dict)\n\n def featuremaps(self, x):\n x = self.conv2d_1a(x)\n x = self.conv2d_2a(x)\n x = self.conv2d_2b(x)\n x = self.maxpool_3a(x)\n x = self.conv2d_3b(x)\n x = self.conv2d_4a(x)\n x = self.maxpool_5a(x)\n x = self.mixed_5b(x)\n x = self.repeat(x)\n x = self.mixed_6a(x)\n x = self.repeat_1(x)\n x = self.mixed_7a(x)\n x = self.repeat_2(x)\n x = self.block8(x)\n x = self.conv2d_7b(x)\n return x\n\n def forward(self, x):\n f = self.featuremaps(x)\n v = self.global_avgpool(f)\n v = v.view(v.size(0), -1)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n\n\ndef inceptionresnetv2(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = InceptionResNetV2(num_classes=num_classes, loss=loss, **kwargs)\n if pretrained:\n model.load_imagenet_weights()\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/inceptionv4.py", "content": "from __future__ import division, absolute_import\nimport torch\nimport torch.nn as nn\nimport torch.utils.model_zoo as model_zoo\n\n__all__ = ['inceptionv4']\n\"\"\"\nCode imported from https://github.com/Cadene/pretrained-models.pytorch\n\"\"\"\n\npretrained_settings = {\n 'inceptionv4': {\n 'imagenet': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionv4-8e4777a0.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 299, 299],\n 'input_range': [0, 1],\n 'mean': [0.5, 0.5, 0.5],\n 'std': [0.5, 0.5, 0.5],\n 'num_classes': 1000\n },\n 'imagenet+background': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionv4-8e4777a0.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 299, 299],\n 'input_range': [0, 1],\n 'mean': [0.5, 0.5, 0.5],\n 'std': [0.5, 0.5, 0.5],\n 'num_classes': 1001\n }\n }\n}\n\n\nclass BasicConv2d(nn.Module):\n\n def __init__(self, in_planes, out_planes, kernel_size, stride, padding=0):\n super(BasicConv2d, self).__init__()\n self.conv = nn.Conv2d(\n in_planes,\n out_planes,\n kernel_size=kernel_size,\n stride=stride,\n padding=padding,\n bias=False\n ) # verify bias false\n self.bn = nn.BatchNorm2d(\n out_planes,\n eps=0.001, # value found in tensorflow\n momentum=0.1, # default pytorch value\n affine=True\n )\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n x = self.relu(x)\n return x\n\n\nclass Mixed_3a(nn.Module):\n\n def __init__(self):\n super(Mixed_3a, self).__init__()\n self.maxpool = nn.MaxPool2d(3, stride=2)\n self.conv = BasicConv2d(64, 96, kernel_size=3, stride=2)\n\n def forward(self, x):\n x0 = self.maxpool(x)\n x1 = self.conv(x)\n out = torch.cat((x0, x1), 1)\n return out\n\n\nclass Mixed_4a(nn.Module):\n\n def __init__(self):\n super(Mixed_4a, self).__init__()\n\n self.branch0 = nn.Sequential(\n BasicConv2d(160, 64, kernel_size=1, stride=1),\n BasicConv2d(64, 96, kernel_size=3, stride=1)\n )\n\n self.branch1 = nn.Sequential(\n BasicConv2d(160, 64, kernel_size=1, stride=1),\n BasicConv2d(64, 64, kernel_size=(1, 7), stride=1, padding=(0, 3)),\n BasicConv2d(64, 64, kernel_size=(7, 1), stride=1, padding=(3, 0)),\n BasicConv2d(64, 96, kernel_size=(3, 3), stride=1)\n )\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n out = torch.cat((x0, x1), 1)\n return out\n\n\nclass Mixed_5a(nn.Module):\n\n def __init__(self):\n super(Mixed_5a, self).__init__()\n self.conv = BasicConv2d(192, 192, kernel_size=3, stride=2)\n self.maxpool = nn.MaxPool2d(3, stride=2)\n\n def forward(self, x):\n x0 = self.conv(x)\n x1 = self.maxpool(x)\n out = torch.cat((x0, x1), 1)\n return out\n\n\nclass Inception_A(nn.Module):\n\n def __init__(self):\n super(Inception_A, self).__init__()\n self.branch0 = BasicConv2d(384, 96, kernel_size=1, stride=1)\n\n self.branch1 = nn.Sequential(\n BasicConv2d(384, 64, kernel_size=1, stride=1),\n BasicConv2d(64, 96, kernel_size=3, stride=1, padding=1)\n )\n\n self.branch2 = nn.Sequential(\n BasicConv2d(384, 64, kernel_size=1, stride=1),\n BasicConv2d(64, 96, kernel_size=3, stride=1, padding=1),\n BasicConv2d(96, 96, kernel_size=3, stride=1, padding=1)\n )\n\n self.branch3 = nn.Sequential(\n nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),\n BasicConv2d(384, 96, kernel_size=1, stride=1)\n )\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n x2 = self.branch2(x)\n x3 = self.branch3(x)\n out = torch.cat((x0, x1, x2, x3), 1)\n return out\n\n\nclass Reduction_A(nn.Module):\n\n def __init__(self):\n super(Reduction_A, self).__init__()\n self.branch0 = BasicConv2d(384, 384, kernel_size=3, stride=2)\n\n self.branch1 = nn.Sequential(\n BasicConv2d(384, 192, kernel_size=1, stride=1),\n BasicConv2d(192, 224, kernel_size=3, stride=1, padding=1),\n BasicConv2d(224, 256, kernel_size=3, stride=2)\n )\n\n self.branch2 = nn.MaxPool2d(3, stride=2)\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n x2 = self.branch2(x)\n out = torch.cat((x0, x1, x2), 1)\n return out\n\n\nclass Inception_B(nn.Module):\n\n def __init__(self):\n super(Inception_B, self).__init__()\n self.branch0 = BasicConv2d(1024, 384, kernel_size=1, stride=1)\n\n self.branch1 = nn.Sequential(\n BasicConv2d(1024, 192, kernel_size=1, stride=1),\n BasicConv2d(\n 192, 224, kernel_size=(1, 7), stride=1, padding=(0, 3)\n ),\n BasicConv2d(\n 224, 256, kernel_size=(7, 1), stride=1, padding=(3, 0)\n )\n )\n\n self.branch2 = nn.Sequential(\n BasicConv2d(1024, 192, kernel_size=1, stride=1),\n BasicConv2d(\n 192, 192, kernel_size=(7, 1), stride=1, padding=(3, 0)\n ),\n BasicConv2d(\n 192, 224, kernel_size=(1, 7), stride=1, padding=(0, 3)\n ),\n BasicConv2d(\n 224, 224, kernel_size=(7, 1), stride=1, padding=(3, 0)\n ),\n BasicConv2d(\n 224, 256, kernel_size=(1, 7), stride=1, padding=(0, 3)\n )\n )\n\n self.branch3 = nn.Sequential(\n nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),\n BasicConv2d(1024, 128, kernel_size=1, stride=1)\n )\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n x2 = self.branch2(x)\n x3 = self.branch3(x)\n out = torch.cat((x0, x1, x2, x3), 1)\n return out\n\n\nclass Reduction_B(nn.Module):\n\n def __init__(self):\n super(Reduction_B, self).__init__()\n\n self.branch0 = nn.Sequential(\n BasicConv2d(1024, 192, kernel_size=1, stride=1),\n BasicConv2d(192, 192, kernel_size=3, stride=2)\n )\n\n self.branch1 = nn.Sequential(\n BasicConv2d(1024, 256, kernel_size=1, stride=1),\n BasicConv2d(\n 256, 256, kernel_size=(1, 7), stride=1, padding=(0, 3)\n ),\n BasicConv2d(\n 256, 320, kernel_size=(7, 1), stride=1, padding=(3, 0)\n ), BasicConv2d(320, 320, kernel_size=3, stride=2)\n )\n\n self.branch2 = nn.MaxPool2d(3, stride=2)\n\n def forward(self, x):\n x0 = self.branch0(x)\n x1 = self.branch1(x)\n x2 = self.branch2(x)\n out = torch.cat((x0, x1, x2), 1)\n return out\n\n\nclass Inception_C(nn.Module):\n\n def __init__(self):\n super(Inception_C, self).__init__()\n\n self.branch0 = BasicConv2d(1536, 256, kernel_size=1, stride=1)\n\n self.branch1_0 = BasicConv2d(1536, 384, kernel_size=1, stride=1)\n self.branch1_1a = BasicConv2d(\n 384, 256, kernel_size=(1, 3), stride=1, padding=(0, 1)\n )\n self.branch1_1b = BasicConv2d(\n 384, 256, kernel_size=(3, 1), stride=1, padding=(1, 0)\n )\n\n self.branch2_0 = BasicConv2d(1536, 384, kernel_size=1, stride=1)\n self.branch2_1 = BasicConv2d(\n 384, 448, kernel_size=(3, 1), stride=1, padding=(1, 0)\n )\n self.branch2_2 = BasicConv2d(\n 448, 512, kernel_size=(1, 3), stride=1, padding=(0, 1)\n )\n self.branch2_3a = BasicConv2d(\n 512, 256, kernel_size=(1, 3), stride=1, padding=(0, 1)\n )\n self.branch2_3b = BasicConv2d(\n 512, 256, kernel_size=(3, 1), stride=1, padding=(1, 0)\n )\n\n self.branch3 = nn.Sequential(\n nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False),\n BasicConv2d(1536, 256, kernel_size=1, stride=1)\n )\n\n def forward(self, x):\n x0 = self.branch0(x)\n\n x1_0 = self.branch1_0(x)\n x1_1a = self.branch1_1a(x1_0)\n x1_1b = self.branch1_1b(x1_0)\n x1 = torch.cat((x1_1a, x1_1b), 1)\n\n x2_0 = self.branch2_0(x)\n x2_1 = self.branch2_1(x2_0)\n x2_2 = self.branch2_2(x2_1)\n x2_3a = self.branch2_3a(x2_2)\n x2_3b = self.branch2_3b(x2_2)\n x2 = torch.cat((x2_3a, x2_3b), 1)\n\n x3 = self.branch3(x)\n\n out = torch.cat((x0, x1, x2, x3), 1)\n return out\n\n\nclass InceptionV4(nn.Module):\n \"\"\"Inception-v4.\n\n Reference:\n Szegedy et al. Inception-v4, Inception-ResNet and the Impact of Residual\n Connections on Learning. AAAI 2017.\n\n Public keys:\n - ``inceptionv4``: InceptionV4.\n \"\"\"\n\n def __init__(self, num_classes, loss, **kwargs):\n super(InceptionV4, self).__init__()\n self.loss = loss\n\n self.features = nn.Sequential(\n BasicConv2d(3, 32, kernel_size=3, stride=2),\n BasicConv2d(32, 32, kernel_size=3, stride=1),\n BasicConv2d(32, 64, kernel_size=3, stride=1, padding=1),\n Mixed_3a(),\n Mixed_4a(),\n Mixed_5a(),\n Inception_A(),\n Inception_A(),\n Inception_A(),\n Inception_A(),\n Reduction_A(), # Mixed_6a\n Inception_B(),\n Inception_B(),\n Inception_B(),\n Inception_B(),\n Inception_B(),\n Inception_B(),\n Inception_B(),\n Reduction_B(), # Mixed_7a\n Inception_C(),\n Inception_C(),\n Inception_C()\n )\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.classifier = nn.Linear(1536, num_classes)\n\n def forward(self, x):\n f = self.features(x)\n v = self.global_avgpool(f)\n v = v.view(v.size(0), -1)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef inceptionv4(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = InceptionV4(num_classes, loss, **kwargs)\n if pretrained:\n model_url = pretrained_settings['inceptionv4']['imagenet']['url']\n init_pretrained_weights(model, model_url)\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/mlfn.py", "content": "from __future__ import division, absolute_import\nimport torch\nimport torch.utils.model_zoo as model_zoo\nfrom torch import nn\nfrom torch.nn import functional as F\n\n__all__ = ['mlfn']\n\nmodel_urls = {\n # training epoch = 5, top1 = 51.6\n 'imagenet':\n 'https://mega.nz/#!YHxAhaxC!yu9E6zWl0x5zscSouTdbZu8gdFFytDdl-RAdD2DEfpk',\n}\n\n\nclass MLFNBlock(nn.Module):\n\n def __init__(\n self, in_channels, out_channels, stride, fsm_channels, groups=32\n ):\n super(MLFNBlock, self).__init__()\n self.groups = groups\n mid_channels = out_channels // 2\n\n # Factor Modules\n self.fm_conv1 = nn.Conv2d(in_channels, mid_channels, 1, bias=False)\n self.fm_bn1 = nn.BatchNorm2d(mid_channels)\n self.fm_conv2 = nn.Conv2d(\n mid_channels,\n mid_channels,\n 3,\n stride=stride,\n padding=1,\n bias=False,\n groups=self.groups\n )\n self.fm_bn2 = nn.BatchNorm2d(mid_channels)\n self.fm_conv3 = nn.Conv2d(mid_channels, out_channels, 1, bias=False)\n self.fm_bn3 = nn.BatchNorm2d(out_channels)\n\n # Factor Selection Module\n self.fsm = nn.Sequential(\n nn.AdaptiveAvgPool2d(1),\n nn.Conv2d(in_channels, fsm_channels[0], 1),\n nn.BatchNorm2d(fsm_channels[0]),\n nn.ReLU(inplace=True),\n nn.Conv2d(fsm_channels[0], fsm_channels[1], 1),\n nn.BatchNorm2d(fsm_channels[1]),\n nn.ReLU(inplace=True),\n nn.Conv2d(fsm_channels[1], self.groups, 1),\n nn.BatchNorm2d(self.groups),\n nn.Sigmoid(),\n )\n\n self.downsample = None\n if in_channels != out_channels or stride > 1:\n self.downsample = nn.Sequential(\n nn.Conv2d(\n in_channels, out_channels, 1, stride=stride, bias=False\n ),\n nn.BatchNorm2d(out_channels),\n )\n\n def forward(self, x):\n residual = x\n s = self.fsm(x)\n\n # reduce dimension\n x = self.fm_conv1(x)\n x = self.fm_bn1(x)\n x = F.relu(x, inplace=True)\n\n # group convolution\n x = self.fm_conv2(x)\n x = self.fm_bn2(x)\n x = F.relu(x, inplace=True)\n\n # factor selection\n b, c = x.size(0), x.size(1)\n n = c // self.groups\n ss = s.repeat(1, n, 1, 1) # from (b, g, 1, 1) to (b, g*n=c, 1, 1)\n ss = ss.view(b, n, self.groups, 1, 1)\n ss = ss.permute(0, 2, 1, 3, 4).contiguous()\n ss = ss.view(b, c, 1, 1)\n x = ss * x\n\n # recover dimension\n x = self.fm_conv3(x)\n x = self.fm_bn3(x)\n x = F.relu(x, inplace=True)\n\n if self.downsample is not None:\n residual = self.downsample(residual)\n\n return F.relu(residual + x, inplace=True), s\n\n\nclass MLFN(nn.Module):\n \"\"\"Multi-Level Factorisation Net.\n\n Reference:\n Chang et al. Multi-Level Factorisation Net for\n Person Re-Identification. CVPR 2018.\n\n Public keys:\n - ``mlfn``: MLFN (Multi-Level Factorisation Net).\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n loss='softmax',\n groups=32,\n channels=[64, 256, 512, 1024, 2048],\n embed_dim=1024,\n **kwargs\n ):\n super(MLFN, self).__init__()\n self.loss = loss\n self.groups = groups\n\n # first convolutional layer\n self.conv1 = nn.Conv2d(3, channels[0], 7, stride=2, padding=3)\n self.bn1 = nn.BatchNorm2d(channels[0])\n self.maxpool = nn.MaxPool2d(3, stride=2, padding=1)\n\n # main body\n self.feature = nn.ModuleList(\n [\n # layer 1-3\n MLFNBlock(channels[0], channels[1], 1, [128, 64], self.groups),\n MLFNBlock(channels[1], channels[1], 1, [128, 64], self.groups),\n MLFNBlock(channels[1], channels[1], 1, [128, 64], self.groups),\n # layer 4-7\n MLFNBlock(\n channels[1], channels[2], 2, [256, 128], self.groups\n ),\n MLFNBlock(\n channels[2], channels[2], 1, [256, 128], self.groups\n ),\n MLFNBlock(\n channels[2], channels[2], 1, [256, 128], self.groups\n ),\n MLFNBlock(\n channels[2], channels[2], 1, [256, 128], self.groups\n ),\n # layer 8-13\n MLFNBlock(\n channels[2], channels[3], 2, [512, 128], self.groups\n ),\n MLFNBlock(\n channels[3], channels[3], 1, [512, 128], self.groups\n ),\n MLFNBlock(\n channels[3], channels[3], 1, [512, 128], self.groups\n ),\n MLFNBlock(\n channels[3], channels[3], 1, [512, 128], self.groups\n ),\n MLFNBlock(\n channels[3], channels[3], 1, [512, 128], self.groups\n ),\n MLFNBlock(\n channels[3], channels[3], 1, [512, 128], self.groups\n ),\n # layer 14-16\n MLFNBlock(\n channels[3], channels[4], 2, [512, 128], self.groups\n ),\n MLFNBlock(\n channels[4], channels[4], 1, [512, 128], self.groups\n ),\n MLFNBlock(\n channels[4], channels[4], 1, [512, 128], self.groups\n ),\n ]\n )\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n\n # projection functions\n self.fc_x = nn.Sequential(\n nn.Conv2d(channels[4], embed_dim, 1, bias=False),\n nn.BatchNorm2d(embed_dim),\n nn.ReLU(inplace=True),\n )\n self.fc_s = nn.Sequential(\n nn.Conv2d(self.groups * 16, embed_dim, 1, bias=False),\n nn.BatchNorm2d(embed_dim),\n nn.ReLU(inplace=True),\n )\n\n self.classifier = nn.Linear(embed_dim, num_classes)\n\n self.init_params()\n\n def init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = F.relu(x, inplace=True)\n x = self.maxpool(x)\n\n s_hat = []\n for block in self.feature:\n x, s = block(x)\n s_hat.append(s)\n s_hat = torch.cat(s_hat, 1)\n\n x = self.global_avgpool(x)\n x = self.fc_x(x)\n s_hat = self.fc_s(s_hat)\n\n v = (x+s_hat) * 0.5\n v = v.view(v.size(0), -1)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef mlfn(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = MLFN(num_classes, loss, **kwargs)\n if pretrained:\n # init_pretrained_weights(model, model_urls['imagenet'])\n import warnings\n warnings.warn(\n 'The imagenet pretrained weights need to be manually downloaded from {}'\n .format(model_urls['imagenet'])\n )\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/mobilenetv2.py", "content": "from __future__ import division, absolute_import\nimport torch.utils.model_zoo as model_zoo\nfrom torch import nn\nfrom torch.nn import functional as F\n\n__all__ = ['mobilenetv2_x1_0', 'mobilenetv2_x1_4']\n\nmodel_urls = {\n # 1.0: top-1 71.3\n 'mobilenetv2_x1_0':\n 'https://mega.nz/#!NKp2wAIA!1NH1pbNzY_M2hVk_hdsxNM1NUOWvvGPHhaNr-fASF6c',\n # 1.4: top-1 73.9\n 'mobilenetv2_x1_4':\n 'https://mega.nz/#!RGhgEIwS!xN2s2ZdyqI6vQ3EwgmRXLEW3khr9tpXg96G9SUJugGk',\n}\n\n\nclass ConvBlock(nn.Module):\n \"\"\"Basic convolutional block.\n \n convolution (bias discarded) + batch normalization + relu6.\n\n Args:\n in_c (int): number of input channels.\n out_c (int): number of output channels.\n k (int or tuple): kernel size.\n s (int or tuple): stride.\n p (int or tuple): padding.\n g (int): number of blocked connections from input channels\n to output channels (default: 1).\n \"\"\"\n\n def __init__(self, in_c, out_c, k, s=1, p=0, g=1):\n super(ConvBlock, self).__init__()\n self.conv = nn.Conv2d(\n in_c, out_c, k, stride=s, padding=p, bias=False, groups=g\n )\n self.bn = nn.BatchNorm2d(out_c)\n\n def forward(self, x):\n return F.relu6(self.bn(self.conv(x)))\n\n\nclass Bottleneck(nn.Module):\n\n def __init__(self, in_channels, out_channels, expansion_factor, stride=1):\n super(Bottleneck, self).__init__()\n mid_channels = in_channels * expansion_factor\n self.use_residual = stride == 1 and in_channels == out_channels\n self.conv1 = ConvBlock(in_channels, mid_channels, 1)\n self.dwconv2 = ConvBlock(\n mid_channels, mid_channels, 3, stride, 1, g=mid_channels\n )\n self.conv3 = nn.Sequential(\n nn.Conv2d(mid_channels, out_channels, 1, bias=False),\n nn.BatchNorm2d(out_channels),\n )\n\n def forward(self, x):\n m = self.conv1(x)\n m = self.dwconv2(m)\n m = self.conv3(m)\n if self.use_residual:\n return x + m\n else:\n return m\n\n\nclass MobileNetV2(nn.Module):\n \"\"\"MobileNetV2.\n\n Reference:\n Sandler et al. MobileNetV2: Inverted Residuals and\n Linear Bottlenecks. CVPR 2018.\n\n Public keys:\n - ``mobilenetv2_x1_0``: MobileNetV2 x1.0.\n - ``mobilenetv2_x1_4``: MobileNetV2 x1.4.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n width_mult=1,\n loss='softmax',\n fc_dims=None,\n dropout_p=None,\n **kwargs\n ):\n super(MobileNetV2, self).__init__()\n self.loss = loss\n self.in_channels = int(32 * width_mult)\n self.feature_dim = int(1280 * width_mult) if width_mult > 1 else 1280\n\n # construct layers\n self.conv1 = ConvBlock(3, self.in_channels, 3, s=2, p=1)\n self.conv2 = self._make_layer(\n Bottleneck, 1, int(16 * width_mult), 1, 1\n )\n self.conv3 = self._make_layer(\n Bottleneck, 6, int(24 * width_mult), 2, 2\n )\n self.conv4 = self._make_layer(\n Bottleneck, 6, int(32 * width_mult), 3, 2\n )\n self.conv5 = self._make_layer(\n Bottleneck, 6, int(64 * width_mult), 4, 2\n )\n self.conv6 = self._make_layer(\n Bottleneck, 6, int(96 * width_mult), 3, 1\n )\n self.conv7 = self._make_layer(\n Bottleneck, 6, int(160 * width_mult), 3, 2\n )\n self.conv8 = self._make_layer(\n Bottleneck, 6, int(320 * width_mult), 1, 1\n )\n self.conv9 = ConvBlock(self.in_channels, self.feature_dim, 1)\n\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.fc = self._construct_fc_layer(\n fc_dims, self.feature_dim, dropout_p\n )\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n self._init_params()\n\n def _make_layer(self, block, t, c, n, s):\n # t: expansion factor\n # c: output channels\n # n: number of blocks\n # s: stride for first layer\n layers = []\n layers.append(block(self.in_channels, c, t, s))\n self.in_channels = c\n for i in range(1, n):\n layers.append(block(self.in_channels, c, t))\n return nn.Sequential(*layers)\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n \"\"\"Constructs fully connected layer.\n\n Args:\n fc_dims (list or tuple): dimensions of fc layers, if None, no fc layers are constructed\n input_dim (int): input dimension\n dropout_p (float): dropout probability, if None, dropout is unused\n \"\"\"\n if fc_dims is None:\n self.feature_dim = input_dim\n return None\n\n assert isinstance(\n fc_dims, (list, tuple)\n ), 'fc_dims must be either list or tuple, but got {}'.format(\n type(fc_dims)\n )\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.conv2(x)\n x = self.conv3(x)\n x = self.conv4(x)\n x = self.conv5(x)\n x = self.conv6(x)\n x = self.conv7(x)\n x = self.conv8(x)\n x = self.conv9(x)\n return x\n\n def forward(self, x):\n f = self.featuremaps(x)\n v = self.global_avgpool(f)\n v = v.view(v.size(0), -1)\n\n if self.fc is not None:\n v = self.fc(v)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError(\"Unsupported loss: {}\".format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef mobilenetv2_x1_0(num_classes, loss, pretrained=True, **kwargs):\n model = MobileNetV2(\n num_classes,\n loss=loss,\n width_mult=1,\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n # init_pretrained_weights(model, model_urls['mobilenetv2_x1_0'])\n import warnings\n warnings.warn(\n 'The imagenet pretrained weights need to be manually downloaded from {}'\n .format(model_urls['mobilenetv2_x1_0'])\n )\n return model\n\n\ndef mobilenetv2_x1_4(num_classes, loss, pretrained=True, **kwargs):\n model = MobileNetV2(\n num_classes,\n loss=loss,\n width_mult=1.4,\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n # init_pretrained_weights(model, model_urls['mobilenetv2_x1_4'])\n import warnings\n warnings.warn(\n 'The imagenet pretrained weights need to be manually downloaded from {}'\n .format(model_urls['mobilenetv2_x1_4'])\n )\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/mudeep.py", "content": "from __future__ import division, absolute_import\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\n__all__ = ['MuDeep']\n\n\nclass ConvBlock(nn.Module):\n \"\"\"Basic convolutional block.\n \n convolution + batch normalization + relu.\n\n Args:\n in_c (int): number of input channels.\n out_c (int): number of output channels.\n k (int or tuple): kernel size.\n s (int or tuple): stride.\n p (int or tuple): padding.\n \"\"\"\n\n def __init__(self, in_c, out_c, k, s, p):\n super(ConvBlock, self).__init__()\n self.conv = nn.Conv2d(in_c, out_c, k, stride=s, padding=p)\n self.bn = nn.BatchNorm2d(out_c)\n\n def forward(self, x):\n return F.relu(self.bn(self.conv(x)))\n\n\nclass ConvLayers(nn.Module):\n \"\"\"Preprocessing layers.\"\"\"\n\n def __init__(self):\n super(ConvLayers, self).__init__()\n self.conv1 = ConvBlock(3, 48, k=3, s=1, p=1)\n self.conv2 = ConvBlock(48, 96, k=3, s=1, p=1)\n self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.conv2(x)\n x = self.maxpool(x)\n return x\n\n\nclass MultiScaleA(nn.Module):\n \"\"\"Multi-scale stream layer A (Sec.3.1)\"\"\"\n\n def __init__(self):\n super(MultiScaleA, self).__init__()\n self.stream1 = nn.Sequential(\n ConvBlock(96, 96, k=1, s=1, p=0),\n ConvBlock(96, 24, k=3, s=1, p=1),\n )\n self.stream2 = nn.Sequential(\n nn.AvgPool2d(kernel_size=3, stride=1, padding=1),\n ConvBlock(96, 24, k=1, s=1, p=0),\n )\n self.stream3 = ConvBlock(96, 24, k=1, s=1, p=0)\n self.stream4 = nn.Sequential(\n ConvBlock(96, 16, k=1, s=1, p=0),\n ConvBlock(16, 24, k=3, s=1, p=1),\n ConvBlock(24, 24, k=3, s=1, p=1),\n )\n\n def forward(self, x):\n s1 = self.stream1(x)\n s2 = self.stream2(x)\n s3 = self.stream3(x)\n s4 = self.stream4(x)\n y = torch.cat([s1, s2, s3, s4], dim=1)\n return y\n\n\nclass Reduction(nn.Module):\n \"\"\"Reduction layer (Sec.3.1)\"\"\"\n\n def __init__(self):\n super(Reduction, self).__init__()\n self.stream1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n self.stream2 = ConvBlock(96, 96, k=3, s=2, p=1)\n self.stream3 = nn.Sequential(\n ConvBlock(96, 48, k=1, s=1, p=0),\n ConvBlock(48, 56, k=3, s=1, p=1),\n ConvBlock(56, 64, k=3, s=2, p=1),\n )\n\n def forward(self, x):\n s1 = self.stream1(x)\n s2 = self.stream2(x)\n s3 = self.stream3(x)\n y = torch.cat([s1, s2, s3], dim=1)\n return y\n\n\nclass MultiScaleB(nn.Module):\n \"\"\"Multi-scale stream layer B (Sec.3.1)\"\"\"\n\n def __init__(self):\n super(MultiScaleB, self).__init__()\n self.stream1 = nn.Sequential(\n nn.AvgPool2d(kernel_size=3, stride=1, padding=1),\n ConvBlock(256, 256, k=1, s=1, p=0),\n )\n self.stream2 = nn.Sequential(\n ConvBlock(256, 64, k=1, s=1, p=0),\n ConvBlock(64, 128, k=(1, 3), s=1, p=(0, 1)),\n ConvBlock(128, 256, k=(3, 1), s=1, p=(1, 0)),\n )\n self.stream3 = ConvBlock(256, 256, k=1, s=1, p=0)\n self.stream4 = nn.Sequential(\n ConvBlock(256, 64, k=1, s=1, p=0),\n ConvBlock(64, 64, k=(1, 3), s=1, p=(0, 1)),\n ConvBlock(64, 128, k=(3, 1), s=1, p=(1, 0)),\n ConvBlock(128, 128, k=(1, 3), s=1, p=(0, 1)),\n ConvBlock(128, 256, k=(3, 1), s=1, p=(1, 0)),\n )\n\n def forward(self, x):\n s1 = self.stream1(x)\n s2 = self.stream2(x)\n s3 = self.stream3(x)\n s4 = self.stream4(x)\n return s1, s2, s3, s4\n\n\nclass Fusion(nn.Module):\n \"\"\"Saliency-based learning fusion layer (Sec.3.2)\"\"\"\n\n def __init__(self):\n super(Fusion, self).__init__()\n self.a1 = nn.Parameter(torch.rand(1, 256, 1, 1))\n self.a2 = nn.Parameter(torch.rand(1, 256, 1, 1))\n self.a3 = nn.Parameter(torch.rand(1, 256, 1, 1))\n self.a4 = nn.Parameter(torch.rand(1, 256, 1, 1))\n\n # We add an average pooling layer to reduce the spatial dimension\n # of feature maps, which differs from the original paper.\n self.avgpool = nn.AvgPool2d(kernel_size=4, stride=4, padding=0)\n\n def forward(self, x1, x2, x3, x4):\n s1 = self.a1.expand_as(x1) * x1\n s2 = self.a2.expand_as(x2) * x2\n s3 = self.a3.expand_as(x3) * x3\n s4 = self.a4.expand_as(x4) * x4\n y = self.avgpool(s1 + s2 + s3 + s4)\n return y\n\n\nclass MuDeep(nn.Module):\n \"\"\"Multiscale deep neural network.\n\n Reference:\n Qian et al. Multi-scale Deep Learning Architectures\n for Person Re-identification. ICCV 2017.\n\n Public keys:\n - ``mudeep``: Multiscale deep neural network.\n \"\"\"\n\n def __init__(self, num_classes, loss='softmax', **kwargs):\n super(MuDeep, self).__init__()\n self.loss = loss\n\n self.block1 = ConvLayers()\n self.block2 = MultiScaleA()\n self.block3 = Reduction()\n self.block4 = MultiScaleB()\n self.block5 = Fusion()\n\n # Due to this fully connected layer, input image has to be fixed\n # in shape, i.e. (3, 256, 128), such that the last convolutional feature\n # maps are of shape (256, 16, 8). If input shape is changed,\n # the input dimension of this layer has to be changed accordingly.\n self.fc = nn.Sequential(\n nn.Linear(256 * 16 * 8, 4096),\n nn.BatchNorm1d(4096),\n nn.ReLU(),\n )\n self.classifier = nn.Linear(4096, num_classes)\n self.feat_dim = 4096\n\n def featuremaps(self, x):\n x = self.block1(x)\n x = self.block2(x)\n x = self.block3(x)\n x = self.block4(x)\n x = self.block5(*x)\n return x\n\n def forward(self, x):\n x = self.featuremaps(x)\n x = x.view(x.size(0), -1)\n x = self.fc(x)\n y = self.classifier(x)\n\n if not self.training:\n return x\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, x\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/nasnet.py", "content": "from __future__ import division, absolute_import\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\n\n__all__ = ['nasnetamobile']\n\"\"\"\nNASNet Mobile\nThanks to Anastasiia (https://github.com/DagnyT) for the great help, support and motivation!\n\n\n------------------------------------------------------------------------------------\n Architecture | Top-1 Acc | Top-5 Acc | Multiply-Adds | Params (M)\n------------------------------------------------------------------------------------\n| NASNet-A (4 @ 1056) | 74.08% | 91.74% | 564 M | 5.3 |\n------------------------------------------------------------------------------------\n# References:\n - [Learning Transferable Architectures for Scalable Image Recognition]\n (https://arxiv.org/abs/1707.07012)\n\"\"\"\n\"\"\"\nCode imported from https://github.com/Cadene/pretrained-models.pytorch\n\"\"\"\n\npretrained_settings = {\n 'nasnetamobile': {\n 'imagenet': {\n # 'url': 'https://github.com/veronikayurchuk/pretrained-models.pytorch/releases/download/v1.0/nasnetmobile-7e03cead.pth.tar',\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/nasnetamobile-7e03cead.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 224, 224], # resize 256\n 'input_range': [0, 1],\n 'mean': [0.5, 0.5, 0.5],\n 'std': [0.5, 0.5, 0.5],\n 'num_classes': 1000\n },\n # 'imagenet+background': {\n # # 'url': 'http://data.lip6.fr/cadene/pretrainedmodels/nasnetalarge-a1897284.pth',\n # 'input_space': 'RGB',\n # 'input_size': [3, 224, 224], # resize 256\n # 'input_range': [0, 1],\n # 'mean': [0.5, 0.5, 0.5],\n # 'std': [0.5, 0.5, 0.5],\n # 'num_classes': 1001\n # }\n }\n}\n\n\nclass MaxPoolPad(nn.Module):\n\n def __init__(self):\n super(MaxPoolPad, self).__init__()\n self.pad = nn.ZeroPad2d((1, 0, 1, 0))\n self.pool = nn.MaxPool2d(3, stride=2, padding=1)\n\n def forward(self, x):\n x = self.pad(x)\n x = self.pool(x)\n x = x[:, :, 1:, 1:].contiguous()\n return x\n\n\nclass AvgPoolPad(nn.Module):\n\n def __init__(self, stride=2, padding=1):\n super(AvgPoolPad, self).__init__()\n self.pad = nn.ZeroPad2d((1, 0, 1, 0))\n self.pool = nn.AvgPool2d(\n 3, stride=stride, padding=padding, count_include_pad=False\n )\n\n def forward(self, x):\n x = self.pad(x)\n x = self.pool(x)\n x = x[:, :, 1:, 1:].contiguous()\n return x\n\n\nclass SeparableConv2d(nn.Module):\n\n def __init__(\n self,\n in_channels,\n out_channels,\n dw_kernel,\n dw_stride,\n dw_padding,\n bias=False\n ):\n super(SeparableConv2d, self).__init__()\n self.depthwise_conv2d = nn.Conv2d(\n in_channels,\n in_channels,\n dw_kernel,\n stride=dw_stride,\n padding=dw_padding,\n bias=bias,\n groups=in_channels\n )\n self.pointwise_conv2d = nn.Conv2d(\n in_channels, out_channels, 1, stride=1, bias=bias\n )\n\n def forward(self, x):\n x = self.depthwise_conv2d(x)\n x = self.pointwise_conv2d(x)\n return x\n\n\nclass BranchSeparables(nn.Module):\n\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n stride,\n padding,\n name=None,\n bias=False\n ):\n super(BranchSeparables, self).__init__()\n self.relu = nn.ReLU()\n self.separable_1 = SeparableConv2d(\n in_channels, in_channels, kernel_size, stride, padding, bias=bias\n )\n self.bn_sep_1 = nn.BatchNorm2d(\n in_channels, eps=0.001, momentum=0.1, affine=True\n )\n self.relu1 = nn.ReLU()\n self.separable_2 = SeparableConv2d(\n in_channels, out_channels, kernel_size, 1, padding, bias=bias\n )\n self.bn_sep_2 = nn.BatchNorm2d(\n out_channels, eps=0.001, momentum=0.1, affine=True\n )\n self.name = name\n\n def forward(self, x):\n x = self.relu(x)\n if self.name == 'specific':\n x = nn.ZeroPad2d((1, 0, 1, 0))(x)\n x = self.separable_1(x)\n if self.name == 'specific':\n x = x[:, :, 1:, 1:].contiguous()\n\n x = self.bn_sep_1(x)\n x = self.relu1(x)\n x = self.separable_2(x)\n x = self.bn_sep_2(x)\n return x\n\n\nclass BranchSeparablesStem(nn.Module):\n\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n stride,\n padding,\n bias=False\n ):\n super(BranchSeparablesStem, self).__init__()\n self.relu = nn.ReLU()\n self.separable_1 = SeparableConv2d(\n in_channels, out_channels, kernel_size, stride, padding, bias=bias\n )\n self.bn_sep_1 = nn.BatchNorm2d(\n out_channels, eps=0.001, momentum=0.1, affine=True\n )\n self.relu1 = nn.ReLU()\n self.separable_2 = SeparableConv2d(\n out_channels, out_channels, kernel_size, 1, padding, bias=bias\n )\n self.bn_sep_2 = nn.BatchNorm2d(\n out_channels, eps=0.001, momentum=0.1, affine=True\n )\n\n def forward(self, x):\n x = self.relu(x)\n x = self.separable_1(x)\n x = self.bn_sep_1(x)\n x = self.relu1(x)\n x = self.separable_2(x)\n x = self.bn_sep_2(x)\n return x\n\n\nclass BranchSeparablesReduction(BranchSeparables):\n\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n stride,\n padding,\n z_padding=1,\n bias=False\n ):\n BranchSeparables.__init__(\n self, in_channels, out_channels, kernel_size, stride, padding, bias\n )\n self.padding = nn.ZeroPad2d((z_padding, 0, z_padding, 0))\n\n def forward(self, x):\n x = self.relu(x)\n x = self.padding(x)\n x = self.separable_1(x)\n x = x[:, :, 1:, 1:].contiguous()\n x = self.bn_sep_1(x)\n x = self.relu1(x)\n x = self.separable_2(x)\n x = self.bn_sep_2(x)\n return x\n\n\nclass CellStem0(nn.Module):\n\n def __init__(self, stem_filters, num_filters=42):\n super(CellStem0, self).__init__()\n self.num_filters = num_filters\n self.stem_filters = stem_filters\n self.conv_1x1 = nn.Sequential()\n self.conv_1x1.add_module('relu', nn.ReLU())\n self.conv_1x1.add_module(\n 'conv',\n nn.Conv2d(\n self.stem_filters, self.num_filters, 1, stride=1, bias=False\n )\n )\n self.conv_1x1.add_module(\n 'bn',\n nn.BatchNorm2d(\n self.num_filters, eps=0.001, momentum=0.1, affine=True\n )\n )\n\n self.comb_iter_0_left = BranchSeparables(\n self.num_filters, self.num_filters, 5, 2, 2\n )\n self.comb_iter_0_right = BranchSeparablesStem(\n self.stem_filters, self.num_filters, 7, 2, 3, bias=False\n )\n\n self.comb_iter_1_left = nn.MaxPool2d(3, stride=2, padding=1)\n self.comb_iter_1_right = BranchSeparablesStem(\n self.stem_filters, self.num_filters, 7, 2, 3, bias=False\n )\n\n self.comb_iter_2_left = nn.AvgPool2d(\n 3, stride=2, padding=1, count_include_pad=False\n )\n self.comb_iter_2_right = BranchSeparablesStem(\n self.stem_filters, self.num_filters, 5, 2, 2, bias=False\n )\n\n self.comb_iter_3_right = nn.AvgPool2d(\n 3, stride=1, padding=1, count_include_pad=False\n )\n\n self.comb_iter_4_left = BranchSeparables(\n self.num_filters, self.num_filters, 3, 1, 1, bias=False\n )\n self.comb_iter_4_right = nn.MaxPool2d(3, stride=2, padding=1)\n\n def forward(self, x):\n x1 = self.conv_1x1(x)\n\n x_comb_iter_0_left = self.comb_iter_0_left(x1)\n x_comb_iter_0_right = self.comb_iter_0_right(x)\n x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n x_comb_iter_1_left = self.comb_iter_1_left(x1)\n x_comb_iter_1_right = self.comb_iter_1_right(x)\n x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n x_comb_iter_2_left = self.comb_iter_2_left(x1)\n x_comb_iter_2_right = self.comb_iter_2_right(x)\n x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right\n\n x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)\n x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1\n\n x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)\n x_comb_iter_4_right = self.comb_iter_4_right(x1)\n x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right\n\n x_out = torch.cat(\n [x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1\n )\n return x_out\n\n\nclass CellStem1(nn.Module):\n\n def __init__(self, stem_filters, num_filters):\n super(CellStem1, self).__init__()\n self.num_filters = num_filters\n self.stem_filters = stem_filters\n self.conv_1x1 = nn.Sequential()\n self.conv_1x1.add_module('relu', nn.ReLU())\n self.conv_1x1.add_module(\n 'conv',\n nn.Conv2d(\n 2 * self.num_filters,\n self.num_filters,\n 1,\n stride=1,\n bias=False\n )\n )\n self.conv_1x1.add_module(\n 'bn',\n nn.BatchNorm2d(\n self.num_filters, eps=0.001, momentum=0.1, affine=True\n )\n )\n\n self.relu = nn.ReLU()\n self.path_1 = nn.Sequential()\n self.path_1.add_module(\n 'avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False)\n )\n self.path_1.add_module(\n 'conv',\n nn.Conv2d(\n self.stem_filters,\n self.num_filters // 2,\n 1,\n stride=1,\n bias=False\n )\n )\n self.path_2 = nn.ModuleList()\n self.path_2.add_module('pad', nn.ZeroPad2d((0, 1, 0, 1)))\n self.path_2.add_module(\n 'avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False)\n )\n self.path_2.add_module(\n 'conv',\n nn.Conv2d(\n self.stem_filters,\n self.num_filters // 2,\n 1,\n stride=1,\n bias=False\n )\n )\n\n self.final_path_bn = nn.BatchNorm2d(\n self.num_filters, eps=0.001, momentum=0.1, affine=True\n )\n\n self.comb_iter_0_left = BranchSeparables(\n self.num_filters,\n self.num_filters,\n 5,\n 2,\n 2,\n name='specific',\n bias=False\n )\n self.comb_iter_0_right = BranchSeparables(\n self.num_filters,\n self.num_filters,\n 7,\n 2,\n 3,\n name='specific',\n bias=False\n )\n\n # self.comb_iter_1_left = nn.MaxPool2d(3, stride=2, padding=1)\n self.comb_iter_1_left = MaxPoolPad()\n self.comb_iter_1_right = BranchSeparables(\n self.num_filters,\n self.num_filters,\n 7,\n 2,\n 3,\n name='specific',\n bias=False\n )\n\n # self.comb_iter_2_left = nn.AvgPool2d(3, stride=2, padding=1, count_include_pad=False)\n self.comb_iter_2_left = AvgPoolPad()\n self.comb_iter_2_right = BranchSeparables(\n self.num_filters,\n self.num_filters,\n 5,\n 2,\n 2,\n name='specific',\n bias=False\n )\n\n self.comb_iter_3_right = nn.AvgPool2d(\n 3, stride=1, padding=1, count_include_pad=False\n )\n\n self.comb_iter_4_left = BranchSeparables(\n self.num_filters,\n self.num_filters,\n 3,\n 1,\n 1,\n name='specific',\n bias=False\n )\n # self.comb_iter_4_right = nn.MaxPool2d(3, stride=2, padding=1)\n self.comb_iter_4_right = MaxPoolPad()\n\n def forward(self, x_conv0, x_stem_0):\n x_left = self.conv_1x1(x_stem_0)\n\n x_relu = self.relu(x_conv0)\n # path 1\n x_path1 = self.path_1(x_relu)\n # path 2\n x_path2 = self.path_2.pad(x_relu)\n x_path2 = x_path2[:, :, 1:, 1:]\n x_path2 = self.path_2.avgpool(x_path2)\n x_path2 = self.path_2.conv(x_path2)\n # final path\n x_right = self.final_path_bn(torch.cat([x_path1, x_path2], 1))\n\n x_comb_iter_0_left = self.comb_iter_0_left(x_left)\n x_comb_iter_0_right = self.comb_iter_0_right(x_right)\n x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n x_comb_iter_1_left = self.comb_iter_1_left(x_left)\n x_comb_iter_1_right = self.comb_iter_1_right(x_right)\n x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n x_comb_iter_2_left = self.comb_iter_2_left(x_left)\n x_comb_iter_2_right = self.comb_iter_2_right(x_right)\n x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right\n\n x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)\n x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1\n\n x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)\n x_comb_iter_4_right = self.comb_iter_4_right(x_left)\n x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right\n\n x_out = torch.cat(\n [x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1\n )\n return x_out\n\n\nclass FirstCell(nn.Module):\n\n def __init__(\n self, in_channels_left, out_channels_left, in_channels_right,\n out_channels_right\n ):\n super(FirstCell, self).__init__()\n self.conv_1x1 = nn.Sequential()\n self.conv_1x1.add_module('relu', nn.ReLU())\n self.conv_1x1.add_module(\n 'conv',\n nn.Conv2d(\n in_channels_right, out_channels_right, 1, stride=1, bias=False\n )\n )\n self.conv_1x1.add_module(\n 'bn',\n nn.BatchNorm2d(\n out_channels_right, eps=0.001, momentum=0.1, affine=True\n )\n )\n\n self.relu = nn.ReLU()\n self.path_1 = nn.Sequential()\n self.path_1.add_module(\n 'avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False)\n )\n self.path_1.add_module(\n 'conv',\n nn.Conv2d(\n in_channels_left, out_channels_left, 1, stride=1, bias=False\n )\n )\n self.path_2 = nn.ModuleList()\n self.path_2.add_module('pad', nn.ZeroPad2d((0, 1, 0, 1)))\n self.path_2.add_module(\n 'avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False)\n )\n self.path_2.add_module(\n 'conv',\n nn.Conv2d(\n in_channels_left, out_channels_left, 1, stride=1, bias=False\n )\n )\n\n self.final_path_bn = nn.BatchNorm2d(\n out_channels_left * 2, eps=0.001, momentum=0.1, affine=True\n )\n\n self.comb_iter_0_left = BranchSeparables(\n out_channels_right, out_channels_right, 5, 1, 2, bias=False\n )\n self.comb_iter_0_right = BranchSeparables(\n out_channels_right, out_channels_right, 3, 1, 1, bias=False\n )\n\n self.comb_iter_1_left = BranchSeparables(\n out_channels_right, out_channels_right, 5, 1, 2, bias=False\n )\n self.comb_iter_1_right = BranchSeparables(\n out_channels_right, out_channels_right, 3, 1, 1, bias=False\n )\n\n self.comb_iter_2_left = nn.AvgPool2d(\n 3, stride=1, padding=1, count_include_pad=False\n )\n\n self.comb_iter_3_left = nn.AvgPool2d(\n 3, stride=1, padding=1, count_include_pad=False\n )\n self.comb_iter_3_right = nn.AvgPool2d(\n 3, stride=1, padding=1, count_include_pad=False\n )\n\n self.comb_iter_4_left = BranchSeparables(\n out_channels_right, out_channels_right, 3, 1, 1, bias=False\n )\n\n def forward(self, x, x_prev):\n x_relu = self.relu(x_prev)\n # path 1\n x_path1 = self.path_1(x_relu)\n # path 2\n x_path2 = self.path_2.pad(x_relu)\n x_path2 = x_path2[:, :, 1:, 1:]\n x_path2 = self.path_2.avgpool(x_path2)\n x_path2 = self.path_2.conv(x_path2)\n # final path\n x_left = self.final_path_bn(torch.cat([x_path1, x_path2], 1))\n\n x_right = self.conv_1x1(x)\n\n x_comb_iter_0_left = self.comb_iter_0_left(x_right)\n x_comb_iter_0_right = self.comb_iter_0_right(x_left)\n x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n x_comb_iter_1_left = self.comb_iter_1_left(x_left)\n x_comb_iter_1_right = self.comb_iter_1_right(x_left)\n x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n x_comb_iter_2_left = self.comb_iter_2_left(x_right)\n x_comb_iter_2 = x_comb_iter_2_left + x_left\n\n x_comb_iter_3_left = self.comb_iter_3_left(x_left)\n x_comb_iter_3_right = self.comb_iter_3_right(x_left)\n x_comb_iter_3 = x_comb_iter_3_left + x_comb_iter_3_right\n\n x_comb_iter_4_left = self.comb_iter_4_left(x_right)\n x_comb_iter_4 = x_comb_iter_4_left + x_right\n\n x_out = torch.cat(\n [\n x_left, x_comb_iter_0, x_comb_iter_1, x_comb_iter_2,\n x_comb_iter_3, x_comb_iter_4\n ], 1\n )\n return x_out\n\n\nclass NormalCell(nn.Module):\n\n def __init__(\n self, in_channels_left, out_channels_left, in_channels_right,\n out_channels_right\n ):\n super(NormalCell, self).__init__()\n self.conv_prev_1x1 = nn.Sequential()\n self.conv_prev_1x1.add_module('relu', nn.ReLU())\n self.conv_prev_1x1.add_module(\n 'conv',\n nn.Conv2d(\n in_channels_left, out_channels_left, 1, stride=1, bias=False\n )\n )\n self.conv_prev_1x1.add_module(\n 'bn',\n nn.BatchNorm2d(\n out_channels_left, eps=0.001, momentum=0.1, affine=True\n )\n )\n\n self.conv_1x1 = nn.Sequential()\n self.conv_1x1.add_module('relu', nn.ReLU())\n self.conv_1x1.add_module(\n 'conv',\n nn.Conv2d(\n in_channels_right, out_channels_right, 1, stride=1, bias=False\n )\n )\n self.conv_1x1.add_module(\n 'bn',\n nn.BatchNorm2d(\n out_channels_right, eps=0.001, momentum=0.1, affine=True\n )\n )\n\n self.comb_iter_0_left = BranchSeparables(\n out_channels_right, out_channels_right, 5, 1, 2, bias=False\n )\n self.comb_iter_0_right = BranchSeparables(\n out_channels_left, out_channels_left, 3, 1, 1, bias=False\n )\n\n self.comb_iter_1_left = BranchSeparables(\n out_channels_left, out_channels_left, 5, 1, 2, bias=False\n )\n self.comb_iter_1_right = BranchSeparables(\n out_channels_left, out_channels_left, 3, 1, 1, bias=False\n )\n\n self.comb_iter_2_left = nn.AvgPool2d(\n 3, stride=1, padding=1, count_include_pad=False\n )\n\n self.comb_iter_3_left = nn.AvgPool2d(\n 3, stride=1, padding=1, count_include_pad=False\n )\n self.comb_iter_3_right = nn.AvgPool2d(\n 3, stride=1, padding=1, count_include_pad=False\n )\n\n self.comb_iter_4_left = BranchSeparables(\n out_channels_right, out_channels_right, 3, 1, 1, bias=False\n )\n\n def forward(self, x, x_prev):\n x_left = self.conv_prev_1x1(x_prev)\n x_right = self.conv_1x1(x)\n\n x_comb_iter_0_left = self.comb_iter_0_left(x_right)\n x_comb_iter_0_right = self.comb_iter_0_right(x_left)\n x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n x_comb_iter_1_left = self.comb_iter_1_left(x_left)\n x_comb_iter_1_right = self.comb_iter_1_right(x_left)\n x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n x_comb_iter_2_left = self.comb_iter_2_left(x_right)\n x_comb_iter_2 = x_comb_iter_2_left + x_left\n\n x_comb_iter_3_left = self.comb_iter_3_left(x_left)\n x_comb_iter_3_right = self.comb_iter_3_right(x_left)\n x_comb_iter_3 = x_comb_iter_3_left + x_comb_iter_3_right\n\n x_comb_iter_4_left = self.comb_iter_4_left(x_right)\n x_comb_iter_4 = x_comb_iter_4_left + x_right\n\n x_out = torch.cat(\n [\n x_left, x_comb_iter_0, x_comb_iter_1, x_comb_iter_2,\n x_comb_iter_3, x_comb_iter_4\n ], 1\n )\n return x_out\n\n\nclass ReductionCell0(nn.Module):\n\n def __init__(\n self, in_channels_left, out_channels_left, in_channels_right,\n out_channels_right\n ):\n super(ReductionCell0, self).__init__()\n self.conv_prev_1x1 = nn.Sequential()\n self.conv_prev_1x1.add_module('relu', nn.ReLU())\n self.conv_prev_1x1.add_module(\n 'conv',\n nn.Conv2d(\n in_channels_left, out_channels_left, 1, stride=1, bias=False\n )\n )\n self.conv_prev_1x1.add_module(\n 'bn',\n nn.BatchNorm2d(\n out_channels_left, eps=0.001, momentum=0.1, affine=True\n )\n )\n\n self.conv_1x1 = nn.Sequential()\n self.conv_1x1.add_module('relu', nn.ReLU())\n self.conv_1x1.add_module(\n 'conv',\n nn.Conv2d(\n in_channels_right, out_channels_right, 1, stride=1, bias=False\n )\n )\n self.conv_1x1.add_module(\n 'bn',\n nn.BatchNorm2d(\n out_channels_right, eps=0.001, momentum=0.1, affine=True\n )\n )\n\n self.comb_iter_0_left = BranchSeparablesReduction(\n out_channels_right, out_channels_right, 5, 2, 2, bias=False\n )\n self.comb_iter_0_right = BranchSeparablesReduction(\n out_channels_right, out_channels_right, 7, 2, 3, bias=False\n )\n\n self.comb_iter_1_left = MaxPoolPad()\n self.comb_iter_1_right = BranchSeparablesReduction(\n out_channels_right, out_channels_right, 7, 2, 3, bias=False\n )\n\n self.comb_iter_2_left = AvgPoolPad()\n self.comb_iter_2_right = BranchSeparablesReduction(\n out_channels_right, out_channels_right, 5, 2, 2, bias=False\n )\n\n self.comb_iter_3_right = nn.AvgPool2d(\n 3, stride=1, padding=1, count_include_pad=False\n )\n\n self.comb_iter_4_left = BranchSeparablesReduction(\n out_channels_right, out_channels_right, 3, 1, 1, bias=False\n )\n self.comb_iter_4_right = MaxPoolPad()\n\n def forward(self, x, x_prev):\n x_left = self.conv_prev_1x1(x_prev)\n x_right = self.conv_1x1(x)\n\n x_comb_iter_0_left = self.comb_iter_0_left(x_right)\n x_comb_iter_0_right = self.comb_iter_0_right(x_left)\n x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n x_comb_iter_1_left = self.comb_iter_1_left(x_right)\n x_comb_iter_1_right = self.comb_iter_1_right(x_left)\n x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n x_comb_iter_2_left = self.comb_iter_2_left(x_right)\n x_comb_iter_2_right = self.comb_iter_2_right(x_left)\n x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right\n\n x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)\n x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1\n\n x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)\n x_comb_iter_4_right = self.comb_iter_4_right(x_right)\n x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right\n\n x_out = torch.cat(\n [x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1\n )\n return x_out\n\n\nclass ReductionCell1(nn.Module):\n\n def __init__(\n self, in_channels_left, out_channels_left, in_channels_right,\n out_channels_right\n ):\n super(ReductionCell1, self).__init__()\n self.conv_prev_1x1 = nn.Sequential()\n self.conv_prev_1x1.add_module('relu', nn.ReLU())\n self.conv_prev_1x1.add_module(\n 'conv',\n nn.Conv2d(\n in_channels_left, out_channels_left, 1, stride=1, bias=False\n )\n )\n self.conv_prev_1x1.add_module(\n 'bn',\n nn.BatchNorm2d(\n out_channels_left, eps=0.001, momentum=0.1, affine=True\n )\n )\n\n self.conv_1x1 = nn.Sequential()\n self.conv_1x1.add_module('relu', nn.ReLU())\n self.conv_1x1.add_module(\n 'conv',\n nn.Conv2d(\n in_channels_right, out_channels_right, 1, stride=1, bias=False\n )\n )\n self.conv_1x1.add_module(\n 'bn',\n nn.BatchNorm2d(\n out_channels_right, eps=0.001, momentum=0.1, affine=True\n )\n )\n\n self.comb_iter_0_left = BranchSeparables(\n out_channels_right,\n out_channels_right,\n 5,\n 2,\n 2,\n name='specific',\n bias=False\n )\n self.comb_iter_0_right = BranchSeparables(\n out_channels_right,\n out_channels_right,\n 7,\n 2,\n 3,\n name='specific',\n bias=False\n )\n\n # self.comb_iter_1_left = nn.MaxPool2d(3, stride=2, padding=1)\n self.comb_iter_1_left = MaxPoolPad()\n self.comb_iter_1_right = BranchSeparables(\n out_channels_right,\n out_channels_right,\n 7,\n 2,\n 3,\n name='specific',\n bias=False\n )\n\n # self.comb_iter_2_left = nn.AvgPool2d(3, stride=2, padding=1, count_include_pad=False)\n self.comb_iter_2_left = AvgPoolPad()\n self.comb_iter_2_right = BranchSeparables(\n out_channels_right,\n out_channels_right,\n 5,\n 2,\n 2,\n name='specific',\n bias=False\n )\n\n self.comb_iter_3_right = nn.AvgPool2d(\n 3, stride=1, padding=1, count_include_pad=False\n )\n\n self.comb_iter_4_left = BranchSeparables(\n out_channels_right,\n out_channels_right,\n 3,\n 1,\n 1,\n name='specific',\n bias=False\n )\n # self.comb_iter_4_right = nn.MaxPool2d(3, stride=2, padding=1)\n self.comb_iter_4_right = MaxPoolPad()\n\n def forward(self, x, x_prev):\n x_left = self.conv_prev_1x1(x_prev)\n x_right = self.conv_1x1(x)\n\n x_comb_iter_0_left = self.comb_iter_0_left(x_right)\n x_comb_iter_0_right = self.comb_iter_0_right(x_left)\n x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n x_comb_iter_1_left = self.comb_iter_1_left(x_right)\n x_comb_iter_1_right = self.comb_iter_1_right(x_left)\n x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n x_comb_iter_2_left = self.comb_iter_2_left(x_right)\n x_comb_iter_2_right = self.comb_iter_2_right(x_left)\n x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right\n\n x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)\n x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1\n\n x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)\n x_comb_iter_4_right = self.comb_iter_4_right(x_right)\n x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right\n\n x_out = torch.cat(\n [x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1\n )\n return x_out\n\n\nclass NASNetAMobile(nn.Module):\n \"\"\"Neural Architecture Search (NAS).\n\n Reference:\n Zoph et al. Learning Transferable Architectures\n for Scalable Image Recognition. CVPR 2018.\n\n Public keys:\n - ``nasnetamobile``: NASNet-A Mobile.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n loss,\n stem_filters=32,\n penultimate_filters=1056,\n filters_multiplier=2,\n **kwargs\n ):\n super(NASNetAMobile, self).__init__()\n self.stem_filters = stem_filters\n self.penultimate_filters = penultimate_filters\n self.filters_multiplier = filters_multiplier\n self.loss = loss\n\n filters = self.penultimate_filters // 24\n # 24 is default value for the architecture\n\n self.conv0 = nn.Sequential()\n self.conv0.add_module(\n 'conv',\n nn.Conv2d(\n in_channels=3,\n out_channels=self.stem_filters,\n kernel_size=3,\n padding=0,\n stride=2,\n bias=False\n )\n )\n self.conv0.add_module(\n 'bn',\n nn.BatchNorm2d(\n self.stem_filters, eps=0.001, momentum=0.1, affine=True\n )\n )\n\n self.cell_stem_0 = CellStem0(\n self.stem_filters, num_filters=filters // (filters_multiplier**2)\n )\n self.cell_stem_1 = CellStem1(\n self.stem_filters, num_filters=filters // filters_multiplier\n )\n\n self.cell_0 = FirstCell(\n in_channels_left=filters,\n out_channels_left=filters // 2, # 1, 0.5\n in_channels_right=2 * filters,\n out_channels_right=filters\n ) # 2, 1\n self.cell_1 = NormalCell(\n in_channels_left=2 * filters,\n out_channels_left=filters, # 2, 1\n in_channels_right=6 * filters,\n out_channels_right=filters\n ) # 6, 1\n self.cell_2 = NormalCell(\n in_channels_left=6 * filters,\n out_channels_left=filters, # 6, 1\n in_channels_right=6 * filters,\n out_channels_right=filters\n ) # 6, 1\n self.cell_3 = NormalCell(\n in_channels_left=6 * filters,\n out_channels_left=filters, # 6, 1\n in_channels_right=6 * filters,\n out_channels_right=filters\n ) # 6, 1\n\n self.reduction_cell_0 = ReductionCell0(\n in_channels_left=6 * filters,\n out_channels_left=2 * filters, # 6, 2\n in_channels_right=6 * filters,\n out_channels_right=2 * filters\n ) # 6, 2\n\n self.cell_6 = FirstCell(\n in_channels_left=6 * filters,\n out_channels_left=filters, # 6, 1\n in_channels_right=8 * filters,\n out_channels_right=2 * filters\n ) # 8, 2\n self.cell_7 = NormalCell(\n in_channels_left=8 * filters,\n out_channels_left=2 * filters, # 8, 2\n in_channels_right=12 * filters,\n out_channels_right=2 * filters\n ) # 12, 2\n self.cell_8 = NormalCell(\n in_channels_left=12 * filters,\n out_channels_left=2 * filters, # 12, 2\n in_channels_right=12 * filters,\n out_channels_right=2 * filters\n ) # 12, 2\n self.cell_9 = NormalCell(\n in_channels_left=12 * filters,\n out_channels_left=2 * filters, # 12, 2\n in_channels_right=12 * filters,\n out_channels_right=2 * filters\n ) # 12, 2\n\n self.reduction_cell_1 = ReductionCell1(\n in_channels_left=12 * filters,\n out_channels_left=4 * filters, # 12, 4\n in_channels_right=12 * filters,\n out_channels_right=4 * filters\n ) # 12, 4\n\n self.cell_12 = FirstCell(\n in_channels_left=12 * filters,\n out_channels_left=2 * filters, # 12, 2\n in_channels_right=16 * filters,\n out_channels_right=4 * filters\n ) # 16, 4\n self.cell_13 = NormalCell(\n in_channels_left=16 * filters,\n out_channels_left=4 * filters, # 16, 4\n in_channels_right=24 * filters,\n out_channels_right=4 * filters\n ) # 24, 4\n self.cell_14 = NormalCell(\n in_channels_left=24 * filters,\n out_channels_left=4 * filters, # 24, 4\n in_channels_right=24 * filters,\n out_channels_right=4 * filters\n ) # 24, 4\n self.cell_15 = NormalCell(\n in_channels_left=24 * filters,\n out_channels_left=4 * filters, # 24, 4\n in_channels_right=24 * filters,\n out_channels_right=4 * filters\n ) # 24, 4\n\n self.relu = nn.ReLU()\n self.dropout = nn.Dropout()\n self.classifier = nn.Linear(24 * filters, num_classes)\n\n self._init_params()\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def features(self, input):\n x_conv0 = self.conv0(input)\n x_stem_0 = self.cell_stem_0(x_conv0)\n x_stem_1 = self.cell_stem_1(x_conv0, x_stem_0)\n\n x_cell_0 = self.cell_0(x_stem_1, x_stem_0)\n x_cell_1 = self.cell_1(x_cell_0, x_stem_1)\n x_cell_2 = self.cell_2(x_cell_1, x_cell_0)\n x_cell_3 = self.cell_3(x_cell_2, x_cell_1)\n\n x_reduction_cell_0 = self.reduction_cell_0(x_cell_3, x_cell_2)\n\n x_cell_6 = self.cell_6(x_reduction_cell_0, x_cell_3)\n x_cell_7 = self.cell_7(x_cell_6, x_reduction_cell_0)\n x_cell_8 = self.cell_8(x_cell_7, x_cell_6)\n x_cell_9 = self.cell_9(x_cell_8, x_cell_7)\n\n x_reduction_cell_1 = self.reduction_cell_1(x_cell_9, x_cell_8)\n\n x_cell_12 = self.cell_12(x_reduction_cell_1, x_cell_9)\n x_cell_13 = self.cell_13(x_cell_12, x_reduction_cell_1)\n x_cell_14 = self.cell_14(x_cell_13, x_cell_12)\n x_cell_15 = self.cell_15(x_cell_14, x_cell_13)\n\n x_cell_15 = self.relu(x_cell_15)\n x_cell_15 = F.avg_pool2d(\n x_cell_15,\n x_cell_15.size()[2:]\n ) # global average pool\n x_cell_15 = x_cell_15.view(x_cell_15.size(0), -1)\n x_cell_15 = self.dropout(x_cell_15)\n\n return x_cell_15\n\n def forward(self, input):\n v = self.features(input)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef nasnetamobile(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = NASNetAMobile(num_classes, loss, **kwargs)\n if pretrained:\n model_url = pretrained_settings['nasnetamobile']['imagenet']['url']\n init_pretrained_weights(model, model_url)\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/osnet.py", "content": "from __future__ import division, absolute_import\nimport warnings\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\n__all__ = [\n 'osnet_x1_0', 'osnet_x0_75', 'osnet_x0_5', 'osnet_x0_25', 'osnet_ibn_x1_0'\n]\n\npretrained_urls = {\n 'osnet_x1_0':\n 'https://drive.google.com/uc?id=1LaG1EJpHrxdAxKnSCJ_i0u-nbxSAeiFY',\n 'osnet_x0_75':\n 'https://drive.google.com/uc?id=1uwA9fElHOk3ZogwbeY5GkLI6QPTX70Hq',\n 'osnet_x0_5':\n 'https://drive.google.com/uc?id=16DGLbZukvVYgINws8u8deSaOqjybZ83i',\n 'osnet_x0_25':\n 'https://drive.google.com/uc?id=1rb8UN5ZzPKRc_xvtHlyDh-cSz88YX9hs',\n 'osnet_ibn_x1_0':\n 'https://drive.google.com/uc?id=1sr90V6irlYYDd4_4ISU2iruoRG8J__6l'\n}\n\n\n##########\n# Basic layers\n##########\nclass ConvLayer(nn.Module):\n \"\"\"Convolution layer (conv + bn + relu).\"\"\"\n\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n stride=1,\n padding=0,\n groups=1,\n IN=False\n ):\n super(ConvLayer, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n kernel_size,\n stride=stride,\n padding=padding,\n bias=False,\n groups=groups\n )\n if IN:\n self.bn = nn.InstanceNorm2d(out_channels, affine=True)\n else:\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n x = self.relu(x)\n return x\n\n\nclass Conv1x1(nn.Module):\n \"\"\"1x1 convolution + bn + relu.\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, groups=1):\n super(Conv1x1, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n 1,\n stride=stride,\n padding=0,\n bias=False,\n groups=groups\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n x = self.relu(x)\n return x\n\n\nclass Conv1x1Linear(nn.Module):\n \"\"\"1x1 convolution + bn (w/o non-linearity).\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1):\n super(Conv1x1Linear, self).__init__()\n self.conv = nn.Conv2d(\n in_channels, out_channels, 1, stride=stride, padding=0, bias=False\n )\n self.bn = nn.BatchNorm2d(out_channels)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return x\n\n\nclass Conv3x3(nn.Module):\n \"\"\"3x3 convolution + bn + relu.\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, groups=1):\n super(Conv3x3, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n 3,\n stride=stride,\n padding=1,\n bias=False,\n groups=groups\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n x = self.relu(x)\n return x\n\n\nclass LightConv3x3(nn.Module):\n \"\"\"Lightweight 3x3 convolution.\n\n 1x1 (linear) + dw 3x3 (nonlinear).\n \"\"\"\n\n def __init__(self, in_channels, out_channels):\n super(LightConv3x3, self).__init__()\n self.conv1 = nn.Conv2d(\n in_channels, out_channels, 1, stride=1, padding=0, bias=False\n )\n self.conv2 = nn.Conv2d(\n out_channels,\n out_channels,\n 3,\n stride=1,\n padding=1,\n bias=False,\n groups=out_channels\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.conv2(x)\n x = self.bn(x)\n x = self.relu(x)\n return x\n\n\n##########\n# Building blocks for omni-scale feature learning\n##########\nclass ChannelGate(nn.Module):\n \"\"\"A mini-network that generates channel-wise gates conditioned on input tensor.\"\"\"\n\n def __init__(\n self,\n in_channels,\n num_gates=None,\n return_gates=False,\n gate_activation='sigmoid',\n reduction=16,\n layer_norm=False\n ):\n super(ChannelGate, self).__init__()\n if num_gates is None:\n num_gates = in_channels\n self.return_gates = return_gates\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.fc1 = nn.Conv2d(\n in_channels,\n in_channels // reduction,\n kernel_size=1,\n bias=True,\n padding=0\n )\n self.norm1 = None\n if layer_norm:\n self.norm1 = nn.LayerNorm((in_channels // reduction, 1, 1))\n self.relu = nn.ReLU(inplace=True)\n self.fc2 = nn.Conv2d(\n in_channels // reduction,\n num_gates,\n kernel_size=1,\n bias=True,\n padding=0\n )\n if gate_activation == 'sigmoid':\n self.gate_activation = nn.Sigmoid()\n elif gate_activation == 'relu':\n self.gate_activation = nn.ReLU(inplace=True)\n elif gate_activation == 'linear':\n self.gate_activation = None\n else:\n raise RuntimeError(\n \"Unknown gate activation: {}\".format(gate_activation)\n )\n\n def forward(self, x):\n input = x\n x = self.global_avgpool(x)\n x = self.fc1(x)\n if self.norm1 is not None:\n x = self.norm1(x)\n x = self.relu(x)\n x = self.fc2(x)\n if self.gate_activation is not None:\n x = self.gate_activation(x)\n if self.return_gates:\n return x\n return input * x\n\n\nclass OSBlock(nn.Module):\n \"\"\"Omni-scale feature learning block.\"\"\"\n\n def __init__(\n self,\n in_channels,\n out_channels,\n IN=False,\n bottleneck_reduction=4,\n **kwargs\n ):\n super(OSBlock, self).__init__()\n mid_channels = out_channels // bottleneck_reduction\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2a = LightConv3x3(mid_channels, mid_channels)\n self.conv2b = nn.Sequential(\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n )\n self.conv2c = nn.Sequential(\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n )\n self.conv2d = nn.Sequential(\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n LightConv3x3(mid_channels, mid_channels),\n )\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n self.IN = None\n if IN:\n self.IN = nn.InstanceNorm2d(out_channels, affine=True)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2a = self.conv2a(x1)\n x2b = self.conv2b(x1)\n x2c = self.conv2c(x1)\n x2d = self.conv2d(x1)\n x2 = self.gate(x2a) + self.gate(x2b) + self.gate(x2c) + self.gate(x2d)\n x3 = self.conv3(x2)\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n if self.IN is not None:\n out = self.IN(out)\n return F.relu(out)\n\n\n##########\n# Network architecture\n##########\nclass OSNet(nn.Module):\n \"\"\"Omni-Scale Network.\n \n Reference:\n - Zhou et al. Omni-Scale Feature Learning for Person Re-Identification. ICCV, 2019.\n - Zhou et al. Learning Generalisable Omni-Scale Representations\n for Person Re-Identification. TPAMI, 2021.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n blocks,\n layers,\n channels,\n feature_dim=512,\n loss='softmax',\n IN=False,\n **kwargs\n ):\n super(OSNet, self).__init__()\n num_blocks = len(blocks)\n assert num_blocks == len(layers)\n assert num_blocks == len(channels) - 1\n self.loss = loss\n self.feature_dim = feature_dim\n\n # convolutional backbone\n self.conv1 = ConvLayer(3, channels[0], 7, stride=2, padding=3, IN=IN)\n self.maxpool = nn.MaxPool2d(3, stride=2, padding=1)\n self.conv2 = self._make_layer(\n blocks[0],\n layers[0],\n channels[0],\n channels[1],\n reduce_spatial_size=True,\n IN=IN\n )\n self.conv3 = self._make_layer(\n blocks[1],\n layers[1],\n channels[1],\n channels[2],\n reduce_spatial_size=True\n )\n self.conv4 = self._make_layer(\n blocks[2],\n layers[2],\n channels[2],\n channels[3],\n reduce_spatial_size=False\n )\n self.conv5 = Conv1x1(channels[3], channels[3])\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n # fully connected layer\n self.fc = self._construct_fc_layer(\n self.feature_dim, channels[3], dropout_p=None\n )\n # identity classification layer\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n self._init_params()\n\n def _make_layer(\n self,\n block,\n layer,\n in_channels,\n out_channels,\n reduce_spatial_size,\n IN=False\n ):\n layers = []\n\n layers.append(block(in_channels, out_channels, IN=IN))\n for i in range(1, layer):\n layers.append(block(out_channels, out_channels, IN=IN))\n\n if reduce_spatial_size:\n layers.append(\n nn.Sequential(\n Conv1x1(out_channels, out_channels),\n nn.AvgPool2d(2, stride=2)\n )\n )\n\n return nn.Sequential(*layers)\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n if fc_dims is None or fc_dims < 0:\n self.feature_dim = input_dim\n return None\n\n if isinstance(fc_dims, int):\n fc_dims = [fc_dims]\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.maxpool(x)\n x = self.conv2(x)\n x = self.conv3(x)\n x = self.conv4(x)\n x = self.conv5(x)\n return x\n\n def forward(self, x, return_featuremaps=False):\n x = self.featuremaps(x)\n if return_featuremaps:\n return x\n v = self.global_avgpool(x)\n v = v.view(v.size(0), -1)\n if self.fc is not None:\n v = self.fc(v)\n if not self.training:\n return v\n y = self.classifier(v)\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError(\"Unsupported loss: {}\".format(self.loss))\n\n\ndef init_pretrained_weights(model, key=''):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n import os\n import errno\n import gdown\n from collections import OrderedDict\n\n def _get_torch_home():\n ENV_TORCH_HOME = 'TORCH_HOME'\n ENV_XDG_CACHE_HOME = 'XDG_CACHE_HOME'\n DEFAULT_CACHE_DIR = '~/.cache'\n torch_home = os.path.expanduser(\n os.getenv(\n ENV_TORCH_HOME,\n os.path.join(\n os.getenv(ENV_XDG_CACHE_HOME, DEFAULT_CACHE_DIR), 'torch'\n )\n )\n )\n return torch_home\n\n torch_home = _get_torch_home()\n model_dir = os.path.join(torch_home, 'checkpoints')\n try:\n os.makedirs(model_dir)\n except OSError as e:\n if e.errno == errno.EEXIST:\n # Directory already exists, ignore.\n pass\n else:\n # Unexpected OSError, re-raise.\n raise\n filename = key + '_imagenet.pth'\n cached_file = os.path.join(model_dir, filename)\n\n if not os.path.exists(cached_file):\n gdown.download(pretrained_urls[key], cached_file, quiet=False)\n\n state_dict = torch.load(cached_file)\n model_dict = model.state_dict()\n new_state_dict = OrderedDict()\n matched_layers, discarded_layers = [], []\n\n for k, v in state_dict.items():\n if k.startswith('module.'):\n k = k[7:] # discard module.\n\n if k in model_dict and model_dict[k].size() == v.size():\n new_state_dict[k] = v\n matched_layers.append(k)\n else:\n discarded_layers.append(k)\n\n model_dict.update(new_state_dict)\n model.load_state_dict(model_dict)\n\n if len(matched_layers) == 0:\n warnings.warn(\n 'The pretrained weights from \"{}\" cannot be loaded, '\n 'please check the key names manually '\n '(** ignored and continue **)'.format(cached_file)\n )\n else:\n print(\n 'Successfully loaded imagenet pretrained weights from \"{}\"'.\n format(cached_file)\n )\n if len(discarded_layers) > 0:\n print(\n '** The following layers are discarded '\n 'due to unmatched keys or layer size: {}'.\n format(discarded_layers)\n )\n\n\n##########\n# Instantiation\n##########\ndef osnet_x1_0(num_classes=1000, pretrained=True, loss='softmax', **kwargs):\n # standard size (width x1.0)\n model = OSNet(\n num_classes,\n blocks=[OSBlock, OSBlock, OSBlock],\n layers=[2, 2, 2],\n channels=[64, 256, 384, 512],\n loss=loss,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, key='osnet_x1_0')\n return model\n\n\ndef osnet_x0_75(num_classes=1000, pretrained=True, loss='softmax', **kwargs):\n # medium size (width x0.75)\n model = OSNet(\n num_classes,\n blocks=[OSBlock, OSBlock, OSBlock],\n layers=[2, 2, 2],\n channels=[48, 192, 288, 384],\n loss=loss,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, key='osnet_x0_75')\n return model\n\n\ndef osnet_x0_5(num_classes=1000, pretrained=True, loss='softmax', **kwargs):\n # tiny size (width x0.5)\n model = OSNet(\n num_classes,\n blocks=[OSBlock, OSBlock, OSBlock],\n layers=[2, 2, 2],\n channels=[32, 128, 192, 256],\n loss=loss,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, key='osnet_x0_5')\n return model\n\n\ndef osnet_x0_25(num_classes=1000, pretrained=True, loss='softmax', **kwargs):\n # very tiny size (width x0.25)\n model = OSNet(\n num_classes,\n blocks=[OSBlock, OSBlock, OSBlock],\n layers=[2, 2, 2],\n channels=[16, 64, 96, 128],\n loss=loss,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, key='osnet_x0_25')\n return model\n\n\ndef osnet_ibn_x1_0(\n num_classes=1000, pretrained=True, loss='softmax', **kwargs\n):\n # standard size (width x1.0) + IBN layer\n # Ref: Pan et al. Two at Once: Enhancing Learning and Generalization Capacities via IBN-Net. ECCV, 2018.\n model = OSNet(\n num_classes,\n blocks=[OSBlock, OSBlock, OSBlock],\n layers=[2, 2, 2],\n channels=[64, 256, 384, 512],\n loss=loss,\n IN=True,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, key='osnet_ibn_x1_0')\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/osnet_ain.py", "content": "from __future__ import division, absolute_import\nimport warnings\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\n__all__ = [\n 'osnet_ain_x1_0', 'osnet_ain_x0_75', 'osnet_ain_x0_5', 'osnet_ain_x0_25'\n]\n\npretrained_urls = {\n 'osnet_ain_x1_0':\n 'https://drive.google.com/uc?id=1-CaioD9NaqbHK_kzSMW8VE4_3KcsRjEo',\n 'osnet_ain_x0_75':\n 'https://drive.google.com/uc?id=1apy0hpsMypqstfencdH-jKIUEFOW4xoM',\n 'osnet_ain_x0_5':\n 'https://drive.google.com/uc?id=1KusKvEYyKGDTUBVRxRiz55G31wkihB6l',\n 'osnet_ain_x0_25':\n 'https://drive.google.com/uc?id=1SxQt2AvmEcgWNhaRb2xC4rP6ZwVDP0Wt'\n}\n\n\n##########\n# Basic layers\n##########\nclass ConvLayer(nn.Module):\n \"\"\"Convolution layer (conv + bn + relu).\"\"\"\n\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size,\n stride=1,\n padding=0,\n groups=1,\n IN=False\n ):\n super(ConvLayer, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n kernel_size,\n stride=stride,\n padding=padding,\n bias=False,\n groups=groups\n )\n if IN:\n self.bn = nn.InstanceNorm2d(out_channels, affine=True)\n else:\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU()\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass Conv1x1(nn.Module):\n \"\"\"1x1 convolution + bn + relu.\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, groups=1):\n super(Conv1x1, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n 1,\n stride=stride,\n padding=0,\n bias=False,\n groups=groups\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU()\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass Conv1x1Linear(nn.Module):\n \"\"\"1x1 convolution + bn (w/o non-linearity).\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, bn=True):\n super(Conv1x1Linear, self).__init__()\n self.conv = nn.Conv2d(\n in_channels, out_channels, 1, stride=stride, padding=0, bias=False\n )\n self.bn = None\n if bn:\n self.bn = nn.BatchNorm2d(out_channels)\n\n def forward(self, x):\n x = self.conv(x)\n if self.bn is not None:\n x = self.bn(x)\n return x\n\n\nclass Conv3x3(nn.Module):\n \"\"\"3x3 convolution + bn + relu.\"\"\"\n\n def __init__(self, in_channels, out_channels, stride=1, groups=1):\n super(Conv3x3, self).__init__()\n self.conv = nn.Conv2d(\n in_channels,\n out_channels,\n 3,\n stride=stride,\n padding=1,\n bias=False,\n groups=groups\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU()\n\n def forward(self, x):\n x = self.conv(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass LightConv3x3(nn.Module):\n \"\"\"Lightweight 3x3 convolution.\n\n 1x1 (linear) + dw 3x3 (nonlinear).\n \"\"\"\n\n def __init__(self, in_channels, out_channels):\n super(LightConv3x3, self).__init__()\n self.conv1 = nn.Conv2d(\n in_channels, out_channels, 1, stride=1, padding=0, bias=False\n )\n self.conv2 = nn.Conv2d(\n out_channels,\n out_channels,\n 3,\n stride=1,\n padding=1,\n bias=False,\n groups=out_channels\n )\n self.bn = nn.BatchNorm2d(out_channels)\n self.relu = nn.ReLU()\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.conv2(x)\n x = self.bn(x)\n return self.relu(x)\n\n\nclass LightConvStream(nn.Module):\n \"\"\"Lightweight convolution stream.\"\"\"\n\n def __init__(self, in_channels, out_channels, depth):\n super(LightConvStream, self).__init__()\n assert depth >= 1, 'depth must be equal to or larger than 1, but got {}'.format(\n depth\n )\n layers = []\n layers += [LightConv3x3(in_channels, out_channels)]\n for i in range(depth - 1):\n layers += [LightConv3x3(out_channels, out_channels)]\n self.layers = nn.Sequential(*layers)\n\n def forward(self, x):\n return self.layers(x)\n\n\n##########\n# Building blocks for omni-scale feature learning\n##########\nclass ChannelGate(nn.Module):\n \"\"\"A mini-network that generates channel-wise gates conditioned on input tensor.\"\"\"\n\n def __init__(\n self,\n in_channels,\n num_gates=None,\n return_gates=False,\n gate_activation='sigmoid',\n reduction=16,\n layer_norm=False\n ):\n super(ChannelGate, self).__init__()\n if num_gates is None:\n num_gates = in_channels\n self.return_gates = return_gates\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.fc1 = nn.Conv2d(\n in_channels,\n in_channels // reduction,\n kernel_size=1,\n bias=True,\n padding=0\n )\n self.norm1 = None\n if layer_norm:\n self.norm1 = nn.LayerNorm((in_channels // reduction, 1, 1))\n self.relu = nn.ReLU()\n self.fc2 = nn.Conv2d(\n in_channels // reduction,\n num_gates,\n kernel_size=1,\n bias=True,\n padding=0\n )\n if gate_activation == 'sigmoid':\n self.gate_activation = nn.Sigmoid()\n elif gate_activation == 'relu':\n self.gate_activation = nn.ReLU()\n elif gate_activation == 'linear':\n self.gate_activation = None\n else:\n raise RuntimeError(\n \"Unknown gate activation: {}\".format(gate_activation)\n )\n\n def forward(self, x):\n input = x\n x = self.global_avgpool(x)\n x = self.fc1(x)\n if self.norm1 is not None:\n x = self.norm1(x)\n x = self.relu(x)\n x = self.fc2(x)\n if self.gate_activation is not None:\n x = self.gate_activation(x)\n if self.return_gates:\n return x\n return input * x\n\n\nclass OSBlock(nn.Module):\n \"\"\"Omni-scale feature learning block.\"\"\"\n\n def __init__(self, in_channels, out_channels, reduction=4, T=4, **kwargs):\n super(OSBlock, self).__init__()\n assert T >= 1\n assert out_channels >= reduction and out_channels % reduction == 0\n mid_channels = out_channels // reduction\n\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2 = nn.ModuleList()\n for t in range(1, T + 1):\n self.conv2 += [LightConvStream(mid_channels, mid_channels, t)]\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2 = 0\n for conv2_t in self.conv2:\n x2_t = conv2_t(x1)\n x2 = x2 + self.gate(x2_t)\n x3 = self.conv3(x2)\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n return F.relu(out)\n\n\nclass OSBlockINin(nn.Module):\n \"\"\"Omni-scale feature learning block with instance normalization.\"\"\"\n\n def __init__(self, in_channels, out_channels, reduction=4, T=4, **kwargs):\n super(OSBlockINin, self).__init__()\n assert T >= 1\n assert out_channels >= reduction and out_channels % reduction == 0\n mid_channels = out_channels // reduction\n\n self.conv1 = Conv1x1(in_channels, mid_channels)\n self.conv2 = nn.ModuleList()\n for t in range(1, T + 1):\n self.conv2 += [LightConvStream(mid_channels, mid_channels, t)]\n self.gate = ChannelGate(mid_channels)\n self.conv3 = Conv1x1Linear(mid_channels, out_channels, bn=False)\n self.downsample = None\n if in_channels != out_channels:\n self.downsample = Conv1x1Linear(in_channels, out_channels)\n self.IN = nn.InstanceNorm2d(out_channels, affine=True)\n\n def forward(self, x):\n identity = x\n x1 = self.conv1(x)\n x2 = 0\n for conv2_t in self.conv2:\n x2_t = conv2_t(x1)\n x2 = x2 + self.gate(x2_t)\n x3 = self.conv3(x2)\n x3 = self.IN(x3) # IN inside residual\n if self.downsample is not None:\n identity = self.downsample(identity)\n out = x3 + identity\n return F.relu(out)\n\n\n##########\n# Network architecture\n##########\nclass OSNet(nn.Module):\n \"\"\"Omni-Scale Network.\n \n Reference:\n - Zhou et al. Omni-Scale Feature Learning for Person Re-Identification. ICCV, 2019.\n - Zhou et al. Learning Generalisable Omni-Scale Representations\n for Person Re-Identification. TPAMI, 2021.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n blocks,\n layers,\n channels,\n feature_dim=512,\n loss='softmax',\n conv1_IN=False,\n **kwargs\n ):\n super(OSNet, self).__init__()\n num_blocks = len(blocks)\n assert num_blocks == len(layers)\n assert num_blocks == len(channels) - 1\n self.loss = loss\n self.feature_dim = feature_dim\n\n # convolutional backbone\n self.conv1 = ConvLayer(\n 3, channels[0], 7, stride=2, padding=3, IN=conv1_IN\n )\n self.maxpool = nn.MaxPool2d(3, stride=2, padding=1)\n self.conv2 = self._make_layer(\n blocks[0], layers[0], channels[0], channels[1]\n )\n self.pool2 = nn.Sequential(\n Conv1x1(channels[1], channels[1]), nn.AvgPool2d(2, stride=2)\n )\n self.conv3 = self._make_layer(\n blocks[1], layers[1], channels[1], channels[2]\n )\n self.pool3 = nn.Sequential(\n Conv1x1(channels[2], channels[2]), nn.AvgPool2d(2, stride=2)\n )\n self.conv4 = self._make_layer(\n blocks[2], layers[2], channels[2], channels[3]\n )\n self.conv5 = Conv1x1(channels[3], channels[3])\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n # fully connected layer\n self.fc = self._construct_fc_layer(\n self.feature_dim, channels[3], dropout_p=None\n )\n # identity classification layer\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n self._init_params()\n\n def _make_layer(self, blocks, layer, in_channels, out_channels):\n layers = []\n layers += [blocks[0](in_channels, out_channels)]\n for i in range(1, len(blocks)):\n layers += [blocks[i](out_channels, out_channels)]\n return nn.Sequential(*layers)\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n if fc_dims is None or fc_dims < 0:\n self.feature_dim = input_dim\n return None\n\n if isinstance(fc_dims, int):\n fc_dims = [fc_dims]\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU())\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.InstanceNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.maxpool(x)\n x = self.conv2(x)\n x = self.pool2(x)\n x = self.conv3(x)\n x = self.pool3(x)\n x = self.conv4(x)\n x = self.conv5(x)\n return x\n\n def forward(self, x, return_featuremaps=False):\n x = self.featuremaps(x)\n if return_featuremaps:\n return x\n v = self.global_avgpool(x)\n v = v.view(v.size(0), -1)\n if self.fc is not None:\n v = self.fc(v)\n if not self.training:\n return v\n y = self.classifier(v)\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError(\"Unsupported loss: {}\".format(self.loss))\n\n\ndef init_pretrained_weights(model, key=''):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n import os\n import errno\n import gdown\n from collections import OrderedDict\n\n def _get_torch_home():\n ENV_TORCH_HOME = 'TORCH_HOME'\n ENV_XDG_CACHE_HOME = 'XDG_CACHE_HOME'\n DEFAULT_CACHE_DIR = '~/.cache'\n torch_home = os.path.expanduser(\n os.getenv(\n ENV_TORCH_HOME,\n os.path.join(\n os.getenv(ENV_XDG_CACHE_HOME, DEFAULT_CACHE_DIR), 'torch'\n )\n )\n )\n return torch_home\n\n torch_home = _get_torch_home()\n model_dir = os.path.join(torch_home, 'checkpoints')\n try:\n os.makedirs(model_dir)\n except OSError as e:\n if e.errno == errno.EEXIST:\n # Directory already exists, ignore.\n pass\n else:\n # Unexpected OSError, re-raise.\n raise\n filename = key + '_imagenet.pth'\n cached_file = os.path.join(model_dir, filename)\n\n if not os.path.exists(cached_file):\n gdown.download(pretrained_urls[key], cached_file, quiet=False)\n\n state_dict = torch.load(cached_file)\n model_dict = model.state_dict()\n new_state_dict = OrderedDict()\n matched_layers, discarded_layers = [], []\n\n for k, v in state_dict.items():\n if k.startswith('module.'):\n k = k[7:] # discard module.\n\n if k in model_dict and model_dict[k].size() == v.size():\n new_state_dict[k] = v\n matched_layers.append(k)\n else:\n discarded_layers.append(k)\n\n model_dict.update(new_state_dict)\n model.load_state_dict(model_dict)\n\n if len(matched_layers) == 0:\n warnings.warn(\n 'The pretrained weights from \"{}\" cannot be loaded, '\n 'please check the key names manually '\n '(** ignored and continue **)'.format(cached_file)\n )\n else:\n print(\n 'Successfully loaded imagenet pretrained weights from \"{}\"'.\n format(cached_file)\n )\n if len(discarded_layers) > 0:\n print(\n '** The following layers are discarded '\n 'due to unmatched keys or layer size: {}'.\n format(discarded_layers)\n )\n\n\n##########\n# Instantiation\n##########\ndef osnet_ain_x1_0(\n num_classes=1000, pretrained=True, loss='softmax', **kwargs\n):\n model = OSNet(\n num_classes,\n blocks=[\n [OSBlockINin, OSBlockINin], [OSBlock, OSBlockINin],\n [OSBlockINin, OSBlock]\n ],\n layers=[2, 2, 2],\n channels=[64, 256, 384, 512],\n loss=loss,\n conv1_IN=True,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, key='osnet_ain_x1_0')\n return model\n\n\ndef osnet_ain_x0_75(\n num_classes=1000, pretrained=True, loss='softmax', **kwargs\n):\n model = OSNet(\n num_classes,\n blocks=[\n [OSBlockINin, OSBlockINin], [OSBlock, OSBlockINin],\n [OSBlockINin, OSBlock]\n ],\n layers=[2, 2, 2],\n channels=[48, 192, 288, 384],\n loss=loss,\n conv1_IN=True,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, key='osnet_ain_x0_75')\n return model\n\n\ndef osnet_ain_x0_5(\n num_classes=1000, pretrained=True, loss='softmax', **kwargs\n):\n model = OSNet(\n num_classes,\n blocks=[\n [OSBlockINin, OSBlockINin], [OSBlock, OSBlockINin],\n [OSBlockINin, OSBlock]\n ],\n layers=[2, 2, 2],\n channels=[32, 128, 192, 256],\n loss=loss,\n conv1_IN=True,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, key='osnet_ain_x0_5')\n return model\n\n\ndef osnet_ain_x0_25(\n num_classes=1000, pretrained=True, loss='softmax', **kwargs\n):\n model = OSNet(\n num_classes,\n blocks=[\n [OSBlockINin, OSBlockINin], [OSBlock, OSBlockINin],\n [OSBlockINin, OSBlock]\n ],\n layers=[2, 2, 2],\n channels=[16, 64, 96, 128],\n loss=loss,\n conv1_IN=True,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, key='osnet_ain_x0_25')\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/pcb.py", "content": "from __future__ import division, absolute_import\nimport torch.utils.model_zoo as model_zoo\nfrom torch import nn\nfrom torch.nn import functional as F\n\n__all__ = ['pcb_p6', 'pcb_p4']\n\nmodel_urls = {\n 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',\n 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',\n 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n}\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n \"\"\"3x3 convolution with padding\"\"\"\n return nn.Conv2d(\n in_planes,\n out_planes,\n kernel_size=3,\n stride=stride,\n padding=1,\n bias=False\n )\n\n\nclass BasicBlock(nn.Module):\n expansion = 1\n\n def __init__(self, inplanes, planes, stride=1, downsample=None):\n super(BasicBlock, self).__init__()\n self.conv1 = conv3x3(inplanes, planes, stride)\n self.bn1 = nn.BatchNorm2d(planes)\n self.relu = nn.ReLU(inplace=True)\n self.conv2 = conv3x3(planes, planes)\n self.bn2 = nn.BatchNorm2d(planes)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n out = self.relu(out)\n\n return out\n\n\nclass Bottleneck(nn.Module):\n expansion = 4\n\n def __init__(self, inplanes, planes, stride=1, downsample=None):\n super(Bottleneck, self).__init__()\n self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n self.bn1 = nn.BatchNorm2d(planes)\n self.conv2 = nn.Conv2d(\n planes,\n planes,\n kernel_size=3,\n stride=stride,\n padding=1,\n bias=False\n )\n self.bn2 = nn.BatchNorm2d(planes)\n self.conv3 = nn.Conv2d(\n planes, planes * self.expansion, kernel_size=1, bias=False\n )\n self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n self.relu = nn.ReLU(inplace=True)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n out = self.relu(out)\n\n out = self.conv3(out)\n out = self.bn3(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n out = self.relu(out)\n\n return out\n\n\nclass DimReduceLayer(nn.Module):\n\n def __init__(self, in_channels, out_channels, nonlinear):\n super(DimReduceLayer, self).__init__()\n layers = []\n layers.append(\n nn.Conv2d(\n in_channels, out_channels, 1, stride=1, padding=0, bias=False\n )\n )\n layers.append(nn.BatchNorm2d(out_channels))\n\n if nonlinear == 'relu':\n layers.append(nn.ReLU(inplace=True))\n elif nonlinear == 'leakyrelu':\n layers.append(nn.LeakyReLU(0.1))\n\n self.layers = nn.Sequential(*layers)\n\n def forward(self, x):\n return self.layers(x)\n\n\nclass PCB(nn.Module):\n \"\"\"Part-based Convolutional Baseline.\n\n Reference:\n Sun et al. Beyond Part Models: Person Retrieval with Refined\n Part Pooling (and A Strong Convolutional Baseline). ECCV 2018.\n\n Public keys:\n - ``pcb_p4``: PCB with 4-part strips.\n - ``pcb_p6``: PCB with 6-part strips.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n loss,\n block,\n layers,\n parts=6,\n reduced_dim=256,\n nonlinear='relu',\n **kwargs\n ):\n self.inplanes = 64\n super(PCB, self).__init__()\n self.loss = loss\n self.parts = parts\n self.feature_dim = 512 * block.expansion\n\n # backbone network\n self.conv1 = nn.Conv2d(\n 3, 64, kernel_size=7, stride=2, padding=3, bias=False\n )\n self.bn1 = nn.BatchNorm2d(64)\n self.relu = nn.ReLU(inplace=True)\n self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n self.layer1 = self._make_layer(block, 64, layers[0])\n self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n self.layer4 = self._make_layer(block, 512, layers[3], stride=1)\n\n # pcb layers\n self.parts_avgpool = nn.AdaptiveAvgPool2d((self.parts, 1))\n self.dropout = nn.Dropout(p=0.5)\n self.conv5 = DimReduceLayer(\n 512 * block.expansion, reduced_dim, nonlinear=nonlinear\n )\n self.feature_dim = reduced_dim\n self.classifier = nn.ModuleList(\n [\n nn.Linear(self.feature_dim, num_classes)\n for _ in range(self.parts)\n ]\n )\n\n self._init_params()\n\n def _make_layer(self, block, planes, blocks, stride=1):\n downsample = None\n if stride != 1 or self.inplanes != planes * block.expansion:\n downsample = nn.Sequential(\n nn.Conv2d(\n self.inplanes,\n planes * block.expansion,\n kernel_size=1,\n stride=stride,\n bias=False\n ),\n nn.BatchNorm2d(planes * block.expansion),\n )\n\n layers = []\n layers.append(block(self.inplanes, planes, stride, downsample))\n self.inplanes = planes * block.expansion\n for i in range(1, blocks):\n layers.append(block(self.inplanes, planes))\n\n return nn.Sequential(*layers)\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = self.relu(x)\n x = self.maxpool(x)\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x = self.layer4(x)\n return x\n\n def forward(self, x):\n f = self.featuremaps(x)\n v_g = self.parts_avgpool(f)\n\n if not self.training:\n v_g = F.normalize(v_g, p=2, dim=1)\n return v_g.view(v_g.size(0), -1)\n\n v_g = self.dropout(v_g)\n v_h = self.conv5(v_g)\n\n y = []\n for i in range(self.parts):\n v_h_i = v_h[:, :, i, :]\n v_h_i = v_h_i.view(v_h_i.size(0), -1)\n y_i = self.classifier[i](v_h_i)\n y.append(y_i)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n v_g = F.normalize(v_g, p=2, dim=1)\n return y, v_g.view(v_g.size(0), -1)\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef pcb_p6(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = PCB(\n num_classes=num_classes,\n loss=loss,\n block=Bottleneck,\n layers=[3, 4, 6, 3],\n last_stride=1,\n parts=6,\n reduced_dim=256,\n nonlinear='relu',\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet50'])\n return model\n\n\ndef pcb_p4(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = PCB(\n num_classes=num_classes,\n loss=loss,\n block=Bottleneck,\n layers=[3, 4, 6, 3],\n last_stride=1,\n parts=4,\n reduced_dim=256,\n nonlinear='relu',\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet50'])\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/resnet.py", "content": "\"\"\"\nCode source: https://github.com/pytorch/vision\n\"\"\"\nfrom __future__ import division, absolute_import\nimport torch.utils.model_zoo as model_zoo\nfrom torch import nn\n\n__all__ = [\n 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152',\n 'resnext50_32x4d', 'resnext101_32x8d', 'resnet50_fc512'\n]\n\nmodel_urls = {\n 'resnet18':\n 'https://download.pytorch.org/models/resnet18-5c106cde.pth',\n 'resnet34':\n 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',\n 'resnet50':\n 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n 'resnet101':\n 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n 'resnet152':\n 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n 'resnext50_32x4d':\n 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth',\n 'resnext101_32x8d':\n 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth',\n}\n\n\ndef conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):\n \"\"\"3x3 convolution with padding\"\"\"\n return nn.Conv2d(\n in_planes,\n out_planes,\n kernel_size=3,\n stride=stride,\n padding=dilation,\n groups=groups,\n bias=False,\n dilation=dilation\n )\n\n\ndef conv1x1(in_planes, out_planes, stride=1):\n \"\"\"1x1 convolution\"\"\"\n return nn.Conv2d(\n in_planes, out_planes, kernel_size=1, stride=stride, bias=False\n )\n\n\nclass BasicBlock(nn.Module):\n expansion = 1\n\n def __init__(\n self,\n inplanes,\n planes,\n stride=1,\n downsample=None,\n groups=1,\n base_width=64,\n dilation=1,\n norm_layer=None\n ):\n super(BasicBlock, self).__init__()\n if norm_layer is None:\n norm_layer = nn.BatchNorm2d\n if groups != 1 or base_width != 64:\n raise ValueError(\n 'BasicBlock only supports groups=1 and base_width=64'\n )\n if dilation > 1:\n raise NotImplementedError(\n \"Dilation > 1 not supported in BasicBlock\"\n )\n # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n self.conv1 = conv3x3(inplanes, planes, stride)\n self.bn1 = norm_layer(planes)\n self.relu = nn.ReLU(inplace=True)\n self.conv2 = conv3x3(planes, planes)\n self.bn2 = norm_layer(planes)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n identity = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n\n if self.downsample is not None:\n identity = self.downsample(x)\n\n out += identity\n out = self.relu(out)\n\n return out\n\n\nclass Bottleneck(nn.Module):\n expansion = 4\n\n def __init__(\n self,\n inplanes,\n planes,\n stride=1,\n downsample=None,\n groups=1,\n base_width=64,\n dilation=1,\n norm_layer=None\n ):\n super(Bottleneck, self).__init__()\n if norm_layer is None:\n norm_layer = nn.BatchNorm2d\n width = int(planes * (base_width/64.)) * groups\n # Both self.conv2 and self.downsample layers downsample the input when stride != 1\n self.conv1 = conv1x1(inplanes, width)\n self.bn1 = norm_layer(width)\n self.conv2 = conv3x3(width, width, stride, groups, dilation)\n self.bn2 = norm_layer(width)\n self.conv3 = conv1x1(width, planes * self.expansion)\n self.bn3 = norm_layer(planes * self.expansion)\n self.relu = nn.ReLU(inplace=True)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n identity = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n out = self.relu(out)\n\n out = self.conv3(out)\n out = self.bn3(out)\n\n if self.downsample is not None:\n identity = self.downsample(x)\n\n out += identity\n out = self.relu(out)\n\n return out\n\n\nclass ResNet(nn.Module):\n \"\"\"Residual network.\n \n Reference:\n - He et al. Deep Residual Learning for Image Recognition. CVPR 2016.\n - Xie et al. Aggregated Residual Transformations for Deep Neural Networks. CVPR 2017.\n\n Public keys:\n - ``resnet18``: ResNet18.\n - ``resnet34``: ResNet34.\n - ``resnet50``: ResNet50.\n - ``resnet101``: ResNet101.\n - ``resnet152``: ResNet152.\n - ``resnext50_32x4d``: ResNeXt50.\n - ``resnext101_32x8d``: ResNeXt101.\n - ``resnet50_fc512``: ResNet50 + FC.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n loss,\n block,\n layers,\n zero_init_residual=False,\n groups=1,\n width_per_group=64,\n replace_stride_with_dilation=None,\n norm_layer=None,\n last_stride=2,\n fc_dims=None,\n dropout_p=None,\n **kwargs\n ):\n super(ResNet, self).__init__()\n if norm_layer is None:\n norm_layer = nn.BatchNorm2d\n self._norm_layer = norm_layer\n self.loss = loss\n self.feature_dim = 512 * block.expansion\n self.inplanes = 64\n self.dilation = 1\n if replace_stride_with_dilation is None:\n # each element in the tuple indicates if we should replace\n # the 2x2 stride with a dilated convolution instead\n replace_stride_with_dilation = [False, False, False]\n if len(replace_stride_with_dilation) != 3:\n raise ValueError(\n \"replace_stride_with_dilation should be None \"\n \"or a 3-element tuple, got {}\".\n format(replace_stride_with_dilation)\n )\n self.groups = groups\n self.base_width = width_per_group\n self.conv1 = nn.Conv2d(\n 3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False\n )\n self.bn1 = norm_layer(self.inplanes)\n self.relu = nn.ReLU(inplace=True)\n self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n self.layer1 = self._make_layer(block, 64, layers[0])\n self.layer2 = self._make_layer(\n block,\n 128,\n layers[1],\n stride=2,\n dilate=replace_stride_with_dilation[0]\n )\n self.layer3 = self._make_layer(\n block,\n 256,\n layers[2],\n stride=2,\n dilate=replace_stride_with_dilation[1]\n )\n self.layer4 = self._make_layer(\n block,\n 512,\n layers[3],\n stride=last_stride,\n dilate=replace_stride_with_dilation[2]\n )\n self.global_avgpool = nn.AdaptiveAvgPool2d((1, 1))\n self.fc = self._construct_fc_layer(\n fc_dims, 512 * block.expansion, dropout_p\n )\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n self._init_params()\n\n # Zero-initialize the last BN in each residual branch,\n # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n if zero_init_residual:\n for m in self.modules():\n if isinstance(m, Bottleneck):\n nn.init.constant_(m.bn3.weight, 0)\n elif isinstance(m, BasicBlock):\n nn.init.constant_(m.bn2.weight, 0)\n\n def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n norm_layer = self._norm_layer\n downsample = None\n previous_dilation = self.dilation\n if dilate:\n self.dilation *= stride\n stride = 1\n if stride != 1 or self.inplanes != planes * block.expansion:\n downsample = nn.Sequential(\n conv1x1(self.inplanes, planes * block.expansion, stride),\n norm_layer(planes * block.expansion),\n )\n\n layers = []\n layers.append(\n block(\n self.inplanes, planes, stride, downsample, self.groups,\n self.base_width, previous_dilation, norm_layer\n )\n )\n self.inplanes = planes * block.expansion\n for _ in range(1, blocks):\n layers.append(\n block(\n self.inplanes,\n planes,\n groups=self.groups,\n base_width=self.base_width,\n dilation=self.dilation,\n norm_layer=norm_layer\n )\n )\n\n return nn.Sequential(*layers)\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n \"\"\"Constructs fully connected layer\n\n Args:\n fc_dims (list or tuple): dimensions of fc layers, if None, no fc layers are constructed\n input_dim (int): input dimension\n dropout_p (float): dropout probability, if None, dropout is unused\n \"\"\"\n if fc_dims is None:\n self.feature_dim = input_dim\n return None\n\n assert isinstance(\n fc_dims, (list, tuple)\n ), 'fc_dims must be either list or tuple, but got {}'.format(\n type(fc_dims)\n )\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = self.relu(x)\n x = self.maxpool(x)\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x = self.layer4(x)\n return x\n\n def forward(self, x):\n f = self.featuremaps(x)\n v = self.global_avgpool(f)\n v = v.view(v.size(0), -1)\n\n if self.fc is not None:\n v = self.fc(v)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError(\"Unsupported loss: {}\".format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\n\"\"\"ResNet\"\"\"\n\n\ndef resnet18(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ResNet(\n num_classes=num_classes,\n loss=loss,\n block=BasicBlock,\n layers=[2, 2, 2, 2],\n last_stride=2,\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet18'])\n return model\n\n\ndef resnet34(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ResNet(\n num_classes=num_classes,\n loss=loss,\n block=BasicBlock,\n layers=[3, 4, 6, 3],\n last_stride=2,\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet34'])\n return model\n\n\ndef resnet50(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ResNet(\n num_classes=num_classes,\n loss=loss,\n block=Bottleneck,\n layers=[3, 4, 6, 3],\n last_stride=2,\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet50'])\n return model\n\n\ndef resnet101(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ResNet(\n num_classes=num_classes,\n loss=loss,\n block=Bottleneck,\n layers=[3, 4, 23, 3],\n last_stride=2,\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet101'])\n return model\n\n\ndef resnet152(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ResNet(\n num_classes=num_classes,\n loss=loss,\n block=Bottleneck,\n layers=[3, 8, 36, 3],\n last_stride=2,\n fc_dims=None,\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet152'])\n return model\n\n\n\"\"\"ResNeXt\"\"\"\n\n\ndef resnext50_32x4d(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ResNet(\n num_classes=num_classes,\n loss=loss,\n block=Bottleneck,\n layers=[3, 4, 6, 3],\n last_stride=2,\n fc_dims=None,\n dropout_p=None,\n groups=32,\n width_per_group=4,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnext50_32x4d'])\n return model\n\n\ndef resnext101_32x8d(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ResNet(\n num_classes=num_classes,\n loss=loss,\n block=Bottleneck,\n layers=[3, 4, 23, 3],\n last_stride=2,\n fc_dims=None,\n dropout_p=None,\n groups=32,\n width_per_group=8,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnext101_32x8d'])\n return model\n\n\n\"\"\"\nResNet + FC\n\"\"\"\n\n\ndef resnet50_fc512(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ResNet(\n num_classes=num_classes,\n loss=loss,\n block=Bottleneck,\n layers=[3, 4, 6, 3],\n last_stride=1,\n fc_dims=[512],\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet50'])\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/resnet_ibn_a.py", "content": "\"\"\"\nCredit to https://github.com/XingangPan/IBN-Net.\n\"\"\"\nfrom __future__ import division, absolute_import\nimport math\nimport torch\nimport torch.nn as nn\nimport torch.utils.model_zoo as model_zoo\n\n__all__ = ['resnet50_ibn_a']\n\nmodel_urls = {\n 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n}\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n \"3x3 convolution with padding\"\n return nn.Conv2d(\n in_planes,\n out_planes,\n kernel_size=3,\n stride=stride,\n padding=1,\n bias=False\n )\n\n\nclass BasicBlock(nn.Module):\n expansion = 1\n\n def __init__(self, inplanes, planes, stride=1, downsample=None):\n super(BasicBlock, self).__init__()\n self.conv1 = conv3x3(inplanes, planes, stride)\n self.bn1 = nn.BatchNorm2d(planes)\n self.relu = nn.ReLU(inplace=True)\n self.conv2 = conv3x3(planes, planes)\n self.bn2 = nn.BatchNorm2d(planes)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n out = self.relu(out)\n\n return out\n\n\nclass IBN(nn.Module):\n\n def __init__(self, planes):\n super(IBN, self).__init__()\n half1 = int(planes / 2)\n self.half = half1\n half2 = planes - half1\n self.IN = nn.InstanceNorm2d(half1, affine=True)\n self.BN = nn.BatchNorm2d(half2)\n\n def forward(self, x):\n split = torch.split(x, self.half, 1)\n out1 = self.IN(split[0].contiguous())\n out2 = self.BN(split[1].contiguous())\n out = torch.cat((out1, out2), 1)\n return out\n\n\nclass Bottleneck(nn.Module):\n expansion = 4\n\n def __init__(self, inplanes, planes, ibn=False, stride=1, downsample=None):\n super(Bottleneck, self).__init__()\n self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n if ibn:\n self.bn1 = IBN(planes)\n else:\n self.bn1 = nn.BatchNorm2d(planes)\n self.conv2 = nn.Conv2d(\n planes,\n planes,\n kernel_size=3,\n stride=stride,\n padding=1,\n bias=False\n )\n self.bn2 = nn.BatchNorm2d(planes)\n self.conv3 = nn.Conv2d(\n planes, planes * self.expansion, kernel_size=1, bias=False\n )\n self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n self.relu = nn.ReLU(inplace=True)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n out = self.relu(out)\n\n out = self.conv3(out)\n out = self.bn3(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n out = self.relu(out)\n\n return out\n\n\nclass ResNet(nn.Module):\n \"\"\"Residual network + IBN layer.\n \n Reference:\n - He et al. Deep Residual Learning for Image Recognition. CVPR 2016.\n - Pan et al. Two at Once: Enhancing Learning and Generalization\n Capacities via IBN-Net. ECCV 2018.\n \"\"\"\n\n def __init__(\n self,\n block,\n layers,\n num_classes=1000,\n loss='softmax',\n fc_dims=None,\n dropout_p=None,\n **kwargs\n ):\n scale = 64\n self.inplanes = scale\n super(ResNet, self).__init__()\n self.loss = loss\n self.feature_dim = scale * 8 * block.expansion\n\n self.conv1 = nn.Conv2d(\n 3, scale, kernel_size=7, stride=2, padding=3, bias=False\n )\n self.bn1 = nn.BatchNorm2d(scale)\n self.relu = nn.ReLU(inplace=True)\n self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n self.layer1 = self._make_layer(block, scale, layers[0])\n self.layer2 = self._make_layer(block, scale * 2, layers[1], stride=2)\n self.layer3 = self._make_layer(block, scale * 4, layers[2], stride=2)\n self.layer4 = self._make_layer(block, scale * 8, layers[3], stride=2)\n self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n self.fc = self._construct_fc_layer(\n fc_dims, scale * 8 * block.expansion, dropout_p\n )\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n m.weight.data.normal_(0, math.sqrt(2. / n))\n elif isinstance(m, nn.BatchNorm2d):\n m.weight.data.fill_(1)\n m.bias.data.zero_()\n elif isinstance(m, nn.InstanceNorm2d):\n m.weight.data.fill_(1)\n m.bias.data.zero_()\n\n def _make_layer(self, block, planes, blocks, stride=1):\n downsample = None\n if stride != 1 or self.inplanes != planes * block.expansion:\n downsample = nn.Sequential(\n nn.Conv2d(\n self.inplanes,\n planes * block.expansion,\n kernel_size=1,\n stride=stride,\n bias=False\n ),\n nn.BatchNorm2d(planes * block.expansion),\n )\n\n layers = []\n ibn = True\n if planes == 512:\n ibn = False\n layers.append(block(self.inplanes, planes, ibn, stride, downsample))\n self.inplanes = planes * block.expansion\n for i in range(1, blocks):\n layers.append(block(self.inplanes, planes, ibn))\n\n return nn.Sequential(*layers)\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n \"\"\"Constructs fully connected layer\n\n Args:\n fc_dims (list or tuple): dimensions of fc layers, if None, no fc layers are constructed\n input_dim (int): input dimension\n dropout_p (float): dropout probability, if None, dropout is unused\n \"\"\"\n if fc_dims is None:\n self.feature_dim = input_dim\n return None\n\n assert isinstance(\n fc_dims, (list, tuple)\n ), 'fc_dims must be either list or tuple, but got {}'.format(\n type(fc_dims)\n )\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = self.relu(x)\n x = self.maxpool(x)\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x = self.layer4(x)\n return x\n\n def forward(self, x):\n f = self.featuremaps(x)\n v = self.avgpool(f)\n v = v.view(v.size(0), -1)\n if self.fc is not None:\n v = self.fc(v)\n if not self.training:\n return v\n y = self.classifier(v)\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError(\"Unsupported loss: {}\".format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef resnet50_ibn_a(num_classes, loss='softmax', pretrained=False, **kwargs):\n model = ResNet(\n Bottleneck, [3, 4, 6, 3], num_classes=num_classes, loss=loss, **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet50'])\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/resnet_ibn_b.py", "content": "\"\"\"\nCredit to https://github.com/XingangPan/IBN-Net.\n\"\"\"\nfrom __future__ import division, absolute_import\nimport math\nimport torch.nn as nn\nimport torch.utils.model_zoo as model_zoo\n\n__all__ = ['resnet50_ibn_b']\n\nmodel_urls = {\n 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n}\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n \"3x3 convolution with padding\"\n return nn.Conv2d(\n in_planes,\n out_planes,\n kernel_size=3,\n stride=stride,\n padding=1,\n bias=False\n )\n\n\nclass BasicBlock(nn.Module):\n expansion = 1\n\n def __init__(self, inplanes, planes, stride=1, downsample=None):\n super(BasicBlock, self).__init__()\n self.conv1 = conv3x3(inplanes, planes, stride)\n self.bn1 = nn.BatchNorm2d(planes)\n self.relu = nn.ReLU(inplace=True)\n self.conv2 = conv3x3(planes, planes)\n self.bn2 = nn.BatchNorm2d(planes)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n out = self.relu(out)\n\n return out\n\n\nclass Bottleneck(nn.Module):\n expansion = 4\n\n def __init__(self, inplanes, planes, stride=1, downsample=None, IN=False):\n super(Bottleneck, self).__init__()\n self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n self.bn1 = nn.BatchNorm2d(planes)\n self.conv2 = nn.Conv2d(\n planes,\n planes,\n kernel_size=3,\n stride=stride,\n padding=1,\n bias=False\n )\n self.bn2 = nn.BatchNorm2d(planes)\n self.conv3 = nn.Conv2d(\n planes, planes * self.expansion, kernel_size=1, bias=False\n )\n self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n self.IN = None\n if IN:\n self.IN = nn.InstanceNorm2d(planes * 4, affine=True)\n self.relu = nn.ReLU(inplace=True)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n out = self.relu(out)\n\n out = self.conv3(out)\n out = self.bn3(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n if self.IN is not None:\n out = self.IN(out)\n out = self.relu(out)\n\n return out\n\n\nclass ResNet(nn.Module):\n \"\"\"Residual network + IBN layer.\n \n Reference:\n - He et al. Deep Residual Learning for Image Recognition. CVPR 2016.\n - Pan et al. Two at Once: Enhancing Learning and Generalization\n Capacities via IBN-Net. ECCV 2018.\n \"\"\"\n\n def __init__(\n self,\n block,\n layers,\n num_classes=1000,\n loss='softmax',\n fc_dims=None,\n dropout_p=None,\n **kwargs\n ):\n scale = 64\n self.inplanes = scale\n super(ResNet, self).__init__()\n self.loss = loss\n self.feature_dim = scale * 8 * block.expansion\n\n self.conv1 = nn.Conv2d(\n 3, scale, kernel_size=7, stride=2, padding=3, bias=False\n )\n self.bn1 = nn.InstanceNorm2d(scale, affine=True)\n self.relu = nn.ReLU(inplace=True)\n self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n self.layer1 = self._make_layer(\n block, scale, layers[0], stride=1, IN=True\n )\n self.layer2 = self._make_layer(\n block, scale * 2, layers[1], stride=2, IN=True\n )\n self.layer3 = self._make_layer(block, scale * 4, layers[2], stride=2)\n self.layer4 = self._make_layer(block, scale * 8, layers[3], stride=2)\n self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n self.fc = self._construct_fc_layer(\n fc_dims, scale * 8 * block.expansion, dropout_p\n )\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n m.weight.data.normal_(0, math.sqrt(2. / n))\n elif isinstance(m, nn.BatchNorm2d):\n m.weight.data.fill_(1)\n m.bias.data.zero_()\n elif isinstance(m, nn.InstanceNorm2d):\n m.weight.data.fill_(1)\n m.bias.data.zero_()\n\n def _make_layer(self, block, planes, blocks, stride=1, IN=False):\n downsample = None\n if stride != 1 or self.inplanes != planes * block.expansion:\n downsample = nn.Sequential(\n nn.Conv2d(\n self.inplanes,\n planes * block.expansion,\n kernel_size=1,\n stride=stride,\n bias=False\n ),\n nn.BatchNorm2d(planes * block.expansion),\n )\n\n layers = []\n layers.append(block(self.inplanes, planes, stride, downsample))\n self.inplanes = planes * block.expansion\n for i in range(1, blocks - 1):\n layers.append(block(self.inplanes, planes))\n layers.append(block(self.inplanes, planes, IN=IN))\n\n return nn.Sequential(*layers)\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n \"\"\"Constructs fully connected layer\n\n Args:\n fc_dims (list or tuple): dimensions of fc layers, if None, no fc layers are constructed\n input_dim (int): input dimension\n dropout_p (float): dropout probability, if None, dropout is unused\n \"\"\"\n if fc_dims is None:\n self.feature_dim = input_dim\n return None\n\n assert isinstance(\n fc_dims, (list, tuple)\n ), 'fc_dims must be either list or tuple, but got {}'.format(\n type(fc_dims)\n )\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = self.relu(x)\n x = self.maxpool(x)\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x = self.layer4(x)\n return x\n\n def forward(self, x):\n f = self.featuremaps(x)\n v = self.avgpool(f)\n v = v.view(v.size(0), -1)\n if self.fc is not None:\n v = self.fc(v)\n if not self.training:\n return v\n y = self.classifier(v)\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError(\"Unsupported loss: {}\".format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef resnet50_ibn_b(num_classes, loss='softmax', pretrained=False, **kwargs):\n model = ResNet(\n Bottleneck, [3, 4, 6, 3], num_classes=num_classes, loss=loss, **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet50'])\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/resnetmid.py", "content": "from __future__ import division, absolute_import\nimport torch\nimport torch.utils.model_zoo as model_zoo\nfrom torch import nn\n\n__all__ = ['resnet50mid']\n\nmodel_urls = {\n 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',\n 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',\n 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n}\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n \"\"\"3x3 convolution with padding\"\"\"\n return nn.Conv2d(\n in_planes,\n out_planes,\n kernel_size=3,\n stride=stride,\n padding=1,\n bias=False\n )\n\n\nclass BasicBlock(nn.Module):\n expansion = 1\n\n def __init__(self, inplanes, planes, stride=1, downsample=None):\n super(BasicBlock, self).__init__()\n self.conv1 = conv3x3(inplanes, planes, stride)\n self.bn1 = nn.BatchNorm2d(planes)\n self.relu = nn.ReLU(inplace=True)\n self.conv2 = conv3x3(planes, planes)\n self.bn2 = nn.BatchNorm2d(planes)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n out = self.relu(out)\n\n return out\n\n\nclass Bottleneck(nn.Module):\n expansion = 4\n\n def __init__(self, inplanes, planes, stride=1, downsample=None):\n super(Bottleneck, self).__init__()\n self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n self.bn1 = nn.BatchNorm2d(planes)\n self.conv2 = nn.Conv2d(\n planes,\n planes,\n kernel_size=3,\n stride=stride,\n padding=1,\n bias=False\n )\n self.bn2 = nn.BatchNorm2d(planes)\n self.conv3 = nn.Conv2d(\n planes, planes * self.expansion, kernel_size=1, bias=False\n )\n self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n self.relu = nn.ReLU(inplace=True)\n self.downsample = downsample\n self.stride = stride\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n out = self.relu(out)\n\n out = self.conv3(out)\n out = self.bn3(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out += residual\n out = self.relu(out)\n\n return out\n\n\nclass ResNetMid(nn.Module):\n \"\"\"Residual network + mid-level features.\n \n Reference:\n Yu et al. The Devil is in the Middle: Exploiting Mid-level Representations for\n Cross-Domain Instance Matching. arXiv:1711.08106.\n\n Public keys:\n - ``resnet50mid``: ResNet50 + mid-level feature fusion.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n loss,\n block,\n layers,\n last_stride=2,\n fc_dims=None,\n **kwargs\n ):\n self.inplanes = 64\n super(ResNetMid, self).__init__()\n self.loss = loss\n self.feature_dim = 512 * block.expansion\n\n # backbone network\n self.conv1 = nn.Conv2d(\n 3, 64, kernel_size=7, stride=2, padding=3, bias=False\n )\n self.bn1 = nn.BatchNorm2d(64)\n self.relu = nn.ReLU(inplace=True)\n self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n self.layer1 = self._make_layer(block, 64, layers[0])\n self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n self.layer4 = self._make_layer(\n block, 512, layers[3], stride=last_stride\n )\n\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n assert fc_dims is not None\n self.fc_fusion = self._construct_fc_layer(\n fc_dims, 512 * block.expansion * 2\n )\n self.feature_dim += 512 * block.expansion\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n self._init_params()\n\n def _make_layer(self, block, planes, blocks, stride=1):\n downsample = None\n if stride != 1 or self.inplanes != planes * block.expansion:\n downsample = nn.Sequential(\n nn.Conv2d(\n self.inplanes,\n planes * block.expansion,\n kernel_size=1,\n stride=stride,\n bias=False\n ),\n nn.BatchNorm2d(planes * block.expansion),\n )\n\n layers = []\n layers.append(block(self.inplanes, planes, stride, downsample))\n self.inplanes = planes * block.expansion\n for i in range(1, blocks):\n layers.append(block(self.inplanes, planes))\n\n return nn.Sequential(*layers)\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n \"\"\"Constructs fully connected layer\n\n Args:\n fc_dims (list or tuple): dimensions of fc layers, if None, no fc layers are constructed\n input_dim (int): input dimension\n dropout_p (float): dropout probability, if None, dropout is unused\n \"\"\"\n if fc_dims is None:\n self.feature_dim = input_dim\n return None\n\n assert isinstance(\n fc_dims, (list, tuple)\n ), 'fc_dims must be either list or tuple, but got {}'.format(\n type(fc_dims)\n )\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = self.relu(x)\n x = self.maxpool(x)\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x4a = self.layer4[0](x)\n x4b = self.layer4[1](x4a)\n x4c = self.layer4[2](x4b)\n return x4a, x4b, x4c\n\n def forward(self, x):\n x4a, x4b, x4c = self.featuremaps(x)\n\n v4a = self.global_avgpool(x4a)\n v4b = self.global_avgpool(x4b)\n v4c = self.global_avgpool(x4c)\n v4ab = torch.cat([v4a, v4b], 1)\n v4ab = v4ab.view(v4ab.size(0), -1)\n v4ab = self.fc_fusion(v4ab)\n v4c = v4c.view(v4c.size(0), -1)\n v = torch.cat([v4ab, v4c], 1)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\n\"\"\"\nResidual network configurations:\n--\nresnet18: block=BasicBlock, layers=[2, 2, 2, 2]\nresnet34: block=BasicBlock, layers=[3, 4, 6, 3]\nresnet50: block=Bottleneck, layers=[3, 4, 6, 3]\nresnet101: block=Bottleneck, layers=[3, 4, 23, 3]\nresnet152: block=Bottleneck, layers=[3, 8, 36, 3]\n\"\"\"\n\n\ndef resnet50mid(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ResNetMid(\n num_classes=num_classes,\n loss=loss,\n block=Bottleneck,\n layers=[3, 4, 6, 3],\n last_stride=2,\n fc_dims=[1024],\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['resnet50'])\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/senet.py", "content": "from __future__ import division, absolute_import\nimport math\nfrom collections import OrderedDict\nimport torch.nn as nn\nfrom torch.utils import model_zoo\n\n__all__ = [\n 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',\n 'se_resnext50_32x4d', 'se_resnext101_32x4d', 'se_resnet50_fc512'\n]\n\"\"\"\nCode imported from https://github.com/Cadene/pretrained-models.pytorch\n\"\"\"\n\npretrained_settings = {\n 'senet154': {\n 'imagenet': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 224, 224],\n 'input_range': [0, 1],\n 'mean': [0.485, 0.456, 0.406],\n 'std': [0.229, 0.224, 0.225],\n 'num_classes': 1000\n }\n },\n 'se_resnet50': {\n 'imagenet': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 224, 224],\n 'input_range': [0, 1],\n 'mean': [0.485, 0.456, 0.406],\n 'std': [0.229, 0.224, 0.225],\n 'num_classes': 1000\n }\n },\n 'se_resnet101': {\n 'imagenet': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 224, 224],\n 'input_range': [0, 1],\n 'mean': [0.485, 0.456, 0.406],\n 'std': [0.229, 0.224, 0.225],\n 'num_classes': 1000\n }\n },\n 'se_resnet152': {\n 'imagenet': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 224, 224],\n 'input_range': [0, 1],\n 'mean': [0.485, 0.456, 0.406],\n 'std': [0.229, 0.224, 0.225],\n 'num_classes': 1000\n }\n },\n 'se_resnext50_32x4d': {\n 'imagenet': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 224, 224],\n 'input_range': [0, 1],\n 'mean': [0.485, 0.456, 0.406],\n 'std': [0.229, 0.224, 0.225],\n 'num_classes': 1000\n }\n },\n 'se_resnext101_32x4d': {\n 'imagenet': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 224, 224],\n 'input_range': [0, 1],\n 'mean': [0.485, 0.456, 0.406],\n 'std': [0.229, 0.224, 0.225],\n 'num_classes': 1000\n }\n },\n}\n\n\nclass SEModule(nn.Module):\n\n def __init__(self, channels, reduction):\n super(SEModule, self).__init__()\n self.avg_pool = nn.AdaptiveAvgPool2d(1)\n self.fc1 = nn.Conv2d(\n channels, channels // reduction, kernel_size=1, padding=0\n )\n self.relu = nn.ReLU(inplace=True)\n self.fc2 = nn.Conv2d(\n channels // reduction, channels, kernel_size=1, padding=0\n )\n self.sigmoid = nn.Sigmoid()\n\n def forward(self, x):\n module_input = x\n x = self.avg_pool(x)\n x = self.fc1(x)\n x = self.relu(x)\n x = self.fc2(x)\n x = self.sigmoid(x)\n return module_input * x\n\n\nclass Bottleneck(nn.Module):\n \"\"\"\n Base class for bottlenecks that implements `forward()` method.\n \"\"\"\n\n def forward(self, x):\n residual = x\n\n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n\n out = self.conv2(out)\n out = self.bn2(out)\n out = self.relu(out)\n\n out = self.conv3(out)\n out = self.bn3(out)\n\n if self.downsample is not None:\n residual = self.downsample(x)\n\n out = self.se_module(out) + residual\n out = self.relu(out)\n\n return out\n\n\nclass SEBottleneck(Bottleneck):\n \"\"\"\n Bottleneck for SENet154.\n \"\"\"\n expansion = 4\n\n def __init__(\n self, inplanes, planes, groups, reduction, stride=1, downsample=None\n ):\n super(SEBottleneck, self).__init__()\n self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)\n self.bn1 = nn.BatchNorm2d(planes * 2)\n self.conv2 = nn.Conv2d(\n planes * 2,\n planes * 4,\n kernel_size=3,\n stride=stride,\n padding=1,\n groups=groups,\n bias=False\n )\n self.bn2 = nn.BatchNorm2d(planes * 4)\n self.conv3 = nn.Conv2d(\n planes * 4, planes * 4, kernel_size=1, bias=False\n )\n self.bn3 = nn.BatchNorm2d(planes * 4)\n self.relu = nn.ReLU(inplace=True)\n self.se_module = SEModule(planes * 4, reduction=reduction)\n self.downsample = downsample\n self.stride = stride\n\n\nclass SEResNetBottleneck(Bottleneck):\n \"\"\"\n ResNet bottleneck with a Squeeze-and-Excitation module. It follows Caffe\n implementation and uses `stride=stride` in `conv1` and not in `conv2`\n (the latter is used in the torchvision implementation of ResNet).\n \"\"\"\n expansion = 4\n\n def __init__(\n self, inplanes, planes, groups, reduction, stride=1, downsample=None\n ):\n super(SEResNetBottleneck, self).__init__()\n self.conv1 = nn.Conv2d(\n inplanes, planes, kernel_size=1, bias=False, stride=stride\n )\n self.bn1 = nn.BatchNorm2d(planes)\n self.conv2 = nn.Conv2d(\n planes,\n planes,\n kernel_size=3,\n padding=1,\n groups=groups,\n bias=False\n )\n self.bn2 = nn.BatchNorm2d(planes)\n self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n self.bn3 = nn.BatchNorm2d(planes * 4)\n self.relu = nn.ReLU(inplace=True)\n self.se_module = SEModule(planes * 4, reduction=reduction)\n self.downsample = downsample\n self.stride = stride\n\n\nclass SEResNeXtBottleneck(Bottleneck):\n \"\"\"ResNeXt bottleneck type C with a Squeeze-and-Excitation module\"\"\"\n expansion = 4\n\n def __init__(\n self,\n inplanes,\n planes,\n groups,\n reduction,\n stride=1,\n downsample=None,\n base_width=4\n ):\n super(SEResNeXtBottleneck, self).__init__()\n width = int(math.floor(planes * (base_width/64.)) * groups)\n self.conv1 = nn.Conv2d(\n inplanes, width, kernel_size=1, bias=False, stride=1\n )\n self.bn1 = nn.BatchNorm2d(width)\n self.conv2 = nn.Conv2d(\n width,\n width,\n kernel_size=3,\n stride=stride,\n padding=1,\n groups=groups,\n bias=False\n )\n self.bn2 = nn.BatchNorm2d(width)\n self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)\n self.bn3 = nn.BatchNorm2d(planes * 4)\n self.relu = nn.ReLU(inplace=True)\n self.se_module = SEModule(planes * 4, reduction=reduction)\n self.downsample = downsample\n self.stride = stride\n\n\nclass SENet(nn.Module):\n \"\"\"Squeeze-and-excitation network.\n \n Reference:\n Hu et al. Squeeze-and-Excitation Networks. CVPR 2018.\n\n Public keys:\n - ``senet154``: SENet154.\n - ``se_resnet50``: ResNet50 + SE.\n - ``se_resnet101``: ResNet101 + SE.\n - ``se_resnet152``: ResNet152 + SE.\n - ``se_resnext50_32x4d``: ResNeXt50 (groups=32, width=4) + SE.\n - ``se_resnext101_32x4d``: ResNeXt101 (groups=32, width=4) + SE.\n - ``se_resnet50_fc512``: (ResNet50 + SE) + FC.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n loss,\n block,\n layers,\n groups,\n reduction,\n dropout_p=0.2,\n inplanes=128,\n input_3x3=True,\n downsample_kernel_size=3,\n downsample_padding=1,\n last_stride=2,\n fc_dims=None,\n **kwargs\n ):\n \"\"\"\n Parameters\n ----------\n block (nn.Module): Bottleneck class.\n - For SENet154: SEBottleneck\n - For SE-ResNet models: SEResNetBottleneck\n - For SE-ResNeXt models: SEResNeXtBottleneck\n layers (list of ints): Number of residual blocks for 4 layers of the\n network (layer1...layer4).\n groups (int): Number of groups for the 3x3 convolution in each\n bottleneck block.\n - For SENet154: 64\n - For SE-ResNet models: 1\n - For SE-ResNeXt models: 32\n reduction (int): Reduction ratio for Squeeze-and-Excitation modules.\n - For all models: 16\n dropout_p (float or None): Drop probability for the Dropout layer.\n If `None` the Dropout layer is not used.\n - For SENet154: 0.2\n - For SE-ResNet models: None\n - For SE-ResNeXt models: None\n inplanes (int): Number of input channels for layer1.\n - For SENet154: 128\n - For SE-ResNet models: 64\n - For SE-ResNeXt models: 64\n input_3x3 (bool): If `True`, use three 3x3 convolutions instead of\n a single 7x7 convolution in layer0.\n - For SENet154: True\n - For SE-ResNet models: False\n - For SE-ResNeXt models: False\n downsample_kernel_size (int): Kernel size for downsampling convolutions\n in layer2, layer3 and layer4.\n - For SENet154: 3\n - For SE-ResNet models: 1\n - For SE-ResNeXt models: 1\n downsample_padding (int): Padding for downsampling convolutions in\n layer2, layer3 and layer4.\n - For SENet154: 1\n - For SE-ResNet models: 0\n - For SE-ResNeXt models: 0\n num_classes (int): Number of outputs in `classifier` layer.\n \"\"\"\n super(SENet, self).__init__()\n self.inplanes = inplanes\n self.loss = loss\n\n if input_3x3:\n layer0_modules = [\n (\n 'conv1',\n nn.Conv2d(3, 64, 3, stride=2, padding=1, bias=False)\n ),\n ('bn1', nn.BatchNorm2d(64)),\n ('relu1', nn.ReLU(inplace=True)),\n (\n 'conv2',\n nn.Conv2d(64, 64, 3, stride=1, padding=1, bias=False)\n ),\n ('bn2', nn.BatchNorm2d(64)),\n ('relu2', nn.ReLU(inplace=True)),\n (\n 'conv3',\n nn.Conv2d(\n 64, inplanes, 3, stride=1, padding=1, bias=False\n )\n ),\n ('bn3', nn.BatchNorm2d(inplanes)),\n ('relu3', nn.ReLU(inplace=True)),\n ]\n else:\n layer0_modules = [\n (\n 'conv1',\n nn.Conv2d(\n 3,\n inplanes,\n kernel_size=7,\n stride=2,\n padding=3,\n bias=False\n )\n ),\n ('bn1', nn.BatchNorm2d(inplanes)),\n ('relu1', nn.ReLU(inplace=True)),\n ]\n # To preserve compatibility with Caffe weights `ceil_mode=True`\n # is used instead of `padding=1`.\n layer0_modules.append(\n ('pool', nn.MaxPool2d(3, stride=2, ceil_mode=True))\n )\n self.layer0 = nn.Sequential(OrderedDict(layer0_modules))\n self.layer1 = self._make_layer(\n block,\n planes=64,\n blocks=layers[0],\n groups=groups,\n reduction=reduction,\n downsample_kernel_size=1,\n downsample_padding=0\n )\n self.layer2 = self._make_layer(\n block,\n planes=128,\n blocks=layers[1],\n stride=2,\n groups=groups,\n reduction=reduction,\n downsample_kernel_size=downsample_kernel_size,\n downsample_padding=downsample_padding\n )\n self.layer3 = self._make_layer(\n block,\n planes=256,\n blocks=layers[2],\n stride=2,\n groups=groups,\n reduction=reduction,\n downsample_kernel_size=downsample_kernel_size,\n downsample_padding=downsample_padding\n )\n self.layer4 = self._make_layer(\n block,\n planes=512,\n blocks=layers[3],\n stride=last_stride,\n groups=groups,\n reduction=reduction,\n downsample_kernel_size=downsample_kernel_size,\n downsample_padding=downsample_padding\n )\n\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.fc = self._construct_fc_layer(\n fc_dims, 512 * block.expansion, dropout_p\n )\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n def _make_layer(\n self,\n block,\n planes,\n blocks,\n groups,\n reduction,\n stride=1,\n downsample_kernel_size=1,\n downsample_padding=0\n ):\n downsample = None\n if stride != 1 or self.inplanes != planes * block.expansion:\n downsample = nn.Sequential(\n nn.Conv2d(\n self.inplanes,\n planes * block.expansion,\n kernel_size=downsample_kernel_size,\n stride=stride,\n padding=downsample_padding,\n bias=False\n ),\n nn.BatchNorm2d(planes * block.expansion),\n )\n\n layers = []\n layers.append(\n block(\n self.inplanes, planes, groups, reduction, stride, downsample\n )\n )\n self.inplanes = planes * block.expansion\n for i in range(1, blocks):\n layers.append(block(self.inplanes, planes, groups, reduction))\n\n return nn.Sequential(*layers)\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n \"\"\"\n Construct fully connected layer\n\n - fc_dims (list or tuple): dimensions of fc layers, if None,\n no fc layers are constructed\n - input_dim (int): input dimension\n - dropout_p (float): dropout probability, if None, dropout is unused\n \"\"\"\n if fc_dims is None:\n self.feature_dim = input_dim\n return None\n\n assert isinstance(\n fc_dims, (list, tuple)\n ), 'fc_dims must be either list or tuple, but got {}'.format(\n type(fc_dims)\n )\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def featuremaps(self, x):\n x = self.layer0(x)\n x = self.layer1(x)\n x = self.layer2(x)\n x = self.layer3(x)\n x = self.layer4(x)\n return x\n\n def forward(self, x):\n f = self.featuremaps(x)\n v = self.global_avgpool(f)\n v = v.view(v.size(0), -1)\n\n if self.fc is not None:\n v = self.fc(v)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError(\"Unsupported loss: {}\".format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef senet154(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = SENet(\n num_classes=num_classes,\n loss=loss,\n block=SEBottleneck,\n layers=[3, 8, 36, 3],\n groups=64,\n reduction=16,\n dropout_p=0.2,\n last_stride=2,\n fc_dims=None,\n **kwargs\n )\n if pretrained:\n model_url = pretrained_settings['senet154']['imagenet']['url']\n init_pretrained_weights(model, model_url)\n return model\n\n\ndef se_resnet50(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = SENet(\n num_classes=num_classes,\n loss=loss,\n block=SEResNetBottleneck,\n layers=[3, 4, 6, 3],\n groups=1,\n reduction=16,\n dropout_p=None,\n inplanes=64,\n input_3x3=False,\n downsample_kernel_size=1,\n downsample_padding=0,\n last_stride=2,\n fc_dims=None,\n **kwargs\n )\n if pretrained:\n model_url = pretrained_settings['se_resnet50']['imagenet']['url']\n init_pretrained_weights(model, model_url)\n return model\n\n\ndef se_resnet50_fc512(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = SENet(\n num_classes=num_classes,\n loss=loss,\n block=SEResNetBottleneck,\n layers=[3, 4, 6, 3],\n groups=1,\n reduction=16,\n dropout_p=None,\n inplanes=64,\n input_3x3=False,\n downsample_kernel_size=1,\n downsample_padding=0,\n last_stride=1,\n fc_dims=[512],\n **kwargs\n )\n if pretrained:\n model_url = pretrained_settings['se_resnet50']['imagenet']['url']\n init_pretrained_weights(model, model_url)\n return model\n\n\ndef se_resnet101(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = SENet(\n num_classes=num_classes,\n loss=loss,\n block=SEResNetBottleneck,\n layers=[3, 4, 23, 3],\n groups=1,\n reduction=16,\n dropout_p=None,\n inplanes=64,\n input_3x3=False,\n downsample_kernel_size=1,\n downsample_padding=0,\n last_stride=2,\n fc_dims=None,\n **kwargs\n )\n if pretrained:\n model_url = pretrained_settings['se_resnet101']['imagenet']['url']\n init_pretrained_weights(model, model_url)\n return model\n\n\ndef se_resnet152(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = SENet(\n num_classes=num_classes,\n loss=loss,\n block=SEResNetBottleneck,\n layers=[3, 8, 36, 3],\n groups=1,\n reduction=16,\n dropout_p=None,\n inplanes=64,\n input_3x3=False,\n downsample_kernel_size=1,\n downsample_padding=0,\n last_stride=2,\n fc_dims=None,\n **kwargs\n )\n if pretrained:\n model_url = pretrained_settings['se_resnet152']['imagenet']['url']\n init_pretrained_weights(model, model_url)\n return model\n\n\ndef se_resnext50_32x4d(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = SENet(\n num_classes=num_classes,\n loss=loss,\n block=SEResNeXtBottleneck,\n layers=[3, 4, 6, 3],\n groups=32,\n reduction=16,\n dropout_p=None,\n inplanes=64,\n input_3x3=False,\n downsample_kernel_size=1,\n downsample_padding=0,\n last_stride=2,\n fc_dims=None,\n **kwargs\n )\n if pretrained:\n model_url = pretrained_settings['se_resnext50_32x4d']['imagenet']['url'\n ]\n init_pretrained_weights(model, model_url)\n return model\n\n\ndef se_resnext101_32x4d(\n num_classes, loss='softmax', pretrained=True, **kwargs\n):\n model = SENet(\n num_classes=num_classes,\n loss=loss,\n block=SEResNeXtBottleneck,\n layers=[3, 4, 23, 3],\n groups=32,\n reduction=16,\n dropout_p=None,\n inplanes=64,\n input_3x3=False,\n downsample_kernel_size=1,\n downsample_padding=0,\n last_stride=2,\n fc_dims=None,\n **kwargs\n )\n if pretrained:\n model_url = pretrained_settings['se_resnext101_32x4d']['imagenet'][\n 'url']\n init_pretrained_weights(model, model_url)\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/shufflenet.py", "content": "from __future__ import division, absolute_import\nimport torch\nimport torch.utils.model_zoo as model_zoo\nfrom torch import nn\nfrom torch.nn import functional as F\n\n__all__ = ['shufflenet']\n\nmodel_urls = {\n # training epoch = 90, top1 = 61.8\n 'imagenet':\n 'https://mega.nz/#!RDpUlQCY!tr_5xBEkelzDjveIYBBcGcovNCOrgfiJO9kiidz9fZM',\n}\n\n\nclass ChannelShuffle(nn.Module):\n\n def __init__(self, num_groups):\n super(ChannelShuffle, self).__init__()\n self.g = num_groups\n\n def forward(self, x):\n b, c, h, w = x.size()\n n = c // self.g\n # reshape\n x = x.view(b, self.g, n, h, w)\n # transpose\n x = x.permute(0, 2, 1, 3, 4).contiguous()\n # flatten\n x = x.view(b, c, h, w)\n return x\n\n\nclass Bottleneck(nn.Module):\n\n def __init__(\n self,\n in_channels,\n out_channels,\n stride,\n num_groups,\n group_conv1x1=True\n ):\n super(Bottleneck, self).__init__()\n assert stride in [1, 2], 'Warning: stride must be either 1 or 2'\n self.stride = stride\n mid_channels = out_channels // 4\n if stride == 2:\n out_channels -= in_channels\n # group conv is not applied to first conv1x1 at stage 2\n num_groups_conv1x1 = num_groups if group_conv1x1 else 1\n self.conv1 = nn.Conv2d(\n in_channels,\n mid_channels,\n 1,\n groups=num_groups_conv1x1,\n bias=False\n )\n self.bn1 = nn.BatchNorm2d(mid_channels)\n self.shuffle1 = ChannelShuffle(num_groups)\n self.conv2 = nn.Conv2d(\n mid_channels,\n mid_channels,\n 3,\n stride=stride,\n padding=1,\n groups=mid_channels,\n bias=False\n )\n self.bn2 = nn.BatchNorm2d(mid_channels)\n self.conv3 = nn.Conv2d(\n mid_channels, out_channels, 1, groups=num_groups, bias=False\n )\n self.bn3 = nn.BatchNorm2d(out_channels)\n if stride == 2:\n self.shortcut = nn.AvgPool2d(3, stride=2, padding=1)\n\n def forward(self, x):\n out = F.relu(self.bn1(self.conv1(x)))\n out = self.shuffle1(out)\n out = self.bn2(self.conv2(out))\n out = self.bn3(self.conv3(out))\n if self.stride == 2:\n res = self.shortcut(x)\n out = F.relu(torch.cat([res, out], 1))\n else:\n out = F.relu(x + out)\n return out\n\n\n# configuration of (num_groups: #out_channels) based on Table 1 in the paper\ncfg = {\n 1: [144, 288, 576],\n 2: [200, 400, 800],\n 3: [240, 480, 960],\n 4: [272, 544, 1088],\n 8: [384, 768, 1536],\n}\n\n\nclass ShuffleNet(nn.Module):\n \"\"\"ShuffleNet.\n\n Reference:\n Zhang et al. ShuffleNet: An Extremely Efficient Convolutional Neural\n Network for Mobile Devices. CVPR 2018.\n\n Public keys:\n - ``shufflenet``: ShuffleNet (groups=3).\n \"\"\"\n\n def __init__(self, num_classes, loss='softmax', num_groups=3, **kwargs):\n super(ShuffleNet, self).__init__()\n self.loss = loss\n\n self.conv1 = nn.Sequential(\n nn.Conv2d(3, 24, 3, stride=2, padding=1, bias=False),\n nn.BatchNorm2d(24),\n nn.ReLU(),\n nn.MaxPool2d(3, stride=2, padding=1),\n )\n\n self.stage2 = nn.Sequential(\n Bottleneck(\n 24, cfg[num_groups][0], 2, num_groups, group_conv1x1=False\n ),\n Bottleneck(cfg[num_groups][0], cfg[num_groups][0], 1, num_groups),\n Bottleneck(cfg[num_groups][0], cfg[num_groups][0], 1, num_groups),\n Bottleneck(cfg[num_groups][0], cfg[num_groups][0], 1, num_groups),\n )\n\n self.stage3 = nn.Sequential(\n Bottleneck(cfg[num_groups][0], cfg[num_groups][1], 2, num_groups),\n Bottleneck(cfg[num_groups][1], cfg[num_groups][1], 1, num_groups),\n Bottleneck(cfg[num_groups][1], cfg[num_groups][1], 1, num_groups),\n Bottleneck(cfg[num_groups][1], cfg[num_groups][1], 1, num_groups),\n Bottleneck(cfg[num_groups][1], cfg[num_groups][1], 1, num_groups),\n Bottleneck(cfg[num_groups][1], cfg[num_groups][1], 1, num_groups),\n Bottleneck(cfg[num_groups][1], cfg[num_groups][1], 1, num_groups),\n Bottleneck(cfg[num_groups][1], cfg[num_groups][1], 1, num_groups),\n )\n\n self.stage4 = nn.Sequential(\n Bottleneck(cfg[num_groups][1], cfg[num_groups][2], 2, num_groups),\n Bottleneck(cfg[num_groups][2], cfg[num_groups][2], 1, num_groups),\n Bottleneck(cfg[num_groups][2], cfg[num_groups][2], 1, num_groups),\n Bottleneck(cfg[num_groups][2], cfg[num_groups][2], 1, num_groups),\n )\n\n self.classifier = nn.Linear(cfg[num_groups][2], num_classes)\n self.feat_dim = cfg[num_groups][2]\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.stage2(x)\n x = self.stage3(x)\n x = self.stage4(x)\n x = F.avg_pool2d(x, x.size()[2:]).view(x.size(0), -1)\n\n if not self.training:\n return x\n\n y = self.classifier(x)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, x\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef shufflenet(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ShuffleNet(num_classes, loss, **kwargs)\n if pretrained:\n # init_pretrained_weights(model, model_urls['imagenet'])\n import warnings\n warnings.warn(\n 'The imagenet pretrained weights need to be manually downloaded from {}'\n .format(model_urls['imagenet'])\n )\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/shufflenetv2.py", "content": "\"\"\"\nCode source: https://github.com/pytorch/vision\n\"\"\"\nfrom __future__ import division, absolute_import\nimport torch\nimport torch.utils.model_zoo as model_zoo\nfrom torch import nn\n\n__all__ = [\n 'shufflenet_v2_x0_5', 'shufflenet_v2_x1_0', 'shufflenet_v2_x1_5',\n 'shufflenet_v2_x2_0'\n]\n\nmodel_urls = {\n 'shufflenetv2_x0.5':\n 'https://download.pytorch.org/models/shufflenetv2_x0.5-f707e7126e.pth',\n 'shufflenetv2_x1.0':\n 'https://download.pytorch.org/models/shufflenetv2_x1-5666bf0f80.pth',\n 'shufflenetv2_x1.5': None,\n 'shufflenetv2_x2.0': None,\n}\n\n\ndef channel_shuffle(x, groups):\n batchsize, num_channels, height, width = x.data.size()\n channels_per_group = num_channels // groups\n\n # reshape\n x = x.view(batchsize, groups, channels_per_group, height, width)\n\n x = torch.transpose(x, 1, 2).contiguous()\n\n # flatten\n x = x.view(batchsize, -1, height, width)\n\n return x\n\n\nclass InvertedResidual(nn.Module):\n\n def __init__(self, inp, oup, stride):\n super(InvertedResidual, self).__init__()\n\n if not (1 <= stride <= 3):\n raise ValueError('illegal stride value')\n self.stride = stride\n\n branch_features = oup // 2\n assert (self.stride != 1) or (inp == branch_features << 1)\n\n if self.stride > 1:\n self.branch1 = nn.Sequential(\n self.depthwise_conv(\n inp, inp, kernel_size=3, stride=self.stride, padding=1\n ),\n nn.BatchNorm2d(inp),\n nn.Conv2d(\n inp,\n branch_features,\n kernel_size=1,\n stride=1,\n padding=0,\n bias=False\n ),\n nn.BatchNorm2d(branch_features),\n nn.ReLU(inplace=True),\n )\n\n self.branch2 = nn.Sequential(\n nn.Conv2d(\n inp if (self.stride > 1) else branch_features,\n branch_features,\n kernel_size=1,\n stride=1,\n padding=0,\n bias=False\n ),\n nn.BatchNorm2d(branch_features),\n nn.ReLU(inplace=True),\n self.depthwise_conv(\n branch_features,\n branch_features,\n kernel_size=3,\n stride=self.stride,\n padding=1\n ),\n nn.BatchNorm2d(branch_features),\n nn.Conv2d(\n branch_features,\n branch_features,\n kernel_size=1,\n stride=1,\n padding=0,\n bias=False\n ),\n nn.BatchNorm2d(branch_features),\n nn.ReLU(inplace=True),\n )\n\n @staticmethod\n def depthwise_conv(i, o, kernel_size, stride=1, padding=0, bias=False):\n return nn.Conv2d(\n i, o, kernel_size, stride, padding, bias=bias, groups=i\n )\n\n def forward(self, x):\n if self.stride == 1:\n x1, x2 = x.chunk(2, dim=1)\n out = torch.cat((x1, self.branch2(x2)), dim=1)\n else:\n out = torch.cat((self.branch1(x), self.branch2(x)), dim=1)\n\n out = channel_shuffle(out, 2)\n\n return out\n\n\nclass ShuffleNetV2(nn.Module):\n \"\"\"ShuffleNetV2.\n \n Reference:\n Ma et al. ShuffleNet V2: Practical Guidelines for Efficient CNN Architecture Design. ECCV 2018.\n\n Public keys:\n - ``shufflenet_v2_x0_5``: ShuffleNetV2 x0.5.\n - ``shufflenet_v2_x1_0``: ShuffleNetV2 x1.0.\n - ``shufflenet_v2_x1_5``: ShuffleNetV2 x1.5.\n - ``shufflenet_v2_x2_0``: ShuffleNetV2 x2.0.\n \"\"\"\n\n def __init__(\n self, num_classes, loss, stages_repeats, stages_out_channels, **kwargs\n ):\n super(ShuffleNetV2, self).__init__()\n self.loss = loss\n\n if len(stages_repeats) != 3:\n raise ValueError(\n 'expected stages_repeats as list of 3 positive ints'\n )\n if len(stages_out_channels) != 5:\n raise ValueError(\n 'expected stages_out_channels as list of 5 positive ints'\n )\n self._stage_out_channels = stages_out_channels\n\n input_channels = 3\n output_channels = self._stage_out_channels[0]\n self.conv1 = nn.Sequential(\n nn.Conv2d(input_channels, output_channels, 3, 2, 1, bias=False),\n nn.BatchNorm2d(output_channels),\n nn.ReLU(inplace=True),\n )\n input_channels = output_channels\n\n self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n\n stage_names = ['stage{}'.format(i) for i in [2, 3, 4]]\n for name, repeats, output_channels in zip(\n stage_names, stages_repeats, self._stage_out_channels[1:]\n ):\n seq = [InvertedResidual(input_channels, output_channels, 2)]\n for i in range(repeats - 1):\n seq.append(\n InvertedResidual(output_channels, output_channels, 1)\n )\n setattr(self, name, nn.Sequential(*seq))\n input_channels = output_channels\n\n output_channels = self._stage_out_channels[-1]\n self.conv5 = nn.Sequential(\n nn.Conv2d(input_channels, output_channels, 1, 1, 0, bias=False),\n nn.BatchNorm2d(output_channels),\n nn.ReLU(inplace=True),\n )\n self.global_avgpool = nn.AdaptiveAvgPool2d((1, 1))\n\n self.classifier = nn.Linear(output_channels, num_classes)\n\n def featuremaps(self, x):\n x = self.conv1(x)\n x = self.maxpool(x)\n x = self.stage2(x)\n x = self.stage3(x)\n x = self.stage4(x)\n x = self.conv5(x)\n return x\n\n def forward(self, x):\n f = self.featuremaps(x)\n v = self.global_avgpool(f)\n v = v.view(v.size(0), -1)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError(\"Unsupported loss: {}\".format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n if model_url is None:\n import warnings\n warnings.warn(\n 'ImageNet pretrained weights are unavailable for this model'\n )\n return\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef shufflenet_v2_x0_5(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ShuffleNetV2(\n num_classes, loss, [4, 8, 4], [24, 48, 96, 192, 1024], **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['shufflenetv2_x0.5'])\n return model\n\n\ndef shufflenet_v2_x1_0(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ShuffleNetV2(\n num_classes, loss, [4, 8, 4], [24, 116, 232, 464, 1024], **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['shufflenetv2_x1.0'])\n return model\n\n\ndef shufflenet_v2_x1_5(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ShuffleNetV2(\n num_classes, loss, [4, 8, 4], [24, 176, 352, 704, 1024], **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['shufflenetv2_x1.5'])\n return model\n\n\ndef shufflenet_v2_x2_0(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = ShuffleNetV2(\n num_classes, loss, [4, 8, 4], [24, 244, 488, 976, 2048], **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['shufflenetv2_x2.0'])\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/squeezenet.py", "content": "\"\"\"\nCode source: https://github.com/pytorch/vision\n\"\"\"\nfrom __future__ import division, absolute_import\nimport torch\nimport torch.nn as nn\nimport torch.utils.model_zoo as model_zoo\n\n__all__ = ['squeezenet1_0', 'squeezenet1_1', 'squeezenet1_0_fc512']\n\nmodel_urls = {\n 'squeezenet1_0':\n 'https://download.pytorch.org/models/squeezenet1_0-a815701f.pth',\n 'squeezenet1_1':\n 'https://download.pytorch.org/models/squeezenet1_1-f364aa15.pth',\n}\n\n\nclass Fire(nn.Module):\n\n def __init__(\n self, inplanes, squeeze_planes, expand1x1_planes, expand3x3_planes\n ):\n super(Fire, self).__init__()\n self.inplanes = inplanes\n self.squeeze = nn.Conv2d(inplanes, squeeze_planes, kernel_size=1)\n self.squeeze_activation = nn.ReLU(inplace=True)\n self.expand1x1 = nn.Conv2d(\n squeeze_planes, expand1x1_planes, kernel_size=1\n )\n self.expand1x1_activation = nn.ReLU(inplace=True)\n self.expand3x3 = nn.Conv2d(\n squeeze_planes, expand3x3_planes, kernel_size=3, padding=1\n )\n self.expand3x3_activation = nn.ReLU(inplace=True)\n\n def forward(self, x):\n x = self.squeeze_activation(self.squeeze(x))\n return torch.cat(\n [\n self.expand1x1_activation(self.expand1x1(x)),\n self.expand3x3_activation(self.expand3x3(x))\n ], 1\n )\n\n\nclass SqueezeNet(nn.Module):\n \"\"\"SqueezeNet.\n\n Reference:\n Iandola et al. SqueezeNet: AlexNet-level accuracy with 50x fewer parameters\n and< 0.5 MB model size. arXiv:1602.07360.\n\n Public keys:\n - ``squeezenet1_0``: SqueezeNet (version=1.0).\n - ``squeezenet1_1``: SqueezeNet (version=1.1).\n - ``squeezenet1_0_fc512``: SqueezeNet (version=1.0) + FC.\n \"\"\"\n\n def __init__(\n self,\n num_classes,\n loss,\n version=1.0,\n fc_dims=None,\n dropout_p=None,\n **kwargs\n ):\n super(SqueezeNet, self).__init__()\n self.loss = loss\n self.feature_dim = 512\n\n if version not in [1.0, 1.1]:\n raise ValueError(\n 'Unsupported SqueezeNet version {version}:'\n '1.0 or 1.1 expected'.format(version=version)\n )\n\n if version == 1.0:\n self.features = nn.Sequential(\n nn.Conv2d(3, 96, kernel_size=7, stride=2),\n nn.ReLU(inplace=True),\n nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),\n Fire(96, 16, 64, 64),\n Fire(128, 16, 64, 64),\n Fire(128, 32, 128, 128),\n nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),\n Fire(256, 32, 128, 128),\n Fire(256, 48, 192, 192),\n Fire(384, 48, 192, 192),\n Fire(384, 64, 256, 256),\n nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),\n Fire(512, 64, 256, 256),\n )\n else:\n self.features = nn.Sequential(\n nn.Conv2d(3, 64, kernel_size=3, stride=2),\n nn.ReLU(inplace=True),\n nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),\n Fire(64, 16, 64, 64),\n Fire(128, 16, 64, 64),\n nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),\n Fire(128, 32, 128, 128),\n Fire(256, 32, 128, 128),\n nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),\n Fire(256, 48, 192, 192),\n Fire(384, 48, 192, 192),\n Fire(384, 64, 256, 256),\n Fire(512, 64, 256, 256),\n )\n\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.fc = self._construct_fc_layer(fc_dims, 512, dropout_p)\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n self._init_params()\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n \"\"\"Constructs fully connected layer\n\n Args:\n fc_dims (list or tuple): dimensions of fc layers, if None, no fc layers are constructed\n input_dim (int): input dimension\n dropout_p (float): dropout probability, if None, dropout is unused\n \"\"\"\n if fc_dims is None:\n self.feature_dim = input_dim\n return None\n\n assert isinstance(\n fc_dims, (list, tuple)\n ), 'fc_dims must be either list or tuple, but got {}'.format(\n type(fc_dims)\n )\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def forward(self, x):\n f = self.features(x)\n v = self.global_avgpool(f)\n v = v.view(v.size(0), -1)\n\n if self.fc is not None:\n v = self.fc(v)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initializes model with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url, map_location=None)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef squeezenet1_0(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = SqueezeNet(\n num_classes, loss, version=1.0, fc_dims=None, dropout_p=None, **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['squeezenet1_0'])\n return model\n\n\ndef squeezenet1_0_fc512(\n num_classes, loss='softmax', pretrained=True, **kwargs\n):\n model = SqueezeNet(\n num_classes,\n loss,\n version=1.0,\n fc_dims=[512],\n dropout_p=None,\n **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['squeezenet1_0'])\n return model\n\n\ndef squeezenet1_1(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = SqueezeNet(\n num_classes, loss, version=1.1, fc_dims=None, dropout_p=None, **kwargs\n )\n if pretrained:\n init_pretrained_weights(model, model_urls['squeezenet1_1'])\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/models/xception.py", "content": "from __future__ import division, absolute_import\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.model_zoo as model_zoo\n\n__all__ = ['xception']\n\npretrained_settings = {\n 'xception': {\n 'imagenet': {\n 'url':\n 'http://data.lip6.fr/cadene/pretrainedmodels/xception-43020ad28.pth',\n 'input_space': 'RGB',\n 'input_size': [3, 299, 299],\n 'input_range': [0, 1],\n 'mean': [0.5, 0.5, 0.5],\n 'std': [0.5, 0.5, 0.5],\n 'num_classes': 1000,\n 'scale':\n 0.8975 # The resize parameter of the validation transform should be 333, and make sure to center crop at 299x299\n }\n }\n}\n\n\nclass SeparableConv2d(nn.Module):\n\n def __init__(\n self,\n in_channels,\n out_channels,\n kernel_size=1,\n stride=1,\n padding=0,\n dilation=1,\n bias=False\n ):\n super(SeparableConv2d, self).__init__()\n\n self.conv1 = nn.Conv2d(\n in_channels,\n in_channels,\n kernel_size,\n stride,\n padding,\n dilation,\n groups=in_channels,\n bias=bias\n )\n self.pointwise = nn.Conv2d(\n in_channels, out_channels, 1, 1, 0, 1, 1, bias=bias\n )\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.pointwise(x)\n return x\n\n\nclass Block(nn.Module):\n\n def __init__(\n self,\n in_filters,\n out_filters,\n reps,\n strides=1,\n start_with_relu=True,\n grow_first=True\n ):\n super(Block, self).__init__()\n\n if out_filters != in_filters or strides != 1:\n self.skip = nn.Conv2d(\n in_filters, out_filters, 1, stride=strides, bias=False\n )\n self.skipbn = nn.BatchNorm2d(out_filters)\n else:\n self.skip = None\n\n self.relu = nn.ReLU(inplace=True)\n rep = []\n\n filters = in_filters\n if grow_first:\n rep.append(self.relu)\n rep.append(\n SeparableConv2d(\n in_filters,\n out_filters,\n 3,\n stride=1,\n padding=1,\n bias=False\n )\n )\n rep.append(nn.BatchNorm2d(out_filters))\n filters = out_filters\n\n for i in range(reps - 1):\n rep.append(self.relu)\n rep.append(\n SeparableConv2d(\n filters, filters, 3, stride=1, padding=1, bias=False\n )\n )\n rep.append(nn.BatchNorm2d(filters))\n\n if not grow_first:\n rep.append(self.relu)\n rep.append(\n SeparableConv2d(\n in_filters,\n out_filters,\n 3,\n stride=1,\n padding=1,\n bias=False\n )\n )\n rep.append(nn.BatchNorm2d(out_filters))\n\n if not start_with_relu:\n rep = rep[1:]\n else:\n rep[0] = nn.ReLU(inplace=False)\n\n if strides != 1:\n rep.append(nn.MaxPool2d(3, strides, 1))\n self.rep = nn.Sequential(*rep)\n\n def forward(self, inp):\n x = self.rep(inp)\n\n if self.skip is not None:\n skip = self.skip(inp)\n skip = self.skipbn(skip)\n else:\n skip = inp\n\n x += skip\n return x\n\n\nclass Xception(nn.Module):\n \"\"\"Xception.\n \n Reference:\n Chollet. Xception: Deep Learning with Depthwise\n Separable Convolutions. CVPR 2017.\n\n Public keys:\n - ``xception``: Xception.\n \"\"\"\n\n def __init__(\n self, num_classes, loss, fc_dims=None, dropout_p=None, **kwargs\n ):\n super(Xception, self).__init__()\n self.loss = loss\n\n self.conv1 = nn.Conv2d(3, 32, 3, 2, 0, bias=False)\n self.bn1 = nn.BatchNorm2d(32)\n\n self.conv2 = nn.Conv2d(32, 64, 3, bias=False)\n self.bn2 = nn.BatchNorm2d(64)\n\n self.block1 = Block(\n 64, 128, 2, 2, start_with_relu=False, grow_first=True\n )\n self.block2 = Block(\n 128, 256, 2, 2, start_with_relu=True, grow_first=True\n )\n self.block3 = Block(\n 256, 728, 2, 2, start_with_relu=True, grow_first=True\n )\n\n self.block4 = Block(\n 728, 728, 3, 1, start_with_relu=True, grow_first=True\n )\n self.block5 = Block(\n 728, 728, 3, 1, start_with_relu=True, grow_first=True\n )\n self.block6 = Block(\n 728, 728, 3, 1, start_with_relu=True, grow_first=True\n )\n self.block7 = Block(\n 728, 728, 3, 1, start_with_relu=True, grow_first=True\n )\n\n self.block8 = Block(\n 728, 728, 3, 1, start_with_relu=True, grow_first=True\n )\n self.block9 = Block(\n 728, 728, 3, 1, start_with_relu=True, grow_first=True\n )\n self.block10 = Block(\n 728, 728, 3, 1, start_with_relu=True, grow_first=True\n )\n self.block11 = Block(\n 728, 728, 3, 1, start_with_relu=True, grow_first=True\n )\n\n self.block12 = Block(\n 728, 1024, 2, 2, start_with_relu=True, grow_first=False\n )\n\n self.conv3 = SeparableConv2d(1024, 1536, 3, 1, 1)\n self.bn3 = nn.BatchNorm2d(1536)\n\n self.conv4 = SeparableConv2d(1536, 2048, 3, 1, 1)\n self.bn4 = nn.BatchNorm2d(2048)\n\n self.global_avgpool = nn.AdaptiveAvgPool2d(1)\n self.feature_dim = 2048\n self.fc = self._construct_fc_layer(fc_dims, 2048, dropout_p)\n self.classifier = nn.Linear(self.feature_dim, num_classes)\n\n self._init_params()\n\n def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None):\n \"\"\"Constructs fully connected layer.\n\n Args:\n fc_dims (list or tuple): dimensions of fc layers, if None, no fc layers are constructed\n input_dim (int): input dimension\n dropout_p (float): dropout probability, if None, dropout is unused\n \"\"\"\n if fc_dims is None:\n self.feature_dim = input_dim\n return None\n\n assert isinstance(\n fc_dims, (list, tuple)\n ), 'fc_dims must be either list or tuple, but got {}'.format(\n type(fc_dims)\n )\n\n layers = []\n for dim in fc_dims:\n layers.append(nn.Linear(input_dim, dim))\n layers.append(nn.BatchNorm1d(dim))\n layers.append(nn.ReLU(inplace=True))\n if dropout_p is not None:\n layers.append(nn.Dropout(p=dropout_p))\n input_dim = dim\n\n self.feature_dim = fc_dims[-1]\n\n return nn.Sequential(*layers)\n\n def _init_params(self):\n for m in self.modules():\n if isinstance(m, nn.Conv2d):\n nn.init.kaiming_normal_(\n m.weight, mode='fan_out', nonlinearity='relu'\n )\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm2d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.BatchNorm1d):\n nn.init.constant_(m.weight, 1)\n nn.init.constant_(m.bias, 0)\n elif isinstance(m, nn.Linear):\n nn.init.normal_(m.weight, 0, 0.01)\n if m.bias is not None:\n nn.init.constant_(m.bias, 0)\n\n def featuremaps(self, input):\n x = self.conv1(input)\n x = self.bn1(x)\n x = F.relu(x, inplace=True)\n\n x = self.conv2(x)\n x = self.bn2(x)\n x = F.relu(x, inplace=True)\n\n x = self.block1(x)\n x = self.block2(x)\n x = self.block3(x)\n x = self.block4(x)\n x = self.block5(x)\n x = self.block6(x)\n x = self.block7(x)\n x = self.block8(x)\n x = self.block9(x)\n x = self.block10(x)\n x = self.block11(x)\n x = self.block12(x)\n\n x = self.conv3(x)\n x = self.bn3(x)\n x = F.relu(x, inplace=True)\n\n x = self.conv4(x)\n x = self.bn4(x)\n x = F.relu(x, inplace=True)\n return x\n\n def forward(self, x):\n f = self.featuremaps(x)\n v = self.global_avgpool(f)\n v = v.view(v.size(0), -1)\n\n if self.fc is not None:\n v = self.fc(v)\n\n if not self.training:\n return v\n\n y = self.classifier(v)\n\n if self.loss == 'softmax':\n return y\n elif self.loss == 'triplet':\n return y, v\n else:\n raise KeyError('Unsupported loss: {}'.format(self.loss))\n\n\ndef init_pretrained_weights(model, model_url):\n \"\"\"Initialize models with pretrained weights.\n \n Layers that don't match with pretrained layers in name or size are kept unchanged.\n \"\"\"\n pretrain_dict = model_zoo.load_url(model_url)\n model_dict = model.state_dict()\n pretrain_dict = {\n k: v\n for k, v in pretrain_dict.items()\n if k in model_dict and model_dict[k].size() == v.size()\n }\n model_dict.update(pretrain_dict)\n model.load_state_dict(model_dict)\n\n\ndef xception(num_classes, loss='softmax', pretrained=True, **kwargs):\n model = Xception(num_classes, loss, fc_dims=None, dropout_p=None, **kwargs)\n if pretrained:\n model_url = pretrained_settings['xception']['imagenet']['url']\n init_pretrained_weights(model, model_url)\n return model\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/optim/__init__.py", "content": "from __future__ import absolute_import\n\nfrom .optimizer import build_optimizer\nfrom .lr_scheduler import build_lr_scheduler\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/optim/lr_scheduler.py", "content": "from __future__ import print_function, absolute_import\nimport torch\n\nAVAI_SCH = ['single_step', 'multi_step', 'cosine']\n\n\ndef build_lr_scheduler(\n optimizer, lr_scheduler='single_step', stepsize=1, gamma=0.1, max_epoch=1\n):\n \"\"\"A function wrapper for building a learning rate scheduler.\n\n Args:\n optimizer (Optimizer): an Optimizer.\n lr_scheduler (str, optional): learning rate scheduler method. Default is single_step.\n stepsize (int or list, optional): step size to decay learning rate. When ``lr_scheduler``\n is \"single_step\", ``stepsize`` should be an integer. When ``lr_scheduler`` is\n \"multi_step\", ``stepsize`` is a list. Default is 1.\n gamma (float, optional): decay rate. Default is 0.1.\n max_epoch (int, optional): maximum epoch (for cosine annealing). Default is 1.\n\n Examples::\n >>> # Decay learning rate by every 20 epochs.\n >>> scheduler = torchreid.optim.build_lr_scheduler(\n >>> optimizer, lr_scheduler='single_step', stepsize=20\n >>> )\n >>> # Decay learning rate at 30, 50 and 55 epochs.\n >>> scheduler = torchreid.optim.build_lr_scheduler(\n >>> optimizer, lr_scheduler='multi_step', stepsize=[30, 50, 55]\n >>> )\n \"\"\"\n if lr_scheduler not in AVAI_SCH:\n raise ValueError(\n 'Unsupported scheduler: {}. Must be one of {}'.format(\n lr_scheduler, AVAI_SCH\n )\n )\n\n if lr_scheduler == 'single_step':\n if isinstance(stepsize, list):\n stepsize = stepsize[-1]\n\n if not isinstance(stepsize, int):\n raise TypeError(\n 'For single_step lr_scheduler, stepsize must '\n 'be an integer, but got {}'.format(type(stepsize))\n )\n\n scheduler = torch.optim.lr_scheduler.StepLR(\n optimizer, step_size=stepsize, gamma=gamma\n )\n\n elif lr_scheduler == 'multi_step':\n if not isinstance(stepsize, list):\n raise TypeError(\n 'For multi_step lr_scheduler, stepsize must '\n 'be a list, but got {}'.format(type(stepsize))\n )\n\n scheduler = torch.optim.lr_scheduler.MultiStepLR(\n optimizer, milestones=stepsize, gamma=gamma\n )\n\n elif lr_scheduler == 'cosine':\n scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n optimizer, float(max_epoch)\n )\n\n return scheduler\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/optim/optimizer.py", "content": "from __future__ import print_function, absolute_import\nimport warnings\nimport torch\nimport torch.nn as nn\n\nfrom .radam import RAdam\n\nAVAI_OPTIMS = ['adam', 'amsgrad', 'sgd', 'rmsprop', 'radam']\n\n\ndef build_optimizer(\n model,\n optim='adam',\n lr=0.0003,\n weight_decay=5e-04,\n momentum=0.9,\n sgd_dampening=0,\n sgd_nesterov=False,\n rmsprop_alpha=0.99,\n adam_beta1=0.9,\n adam_beta2=0.99,\n staged_lr=False,\n new_layers='',\n base_lr_mult=0.1\n):\n \"\"\"A function wrapper for building an optimizer.\n\n Args:\n model (nn.Module): model.\n optim (str, optional): optimizer. Default is \"adam\".\n lr (float, optional): learning rate. Default is 0.0003.\n weight_decay (float, optional): weight decay (L2 penalty). Default is 5e-04.\n momentum (float, optional): momentum factor in sgd. Default is 0.9,\n sgd_dampening (float, optional): dampening for momentum. Default is 0.\n sgd_nesterov (bool, optional): enables Nesterov momentum. Default is False.\n rmsprop_alpha (float, optional): smoothing constant for rmsprop. Default is 0.99.\n adam_beta1 (float, optional): beta-1 value in adam. Default is 0.9.\n adam_beta2 (float, optional): beta-2 value in adam. Default is 0.99,\n staged_lr (bool, optional): uses different learning rates for base and new layers. Base\n layers are pretrained layers while new layers are randomly initialized, e.g. the\n identity classification layer. Enabling ``staged_lr`` can allow the base layers to\n be trained with a smaller learning rate determined by ``base_lr_mult``, while the new\n layers will take the ``lr``. Default is False.\n new_layers (str or list): attribute names in ``model``. Default is empty.\n base_lr_mult (float, optional): learning rate multiplier for base layers. Default is 0.1.\n\n Examples::\n >>> # A normal optimizer can be built by\n >>> optimizer = torchreid.optim.build_optimizer(model, optim='sgd', lr=0.01)\n >>> # If you want to use a smaller learning rate for pretrained layers\n >>> # and the attribute name for the randomly initialized layer is 'classifier',\n >>> # you can do\n >>> optimizer = torchreid.optim.build_optimizer(\n >>> model, optim='sgd', lr=0.01, staged_lr=True,\n >>> new_layers='classifier', base_lr_mult=0.1\n >>> )\n >>> # Now the `classifier` has learning rate 0.01 but the base layers\n >>> # have learning rate 0.01 * 0.1.\n >>> # new_layers can also take multiple attribute names. Say the new layers\n >>> # are 'fc' and 'classifier', you can do\n >>> optimizer = torchreid.optim.build_optimizer(\n >>> model, optim='sgd', lr=0.01, staged_lr=True,\n >>> new_layers=['fc', 'classifier'], base_lr_mult=0.1\n >>> )\n \"\"\"\n if optim not in AVAI_OPTIMS:\n raise ValueError(\n 'Unsupported optim: {}. Must be one of {}'.format(\n optim, AVAI_OPTIMS\n )\n )\n\n if not isinstance(model, nn.Module):\n raise TypeError(\n 'model given to build_optimizer must be an instance of nn.Module'\n )\n\n if staged_lr:\n if isinstance(new_layers, str):\n if new_layers is None:\n warnings.warn(\n 'new_layers is empty, therefore, staged_lr is useless'\n )\n new_layers = [new_layers]\n\n if isinstance(model, nn.DataParallel):\n model = model.module\n\n base_params = []\n base_layers = []\n new_params = []\n\n for name, module in model.named_children():\n if name in new_layers:\n new_params += [p for p in module.parameters()]\n else:\n base_params += [p for p in module.parameters()]\n base_layers.append(name)\n\n param_groups = [\n {\n 'params': base_params,\n 'lr': lr * base_lr_mult\n },\n {\n 'params': new_params\n },\n ]\n\n else:\n param_groups = model.parameters()\n\n if optim == 'adam':\n optimizer = torch.optim.Adam(\n param_groups,\n lr=lr,\n weight_decay=weight_decay,\n betas=(adam_beta1, adam_beta2),\n )\n\n elif optim == 'amsgrad':\n optimizer = torch.optim.Adam(\n param_groups,\n lr=lr,\n weight_decay=weight_decay,\n betas=(adam_beta1, adam_beta2),\n amsgrad=True,\n )\n\n elif optim == 'sgd':\n optimizer = torch.optim.SGD(\n param_groups,\n lr=lr,\n momentum=momentum,\n weight_decay=weight_decay,\n dampening=sgd_dampening,\n nesterov=sgd_nesterov,\n )\n\n elif optim == 'rmsprop':\n optimizer = torch.optim.RMSprop(\n param_groups,\n lr=lr,\n momentum=momentum,\n weight_decay=weight_decay,\n alpha=rmsprop_alpha,\n )\n\n elif optim == 'radam':\n optimizer = RAdam(\n param_groups,\n lr=lr,\n weight_decay=weight_decay,\n betas=(adam_beta1, adam_beta2)\n )\n\n return optimizer\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/optim/radam.py", "content": "\"\"\"\nImported from: https://github.com/LiyuanLucasLiu/RAdam\n\nPaper: https://arxiv.org/abs/1908.03265\n\n@article{liu2019radam,\n title={On the Variance of the Adaptive Learning Rate and Beyond},\n author={Liu, Liyuan and Jiang, Haoming and He, Pengcheng and Chen, Weizhu and Liu, Xiaodong and Gao, Jianfeng and Han, Jiawei},\n journal={arXiv preprint arXiv:1908.03265},\n year={2019}\n}\n\"\"\"\nfrom __future__ import print_function, absolute_import\nimport math\nimport torch\nfrom torch.optim.optimizer import Optimizer\n\n\nclass RAdam(Optimizer):\n\n def __init__(\n self,\n params,\n lr=1e-3,\n betas=(0.9, 0.999),\n eps=1e-8,\n weight_decay=0,\n degenerated_to_sgd=True\n ):\n if not 0.0 <= lr:\n raise ValueError(\"Invalid learning rate: {}\".format(lr))\n if not 0.0 <= eps:\n raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n if not 0.0 <= betas[0] < 1.0:\n raise ValueError(\n \"Invalid beta parameter at index 0: {}\".format(betas[0])\n )\n if not 0.0 <= betas[1] < 1.0:\n raise ValueError(\n \"Invalid beta parameter at index 1: {}\".format(betas[1])\n )\n\n self.degenerated_to_sgd = degenerated_to_sgd\n defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n self.buffer = [[None, None, None] for ind in range(10)]\n super(RAdam, self).__init__(params, defaults)\n\n def __setstate__(self, state):\n super(RAdam, self).__setstate__(state)\n\n def step(self, closure=None):\n\n loss = None\n if closure is not None:\n loss = closure()\n\n for group in self.param_groups:\n\n for p in group['params']:\n if p.grad is None:\n continue\n grad = p.grad.data.float()\n if grad.is_sparse:\n raise RuntimeError(\n 'RAdam does not support sparse gradients'\n )\n\n p_data_fp32 = p.data.float()\n\n state = self.state[p]\n\n if len(state) == 0:\n state['step'] = 0\n state['exp_avg'] = torch.zeros_like(p_data_fp32)\n state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n else:\n state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n state['exp_avg_sq'] = state['exp_avg_sq'].type_as(\n p_data_fp32\n )\n\n exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n beta1, beta2 = group['betas']\n\n exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n state['step'] += 1\n buffered = self.buffer[int(state['step'] % 10)]\n if state['step'] == buffered[0]:\n N_sma, step_size = buffered[1], buffered[2]\n else:\n buffered[0] = state['step']\n beta2_t = beta2**state['step']\n N_sma_max = 2 / (1-beta2) - 1\n N_sma = N_sma_max - 2 * state['step'\n ] * beta2_t / (1-beta2_t)\n buffered[1] = N_sma\n\n # more conservative since it's an approximated value\n if N_sma >= 5:\n step_size = math.sqrt(\n (1-beta2_t) * (N_sma-4) / (N_sma_max-4) *\n (N_sma-2) / N_sma * N_sma_max / (N_sma_max-2)\n ) / (1 - beta1**state['step'])\n elif self.degenerated_to_sgd:\n step_size = 1.0 / (1 - beta1**state['step'])\n else:\n step_size = -1\n buffered[2] = step_size\n\n # more conservative since it's an approximated value\n if N_sma >= 5:\n if group['weight_decay'] != 0:\n p_data_fp32.add_(\n -group['weight_decay'] * group['lr'], p_data_fp32\n )\n denom = exp_avg_sq.sqrt().add_(group['eps'])\n p_data_fp32.addcdiv_(\n -step_size * group['lr'], exp_avg, denom\n )\n p.data.copy_(p_data_fp32)\n elif step_size > 0:\n if group['weight_decay'] != 0:\n p_data_fp32.add_(\n -group['weight_decay'] * group['lr'], p_data_fp32\n )\n p_data_fp32.add_(-step_size * group['lr'], exp_avg)\n p.data.copy_(p_data_fp32)\n\n return loss\n\n\nclass PlainRAdam(Optimizer):\n\n def __init__(\n self,\n params,\n lr=1e-3,\n betas=(0.9, 0.999),\n eps=1e-8,\n weight_decay=0,\n degenerated_to_sgd=True\n ):\n if not 0.0 <= lr:\n raise ValueError(\"Invalid learning rate: {}\".format(lr))\n if not 0.0 <= eps:\n raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n if not 0.0 <= betas[0] < 1.0:\n raise ValueError(\n \"Invalid beta parameter at index 0: {}\".format(betas[0])\n )\n if not 0.0 <= betas[1] < 1.0:\n raise ValueError(\n \"Invalid beta parameter at index 1: {}\".format(betas[1])\n )\n\n self.degenerated_to_sgd = degenerated_to_sgd\n defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n\n super(PlainRAdam, self).__init__(params, defaults)\n\n def __setstate__(self, state):\n super(PlainRAdam, self).__setstate__(state)\n\n def step(self, closure=None):\n\n loss = None\n if closure is not None:\n loss = closure()\n\n for group in self.param_groups:\n\n for p in group['params']:\n if p.grad is None:\n continue\n grad = p.grad.data.float()\n if grad.is_sparse:\n raise RuntimeError(\n 'RAdam does not support sparse gradients'\n )\n\n p_data_fp32 = p.data.float()\n\n state = self.state[p]\n\n if len(state) == 0:\n state['step'] = 0\n state['exp_avg'] = torch.zeros_like(p_data_fp32)\n state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n else:\n state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n state['exp_avg_sq'] = state['exp_avg_sq'].type_as(\n p_data_fp32\n )\n\n exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n beta1, beta2 = group['betas']\n\n exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n state['step'] += 1\n beta2_t = beta2**state['step']\n N_sma_max = 2 / (1-beta2) - 1\n N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1-beta2_t)\n\n # more conservative since it's an approximated value\n if N_sma >= 5:\n if group['weight_decay'] != 0:\n p_data_fp32.add_(\n -group['weight_decay'] * group['lr'], p_data_fp32\n )\n step_size = group['lr'] * math.sqrt(\n (1-beta2_t) * (N_sma-4) / (N_sma_max-4) *\n (N_sma-2) / N_sma * N_sma_max / (N_sma_max-2)\n ) / (1 - beta1**state['step'])\n denom = exp_avg_sq.sqrt().add_(group['eps'])\n p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n p.data.copy_(p_data_fp32)\n elif self.degenerated_to_sgd:\n if group['weight_decay'] != 0:\n p_data_fp32.add_(\n -group['weight_decay'] * group['lr'], p_data_fp32\n )\n step_size = group['lr'] / (1 - beta1**state['step'])\n p_data_fp32.add_(-step_size, exp_avg)\n p.data.copy_(p_data_fp32)\n\n return loss\n\n\nclass AdamW(Optimizer):\n\n def __init__(\n self,\n params,\n lr=1e-3,\n betas=(0.9, 0.999),\n eps=1e-8,\n weight_decay=0,\n warmup=0\n ):\n if not 0.0 <= lr:\n raise ValueError(\"Invalid learning rate: {}\".format(lr))\n if not 0.0 <= eps:\n raise ValueError(\"Invalid epsilon value: {}\".format(eps))\n if not 0.0 <= betas[0] < 1.0:\n raise ValueError(\n \"Invalid beta parameter at index 0: {}\".format(betas[0])\n )\n if not 0.0 <= betas[1] < 1.0:\n raise ValueError(\n \"Invalid beta parameter at index 1: {}\".format(betas[1])\n )\n\n defaults = dict(\n lr=lr,\n betas=betas,\n eps=eps,\n weight_decay=weight_decay,\n warmup=warmup\n )\n super(AdamW, self).__init__(params, defaults)\n\n def __setstate__(self, state):\n super(AdamW, self).__setstate__(state)\n\n def step(self, closure=None):\n loss = None\n if closure is not None:\n loss = closure()\n\n for group in self.param_groups:\n\n for p in group['params']:\n if p.grad is None:\n continue\n grad = p.grad.data.float()\n if grad.is_sparse:\n raise RuntimeError(\n 'Adam does not support sparse gradients, please consider SparseAdam instead'\n )\n\n p_data_fp32 = p.data.float()\n\n state = self.state[p]\n\n if len(state) == 0:\n state['step'] = 0\n state['exp_avg'] = torch.zeros_like(p_data_fp32)\n state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n else:\n state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n state['exp_avg_sq'] = state['exp_avg_sq'].type_as(\n p_data_fp32\n )\n\n exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n beta1, beta2 = group['betas']\n\n state['step'] += 1\n\n exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n denom = exp_avg_sq.sqrt().add_(group['eps'])\n bias_correction1 = 1 - beta1**state['step']\n bias_correction2 = 1 - beta2**state['step']\n\n if group['warmup'] > state['step']:\n scheduled_lr = 1e-8 + state['step'] * group['lr'] / group[\n 'warmup']\n else:\n scheduled_lr = group['lr']\n\n step_size = scheduled_lr * math.sqrt(\n bias_correction2\n ) / bias_correction1\n\n if group['weight_decay'] != 0:\n p_data_fp32.add_(\n -group['weight_decay'] * scheduled_lr, p_data_fp32\n )\n\n p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n\n p.data.copy_(p_data_fp32)\n\n return loss\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/README.md", "content": "# Understanding Image Retrieval Re-Ranking: A Graph Neural Network Perspective\n\n[[Paper]](https://arxiv.org/abs/2012.07620v2)\n\nOn the Market-1501 dataset, we accelerate the re-ranking processing from **89.2s** to **9.4ms** with one K40m GPU, facilitating the real-time post-processing. \nSimilarly, we observe that our method achieves comparable or even better retrieval results on the other four image retrieval benchmarks, \ni.e., VeRi-776, Oxford-5k, Paris-6k and University-1652, with limited time cost.\n\n## Prerequisites\n\nThe code was mainly developed and tested with python 3.7, PyTorch 1.4.1, CUDA 10.2, and CentOS release 6.10.\n\nThe code has been included in `/extension`. To compile it:\n\n```shell\ncd extension\nsh make.sh\n```\n\n## Demo\n\nThe demo script `main.py` provides the gnn re-ranking method using the prepared feature. \n\n```shell\npython main.py --data_path PATH_TO_DATA --k1 26 --k2 7\n```\n\n## Citation\n```bibtex\n@article{zhang2020understanding,\n title={Understanding Image Retrieval Re-Ranking: A Graph Neural Network Perspective},\n author={Xuanmeng Zhang, Minyue Jiang, Zhedong Zheng, Xiao Tan, Errui Ding, Yi Yang},\n journal={arXiv preprint arXiv:2012.07620},\n year={2020}\n}\n```\n\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/extension/adjacency_matrix/build_adjacency_matrix.cpp", "content": "#include \n#include \n#include \n\nat::Tensor build_adjacency_matrix_forward(torch::Tensor initial_rank);\n\n\n#define CHECK_CUDA(x) AT_ASSERTM(x.type().is_cuda(), #x \" must be a CUDA tensor\")\n#define CHECK_CONTIGUOUS(x) AT_ASSERTM(x.is_contiguous(), #x \" must be contiguous\")\n#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)\n\nat::Tensor build_adjacency_matrix(at::Tensor initial_rank) {\n CHECK_INPUT(initial_rank);\n return build_adjacency_matrix_forward(initial_rank);\n}\n\nPYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {\n m.def(\"forward\", &build_adjacency_matrix, \"build_adjacency_matrix (CUDA)\");\n}\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/extension/adjacency_matrix/build_adjacency_matrix_kernel.cu", "content": "#include \n\n#include \n#include \n#include \n\n#define CUDA_1D_KERNEL_LOOP(i, n) for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += blockDim.x * gridDim.x)\n\n\n__global__ void build_adjacency_matrix_kernel(float* initial_rank, float* A, const int total_num, const int topk, const int nthreads, const int all_num) {\n int index = blockIdx.x * blockDim.x + threadIdx.x;\n int stride = blockDim.x * gridDim.x;\n for (int i = index; i < all_num; i += stride) {\n int ii = i / topk;\n A[ii * total_num + int(initial_rank[i])] = float(1.0);\n }\n}\n\nat::Tensor build_adjacency_matrix_forward(at::Tensor initial_rank) {\n const auto total_num = initial_rank.size(0);\n const auto topk = initial_rank.size(1);\n const auto all_num = total_num * topk;\n auto A = torch::zeros({total_num, total_num}, at::device(initial_rank.device()).dtype(at::ScalarType::Float));\n\n const int threads = 1024;\n const int blocks = (all_num + threads - 1) / threads;\n\n build_adjacency_matrix_kernel<<>>(initial_rank.data_ptr(), A.data_ptr(), total_num, topk, threads, all_num);\n return A;\n\n}\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/extension/adjacency_matrix/setup.py", "content": "\"\"\"\n Understanding Image Retrieval Re-Ranking: A Graph Neural Network Perspective\n\n Xuanmeng Zhang, Minyue Jiang, Zhedong Zheng, Xiao Tan, Errui Ding, Yi Yang\n\n Project Page : https://github.com/Xuanmeng-Zhang/gnn-re-ranking\n\n Paper: https://arxiv.org/abs/2012.07620v2\n\n ======================================================================\n \n On the Market-1501 dataset, we accelerate the re-ranking processing from 89.2s to 9.4ms\n with one K40m GPU, facilitating the real-time post-processing. Similarly, we observe \n that our method achieves comparable or even better retrieval results on the other four \n image retrieval benchmarks, i.e., VeRi-776, Oxford-5k, Paris-6k and University-1652, \n with limited time cost.\n\"\"\"\n\nfrom setuptools import Extension, setup\nimport torch\nimport torch.nn as nn\nfrom torch.autograd import Function\nfrom torch.utils.cpp_extension import CUDAExtension, BuildExtension\n\nsetup(\n name='build_adjacency_matrix',\n ext_modules=[\n CUDAExtension(\n 'build_adjacency_matrix', [\n 'build_adjacency_matrix.cpp',\n 'build_adjacency_matrix_kernel.cu',\n ]\n ),\n ],\n cmdclass={'build_ext': BuildExtension}\n)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/extension/make.sh", "content": "cd adjacency_matrix\npython setup.py install\ncd ../propagation\npython setup.py install" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/extension/propagation/gnn_propagate.cpp", "content": "#include \n#include \n#include \n\nat::Tensor gnn_propagate_forward(at::Tensor A, at::Tensor initial_rank, at::Tensor S);\n\n\n#define CHECK_CUDA(x) AT_ASSERTM(x.type().is_cuda(), #x \" must be a CUDA tensor\")\n#define CHECK_CONTIGUOUS(x) AT_ASSERTM(x.is_contiguous(), #x \" must be contiguous\")\n#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)\n\nat::Tensor gnn_propagate(at::Tensor A ,at::Tensor initial_rank, at::Tensor S) {\n CHECK_INPUT(A);\n CHECK_INPUT(initial_rank);\n CHECK_INPUT(S);\n return gnn_propagate_forward(A, initial_rank, S);\n}\n\nPYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {\n m.def(\"forward\", &gnn_propagate, \"gnn propagate (CUDA)\");\n}" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/extension/propagation/gnn_propagate_kernel.cu", "content": "#include \n\n#include \n#include \n#include \n#include \n\n__global__ void gnn_propagate_forward_kernel(float* initial_rank, float* A, float* A_qe, float* S, const int sample_num, const int topk, const int total_num) {\n int index = blockIdx.x * blockDim.x + threadIdx.x;\n int stride = blockDim.x * gridDim.x;\n for (int i = index; i < total_num; i += stride) {\n int fea = i % sample_num;\n int sample_index = i / sample_num;\n float sum = 0.0;\n for (int j = 0; j < topk ; j++) {\n int topk_fea_index = int(initial_rank[sample_index*topk+j]) * sample_num + fea;\n sum += A[ topk_fea_index] * S[sample_index*topk+j];\n }\n A_qe[i] = sum;\n }\n}\n\nat::Tensor gnn_propagate_forward(at::Tensor A, at::Tensor initial_rank, at::Tensor S) {\n const auto sample_num = A.size(0);\n const auto topk = initial_rank.size(1);\n\n const auto total_num = sample_num * sample_num ; \n auto A_qe = torch::zeros({sample_num, sample_num}, at::device(initial_rank.device()).dtype(at::ScalarType::Float));\n\n const int threads = 1024;\n const int blocks = (total_num + threads - 1) / threads;\n\n gnn_propagate_forward_kernel<<>>(initial_rank.data_ptr(), A.data_ptr(), A_qe.data_ptr(), S.data_ptr(), sample_num, topk, total_num);\n return A_qe;\n\n}" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/extension/propagation/setup.py", "content": "\"\"\"\n Understanding Image Retrieval Re-Ranking: A Graph Neural Network Perspective\n\n Xuanmeng Zhang, Minyue Jiang, Zhedong Zheng, Xiao Tan, Errui Ding, Yi Yang\n\n Project Page : https://github.com/Xuanmeng-Zhang/gnn-re-ranking\n\n Paper: https://arxiv.org/abs/2012.07620v2\n\n ======================================================================\n \n On the Market-1501 dataset, we accelerate the re-ranking processing from 89.2s to 9.4ms\n with one K40m GPU, facilitating the real-time post-processing. Similarly, we observe \n that our method achieves comparable or even better retrieval results on the other four \n image retrieval benchmarks, i.e., VeRi-776, Oxford-5k, Paris-6k and University-1652, \n with limited time cost.\n\"\"\"\n\nfrom setuptools import Extension, setup\nimport torch\nimport torch.nn as nn\nfrom torch.autograd import Function\nfrom torch.utils.cpp_extension import CUDAExtension, BuildExtension\n\nsetup(\n name='gnn_propagate',\n ext_modules=[\n CUDAExtension(\n 'gnn_propagate', [\n 'gnn_propagate.cpp',\n 'gnn_propagate_kernel.cu',\n ]\n ),\n ],\n cmdclass={'build_ext': BuildExtension}\n)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/gnn_reranking.py", "content": "\"\"\"\n Understanding Image Retrieval Re-Ranking: A Graph Neural Network Perspective\n\n Xuanmeng Zhang, Minyue Jiang, Zhedong Zheng, Xiao Tan, Errui Ding, Yi Yang\n\n Project Page : https://github.com/Xuanmeng-Zhang/gnn-re-ranking\n\n Paper: https://arxiv.org/abs/2012.07620v2\n\n ======================================================================\n \n On the Market-1501 dataset, we accelerate the re-ranking processing from 89.2s to 9.4ms\n with one K40m GPU, facilitating the real-time post-processing. Similarly, we observe \n that our method achieves comparable or even better retrieval results on the other four \n image retrieval benchmarks, i.e., VeRi-776, Oxford-5k, Paris-6k and University-1652, \n with limited time cost.\n\"\"\"\n\nimport numpy as np\nimport torch\n\nimport gnn_propagate\nimport build_adjacency_matrix\nfrom utils import *\n\n\ndef gnn_reranking(X_q, X_g, k1, k2):\n query_num, gallery_num = X_q.shape[0], X_g.shape[0]\n\n X_u = torch.cat((X_q, X_g), axis=0)\n original_score = torch.mm(X_u, X_u.t())\n del X_u, X_q, X_g\n\n # initial ranking list\n S, initial_rank = original_score.topk(\n k=k1, dim=-1, largest=True, sorted=True\n )\n\n # stage 1\n A = build_adjacency_matrix.forward(initial_rank.float())\n S = S * S\n\n # stage 2\n if k2 != 1:\n for i in range(2):\n A = A + A.T\n A = gnn_propagate.forward(\n A, initial_rank[:, :k2].contiguous().float(),\n S[:, :k2].contiguous().float()\n )\n A_norm = torch.norm(A, p=2, dim=1, keepdim=True)\n A = A.div(A_norm.expand_as(A))\n\n cosine_similarity = torch.mm(A[:query_num, ], A[query_num:, ].t())\n del A, S\n\n L = torch.sort(-cosine_similarity, dim=1)[1]\n L = L.data.cpu().numpy()\n return L\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/main.py", "content": "\"\"\"\n Understanding Image Retrieval Re-Ranking: A Graph Neural Network Perspective\n\n Xuanmeng Zhang, Minyue Jiang, Zhedong Zheng, Xiao Tan, Errui Ding, Yi Yang\n\n Project Page : https://github.com/Xuanmeng-Zhang/gnn-re-ranking\n\n Paper: https://arxiv.org/abs/2012.07620v2\n\n ======================================================================\n \n On the Market-1501 dataset, we accelerate the re-ranking processing from 89.2s to 9.4ms\n with one K40m GPU, facilitating the real-time post-processing. Similarly, we observe \n that our method achieves comparable or even better retrieval results on the other four \n image retrieval benchmarks, i.e., VeRi-776, Oxford-5k, Paris-6k and University-1652, \n with limited time cost.\n\"\"\"\n\nimport os\nimport numpy as np\nimport argparse\nimport torch\n\nfrom utils import *\nfrom gnn_reranking import *\n\nparser = argparse.ArgumentParser(description='Reranking_is_GNN')\nparser.add_argument(\n '--data_path',\n type=str,\n default='../xm_rerank_gpu_2/features/market_88_test.pkl',\n help='path to dataset'\n)\nparser.add_argument(\n '--k1',\n type=int,\n default=26, # Market-1501\n # default=60, # Veri-776\n help='parameter k1'\n)\nparser.add_argument(\n '--k2',\n type=int,\n default=7, # Market-1501\n # default=10, # Veri-776\n help='parameter k2'\n)\n\nargs = parser.parse_args()\n\n\ndef main():\n data = load_pickle(args.data_path)\n\n query_cam = data['query_cam']\n query_label = data['query_label']\n gallery_cam = data['gallery_cam']\n gallery_label = data['gallery_label']\n\n gallery_feature = torch.FloatTensor(data['gallery_f'])\n query_feature = torch.FloatTensor(data['query_f'])\n query_feature = query_feature.cuda()\n gallery_feature = gallery_feature.cuda()\n\n indices = gnn_reranking(query_feature, gallery_feature, args.k1, args.k2)\n evaluate_ranking_list(\n indices, query_label, query_cam, gallery_label, gallery_cam\n )\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/GPU-Re-Ranking/utils.py", "content": "\"\"\"\n Understanding Image Retrieval Re-Ranking: A Graph Neural Network Perspective\n\n Xuanmeng Zhang, Minyue Jiang, Zhedong Zheng, Xiao Tan, Errui Ding, Yi Yang\n\n Project Page : https://github.com/Xuanmeng-Zhang/gnn-re-ranking\n\n Paper: https://arxiv.org/abs/2012.07620v2\n\n ======================================================================\n \n On the Market-1501 dataset, we accelerate the re-ranking processing from 89.2s to 9.4ms\n with one K40m GPU, facilitating the real-time post-processing. Similarly, we observe \n that our method achieves comparable or even better retrieval results on the other four \n image retrieval benchmarks, i.e., VeRi-776, Oxford-5k, Paris-6k and University-1652, \n with limited time cost.\n\"\"\"\n\nimport os\nimport numpy as np\nimport pickle\nimport torch\n\n\ndef load_pickle(pickle_path):\n with open(pickle_path, 'rb') as f:\n data = pickle.load(f)\n return data\n\n\ndef save_pickle(pickle_path, data):\n with open(pickle_path, 'wb') as f:\n pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)\n\n\ndef pairwise_squared_distance(x):\n '''\n x : (n_samples, n_points, dims)\n return : (n_samples, n_points, n_points)\n '''\n x2s = (x * x).sum(-1, keepdim=True)\n return x2s + x2s.transpose(-1, -2) - 2 * x @ x.transpose(-1, -2)\n\n\ndef pairwise_distance(x, y):\n m, n = x.size(0), y.size(0)\n\n x = x.view(m, -1)\n y = y.view(n, -1)\n\n dist = torch.pow(x, 2).sum(\n dim=1, keepdim=True\n ).expand(m, n) + torch.pow(y, 2).sum(\n dim=1, keepdim=True\n ).expand(n, m).t()\n dist.addmm_(1, -2, x, y.t())\n\n return dist\n\n\ndef cosine_similarity(x, y):\n m, n = x.size(0), y.size(0)\n\n x = x.view(m, -1)\n y = y.view(n, -1)\n\n y = y.t()\n score = torch.mm(x, y)\n\n return score\n\n\ndef evaluate_ranking_list(\n indices, query_label, query_cam, gallery_label, gallery_cam\n):\n CMC = np.zeros((len(gallery_label)), dtype=np.int)\n ap = 0.0\n\n for i in range(len(query_label)):\n ap_tmp, CMC_tmp = evaluate(\n indices[i], query_label[i], query_cam[i], gallery_label,\n gallery_cam\n )\n if CMC_tmp[0] == -1:\n continue\n CMC = CMC + CMC_tmp\n ap += ap_tmp\n\n CMC = CMC.astype(np.float32)\n CMC = CMC / len(query_label) #average CMC\n print(\n 'Rank@1:%f Rank@5:%f Rank@10:%f mAP:%f' %\n (CMC[0], CMC[4], CMC[9], ap / len(query_label))\n )\n\n\ndef evaluate(index, ql, qc, gl, gc):\n query_index = np.argwhere(gl == ql)\n camera_index = np.argwhere(gc == qc)\n\n good_index = np.setdiff1d(query_index, camera_index, assume_unique=True)\n junk_index1 = np.argwhere(gl == -1)\n junk_index2 = np.intersect1d(query_index, camera_index)\n junk_index = np.append(junk_index2, junk_index1) #.flatten())\n\n CMC_tmp = compute_mAP(index, good_index, junk_index)\n return CMC_tmp\n\n\ndef compute_mAP(index, good_index, junk_index):\n ap = 0\n cmc = np.zeros((len(index)), dtype=np.int)\n if good_index.size == 0: # if empty\n cmc[0] = -1\n return ap, cmc\n\n # remove junk_index\n mask = np.in1d(index, junk_index, invert=True)\n index = index[mask]\n\n # find good_index index\n ngood = len(good_index)\n mask = np.in1d(index, good_index)\n rows_good = np.argwhere(mask == True)\n rows_good = rows_good.flatten()\n\n cmc[rows_good[0]:] = 1\n for i in range(ngood):\n d_recall = 1.0 / ngood\n precision = (i+1) * 1.0 / (rows_good[i] + 1)\n if rows_good[i] != 0:\n old_precision = i * 1.0 / rows_good[i]\n else:\n old_precision = 1.0\n ap = ap + d_recall * (old_precision+precision) / 2\n\n return ap, cmc\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/__init__.py", "content": "from __future__ import absolute_import\n\nfrom .tools import *\nfrom .rerank import re_ranking\nfrom .loggers import *\nfrom .avgmeter import *\nfrom .reidtools import *\nfrom .torchtools import *\nfrom .model_complexity import compute_model_complexity\nfrom .feature_extractor import FeatureExtractor\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/avgmeter.py", "content": "from __future__ import division, absolute_import\nfrom collections import defaultdict\nimport torch\n\n__all__ = ['AverageMeter', 'MetricMeter']\n\n\nclass AverageMeter(object):\n \"\"\"Computes and stores the average and current value.\n\n Examples::\n >>> # Initialize a meter to record loss\n >>> losses = AverageMeter()\n >>> # Update meter after every minibatch update\n >>> losses.update(loss_value, batch_size)\n \"\"\"\n\n def __init__(self):\n self.reset()\n\n def reset(self):\n self.val = 0\n self.avg = 0\n self.sum = 0\n self.count = 0\n\n def update(self, val, n=1):\n self.val = val\n self.sum += val * n\n self.count += n\n self.avg = self.sum / self.count\n\n\nclass MetricMeter(object):\n \"\"\"A collection of metrics.\n\n Source: https://github.com/KaiyangZhou/Dassl.pytorch\n\n Examples::\n >>> # 1. Create an instance of MetricMeter\n >>> metric = MetricMeter()\n >>> # 2. Update using a dictionary as input\n >>> input_dict = {'loss_1': value_1, 'loss_2': value_2}\n >>> metric.update(input_dict)\n >>> # 3. Convert to string and print\n >>> print(str(metric))\n \"\"\"\n\n def __init__(self, delimiter='\\t'):\n self.meters = defaultdict(AverageMeter)\n self.delimiter = delimiter\n\n def update(self, input_dict):\n if input_dict is None:\n return\n\n if not isinstance(input_dict, dict):\n raise TypeError(\n 'Input to MetricMeter.update() must be a dictionary'\n )\n\n for k, v in input_dict.items():\n if isinstance(v, torch.Tensor):\n v = v.item()\n self.meters[k].update(v)\n\n def __str__(self):\n output_str = []\n for name, meter in self.meters.items():\n output_str.append(\n '{} {:.4f} ({:.4f})'.format(name, meter.val, meter.avg)\n )\n return self.delimiter.join(output_str)\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/feature_extractor.py", "content": "from __future__ import absolute_import\nimport numpy as np\nimport torch\nimport torchvision.transforms as T\nfrom PIL import Image\n\nfrom torchreid.utils import (\n check_isfile, load_pretrained_weights, compute_model_complexity\n)\nfrom torchreid.models import build_model\n\n\nclass FeatureExtractor(object):\n \"\"\"A simple API for feature extraction.\n\n FeatureExtractor can be used like a python function, which\n accepts input of the following types:\n - a list of strings (image paths)\n - a list of numpy.ndarray each with shape (H, W, C)\n - a single string (image path)\n - a single numpy.ndarray with shape (H, W, C)\n - a torch.Tensor with shape (B, C, H, W) or (C, H, W)\n\n Returned is a torch tensor with shape (B, D) where D is the\n feature dimension.\n\n Args:\n model_name (str): model name.\n model_path (str): path to model weights.\n image_size (sequence or int): image height and width.\n pixel_mean (list): pixel mean for normalization.\n pixel_std (list): pixel std for normalization.\n pixel_norm (bool): whether to normalize pixels.\n device (str): 'cpu' or 'cuda' (could be specific gpu devices).\n verbose (bool): show model details.\n\n Examples::\n\n from torchreid.utils import FeatureExtractor\n\n extractor = FeatureExtractor(\n model_name='osnet_x1_0',\n model_path='a/b/c/model.pth.tar',\n device='cuda'\n )\n\n image_list = [\n 'a/b/c/image001.jpg',\n 'a/b/c/image002.jpg',\n 'a/b/c/image003.jpg',\n 'a/b/c/image004.jpg',\n 'a/b/c/image005.jpg'\n ]\n\n features = extractor(image_list)\n print(features.shape) # output (5, 512)\n \"\"\"\n\n def __init__(\n self,\n model_name='',\n model_path='',\n image_size=(256, 128),\n pixel_mean=[0.485, 0.456, 0.406],\n pixel_std=[0.229, 0.224, 0.225],\n pixel_norm=True,\n device='cuda',\n verbose=True\n ):\n # Build model\n model = build_model(\n model_name,\n num_classes=1,\n pretrained=not (model_path and check_isfile(model_path)),\n use_gpu=device.startswith('cuda')\n )\n model.eval()\n\n if verbose:\n num_params, flops = compute_model_complexity(\n model, (1, 3, image_size[0], image_size[1])\n )\n print('Model: {}'.format(model_name))\n print('- params: {:,}'.format(num_params))\n print('- flops: {:,}'.format(flops))\n\n if model_path and check_isfile(model_path):\n load_pretrained_weights(model, model_path)\n\n # Build transform functions\n transforms = []\n transforms += [T.Resize(image_size)]\n transforms += [T.ToTensor()]\n if pixel_norm:\n transforms += [T.Normalize(mean=pixel_mean, std=pixel_std)]\n preprocess = T.Compose(transforms)\n\n to_pil = T.ToPILImage()\n\n device = torch.device(device)\n model.to(device)\n\n # Class attributes\n self.model = model\n self.preprocess = preprocess\n self.to_pil = to_pil\n self.device = device\n\n def __call__(self, input):\n if isinstance(input, list):\n images = []\n\n for element in input:\n if isinstance(element, str):\n image = Image.open(element).convert('RGB')\n\n elif isinstance(element, np.ndarray):\n image = self.to_pil(element)\n\n else:\n raise TypeError(\n 'Type of each element must belong to [str | numpy.ndarray]'\n )\n\n image = self.preprocess(image)\n images.append(image)\n\n images = torch.stack(images, dim=0)\n images = images.to(self.device)\n\n elif isinstance(input, str):\n image = Image.open(input).convert('RGB')\n image = self.preprocess(image)\n images = image.unsqueeze(0).to(self.device)\n\n elif isinstance(input, np.ndarray):\n image = self.to_pil(input)\n image = self.preprocess(image)\n images = image.unsqueeze(0).to(self.device)\n\n elif isinstance(input, torch.Tensor):\n if input.dim() == 3:\n input = input.unsqueeze(0)\n images = input.to(self.device)\n\n else:\n raise NotImplementedError\n\n with torch.no_grad():\n features = self.model(images)\n\n return features\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/loggers.py", "content": "from __future__ import absolute_import\nimport os\nimport sys\nimport os.path as osp\n\nfrom .tools import mkdir_if_missing\n\n__all__ = ['Logger', 'RankLogger']\n\n\nclass Logger(object):\n \"\"\"Writes console output to external text file.\n\n Imported from ``_\n\n Args:\n fpath (str): directory to save logging file.\n\n Examples::\n >>> import sys\n >>> import os\n >>> import os.path as osp\n >>> from torchreid.utils import Logger\n >>> save_dir = 'log/resnet50-softmax-market1501'\n >>> log_name = 'train.log'\n >>> sys.stdout = Logger(osp.join(args.save_dir, log_name))\n \"\"\"\n\n def __init__(self, fpath=None):\n self.console = sys.stdout\n self.file = None\n if fpath is not None:\n mkdir_if_missing(osp.dirname(fpath))\n self.file = open(fpath, 'w')\n\n def __del__(self):\n self.close()\n\n def __enter__(self):\n pass\n\n def __exit__(self, *args):\n self.close()\n\n def write(self, msg):\n self.console.write(msg)\n if self.file is not None:\n self.file.write(msg)\n\n def flush(self):\n self.console.flush()\n if self.file is not None:\n self.file.flush()\n os.fsync(self.file.fileno())\n\n def close(self):\n self.console.close()\n if self.file is not None:\n self.file.close()\n\n\nclass RankLogger(object):\n \"\"\"Records the rank1 matching accuracy obtained for each\n test dataset at specified evaluation steps and provides a function\n to show the summarized results, which are convenient for analysis.\n\n Args:\n sources (str or list): source dataset name(s).\n targets (str or list): target dataset name(s).\n\n Examples::\n >>> from torchreid.utils import RankLogger\n >>> s = 'market1501'\n >>> t = 'market1501'\n >>> ranklogger = RankLogger(s, t)\n >>> ranklogger.write(t, 10, 0.5)\n >>> ranklogger.write(t, 20, 0.7)\n >>> ranklogger.write(t, 30, 0.9)\n >>> ranklogger.show_summary()\n >>> # You will see:\n >>> # => Show performance summary\n >>> # market1501 (source)\n >>> # - epoch 10 rank1 50.0%\n >>> # - epoch 20 rank1 70.0%\n >>> # - epoch 30 rank1 90.0%\n >>> # If there are multiple test datasets\n >>> t = ['market1501', 'dukemtmcreid']\n >>> ranklogger = RankLogger(s, t)\n >>> ranklogger.write(t[0], 10, 0.5)\n >>> ranklogger.write(t[0], 20, 0.7)\n >>> ranklogger.write(t[0], 30, 0.9)\n >>> ranklogger.write(t[1], 10, 0.1)\n >>> ranklogger.write(t[1], 20, 0.2)\n >>> ranklogger.write(t[1], 30, 0.3)\n >>> ranklogger.show_summary()\n >>> # You can see:\n >>> # => Show performance summary\n >>> # market1501 (source)\n >>> # - epoch 10 rank1 50.0%\n >>> # - epoch 20 rank1 70.0%\n >>> # - epoch 30 rank1 90.0%\n >>> # dukemtmcreid (target)\n >>> # - epoch 10 rank1 10.0%\n >>> # - epoch 20 rank1 20.0%\n >>> # - epoch 30 rank1 30.0%\n \"\"\"\n\n def __init__(self, sources, targets):\n self.sources = sources\n self.targets = targets\n\n if isinstance(self.sources, str):\n self.sources = [self.sources]\n\n if isinstance(self.targets, str):\n self.targets = [self.targets]\n\n self.logger = {\n name: {\n 'epoch': [],\n 'rank1': []\n }\n for name in self.targets\n }\n\n def write(self, name, epoch, rank1):\n \"\"\"Writes result.\n\n Args:\n name (str): dataset name.\n epoch (int): current epoch.\n rank1 (float): rank1 result.\n \"\"\"\n self.logger[name]['epoch'].append(epoch)\n self.logger[name]['rank1'].append(rank1)\n\n def show_summary(self):\n \"\"\"Shows saved results.\"\"\"\n print('=> Show performance summary')\n for name in self.targets:\n from_where = 'source' if name in self.sources else 'target'\n print('{} ({})'.format(name, from_where))\n for epoch, rank1 in zip(\n self.logger[name]['epoch'], self.logger[name]['rank1']\n ):\n print('- epoch {}\\t rank1 {:.1%}'.format(epoch, rank1))\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/model_complexity.py", "content": "from __future__ import division, print_function, absolute_import\nimport math\nimport numpy as np\nfrom itertools import repeat\nfrom collections import namedtuple, defaultdict\nimport torch\n\n__all__ = ['compute_model_complexity']\n\"\"\"\nUtility\n\"\"\"\n\n\ndef _ntuple(n):\n\n def parse(x):\n if isinstance(x, int):\n return tuple(repeat(x, n))\n return x\n\n return parse\n\n\n_single = _ntuple(1)\n_pair = _ntuple(2)\n_triple = _ntuple(3)\n\"\"\"\nConvolution\n\"\"\"\n\n\ndef hook_convNd(m, x, y):\n k = torch.prod(torch.Tensor(m.kernel_size)).item()\n cin = m.in_channels\n flops_per_ele = k * cin # + (k*cin-1)\n if m.bias is not None:\n flops_per_ele += 1\n flops = flops_per_ele * y.numel() / m.groups\n return int(flops)\n\n\n\"\"\"\nPooling\n\"\"\"\n\n\ndef hook_maxpool1d(m, x, y):\n flops_per_ele = m.kernel_size - 1\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_maxpool2d(m, x, y):\n k = _pair(m.kernel_size)\n k = torch.prod(torch.Tensor(k)).item()\n # ops: compare\n flops_per_ele = k - 1\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_maxpool3d(m, x, y):\n k = _triple(m.kernel_size)\n k = torch.prod(torch.Tensor(k)).item()\n flops_per_ele = k - 1\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_avgpool1d(m, x, y):\n flops_per_ele = m.kernel_size\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_avgpool2d(m, x, y):\n k = _pair(m.kernel_size)\n k = torch.prod(torch.Tensor(k)).item()\n flops_per_ele = k\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_avgpool3d(m, x, y):\n k = _triple(m.kernel_size)\n k = torch.prod(torch.Tensor(k)).item()\n flops_per_ele = k\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_adapmaxpool1d(m, x, y):\n x = x[0]\n out_size = m.output_size\n k = math.ceil(x.size(2) / out_size)\n flops_per_ele = k - 1\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_adapmaxpool2d(m, x, y):\n x = x[0]\n out_size = _pair(m.output_size)\n k = torch.Tensor(list(x.size()[2:])) / torch.Tensor(out_size)\n k = torch.prod(torch.ceil(k)).item()\n flops_per_ele = k - 1\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_adapmaxpool3d(m, x, y):\n x = x[0]\n out_size = _triple(m.output_size)\n k = torch.Tensor(list(x.size()[2:])) / torch.Tensor(out_size)\n k = torch.prod(torch.ceil(k)).item()\n flops_per_ele = k - 1\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_adapavgpool1d(m, x, y):\n x = x[0]\n out_size = m.output_size\n k = math.ceil(x.size(2) / out_size)\n flops_per_ele = k\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_adapavgpool2d(m, x, y):\n x = x[0]\n out_size = _pair(m.output_size)\n k = torch.Tensor(list(x.size()[2:])) / torch.Tensor(out_size)\n k = torch.prod(torch.ceil(k)).item()\n flops_per_ele = k\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\ndef hook_adapavgpool3d(m, x, y):\n x = x[0]\n out_size = _triple(m.output_size)\n k = torch.Tensor(list(x.size()[2:])) / torch.Tensor(out_size)\n k = torch.prod(torch.ceil(k)).item()\n flops_per_ele = k\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\n\"\"\"\nNon-linear activations\n\"\"\"\n\n\ndef hook_relu(m, x, y):\n # eq: max(0, x)\n num_ele = y.numel()\n return int(num_ele)\n\n\ndef hook_leakyrelu(m, x, y):\n # eq: max(0, x) + negative_slope*min(0, x)\n num_ele = y.numel()\n flops = 3 * num_ele\n return int(flops)\n\n\n\"\"\"\nNormalization\n\"\"\"\n\n\ndef hook_batchnormNd(m, x, y):\n num_ele = y.numel()\n flops = 2 * num_ele # mean and std\n if m.affine:\n flops += 2 * num_ele # gamma and beta\n return int(flops)\n\n\ndef hook_instancenormNd(m, x, y):\n return hook_batchnormNd(m, x, y)\n\n\ndef hook_groupnorm(m, x, y):\n return hook_batchnormNd(m, x, y)\n\n\ndef hook_layernorm(m, x, y):\n num_ele = y.numel()\n flops = 2 * num_ele # mean and std\n if m.elementwise_affine:\n flops += 2 * num_ele # gamma and beta\n return int(flops)\n\n\n\"\"\"\nLinear\n\"\"\"\n\n\ndef hook_linear(m, x, y):\n flops_per_ele = m.in_features # + (m.in_features-1)\n if m.bias is not None:\n flops_per_ele += 1\n flops = flops_per_ele * y.numel()\n return int(flops)\n\n\n__generic_flops_counter = {\n # Convolution\n 'Conv1d': hook_convNd,\n 'Conv2d': hook_convNd,\n 'Conv3d': hook_convNd,\n # Pooling\n 'MaxPool1d': hook_maxpool1d,\n 'MaxPool2d': hook_maxpool2d,\n 'MaxPool3d': hook_maxpool3d,\n 'AvgPool1d': hook_avgpool1d,\n 'AvgPool2d': hook_avgpool2d,\n 'AvgPool3d': hook_avgpool3d,\n 'AdaptiveMaxPool1d': hook_adapmaxpool1d,\n 'AdaptiveMaxPool2d': hook_adapmaxpool2d,\n 'AdaptiveMaxPool3d': hook_adapmaxpool3d,\n 'AdaptiveAvgPool1d': hook_adapavgpool1d,\n 'AdaptiveAvgPool2d': hook_adapavgpool2d,\n 'AdaptiveAvgPool3d': hook_adapavgpool3d,\n # Non-linear activations\n 'ReLU': hook_relu,\n 'ReLU6': hook_relu,\n 'LeakyReLU': hook_leakyrelu,\n # Normalization\n 'BatchNorm1d': hook_batchnormNd,\n 'BatchNorm2d': hook_batchnormNd,\n 'BatchNorm3d': hook_batchnormNd,\n 'InstanceNorm1d': hook_instancenormNd,\n 'InstanceNorm2d': hook_instancenormNd,\n 'InstanceNorm3d': hook_instancenormNd,\n 'GroupNorm': hook_groupnorm,\n 'LayerNorm': hook_layernorm,\n # Linear\n 'Linear': hook_linear,\n}\n\n__conv_linear_flops_counter = {\n # Convolution\n 'Conv1d': hook_convNd,\n 'Conv2d': hook_convNd,\n 'Conv3d': hook_convNd,\n # Linear\n 'Linear': hook_linear,\n}\n\n\ndef _get_flops_counter(only_conv_linear):\n if only_conv_linear:\n return __conv_linear_flops_counter\n return __generic_flops_counter\n\n\ndef compute_model_complexity(\n model, input_size, verbose=False, only_conv_linear=True\n):\n \"\"\"Returns number of parameters and FLOPs.\n\n .. note::\n (1) this function only provides an estimate of the theoretical time complexity\n rather than the actual running time which depends on implementations and hardware,\n and (2) the FLOPs is only counted for layers that are used at test time. This means\n that redundant layers such as person ID classification layer will be ignored as it\n is discarded when doing feature extraction. Note that the inference graph depends on\n how you construct the computations in ``forward()``.\n\n Args:\n model (nn.Module): network model.\n input_size (tuple): input size, e.g. (1, 3, 256, 128).\n verbose (bool, optional): shows detailed complexity of\n each module. Default is False.\n only_conv_linear (bool, optional): only considers convolution\n and linear layers when counting flops. Default is True.\n If set to False, flops of all layers will be counted.\n\n Examples::\n >>> from torchreid import models, utils\n >>> model = models.build_model(name='resnet50', num_classes=1000)\n >>> num_params, flops = utils.compute_model_complexity(model, (1, 3, 256, 128), verbose=True)\n \"\"\"\n registered_handles = []\n layer_list = []\n layer = namedtuple('layer', ['class_name', 'params', 'flops'])\n\n def _add_hooks(m):\n\n def _has_submodule(m):\n return len(list(m.children())) > 0\n\n def _hook(m, x, y):\n params = sum(p.numel() for p in m.parameters())\n class_name = str(m.__class__.__name__)\n flops_counter = _get_flops_counter(only_conv_linear)\n if class_name in flops_counter:\n flops = flops_counter[class_name](m, x, y)\n else:\n flops = 0\n layer_list.append(\n layer(class_name=class_name, params=params, flops=flops)\n )\n\n # only consider the very basic nn layer\n if _has_submodule(m):\n return\n\n handle = m.register_forward_hook(_hook)\n registered_handles.append(handle)\n\n default_train_mode = model.training\n\n model.eval().apply(_add_hooks)\n input = torch.rand(input_size)\n if next(model.parameters()).is_cuda:\n input = input.cuda()\n model(input) # forward\n\n for handle in registered_handles:\n handle.remove()\n\n model.train(default_train_mode)\n\n if verbose:\n per_module_params = defaultdict(list)\n per_module_flops = defaultdict(list)\n\n total_params, total_flops = 0, 0\n\n for layer in layer_list:\n total_params += layer.params\n total_flops += layer.flops\n if verbose:\n per_module_params[layer.class_name].append(layer.params)\n per_module_flops[layer.class_name].append(layer.flops)\n\n if verbose:\n num_udscore = 55\n print(' {}'.format('-' * num_udscore))\n print(' Model complexity with input size {}'.format(input_size))\n print(' {}'.format('-' * num_udscore))\n for class_name in per_module_params:\n params = int(np.sum(per_module_params[class_name]))\n flops = int(np.sum(per_module_flops[class_name]))\n print(\n ' {} (params={:,}, flops={:,})'.format(\n class_name, params, flops\n )\n )\n print(' {}'.format('-' * num_udscore))\n print(\n ' Total (params={:,}, flops={:,})'.format(\n total_params, total_flops\n )\n )\n print(' {}'.format('-' * num_udscore))\n\n return total_params, total_flops\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/reidtools.py", "content": "from __future__ import print_function, absolute_import\nimport numpy as np\nimport shutil\nimport os.path as osp\nimport cv2\n\nfrom .tools import mkdir_if_missing\n\n__all__ = ['visualize_ranked_results']\n\nGRID_SPACING = 10\nQUERY_EXTRA_SPACING = 90\nBW = 5 # border width\nGREEN = (0, 255, 0)\nRED = (0, 0, 255)\n\n\ndef visualize_ranked_results(\n distmat, dataset, data_type, width=128, height=256, save_dir='', topk=10\n):\n \"\"\"Visualizes ranked results.\n\n Supports both image-reid and video-reid.\n\n For image-reid, ranks will be plotted in a single figure. For video-reid, ranks will be\n saved in folders each containing a tracklet.\n\n Args:\n distmat (numpy.ndarray): distance matrix of shape (num_query, num_gallery).\n dataset (tuple): a 2-tuple containing (query, gallery), each of which contains\n tuples of (img_path(s), pid, camid, dsetid).\n data_type (str): \"image\" or \"video\".\n width (int, optional): resized image width. Default is 128.\n height (int, optional): resized image height. Default is 256.\n save_dir (str): directory to save output images.\n topk (int, optional): denoting top-k images in the rank list to be visualized.\n Default is 10.\n \"\"\"\n num_q, num_g = distmat.shape\n mkdir_if_missing(save_dir)\n\n print('# query: {}\\n# gallery {}'.format(num_q, num_g))\n print('Visualizing top-{} ranks ...'.format(topk))\n\n query, gallery = dataset\n assert num_q == len(query)\n assert num_g == len(gallery)\n\n indices = np.argsort(distmat, axis=1)\n\n def _cp_img_to(src, dst, rank, prefix, matched=False):\n \"\"\"\n Args:\n src: image path or tuple (for vidreid)\n dst: target directory\n rank: int, denoting ranked position, starting from 1\n prefix: string\n matched: bool\n \"\"\"\n if isinstance(src, (tuple, list)):\n if prefix == 'gallery':\n suffix = 'TRUE' if matched else 'FALSE'\n dst = osp.join(\n dst, prefix + '_top' + str(rank).zfill(3)\n ) + '_' + suffix\n else:\n dst = osp.join(dst, prefix + '_top' + str(rank).zfill(3))\n mkdir_if_missing(dst)\n for img_path in src:\n shutil.copy(img_path, dst)\n else:\n dst = osp.join(\n dst, prefix + '_top' + str(rank).zfill(3) + '_name_' +\n osp.basename(src)\n )\n shutil.copy(src, dst)\n\n for q_idx in range(num_q):\n qimg_path, qpid, qcamid = query[q_idx][:3]\n qimg_path_name = qimg_path[0] if isinstance(\n qimg_path, (tuple, list)\n ) else qimg_path\n\n if data_type == 'image':\n qimg = cv2.imread(qimg_path)\n qimg = cv2.resize(qimg, (width, height))\n qimg = cv2.copyMakeBorder(\n qimg, BW, BW, BW, BW, cv2.BORDER_CONSTANT, value=(0, 0, 0)\n )\n # resize twice to ensure that the border width is consistent across images\n qimg = cv2.resize(qimg, (width, height))\n num_cols = topk + 1\n grid_img = 255 * np.ones(\n (\n height,\n num_cols*width + topk*GRID_SPACING + QUERY_EXTRA_SPACING, 3\n ),\n dtype=np.uint8\n )\n grid_img[:, :width, :] = qimg\n else:\n qdir = osp.join(\n save_dir, osp.basename(osp.splitext(qimg_path_name)[0])\n )\n mkdir_if_missing(qdir)\n _cp_img_to(qimg_path, qdir, rank=0, prefix='query')\n\n rank_idx = 1\n for g_idx in indices[q_idx, :]:\n gimg_path, gpid, gcamid = gallery[g_idx][:3]\n invalid = (qpid == gpid) & (qcamid == gcamid)\n\n if not invalid:\n matched = gpid == qpid\n if data_type == 'image':\n border_color = GREEN if matched else RED\n gimg = cv2.imread(gimg_path)\n gimg = cv2.resize(gimg, (width, height))\n gimg = cv2.copyMakeBorder(\n gimg,\n BW,\n BW,\n BW,\n BW,\n cv2.BORDER_CONSTANT,\n value=border_color\n )\n gimg = cv2.resize(gimg, (width, height))\n start = rank_idx*width + rank_idx*GRID_SPACING + QUERY_EXTRA_SPACING\n end = (\n rank_idx+1\n ) * width + rank_idx*GRID_SPACING + QUERY_EXTRA_SPACING\n grid_img[:, start:end, :] = gimg\n else:\n _cp_img_to(\n gimg_path,\n qdir,\n rank=rank_idx,\n prefix='gallery',\n matched=matched\n )\n\n rank_idx += 1\n if rank_idx > topk:\n break\n\n if data_type == 'image':\n imname = osp.basename(osp.splitext(qimg_path_name)[0])\n cv2.imwrite(osp.join(save_dir, imname + '.jpg'), grid_img)\n\n if (q_idx+1) % 100 == 0:\n print('- done {}/{}'.format(q_idx + 1, num_q))\n\n print('Done. Images have been saved to \"{}\" ...'.format(save_dir))\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/rerank.py", "content": "#!/usr/bin/env python2/python3\n# -*- coding: utf-8 -*-\n\"\"\"\nSource: https://github.com/zhunzhong07/person-re-ranking\n\nCreated on Mon Jun 26 14:46:56 2017\n@author: luohao\nModified by Houjing Huang, 2017-12-22.\n- This version accepts distance matrix instead of raw features.\n- The difference of `/` division between python 2 and 3 is handled.\n- numpy.float16 is replaced by numpy.float32 for numerical precision.\n\nCVPR2017 paper:Zhong Z, Zheng L, Cao D, et al. Re-ranking Person Re-identification with k-reciprocal Encoding[J]. 2017.\nurl:http://openaccess.thecvf.com/content_cvpr_2017/papers/Zhong_Re-Ranking_Person_Re-Identification_CVPR_2017_paper.pdf\nMatlab version: https://github.com/zhunzhong07/person-re-ranking\n\nAPI\nq_g_dist: query-gallery distance matrix, numpy array, shape [num_query, num_gallery]\nq_q_dist: query-query distance matrix, numpy array, shape [num_query, num_query]\ng_g_dist: gallery-gallery distance matrix, numpy array, shape [num_gallery, num_gallery]\nk1, k2, lambda_value: parameters, the original paper is (k1=20, k2=6, lambda_value=0.3)\nReturns:\n final_dist: re-ranked distance, numpy array, shape [num_query, num_gallery]\n\"\"\"\nfrom __future__ import division, print_function, absolute_import\nimport numpy as np\n\n__all__ = ['re_ranking']\n\n\ndef re_ranking(q_g_dist, q_q_dist, g_g_dist, k1=20, k2=6, lambda_value=0.3):\n\n # The following naming, e.g. gallery_num, is different from outer scope.\n # Don't care about it.\n\n original_dist = np.concatenate(\n [\n np.concatenate([q_q_dist, q_g_dist], axis=1),\n np.concatenate([q_g_dist.T, g_g_dist], axis=1)\n ],\n axis=0\n )\n original_dist = np.power(original_dist, 2).astype(np.float32)\n original_dist = np.transpose(\n 1. * original_dist / np.max(original_dist, axis=0)\n )\n V = np.zeros_like(original_dist).astype(np.float32)\n initial_rank = np.argsort(original_dist).astype(np.int32)\n\n query_num = q_g_dist.shape[0]\n gallery_num = q_g_dist.shape[0] + q_g_dist.shape[1]\n all_num = gallery_num\n\n for i in range(all_num):\n # k-reciprocal neighbors\n forward_k_neigh_index = initial_rank[i, :k1 + 1]\n backward_k_neigh_index = initial_rank[forward_k_neigh_index, :k1 + 1]\n fi = np.where(backward_k_neigh_index == i)[0]\n k_reciprocal_index = forward_k_neigh_index[fi]\n k_reciprocal_expansion_index = k_reciprocal_index\n for j in range(len(k_reciprocal_index)):\n candidate = k_reciprocal_index[j]\n candidate_forward_k_neigh_index = initial_rank[\n candidate, :int(np.around(k1 / 2.)) + 1]\n candidate_backward_k_neigh_index = initial_rank[\n candidate_forward_k_neigh_index, :int(np.around(k1 / 2.)) + 1]\n fi_candidate = np.where(\n candidate_backward_k_neigh_index == candidate\n )[0]\n candidate_k_reciprocal_index = candidate_forward_k_neigh_index[\n fi_candidate]\n if len(\n np.\n intersect1d(candidate_k_reciprocal_index, k_reciprocal_index)\n ) > 2. / 3 * len(candidate_k_reciprocal_index):\n k_reciprocal_expansion_index = np.append(\n k_reciprocal_expansion_index, candidate_k_reciprocal_index\n )\n\n k_reciprocal_expansion_index = np.unique(k_reciprocal_expansion_index)\n weight = np.exp(-original_dist[i, k_reciprocal_expansion_index])\n V[i, k_reciprocal_expansion_index] = 1. * weight / np.sum(weight)\n original_dist = original_dist[:query_num, ]\n if k2 != 1:\n V_qe = np.zeros_like(V, dtype=np.float32)\n for i in range(all_num):\n V_qe[i, :] = np.mean(V[initial_rank[i, :k2], :], axis=0)\n V = V_qe\n del V_qe\n del initial_rank\n invIndex = []\n for i in range(gallery_num):\n invIndex.append(np.where(V[:, i] != 0)[0])\n\n jaccard_dist = np.zeros_like(original_dist, dtype=np.float32)\n\n for i in range(query_num):\n temp_min = np.zeros(shape=[1, gallery_num], dtype=np.float32)\n indNonZero = np.where(V[i, :] != 0)[0]\n indImages = []\n indImages = [invIndex[ind] for ind in indNonZero]\n for j in range(len(indNonZero)):\n temp_min[0, indImages[j]] = temp_min[0, indImages[j]] + np.minimum(\n V[i, indNonZero[j]], V[indImages[j], indNonZero[j]]\n )\n jaccard_dist[i] = 1 - temp_min / (2.-temp_min)\n\n final_dist = jaccard_dist * (1-lambda_value) + original_dist*lambda_value\n del original_dist\n del V\n del jaccard_dist\n final_dist = final_dist[:query_num, query_num:]\n return final_dist\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/tools.py", "content": "from __future__ import division, print_function, absolute_import\nimport os\nimport sys\nimport json\nimport time\nimport errno\nimport numpy as np\nimport random\nimport os.path as osp\nimport warnings\nimport PIL\nimport torch\nfrom PIL import Image\n\n__all__ = [\n 'mkdir_if_missing', 'check_isfile', 'read_json', 'write_json',\n 'set_random_seed', 'download_url', 'read_image', 'collect_env_info',\n 'listdir_nohidden'\n]\n\n\ndef mkdir_if_missing(dirname):\n \"\"\"Creates dirname if it is missing.\"\"\"\n if not osp.exists(dirname):\n try:\n os.makedirs(dirname)\n except OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\n\ndef check_isfile(fpath):\n \"\"\"Checks if the given path is a file.\n\n Args:\n fpath (str): file path.\n\n Returns:\n bool\n \"\"\"\n isfile = osp.isfile(fpath)\n if not isfile:\n warnings.warn('No file found at \"{}\"'.format(fpath))\n return isfile\n\n\ndef read_json(fpath):\n \"\"\"Reads json file from a path.\"\"\"\n with open(fpath, 'r') as f:\n obj = json.load(f)\n return obj\n\n\ndef write_json(obj, fpath):\n \"\"\"Writes to a json file.\"\"\"\n mkdir_if_missing(osp.dirname(fpath))\n with open(fpath, 'w') as f:\n json.dump(obj, f, indent=4, separators=(',', ': '))\n\n\ndef set_random_seed(seed):\n random.seed(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed_all(seed)\n\n\ndef download_url(url, dst):\n \"\"\"Downloads file from a url to a destination.\n\n Args:\n url (str): url to download file.\n dst (str): destination path.\n \"\"\"\n from six.moves import urllib\n print('* url=\"{}\"'.format(url))\n print('* destination=\"{}\"'.format(dst))\n\n def _reporthook(count, block_size, total_size):\n global start_time\n if count == 0:\n start_time = time.time()\n return\n duration = time.time() - start_time\n progress_size = int(count * block_size)\n speed = int(progress_size / (1024*duration))\n percent = int(count * block_size * 100 / total_size)\n sys.stdout.write(\n '\\r...%d%%, %d MB, %d KB/s, %d seconds passed' %\n (percent, progress_size / (1024*1024), speed, duration)\n )\n sys.stdout.flush()\n\n urllib.request.urlretrieve(url, dst, _reporthook)\n sys.stdout.write('\\n')\n\n\ndef read_image(path):\n \"\"\"Reads image from path using ``PIL.Image``.\n\n Args:\n path (str): path to an image.\n\n Returns:\n PIL image\n \"\"\"\n got_img = False\n if not osp.exists(path):\n raise IOError('\"{}\" does not exist'.format(path))\n while not got_img:\n try:\n img = Image.open(path).convert('RGB')\n got_img = True\n except IOError:\n print(\n 'IOError incurred when reading \"{}\". Will redo. Don\\'t worry. Just chill.'\n .format(path)\n )\n return img\n\n\ndef collect_env_info():\n \"\"\"Returns env info as a string.\n\n Code source: github.com/facebookresearch/maskrcnn-benchmark\n \"\"\"\n from torch.utils.collect_env import get_pretty_env_info\n env_str = get_pretty_env_info()\n env_str += '\\n Pillow ({})'.format(PIL.__version__)\n return env_str\n\n\ndef listdir_nohidden(path, sort=False):\n \"\"\"List non-hidden items in a directory.\n\n Args:\n path (str): directory path.\n sort (bool): sort the items.\n \"\"\"\n items = [f for f in os.listdir(path) if not f.startswith('.')]\n if sort:\n items.sort()\n return items\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid/torchreid/utils/torchtools.py", "content": "from __future__ import division, print_function, absolute_import\nimport pickle\nimport shutil\nimport os.path as osp\nimport warnings\nfrom functools import partial\nfrom collections import OrderedDict\nimport torch\nimport torch.nn as nn\n\nfrom .tools import mkdir_if_missing\n\n__all__ = [\n 'save_checkpoint', 'load_checkpoint', 'resume_from_checkpoint',\n 'open_all_layers', 'open_specified_layers', 'count_num_param',\n 'load_pretrained_weights'\n]\n\n\ndef save_checkpoint(\n state, save_dir, is_best=False, remove_module_from_keys=False\n):\n r\"\"\"Saves checkpoint.\n\n Args:\n state (dict): dictionary.\n save_dir (str): directory to save checkpoint.\n is_best (bool, optional): if True, this checkpoint will be copied and named\n ``model-best.pth.tar``. Default is False.\n remove_module_from_keys (bool, optional): whether to remove \"module.\"\n from layer names. Default is False.\n\n Examples::\n >>> state = {\n >>> 'state_dict': model.state_dict(),\n >>> 'epoch': 10,\n >>> 'rank1': 0.5,\n >>> 'optimizer': optimizer.state_dict()\n >>> }\n >>> save_checkpoint(state, 'log/my_model')\n \"\"\"\n mkdir_if_missing(save_dir)\n if remove_module_from_keys:\n # remove 'module.' in state_dict's keys\n state_dict = state['state_dict']\n new_state_dict = OrderedDict()\n for k, v in state_dict.items():\n if k.startswith('module.'):\n k = k[7:]\n new_state_dict[k] = v\n state['state_dict'] = new_state_dict\n # save\n epoch = state['epoch']\n fpath = osp.join(save_dir, 'model.pth.tar-' + str(epoch))\n torch.save(state, fpath)\n print('Checkpoint saved to \"{}\"'.format(fpath))\n if is_best:\n shutil.copy(fpath, osp.join(osp.dirname(fpath), 'model-best.pth.tar'))\n\n\ndef load_checkpoint(fpath):\n r\"\"\"Loads checkpoint.\n\n ``UnicodeDecodeError`` can be well handled, which means\n python2-saved files can be read from python3.\n\n Args:\n fpath (str): path to checkpoint.\n\n Returns:\n dict\n\n Examples::\n >>> from torchreid.utils import load_checkpoint\n >>> fpath = 'log/my_model/model.pth.tar-10'\n >>> checkpoint = load_checkpoint(fpath)\n \"\"\"\n if fpath is None:\n raise ValueError('File path is None')\n fpath = osp.abspath(osp.expanduser(fpath))\n if not osp.exists(fpath):\n raise FileNotFoundError('File is not found at \"{}\"'.format(fpath))\n map_location = None if torch.cuda.is_available() else 'cpu'\n try:\n checkpoint = torch.load(fpath, map_location=map_location)\n except UnicodeDecodeError:\n pickle.load = partial(pickle.load, encoding=\"latin1\")\n pickle.Unpickler = partial(pickle.Unpickler, encoding=\"latin1\")\n checkpoint = torch.load(\n fpath, pickle_module=pickle, map_location=map_location\n )\n except Exception:\n print('Unable to load checkpoint from \"{}\"'.format(fpath))\n raise\n return checkpoint\n\n\ndef resume_from_checkpoint(fpath, model, optimizer=None, scheduler=None):\n r\"\"\"Resumes training from a checkpoint.\n\n This will load (1) model weights and (2) ``state_dict``\n of optimizer if ``optimizer`` is not None.\n\n Args:\n fpath (str): path to checkpoint.\n model (nn.Module): model.\n optimizer (Optimizer, optional): an Optimizer.\n scheduler (LRScheduler, optional): an LRScheduler.\n\n Returns:\n int: start_epoch.\n\n Examples::\n >>> from torchreid.utils import resume_from_checkpoint\n >>> fpath = 'log/my_model/model.pth.tar-10'\n >>> start_epoch = resume_from_checkpoint(\n >>> fpath, model, optimizer, scheduler\n >>> )\n \"\"\"\n print('Loading checkpoint from \"{}\"'.format(fpath))\n checkpoint = load_checkpoint(fpath)\n model.load_state_dict(checkpoint['state_dict'])\n print('Loaded model weights')\n if optimizer is not None and 'optimizer' in checkpoint.keys():\n optimizer.load_state_dict(checkpoint['optimizer'])\n print('Loaded optimizer')\n if scheduler is not None and 'scheduler' in checkpoint.keys():\n scheduler.load_state_dict(checkpoint['scheduler'])\n print('Loaded scheduler')\n start_epoch = checkpoint['epoch']\n print('Last epoch = {}'.format(start_epoch))\n if 'rank1' in checkpoint.keys():\n print('Last rank1 = {:.1%}'.format(checkpoint['rank1']))\n return start_epoch\n\n\ndef adjust_learning_rate(\n optimizer,\n base_lr,\n epoch,\n stepsize=20,\n gamma=0.1,\n linear_decay=False,\n final_lr=0,\n max_epoch=100\n):\n r\"\"\"Adjusts learning rate.\n\n Deprecated.\n \"\"\"\n if linear_decay:\n # linearly decay learning rate from base_lr to final_lr\n frac_done = epoch / max_epoch\n lr = frac_done*final_lr + (1.-frac_done) * base_lr\n else:\n # decay learning rate by gamma for every stepsize\n lr = base_lr * (gamma**(epoch // stepsize))\n\n for param_group in optimizer.param_groups:\n param_group['lr'] = lr\n\n\ndef set_bn_to_eval(m):\n r\"\"\"Sets BatchNorm layers to eval mode.\"\"\"\n # 1. no update for running mean and var\n # 2. scale and shift parameters are still trainable\n classname = m.__class__.__name__\n if classname.find('BatchNorm') != -1:\n m.eval()\n\n\ndef open_all_layers(model):\n r\"\"\"Opens all layers in model for training.\n\n Examples::\n >>> from torchreid.utils import open_all_layers\n >>> open_all_layers(model)\n \"\"\"\n model.train()\n for p in model.parameters():\n p.requires_grad = True\n\n\ndef open_specified_layers(model, open_layers):\n r\"\"\"Opens specified layers in model for training while keeping\n other layers frozen.\n\n Args:\n model (nn.Module): neural net model.\n open_layers (str or list): layers open for training.\n\n Examples::\n >>> from torchreid.utils import open_specified_layers\n >>> # Only model.classifier will be updated.\n >>> open_layers = 'classifier'\n >>> open_specified_layers(model, open_layers)\n >>> # Only model.fc and model.classifier will be updated.\n >>> open_layers = ['fc', 'classifier']\n >>> open_specified_layers(model, open_layers)\n \"\"\"\n if isinstance(model, nn.DataParallel):\n model = model.module\n\n if isinstance(open_layers, str):\n open_layers = [open_layers]\n\n for layer in open_layers:\n assert hasattr(\n model, layer\n ), '\"{}\" is not an attribute of the model, please provide the correct name'.format(\n layer\n )\n\n for name, module in model.named_children():\n if name in open_layers:\n module.train()\n for p in module.parameters():\n p.requires_grad = True\n else:\n module.eval()\n for p in module.parameters():\n p.requires_grad = False\n\n\ndef count_num_param(model):\n r\"\"\"Counts number of parameters in a model while ignoring ``self.classifier``.\n\n Args:\n model (nn.Module): network model.\n\n Examples::\n >>> from torchreid.utils import count_num_param\n >>> model_size = count_num_param(model)\n\n .. warning::\n \n This method is deprecated in favor of\n ``torchreid.utils.compute_model_complexity``.\n \"\"\"\n warnings.warn(\n 'This method is deprecated and will be removed in the future.'\n )\n\n num_param = sum(p.numel() for p in model.parameters())\n\n if isinstance(model, nn.DataParallel):\n model = model.module\n\n if hasattr(model,\n 'classifier') and isinstance(model.classifier, nn.Module):\n # we ignore the classifier because it is unused at test time\n num_param -= sum(p.numel() for p in model.classifier.parameters())\n\n return num_param\n\n\ndef load_pretrained_weights(model, weight_path):\n r\"\"\"Loads pretrianed weights to model.\n\n Features::\n - Incompatible layers (unmatched in name or size) will be ignored.\n - Can automatically deal with keys containing \"module.\".\n\n Args:\n model (nn.Module): network model.\n weight_path (str): path to pretrained weights.\n\n Examples::\n >>> from torchreid.utils import load_pretrained_weights\n >>> weight_path = 'log/my_model/model-best.pth.tar'\n >>> load_pretrained_weights(model, weight_path)\n \"\"\"\n checkpoint = load_checkpoint(weight_path)\n if 'state_dict' in checkpoint:\n state_dict = checkpoint['state_dict']\n else:\n state_dict = checkpoint\n\n model_dict = model.state_dict()\n new_state_dict = OrderedDict()\n matched_layers, discarded_layers = [], []\n\n for k, v in state_dict.items():\n if k.startswith('module.'):\n k = k[7:] # discard module.\n\n if k in model_dict and model_dict[k].size() == v.size():\n new_state_dict[k] = v\n matched_layers.append(k)\n else:\n discarded_layers.append(k)\n\n model_dict.update(new_state_dict)\n model.load_state_dict(model_dict)\n\n if len(matched_layers) == 0:\n warnings.warn(\n 'The pretrained weights \"{}\" cannot be loaded, '\n 'please check the key names manually '\n '(** ignored and continue **)'.format(weight_path)\n )\n else:\n print(\n 'Successfully loaded pretrained weights from \"{}\"'.\n format(weight_path)\n )\n if len(discarded_layers) > 0:\n print(\n '** The following layers are discarded '\n 'due to unmatched keys or layer size: {}'.\n format(discarded_layers)\n )\n" }, { "path": "asone/trackers/strong_sort/tracker/deep/reid_model_factory.py", "content": "__model_types = [\n 'resnet50', 'mlfn', 'hacnn', 'mobilenetv2_x1_0', 'mobilenetv2_x1_4',\n 'osnet_x1_0', 'osnet_x0_75', 'osnet_x0_5', 'osnet_x0_25',\n 'osnet_ibn_x1_0', 'osnet_ain_x1_0']\n\n__trained_urls = {\n\n # market1501 models ########################################################\n 'resnet50_market1501.pt':\n 'https://drive.google.com/uc?id=1dUUZ4rHDWohmsQXCRe2C_HbYkzz94iBV',\n 'resnet50_dukemtmcreid.pt':\n 'https://drive.google.com/uc?id=17ymnLglnc64NRvGOitY3BqMRS9UWd1wg',\n 'resnet50_msmt17.pt':\n 'https://drive.google.com/uc?id=1ep7RypVDOthCRIAqDnn4_N-UhkkFHJsj',\n\n 'resnet50_fc512_market1501.pt':\n 'https://drive.google.com/uc?id=1kv8l5laX_YCdIGVCetjlNdzKIA3NvsSt',\n 'resnet50_fc512_dukemtmcreid.pt':\n 'https://drive.google.com/uc?id=13QN8Mp3XH81GK4BPGXobKHKyTGH50Rtx',\n 'resnet50_fc512_msmt17.pt':\n 'https://drive.google.com/uc?id=1fDJLcz4O5wxNSUvImIIjoaIF9u1Rwaud',\n\n 'mlfn_market1501.pt':\n 'https://drive.google.com/uc?id=1wXcvhA_b1kpDfrt9s2Pma-MHxtj9pmvS',\n 'mlfn_dukemtmcreid.pt':\n 'https://drive.google.com/uc?id=1rExgrTNb0VCIcOnXfMsbwSUW1h2L1Bum',\n 'mlfn_msmt17.pt':\n 'https://drive.google.com/uc?id=18JzsZlJb3Wm7irCbZbZ07TN4IFKvR6p-',\n\n 'hacnn_market1501.pt':\n 'https://drive.google.com/uc?id=1LRKIQduThwGxMDQMiVkTScBwR7WidmYF',\n 'hacnn_dukemtmcreid.pt':\n 'https://drive.google.com/uc?id=1zNm6tP4ozFUCUQ7Sv1Z98EAJWXJEhtYH',\n 'hacnn_msmt17.pt':\n 'https://drive.google.com/uc?id=1MsKRtPM5WJ3_Tk2xC0aGOO7pM3VaFDNZ',\n\n 'mobilenetv2_x1_0_market1501.pt':\n 'https://drive.google.com/uc?id=18DgHC2ZJkjekVoqBWszD8_Xiikz-fewp',\n 'mobilenetv2_x1_0_dukemtmcreid.pt':\n 'https://drive.google.com/uc?id=1q1WU2FETRJ3BXcpVtfJUuqq4z3psetds',\n 'mobilenetv2_x1_0_msmt17.pt':\n 'https://drive.google.com/uc?id=1j50Hv14NOUAg7ZeB3frzfX-WYLi7SrhZ',\n\n 'mobilenetv2_x1_4_market1501.pt':\n 'https://drive.google.com/uc?id=1t6JCqphJG-fwwPVkRLmGGyEBhGOf2GO5',\n 'mobilenetv2_x1_4_dukemtmcreid.pt':\n 'https://drive.google.com/uc?id=12uD5FeVqLg9-AFDju2L7SQxjmPb4zpBN',\n 'mobilenetv2_x1_4_msmt17.pt':\n 'https://drive.google.com/uc?id=1ZY5P2Zgm-3RbDpbXM0kIBMPvspeNIbXz',\n\n 'osnet_x1_0_market1501.pt':\n 'https://drive.google.com/uc?id=1vduhq5DpN2q1g4fYEZfPI17MJeh9qyrA',\n 'osnet_x1_0_dukemtmcreid.pt':\n 'https://drive.google.com/uc?id=1QZO_4sNf4hdOKKKzKc-TZU9WW1v6zQbq',\n 'osnet_x1_0_msmt17.pt':\n 'https://drive.google.com/uc?id=112EMUfBPYeYg70w-syK6V6Mx8-Qb9Q1M',\n\n 'osnet_x0_75_market1501.pt':\n 'https://drive.google.com/uc?id=1ozRaDSQw_EQ8_93OUmjDbvLXw9TnfPer',\n 'osnet_x0_75_dukemtmcreid.pt':\n 'https://drive.google.com/uc?id=1IE3KRaTPp4OUa6PGTFL_d5_KQSJbP0Or',\n 'osnet_x0_75_msmt17.pt':\n 'https://drive.google.com/uc?id=1QEGO6WnJ-BmUzVPd3q9NoaO_GsPNlmWc',\n\n 'osnet_x0_5_market1501.pt':\n 'https://drive.google.com/uc?id=1PLB9rgqrUM7blWrg4QlprCuPT7ILYGKT',\n 'osnet_x0_5_dukemtmcreid.pt':\n 'https://drive.google.com/uc?id=1KoUVqmiST175hnkALg9XuTi1oYpqcyTu',\n 'osnet_x0_5_msmt17.pt':\n 'https://drive.google.com/uc?id=1UT3AxIaDvS2PdxzZmbkLmjtiqq7AIKCv',\n\n 'osnet_x0_25_market1501.pt':\n 'https://drive.google.com/uc?id=1z1UghYvOTtjx7kEoRfmqSMu-z62J6MAj',\n 'osnet_x0_25_dukemtmcreid.pt':\n 'https://drive.google.com/uc?id=1eumrtiXT4NOspjyEV4j8cHmlOaaCGk5l',\n 'osnet_x0_25_msmt17.pt':\n 'https://drive.google.com/uc?id=1sSwXSUlj4_tHZequ_iZ8w_Jh0VaRQMqF',\n\n ####### market1501 models ##################################################\n 'resnet50_msmt17.pt':\n 'https://drive.google.com/uc?id=1yiBteqgIZoOeywE8AhGmEQl7FTVwrQmf',\n 'osnet_x1_0_msmt17.pt':\n 'https://drive.google.com/uc?id=1IosIFlLiulGIjwW3H8uMRmx3MzPwf86x',\n 'osnet_x0_75_msmt17.pt':\n 'https://drive.google.com/uc?id=1fhjSS_7SUGCioIf2SWXaRGPqIY9j7-uw',\n\n 'osnet_x0_5_msmt17.pt':\n 'https://drive.google.com/uc?id=1DHgmb6XV4fwG3n-CnCM0zdL9nMsZ9_RF',\n 'osnet_x0_25_msmt17.pt':\n 'https://drive.google.com/uc?id=1Kkx2zW89jq_NETu4u42CFZTMVD5Hwm6e',\n 'osnet_ibn_x1_0_msmt17.pt':\n 'https://drive.google.com/uc?id=1q3Sj2ii34NlfxA4LvmHdWO_75NDRmECJ',\n 'osnet_ain_x1_0_msmt17.pt':\n 'https://drive.google.com/uc?id=1SigwBE6mPdqiJMqhuIY4aqC7--5CsMal',\n}\n\n\ndef show_downloadeable_models():\n print('\\nAvailable ReID models for automatic download')\n print(list(__trained_urls.keys()))\n\n\ndef get_model_url(model):\n model = str(model).rsplit('/', 1)[-1]\n if model in __trained_urls:\n return __trained_urls[model]\n else:\n None\n\n\ndef is_model_in_model_types(model):\n model = str(model).rsplit('/', 1)[-1].split('.')[0]\n if model in __model_types:\n return True\n else:\n return False\n\n\ndef get_model_name(model):\n model = str(model).rsplit('/', 1)[-1].split('.')[0]\n for x in __model_types:\n if x in model:\n return x\n return None\n\n" }, { "path": "asone/trackers/strong_sort/tracker/sort/__init__.py", "content": "" }, { "path": "asone/trackers/strong_sort/tracker/sort/detection.py", "content": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\n\n\nclass Detection(object):\n \"\"\"\n This class represents a bounding box detection in a single image.\n\n Parameters\n ----------\n tlwh : array_like\n Bounding box in format `(x, y, w, h)`.\n confidence : float\n Detector confidence score.\n feature : array_like\n A feature vector that describes the object contained in this image.\n\n Attributes\n ----------\n tlwh : ndarray\n Bounding box in format `(top left x, top left y, width, height)`.\n confidence : ndarray\n Detector confidence score.\n feature : ndarray | NoneType\n A feature vector that describes the object contained in this image.\n\n \"\"\"\n\n def __init__(self, tlwh, confidence, feature):\n self.tlwh = np.asarray(tlwh, dtype=np.float)\n self.confidence = float(confidence)\n self.feature = np.asarray(feature.cpu(), dtype=np.float32)\n\n def to_tlbr(self):\n \"\"\"Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,\n `(top left, bottom right)`.\n \"\"\"\n ret = self.tlwh.copy()\n ret[2:] += ret[:2]\n return ret\n\n def to_xyah(self):\n \"\"\"Convert bounding box to format `(center x, center y, aspect ratio,\n height)`, where the aspect ratio is `width / height`.\n \"\"\"\n ret = self.tlwh.copy()\n ret[:2] += ret[2:] / 2\n ret[2] /= ret[3]\n return ret\n" }, { "path": "asone/trackers/strong_sort/tracker/sort/iou_matching.py", "content": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom . import linear_assignment\n\n\ndef iou(bbox, candidates):\n \"\"\"Computer intersection over union.\n\n Parameters\n ----------\n bbox : ndarray\n A bounding box in format `(top left x, top left y, width, height)`.\n candidates : ndarray\n A matrix of candidate bounding boxes (one per row) in the same format\n as `bbox`.\n\n Returns\n -------\n ndarray\n The intersection over union in [0, 1] between the `bbox` and each\n candidate. A higher score means a larger fraction of the `bbox` is\n occluded by the candidate.\n\n \"\"\"\n bbox_tl, bbox_br = bbox[:2], bbox[:2] + bbox[2:]\n candidates_tl = candidates[:, :2]\n candidates_br = candidates[:, :2] + candidates[:, 2:]\n\n tl = np.c_[np.maximum(bbox_tl[0], candidates_tl[:, 0])[:, np.newaxis],\n np.maximum(bbox_tl[1], candidates_tl[:, 1])[:, np.newaxis]]\n br = np.c_[np.minimum(bbox_br[0], candidates_br[:, 0])[:, np.newaxis],\n np.minimum(bbox_br[1], candidates_br[:, 1])[:, np.newaxis]]\n wh = np.maximum(0., br - tl)\n\n area_intersection = wh.prod(axis=1)\n area_bbox = bbox[2:].prod()\n area_candidates = candidates[:, 2:].prod(axis=1)\n return area_intersection / (area_bbox + area_candidates - area_intersection)\n\n\ndef iou_cost(tracks, detections, track_indices=None,\n detection_indices=None):\n \"\"\"An intersection over union distance metric.\n\n Parameters\n ----------\n tracks : List[deep_sort.track.Track]\n A list of tracks.\n detections : List[deep_sort.detection.Detection]\n A list of detections.\n track_indices : Optional[List[int]]\n A list of indices to tracks that should be matched. Defaults to\n all `tracks`.\n detection_indices : Optional[List[int]]\n A list of indices to detections that should be matched. Defaults\n to all `detections`.\n\n Returns\n -------\n ndarray\n Returns a cost matrix of shape\n len(track_indices), len(detection_indices) where entry (i, j) is\n `1 - iou(tracks[track_indices[i]], detections[detection_indices[j]])`.\n\n \"\"\"\n if track_indices is None:\n track_indices = np.arange(len(tracks))\n if detection_indices is None:\n detection_indices = np.arange(len(detections))\n\n cost_matrix = np.zeros((len(track_indices), len(detection_indices)))\n for row, track_idx in enumerate(track_indices):\n if tracks[track_idx].time_since_update > 1:\n cost_matrix[row, :] = linear_assignment.INFTY_COST\n continue\n\n bbox = tracks[track_idx].to_tlwh()\n candidates = np.asarray(\n [detections[i].tlwh for i in detection_indices])\n cost_matrix[row, :] = 1. - iou(bbox, candidates)\n return cost_matrix\n" }, { "path": "asone/trackers/strong_sort/tracker/sort/kalman_filter.py", "content": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport scipy.linalg\n\"\"\"\nTable for the 0.95 quantile of the chi-square distribution with N degrees of\nfreedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv\nfunction and used as Mahalanobis gating threshold.\n\"\"\"\nchi2inv95 = {\n 1: 3.8415,\n 2: 5.9915,\n 3: 7.8147,\n 4: 9.4877,\n 5: 11.070,\n 6: 12.592,\n 7: 14.067,\n 8: 15.507,\n 9: 16.919}\n\n\nclass KalmanFilter(object):\n \"\"\"\n A simple Kalman filter for tracking bounding boxes in image space.\n The 8-dimensional state space\n x, y, a, h, vx, vy, va, vh\n contains the bounding box center position (x, y), aspect ratio a, height h,\n and their respective velocities.\n Object motion follows a constant velocity model. The bounding box location\n (x, y, a, h) is taken as direct observation of the state space (linear\n observation model).\n \"\"\"\n\n def __init__(self):\n ndim, dt = 4, 1.\n\n # Create Kalman filter model matrices.\n self._motion_mat = np.eye(2 * ndim, 2 * ndim)\n for i in range(ndim):\n self._motion_mat[i, ndim + i] = dt\n\n self._update_mat = np.eye(ndim, 2 * ndim)\n\n # Motion and observation uncertainty are chosen relative to the current\n # state estimate. These weights control the amount of uncertainty in\n # the model. This is a bit hacky.\n self._std_weight_position = 1. / 20\n self._std_weight_velocity = 1. / 160\n\n def initiate(self, measurement):\n \"\"\"Create track from unassociated measurement.\n Parameters\n ----------\n measurement : ndarray\n Bounding box coordinates (x, y, a, h) with center position (x, y),\n aspect ratio a, and height h.\n Returns\n -------\n (ndarray, ndarray)\n Returns the mean vector (8 dimensional) and covariance matrix (8x8\n dimensional) of the new track. Unobserved velocities are initialized\n to 0 mean.\n \"\"\"\n mean_pos = measurement\n mean_vel = np.zeros_like(mean_pos)\n mean = np.r_[mean_pos, mean_vel]\n\n std = [\n 2 * self._std_weight_position * measurement[0], # the center point x\n 2 * self._std_weight_position * measurement[1], # the center point y\n 1 * measurement[2], # the ratio of width/height\n 2 * self._std_weight_position * measurement[3], # the height\n 10 * self._std_weight_velocity * measurement[0],\n 10 * self._std_weight_velocity * measurement[1],\n 0.1 * measurement[2],\n 10 * self._std_weight_velocity * measurement[3]]\n covariance = np.diag(np.square(std))\n return mean, covariance\n\n def predict(self, mean, covariance):\n \"\"\"Run Kalman filter prediction step.\n Parameters\n ----------\n mean : ndarray\n The 8 dimensional mean vector of the object state at the previous\n time step.\n covariance : ndarray\n The 8x8 dimensional covariance matrix of the object state at the\n previous time step.\n Returns\n -------\n (ndarray, ndarray)\n Returns the mean vector and covariance matrix of the predicted\n state. Unobserved velocities are initialized to 0 mean.\n \"\"\"\n std_pos = [\n self._std_weight_position * mean[0],\n self._std_weight_position * mean[1],\n 1 * mean[2],\n self._std_weight_position * mean[3]]\n std_vel = [\n self._std_weight_velocity * mean[0],\n self._std_weight_velocity * mean[1],\n 0.1 * mean[2],\n self._std_weight_velocity * mean[3]]\n motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))\n\n mean = np.dot(self._motion_mat, mean)\n covariance = np.linalg.multi_dot((\n self._motion_mat, covariance, self._motion_mat.T)) + motion_cov\n\n return mean, covariance\n\n def project(self, mean, covariance, confidence=.0):\n \"\"\"Project state distribution to measurement space.\n Parameters\n ----------\n mean : ndarray\n The state's mean vector (8 dimensional array).\n covariance : ndarray\n The state's covariance matrix (8x8 dimensional).\n confidence: (dyh) 检测框置信度\n Returns\n -------\n (ndarray, ndarray)\n Returns the projected mean and covariance matrix of the given state\n estimate.\n \"\"\"\n std = [\n self._std_weight_position * mean[3],\n self._std_weight_position * mean[3],\n 1e-1,\n self._std_weight_position * mean[3]]\n\n\n std = [(1 - confidence) * x for x in std]\n\n innovation_cov = np.diag(np.square(std))\n\n mean = np.dot(self._update_mat, mean)\n covariance = np.linalg.multi_dot((\n self._update_mat, covariance, self._update_mat.T))\n return mean, covariance + innovation_cov\n\n def update(self, mean, covariance, measurement, confidence=.0):\n \"\"\"Run Kalman filter correction step.\n Parameters\n ----------\n mean : ndarray\n The predicted state's mean vector (8 dimensional).\n covariance : ndarray\n The state's covariance matrix (8x8 dimensional).\n measurement : ndarray\n The 4 dimensional measurement vector (x, y, a, h), where (x, y)\n is the center position, a the aspect ratio, and h the height of the\n bounding box.\n confidence: (dyh)检测框置信度\n Returns\n -------\n (ndarray, ndarray)\n Returns the measurement-corrected state distribution.\n \"\"\"\n projected_mean, projected_cov = self.project(mean, covariance, confidence)\n\n chol_factor, lower = scipy.linalg.cho_factor(\n projected_cov, lower=True, check_finite=False)\n kalman_gain = scipy.linalg.cho_solve(\n (chol_factor, lower), np.dot(covariance, self._update_mat.T).T,\n check_finite=False).T\n innovation = measurement - projected_mean\n\n new_mean = mean + np.dot(innovation, kalman_gain.T)\n new_covariance = covariance - np.linalg.multi_dot((\n kalman_gain, projected_cov, kalman_gain.T))\n return new_mean, new_covariance\n\n def gating_distance(self, mean, covariance, measurements,\n only_position=False):\n \"\"\"Compute gating distance between state distribution and measurements.\n A suitable distance threshold can be obtained from `chi2inv95`. If\n `only_position` is False, the chi-square distribution has 4 degrees of\n freedom, otherwise 2.\n Parameters\n ----------\n mean : ndarray\n Mean vector over the state distribution (8 dimensional).\n covariance : ndarray\n Covariance of the state distribution (8x8 dimensional).\n measurements : ndarray\n An Nx4 dimensional matrix of N measurements, each in\n format (x, y, a, h) where (x, y) is the bounding box center\n position, a the aspect ratio, and h the height.\n only_position : Optional[bool]\n If True, distance computation is done with respect to the bounding\n box center position only.\n Returns\n -------\n ndarray\n Returns an array of length N, where the i-th element contains the\n squared Mahalanobis distance between (mean, covariance) and\n `measurements[i]`.\n \"\"\"\n mean, covariance = self.project(mean, covariance)\n\n if only_position:\n mean, covariance = mean[:2], covariance[:2, :2]\n measurements = measurements[:, :2]\n\n cholesky_factor = np.linalg.cholesky(covariance)\n d = measurements - mean\n z = scipy.linalg.solve_triangular(\n cholesky_factor, d.T, lower=True, check_finite=False,\n overwrite_b=True)\n squared_maha = np.sum(z * z, axis=0)\n return squared_maha" }, { "path": "asone/trackers/strong_sort/tracker/sort/linear_assignment.py", "content": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom scipy.optimize import linear_sum_assignment\nfrom . import kalman_filter\n\n\nINFTY_COST = 1e+5\n\n\ndef min_cost_matching(\n distance_metric, max_distance, tracks, detections, track_indices=None,\n detection_indices=None):\n \"\"\"Solve linear assignment problem.\n Parameters\n ----------\n distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray\n The distance metric is given a list of tracks and detections as well as\n a list of N track indices and M detection indices. The metric should\n return the NxM dimensional cost matrix, where element (i, j) is the\n association cost between the i-th track in the given track indices and\n the j-th detection in the given detection_indices.\n max_distance : float\n Gating threshold. Associations with cost larger than this value are\n disregarded.\n tracks : List[track.Track]\n A list of predicted tracks at the current time step.\n detections : List[detection.Detection]\n A list of detections at the current time step.\n track_indices : List[int]\n List of track indices that maps rows in `cost_matrix` to tracks in\n `tracks` (see description above).\n detection_indices : List[int]\n List of detection indices that maps columns in `cost_matrix` to\n detections in `detections` (see description above).\n Returns\n -------\n (List[(int, int)], List[int], List[int])\n Returns a tuple with the following three entries:\n * A list of matched track and detection indices.\n * A list of unmatched track indices.\n * A list of unmatched detection indices.\n \"\"\"\n if track_indices is None:\n track_indices = np.arange(len(tracks))\n if detection_indices is None:\n detection_indices = np.arange(len(detections))\n\n if len(detection_indices) == 0 or len(track_indices) == 0:\n return [], track_indices, detection_indices # Nothing to match.\n\n cost_matrix = distance_metric(\n tracks, detections, track_indices, detection_indices)\n cost_matrix[cost_matrix > max_distance] = max_distance + 1e-5\n row_indices, col_indices = linear_sum_assignment(cost_matrix)\n\n matches, unmatched_tracks, unmatched_detections = [], [], []\n for col, detection_idx in enumerate(detection_indices):\n if col not in col_indices:\n unmatched_detections.append(detection_idx)\n for row, track_idx in enumerate(track_indices):\n if row not in row_indices:\n unmatched_tracks.append(track_idx)\n for row, col in zip(row_indices, col_indices):\n track_idx = track_indices[row]\n detection_idx = detection_indices[col]\n if cost_matrix[row, col] > max_distance:\n unmatched_tracks.append(track_idx)\n unmatched_detections.append(detection_idx)\n else:\n matches.append((track_idx, detection_idx))\n return matches, unmatched_tracks, unmatched_detections\n\n\ndef matching_cascade(\n distance_metric, max_distance, cascade_depth, tracks, detections,\n track_indices=None, detection_indices=None):\n \"\"\"Run matching cascade.\n Parameters\n ----------\n distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray\n The distance metric is given a list of tracks and detections as well as\n a list of N track indices and M detection indices. The metric should\n return the NxM dimensional cost matrix, where element (i, j) is the\n association cost between the i-th track in the given track indices and\n the j-th detection in the given detection indices.\n max_distance : float\n Gating threshold. Associations with cost larger than this value are\n disregarded.\n cascade_depth: int\n The cascade depth, should be se to the maximum track age.\n tracks : List[track.Track]\n A list of predicted tracks at the current time step.\n detections : List[detection.Detection]\n A list of detections at the current time step.\n track_indices : Optional[List[int]]\n List of track indices that maps rows in `cost_matrix` to tracks in\n `tracks` (see description above). Defaults to all tracks.\n detection_indices : Optional[List[int]]\n List of detection indices that maps columns in `cost_matrix` to\n detections in `detections` (see description above). Defaults to all\n detections.\n Returns\n -------\n (List[(int, int)], List[int], List[int])\n Returns a tuple with the following three entries:\n * A list of matched track and detection indices.\n * A list of unmatched track indices.\n * A list of unmatched detection indices.\n \"\"\"\n if track_indices is None:\n track_indices = list(range(len(tracks)))\n if detection_indices is None:\n detection_indices = list(range(len(detections)))\n\n unmatched_detections = detection_indices\n matches = []\n track_indices_l = [\n k for k in track_indices\n # if tracks[k].time_since_update == 1 + level\n ]\n matches_l, _, unmatched_detections = \\\n min_cost_matching(\n distance_metric, max_distance, tracks, detections,\n track_indices_l, unmatched_detections)\n matches += matches_l\n unmatched_tracks = list(set(track_indices) - set(k for k, _ in matches))\n return matches, unmatched_tracks, unmatched_detections\n\n\ndef gate_cost_matrix(\n cost_matrix, tracks, detections, track_indices, detection_indices,\n gated_cost=INFTY_COST, only_position=False):\n \"\"\"Invalidate infeasible entries in cost matrix based on the state\n distributions obtained by Kalman filtering.\n Parameters\n ----------\n kf : The Kalman filter.\n cost_matrix : ndarray\n The NxM dimensional cost matrix, where N is the number of track indices\n and M is the number of detection indices, such that entry (i, j) is the\n association cost between `tracks[track_indices[i]]` and\n `detections[detection_indices[j]]`.\n tracks : List[track.Track]\n A list of predicted tracks at the current time step.\n detections : List[detection.Detection]\n A list of detections at the current time step.\n track_indices : List[int]\n List of track indices that maps rows in `cost_matrix` to tracks in\n `tracks` (see description above).\n detection_indices : List[int]\n List of detection indices that maps columns in `cost_matrix` to\n detections in `detections` (see description above).\n gated_cost : Optional[float]\n Entries in the cost matrix corresponding to infeasible associations are\n set this value. Defaults to a very large value.\n only_position : Optional[bool]\n If True, only the x, y position of the state distribution is considered\n during gating. Defaults to False.\n Returns\n -------\n ndarray\n Returns the modified cost matrix.\n \"\"\"\n gating_dim = 2 if only_position else 4\n gating_threshold = kalman_filter.chi2inv95[gating_dim]\n measurements = np.asarray(\n [detections[i].to_xyah() for i in detection_indices])\n for row, track_idx in enumerate(track_indices):\n track = tracks[track_idx]\n gating_distance = track.kf.gating_distance(track.mean, track.covariance, measurements, only_position)\n cost_matrix[row, gating_distance > gating_threshold] = gated_cost\n cost_matrix[row] = 0.995 * cost_matrix[row] + (1 - 0.995) * gating_distance\n return cost_matrix" }, { "path": "asone/trackers/strong_sort/tracker/sort/nn_matching.py", "content": "# vim: expandtab:ts=4:sw=4\nimport sys\n\nimport numpy as np\nimport torch\n\n\ndef _pdist(a, b):\n \"\"\"Compute pair-wise squared distance between points in `a` and `b`.\n Parameters\n ----------\n a : array_like\n An NxM matrix of N samples of dimensionality M.\n b : array_like\n An LxM matrix of L samples of dimensionality M.\n Returns\n -------\n ndarray\n Returns a matrix of size len(a), len(b) such that eleement (i, j)\n contains the squared distance between `a[i]` and `b[j]`.\n \"\"\"\n a, b = np.asarray(a), np.asarray(b)\n if len(a) == 0 or len(b) == 0:\n return np.zeros((len(a), len(b)))\n a2, b2 = np.square(a).sum(axis=1), np.square(b).sum(axis=1)\n r2 = -2.0 * np.dot(a, b.T) + a2[:, None] + b2[None, :]\n r2 = np.clip(r2, 0.0, float(np.inf))\n return r2\n\n\ndef _cosine_distance(a, b, data_is_normalized=False):\n \"\"\"Compute pair-wise cosine distance between points in `a` and `b`.\n Parameters\n ----------\n a : array_like\n An NxM matrix of N samples of dimensionality M.\n b : array_like\n An LxM matrix of L samples of dimensionality M.\n data_is_normalized : Optional[bool]\n If True, assumes rows in a and b are unit length vectors.\n Otherwise, a and b are explicitly normalized to lenght 1.\n Returns\n -------\n ndarray\n Returns a matrix of size len(a), len(b) such that eleement (i, j)\n contains the squared distance between `a[i]` and `b[j]`.\n \"\"\"\n if not data_is_normalized:\n a = np.asarray(a) / np.linalg.norm(a, axis=1, keepdims=True)\n b = np.asarray(b) / np.linalg.norm(b, axis=1, keepdims=True)\n return 1.0 - np.dot(a, b.T)\n\n\ndef _nn_euclidean_distance(x, y):\n \"\"\"Helper function for nearest neighbor distance metric (Euclidean).\n Parameters\n ----------\n x : ndarray\n A matrix of N row-vectors (sample points).\n y : ndarray\n A matrix of M row-vectors (query points).\n Returns\n -------\n ndarray\n A vector of length M that contains for each entry in `y` the\n smallest Euclidean distance to a sample in `x`.\n \"\"\"\n # x_ = torch.from_numpy(np.asarray(x) / np.linalg.norm(x, axis=1, keepdims=True))\n # y_ = torch.from_numpy(np.asarray(y) / np.linalg.norm(y, axis=1, keepdims=True))\n distances = distances = _pdist(x, y)\n return np.maximum(0.0, torch.min(distances, axis=0)[0].numpy())\n\n\ndef _nn_cosine_distance(x, y):\n \"\"\"Helper function for nearest neighbor distance metric (cosine).\n Parameters\n ----------\n x : ndarray\n A matrix of N row-vectors (sample points).\n y : ndarray\n A matrix of M row-vectors (query points).\n Returns\n -------\n ndarray\n A vector of length M that contains for each entry in `y` the\n smallest cosine distance to a sample in `x`.\n \"\"\"\n x_ = torch.from_numpy(np.asarray(x))\n y_ = torch.from_numpy(np.asarray(y))\n distances = _cosine_distance(x_, y_)\n distances = distances\n return distances.min(axis=0)\n\n\nclass NearestNeighborDistanceMetric(object):\n \"\"\"\n A nearest neighbor distance metric that, for each target, returns\n the closest distance to any sample that has been observed so far.\n Parameters\n ----------\n metric : str\n Either \"euclidean\" or \"cosine\".\n matching_threshold: float\n The matching threshold. Samples with larger distance are considered an\n invalid match.\n budget : Optional[int]\n If not None, fix samples per class to at most this number. Removes\n the oldest samples when the budget is reached.\n Attributes\n ----------\n samples : Dict[int -> List[ndarray]]\n A dictionary that maps from target identities to the list of samples\n that have been observed so far.\n \"\"\"\n\n def __init__(self, metric, matching_threshold, budget=None):\n if metric == \"euclidean\":\n self._metric = _nn_euclidean_distance\n elif metric == \"cosine\":\n self._metric = _nn_cosine_distance\n else:\n raise ValueError(\"Invalid metric; must be either 'euclidean' or 'cosine'\")\n self.matching_threshold = matching_threshold\n self.budget = budget\n self.samples = {}\n\n def partial_fit(self, features, targets, active_targets):\n \"\"\"Update the distance metric with new data.\n Parameters\n ----------\n features : ndarray\n An NxM matrix of N features of dimensionality M.\n targets : ndarray\n An integer array of associated target identities.\n active_targets : List[int]\n A list of targets that are currently present in the scene.\n \"\"\"\n for feature, target in zip(features, targets):\n self.samples.setdefault(target, []).append(feature)\n if self.budget is not None:\n self.samples[target] = self.samples[target][-self.budget :]\n self.samples = {k: self.samples[k] for k in active_targets}\n\n def distance(self, features, targets):\n \"\"\"Compute distance between features and targets.\n Parameters\n ----------\n features : ndarray\n An NxM matrix of N features of dimensionality M.\n targets : List[int]\n A list of targets to match the given `features` against.\n Returns\n -------\n ndarray\n Returns a cost matrix of shape len(targets), len(features), where\n element (i, j) contains the closest squared distance between\n `targets[i]` and `features[j]`.\n \"\"\"\n cost_matrix = np.zeros((len(targets), len(features)))\n for i, target in enumerate(targets):\n cost_matrix[i, :] = self._metric(self.samples[target], features)\n return cost_matrix\n" }, { "path": "asone/trackers/strong_sort/tracker/sort/preprocessing.py", "content": "# vim: expandtab:ts=4:sw=4\nimport numpy as np\nimport cv2\n\n\ndef non_max_suppression(boxes, max_bbox_overlap, scores=None):\n \"\"\"Suppress overlapping detections.\n\n Original code from [1]_ has been adapted to include confidence score.\n\n .. [1] http://www.pyimagesearch.com/2015/02/16/\n faster-non-maximum-suppression-python/\n\n Examples\n --------\n\n >>> boxes = [d.roi for d in detections]\n >>> scores = [d.confidence for d in detections]\n >>> indices = non_max_suppression(boxes, max_bbox_overlap, scores)\n >>> detections = [detections[i] for i in indices]\n\n Parameters\n ----------\n boxes : ndarray\n Array of ROIs (x, y, width, height).\n max_bbox_overlap : float\n ROIs that overlap more than this values are suppressed.\n scores : Optional[array_like]\n Detector confidence score.\n\n Returns\n -------\n List[int]\n Returns indices of detections that have survived non-maxima suppression.\n\n \"\"\"\n if len(boxes) == 0:\n return []\n\n boxes = boxes.astype(np.float)\n pick = []\n\n x1 = boxes[:, 0]\n y1 = boxes[:, 1]\n x2 = boxes[:, 2] + boxes[:, 0]\n y2 = boxes[:, 3] + boxes[:, 1]\n\n area = (x2 - x1 + 1) * (y2 - y1 + 1)\n if scores is not None:\n idxs = np.argsort(scores)\n else:\n idxs = np.argsort(y2)\n\n while len(idxs) > 0:\n last = len(idxs) - 1\n i = idxs[last]\n pick.append(i)\n\n xx1 = np.maximum(x1[i], x1[idxs[:last]])\n yy1 = np.maximum(y1[i], y1[idxs[:last]])\n xx2 = np.minimum(x2[i], x2[idxs[:last]])\n yy2 = np.minimum(y2[i], y2[idxs[:last]])\n\n w = np.maximum(0, xx2 - xx1 + 1)\n h = np.maximum(0, yy2 - yy1 + 1)\n\n overlap = (w * h) / area[idxs[:last]]\n\n idxs = np.delete(\n idxs, np.concatenate(\n ([last], np.where(overlap > max_bbox_overlap)[0])))\n\n return pick\n" }, { "path": "asone/trackers/strong_sort/tracker/sort/track.py", "content": "# vim: expandtab:ts=4:sw=4\nimport cv2\nimport numpy as np\nfrom asone.trackers.strong_sort.tracker.sort.kalman_filter import KalmanFilter\n\n\nclass TrackState:\n \"\"\"\n Enumeration type for the single target track state. Newly created tracks are\n classified as `tentative` until enough evidence has been collected. Then,\n the track state is changed to `confirmed`. Tracks that are no longer alive\n are classified as `deleted` to mark them for removal from the set of active\n tracks.\n\n \"\"\"\n\n Tentative = 1\n Confirmed = 2\n Deleted = 3\n\n\nclass Track:\n \"\"\"\n A single target track with state space `(x, y, a, h)` and associated\n velocities, where `(x, y)` is the center of the bounding box, `a` is the\n aspect ratio and `h` is the height.\n\n Parameters\n ----------\n mean : ndarray\n Mean vector of the initial state distribution.\n covariance : ndarray\n Covariance matrix of the initial state distribution.\n track_id : int\n A unique track identifier.\n n_init : int\n Number of consecutive detections before the track is confirmed. The\n track state is set to `Deleted` if a miss occurs within the first\n `n_init` frames.\n max_age : int\n The maximum number of consecutive misses before the track state is\n set to `Deleted`.\n feature : Optional[ndarray]\n Feature vector of the detection this track originates from. If not None,\n this feature is added to the `features` cache.\n\n Attributes\n ----------\n mean : ndarray\n Mean vector of the initial state distribution.\n covariance : ndarray\n Covariance matrix of the initial state distribution.\n track_id : int\n A unique track identifier.\n hits : int\n Total number of measurement updates.\n age : int\n Total number of frames since first occurance.\n time_since_update : int\n Total number of frames since last measurement update.\n state : TrackState\n The current track state.\n features : List[ndarray]\n A cache of features. On each measurement update, the associated feature\n vector is added to this list.\n\n \"\"\"\n\n def __init__(self, detection, track_id, class_id, conf, n_init, max_age, ema_alpha,\n feature=None):\n self.track_id = track_id\n self.class_id = int(class_id)\n self.hits = 1\n self.age = 1\n self.time_since_update = 0\n self.ema_alpha = ema_alpha\n\n self.state = TrackState.Tentative\n self.features = []\n if feature is not None:\n feature /= np.linalg.norm(feature)\n self.features.append(feature)\n\n self.conf = conf\n self._n_init = n_init\n self._max_age = max_age\n\n self.kf = KalmanFilter()\n self.mean, self.covariance = self.kf.initiate(detection)\n\n def to_tlwh(self):\n \"\"\"Get current position in bounding box format `(top left x, top left y,\n width, height)`.\n\n Returns\n -------\n ndarray\n The bounding box.\n\n \"\"\"\n ret = self.mean[:4].copy()\n ret[2] *= ret[3]\n ret[:2] -= ret[2:] / 2\n return ret\n\n def to_tlbr(self):\n \"\"\"Get kf estimated current position in bounding box format `(min x, miny, max x,\n max y)`.\n\n Returns\n -------\n ndarray\n The predicted kf bounding box.\n\n \"\"\"\n ret = self.to_tlwh()\n ret[2:] = ret[:2] + ret[2:]\n return ret\n\n\n def ECC(self, src, dst, warp_mode = cv2.MOTION_EUCLIDEAN, eps = 1e-5,\n max_iter = 100, scale = 0.1, align = False):\n \"\"\"Compute the warp matrix from src to dst.\n Parameters\n ----------\n src : ndarray \n An NxM matrix of source img(BGR or Gray), it must be the same format as dst.\n dst : ndarray\n An NxM matrix of target img(BGR or Gray).\n warp_mode: flags of opencv\n translation: cv2.MOTION_TRANSLATION\n rotated and shifted: cv2.MOTION_EUCLIDEAN\n affine(shift,rotated,shear): cv2.MOTION_AFFINE\n homography(3d): cv2.MOTION_HOMOGRAPHY\n eps: float\n the threshold of the increment in the correlation coefficient between two iterations\n max_iter: int\n the number of iterations.\n scale: float or [int, int]\n scale_ratio: float\n scale_size: [W, H]\n align: bool\n whether to warp affine or perspective transforms to the source image\n Returns\n -------\n warp matrix : ndarray\n Returns the warp matrix from src to dst.\n if motion models is homography, the warp matrix will be 3x3, otherwise 2x3\n src_aligned: ndarray\n aligned source image of gray\n \"\"\"\n\n # skip if current and previous frame are not initialized (1st inference)\n if (src.any() or dst.any() is None):\n return None, None\n # skip if current and previous fames are not the same size\n elif (src.shape != dst.shape):\n return None, None\n\n # BGR2GRAY\n if src.ndim == 3:\n # Convert images to grayscale\n src = cv2.cvtColor(src, cv2.COLOR_BGR2GRAY)\n dst = cv2.cvtColor(dst, cv2.COLOR_BGR2GRAY)\n\n # make the imgs smaller to speed up\n if scale is not None:\n if isinstance(scale, float) or isinstance(scale, int):\n if scale != 1:\n src_r = cv2.resize(src, (0, 0), fx = scale, fy = scale,interpolation = cv2.INTER_LINEAR)\n dst_r = cv2.resize(dst, (0, 0), fx = scale, fy = scale,interpolation = cv2.INTER_LINEAR)\n scale = [scale, scale]\n else:\n src_r, dst_r = src, dst\n scale = None\n else:\n if scale[0] != src.shape[1] and scale[1] != src.shape[0]:\n src_r = cv2.resize(src, (scale[0], scale[1]), interpolation = cv2.INTER_LINEAR)\n dst_r = cv2.resize(dst, (scale[0], scale[1]), interpolation=cv2.INTER_LINEAR)\n scale = [scale[0] / src.shape[1], scale[1] / src.shape[0]]\n else:\n src_r, dst_r = src, dst\n scale = None\n else:\n src_r, dst_r = src, dst\n\n # Define 2x3 or 3x3 matrices and initialize the matrix to identity\n if warp_mode == cv2.MOTION_HOMOGRAPHY :\n warp_matrix = np.eye(3, 3, dtype=np.float32)\n else :\n warp_matrix = np.eye(2, 3, dtype=np.float32)\n\n # Define termination criteria\n criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, max_iter, eps)\n\n # Run the ECC algorithm. The results are stored in warp_matrix.\n try:\n (cc, warp_matrix) = cv2.findTransformECC (src_r, dst_r, warp_matrix, warp_mode, criteria, None, 1)\n except cv2.error as e:\n return None, None\n \n\n if scale is not None:\n warp_matrix[0, 2] = warp_matrix[0, 2] / scale[0]\n warp_matrix[1, 2] = warp_matrix[1, 2] / scale[1]\n\n if align:\n sz = src.shape\n if warp_mode == cv2.MOTION_HOMOGRAPHY:\n # Use warpPerspective for Homography\n src_aligned = cv2.warpPerspective(src, warp_matrix, (sz[1],sz[0]), flags=cv2.INTER_LINEAR)\n else :\n # Use warpAffine for Translation, Euclidean and Affine\n src_aligned = cv2.warpAffine(src, warp_matrix, (sz[1],sz[0]), flags=cv2.INTER_LINEAR)\n return warp_matrix, src_aligned\n else:\n return warp_matrix, None\n\n\n def get_matrix(self, matrix):\n eye = np.eye(3)\n dist = np.linalg.norm(eye - matrix)\n if dist < 100:\n return matrix\n else:\n return eye\n\n def camera_update(self, previous_frame, next_frame):\n warp_matrix, src_aligned = self.ECC(previous_frame, next_frame)\n if warp_matrix is None and src_aligned is None:\n return\n [a,b] = warp_matrix\n warp_matrix=np.array([a,b,[0,0,1]])\n warp_matrix = warp_matrix.tolist()\n matrix = self.get_matrix(warp_matrix)\n\n x1, y1, x2, y2 = self.to_tlbr()\n x1_, y1_, _ = matrix @ np.array([x1, y1, 1]).T\n x2_, y2_, _ = matrix @ np.array([x2, y2, 1]).T\n w, h = x2_ - x1_, y2_ - y1_\n cx, cy = x1_ + w / 2, y1_ + h / 2\n self.mean[:4] = [cx, cy, w / h, h]\n\n\n def increment_age(self):\n self.age += 1\n self.time_since_update += 1\n\n def predict(self, kf):\n \"\"\"Propagate the state distribution to the current time step using a\n Kalman filter prediction step.\n\n Parameters\n ----------\n kf : kalman_filter.KalmanFilter\n The Kalman filter.\n\n \"\"\"\n self.mean, self.covariance = self.kf.predict(self.mean, self.covariance)\n self.age += 1\n self.time_since_update += 1\n\n def update(self, detection, class_id, conf):\n \"\"\"Perform Kalman filter measurement update step and update the feature\n cache.\n Parameters\n ----------\n detection : Detection\n The associated detection.\n \"\"\"\n self.conf = conf\n self.class_id = int(class_id)\n self.mean, self.covariance = self.kf.update(self.mean, self.covariance, detection.to_xyah(), detection.confidence)\n\n feature = detection.feature / np.linalg.norm(detection.feature)\n\n smooth_feat = self.ema_alpha * self.features[-1] + (1 - self.ema_alpha) * feature\n smooth_feat /= np.linalg.norm(smooth_feat)\n self.features = [smooth_feat]\n\n self.hits += 1\n self.time_since_update = 0\n if self.state == TrackState.Tentative and self.hits >= self._n_init:\n self.state = TrackState.Confirmed\n\n def mark_missed(self):\n \"\"\"Mark this track as missed (no association at the current time step).\n \"\"\"\n if self.state == TrackState.Tentative:\n self.state = TrackState.Deleted\n elif self.time_since_update > self._max_age:\n self.state = TrackState.Deleted\n\n def is_tentative(self):\n \"\"\"Returns True if this track is tentative (unconfirmed).\n \"\"\"\n return self.state == TrackState.Tentative\n\n def is_confirmed(self):\n \"\"\"Returns True if this track is confirmed.\"\"\"\n return self.state == TrackState.Confirmed\n\n def is_deleted(self):\n \"\"\"Returns True if this track is dead and should be deleted.\"\"\"\n return self.state == TrackState.Deleted\n" }, { "path": "asone/trackers/strong_sort/tracker/sort/tracker.py", "content": "# vim: expandtab:ts=4:sw=4\nfrom __future__ import absolute_import\nimport numpy as np\nfrom . import kalman_filter\nfrom . import linear_assignment\nfrom . import iou_matching\nfrom .track import Track\n\n\nclass Tracker:\n \"\"\"\n This is the multi-target tracker.\n Parameters\n ----------\n metric : nn_matching.NearestNeighborDistanceMetric\n A distance metric for measurement-to-track association.\n max_age : int\n Maximum number of missed misses before a track is deleted.\n n_init : int\n Number of consecutive detections before the track is confirmed. The\n track state is set to `Deleted` if a miss occurs within the first\n `n_init` frames.\n Attributes\n ----------\n metric : nn_matching.NearestNeighborDistanceMetric\n The distance metric used for measurement to track association.\n max_age : int\n Maximum number of missed misses before a track is deleted.\n n_init : int\n Number of frames that a track remains in initialization phase.\n kf : kalman_filter.KalmanFilter\n A Kalman filter to filter target trajectories in image space.\n tracks : List[Track]\n The list of active tracks at the current time step.\n \"\"\"\n GATING_THRESHOLD = np.sqrt(kalman_filter.chi2inv95[4])\n\n def __init__(self, metric, max_iou_distance=0.9, max_age=30, n_init=3, _lambda=0, ema_alpha=0.9, mc_lambda=0.995):\n self.metric = metric\n self.max_iou_distance = max_iou_distance\n self.max_age = max_age\n self.n_init = n_init\n self._lambda = _lambda\n self.ema_alpha = ema_alpha\n self.mc_lambda = mc_lambda\n\n self.kf = kalman_filter.KalmanFilter()\n self.tracks = []\n self._next_id = 1\n\n def predict(self):\n \"\"\"Propagate track state distributions one time step forward.\n\n This function should be called once every time step, before `update`.\n \"\"\"\n for track in self.tracks:\n track.predict(self.kf)\n\n def increment_ages(self):\n for track in self.tracks:\n track.increment_age()\n track.mark_missed()\n\n def camera_update(self, previous_img, current_img):\n for track in self.tracks:\n track.camera_update(previous_img, current_img)\n\n def update(self, detections, classes, confidences):\n \"\"\"Perform measurement update and track management.\n\n Parameters\n ----------\n detections : List[deep_sort.detection.Detection]\n A list of detections at the current time step.\n\n \"\"\"\n # Run matching cascade.\n matches, unmatched_tracks, unmatched_detections = \\\n self._match(detections)\n\n # Update track set.\n for track_idx, detection_idx in matches:\n self.tracks[track_idx].update(\n detections[detection_idx], classes[detection_idx], confidences[detection_idx])\n for track_idx in unmatched_tracks:\n self.tracks[track_idx].mark_missed()\n for detection_idx in unmatched_detections:\n self._initiate_track(detections[detection_idx], classes[detection_idx].item(), confidences[detection_idx].item())\n self.tracks = [t for t in self.tracks if not t.is_deleted()]\n\n # Update distance metric.\n active_targets = [t.track_id for t in self.tracks if t.is_confirmed()]\n features, targets = [], []\n for track in self.tracks:\n if not track.is_confirmed():\n continue\n features += track.features\n targets += [track.track_id for _ in track.features]\n self.metric.partial_fit(np.asarray(features), np.asarray(targets), active_targets)\n\n def _full_cost_metric(self, tracks, dets, track_indices, detection_indices):\n \"\"\"\n This implements the full lambda-based cost-metric. However, in doing so, it disregards\n the possibility to gate the position only which is provided by\n linear_assignment.gate_cost_matrix(). Instead, I gate by everything.\n Note that the Mahalanobis distance is itself an unnormalised metric. Given the cosine\n distance being normalised, we employ a quick and dirty normalisation based on the\n threshold: that is, we divide the positional-cost by the gating threshold, thus ensuring\n that the valid values range 0-1.\n Note also that the authors work with the squared distance. I also sqrt this, so that it\n is more intuitive in terms of values.\n \"\"\"\n # Compute First the Position-based Cost Matrix\n pos_cost = np.empty([len(track_indices), len(detection_indices)])\n msrs = np.asarray([dets[i].to_xyah() for i in detection_indices])\n for row, track_idx in enumerate(track_indices):\n pos_cost[row, :] = np.sqrt(\n self.kf.gating_distance(\n tracks[track_idx].mean, tracks[track_idx].covariance, msrs, False\n )\n ) / self.GATING_THRESHOLD\n pos_gate = pos_cost > 1.0\n # Now Compute the Appearance-based Cost Matrix\n app_cost = self.metric.distance(\n np.array([dets[i].feature for i in detection_indices]),\n np.array([tracks[i].track_id for i in track_indices]),\n )\n app_gate = app_cost > self.metric.matching_threshold\n # Now combine and threshold\n cost_matrix = self._lambda * pos_cost + (1 - self._lambda) * app_cost\n cost_matrix[np.logical_or(pos_gate, app_gate)] = linear_assignment.INFTY_COST\n # Return Matrix\n return cost_matrix\n\n def _match(self, detections):\n\n def gated_metric(tracks, dets, track_indices, detection_indices):\n features = np.array([dets[i].feature for i in detection_indices])\n targets = np.array([tracks[i].track_id for i in track_indices])\n cost_matrix = self.metric.distance(features, targets)\n cost_matrix = linear_assignment.gate_cost_matrix(cost_matrix, tracks, dets, track_indices, detection_indices)\n\n return cost_matrix\n\n # Split track set into confirmed and unconfirmed tracks.\n confirmed_tracks = [\n i for i, t in enumerate(self.tracks) if t.is_confirmed()]\n unconfirmed_tracks = [\n i for i, t in enumerate(self.tracks) if not t.is_confirmed()]\n\n # Associate confirmed tracks using appearance features.\n matches_a, unmatched_tracks_a, unmatched_detections = \\\n linear_assignment.matching_cascade(\n gated_metric, self.metric.matching_threshold, self.max_age,\n self.tracks, detections, confirmed_tracks)\n\n # Associate remaining tracks together with unconfirmed tracks using IOU.\n iou_track_candidates = unconfirmed_tracks + [\n k for k in unmatched_tracks_a if\n self.tracks[k].time_since_update == 1]\n unmatched_tracks_a = [\n k for k in unmatched_tracks_a if\n self.tracks[k].time_since_update != 1]\n matches_b, unmatched_tracks_b, unmatched_detections = \\\n linear_assignment.min_cost_matching(\n iou_matching.iou_cost, self.max_iou_distance, self.tracks,\n detections, iou_track_candidates, unmatched_detections)\n\n matches = matches_a + matches_b\n unmatched_tracks = list(set(unmatched_tracks_a + unmatched_tracks_b))\n return matches, unmatched_tracks, unmatched_detections\n\n def _initiate_track(self, detection, class_id, conf):\n self.tracks.append(Track(\n detection.to_xyah(), self._next_id, class_id, conf, self.n_init, self.max_age, self.ema_alpha,\n detection.feature))\n self._next_id += 1\n" }, { "path": "asone/trackers/strong_sort/tracker/strong_sort.py", "content": "import numpy as np\nimport torch\nimport sys\nimport gdown\nfrom os.path import exists as file_exists, join\n\nfrom .sort.nn_matching import NearestNeighborDistanceMetric\nfrom .sort.detection import Detection\nfrom .sort.tracker import Tracker\nfrom .deep.reid_model_factory import show_downloadeable_models, get_model_url, get_model_name\n\nfrom torchreid.reid.utils import FeatureExtractor\nfrom torchreid.reid.utils.tools import download_url\n\n__all__ = ['StrongSORT']\n\ndef xyxy2xywh(x):\n # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right\n y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n y[:, 0] = (x[:, 0] + x[:, 2]) / 2 # x center\n y[:, 1] = (x[:, 1] + x[:, 3]) / 2 # y center\n y[:, 2] = x[:, 2] - x[:, 0] # width\n y[:, 3] = x[:, 3] - x[:, 1] # height\n return y\n\nclass StrongSORT(object):\n def __init__(self, \n model_weights,\n device, max_dist=0.2,\n max_iou_distance=0.7,\n max_age=70, n_init=3,\n nn_budget=100,\n mc_lambda=0.995,\n ema_alpha=0.9\n ):\n model_name = get_model_name(model_weights)\n model_url = get_model_url(model_weights)\n\n if not file_exists(model_weights) and model_url is not None:\n gdown.download(model_url, str(model_weights), quiet=False)\n elif file_exists(model_weights):\n pass\n elif model_url is None:\n print('No URL associated to the chosen DeepSort weights. Choose between:')\n show_downloadeable_models()\n exit()\n\n self.extractor = FeatureExtractor(\n # get rid of dataset information DeepSort model name\n model_name=model_name,\n model_path=model_weights,\n device=str(device)\n )\n\n self.max_dist = max_dist\n metric = NearestNeighborDistanceMetric(\n \"cosine\", self.max_dist, nn_budget)\n self.tracker = Tracker(\n metric, max_iou_distance=max_iou_distance, max_age=max_age, n_init=n_init)\n\n def update(self, bbox_xywh, confs, classes, ori_img):\n \n xywhs = xyxy2xywh(bbox_xywh)\n confs = confs\n self.height, self.width = ori_img.shape[:2]\n\n # generate detections\n features = self._get_features(xywhs, ori_img)\n bbox_tlwh = self._xywh_to_tlwh(xywhs)\n detections = [Detection(bbox_tlwh[i], conf, features[i]) for i, conf in enumerate(confs)]\n\n\n # run on non-maximum supression\n boxes = np.array([d.tlwh for d in detections])\n scores = np.array([d.confidence for d in detections])\n\n # update tracker\n self.tracker.predict()\n self.tracker.update(detections, classes, confs)\n\n # output bbox identities\n outputs = []\n for track in self.tracker.tracks:\n if not track.is_confirmed() or track.time_since_update > 1:\n continue\n\n box = track.to_tlwh()\n x1, y1, x2, y2 = self._tlwh_to_xyxy(box)\n \n track_id = track.track_id\n class_id = track.class_id\n conf = track.conf\n outputs.append(np.array([x1, y1, x2, y2, track_id, class_id, conf]))\n if len(outputs) > 0:\n outputs = np.stack(outputs, axis=0)\n return outputs\n\n \"\"\"\n TODO:\n Convert bbox from xc_yc_w_h to xtl_ytl_w_h\n Thanks JieChen91@github.com for reporting this bug!\n \"\"\"\n @staticmethod\n def _xywh_to_tlwh(bbox_xywh):\n if isinstance(bbox_xywh, np.ndarray):\n bbox_tlwh = bbox_xywh.copy()\n elif isinstance(bbox_xywh, torch.Tensor):\n bbox_tlwh = bbox_xywh.clone()\n bbox_tlwh[:, 0] = bbox_xywh[:, 0] - bbox_xywh[:, 2] / 2.\n bbox_tlwh[:, 1] = bbox_xywh[:, 1] - bbox_xywh[:, 3] / 2.\n return bbox_tlwh\n\n def _xywh_to_xyxy(self, bbox_xywh):\n x, y, w, h = bbox_xywh\n x1 = max(int(x - w / 2), 0)\n x2 = min(int(x + w / 2), self.width - 1)\n y1 = max(int(y - h / 2), 0)\n y2 = min(int(y + h / 2), self.height - 1)\n return x1, y1, x2, y2\n\n def _tlwh_to_xyxy(self, bbox_tlwh):\n \"\"\"\n TODO:\n Convert bbox from xtl_ytl_w_h to xc_yc_w_h\n Thanks JieChen91@github.com for reporting this bug!\n \"\"\"\n x, y, w, h = bbox_tlwh\n x1 = max(int(x), 0)\n x2 = min(int(x+w), self.width - 1)\n y1 = max(int(y), 0)\n y2 = min(int(y+h), self.height - 1)\n return x1, y1, x2, y2\n\n def increment_ages(self):\n self.tracker.increment_ages()\n\n def _xyxy_to_tlwh(self, bbox_xyxy):\n x1, y1, x2, y2 = bbox_xyxy\n\n t = x1\n l = y1\n w = int(x2 - x1)\n h = int(y2 - y1)\n return t, l, w, h\n\n def _get_features(self, bbox_xywh, ori_img):\n im_crops = []\n for box in bbox_xywh:\n x1, y1, x2, y2 = self._xywh_to_xyxy(box)\n im = ori_img[y1:y2, x1:x2]\n im_crops.append(im)\n if im_crops:\n features = self.extractor(im_crops)\n else:\n features = np.array([])\n return features\n" }, { "path": "asone/trackers/strong_sort/tracker/utils/__init__.py", "content": "" }, { "path": "asone/trackers/strong_sort/tracker/utils/asserts.py", "content": "from os import environ\n\n\ndef assert_in(file, files_to_check):\n if file not in files_to_check:\n raise AssertionError(\"{} does not exist in the list\".format(str(file)))\n return True\n\n\ndef assert_in_env(check_list: list):\n for item in check_list:\n assert_in(item, environ.keys())\n return True\n" }, { "path": "asone/trackers/strong_sort/tracker/utils/draw.py", "content": "import numpy as np\nimport cv2\n\npalette = (2 ** 11 - 1, 2 ** 15 - 1, 2 ** 20 - 1)\n\n\ndef compute_color_for_labels(label):\n \"\"\"\n Simple function that adds fixed color depending on the class\n \"\"\"\n color = [int((p * (label ** 2 - label + 1)) % 255) for p in palette]\n return tuple(color)\n\n\ndef draw_boxes(img, bbox, identities=None, offset=(0,0)):\n for i,box in enumerate(bbox):\n x1,y1,x2,y2 = [int(i) for i in box]\n x1 += offset[0]\n x2 += offset[0]\n y1 += offset[1]\n y2 += offset[1]\n # box text and bar\n id = int(identities[i]) if identities is not None else 0 \n color = compute_color_for_labels(id)\n label = '{}{:d}'.format(\"\", id)\n t_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_PLAIN, 2 , 2)[0]\n cv2.rectangle(img,(x1, y1),(x2,y2),color,3)\n cv2.rectangle(img,(x1, y1),(x1+t_size[0]+3,y1+t_size[1]+4), color,-1)\n cv2.putText(img,label,(x1,y1+t_size[1]+4), cv2.FONT_HERSHEY_PLAIN, 2, [255,255,255], 2)\n return img\n\n\n\nif __name__ == '__main__':\n for i in range(82):\n print(compute_color_for_labels(i))\n" }, { "path": "asone/trackers/strong_sort/tracker/utils/evaluation.py", "content": "import os\nimport numpy as np\nimport copy\nimport motmetrics as mm\nmm.lap.default_solver = 'lap'\nfrom utils.io import read_results, unzip_objs\n\n\nclass Evaluator(object):\n\n def __init__(self, data_root, seq_name, data_type):\n self.data_root = data_root\n self.seq_name = seq_name\n self.data_type = data_type\n\n self.load_annotations()\n self.reset_accumulator()\n\n def load_annotations(self):\n assert self.data_type == 'mot'\n\n gt_filename = os.path.join(self.data_root, self.seq_name, 'gt', 'gt.txt')\n self.gt_frame_dict = read_results(gt_filename, self.data_type, is_gt=True)\n self.gt_ignore_frame_dict = read_results(gt_filename, self.data_type, is_ignore=True)\n\n def reset_accumulator(self):\n self.acc = mm.MOTAccumulator(auto_id=True)\n\n def eval_frame(self, frame_id, trk_tlwhs, trk_ids, rtn_events=False):\n # results\n trk_tlwhs = np.copy(trk_tlwhs)\n trk_ids = np.copy(trk_ids)\n\n # gts\n gt_objs = self.gt_frame_dict.get(frame_id, [])\n gt_tlwhs, gt_ids = unzip_objs(gt_objs)[:2]\n\n # ignore boxes\n ignore_objs = self.gt_ignore_frame_dict.get(frame_id, [])\n ignore_tlwhs = unzip_objs(ignore_objs)[0]\n\n\n # remove ignored results\n keep = np.ones(len(trk_tlwhs), dtype=bool)\n iou_distance = mm.distances.iou_matrix(ignore_tlwhs, trk_tlwhs, max_iou=0.5)\n if len(iou_distance) > 0:\n match_is, match_js = mm.lap.linear_sum_assignment(iou_distance)\n match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js])\n match_ious = iou_distance[match_is, match_js]\n\n match_js = np.asarray(match_js, dtype=int)\n match_js = match_js[np.logical_not(np.isnan(match_ious))]\n keep[match_js] = False\n trk_tlwhs = trk_tlwhs[keep]\n trk_ids = trk_ids[keep]\n\n # get distance matrix\n iou_distance = mm.distances.iou_matrix(gt_tlwhs, trk_tlwhs, max_iou=0.5)\n\n # acc\n self.acc.update(gt_ids, trk_ids, iou_distance)\n\n if rtn_events and iou_distance.size > 0 and hasattr(self.acc, 'last_mot_events'):\n events = self.acc.last_mot_events # only supported by https://github.com/longcw/py-motmetrics\n else:\n events = None\n return events\n\n def eval_file(self, filename):\n self.reset_accumulator()\n\n result_frame_dict = read_results(filename, self.data_type, is_gt=False)\n frames = sorted(list(set(self.gt_frame_dict.keys()) | set(result_frame_dict.keys())))\n for frame_id in frames:\n trk_objs = result_frame_dict.get(frame_id, [])\n trk_tlwhs, trk_ids = unzip_objs(trk_objs)[:2]\n self.eval_frame(frame_id, trk_tlwhs, trk_ids, rtn_events=False)\n\n return self.acc\n\n @staticmethod\n def get_summary(accs, names, metrics=('mota', 'num_switches', 'idp', 'idr', 'idf1', 'precision', 'recall')):\n names = copy.deepcopy(names)\n if metrics is None:\n metrics = mm.metrics.motchallenge_metrics\n metrics = copy.deepcopy(metrics)\n\n mh = mm.metrics.create()\n summary = mh.compute_many(\n accs,\n metrics=metrics,\n names=names,\n generate_overall=True\n )\n\n return summary\n\n @staticmethod\n def save_summary(summary, filename):\n import pandas as pd\n writer = pd.ExcelWriter(filename)\n summary.to_excel(writer)\n writer.save()\n" }, { "path": "asone/trackers/strong_sort/tracker/utils/io.py", "content": "import os\nfrom typing import Dict\nimport numpy as np\n\n# from utils.log import get_logger\n\n\ndef write_results(filename, results, data_type):\n if data_type == 'mot':\n save_format = '{frame},{id},{x1},{y1},{w},{h},-1,-1,-1,-1\\n'\n elif data_type == 'kitti':\n save_format = '{frame} {id} pedestrian 0 0 -10 {x1} {y1} {x2} {y2} -10 -10 -10 -1000 -1000 -1000 -10\\n'\n else:\n raise ValueError(data_type)\n\n with open(filename, 'w') as f:\n for frame_id, tlwhs, track_ids in results:\n if data_type == 'kitti':\n frame_id -= 1\n for tlwh, track_id in zip(tlwhs, track_ids):\n if track_id < 0:\n continue\n x1, y1, w, h = tlwh\n x2, y2 = x1 + w, y1 + h\n line = save_format.format(frame=frame_id, id=track_id, x1=x1, y1=y1, x2=x2, y2=y2, w=w, h=h)\n f.write(line)\n\n\n# def write_results(filename, results_dict: Dict, data_type: str):\n# if not filename:\n# return\n# path = os.path.dirname(filename)\n# if not os.path.exists(path):\n# os.makedirs(path)\n\n# if data_type in ('mot', 'mcmot', 'lab'):\n# save_format = '{frame},{id},{x1},{y1},{w},{h},1,-1,-1,-1\\n'\n# elif data_type == 'kitti':\n# save_format = '{frame} {id} pedestrian -1 -1 -10 {x1} {y1} {x2} {y2} -1 -1 -1 -1000 -1000 -1000 -10 {score}\\n'\n# else:\n# raise ValueError(data_type)\n\n# with open(filename, 'w') as f:\n# for frame_id, frame_data in results_dict.items():\n# if data_type == 'kitti':\n# frame_id -= 1\n# for tlwh, track_id in frame_data:\n# if track_id < 0:\n# continue\n# x1, y1, w, h = tlwh\n# x2, y2 = x1 + w, y1 + h\n# line = save_format.format(frame=frame_id, id=track_id, x1=x1, y1=y1, x2=x2, y2=y2, w=w, h=h, score=1.0)\n# f.write(line)\n# logger.info('Save results to {}'.format(filename))\n\n\ndef read_results(filename, data_type: str, is_gt=False, is_ignore=False):\n if data_type in ('mot', 'lab'):\n read_fun = read_mot_results\n else:\n raise ValueError('Unknown data type: {}'.format(data_type))\n\n return read_fun(filename, is_gt, is_ignore)\n\n\n\"\"\"\nlabels={'ped', ...\t\t\t% 1\n'person_on_vhcl', ...\t% 2\n'car', ...\t\t\t\t% 3\n'bicycle', ...\t\t\t% 4\n'mbike', ...\t\t\t% 5\n'non_mot_vhcl', ...\t\t% 6\n'static_person', ...\t% 7\n'distractor', ...\t\t% 8\n'occluder', ...\t\t\t% 9\n'occluder_on_grnd', ...\t\t%10\n'occluder_full', ...\t\t% 11\n'reflection', ...\t\t% 12\n'crowd' ...\t\t\t% 13\n};\n\"\"\"\n\n\ndef read_mot_results(filename, is_gt, is_ignore):\n valid_labels = {1}\n ignore_labels = {2, 7, 8, 12}\n results_dict = dict()\n if os.path.isfile(filename):\n with open(filename, 'r') as f:\n for line in f.readlines():\n linelist = line.split(',')\n if len(linelist) < 7:\n continue\n fid = int(linelist[0])\n if fid < 1:\n continue\n results_dict.setdefault(fid, list())\n\n if is_gt:\n if 'MOT16-' in filename or 'MOT17-' in filename:\n label = int(float(linelist[7]))\n mark = int(float(linelist[6]))\n if mark == 0 or label not in valid_labels:\n continue\n score = 1\n elif is_ignore:\n if 'MOT16-' in filename or 'MOT17-' in filename:\n label = int(float(linelist[7]))\n vis_ratio = float(linelist[8])\n if label not in ignore_labels and vis_ratio >= 0:\n continue\n else:\n continue\n score = 1\n else:\n score = float(linelist[6])\n\n tlwh = tuple(map(float, linelist[2:6]))\n target_id = int(linelist[1])\n\n results_dict[fid].append((tlwh, target_id, score))\n\n return results_dict\n\n\ndef unzip_objs(objs):\n if len(objs) > 0:\n tlwhs, ids, scores = zip(*objs)\n else:\n tlwhs, ids, scores = [], [], []\n tlwhs = np.asarray(tlwhs, dtype=float).reshape(-1, 4)\n\n return tlwhs, ids, scores" }, { "path": "asone/trackers/strong_sort/tracker/utils/json_logger.py", "content": "\"\"\"\nReferences:\n https://medium.com/analytics-vidhya/creating-a-custom-logging-mechanism-for-real-time-object-detection-using-tdd-4ca2cfcd0a2f\n\"\"\"\nimport json\nfrom os import makedirs\nfrom os.path import exists, join\nfrom datetime import datetime\n\n\nclass JsonMeta(object):\n HOURS = 3\n MINUTES = 59\n SECONDS = 59\n PATH_TO_SAVE = 'LOGS'\n DEFAULT_FILE_NAME = 'remaining'\n\n\nclass BaseJsonLogger(object):\n \"\"\"\n This is the base class that returns __dict__ of its own\n it also returns the dicts of objects in the attributes that are list instances\n\n \"\"\"\n\n def dic(self):\n # returns dicts of objects\n out = {}\n for k, v in self.__dict__.items():\n if hasattr(v, 'dic'):\n out[k] = v.dic()\n elif isinstance(v, list):\n out[k] = self.list(v)\n else:\n out[k] = v\n return out\n\n @staticmethod\n def list(values):\n # applies the dic method on items in the list\n return [v.dic() if hasattr(v, 'dic') else v for v in values]\n\n\nclass Label(BaseJsonLogger):\n \"\"\"\n For each bounding box there are various categories with confidences. Label class keeps track of that information.\n \"\"\"\n\n def __init__(self, category: str, confidence: float):\n self.category = category\n self.confidence = confidence\n\n\nclass Bbox(BaseJsonLogger):\n \"\"\"\n This module stores the information for each frame and use them in JsonParser\n Attributes:\n labels (list): List of label module.\n top (int):\n left (int):\n width (int):\n height (int):\n\n Args:\n bbox_id (float):\n top (int):\n left (int):\n width (int):\n height (int):\n\n References:\n Check Label module for better understanding.\n\n\n \"\"\"\n\n def __init__(self, bbox_id, top, left, width, height):\n self.labels = []\n self.bbox_id = bbox_id\n self.top = top\n self.left = left\n self.width = width\n self.height = height\n\n def add_label(self, category, confidence):\n # adds category and confidence only if top_k is not exceeded.\n self.labels.append(Label(category, confidence))\n\n def labels_full(self, value):\n return len(self.labels) == value\n\n\nclass Frame(BaseJsonLogger):\n \"\"\"\n This module stores the information for each frame and use them in JsonParser\n Attributes:\n timestamp (float): The elapsed time of captured frame\n frame_id (int): The frame number of the captured video\n bboxes (list of Bbox objects): Stores the list of bbox objects.\n\n References:\n Check Bbox class for better information\n\n Args:\n timestamp (float):\n frame_id (int):\n\n \"\"\"\n\n def __init__(self, frame_id: int, timestamp: float = None):\n self.frame_id = frame_id\n self.timestamp = timestamp\n self.bboxes = []\n\n def add_bbox(self, bbox_id: int, top: int, left: int, width: int, height: int):\n bboxes_ids = [bbox.bbox_id for bbox in self.bboxes]\n if bbox_id not in bboxes_ids:\n self.bboxes.append(Bbox(bbox_id, top, left, width, height))\n else:\n raise ValueError(\"Frame with id: {} already has a Bbox with id: {}\".format(self.frame_id, bbox_id))\n\n def add_label_to_bbox(self, bbox_id: int, category: str, confidence: float):\n bboxes = {bbox.id: bbox for bbox in self.bboxes}\n if bbox_id in bboxes.keys():\n res = bboxes.get(bbox_id)\n res.add_label(category, confidence)\n else:\n raise ValueError('the bbox with id: {} does not exists!'.format(bbox_id))\n\n\nclass BboxToJsonLogger(BaseJsonLogger):\n \"\"\"\n ُ This module is designed to automate the task of logging jsons. An example json is used\n to show the contents of json file shortly\n Example:\n {\n \"video_details\": {\n \"frame_width\": 1920,\n \"frame_height\": 1080,\n \"frame_rate\": 20,\n \"video_name\": \"/home/gpu/codes/MSD/pedestrian_2/project/public/camera1.avi\"\n },\n \"frames\": [\n {\n \"frame_id\": 329,\n \"timestamp\": 3365.1254\n \"bboxes\": [\n {\n \"labels\": [\n {\n \"category\": \"pedestrian\",\n \"confidence\": 0.9\n }\n ],\n \"bbox_id\": 0,\n \"top\": 1257,\n \"left\": 138,\n \"width\": 68,\n \"height\": 109\n }\n ]\n }],\n\n Attributes:\n frames (dict): It's a dictionary that maps each frame_id to json attributes.\n video_details (dict): information about video file.\n top_k_labels (int): shows the allowed number of labels\n start_time (datetime object): we use it to automate the json output by time.\n\n Args:\n top_k_labels (int): shows the allowed number of labels\n\n \"\"\"\n\n def __init__(self, top_k_labels: int = 1):\n self.frames = {}\n self.video_details = self.video_details = dict(frame_width=None, frame_height=None, frame_rate=None,\n video_name=None)\n self.top_k_labels = top_k_labels\n self.start_time = datetime.now()\n\n def set_top_k(self, value):\n self.top_k_labels = value\n\n def frame_exists(self, frame_id: int) -> bool:\n \"\"\"\n Args:\n frame_id (int):\n\n Returns:\n bool: true if frame_id is recognized\n \"\"\"\n return frame_id in self.frames.keys()\n\n def add_frame(self, frame_id: int, timestamp: float = None) -> None:\n \"\"\"\n Args:\n frame_id (int):\n timestamp (float): opencv captured frame time property\n\n Raises:\n ValueError: if frame_id would not exist in class frames attribute\n\n Returns:\n None\n\n \"\"\"\n if not self.frame_exists(frame_id):\n self.frames[frame_id] = Frame(frame_id, timestamp)\n else:\n raise ValueError(\"Frame id: {} already exists\".format(frame_id))\n\n def bbox_exists(self, frame_id: int, bbox_id: int) -> bool:\n \"\"\"\n Args:\n frame_id:\n bbox_id:\n\n Returns:\n bool: if bbox exists in frame bboxes list\n \"\"\"\n bboxes = []\n if self.frame_exists(frame_id=frame_id):\n bboxes = [bbox.bbox_id for bbox in self.frames[frame_id].bboxes]\n return bbox_id in bboxes\n\n def find_bbox(self, frame_id: int, bbox_id: int):\n \"\"\"\n\n Args:\n frame_id:\n bbox_id:\n\n Returns:\n bbox_id (int):\n\n Raises:\n ValueError: if bbox_id does not exist in the bbox list of specific frame.\n \"\"\"\n if not self.bbox_exists(frame_id, bbox_id):\n raise ValueError(\"frame with id: {} does not contain bbox with id: {}\".format(frame_id, bbox_id))\n bboxes = {bbox.bbox_id: bbox for bbox in self.frames[frame_id].bboxes}\n return bboxes.get(bbox_id)\n\n def add_bbox_to_frame(self, frame_id: int, bbox_id: int, top: int, left: int, width: int, height: int) -> None:\n \"\"\"\n\n Args:\n frame_id (int):\n bbox_id (int):\n top (int):\n left (int):\n width (int):\n height (int):\n\n Returns:\n None\n\n Raises:\n ValueError: if bbox_id already exist in frame information with frame_id\n ValueError: if frame_id does not exist in frames attribute\n \"\"\"\n if self.frame_exists(frame_id):\n frame = self.frames[frame_id]\n if not self.bbox_exists(frame_id, bbox_id):\n frame.add_bbox(bbox_id, top, left, width, height)\n else:\n raise ValueError(\n \"frame with frame_id: {} already contains the bbox with id: {} \".format(frame_id, bbox_id))\n else:\n raise ValueError(\"frame with frame_id: {} does not exist\".format(frame_id))\n\n def add_label_to_bbox(self, frame_id: int, bbox_id: int, category: str, confidence: float):\n \"\"\"\n Args:\n frame_id:\n bbox_id:\n category:\n confidence: the confidence value returned from yolo detection\n\n Returns:\n None\n\n Raises:\n ValueError: if labels quota (top_k_labels) exceeds.\n \"\"\"\n bbox = self.find_bbox(frame_id, bbox_id)\n if not bbox.labels_full(self.top_k_labels):\n bbox.add_label(category, confidence)\n else:\n raise ValueError(\"labels in frame_id: {}, bbox_id: {} is fulled\".format(frame_id, bbox_id))\n\n def add_video_details(self, frame_width: int = None, frame_height: int = None, frame_rate: int = None,\n video_name: str = None):\n self.video_details['frame_width'] = frame_width\n self.video_details['frame_height'] = frame_height\n self.video_details['frame_rate'] = frame_rate\n self.video_details['video_name'] = video_name\n\n def output(self):\n output = {'video_details': self.video_details}\n result = list(self.frames.values())\n output['frames'] = [item.dic() for item in result]\n return output\n\n def json_output(self, output_name):\n \"\"\"\n Args:\n output_name:\n\n Returns:\n None\n\n Notes:\n It creates the json output with `output_name` name.\n \"\"\"\n if not output_name.endswith('.json'):\n output_name += '.json'\n with open(output_name, 'w') as file:\n json.dump(self.output(), file)\n file.close()\n\n def set_start(self):\n self.start_time = datetime.now()\n\n def schedule_output_by_time(self, output_dir=JsonMeta.PATH_TO_SAVE, hours: int = 0, minutes: int = 0,\n seconds: int = 60) -> None:\n \"\"\"\n Notes:\n Creates folder and then periodically stores the jsons on that address.\n\n Args:\n output_dir (str): the directory where output files will be stored\n hours (int):\n minutes (int):\n seconds (int):\n\n Returns:\n None\n\n \"\"\"\n end = datetime.now()\n interval = 0\n interval += abs(min([hours, JsonMeta.HOURS]) * 3600)\n interval += abs(min([minutes, JsonMeta.MINUTES]) * 60)\n interval += abs(min([seconds, JsonMeta.SECONDS]))\n diff = (end - self.start_time).seconds\n\n if diff > interval:\n output_name = self.start_time.strftime('%Y-%m-%d %H-%M-%S') + '.json'\n if not exists(output_dir):\n makedirs(output_dir)\n output = join(output_dir, output_name)\n self.json_output(output_name=output)\n self.frames = {}\n self.start_time = datetime.now()\n\n def schedule_output_by_frames(self, frames_quota, frame_counter, output_dir=JsonMeta.PATH_TO_SAVE):\n \"\"\"\n saves as the number of frames quota increases higher.\n :param frames_quota:\n :param frame_counter:\n :param output_dir:\n :return:\n \"\"\"\n pass\n\n def flush(self, output_dir):\n \"\"\"\n Notes:\n We use this function to output jsons whenever possible.\n like the time that we exit the while loop of opencv.\n\n Args:\n output_dir:\n\n Returns:\n None\n\n \"\"\"\n filename = self.start_time.strftime('%Y-%m-%d %H-%M-%S') + '-remaining.json'\n output = join(output_dir, filename)\n self.json_output(output_name=output)\n" }, { "path": "asone/trackers/strong_sort/tracker/utils/log.py", "content": "import logging\n\n\ndef get_logger(name='root'):\n formatter = logging.Formatter(\n # fmt='%(asctime)s [%(levelname)s]: %(filename)s(%(funcName)s:%(lineno)s) >> %(message)s')\n fmt='%(asctime)s [%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')\n\n handler = logging.StreamHandler()\n handler.setFormatter(formatter)\n\n logger = logging.getLogger(name)\n logger.setLevel(logging.INFO)\n logger.addHandler(handler)\n return logger\n\n\n" }, { "path": "asone/trackers/strong_sort/tracker/utils/parser.py", "content": "import os\nimport yaml\nfrom easydict import EasyDict as edict\n\n\nclass YamlParser(edict):\n \"\"\"\n This is yaml parser based on EasyDict.\n \"\"\"\n\n def __init__(self, cfg_dict=None, config_file=None):\n if cfg_dict is None:\n cfg_dict = {}\n\n if config_file is not None:\n assert(os.path.isfile(config_file))\n with open(config_file, 'r') as fo:\n yaml_ = yaml.load(fo.read(), Loader=yaml.FullLoader)\n cfg_dict.update(yaml_)\n\n super(YamlParser, self).__init__(cfg_dict)\n\n def merge_from_file(self, config_file):\n with open(config_file, 'r') as fo:\n yaml_ = yaml.load(fo.read(), Loader=yaml.FullLoader)\n self.update(yaml_)\n\n def merge_from_dict(self, config_dict):\n self.update(config_dict)\n\n\ndef get_config(config_file=None):\n return YamlParser(config_file=config_file)\n\n\nif __name__ == \"__main__\":\n cfg = YamlParser(config_file=\"../configs/yolov3.yaml\")\n cfg.merge_from_file(\"../configs/strong_sort.yaml\")\n\n import ipdb\n ipdb.set_trace()\n" }, { "path": "asone/trackers/strong_sort/tracker/utils/tools.py", "content": "from functools import wraps\nfrom time import time\n\n\ndef is_video(ext: str):\n \"\"\"\n Returns true if ext exists in\n allowed_exts for video files.\n\n Args:\n ext:\n\n Returns:\n\n \"\"\"\n\n allowed_exts = ('.mp4', '.webm', '.ogg', '.avi', '.wmv', '.mkv', '.3gp')\n return any((ext.endswith(x) for x in allowed_exts))\n\n\ndef tik_tok(func):\n \"\"\"\n keep track of time for each process.\n Args:\n func:\n\n Returns:\n\n \"\"\"\n @wraps(func)\n def _time_it(*args, **kwargs):\n start = time()\n try:\n return func(*args, **kwargs)\n finally:\n end_ = time()\n print(\"time: {:.03f}s, fps: {:.03f}\".format(end_ - start, 1 / (end_ - start)))\n\n return _time_it\n" }, { "path": "asone/trackers/tracker.py", "content": "from asone.trackers import ByteTrack\nfrom asone.trackers import NorFair\nfrom asone.trackers import DeepSort\nfrom asone.trackers import Motpy\nfrom asone.trackers import OcSort\nfrom asone.trackers import StrongSort\n\n\nclass Tracker:\n def __init__(self, tracker: int, detector: object, use_cuda=True) -> None:\n \n self.trackers = {\n '0': ByteTrack,\n '1': DeepSort,\n '2': NorFair,\n '3': Motpy,\n '4': OcSort,\n '5': StrongSort\n }\n\n self.tracker = self._select_tracker(tracker, detector, use_cuda=use_cuda)\n\n def _select_tracker(self, tracker, detector, use_cuda):\n _tracker = self.trackers.get(str(tracker), None)\n\n if _tracker is not None:\n if _tracker is DeepSort:\n return _tracker(detector, use_cuda=use_cuda)\n else:\n return _tracker(detector)\n else:\n raise ValueError(f'Invalid tracker: {tracker}')\n\n def detect_and_track(self, image, config: dict):\n \n return self.tracker.detect_and_track(image, config)\n\n def get_tracker(self):\n return self.tracker" }, { "path": "asone/utils/__init__.py", "content": "from asone.utils.classes import get_names\nfrom asone.utils.download import download_weights\nfrom asone.utils.colors import compute_color_for_labels\nfrom asone.utils.counting import estimateSpeed, intersect\nfrom asone.utils.ponits_conversion import xyxy_to_tlwh, xyxy_to_xywh, tlwh_to_xyxy\nfrom asone.utils.temp_loader import get_detector, get_tracker\n\nfrom asone.utils.draw import draw_boxes\nfrom asone.utils.draw import draw_text\nfrom asone.utils.draw import draw_kpts\nfrom asone.utils.draw import plot_skeleton_kpts\nfrom asone.utils.pose_estimators_weights import get_weight_path" }, { "path": "asone/utils/classes.py", "content": "\n# names = []\n# with open('classes.txt') as f:\n# names.append(f.read())\n\nglobal names\nnames = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light',\n 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow',\n 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',\n 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard',\n 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',\n 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',\n 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone',\n 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear',\n 'hair drier', 'toothbrush', \"text\"]\n\ndef get_names():\n return names" }, { "path": "asone/utils/colors.py", "content": "from numpy import random\nfrom asone.utils import get_names\nnames = get_names()\n\ncolors = [[random.randint(0, 255) for _ in range(3)]\n for _ in range(len(names))]\npalette = (2 ** 11 - 1, 2 ** 15 - 1, 2 ** 20 - 1)\n\n\ndef compute_color_for_labels(label):\n \"\"\"\n Simple function that adds fixed color depending on the class\n \"\"\"\n if label == 0: # person #BGR\n color = (85, 45, 255)\n elif label == 2: # Car\n color = (222, 82, 175)\n elif label == 3: # Motobike\n color = (0, 204, 255)\n elif label == 5: # Bus\n color = (0, 149, 255)\n else:\n color = [int((p * (label ** 2 - label + 1)) % 255) for p in palette]\n return tuple(color)\n" }, { "path": "asone/utils/counting.py", "content": "import math\n\ndef estimateSpeed(location1, location2):\n\n d_pixels = math.sqrt(math.pow(\n location2[0] - location1[0], 2) + math.pow(location2[1] - location1[1], 2))\n ppm = 8 # Pixels per Meter\n d_meters = d_pixels / ppm\n time_constant = 15 * 3.6\n speed = d_meters * time_constant\n return speed\n\n# Return true if line segments AB and CD intersect\n\n\ndef intersect(A, B, C, D):\n return ccw(A, C, D) != ccw(B, C, D) and ccw(A, B, C) != ccw(A, B, D)\n\n\ndef ccw(A, B, C):\n return (C[1]-A[1]) * (B[0]-A[0]) > (B[1]-A[1]) * (C[0]-A[0])\n" }, { "path": "asone/utils/default_cfg.py", "content": "config = {\n \"output_dir\": \"results\",\n \"filename\": None,\n \"fps\": None,\n \"save_result\": True,\n \"display\": False,\n \"draw_trails\": False,\n \"filter_classes\": None,\n \"class_names\": None,\n \"input_shape\" : (640, 640),\n \"conf_thres\": 0.25,\n \"iou_thres\" : 0.45,\n \"max_det\" : 1000,\n \"agnostic_nms\" : False,\n \"with_p6\" : False\n}\n" }, { "path": "asone/utils/download.py", "content": "import gdown\nimport os\nimport zipfile\n\n\ndef exractfile(file, dest):\n with zipfile.ZipFile(file, 'r') as zip_ref:\n zip_ref.extractall(dest)\n\n\ndef download_weights(weights):\n\n outputpath = os.path.dirname(weights)\n model = os.path.splitext(os.path.basename(weights))[0]\n filename = f'{model}.zip'\n if model == 'yolov5s':\n model_key = '1W5ypZmrYE4_Aqu3Jqsl-IDvLK6SOtJCK'\n elif model == 'yolov5x6':\n model_key = '1CTYtGC8VFZD0uJbU4fcSPSjcYb8Be0jU'\n elif model == 'yolov5n':\n model_key = '1q9_e76T_b353QmG5xGi3zkselGQrxuBk'\n elif model == 'yolov5m':\n model_key = '1Vv3VEkgYd7WB-3e2MPo0QUKq_-F7biBP'\n elif model == 'yolov5l':\n model_key = '1Wr4S7BTqqCOCP14T_aDgVxGV5h9pMM-n'\n elif model == 'yolov5x':\n model_key = '18g_pjpwsnOlBKbhApuaTsbQUIDO75BSt'\n elif model == 'yolov5n6':\n model_key = '1gOAZ90nKcvo7bhNZCZM-JRuY8FBeEkhl'\n elif model == 'yolov5s6':\n model_key = '12W1Z0esjFc9UhiWWxCESjhm5KA3nzSwt'\n elif model == 'yolov5m6':\n model_key = '1O-bbJ8WcqLig40IUyJ8ulKwj-J_KXvHr'\n elif model == 'yolov5l6':\n model_key = '1sPZ-YpenYojZSIB5G1SMv2hPWX5oRwlb'\n elif model == 'yolov6n':\n model_key = '1NA_u4BkPE_N8HcPmZrd7HyLmvFHOk8qd'\n elif model == 'yolov6t':\n model_key = '16OWncBp-vh-sLDMOR58th3WOGv4envQ1'\n elif model == 'yolov6s':\n model_key = '14BE0j654ClLxMq2ySWZNhTCmf48mLyXi'\n elif model == 'yolov6l_relu':\n model_key = '14UfY057QUQoAj6q39PX_qE7U1bBIrIGi'\n elif model == 'yolov6l':\n model_key = '1HdRIs0uMPbqs5E2aEX8O3d3dJTh-KBTf'\n elif model == 'yolov6m':\n model_key = '1t_w9SCwbZAW7icwX_z97-SQz-plXzBgM'\n elif model == 'yolov6s_repopt':\n model_key = '1L_1Crxx-4059xDDUZEf_asWRBVd3PF05'\n elif model == 'yolov7-e6e':\n model_key = '1W9vEbEv8expm0Yz9R9VIn8ToqDM9TgIt' \n elif model == 'yolov7-d6':\n model_key = '1_Ybtx7EAXnBwIZ59Vgo0FkXsCdjRuq2s'\n elif model == 'yolov7':\n model_key = '17iFeNfq5hKZVpQLQEgZzxF9Da5o5llLG'\n elif model == 'yolov7-tiny':\n model_key = '18Fels44wVJ1vG7yDuDPuWwuiAqeFxKI7'\n elif model == 'yolov7-e6':\n model_key = '1g_2nYpeJ28cLYcOAUeztHUA5R-stk3Cm'\n elif model == 'yolov7-w6':\n model_key = '1wv3M23RFo0MhaujegBPY6gZ30IA894CO'\n elif model == 'yolov7x':\n model_key = '1zZskyvdgU45Ke8TtCA6csdCzqLrhmFYx'\n elif model == 'yolor_csp':\n model_key = '1G3FBZKrznW_64mGfs6b3nAJiJv6GmmV0'\n elif model == 'yolor_csp_star':\n model_key = '15WDl46ZthFGZfpOyI3qXx6gC9FQLH_wH'\n elif model == 'yolor_csp_x':\n model_key = '1LU2ckh7eSpVD0nyPSdq1n34lKmNAX39T'\n elif model == 'yolor_csp_x_star':\n model_key = '1jheqFDm7BpHQpR60wuWSBpbuyK5SoKdV'\n elif model == 'yolor_p6':\n model_key = '1XKREKdxQCO8OXiW2IWGFhczolIaIr9sm'\n elif model == 'yolox_l':\n model_key = '1jX1KHerOdZ5-dmXh6cWcRAn80aKD-7sP'\n elif model == 'yolox_nano':\n model_key = '1783Os6uTpYjunL-MfK0WE1Wcwk58fIUi'\n elif model == 'yolox_tiny':\n model_key = '1Lcv1ITvfPdWsu6Kb8Hq6cOSfJE7lbbf2'\n elif model == 'yolox_darknet':\n model_key = '17f4UI06TWJ25Oqo2OoQGu8AoGVX1lPta'\n elif model == 'yolox_s':\n model_key = '1IUAOv62XuwkwwTCVD2y3xJg7KUA3M0-M'\n elif model == 'yolox_m':\n model_key = '1ktHj8UEwl0V8Qz9G74E-yj-o13FLeD0-'\n elif model == 'yolox_x':\n model_key = '13HNnlILCx_XamNJWwJ1MG5x0XfP6HL1U'\n elif model == 'ckpt':\n model_key = '1VZ05gzg249Q1m8BJVQxl3iHoNIbjzJf8'\n\n elif model == 'yolov8s':\n model_key = '1hUhjQWw1cJL7TtBG0zD3MO52iCYh0DEn'\n elif model == 'yolov8n':\n model_key = '1x6zHzsEcyhuyWy2xY3swAQ4vIQvBYrsr'\n elif model == 'yolov8l':\n model_key = '1xQxHTEIpoiP4d73F6dtedU7hIZpFTqY2'\n elif model == 'yolov8m':\n model_key = '1_FoKnqkaoWchVy4B24Hn2PanEKfh-eSp'\n elif model == 'yolov8x':\n model_key = '1s60fsjiyDlPQ1L5H_GAoWahHONLbvz7T'\n \n elif model == 'yolov8s-pose':\n model_key = '1Gfehs7CzQJgBOceLK6WrWpHXilUlgRyc'\n elif model == 'yolov8n-pose':\n model_key = '1NE0BR9OuwWKwadrBQN904R7NBXMpdyqy'\n elif model == 'yolov8l-pose':\n model_key = '1nzArC823KleZk0_DCA-D6Jm6kHLrPQw_'\n elif model == 'yolov8m-pose':\n model_key = '1p8W_FNUsY9_fIVsDAZjbLnqA2ViF0j8P'\n elif model == 'yolov8x-pose':\n model_key = '1Z0tZg1OYStkbzFNa5DJS_5qegNM_YtWH'\n elif model == 'yolov7-w6-pose':\n model_key = '1R_7QH-Y9TJ0jnDn3STfZvZzbx1fqs1mZ'\n elif model == 'yolo_nas_l':\n model_key = '1FYDcp-LNxlVfxvKU2kG8FmkqZNIOSAnN'\n elif model == 'yolo_nas_m':\n model_key = '1wHf3q5Jl5Uk1iGgRjm6TYChInAoYGNhf'\n elif model == 'yolo_nas_s':\n model_key = '1jtI-L9J8G7sRBa-sziqryMp11vlFKWyh'\n \n elif model == 'yolov9-c-converted':\n model_key = '1PCT4dCKosBB0B26sLdQ5xdjQ2vTPjWaH'\n elif model == 'yolov9-e-converted':\n model_key = '16rNqbrlmN3YgsaxOsYLad1IRK2eo_3AL'\n elif model == 'yolov9-c':\n model_key = '1FRnyftVwTYyH2Or8BkMWE7HB_08dsNkA'\n elif model == 'yolov9-e':\n model_key = '1ayY7dJH6r9hzl1MSOdVFx0UqmWixhEJu'\n elif model == 'gelan-c':\n model_key = '1AhGd8Ex0Kr1iXiSytu-OBpyvLykL9tMF'\n elif model == 'gelan-e':\n model_key = '1vc7nBJTMm3tt_gLD-zJSbmFAaWoP8ZbS'\n \n elif model == \"sam_vit_h_4b8939\":\n model_key = '1-QSw_IqF4WczsC3pdbiIpmjTEaxPc5nO'\n else:\n raise ValueError(f'No model named {model} found.')\n\n url = f'https://drive.google.com/uc?id={model_key}&confirm=t'\n gdown.download(url, output=filename, quiet=False)\n\n if not os.path.exists(outputpath):\n os.makedirs(outputpath)\n\n exractfile(filename, outputpath)\n os.remove(filename)\n\n# def download_weights(weights):\n \n# f = open('asone/utils/weights.json')\n# data = json.load(f)\n \n# outputpath = os.path.dirname(weights)\n# model = os.path.splitext(os.path.basename(weights))[0]\n# filename = f'{model}.zip'\n# if model in data:\n# model_key = data[model]\n# else:\n# raise ValueError(f'No model named {model} found.')\n\n# url = f'https://drive.google.com/uc?id={model_key}&confirm=t'\n# gdown.download(url, output=filename, quiet=False)\n\n# if not os.path.exists(outputpath):\n# os.makedirs(outputpath)\n\n# exractfile(filename, outputpath)\n# os.remove(filename)\n" }, { "path": "asone/utils/draw.py", "content": "import cv2\nfrom numpy import random\nimport numpy as np\nfrom asone.utils import compute_color_for_labels\nfrom asone.utils import get_names\nfrom collections import deque\nfrom asone.schemas.output_schemas import ModelOutput\n\nnames = get_names()\ndata_deque = {}\n\n\ndef draw_ui_box(x, img, label=None, color=None, line_thickness=None):\n # Plots one bounding box on image img\n tl = line_thickness or round(\n 0.002 * (img.shape[0] + img.shape[1]) / 2) + 1 # line/font thickness\n color = color or [random.randint(0, 255) for _ in range(3)]\n c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3]))\n cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)\n \n if label:\n tf = max(tl - 1, 1) # font thickness\n t_size = cv2.getTextSize(str(label), 0, fontScale=tl / 3, thickness=tf)[0]\n # c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3\n img = draw_border(img, (c1[0], c1[1] - t_size[1] - 3),\n (c1[0] + t_size[0], c1[1]+3), color, 1, 8, 2)\n\n # cv2.line(img, c1, c2, color, 30)\n # cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled\n cv2.putText(img, str(label), (c1[0], c1[1] - 2), 0, tl / 3,\n [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)\n return img\n\ndef draw_border(img, pt1, pt2, color, thickness, r, d):\n x1, y1 = pt1\n x2, y2 = pt2\n # Top leftfrom collections import deque (x1, y1 + r + d), color, thickness)\n cv2.ellipse(img, (x1 + r, y1 + r), (r, r), 180, 0, 90, color, thickness)\n\n # Top right\n cv2.line(img, (x2 - r, y1), (x2 - r - d, y1), color, thickness)\n cv2.line(img, (x2, y1 + r), (x2, y1 + r + d), color, thickness)\n cv2.ellipse(img, (x2 - r, y1 + r), (r, r), 270, 0, 90, color, thickness)\n # Bottom left\n cv2.line(img, (x1 + r, y2), (x1 + r + d, y2), color, thickness)\n cv2.line(img, (x1, y2 - r), (x1, y2 - r - d), color, thickness)\n cv2.ellipse(img, (x1 + r, y2 - r), (r, r), 90, 0, 90, color, thickness)\n # Bottom right\n cv2.line(img, (x2 - r, y2), (x2 - r - d, y2), color, thickness)\n cv2.line(img, (x2, y2 - r), (x2, y2 - r - d), color, thickness)\n cv2.ellipse(img, (x2 - r, y2 - r), (r, r), 0, 0, 90, color, thickness)\n\n cv2.rectangle(img, (x1 + r, y1), (x2 - r, y2), color, -1, cv2.LINE_AA)\n cv2.rectangle(img, (x1, y1 + r), (x2, y2 - r - d), color, -1, cv2.LINE_AA)\n\n cv2.circle(img, (x1 + r, y1+r), 2, color, 12)\n cv2.circle(img, (x2 - r, y1+r), 2, color, 12)\n cv2.circle(img, (x1 + r, y2-r), 2, color, 12)\n cv2.circle(img, (x2 - r, y2-r), 2, color, 12)\n\n return img\n\n\ndef draw_boxes(img, bbox_xyxy, class_ids, identities=None, draw_trails=False, offset=(0, 0), class_names=None):\n # cv2.line(img, line2[0], line2[1], (0,200,0), 3)\n height, width, _ = img.shape\n\n # remove tracked point from buffer if object is lost\n if draw_trails:\n for key in list(data_deque):\n if key not in identities:\n data_deque.pop(key)\n \n for i, box in enumerate(bbox_xyxy):\n x1, y1, x2, y2 = [int(i) for i in box]\n x1 += offset[0]\n x2 += offset[0]\n y1 += offset[1]\n y2 += offset[1]\n\n # get ID of object\n id = int(identities[i]) if identities is not None else None\n \n # if class_ids is not None:\n color = compute_color_for_labels(int(class_ids[i]))\n\n if class_names:\n obj_name = class_names[int(class_ids[i])]\n else: \n obj_name = names[int(class_ids[i])]\n \n label = f'{obj_name}' if id is None else f'{id}'\n\n draw_ui_box(box, img, label=label, color=color, line_thickness=2)\n\n # Draw trails \n # code to find center of bottom edge\n center = (int((x2+x1) / 2), int((y2+y2)/2))\n\n if draw_trails:\n # create new buffer for new object\n if id not in data_deque: \n data_deque[id] = deque(maxlen= 64)\n \n data_deque[id].appendleft(center) \n drawtrails(data_deque, id, color, img)\n \n return img\n\ndef drawtrails(data_deque, id, color, img):\n # draw trail\n for i in range(1, len(data_deque[id])):\n # check if on buffer value is none\n if data_deque[id][i - 1] is None or data_deque[id][i] is None:\n continue\n\n # generate dynamic thickness of trails\n thickness = int(np.sqrt(64 / float(i + i)) * 1.5)\n \n # draw trails\n cv2.line(img, data_deque[id][i - 1], data_deque[id][i], color, thickness)\n\ndef draw_text(img, results, offset=(0, 0), display: bool = False):\n color = compute_color_for_labels(int(0))\n for res in results:\n box = res[:4]\n text = res[4]\n x1, y1, x2, y2 = box\n x1 += offset[0]\n x2 += offset[0]\n y1 += offset[1]\n y2 += offset[1]\n label = text\n\n img = draw_ui_box(box, img, label=label, color=color, line_thickness=2)\n center = (int((x2+x1) / 2), int((y2+y2)/2))\n if display:\n cv2.imshow(' Sample', img)\n \n return img\n\n# Utils for code estimation \n\nclass Colors:\n # Ultralytics color palette https://ultralytics.com/\n def __init__(self):\n \"\"\"Initialize colors as hex = matplotlib.colors.TABLEAU_COLORS.values().\"\"\"\n hexs = ('FF3838', 'FF9D97', 'FF701F', 'FFB21D', 'CFD231', '48F90A', '92CC17', '3DDB86', '1A9334', '00D4BB',\n '2C99A8', '00C2FF', '344593', '6473FF', '0018EC', '8438FF', '520085', 'CB38FF', 'FF95C8', 'FF37C7')\n self.palette = [self.hex2rgb(f'#{c}') for c in hexs]\n self.n = len(self.palette)\n self.pose_palette = np.array([[255, 128, 0], [255, 153, 51], [255, 178, 102], [230, 230, 0], [255, 153, 255],\n [153, 204, 255], [255, 102, 255], [255, 51, 255], [102, 178, 255], [51, 153, 255],\n [255, 153, 153], [255, 102, 102], [255, 51, 51], [153, 255, 153], [102, 255, 102],\n [51, 255, 51], [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 255]],\n dtype=np.uint8)\n\n def __call__(self, i, bgr=False):\n \"\"\"Converts hex color codes to rgb values.\"\"\"\n c = self.palette[int(i) % self.n]\n return (c[2], c[1], c[0]) if bgr else c\n\n @staticmethod\n def hex2rgb(h): # rgb order (PIL)\n return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4))\n\n\ncolors = Colors() # create instance for 'from utils.plots import colors'\n\n\ndef draw_kpts(keypoints, image=None, shape=(640, 640), radius=5, kpt_line=True,\n display: bool=True):\n \"\"\"Plot keypoints on the image.\n Args:\n kpts (tensor): Predicted keypoints with shape [17, 3]. Each keypoint has (x, y, confidence).\n shape (tuple): Image shape as a tuple (h, w), where h is the height and w is the width.\n radius (int, optional): Radius of the drawn keypoints. Default is 5.\n kpt_line (bool, optional): If True, the function will draw lines connecting keypoints\n for human pose. Default is True.\n Note: `kpt_line=True` currently only supports human pose plotting.\n \"\"\"\n # if self.pil:\n # # Convert to numpy first\n # self.im = np.asarray(self.im).copy()\n if isinstance(keypoints, ModelOutput):\n image = keypoints.info.image\n keypoints = keypoints.dets.bbox\n if keypoints is not None:\n for kpts in reversed(keypoints):\n \n limb_color = colors.pose_palette[[9, 9, 9, 9, 7, 7, 7, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16]]\n kpt_color = colors.pose_palette[[16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9]]\n skeleton = [[16, 14], [14, 12], [17, 15], [15, 13], [12, 13], [6, 12], [7, 13], [6, 7], [6, 8], [7, 9],\n [8, 10], [9, 11], [2, 3], [1, 2], [1, 3], [2, 4], [3, 5], [4, 6], [5, 7]]\n nkpt, ndim = kpts.shape\n is_pose = nkpt == 17 and ndim in [2, 3]\n kpt_line &= is_pose # `kpt_line=True` for now only supports human pose plotting\n for i, k in enumerate(kpts):\n color_k = [int(x) for x in kpt_color[i]] if is_pose else colors(i)\n x_coord, y_coord = k[0], k[1]\n if x_coord % shape[1] != 0 and y_coord % shape[0] != 0:\n if len(k) == 3:\n conf = k[2]\n if conf < 0.5:\n continue\n cv2.circle(image, (int(x_coord), int(y_coord)), radius, color_k, -1, lineType=cv2.LINE_AA)\n\n if kpt_line:\n ndim = kpts.shape[-1]\n for i, sk in enumerate(skeleton):\n pos1 = (int(kpts[(sk[0] - 1), 0]), int(kpts[(sk[0] - 1), 1]))\n pos2 = (int(kpts[(sk[1] - 1), 0]), int(kpts[(sk[1] - 1), 1]))\n if ndim == 3:\n conf1 = kpts[(sk[0] - 1), 2]\n conf2 = kpts[(sk[1] - 1), 2]\n if conf1 < 0.5 or conf2 < 0.5:\n continue\n if pos1[0] % shape[1] == 0 or pos1[1] % shape[0] == 0 or pos1[0] < 0 or pos1[1] < 0:\n continue\n if pos2[0] % shape[1] == 0 or pos2[1] % shape[0] == 0 or pos2[0] < 0 or pos2[1] < 0:\n continue\n cv2.line(image, pos1, pos2, [int(x) for x in limb_color[i]], thickness=2, lineType=cv2.LINE_AA)\n if display:\n cv2.imshow(' Sample', image)\n return image\n\ndef plot_skeleton_kpts(im, kpts, steps, orig_shape=None):\n #Plot the skeleton and keypointsfor coco datatset\n palette = np.array([[255, 128, 0], [255, 153, 51], [255, 178, 102],\n [230, 230, 0], [255, 153, 255], [153, 204, 255],\n [255, 102, 255], [255, 51, 255], [102, 178, 255],\n [51, 153, 255], [255, 153, 153], [255, 102, 102],\n [255, 51, 51], [153, 255, 153], [102, 255, 102],\n [51, 255, 51], [0, 255, 0], [0, 0, 255], [255, 0, 0],\n [255, 255, 255]])\n\n skeleton = [[16, 14], [14, 12], [17, 15], [15, 13], [12, 13], [6, 12],\n [7, 13], [6, 7], [6, 8], [7, 9], [8, 10], [9, 11], [2, 3],\n [1, 2], [1, 3], [2, 4], [3, 5], [4, 6], [5, 7]]\n\n pose_limb_color = palette[[9, 9, 9, 9, 7, 7, 7, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16]]\n pose_kpt_color = palette[[16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9]]\n radius = 5\n num_kpts = len(kpts) // steps\n\n for kid in range(num_kpts):\n r, g, b = pose_kpt_color[kid]\n x_coord, y_coord = kpts[steps * kid], kpts[steps * kid + 1]\n if not (x_coord % 640 == 0 or y_coord % 640 == 0):\n if steps == 3:\n conf = kpts[steps * kid + 2]\n if conf < 0.5:\n continue\n overlay = im.copy()\n alpha = 0.4\n cv2.circle(overlay, (int(x_coord), int(y_coord)), 8, (int(220), int(237), int(245)), 8)\n cv2.circle(im, (int(x_coord), int(y_coord)), 5, (int(255), int(255), int(255)), -1)\n # im = output\n cv2.addWeighted(overlay, alpha, im, 1 - alpha, 0, im)\n\n for sk_id, sk in enumerate(skeleton):\n r, g, b = pose_limb_color[sk_id]\n pos1 = (int(kpts[(sk[0]-1)*steps]), int(kpts[(sk[0]-1)*steps+1]))\n pos2 = (int(kpts[(sk[1]-1)*steps]), int(kpts[(sk[1]-1)*steps+1]))\n if steps == 3:\n conf1 = kpts[(sk[0]-1)*steps+2]\n conf2 = kpts[(sk[1]-1)*steps+2]\n if conf1<0.5 or conf2<0.5:\n continue\n if pos1[0]%640 == 0 or pos1[1]%640==0 or pos1[0]<0 or pos1[1]<0:\n continue\n if pos2[0] % 640 == 0 or pos2[1] % 640 == 0 or pos2[0]<0 or pos2[1]<0:\n continue\n cv2.line(im, pos1, pos2, (int(255), int(255), int(255)), thickness=2)\n" }, { "path": "asone/utils/ponits_conversion.py", "content": "def xyxy_to_xywh(xyxy):\n \"\"\"\" Calculates the relative bounding box from absolute pixel values. \"\"\"\n bbox_left = min([xyxy[0], xyxy[2]])\n bbox_top = min([xyxy[1], xyxy[3]])\n bbox_w = abs(xyxy[0] - xyxy[2])\n bbox_h = abs(xyxy[1] - xyxy[3])\n x_c = (bbox_left + bbox_w / 2)\n y_c = (bbox_top + bbox_h / 2)\n w = bbox_w\n h = bbox_h\n return [x_c, y_c, w, h]\n\n\ndef tlwh_to_xyxy(tlwh):\n \"\"\"\" Convert tlwh to xyxy \"\"\"\n x1 = tlwh[0]\n y1 = tlwh[1]\n x2 = tlwh[2] + x1\n y2 = tlwh[3] + y1\n return [x1, y1, x2, y2]\n\n\ndef xyxy_to_tlwh(bbox_xyxy):\n tlwh_bboxs = []\n for i, box in enumerate(bbox_xyxy):\n x1, y1, x2, y2 = [int(i) for i in box]\n top = x1\n left = y1\n w = int(x2 - x1)\n h = int(y2 - y1)\n tlwh_obj = [top, left, w, h]\n tlwh_bboxs.append(tlwh_obj)\n return tlwh_bboxs\n" }, { "path": "asone/utils/pose_estimators_weights.py", "content": "import os\n\nweights = { \n \n # '130': os.path.join('yolov7','weights','yolov7-tiny.mlmodel'),\n # '131': os.path.join('yolov7','weights','yolov7.mlmodel'),\n # '132': os.path.join('yolov7','weights','yolov7x.mlmodel'),\n '149': os.path.join('yolov7','weights','yolov7-w6-pose.pt'),\n # '134': os.path.join('yolov7','weights','yolov7-e6.mlmodel'),\n # '135': os.path.join('yolov7','weights','yolov7-d6.mlmodel'),\n # '136': os.path.join('yolov7','weights','yolov7-e6e.mlmodel'),\n \n '144': os.path.join('yolov8','weights','yolov8n-pose.pt'),\n '145': os.path.join('yolov8','weights','yolov8s-pose.pt'),\n '146': os.path.join('yolov8','weights','yolov8m-pose.pt'),\n '147': os.path.join('yolov8','weights','yolov8l-pose.pt'),\n '148': os.path.join('yolov8','weights','yolov8x-pose.pt') \n}\n\ndef get_weight_path(model_flag):\n weight = weights[str(model_flag)]\n return weight" }, { "path": "asone/utils/temp_loader.py", "content": "from asone.detectors import YOLOv5Detector\nfrom asone.detectors import YOLOv7Detector\nfrom asone.trackers import ByteTrack\nfrom asone.trackers import NorFair\nfrom asone.trackers import DeepSort\n\ndetectors = {\n 'yolov5s': YOLOv5Detector,\n 'yolov7': YOLOv7Detector\n}\n\ntrackers = {\n 'byte_track': ByteTrack,\n 'norfair': NorFair,\n 'deepsort': DeepSort\n}\n\n\ndef get_detector(detector, use_cuda=True, use_onnx=False):\n detector = detectors.get(detector, None)\n\n if detector is not None:\n return detector(use_cuda=use_cuda, use_onnx=use_onnx)\n else:\n return None\n\n\ndef get_tracker(tracker, detector, use_cuda=True, use_onnx=False):\n tracker = trackers.get(tracker, None)\n\n if tracker is not None:\n return tracker(detector)\n else:\n return None\n" }, { "path": "asone/utils/utils.py", "content": "import os\nimport sys\nimport traceback\n\n\nclass PathResolver:\n def __init__(self, path: str = None) -> None:\n if path is None:\n stack = traceback.extract_stack()\n file_path = stack[-2].filename\n dir_path = os.path.dirname(file_path)\n path = os.path.join(dir_path, os.path.basename(dir_path))\n \n self.path = path \n \n def __enter__(self):\n sys.path.insert(0, self.path)\n\n def __exit__(self, exc_type, exc_value, traceback):\n sys.path.pop(0)\n " }, { "path": "asone/utils/video_reader.py", "content": "import cv2\nfrom tqdm import tqdm\n\nclass VideoReader:\n def __init__(\n self,\n video_path: str):\n \"\"\"\n A simple wrapper class over OpenCVs cv2.VideoCapture that\n has the ability to return frames in batches as opposed to\n one by one.\n\n Args:\n video_path: path to the video file\n \n \"\"\"\n self.video_path = video_path \n self.video = cv2.VideoCapture(video_path)\n\n def __len__(self):\n return self.video.get(cv2.CAP_PROP_FRAME_COUNT)\n\n @property\n def frame_counts(self):\n return self.__len__()\n\n @property\n def fps(self):\n return int(self.video.get(cv2.CAP_PROP_FPS))\n \n @property\n def frame_size(self):\n return (\n int(self.video.get(cv2.CAP_PROP_FRAME_WIDTH)),\n int(self.video.get(cv2.CAP_PROP_FRAME_HEIGHT)),\n )\n \n def __iter__(self):\n while True:\n success, frame = self.video.read()\n if success:\n yield frame\n else:\n break\n " }, { "path": "asone/utils/weights.json", "content": "{\n \"yolov5s\": \"1W5ypZmrYE4_Aqu3Jqsl-IDvLK6SOtJCK\",\n \"yolov5x6\": \"1CTYtGC8VFZD0uJbU4fcSPSjcYb8Be0jU\",\n \"yolov5n\": \"1q9_e76T_b353QmG5xGi3zkselGQrxuBk\",\n \"yolov5m\": \"1Vv3VEkgYd7WB-3e2MPo0QUKq_-F7biBP\",\n \"yolov5l\": \"1Wr4S7BTqqCOCP14T_aDgVxGV5h9pMM-n\",\n \"yolov5x\": \"18g_pjpwsnOlBKbhApuaTsbQUIDO75BSt\",\n \"yolov5n6\": \"1gOAZ90nKcvo7bhNZCZM-JRuY8FBeEkhl\",\n \"yolov5s6\": \"12W1Z0esjFc9UhiWWxCESjhm5KA3nzSwt\",\n \"yolov5m6\": \"1O-bbJ8WcqLig40IUyJ8ulKwj-J_KXvHr\",\n \"yolov5l6\": \"1sPZ-YpenYojZSIB5G1SMv2hPWX5oRwlb\",\n \"yolov6n\": \"1NA_u4BkPE_N8HcPmZrd7HyLmvFHOk8qd\",\n \"yolov6t\": \"16OWncBp-vh-sLDMOR58th3WOGv4envQ1\",\n \"yolov6s\": \"14BE0j654ClLxMq2ySWZNhTCmf48mLyXi\",\n \"yolov6l_relu\": \"14UfY057QUQoAj6q39PX_qE7U1bBIrIGi\",\n \"yolov6l\": \"1HdRIs0uMPbqs5E2aEX8O3d3dJTh-KBTf\",\n \"yolov6m\": \"1t_w9SCwbZAW7icwX_z97-SQz-plXzBgM\",\n \"yolov6s_repopt\": \"1L_1Crxx-4059xDDUZEf_asWRBVd3PF05\",\n \"yolov7-d6\": \"1_Ybtx7EAXnBwIZ59Vgo0FkXsCdjRuq2s\",\n \"yolov7-e6\": \"1g_2nYpeJ28cLYcOAUeztHUA5R-stk3Cm\",\n \"yolov7-e6e\": \"1PZwpKdHEP4Li3FkKNYn90HmGNiIjMMfi\",\n \"yolov7-tiny\": \"18Fels44wVJ1vG7yDuDPuWwuiAqeFxKI7\",\n \"yolov7-w6\": \"1wv3M23RFo0MhaujegBPY6gZ30IA894CO\",\n \"yolov7\": \"17iFeNfq5hKZVpQLQEgZzxF9Da5o5llLG\",\n \"yolov7x\": \"1zZskyvdgU45Ke8TtCA6csdCzqLrhmFYx\",\n \"yolor_csp\": \"1G3FBZKrznW_64mGfs6b3nAJiJv6GmmV0\",\n \"yolor_csp_star\": \"15WDl46ZthFGZfpOyI3qXx6gC9FQLH_wH\",\n \"yolor_csp_x\": \"1LU2ckh7eSpVD0nyPSdq1n34lKmNAX39T\",\n \"yolor_csp_x_star\": \"1jheqFDm7BpHQpR60wuWSBpbuyK5SoKdV\",\n \"yolor_p6\": \"1XKREKdxQCO8OXiW2IWGFhczolIaIr9sm\",\n \"yolox_l\": \"1jX1KHerOdZ5-dmXh6cWcRAn80aKD-7sP\",\n \"yolox_nano\": \"1783Os6uTpYjunL-MfK0WE1Wcwk58fIUi\",\n \"yolox_tiny\": \"1Lcv1ITvfPdWsu6Kb8Hq6cOSfJE7lbbf2\",\n \"yolox_darknet\": \"17f4UI06TWJ25Oqo2OoQGu8AoGVX1lPta\",\n \"yolox_s\": \"1IUAOv62XuwkwwTCVD2y3xJg7KUA3M0-M\",\n \"yolox_m\": \"1ktHj8UEwl0V8Qz9G74E-yj-o13FLeD0-\",\n \"yolox_x\": \"13HNnlILCx_XamNJWwJ1MG5x0XfP6HL1U\",\n \"ckpt\": \"1VZ05gzg249Q1m8BJVQxl3iHoNIbjzJf8\",\n \"yolov8s\": \"1hUhjQWw1cJL7TtBG0zD3MO52iCYh0DEn\",\n \"yolov8n\": \"1x6zHzsEcyhuyWy2xY3swAQ4vIQvBYrsr\",\n \"yolov8l\": \"1xQxHTEIpoiP4d73F6dtedU7hIZpFTqY2\",\n \"yolov8m\": \"1_FoKnqkaoWchVy4B24Hn2PanEKfh-eSp\",\n \"yolov8x\": \"1s60fsjiyDlPQ1L5H_GAoWahHONLbvz7T\"\n}" }, { "path": "asone/windows/README.md", "content": "# ASOne\n## Docker Installation Instructions For Windows\n#### Table of Contents \n- [System Requirements](#system-requirements) \n- [Installation with Batch and Configuring Devices](#installation-with-batch-and-configuring-devices) \n\n\n\n\n\n\n\n### System Requirements\nWindows machine must meet the following requirements to successfully install the docker:\n\n**With WSL 2 backend**
\nType **winver** in RUN to check the version of the installed windows.\n\n- Windows 11 64-bit: Home or Pro version 21H2 or higher,\\\n or Enterprise or Education version 21H2 or higher\n- Windows 10 64-bit: Home or Pro 21H1 (build 19043) or higher,\\\n or Enterprise or Education 20H2 (build 19042) or higher\n- Enable the WSL 2 feature on Windows. For detailed instructions,\\\n refer to the [wsl installation](https://docs.microsoft.com/en-us/windows/wsl/install)\n\n **Hardware Requirements to run WSL 2 on Windows 10 or Windows 11**
\n\n- 64-bit processor with Second Level Address Translation (SLAT)\n\n- 4GB system RAM\n\n- BIOS-level hardware virtualization support must be enabled in the \\\n BIOS settings. For more information, see [Virtualization](https://docs.docker.com/desktop/troubleshoot/topics/)\n\n## Installation with Batch and Configuring Devices\n1. Download the [enable_feature.bat](enable_feature.bat) and run it as administrator.\n- Reboot your system.\n2. Download the [installation.bat](installation.bat) and run it as administrator.\n- Again Reboot your system.\n3. Open XLaunch and select Multiple windows\n- Select the option Start no client\n- In Extra Settings, select the option \n 1. Clipboard\n 2. Primary Selection\n 3. Native opengl\n 4. Disable access control\n- Save configuration file for later use\n4. Open [cam2ip.exe](cam2ip-1.6-64bit-cv/cam2ip.exe) see \"Listening on: 56000\" \n - IP stream will be on `http://localhost:56000/mjpeg`\n\n You can now go back to [Installation Page](../README.md). \n\n\n\n\n\n\n\n\n" }, { "path": "asone/windows/cam2ip-1.6-64bit-cv/AUTHORS", "content": "Milan Nikolic \n" }, { "path": "asone/windows/cam2ip-1.6-64bit-cv/COPYING", "content": " GNU GENERAL PUBLIC LICENSE\n Version 3, 29 June 2007\n\n Copyright (C) 2007 Free Software Foundation, Inc. \n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n Preamble\n\n The GNU General Public License is a free, copyleft license for\nsoftware and other kinds of works.\n\n The licenses for most software and other practical works are designed\nto take away your freedom to share and change the works. By contrast,\nthe GNU General Public License is intended to guarantee your freedom to\nshare and change all versions of a program--to make sure it remains free\nsoftware for all its users. We, the Free Software Foundation, use the\nGNU General Public License for most of our software; it applies also to\nany other work released this way by its authors. You can apply it to\nyour programs, too.\n\n When we speak of free software, we are referring to freedom, not\nprice. Our General Public Licenses are designed to make sure that you\nhave the freedom to distribute copies of free software (and charge for\nthem if you wish), that you receive source code or can get it if you\nwant it, that you can change the software or use pieces of it in new\nfree programs, and that you know you can do these things.\n\n To protect your rights, we need to prevent others from denying you\nthese rights or asking you to surrender the rights. Therefore, you have\ncertain responsibilities if you distribute copies of the software, or if\nyou modify it: responsibilities to respect the freedom of others.\n\n For example, if you distribute copies of such a program, whether\ngratis or for a fee, you must pass on to the recipients the same\nfreedoms that you received. You must make sure that they, too, receive\nor can get the source code. And you must show them these terms so they\nknow their rights.\n\n Developers that use the GNU GPL protect your rights with two steps:\n(1) assert copyright on the software, and (2) offer you this License\ngiving you legal permission to copy, distribute and/or modify it.\n\n For the developers' and authors' protection, the GPL clearly explains\nthat there is no warranty for this free software. For both users' and\nauthors' sake, the GPL requires that modified versions be marked as\nchanged, so that their problems will not be attributed erroneously to\nauthors of previous versions.\n\n Some devices are designed to deny users access to install or run\nmodified versions of the software inside them, although the manufacturer\ncan do so. This is fundamentally incompatible with the aim of\nprotecting users' freedom to change the software. The systematic\npattern of such abuse occurs in the area of products for individuals to\nuse, which is precisely where it is most unacceptable. Therefore, we\nhave designed this version of the GPL to prohibit the practice for those\nproducts. If such problems arise substantially in other domains, we\nstand ready to extend this provision to those domains in future versions\nof the GPL, as needed to protect the freedom of users.\n\n Finally, every program is threatened constantly by software patents.\nStates should not allow patents to restrict development and use of\nsoftware on general-purpose computers, but in those that do, we wish to\navoid the special danger that patents applied to a free program could\nmake it effectively proprietary. To prevent this, the GPL assures that\npatents cannot be used to render the program non-free.\n\n The precise terms and conditions for copying, distribution and\nmodification follow.\n\n TERMS AND CONDITIONS\n\n 0. Definitions.\n\n \"This License\" refers to version 3 of the GNU General Public License.\n\n \"Copyright\" also means copyright-like laws that apply to other kinds of\nworks, such as semiconductor masks.\n\n \"The Program\" refers to any copyrightable work licensed under this\nLicense. Each licensee is addressed as \"you\". \"Licensees\" and\n\"recipients\" may be individuals or organizations.\n\n To \"modify\" a work means to copy from or adapt all or part of the work\nin a fashion requiring copyright permission, other than the making of an\nexact copy. The resulting work is called a \"modified version\" of the\nearlier work or a work \"based on\" the earlier work.\n\n A \"covered work\" means either the unmodified Program or a work based\non the Program.\n\n To \"propagate\" a work means to do anything with it that, without\npermission, would make you directly or secondarily liable for\ninfringement under applicable copyright law, except executing it on a\ncomputer or modifying a private copy. Propagation includes copying,\ndistribution (with or without modification), making available to the\npublic, and in some countries other activities as well.\n\n To \"convey\" a work means any kind of propagation that enables other\nparties to make or receive copies. Mere interaction with a user through\na computer network, with no transfer of a copy, is not conveying.\n\n An interactive user interface displays \"Appropriate Legal Notices\"\nto the extent that it includes a convenient and prominently visible\nfeature that (1) displays an appropriate copyright notice, and (2)\ntells the user that there is no warranty for the work (except to the\nextent that warranties are provided), that licensees may convey the\nwork under this License, and how to view a copy of this License. If\nthe interface presents a list of user commands or options, such as a\nmenu, a prominent item in the list meets this criterion.\n\n 1. Source Code.\n\n The \"source code\" for a work means the preferred form of the work\nfor making modifications to it. \"Object code\" means any non-source\nform of a work.\n\n A \"Standard Interface\" means an interface that either is an official\nstandard defined by a recognized standards body, or, in the case of\ninterfaces specified for a particular programming language, one that\nis widely used among developers working in that language.\n\n The \"System Libraries\" of an executable work include anything, other\nthan the work as a whole, that (a) is included in the normal form of\npackaging a Major Component, but which is not part of that Major\nComponent, and (b) serves only to enable use of the work with that\nMajor Component, or to implement a Standard Interface for which an\nimplementation is available to the public in source code form. A\n\"Major Component\", in this context, means a major essential component\n(kernel, window system, and so on) of the specific operating system\n(if any) on which the executable work runs, or a compiler used to\nproduce the work, or an object code interpreter used to run it.\n\n The \"Corresponding Source\" for a work in object code form means all\nthe source code needed to generate, install, and (for an executable\nwork) run the object code and to modify the work, including scripts to\ncontrol those activities. However, it does not include the work's\nSystem Libraries, or general-purpose tools or generally available free\nprograms which are used unmodified in performing those activities but\nwhich are not part of the work. For example, Corresponding Source\nincludes interface definition files associated with source files for\nthe work, and the source code for shared libraries and dynamically\nlinked subprograms that the work is specifically designed to require,\nsuch as by intimate data communication or control flow between those\nsubprograms and other parts of the work.\n\n The Corresponding Source need not include anything that users\ncan regenerate automatically from other parts of the Corresponding\nSource.\n\n The Corresponding Source for a work in source code form is that\nsame work.\n\n 2. Basic Permissions.\n\n All rights granted under this License are granted for the term of\ncopyright on the Program, and are irrevocable provided the stated\nconditions are met. This License explicitly affirms your unlimited\npermission to run the unmodified Program. The output from running a\ncovered work is covered by this License only if the output, given its\ncontent, constitutes a covered work. This License acknowledges your\nrights of fair use or other equivalent, as provided by copyright law.\n\n You may make, run and propagate covered works that you do not\nconvey, without conditions so long as your license otherwise remains\nin force. You may convey covered works to others for the sole purpose\nof having them make modifications exclusively for you, or provide you\nwith facilities for running those works, provided that you comply with\nthe terms of this License in conveying all material for which you do\nnot control copyright. Those thus making or running the covered works\nfor you must do so exclusively on your behalf, under your direction\nand control, on terms that prohibit them from making any copies of\nyour copyrighted material outside their relationship with you.\n\n Conveying under any other circumstances is permitted solely under\nthe conditions stated below. Sublicensing is not allowed; section 10\nmakes it unnecessary.\n\n 3. Protecting Users' Legal Rights From Anti-Circumvention Law.\n\n No covered work shall be deemed part of an effective technological\nmeasure under any applicable law fulfilling obligations under article\n11 of the WIPO copyright treaty adopted on 20 December 1996, or\nsimilar laws prohibiting or restricting circumvention of such\nmeasures.\n\n When you convey a covered work, you waive any legal power to forbid\ncircumvention of technological measures to the extent such circumvention\nis effected by exercising rights under this License with respect to\nthe covered work, and you disclaim any intention to limit operation or\nmodification of the work as a means of enforcing, against the work's\nusers, your or third parties' legal rights to forbid circumvention of\ntechnological measures.\n\n 4. Conveying Verbatim Copies.\n\n You may convey verbatim copies of the Program's source code as you\nreceive it, in any medium, provided that you conspicuously and\nappropriately publish on each copy an appropriate copyright notice;\nkeep intact all notices stating that this License and any\nnon-permissive terms added in accord with section 7 apply to the code;\nkeep intact all notices of the absence of any warranty; and give all\nrecipients a copy of this License along with the Program.\n\n You may charge any price or no price for each copy that you convey,\nand you may offer support or warranty protection for a fee.\n\n 5. Conveying Modified Source Versions.\n\n You may convey a work based on the Program, or the modifications to\nproduce it from the Program, in the form of source code under the\nterms of section 4, provided that you also meet all of these conditions:\n\n a) The work must carry prominent notices stating that you modified\n it, and giving a relevant date.\n\n b) The work must carry prominent notices stating that it is\n released under this License and any conditions added under section\n 7. This requirement modifies the requirement in section 4 to\n \"keep intact all notices\".\n\n c) You must license the entire work, as a whole, under this\n License to anyone who comes into possession of a copy. This\n License will therefore apply, along with any applicable section 7\n additional terms, to the whole of the work, and all its parts,\n regardless of how they are packaged. This License gives no\n permission to license the work in any other way, but it does not\n invalidate such permission if you have separately received it.\n\n d) If the work has interactive user interfaces, each must display\n Appropriate Legal Notices; however, if the Program has interactive\n interfaces that do not display Appropriate Legal Notices, your\n work need not make them do so.\n\n A compilation of a covered work with other separate and independent\nworks, which are not by their nature extensions of the covered work,\nand which are not combined with it such as to form a larger program,\nin or on a volume of a storage or distribution medium, is called an\n\"aggregate\" if the compilation and its resulting copyright are not\nused to limit the access or legal rights of the compilation's users\nbeyond what the individual works permit. Inclusion of a covered work\nin an aggregate does not cause this License to apply to the other\nparts of the aggregate.\n\n 6. Conveying Non-Source Forms.\n\n You may convey a covered work in object code form under the terms\nof sections 4 and 5, provided that you also convey the\nmachine-readable Corresponding Source under the terms of this License,\nin one of these ways:\n\n a) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by the\n Corresponding Source fixed on a durable physical medium\n customarily used for software interchange.\n\n b) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by a\n written offer, valid for at least three years and valid for as\n long as you offer spare parts or customer support for that product\n model, to give anyone who possesses the object code either (1) a\n copy of the Corresponding Source for all the software in the\n product that is covered by this License, on a durable physical\n medium customarily used for software interchange, for a price no\n more than your reasonable cost of physically performing this\n conveying of source, or (2) access to copy the\n Corresponding Source from a network server at no charge.\n\n c) Convey individual copies of the object code with a copy of the\n written offer to provide the Corresponding Source. This\n alternative is allowed only occasionally and noncommercially, and\n only if you received the object code with such an offer, in accord\n with subsection 6b.\n\n d) Convey the object code by offering access from a designated\n place (gratis or for a charge), and offer equivalent access to the\n Corresponding Source in the same way through the same place at no\n further charge. You need not require recipients to copy the\n Corresponding Source along with the object code. If the place to\n copy the object code is a network server, the Corresponding Source\n may be on a different server (operated by you or a third party)\n that supports equivalent copying facilities, provided you maintain\n clear directions next to the object code saying where to find the\n Corresponding Source. Regardless of what server hosts the\n Corresponding Source, you remain obligated to ensure that it is\n available for as long as needed to satisfy these requirements.\n\n e) Convey the object code using peer-to-peer transmission, provided\n you inform other peers where the object code and Corresponding\n Source of the work are being offered to the general public at no\n charge under subsection 6d.\n\n A separable portion of the object code, whose source code is excluded\nfrom the Corresponding Source as a System Library, need not be\nincluded in conveying the object code work.\n\n A \"User Product\" is either (1) a \"consumer product\", which means any\ntangible personal property which is normally used for personal, family,\nor household purposes, or (2) anything designed or sold for incorporation\ninto a dwelling. In determining whether a product is a consumer product,\ndoubtful cases shall be resolved in favor of coverage. For a particular\nproduct received by a particular user, \"normally used\" refers to a\ntypical or common use of that class of product, regardless of the status\nof the particular user or of the way in which the particular user\nactually uses, or expects or is expected to use, the product. A product\nis a consumer product regardless of whether the product has substantial\ncommercial, industrial or non-consumer uses, unless such uses represent\nthe only significant mode of use of the product.\n\n \"Installation Information\" for a User Product means any methods,\nprocedures, authorization keys, or other information required to install\nand execute modified versions of a covered work in that User Product from\na modified version of its Corresponding Source. The information must\nsuffice to ensure that the continued functioning of the modified object\ncode is in no case prevented or interfered with solely because\nmodification has been made.\n\n If you convey an object code work under this section in, or with, or\nspecifically for use in, a User Product, and the conveying occurs as\npart of a transaction in which the right of possession and use of the\nUser Product is transferred to the recipient in perpetuity or for a\nfixed term (regardless of how the transaction is characterized), the\nCorresponding Source conveyed under this section must be accompanied\nby the Installation Information. But this requirement does not apply\nif neither you nor any third party retains the ability to install\nmodified object code on the User Product (for example, the work has\nbeen installed in ROM).\n\n The requirement to provide Installation Information does not include a\nrequirement to continue to provide support service, warranty, or updates\nfor a work that has been modified or installed by the recipient, or for\nthe User Product in which it has been modified or installed. Access to a\nnetwork may be denied when the modification itself materially and\nadversely affects the operation of the network or violates the rules and\nprotocols for communication across the network.\n\n Corresponding Source conveyed, and Installation Information provided,\nin accord with this section must be in a format that is publicly\ndocumented (and with an implementation available to the public in\nsource code form), and must require no special password or key for\nunpacking, reading or copying.\n\n 7. Additional Terms.\n\n \"Additional permissions\" are terms that supplement the terms of this\nLicense by making exceptions from one or more of its conditions.\nAdditional permissions that are applicable to the entire Program shall\nbe treated as though they were included in this License, to the extent\nthat they are valid under applicable law. If additional permissions\napply only to part of the Program, that part may be used separately\nunder those permissions, but the entire Program remains governed by\nthis License without regard to the additional permissions.\n\n When you convey a copy of a covered work, you may at your option\nremove any additional permissions from that copy, or from any part of\nit. (Additional permissions may be written to require their own\nremoval in certain cases when you modify the work.) You may place\nadditional permissions on material, added by you to a covered work,\nfor which you have or can give appropriate copyright permission.\n\n Notwithstanding any other provision of this License, for material you\nadd to a covered work, you may (if authorized by the copyright holders of\nthat material) supplement the terms of this License with terms:\n\n a) Disclaiming warranty or limiting liability differently from the\n terms of sections 15 and 16 of this License; or\n\n b) Requiring preservation of specified reasonable legal notices or\n author attributions in that material or in the Appropriate Legal\n Notices displayed by works containing it; or\n\n c) Prohibiting misrepresentation of the origin of that material, or\n requiring that modified versions of such material be marked in\n reasonable ways as different from the original version; or\n\n d) Limiting the use for publicity purposes of names of licensors or\n authors of the material; or\n\n e) Declining to grant rights under trademark law for use of some\n trade names, trademarks, or service marks; or\n\n f) Requiring indemnification of licensors and authors of that\n material by anyone who conveys the material (or modified versions of\n it) with contractual assumptions of liability to the recipient, for\n any liability that these contractual assumptions directly impose on\n those licensors and authors.\n\n All other non-permissive additional terms are considered \"further\nrestrictions\" within the meaning of section 10. If the Program as you\nreceived it, or any part of it, contains a notice stating that it is\ngoverned by this License along with a term that is a further\nrestriction, you may remove that term. If a license document contains\na further restriction but permits relicensing or conveying under this\nLicense, you may add to a covered work material governed by the terms\nof that license document, provided that the further restriction does\nnot survive such relicensing or conveying.\n\n If you add terms to a covered work in accord with this section, you\nmust place, in the relevant source files, a statement of the\nadditional terms that apply to those files, or a notice indicating\nwhere to find the applicable terms.\n\n Additional terms, permissive or non-permissive, may be stated in the\nform of a separately written license, or stated as exceptions;\nthe above requirements apply either way.\n\n 8. Termination.\n\n You may not propagate or modify a covered work except as expressly\nprovided under this License. Any attempt otherwise to propagate or\nmodify it is void, and will automatically terminate your rights under\nthis License (including any patent licenses granted under the third\nparagraph of section 11).\n\n However, if you cease all violation of this License, then your\nlicense from a particular copyright holder is reinstated (a)\nprovisionally, unless and until the copyright holder explicitly and\nfinally terminates your license, and (b) permanently, if the copyright\nholder fails to notify you of the violation by some reasonable means\nprior to 60 days after the cessation.\n\n Moreover, your license from a particular copyright holder is\nreinstated permanently if the copyright holder notifies you of the\nviolation by some reasonable means, this is the first time you have\nreceived notice of violation of this License (for any work) from that\ncopyright holder, and you cure the violation prior to 30 days after\nyour receipt of the notice.\n\n Termination of your rights under this section does not terminate the\nlicenses of parties who have received copies or rights from you under\nthis License. If your rights have been terminated and not permanently\nreinstated, you do not qualify to receive new licenses for the same\nmaterial under section 10.\n\n 9. Acceptance Not Required for Having Copies.\n\n You are not required to accept this License in order to receive or\nrun a copy of the Program. Ancillary propagation of a covered work\noccurring solely as a consequence of using peer-to-peer transmission\nto receive a copy likewise does not require acceptance. However,\nnothing other than this License grants you permission to propagate or\nmodify any covered work. These actions infringe copyright if you do\nnot accept this License. Therefore, by modifying or propagating a\ncovered work, you indicate your acceptance of this License to do so.\n\n 10. Automatic Licensing of Downstream Recipients.\n\n Each time you convey a covered work, the recipient automatically\nreceives a license from the original licensors, to run, modify and\npropagate that work, subject to this License. You are not responsible\nfor enforcing compliance by third parties with this License.\n\n An \"entity transaction\" is a transaction transferring control of an\norganization, or substantially all assets of one, or subdividing an\norganization, or merging organizations. If propagation of a covered\nwork results from an entity transaction, each party to that\ntransaction who receives a copy of the work also receives whatever\nlicenses to the work the party's predecessor in interest had or could\ngive under the previous paragraph, plus a right to possession of the\nCorresponding Source of the work from the predecessor in interest, if\nthe predecessor has it or can get it with reasonable efforts.\n\n You may not impose any further restrictions on the exercise of the\nrights granted or affirmed under this License. For example, you may\nnot impose a license fee, royalty, or other charge for exercise of\nrights granted under this License, and you may not initiate litigation\n(including a cross-claim or counterclaim in a lawsuit) alleging that\nany patent claim is infringed by making, using, selling, offering for\nsale, or importing the Program or any portion of it.\n\n 11. Patents.\n\n A \"contributor\" is a copyright holder who authorizes use under this\nLicense of the Program or a work on which the Program is based. The\nwork thus licensed is called the contributor's \"contributor version\".\n\n A contributor's \"essential patent claims\" are all patent claims\nowned or controlled by the contributor, whether already acquired or\nhereafter acquired, that would be infringed by some manner, permitted\nby this License, of making, using, or selling its contributor version,\nbut do not include claims that would be infringed only as a\nconsequence of further modification of the contributor version. For\npurposes of this definition, \"control\" includes the right to grant\npatent sublicenses in a manner consistent with the requirements of\nthis License.\n\n Each contributor grants you a non-exclusive, worldwide, royalty-free\npatent license under the contributor's essential patent claims, to\nmake, use, sell, offer for sale, import and otherwise run, modify and\npropagate the contents of its contributor version.\n\n In the following three paragraphs, a \"patent license\" is any express\nagreement or commitment, however denominated, not to enforce a patent\n(such as an express permission to practice a patent or covenant not to\nsue for patent infringement). To \"grant\" such a patent license to a\nparty means to make such an agreement or commitment not to enforce a\npatent against the party.\n\n If you convey a covered work, knowingly relying on a patent license,\nand the Corresponding Source of the work is not available for anyone\nto copy, free of charge and under the terms of this License, through a\npublicly available network server or other readily accessible means,\nthen you must either (1) cause the Corresponding Source to be so\navailable, or (2) arrange to deprive yourself of the benefit of the\npatent license for this particular work, or (3) arrange, in a manner\nconsistent with the requirements of this License, to extend the patent\nlicense to downstream recipients. \"Knowingly relying\" means you have\nactual knowledge that, but for the patent license, your conveying the\ncovered work in a country, or your recipient's use of the covered work\nin a country, would infringe one or more identifiable patents in that\ncountry that you have reason to believe are valid.\n\n If, pursuant to or in connection with a single transaction or\narrangement, you convey, or propagate by procuring conveyance of, a\ncovered work, and grant a patent license to some of the parties\nreceiving the covered work authorizing them to use, propagate, modify\nor convey a specific copy of the covered work, then the patent license\nyou grant is automatically extended to all recipients of the covered\nwork and works based on it.\n\n A patent license is \"discriminatory\" if it does not include within\nthe scope of its coverage, prohibits the exercise of, or is\nconditioned on the non-exercise of one or more of the rights that are\nspecifically granted under this License. You may not convey a covered\nwork if you are a party to an arrangement with a third party that is\nin the business of distributing software, under which you make payment\nto the third party based on the extent of your activity of conveying\nthe work, and under which the third party grants, to any of the\nparties who would receive the covered work from you, a discriminatory\npatent license (a) in connection with copies of the covered work\nconveyed by you (or copies made from those copies), or (b) primarily\nfor and in connection with specific products or compilations that\ncontain the covered work, unless you entered into that arrangement,\nor that patent license was granted, prior to 28 March 2007.\n\n Nothing in this License shall be construed as excluding or limiting\nany implied license or other defenses to infringement that may\notherwise be available to you under applicable patent law.\n\n 12. No Surrender of Others' Freedom.\n\n If conditions are imposed on you (whether by court order, agreement or\notherwise) that contradict the conditions of this License, they do not\nexcuse you from the conditions of this License. If you cannot convey a\ncovered work so as to satisfy simultaneously your obligations under this\nLicense and any other pertinent obligations, then as a consequence you may\nnot convey it at all. For example, if you agree to terms that obligate you\nto collect a royalty for further conveying from those to whom you convey\nthe Program, the only way you could satisfy both those terms and this\nLicense would be to refrain entirely from conveying the Program.\n\n 13. Use with the GNU Affero General Public License.\n\n Notwithstanding any other provision of this License, you have\npermission to link or combine any covered work with a work licensed\nunder version 3 of the GNU Affero General Public License into a single\ncombined work, and to convey the resulting work. The terms of this\nLicense will continue to apply to the part which is the covered work,\nbut the special requirements of the GNU Affero General Public License,\nsection 13, concerning interaction through a network will apply to the\ncombination as such.\n\n 14. Revised Versions of this License.\n\n The Free Software Foundation may publish revised and/or new versions of\nthe GNU General Public License from time to time. Such new versions will\nbe similar in spirit to the present version, but may differ in detail to\naddress new problems or concerns.\n\n Each version is given a distinguishing version number. If the\nProgram specifies that a certain numbered version of the GNU General\nPublic License \"or any later version\" applies to it, you have the\noption of following the terms and conditions either of that numbered\nversion or of any later version published by the Free Software\nFoundation. If the Program does not specify a version number of the\nGNU General Public License, you may choose any version ever published\nby the Free Software Foundation.\n\n If the Program specifies that a proxy can decide which future\nversions of the GNU General Public License can be used, that proxy's\npublic statement of acceptance of a version permanently authorizes you\nto choose that version for the Program.\n\n Later license versions may give you additional or different\npermissions. However, no additional obligations are imposed on any\nauthor or copyright holder as a result of your choosing to follow a\nlater version.\n\n 15. Disclaimer of Warranty.\n\n THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY\nAPPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT\nHOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY\nOF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM\nIS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF\nALL NECESSARY SERVICING, REPAIR OR CORRECTION.\n\n 16. Limitation of Liability.\n\n IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\nWILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS\nTHE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY\nGENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE\nUSE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF\nDATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD\nPARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),\nEVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF\nSUCH DAMAGES.\n\n 17. Interpretation of Sections 15 and 16.\n\n If the disclaimer of warranty and limitation of liability provided\nabove cannot be given local legal effect according to their terms,\nreviewing courts shall apply local law that most closely approximates\nan absolute waiver of all civil liability in connection with the\nProgram, unless a warranty or assumption of liability accompanies a\ncopy of the Program in return for a fee.\n\n END OF TERMS AND CONDITIONS\n\n How to Apply These Terms to Your New Programs\n\n If you develop a new program, and you want it to be of the greatest\npossible use to the public, the best way to achieve this is to make it\nfree software which everyone can redistribute and change under these terms.\n\n To do so, attach the following notices to the program. It is safest\nto attach them to the start of each source file to most effectively\nstate the exclusion of warranty; and each file should have at least\nthe \"copyright\" line and a pointer to where the full notice is found.\n\n \n Copyright (C) \n\n This program is free software: you can redistribute it and/or modify\n it under the terms of the GNU General Public License as published by\n the Free Software Foundation, either version 3 of the License, or\n (at your option) any later version.\n\n This program is distributed in the hope that it will be useful,\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n GNU General Public License for more details.\n\n You should have received a copy of the GNU General Public License\n along with this program. If not, see .\n\nAlso add information on how to contact you by electronic and paper mail.\n\n If the program does terminal interaction, make it output a short\nnotice like this when it starts in an interactive mode:\n\n Copyright (C) \n This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.\n This is free software, and you are welcome to redistribute it\n under certain conditions; type `show c' for details.\n\nThe hypothetical commands `show w' and `show c' should show the appropriate\nparts of the General Public License. Of course, your program's commands\nmight be different; for a GUI interface, you would use an \"about box\".\n\n You should also get your employer (if you work as a programmer) or school,\nif any, to sign a \"copyright disclaimer\" for the program, if necessary.\nFor more information on this, and how to apply and follow the GNU GPL, see\n.\n\n The GNU General Public License does not permit incorporating your program\ninto proprietary programs. If your program is a subroutine library, you\nmay consider it more useful to permit linking proprietary applications with\nthe library. If this is what you want to do, use the GNU Lesser General\nPublic License instead of this License. But first, please read\n.\n" }, { "path": "asone/windows/enable_feature.bat", "content": "@echo off\n@\"%SystemRoot%\\System32\\WindowsPowerShell\\v1.0\\powershell.exe\" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command \"[System.Net.ServicePointManager]::SecurityProtocol = 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))\" && SET \"PATH=%PATH%;%ALLUSERSPROFILE%\\chocolatey\\bin\"\npowershell.exe dism.exe /online /enable-feature /featurename:VirtualMachinePlatform /all /norestart\npowershell.exe dism.exe /online /enable-feature /featurename:Microsoft-Windows-Subsystem-Linux /all /norestart\nchoco feature enable -name=exitOnRebootDetected\nchoco feature enable -n allowGlobalConfirmation\necho \"REBOOT YOUR SYSTEM\"\npause" }, { "path": "asone/windows/installation.bat", "content": "@echo on\nchoco install wsl2 --params \"/Version:2 /Retry:true\"\nwsl --set-default-version 2\nchoco install docker-desktop\nchoco install vcxsrv\nfor /F \"tokens=14\" %i in ('\"ipconfig | findstr IPv4 | findstr /i \"192\"\"') do setx DISPLAY %i:0.0\necho \"REBOOT YOUR SYSTEM\"\nPause\n" }, { "path": "asone/windows/test-display.py", "content": "import cv2\n\nimg = cv2.imread('test.jpg')\ncv2.imshow(\"TEST DISPLAY\", cv2.resize(img, (600, 600))\n)\ncv2.waitKey(0)" }, { "path": "asone/windows/test-webcam.py", "content": "import cv2\nimport numpy as np\n\nimg = np.zeros((400, 600, 3))\n\n##Fetch IP Address\n# importing socket module\nimport socket\n# getting the hostname by socket.gethostname() method\nhostname = socket.gethostname()\n# getting the IP address using socket.gethostbyname() method\nip_address = socket.gethostbyname(hostname)\n# capturing the video from ip stream \ncap = cv2.VideoCapture(f'http://{ip_address}:56000/mjpeg')\n# cap.open(\"\")\n\nwhile True:\n if not cap.isOpened():\n print('Unable to load camera. Use the command \"xhost +\"')\n pass\n\n # Capture frame-by-frame\n ret, frame = cap.read()\n print(frame)\n # Display the resulting frame\n cv2.imshow('Video', frame)\n\n\n if cv2.waitKey(1) & 0xFF == ord('q'):\n break\n\n# When everything is done, release the capture\ncap.release()\ncv2.destroyAllWindows()\n" }, { "path": "docker-compose.yml", "content": "version: \"3.9\"\nservices:\n linux:\n build: .\n image: \"asone:latest\"\n volumes:\n - ${PWD}:/workspace\n - $HOME/.Xauthority:/root/.Xauthority:rw\n - /dev/video0:/dev/video0\n network_mode: host\n restart: always\n privileged: true\n environment:\n DISPLAY: $DISPLAY\n\n linux-gpu:\n build: .\n image: \"asone:latest\"\n volumes:\n - ${PWD}:/workspace\n - $HOME/.Xauthority:/root/.Xauthority:rw\n - /dev/video0:/dev/video0\n network_mode: host\n restart: always\n privileged: true\n environment:\n DISPLAY: $DISPLAY\n deploy:\n resources:\n reservations:\n devices:\n - capabilities: [gpu]\n\n windows:\n build: .\n image: \"asone:latest\"\n volumes:\n - ${PWD}:/workspace\n # - $HOME/.Xauthority:/root/.Xauthority:rw\n # - /dev/video0:/dev/video0\n network_mode: host\n restart: always\n privileged: true\n environment:\n DISPLAY: $DISPLAY\n\n windows-gpu:\n build: .\n image: \"asone:latest\"\n volumes:\n - ${PWD}:/workspace\n # - $HOME/.Xauthority:/root/.Xauthority:rw\n # - /dev/video0:/dev/video0\n network_mode: host\n restart: always\n privileged: true\n environment:\n DISPLAY: $DISPLAY\n deploy:\n resources:\n reservations:\n devices:\n - capabilities: [gpu]\n" }, { "path": "main.py", "content": "import sys\nimport argparse\nimport asone\nfrom asone import ASOne\nimport torch\n\n\ndef main(args):\n filter_classes = args.filter_classes\n\n if filter_classes:\n filter_classes = ['person']\n # Check if cuda available\n if args.use_cuda and torch.cuda.is_available():\n args.use_cuda = True\n else:\n args.use_cuda = False\n \n if sys.platform.startswith('darwin'):\n detector = asone.YOLOV7_MLMODEL \n else:\n detector = asone.YOLOV7_PYTORCH\n \n detect = ASOne(\n tracker=asone.BYTETRACK,\n detector=detector,\n weights=args.weights,\n sam_weights=args.sam_weights,\n use_cuda=args.use_cuda\n )\n # Get tracking function\n track = detect.track_video(args.video_path,\n output_dir=args.output_dir,\n conf_thres=args.conf_thres,\n iou_thres=args.iou_thres,\n display=args.display,\n draw_trails=args.draw_trails,\n filter_classes=filter_classes,\n class_names=None) # class_names=['License Plate'] for custom weights\n \n # Loop over track_fn to retrieve outputs of each frame \n for bbox_details, frame_details in track:\n bbox_xyxy, ids, scores, class_ids = bbox_details\n frame, frame_num, fps = frame_details\n print(frame_num)\n \n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n\n parser.add_argument('video_path', help='Path to input video')\n parser.add_argument('--cpu', default=True, action='store_false', dest='use_cuda',\n help='run on cpu if not provided the program will run on gpu.')\n parser.add_argument('--no_save', default=True, action='store_false',\n dest='save_result', help='whether or not save results')\n parser.add_argument('--no_display', default=True, action='store_false',\n dest='display', help='whether or not display results on screen')\n parser.add_argument('--output_dir', default='data/results', help='Path to output directory')\n parser.add_argument('--draw_trails', action='store_true', default=False,\n help='if provided object motion trails will be drawn.')\n parser.add_argument('--filter_classes', default=None, help='Filter class name')\n parser.add_argument('-w', '--weights', default=None, help='Path of trained weights')\n parser.add_argument('--sam_weights', default=None, help='Path of sam weights')\n parser.add_argument('-ct', '--conf_thres', default=0.25, type=float, help='confidence score threshold')\n parser.add_argument('-it', '--iou_thres', default=0.45, type=float, help='iou score threshold')\n\n args = parser.parse_args()\n\n main(args)" }, { "path": "requirements.txt", "content": "torch \ntorchvision\nopencv-python\nlap\nloguru\nnorfair\nnumpy==1.23.3\n#onnxruntime-gpu==1.12.1\nscipy\npyyaml\neasydict\ngdown\npandas\ntabulate\nwheel\nnumpy==1.23.3\nasone-ocr\nmotpy\nultralytics==8.1.30\ntorchreid==0.2.5\ntensorboard\nprotobuf==3.20.*\n# requirements for coreml\nonnxruntime\ncoremltools\nCython==3.0.9\ntyping_extensions\nsuper_gradients\ncython_bbox\nthop\nIPython\nsegment-anything\npillow==9.5.0" }, { "path": "setup.py", "content": "from setuptools import setup, find_packages\nfrom pkg_resources import parse_requirements\nimport pathlib\n\nDISTNAME = 'asone'\nDESCRIPTION = ''\nMAINTAINER = 'AxcelerateAI'\nMAINTAINER_EMAIL = 'umair.imran@axcelerate.ai'\nURL = 'https://github.com/axcelerateai/asone'\nDOWNLOAD_URL = URL\n\nVERSION = '0.3.2'\n\nwith open('README.md') as f:\n long_description = f.read()\n\nrequirements_txt = pathlib.Path('requirements.txt').open()\n\n\ndef setup_package():\n setup(\n name=DISTNAME,\n version=VERSION,\n description=DESCRIPTION,\n long_description = long_description,\n long_description_content_type='text/markdown',\n url=DOWNLOAD_URL,\n author=MAINTAINER,\n author_email=MAINTAINER_EMAIL,\n license='BSD 2-clause',\n keywords='asone bytetrack deepsort norfair yolo yolox yolor yolov5 yolov7 installation inferencing',\n # package_dir={\"\":\"\"},\n packages=find_packages(),\n\n dependency_links=[\n \"https://download.pytorch.org/whl/cu113/\",\n 'https://pypi.python.org/simple/'],\n install_requires=[str(requirement)\n for requirement in parse_requirements(requirements_txt)],\n package_data={\n \"\": [\"detectors/yolor/cfg/*.cfg\", \"detectors/data/*.yaml\",\n \"detectors/data/*.yml\", \"detectors/data/*.names\"],\n },\n\n include_package_data=True,\n classifiers=[\n 'Development Status :: 1 - Planning',\n 'Intended Audience :: Science/Research',\n 'License :: OSI Approved :: MIT License',\n 'Operating System :: POSIX :: Linux',\n 'Operating System :: Microsoft :: Windows :: Windows 10',\n 'Programming Language :: Python :: 3',\n 'Programming Language :: Python :: 3.8',\n 'Programming Language :: Python :: 3.9',\n 'Programming Language :: Python :: 3.10',\n ],\n )\n\n\nif __name__ == \"__main__\":\n setup_package()\n" } ]