[
  {
    "path": ".gitignore",
    "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n#  Usually these files are written by a python script from a template\n#  before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# pipenv\n#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n#   However, in case of collaboration, if having platform-specific dependencies or dependencies\n#   having no cross-platform support, pipenv may install dependencies that don't work, or not\n#   install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# macOS\n**/.DS_Store\n\n# PyCharm project settings\n.idea/\n\n# Xcode\n*.xcworkspace\n\n# FastVLM models\napp/FastVLM/model"
  },
  {
    "path": "ACKNOWLEDGEMENTS",
    "content": "Acknowledgements\nPortions of this Software may utilize the following copyrighted\nmaterial, the use of which is hereby acknowledged.\n\n---------------------------------------------------------------------------------\n\nLLaVA: Large Language and Vision Assistant (LLaVA)\n\n                             Apache License\n                       Version 2.0, January 2004\n                    http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n  \"License\" shall mean the terms and conditions for use, reproduction,\n  and distribution as defined by Sections 1 through 9 of this document.\n\n  \"Licensor\" shall mean the copyright owner or entity authorized by\n  the copyright owner that is granting the License.\n\n  \"Legal Entity\" shall mean the union of the acting entity and all\n  other entities that control, are controlled by, or are under common\n  control with that entity. For the purposes of this definition,\n  \"control\" means (i) the power, direct or indirect, to cause the\n  direction or management of such entity, whether by contract or\n  otherwise, or (ii) ownership of fifty percent (50%) or more of the\n  outstanding shares, or (iii) beneficial ownership of such entity.\n\n  \"You\" (or \"Your\") shall mean an individual or Legal Entity\n  exercising permissions granted by this License.\n\n  \"Source\" form shall mean the preferred form for making modifications,\n  including but not limited to software source code, documentation\n  source, and configuration files.\n\n  \"Object\" form shall mean any form resulting from mechanical\n  transformation or translation of a Source form, including but\n  not limited to compiled object code, generated documentation,\n  and conversions to other media types.\n\n  \"Work\" shall mean the work of authorship, whether in Source or\n  Object form, made available under the License, as indicated by a\n  copyright notice that is included in or attached to the work\n  (an example is provided in the Appendix below).\n\n  \"Derivative Works\" shall mean any work, whether in Source or Object\n  form, that is based on (or derived from) the Work and for which the\n  editorial revisions, annotations, elaborations, or other modifications\n  represent, as a whole, an original work of authorship. For the purposes\n  of this License, Derivative Works shall not include works that remain\n  separable from, or merely link (or bind by name) to the interfaces of,\n  the Work and Derivative Works thereof.\n\n  \"Contribution\" shall mean any work of authorship, including\n  the original version of the Work and any modifications or additions\n  to that Work or Derivative Works thereof, that is intentionally\n  submitted to Licensor for inclusion in the Work by the copyright owner\n  or by an individual or Legal Entity authorized to submit on behalf of\n  the copyright owner. For the purposes of this definition, \"submitted\"\n  means any form of electronic, verbal, or written communication sent\n  to the Licensor or its representatives, including but not limited to\n  communication on electronic mailing lists, source code control systems,\n  and issue tracking systems that are managed by, or on behalf of, the\n  Licensor for the purpose of discussing and improving the Work, but\n  excluding communication that is conspicuously marked or otherwise\n  designated in writing by the copyright owner as \"Not a Contribution.\"\n\n  \"Contributor\" shall mean Licensor and any individual or Legal Entity\n  on behalf of whom a Contribution has been received by Licensor and\n  subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n  this License, each Contributor hereby grants to You a perpetual,\n  worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n  copyright license to reproduce, prepare Derivative Works of,\n  publicly display, publicly perform, sublicense, and distribute the\n  Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n  this License, each Contributor hereby grants to You a perpetual,\n  worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n  (except as stated in this section) patent license to make, have made,\n  use, offer to sell, sell, import, and otherwise transfer the Work,\n  where such license applies only to those patent claims licensable\n  by such Contributor that are necessarily infringed by their\n  Contribution(s) alone or by combination of their Contribution(s)\n  with the Work to which such Contribution(s) was submitted. If You\n  institute patent litigation against any entity (including a\n  cross-claim or counterclaim in a lawsuit) alleging that the Work\n  or a Contribution incorporated within the Work constitutes direct\n  or contributory patent infringement, then any patent licenses\n  granted to You under this License for that Work shall terminate\n  as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n  Work or Derivative Works thereof in any medium, with or without\n  modifications, and in Source or Object form, provided that You\n  meet the following conditions:\n\n  (a) You must give any other recipients of the Work or\n      Derivative Works a copy of this License; and\n\n  (b) You must cause any modified files to carry prominent notices\n      stating that You changed the files; and\n\n  (c) You must retain, in the Source form of any Derivative Works\n      that You distribute, all copyright, patent, trademark, and\n      attribution notices from the Source form of the Work,\n      excluding those notices that do not pertain to any part of\n      the Derivative Works; and\n\n  (d) If the Work includes a \"NOTICE\" text file as part of its\n      distribution, then any Derivative Works that You distribute must\n      include a readable copy of the attribution notices contained\n      within such NOTICE file, excluding those notices that do not\n      pertain to any part of the Derivative Works, in at least one\n      of the following places: within a NOTICE text file distributed\n      as part of the Derivative Works; within the Source form or\n      documentation, if provided along with the Derivative Works; or,\n      within a display generated by the Derivative Works, if and\n      wherever such third-party notices normally appear. The contents\n      of the NOTICE file are for informational purposes only and\n      do not modify the License. You may add Your own attribution\n      notices within Derivative Works that You distribute, alongside\n      or as an addendum to the NOTICE text from the Work, provided\n      that such additional attribution notices cannot be construed\n      as modifying the License.\n\n  You may add Your own copyright statement to Your modifications and\n  may provide additional or different license terms and conditions\n  for use, reproduction, or distribution of Your modifications, or\n  for any such Derivative Works as a whole, provided Your use,\n  reproduction, and distribution of the Work otherwise complies with\n  the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n  any Contribution intentionally submitted for inclusion in the Work\n  by You to the Licensor shall be under the terms and conditions of\n  this License, without any additional terms or conditions.\n  Notwithstanding the above, nothing herein shall supersede or modify\n  the terms of any separate license agreement you may have executed\n  with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n  names, trademarks, service marks, or product names of the Licensor,\n  except as required for reasonable and customary use in describing the\n  origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n  agreed to in writing, Licensor provides the Work (and each\n  Contributor provides its Contributions) on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n  implied, including, without limitation, any warranties or conditions\n  of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n  PARTICULAR PURPOSE. You are solely responsible for determining the\n  appropriateness of using or redistributing the Work and assume any\n  risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n  whether in tort (including negligence), contract, or otherwise,\n  unless required by applicable law (such as deliberate and grossly\n  negligent acts) or agreed to in writing, shall any Contributor be\n  liable to You for damages, including any direct, indirect, special,\n  incidental, or consequential damages of any character arising as a\n  result of this License or out of the use or inability to use the\n  Work (including but not limited to damages for loss of goodwill,\n  work stoppage, computer failure or malfunction, or any and all\n  other commercial damages or losses), even if such Contributor\n  has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n  the Work or Derivative Works thereof, You may choose to offer,\n  and charge a fee for, acceptance of support, warranty, indemnity,\n  or other liability obligations and/or rights consistent with this\n  License. However, in accepting such obligations, You may act only\n  on Your own behalf and on Your sole responsibility, not on behalf\n  of any other Contributor, and only if You agree to indemnify,\n  defend, and hold each Contributor harmless for any liability\n  incurred by, or claims asserted against, such Contributor by reason\n  of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n  To apply the Apache License to your work, attach the following\n  boilerplate notice, with the fields enclosed by brackets \"[]\"\n  replaced with your own identifying information. (Don't include\n  the brackets!)  The text should be enclosed in the appropriate\n  comment syntax for the file format. We also recommend that a\n  file or class name and description of purpose be included on the\n  same \"printed page\" as the copyright notice for easier\n  identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n   http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\n---------------------------------------------------------------------------------\n\nFastViT (ml-fastvit)\n\nCopyright (C) 2023 Apple Inc. All Rights Reserved.\n\nIMPORTANT:  This Apple software is supplied to you by Apple\nInc. (\"Apple\") in consideration of your agreement to the following\nterms, and your use, installation, modification or redistribution of\nthis Apple software constitutes acceptance of these terms.  If you do\nnot agree with these terms, please do not use, install, modify or\nredistribute this Apple software.\n\nIn consideration of your agreement to abide by the following terms, and\nsubject to these terms, Apple grants you a personal, non-exclusive\nlicense, under Apple's copyrights in this original Apple software (the\n\"Apple Software\"), to use, reproduce, modify and redistribute the Apple\nSoftware, with or without modifications, in source and/or binary forms;\nprovided that if you redistribute the Apple Software in its entirety and\nwithout modifications, you must retain this notice and the following\ntext and disclaimers in all such redistributions of the Apple Software.\nNeither the name, trademarks, service marks or logos of Apple Inc. may\nbe used to endorse or promote products derived from the Apple Software\nwithout specific prior written permission from Apple.  Except as\nexpressly stated in this notice, no other rights or licenses, express or\nimplied, are granted by Apple herein, including but not limited to any\npatent rights that may be infringed by your derivative works or by other\nworks in which the Apple Software may be incorporated.\n\nThe Apple Software is provided by Apple on an \"AS IS\" basis.  APPLE\nMAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION\nTHE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS\nFOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND\nOPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS.\n\nIN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL\nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\nSUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS\nINTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION,\nMODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED\nAND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE),\nSTRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE\nPOSSIBILITY OF SUCH DAMAGE.\n\n---------------------------------------------------------------------------------\n\nmlx-vlm\n\nMIT License\n\nCopyright © 2023 Apple Inc.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\n---------------------------------------------------------------------------------\n\nMobileCLIP (ml-mobileclip)\n\nMIT License\n\nCopyright © 2024 Apple Inc.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\n-------------------------------------------------------------------------------\nSOFTWARE DISTRIBUTED WITH ML-MobileCLIP:\n\nThe ML-MobileCLIP model weights and data copyright and license terms can be\nfound in LICENSE_weights_data.\n\nThe ML-MobileCLIP software includes a number of subcomponents with separate\ncopyright notices and license terms - please see the file ACKNOWLEDGEMENTS.\n---------------------------------------------------------------------------------\n"
  },
  {
    "path": "CODE_OF_CONDUCT.md",
    "content": "# Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as\ncontributors and maintainers pledge to making participation in our project and\nour community a harassment-free experience for everyone, regardless of age, body\nsize, disability, ethnicity, sex characteristics, gender identity and expression,\nlevel of experience, education, socio-economic status, nationality, personal\nappearance, race, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment\ninclude:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or\n  advances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic\n  address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n  professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable\nbehavior and are expected to take appropriate and fair corrective action in\nresponse to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or\nreject comments, commits, code, wiki edits, issues, and other contributions\nthat are not aligned to this Code of Conduct, or to ban temporarily or\npermanently any contributor for other behaviors that they deem inappropriate,\nthreatening, offensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies within all project spaces, and it also applies when\nan individual is representing the project or its community in public spaces.\nExamples of representing a project or community include using an official\nproject e-mail address, posting via an official social media account, or acting\nas an appointed representative at an online or offline event. Representation of\na project may be further defined and clarified by project maintainers.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported by contacting the open source team at [opensource-conduct@group.apple.com](mailto:opensource-conduct@group.apple.com). All\ncomplaints will be reviewed and investigated and will result in a response that\nis deemed necessary and appropriate to the circumstances. The project team is\nobligated to maintain confidentiality with regard to the reporter of an incident.\nFurther details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good\nfaith may face temporary or permanent repercussions as determined by other\nmembers of the project's leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 1.4,\navailable at [https://www.contributor-covenant.org/version/1/4/code-of-conduct.html](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html)"
  },
  {
    "path": "CONTRIBUTING.md",
    "content": "# Contribution Guide\n\nThanks for your interest in contributing. This project was released to accompany a research paper for purposes of reproducibility, and beyond its publication there are limited plans for future development of the repository.\n\nWhile we welcome new pull requests and issues please note that our response may be limited. Forks and out-of-tree improvements are strongly encouraged.\n\n## Before you get started\n\nBy submitting a pull request, you represent that you have the right to license your contribution to Apple and the community, and agree by submitting the patch that your contributions are licensed under the [LICENSE](LICENSE).\n\nWe ask that all community members read and observe our [Code of Conduct](CODE_OF_CONDUCT.md).\n"
  },
  {
    "path": "LICENSE",
    "content": "Copyright (C) 2025 Apple Inc. All Rights Reserved.\n\nIMPORTANT:  This Apple software is supplied to you by Apple\nInc. (\"Apple\") in consideration of your agreement to the following\nterms, and your use, installation, modification or redistribution of\nthis Apple software constitutes acceptance of these terms.  If you do\nnot agree with these terms, please do not use, install, modify or\nredistribute this Apple software.\n\nIn consideration of your agreement to abide by the following terms, and\nsubject to these terms, Apple grants you a personal, non-exclusive\nlicense, under Apple's copyrights in this original Apple software (the\n\"Apple Software\"), to use, reproduce, modify and redistribute the Apple\nSoftware, with or without modifications, in source and/or binary forms;\nprovided that if you redistribute the Apple Software in its entirety and\nwithout modifications, you must retain this notice and the following\ntext and disclaimers in all such redistributions of the Apple Software.\nNeither the name, trademarks, service marks or logos of Apple Inc. may\nbe used to endorse or promote products derived from the Apple Software\nwithout specific prior written permission from Apple.  Except as\nexpressly stated in this notice, no other rights or licenses, express or\nimplied, are granted by Apple herein, including but not limited to any\npatent rights that may be infringed by your derivative works or by other\nworks in which the Apple Software may be incorporated.\n\nThe Apple Software is provided by Apple on an \"AS IS\" basis.  APPLE\nMAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION\nTHE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS\nFOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND\nOPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS.\n\nIN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL\nOR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\nSUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS\nINTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION,\nMODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED\nAND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE),\nSTRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE\nPOSSIBILITY OF SUCH DAMAGE.\n\n-------------------------------------------------------------------------------\nSOFTWARE DISTRIBUTED WITH ML-FASTVLM:\n\nThe ml-fastvlm software includes a number of subcomponents with separate\ncopyright notices and license terms - please see the file ACKNOWLEDGEMENTS.\n\nThe ml-fastvlm model weights copyright and license terms can be\nfound in LICENSE_MODEL file.\n-------------------------------------------------------------------------------\n"
  },
  {
    "path": "LICENSE_MODEL",
    "content": "Disclaimer: IMPORTANT: This Apple Machine Learning Research Model is\nspecifically developed and released by Apple Inc. (\"Apple\") for the sole purpose\nof scientific research of artificial intelligence and machine-learning\ntechnology. “Apple Machine Learning Research Model” means the model, including\nbut not limited to algorithms, formulas, trained model weights, parameters,\nconfigurations, checkpoints, and any related materials (including\ndocumentation).\n\nThis Apple Machine Learning Research Model is provided to You by\nApple in consideration of your agreement to the following terms, and your use,\nmodification, creation of Model Derivatives, and or redistribution of the Apple\nMachine Learning Research Model constitutes acceptance of this Agreement. If You\ndo not agree with these terms, please do not use, modify, create Model\nDerivatives of, or distribute this Apple Machine Learning Research Model or\nModel Derivatives.\n\n* License Scope: In consideration of your agreement to abide by the following\n  terms, and subject to these terms, Apple hereby grants you a personal,\n  non-exclusive, worldwide, non-transferable, royalty-free, revocable, and\n  limited license, to use, copy, modify, distribute, and create Model\n  Derivatives (defined below) of the Apple Machine Learning Research Model\n  exclusively for Research Purposes. You agree that any Model Derivatives You\n  may create or that may be created for You will be limited to Research Purposes\n  as well. “Research Purposes” means non-commercial scientific research and\n  academic development activities, such as experimentation, analysis, testing\n  conducted by You with the sole intent to advance scientific knowledge and\n  research. “Research Purposes” does not include any commercial exploitation,\n  product development or use in any commercial product or service.\n\n* Distribution of Apple Machine Learning Research Model and Model Derivatives:\n  If you choose to redistribute Apple Machine Learning Research Model or its\n  Model Derivatives, you must provide a copy of this Agreement to such third\n  party, and ensure that the following attribution notice be provided: “Apple\n  Machine Learning Research Model is licensed under the Apple Machine Learning\n  Research Model License Agreement.” Additionally, all Model Derivatives must\n  clearly be identified as such, including disclosure of modifications and\n  changes made to the Apple Machine Learning Research Model. The name,\n  trademarks, service marks or logos of Apple may not be used to endorse or\n  promote Model Derivatives or the relationship between You and Apple. “Model\n  Derivatives” means any models or any other artifacts created by modifications,\n  improvements, adaptations, alterations to the architecture, algorithm or\n  training processes of the Apple Machine Learning Research Model, or by any\n  retraining, fine-tuning of the Apple Machine Learning Research Model.\n\n* No Other License: Except as expressly stated in this notice, no other rights\n  or licenses, express or implied, are granted by Apple herein, including but\n  not limited to any patent, trademark, and similar intellectual property rights\n  worldwide that may be infringed by the Apple Machine Learning Research Model,\n  the Model Derivatives or by other works in which the Apple Machine Learning\n  Research Model may be incorporated.\n\n* Compliance with Laws: Your use of Apple Machine Learning Research Model must\n  be in compliance with all applicable laws and regulations.\n\n* Term and Termination: The term of this Agreement will begin upon your\n  acceptance of this Agreement or use of the Apple Machine Learning Research\n  Model and will continue until terminated in accordance with the following\n  terms. Apple may terminate this Agreement at any time if You are in breach of\n  any term or condition of this Agreement. Upon termination of this Agreement,\n  You must cease to use all Apple Machine Learning Research Models and Model\n  Derivatives and permanently delete any copy thereof. Sections 3, 6 and 7 will\n  survive termination.\n\n* Disclaimer and Limitation of Liability: This Apple Machine Learning Research\n  Model and any outputs generated by the Apple Machine Learning Research Model\n  are provided on an “AS IS” basis. APPLE MAKES NO WARRANTIES, EXPRESS OR\n  IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF\n  NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE,\n  REGARDING THE APPLE MACHINE LEARNING RESEARCH MODEL OR OUTPUTS GENERATED BY\n  THE APPLE MACHINE LEARNING RESEARCH MODEL. You are solely responsible for\n  determining the appropriateness of using or redistributing the Apple Machine\n  Learning Research Model and any outputs of the Apple Machine Learning Research\n  Model and assume any risks associated with Your use of the Apple Machine\n  Learning Research Model and any output and results. IN NO EVENT SHALL APPLE BE\n  LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n  IN ANY WAY OUT OF THE USE, REPRODUCTION, MODIFICATION AND/OR DISTRIBUTION OF\n  THE APPLE MACHINE LEARNING RESEARCH MODEL AND ANY OUTPUTS OF THE APPLE MACHINE\n  LEARNING RESEARCH MODEL, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT,\n  TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS\n  BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n* Governing Law: This Agreement will be governed by and construed under the laws\n  of the State of California without regard to its choice of law principles. The\n  Convention on Contracts for the International Sale of Goods shall not apply to\n  the Agreement except that the arbitration clause and any arbitration hereunder\n  shall be governed by the Federal Arbitration Act, Chapters 1 and 2.\n\nCopyright (C) 2025 Apple Inc. All Rights Reserved.\n"
  },
  {
    "path": "README.md",
    "content": "# FastVLM: Efficient Vision Encoding for Vision Language Models\n\nThis is the official repository of\n**[FastVLM: Efficient Vision Encoding for Vision Language Models](https://www.arxiv.org/abs/2412.13303). (CVPR 2025)**\n\n[//]: # (![FastViTHD Performance]&#40;docs/acc_vs_latency_qwen-2.png&#41;)\n<p align=\"center\">\n<img src=\"docs/acc_vs_latency_qwen-2.png\" alt=\"Accuracy vs latency figure.\" width=\"400\"/>\n</p>\n\n### Highlights\n* We introduce FastViTHD, a novel hybrid vision encoder designed to output fewer tokens and significantly reduce encoding time for high-resolution images.  \n* Our smallest variant outperforms LLaVA-OneVision-0.5B with 85x faster Time-to-First-Token (TTFT) and 3.4x smaller vision encoder.\n* Our larger variants using Qwen2-7B LLM outperform recent works like Cambrian-1-8B while using a single image encoder with a 7.9x faster TTFT.\n* Demo iOS app to demonstrate the performance of our model on a mobile device.\n\n<table>\n<tr>\n    <td><img src=\"docs/fastvlm-counting.gif\" alt=\"FastVLM - Counting\"></td>\n    <td><img src=\"docs/fastvlm-handwriting.gif\" alt=\"FastVLM - Handwriting\"></td>\n    <td><img src=\"docs/fastvlm-emoji.gif\" alt=\"FastVLM - Emoji\"></td>\n</tr>\n</table>\n\n## Getting Started\nWe use LLaVA codebase to train FastVLM variants. In order to train or finetune your own variants, \nplease follow instructions provided in [LLaVA](https://github.com/haotian-liu/LLaVA) codebase. \nWe provide instructions for running inference with our models.   \n\n### Setup\n```bash\nconda create -n fastvlm python=3.10\nconda activate fastvlm\npip install -e .\n```\n\n### Model Zoo\nFor detailed information on various evaluations, please refer to our [paper](https://www.arxiv.org/abs/2412.13303).\n\n| Model        | Stage |                                            Pytorch Checkpoint (url)                                             |\n|:-------------|:-----:|:---------------------------------------------------------------------------------------------------------------:|\n| FastVLM-0.5B |   2   | [fastvlm_0.5b_stage2](https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_0.5b_stage2.zip) |\n|              |   3   | [fastvlm_0.5b_stage3](https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_0.5b_stage3.zip) |\n| FastVLM-1.5B |   2   | [fastvlm_1.5b_stage2](https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_1.5b_stage2.zip) |\n|              |   3   | [fastvlm_1.5b_stage3](https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_1.5b_stage3.zip)  |\n| FastVLM-7B   |   2   | [fastvlm_7b_stage2](https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_7b_stage2.zip)  |\n|              |   3   | [fastvlm_7b_stage3](https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_7b_stage3.zip)  |\n\nTo download all the pretrained checkpoints run the command below (note that this might take some time depending on your connection so might be good to grab ☕️ while you wait).\n\n```bash\nbash get_models.sh   # Files will be downloaded to `checkpoints` directory.\n```\n\n### Usage Example\nTo run inference of PyTorch checkpoint, follow the instruction below\n```bash\npython predict.py --model-path /path/to/checkpoint-dir \\\n                  --image-file /path/to/image.png \\\n                  --prompt \"Describe the image.\"\n```\n\n### Inference on Apple Silicon\nTo run inference on Apple Silicon, pytorch checkpoints have to be exported to format \nsuitable for running on Apple Silicon, detailed instructions and code can be found [`model_export`](model_export/) subfolder.\nPlease see the README there for more details.\n\nFor convenience, we provide 3 models that are in Apple Silicon compatible format: [fastvlm_0.5b_stage3](https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_0.5b_stage3_llm.fp16.zip), \n[fastvlm_1.5b_stage3](https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_1.5b_stage3_llm.int8.zip), \n[fastvlm_7b_stage3](https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_7b_stage3_llm.int4.zip). \nWe encourage developers to export the model of their choice with the appropriate quantization levels following \nthe instructions in [`model_export`](model_export/).\n\n### Inference on Apple Devices\nTo run inference on Apple devices like iPhone, iPad or Mac, see [`app`](app/) subfolder for more details.\n\n## Citation\nIf you found this code useful, please cite the following paper:\n```\n@InProceedings{fastvlm2025,\n  author = {Pavan Kumar Anasosalu Vasu, Fartash Faghri, Chun-Liang Li, Cem Koc, Nate True, Albert Antony, Gokul Santhanam, James Gabriel, Peter Grasch, Oncel Tuzel, Hadi Pouransari},\n  title = {FastVLM: Efficient Vision Encoding for Vision Language Models},\n  booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n  month = {June},\n  year = {2025},\n}\n```\n\n## Acknowledgements\nOur codebase is built using multiple opensource contributions, please see [ACKNOWLEDGEMENTS](ACKNOWLEDGEMENTS) for more details. \n\n## License\nPlease check out the repository [LICENSE](LICENSE) before using the provided code and\n[LICENSE_MODEL](LICENSE_MODEL) for the released models.\n"
  },
  {
    "path": "app/Configuration/Build.xcconfig",
    "content": "// The `DISAMBIGUATOR` configuration is to make it easier to build\n// and run a sample code project. Once you set your project's development team,\n// you'll have a unique bundle identifier. This is because the bundle identifier\n// is derived based on the 'DISAMBIGUATOR' value. Do not use this\n// approach in your own projects—it's only useful for example projects because\n// they are frequently downloaded and don't have a development team set.\nDISAMBIGUATOR=${DEVELOPMENT_TEAM}\n"
  },
  {
    "path": "app/FastVLM/FastVLM.h",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\n#ifndef FastVLM_h\n#define FastVLM_h\n\n\n#endif /* FastVLM_h */\n"
  },
  {
    "path": "app/FastVLM/FastVLM.swift",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\nimport CoreImage\nimport CoreML\nimport Foundation\nimport MLX\nimport MLXFast\nimport MLXLMCommon\nimport MLXNN\nimport MLXVLM\nimport Tokenizers\n\n// FastVLM is Qwen2VL with a custom vision tower.\n\n// MARK: - Common\n\n/// Rotates half the hidden dims of the input\nprivate func rotateHalf(_ x: MLXArray) -> MLXArray {\n    let index = x.dim(-1) / 2\n    let x1 = x[.ellipsis, 0 ..< index]\n    let x2 = x[.ellipsis, index...]\n    return concatenated([-x2, x1], axis: -1)\n}\n\n// MARK: - Language\n\nprivate enum Language {\n\n    /// Applies Rotary Position Embedding with Multimodal Sections to the query and key tensors\n    static private func applyMultimodalRotaryPositionEmbedding(\n        q: MLXArray, k: MLXArray, cos: MLXArray, sin: MLXArray,\n        positionIds: MLXArray, mropeSection: [Int]\n    ) -> (MLXArray, MLXArray) {\n        var cos = cos[positionIds]\n        var sin = sin[positionIds]\n\n        cos =\n            concatenated(\n                // [m[i % 3] for i, m in enumerate(mx.split(cos, mrope_section, axis=-1))]\n                split(cos, indices: mropeSection, axis: -1).enumerated().map { i, m in m[i % 3] },\n                axis: -1\n            )[0..., .newAxis, 0..., 0...]\n\n        sin =\n            concatenated(\n                split(sin, indices: mropeSection, axis: -1).enumerated().map { i, m in m[i % 3] },\n                axis: -1\n            )[0..., .newAxis, 0..., 0...]\n\n        // Apply rotary embedding\n        let qEmbed = (q * cos) + (rotateHalf(q) * sin)\n        let kEmbed = (k * cos) + (rotateHalf(k) * sin)\n        return (qEmbed, kEmbed)\n    }\n\n    fileprivate class Attention: Module {\n\n        let heads: Int\n        let kvHeads: Int\n        let headDim: Int\n        let scale: Float\n        let mropeSection: [Int]\n\n        @ModuleInfo(key: \"q_proj\") var wq: Linear\n        @ModuleInfo(key: \"k_proj\") var wk: Linear\n        @ModuleInfo(key: \"v_proj\") var wv: Linear\n        @ModuleInfo(key: \"o_proj\") var wo: Linear\n\n        @ModuleInfo(key: \"rotary_emb\") var rotaryEmbedding: RoPE\n\n        public init(_ args: FastVLMConfiguration.TextConfiguration) {\n            let dim = args.hiddenSize\n            self.heads = args.attentionHeads\n            self.kvHeads = args.kvHeads\n            self.headDim = dim / heads\n            self.scale = pow(Float(headDim), -0.5)\n\n            self._wq.wrappedValue = Linear(dim, heads * headDim, bias: true)\n            self._wk.wrappedValue = Linear(dim, kvHeads * headDim, bias: true)\n            self._wv.wrappedValue = Linear(dim, kvHeads * headDim, bias: true)\n            self._wo.wrappedValue = Linear(heads * headDim, dim, bias: false)\n\n            if let v = args.ropeScaling?[\"mrope_section\"], let array = v.asInts() {\n                // mrope_section = np.cumsum(mrope_section * 2)[:-1].tolist()\n                self.mropeSection = sequence(state: (0, array.makeIterator())) { state in\n                    if let v = state.1.next() {\n                        // note the *2\n                        state.0 += v * 2\n                        return state.0\n                    } else {\n                        return nil\n                    }\n                }.dropLast()\n            } else {\n                fatalError(\"rope_scaling['mrope_section'] must be an array of integers\")\n            }\n\n            self._rotaryEmbedding.wrappedValue = RoPE(\n                dimensions: headDim, traditional: args.ropeTraditional, base: args.ropeTheta)\n        }\n\n        public func callAsFunction(\n            _ x: MLXArray, mask: MLXArray? = nil, cache: KVCache?\n        ) -> MLXArray {\n            let (B, L) = (x.dim(0), x.dim(1))\n\n            var queries = wq(x)\n            var keys = wk(x)\n            var values = wv(x)\n\n            // prepare the queries, keys and values for the attention computation\n            queries = queries.reshaped(B, L, heads, headDim).transposed(0, 2, 1, 3)\n            keys = keys.reshaped(B, L, kvHeads, headDim).transposed(0, 2, 1, 3)\n            values = values.reshaped(B, L, kvHeads, headDim).transposed(0, 2, 1, 3)\n\n            let offset = cache?.offset ?? 0\n            let mask = mask?[0..., 0 ..< keys.dim(-2)]\n\n            queries = rotaryEmbedding(queries, offset: offset)\n            keys = rotaryEmbedding(keys, offset: offset)\n\n            if let cache {\n                (keys, values) = cache.update(keys: keys, values: values)\n            }\n\n            let output = MLXFast.scaledDotProductAttention(\n                queries: queries, keys: keys, values: values, scale: scale, mask: mask\n            )\n            .transposed(0, 2, 1, 3)\n            .reshaped(B, L, -1)\n\n            return wo(output)\n        }\n    }\n\n    fileprivate class MLP: Module, UnaryLayer {\n\n        @ModuleInfo(key: \"gate_proj\") var gate: Linear\n        @ModuleInfo(key: \"down_proj\") var down: Linear\n        @ModuleInfo(key: \"up_proj\") var up: Linear\n\n        public init(dimensions: Int, hiddenDimensions: Int) {\n            self._gate.wrappedValue = Linear(dimensions, hiddenDimensions, bias: false)\n            self._down.wrappedValue = Linear(hiddenDimensions, dimensions, bias: false)\n            self._up.wrappedValue = Linear(dimensions, hiddenDimensions, bias: false)\n        }\n\n        public func callAsFunction(_ x: MLXArray) -> MLXArray {\n            down(silu(gate(x)) * up(x))\n        }\n    }\n\n    fileprivate class FastVLMDecoderLayer: Module {\n\n        @ModuleInfo(key: \"self_attn\") var attention: Attention\n        let mlp: MLP\n\n        @ModuleInfo(key: \"input_layernorm\") var inputLayerNorm: RMSNorm\n        @ModuleInfo(key: \"post_attention_layernorm\") var postAttentionLayerNorm: RMSNorm\n\n        public init(_ args: FastVLMConfiguration.TextConfiguration) {\n            self._attention.wrappedValue = Attention(args)\n            self.mlp = MLP(dimensions: args.hiddenSize, hiddenDimensions: args.intermediateSize)\n            self._inputLayerNorm.wrappedValue = RMSNorm(\n                dimensions: args.hiddenSize, eps: args.rmsNormEps)\n            self._postAttentionLayerNorm.wrappedValue = RMSNorm(\n                dimensions: args.hiddenSize, eps: args.rmsNormEps)\n        }\n\n        public func callAsFunction(\n            _ x: MLXArray, mask: MLXArray? = nil, cache: KVCache?\n        ) -> MLXArray {\n            var r = attention(inputLayerNorm(x), mask: mask, cache: cache)\n            let h = x + r\n            r = mlp(postAttentionLayerNorm(h))\n            let out = h + r\n            return out\n        }\n    }\n\n    fileprivate class Qwen2Model: Module {\n\n        @ModuleInfo(key: \"embed_tokens\") var embedTokens: Embedding\n\n        fileprivate let layers: [FastVLMDecoderLayer]\n        fileprivate let norm: RMSNorm\n\n        public init(_ args: FastVLMConfiguration.TextConfiguration) {\n            precondition(args.vocabularySize > 0)\n\n            self._embedTokens.wrappedValue = Embedding(\n                embeddingCount: args.vocabularySize, dimensions: args.hiddenSize)\n\n            self.layers = (0 ..< args.hiddenLayers)\n                .map { _ in\n                    FastVLMDecoderLayer(args)\n                }\n            self.norm = RMSNorm(dimensions: args.hiddenSize, eps: args.rmsNormEps)\n        }\n\n        public func callAsFunction(\n            _ inputs: MLXArray?, cache: [KVCache]? = nil, inputEmbedding: MLXArray? = nil\n        ) -> MLXArray {\n            var h: MLXArray\n            if let inputEmbedding {\n                h = inputEmbedding\n            } else if let inputs {\n                h = embedTokens(inputs)\n            } else {\n                fatalError(\"one of inputs or inputEmbedding must be non-nil\")\n            }\n\n            let mask = createAttentionMask(h: h, cache: cache)\n\n            for (i, layer) in layers.enumerated() {\n                h = layer(h, mask: mask, cache: cache?[i])\n            }\n\n            return norm(h)\n        }\n    }\n\n    fileprivate class LanguageModel: Module, KVCacheDimensionProvider {\n        @ModuleInfo var model: Qwen2Model\n        @ModuleInfo(key: \"lm_head\") var lmHead: Linear?\n\n        var kvHeads: [Int]\n\n        public init(_ args: FastVLMConfiguration.TextConfiguration) {\n            self.model = Qwen2Model(args)\n\n            if !args.tieWordEmbeddings {\n                _lmHead.wrappedValue = Linear(args.hiddenSize, args.vocabularySize, bias: false)\n            }\n\n            self.kvHeads = (0 ..< args.hiddenLayers).map { _ in args.kvHeads }\n        }\n\n        public func callAsFunction(\n            _ inputs: MLXArray?, cache: [KVCache]? = nil, inputEmbedding: MLXArray? = nil\n        ) -> LMOutput {\n            var out = model(inputs, cache: cache, inputEmbedding: inputEmbedding)\n            if let lmHead {\n                out = lmHead(out)\n            } else {\n                out = model.embedTokens.asLinear(out)\n            }\n            return LMOutput(logits: out)\n        }\n    }\n}\n\n// MARK: - Vision\n\nprivate enum Vision {\n\n    fileprivate class VisionModelCoreML {\n\n        let lock = NSLock()\n        var _model: fastvithd?\n\n        init() {\n        }\n\n        func load() throws -> fastvithd {\n            try lock.withLock {\n                if let model = _model { return model }\n                let model = try fastvithd()\n                _model = model\n                return model\n            }\n        }\n\n        public func model() -> fastvithd {\n            try! load()\n        }\n\n        public func encode(_ image: MLXArray) -> MLXArray {\n            // MLMultiArray requires mutable input data\n            var (data, strides) = {\n                let arrayData = image.asType(.float32).asData(access: .noCopyIfContiguous)\n                return (arrayData.data, arrayData.strides)\n            }()\n\n            precondition(image.ndim == 4)\n            precondition(image.dim(0) == 1)\n            precondition(image.dim(1) == 3)\n\n            let h = NSNumber(value: image.dim(2))\n            let w = NSNumber(value: image.dim(3))\n\n            return data.withUnsafeMutableBytes { (ptr: UnsafeMutableRawBufferPointer) in\n                // wrap the backing of the MLXArray\n                let array = try! MLMultiArray(\n                    dataPointer: ptr.baseAddress!, shape: [1, 3, h, w], dataType: .float32,\n                    strides: strides.map { .init(value: $0) })\n\n                // inference\n                let output = try! model().prediction(images: array)\n                precondition(output.image_features.shape == [1, 256, 3072])\n                precondition(output.image_features.dataType == .float32)\n                return output.image_features.withUnsafeBytes { ptr in\n                    MLXArray(ptr, [1, 256, 3072], type: Float32.self)\n                }\n            }\n        }\n    }\n\n    fileprivate class VisionModel: Module {\n\n        let model = VisionModelCoreML()\n\n        public override init() {}\n\n        public func callAsFunction(_ hiddenStates: MLXArray, gridThw: [THW]) -> MLXArray {\n            model.encode(hiddenStates)\n        }\n    }\n}\n\n// MARK: - Processor\n\n/// FastVLM `UserInputProcessor`.\n///\n/// This is meant to be used with ``FastVLM`` and is typically created by ``VLMModelFactory``.\npublic class FastVLMProcessor: UserInputProcessor {\n\n    private let config: FastVLMProcessorConfiguration\n    private let imageProcessingConfig: FastVLMPreProcessorConfiguration\n    private let tokenizer: any Tokenizer\n\n    public init(_ config: FastVLMPreProcessorConfiguration, tokenizer: any Tokenizer) {\n        self.config = FastVLMProcessorConfiguration()\n        self.imageProcessingConfig = config\n        self.tokenizer = tokenizer\n    }\n\n    public func preprocess(image: CIImage, processing: UserInput.Processing?) throws -> (\n        MLXArray, THW\n    ) {\n        // first apply the user requested resizing, etc. if any\n        var image = MediaProcessingExtensions.apply(image, processing: processing)\n\n        // image_processing_clip.py\n        let size = MediaProcessingExtensions.fitIn(\n            image.extent.size, shortestEdge: imageProcessingConfig.size.shortestEdge)\n        image = MediaProcessingExtensions.resampleBicubic(image, to: size)\n\n        image = MediaProcessingExtensions.centerCrop(\n            image, size: imageProcessingConfig.cropSize.size)\n\n        image = MediaProcessing.normalize(\n            image, mean: imageProcessingConfig.imageMeanTuple,\n            std: imageProcessingConfig.imageStdTuple)\n\n        let array = MediaProcessingExtensions.asPlanarMLXArray(image)\n        return (array, .init(0, array.dim(2), array.dim(3)))\n    }\n\n    public func prepare(prompt: UserInput.Prompt, imageTHW: THW?) -> String {\n        var messages = prompt.asMessages()\n        if messages[0][\"role\"] != \"system\" {\n            messages.insert([\"role\": \"system\", \"content\": \"You are a helpful assistant.\"], at: 0)\n        }\n\n        let lastIndex = messages.count - 1\n        var lastMessage = messages[lastIndex][\"content\"] ?? \"\"\n\n        // processing_llava.py\n        if let imageTHW {\n            let height = imageTHW.h\n            let width = imageTHW.w\n            let patchSize = config.patchSize\n\n            var numImageTokens =\n                (height / patchSize) * (width / patchSize) + config.numAdditionalImageTokens\n\n            if config.visionFeatureSelectStrategy == .default {\n                numImageTokens -= 1\n            }\n\n            lastMessage += Array(repeating: config.imageToken, count: numImageTokens)\n                .joined()\n        }\n\n        messages[lastIndex][\"content\"] = lastMessage\n\n        return\n            messages\n            .map {\n                \"<|im_start|>\\($0[\"role\"] ?? \"user\")\\n\\($0[\"content\"] ?? \"\")<|im_end|>\"\n            }\n            .joined(separator: \"\\n\")\n            + \"\\n<|im_start|>assistant\\n\"\n    }\n\n    public func prepare(input: UserInput) throws -> LMInput {\n        if input.images.isEmpty {\n            // just a straight text prompt\n            let prompt = prepare(prompt: input.prompt, imageTHW: nil)\n            let promptTokens = tokenizer.encode(text: prompt)\n            return LMInput(tokens: MLXArray(promptTokens))\n        }\n\n        if input.images.count > 1 {\n            throw VLMError.singleImageAllowed\n        }\n\n        let (pixels, thw) = try preprocess(\n            image: input.images[0].asCIImage(), processing: input.processing)\n        let image = LMInput.ProcessedImage(pixels: pixels, imageGridThw: [thw])\n\n        let prompt = prepare(prompt: input.prompt, imageTHW: thw)\n        let promptTokens = tokenizer.encode(text: prompt)\n        let promptArray = MLXArray(promptTokens).expandedDimensions(axis: 0)\n        let mask = ones(like: promptArray).asType(.int8)\n\n        return LMInput(text: .init(tokens: promptArray, mask: mask), image: image)\n    }\n\n}\n\n// MARK: - Model\n\nprivate class FastVLMMultiModalProjector: Module, UnaryLayer {\n\n    @ModuleInfo(key: \"linear_0\") var linear0: Linear\n    @ModuleInfo(key: \"gelu\") var gelu: GELU\n    @ModuleInfo(key: \"linear_2\") var linear2: Linear\n\n    public init(_ config: FastVLMConfiguration) {\n        self._linear0.wrappedValue = Linear(\n            config.visionConfiguration.hiddenSize,\n            config.textConfiguration.hiddenSize,\n            bias: true)\n        self._gelu.wrappedValue = GELU()\n        self._linear2.wrappedValue = Linear(\n            config.textConfiguration.hiddenSize,\n            config.textConfiguration.hiddenSize,\n            bias: true)\n    }\n\n    public func callAsFunction(_ x: MLXArray) -> MLXArray {\n        var x = linear0(x)\n        x = gelu(x)\n        x = linear2(x)\n        return x\n    }\n}\n\n/// FastVLM\n///\n/// This is typically created by ``VLMModelFactory``.\npublic class FastVLM: Module, VLMModel, KVCacheDimensionProvider {\n\n    static public var modelConfiguration: ModelConfiguration {\n        let bundle = Bundle(for: FastVLM.self)\n        let url = bundle.url(forResource: \"config\", withExtension: \"json\")!\n            .resolvingSymlinksInPath()\n            .deletingLastPathComponent()\n        return ModelConfiguration(directory: url)\n    }\n\n    static public func register(modelFactory: VLMModelFactory) {\n        modelFactory.typeRegistry.registerModelType(\"llava_qwen2\") { url in\n            let configuration = try JSONDecoder().decode(\n                FastVLMConfiguration.self, from: Data(contentsOf: url))\n            return FastVLM(configuration)\n        }\n\n        modelFactory.processorRegistry.registerProcessorType(\"LlavaProcessor\") { url, tokenizer in\n            let configuration = try JSONDecoder().decode(\n                FastVLMPreProcessorConfiguration.self, from: Data(contentsOf: url))\n            return FastVLMProcessor(configuration, tokenizer: tokenizer)\n        }\n    }\n\n    @ModuleInfo(key: \"vision_tower\") private var visionModel: Vision.VisionModel\n    @ModuleInfo(key: \"language_model\") private var languageModel: Language.LanguageModel\n    @ModuleInfo(key: \"multi_modal_projector\") private var multiModalProjector:\n        FastVLMMultiModalProjector\n\n    public let config: FastVLMConfiguration\n\n    public var vocabularySize: Int { config.baseConfiguration.vocabularySize }\n    public var kvHeads: [Int] { languageModel.kvHeads }\n\n    public func loraLinearLayers() -> MLXLMCommon.LoRALinearLayers {\n        languageModel.model.layers.map { ($0.attention, [\"q_proj\", \"v_proj\"]) }\n    }\n\n    public init(_ config: FastVLMConfiguration) {\n        self.config = config\n        self._visionModel.wrappedValue = Vision.VisionModel()\n        self._languageModel.wrappedValue = Language.LanguageModel(config.textConfiguration)\n        self._multiModalProjector.wrappedValue = FastVLMMultiModalProjector(config)\n    }\n\n    private func inputEmbeddings(inputIds: MLXArray, pixelValues: MLXArray?, gridThw: [THW]?)\n        -> MLXArray\n    {\n        guard let pixelValues, let gridThw else {\n            return languageModel(inputIds).logits\n        }\n\n        // Get the input embeddings from the language model\n        let inputEmbeds = languageModel.model.embedTokens(inputIds)\n\n        // Get the ouptut hidden states from the vision model\n        let imageFeaturesCoreML = self.visionModel(pixelValues, gridThw: gridThw)\n        let imageFeatures = multiModalProjector(imageFeaturesCoreML)\n\n        // Insert special image tokens in the input_ids\n        return mergeInputIdsWithImageFeatures(\n            inputIds: inputIds, inputEmbeds: inputEmbeds, imageFeatures: imageFeatures)\n    }\n\n    private func mergeInputIdsWithImageFeatures(\n        inputIds: MLXArray, inputEmbeds: MLXArray, imageFeatures: MLXArray\n    ) -> MLXArray {\n        let imageTokenIndex = config.baseConfiguration.imageTokenId\n\n        var imageIndices = [Int]()\n        for (i, v) in inputIds.asArray(Int.self).enumerated() {\n            if v == imageTokenIndex {\n                imageIndices.append(i)\n            }\n        }\n\n        inputEmbeds[0..., MLXArray(imageIndices), 0...] = imageFeatures\n        return inputEmbeds\n    }\n\n    public func prepare(_ input: LMInput, cache: [any KVCache], windowSize: Int?) throws\n        -> PrepareResult\n    {\n        let gridThw = input.image?.imageGridThw\n\n        let dtype = DType.float32\n        let pixels = input.image?.pixels.asType(dtype)\n\n        let inputEmbeddings = self.inputEmbeddings(\n            inputIds: input.text.tokens, pixelValues: pixels, gridThw: gridThw)\n\n        let result = languageModel(nil, cache: cache, inputEmbedding: inputEmbeddings)\n\n        return .logits(result)\n    }\n\n    public func callAsFunction(_ inputs: MLXArray, cache: [any KVCache]?) -> MLXArray {\n        languageModel(inputs, cache: cache).logits\n    }\n\n    public func sanitize(weights: [String: MLXArray]) -> [String: MLXArray] {\n        _ = try? visionModel.model.load()\n\n        return weights\n    }\n}\n\n// MARK: - Configuration\n\n/// Configuration for ``FastVLM``\npublic struct FastVLMConfiguration: Codable, Sendable {\n\n    public struct VisionConfiguration: Codable, Sendable {\n        public let hiddenSize: Int\n\n        enum CodingKeys: String, CodingKey {\n            case hiddenSize = \"mm_hidden_size\"\n        }\n    }\n\n    public struct TextConfiguration: Codable, Sendable {\n        public let modelType: String\n        public let hiddenSize: Int\n        public let hiddenLayers: Int\n        public let intermediateSize: Int\n        public let attentionHeads: Int\n        private let _rmsNormEps: Float?\n        public var rmsNormEps: Float { _rmsNormEps ?? 1e-6 }\n        public let vocabularySize: Int\n        public let kvHeads: Int\n        private let _maxPositionEmbeddings: Int?\n        public var maxpPositionEmbeddings: Int { _maxPositionEmbeddings ?? 32768 }\n        private let _ropeTheta: Float?\n        public var ropeTheta: Float { _ropeTheta ?? 1_000_000 }\n        private let _ropeTraditional: Bool?\n        public var ropeTraditional: Bool { _ropeTraditional ?? false }\n        public let _ropeScaling: [String: StringOrNumber]?\n        public var ropeScaling: [String: StringOrNumber]? {\n            _ropeScaling ?? [\"mrope_section\": .ints([2, 1, 1])]\n        }\n        private let _tieWordEmbeddings: Bool?\n        public var tieWordEmbeddings: Bool { _tieWordEmbeddings ?? true }\n\n        enum CodingKeys: String, CodingKey {\n            case modelType = \"model_type\"\n            case hiddenSize = \"hidden_size\"\n            case hiddenLayers = \"num_hidden_layers\"\n            case intermediateSize = \"intermediate_size\"\n            case attentionHeads = \"num_attention_heads\"\n            case _rmsNormEps = \"rms_norm_eps\"\n            case vocabularySize = \"vocab_size\"\n            case kvHeads = \"num_key_value_heads\"\n            case _maxPositionEmbeddings = \"max_position_embeddings\"\n            case _ropeTheta = \"rope_theta\"\n            case _ropeTraditional = \"rope_traditional\"\n            case _ropeScaling = \"rope_scaling\"\n            case _tieWordEmbeddings = \"tie_word_embeddings\"\n        }\n    }\n\n    public struct BaseConfiguration: Codable, Sendable {\n        public let modelType: String\n        public let vocabularySize: Int\n        public let imageTokenId: Int\n        public let hiddenSize: Int\n\n        enum CodingKeys: String, CodingKey {\n            case modelType = \"model_type\"\n            case vocabularySize = \"vocab_size\"\n            case imageTokenId = \"image_token_index\"\n            case hiddenSize = \"hidden_size\"\n        }\n    }\n\n    public let visionConfiguration: VisionConfiguration\n    public let textConfiguration: TextConfiguration\n    public let baseConfiguration: BaseConfiguration\n\n    public init(from decoder: any Swift.Decoder) throws {\n        // these are overlaid in the top level\n        self.visionConfiguration = try VisionConfiguration(from: decoder)\n        self.textConfiguration = try TextConfiguration(from: decoder)\n        self.baseConfiguration = try BaseConfiguration(from: decoder)\n    }\n}\n\n/// Configuration for ``FastVLMProcessor``\npublic struct FastVLMPreProcessorConfiguration: Codable, Sendable {\n\n    public struct CropSize: Codable, Sendable {\n        let width: Int\n        let height: Int\n\n        var size: CGSize { .init(width: CGFloat(width), height: CGFloat(height)) }\n    }\n\n    public struct Size: Codable, Sendable {\n        let shortestEdge: Int\n\n        enum CodingKeys: String, CodingKey {\n            case shortestEdge = \"shortest_edge\"\n        }\n    }\n\n    public var imageMean: [CGFloat]\n    public var imageStd: [CGFloat]\n    public var size: Size\n    public var cropSize: CropSize\n\n    public var imageMeanTuple: (CGFloat, CGFloat, CGFloat) {\n        (imageMean[0], imageMean[1], imageMean[2])\n    }\n    public var imageStdTuple: (CGFloat, CGFloat, CGFloat) {\n        (imageStd[0], imageStd[1], imageStd[2])\n    }\n\n    enum CodingKeys: String, CodingKey {\n        case imageMean = \"image_mean\"\n        case imageStd = \"image_std\"\n        case size\n        case cropSize = \"crop_size\"\n    }\n}\n\npublic struct FastVLMProcessorConfiguration: Codable, Sendable {\n\n    public enum Strategy: Codable, Sendable {\n        case `default`\n    }\n\n    public var imageToken = \"<image>\"\n    public var numAdditionalImageTokens = 0\n    public var patchSize = 64\n    public var visionFeatureSelectStrategy: Strategy?\n\n}\n"
  },
  {
    "path": "app/FastVLM/MediaProcessingExtensions.swift",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\nimport Accelerate\nimport CoreImage\nimport MLX\nimport MLXLMCommon\nimport MLXVLM\n\n/// Additions to MediaProcessing -- not currently present in mlx-libraries\nenum MediaProcessingExtensions {\n\n    // this function is not exported in current mlx-swift-examples -- local copy until it is exposed\n    // properly\n    public static func apply(_ image: CIImage, processing: UserInput.Processing?) -> CIImage {\n        var image = image\n\n        if let resize = processing?.resize {\n            let scale = MediaProcessing.bestFitScale(image.extent.size, in: resize)\n            image = image.transformed(by: CGAffineTransform(scaleX: scale, y: scale))\n        }\n\n        return image\n    }\n\n    public static func rectSmallerOrEqual(_ extent: CGRect, size: CGSize) -> Bool {\n        return extent.width <= size.width && extent.height <= size.height\n    }\n\n    public static func centerCrop(_ extent: CGRect, size: CGSize) -> CGRect {\n        let targetWidth = min(extent.width, size.width)\n        let targetHeight = min(extent.height, size.height)\n\n        return CGRect(\n            x: (extent.maxX - targetWidth) / 2,\n            y: (extent.maxY - targetHeight) / 2,\n            width: targetWidth, height: targetHeight\n        )\n    }\n\n    public static func centerCrop(_ image: CIImage, size: CGSize) -> CIImage {\n        let extent = image.extent\n        if rectSmallerOrEqual(extent, size: size) {\n            return image\n        }\n\n        let crop = centerCrop(extent, size: size)\n        return\n            image\n            .cropped(to: crop)\n            .transformed(by: CGAffineTransform(translationX: -crop.minX, y: -crop.minY))\n    }\n\n    public static func fitIn(_ size: CGSize, shortestEdge: Int) -> CGSize {\n        let floatShortestEdge = CGFloat(shortestEdge)\n\n        let (short, long) =\n            size.width <= size.height ? (size.width, size.height) : (size.height, size.width)\n        let newShort = floatShortestEdge\n        let newLong = floatShortestEdge * long / short\n\n        return size.width <= size.height\n            ? CGSize(width: newShort, height: newLong) : CGSize(width: newLong, height: newShort)\n    }\n\n    public static func fitIn(_ size: CGSize, longestEdge: Int) -> CGSize {\n        let floatLongestEdge = CGFloat(longestEdge)\n\n        var (newShort, newLong) =\n            size.width <= size.height ? (size.width, size.height) : (size.height, size.width)\n\n        if newLong > floatLongestEdge {\n            newLong = floatLongestEdge\n            newShort = floatLongestEdge * newShort / newLong\n        }\n\n        return size.width <= size.height\n            ? CGSize(width: newShort, height: newLong) : CGSize(width: newLong, height: newShort)\n    }\n\n    // version of function from https://github.com/ml-explore/mlx-swift-examples/pull/222\n    public static func resampleBicubic(_ image: CIImage, to size: CGSize) -> CIImage {\n        // Create a bicubic scale filter\n\n        let yScale = size.height / image.extent.height\n        let xScale = size.width / image.extent.width\n\n        let filter = CIFilter.bicubicScaleTransform()\n        filter.inputImage = image\n        filter.scale = Float(yScale)\n        filter.aspectRatio = Float(xScale / yScale)\n        let scaledImage = filter.outputImage!\n\n        // Create a rect with the exact dimensions we want\n        let exactRect = CGRect(\n            x: 0,\n            y: 0,\n            width: size.width,\n            height: size.height\n        )\n        // Crop to ensure exact dimensions\n        return scaledImage.cropped(to: exactRect)\n    }\n\n    static let context = CIContext()\n\n    /// Convert the CIImage into a planar 3 channel MLXArray `[1, C, H, W]`.\n    ///\n    /// This physically moves the channels into a planar configuration -- this is\n    /// required for feeding into the CoreML model and is faster to use\n    /// dedicated functions than transforming into contiguous memory\n    /// on readout.\n    static public func asPlanarMLXArray(_ image: CIImage, colorSpace: CGColorSpace? = nil)\n        -> MLXArray\n    {\n        let size = image.extent.size\n        let w = Int(size.width.rounded())\n        let h = Int(size.height.rounded())\n\n        // probably not strictly necessary, but this is what happens in\n        // e.g. image_processing_siglip in transformers (float32)\n        let format = CIFormat.RGBAf\n        let componentsPerPixel = 4\n        let bytesPerComponent: Int = MemoryLayout<Float32>.size\n        let bytesPerPixel = componentsPerPixel * bytesPerComponent\n        let bytesPerRow = w * bytesPerPixel\n\n        var data = Data(count: w * h * bytesPerPixel)\n        var planarData = Data(count: 3 * w * h * bytesPerComponent)\n        data.withUnsafeMutableBytes { ptr in\n            context.render(\n                image, toBitmap: ptr.baseAddress!, rowBytes: bytesPerRow, bounds: image.extent,\n                format: format, colorSpace: colorSpace)\n            context.clearCaches()\n\n            let vh = vImagePixelCount(h)\n            let vw = vImagePixelCount(w)\n\n            // convert from RGBAf -> RGBf in place\n            let rgbBytesPerRow = w * 3 * bytesPerComponent\n            var rgbaSrc = vImage_Buffer(\n                data: ptr.baseAddress!, height: vh, width: vw, rowBytes: bytesPerRow)\n            var rgbDest = vImage_Buffer(\n                data: ptr.baseAddress!, height: vh, width: vw, rowBytes: rgbBytesPerRow)\n\n            vImageConvert_RGBAFFFFtoRGBFFF(&rgbaSrc, &rgbDest, vImage_Flags(kvImageNoFlags))\n\n            // and convert to planar data in a second buffer\n            planarData.withUnsafeMutableBytes { planarPtr in\n                let planeBytesPerRow = w * bytesPerComponent\n\n                var rDest = vImage_Buffer(\n                    data: planarPtr.baseAddress!.advanced(by: 0 * planeBytesPerRow * h), height: vh,\n                    width: vw, rowBytes: planeBytesPerRow)\n                var gDest = vImage_Buffer(\n                    data: planarPtr.baseAddress!.advanced(by: 1 * planeBytesPerRow * h), height: vh,\n                    width: vw, rowBytes: planeBytesPerRow)\n                var bDest = vImage_Buffer(\n                    data: planarPtr.baseAddress!.advanced(by: 2 * planeBytesPerRow * h), height: vh,\n                    width: vw, rowBytes: planeBytesPerRow)\n\n                vImageConvert_RGBFFFtoPlanarF(\n                    &rgbDest, &rDest, &gDest, &bDest, vImage_Flags(kvImageNoFlags))\n            }\n        }\n\n        return MLXArray(planarData, [1, 3, h, w], type: Float32.self)\n    }\n\n}\n"
  },
  {
    "path": "app/FastVLM App/Assets.xcassets/AccentColor.colorset/Contents.json",
    "content": "{\n  \"colors\" : [\n    {\n      \"idiom\" : \"universal\"\n    }\n  ],\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "app/FastVLM App/Assets.xcassets/AppIcon.appiconset/Contents.json",
    "content": "{\n  \"images\" : [\n    {\n      \"filename\" : \"FastVLM - 150 Blue - Light@2x.png\",\n      \"idiom\" : \"universal\",\n      \"platform\" : \"ios\",\n      \"size\" : \"1024x1024\"\n    },\n    {\n      \"appearances\" : [\n        {\n          \"appearance\" : \"luminosity\",\n          \"value\" : \"dark\"\n        }\n      ],\n      \"filename\" : \"FastVLM - 150 Blue - Dark@2x.png\",\n      \"idiom\" : \"universal\",\n      \"platform\" : \"ios\",\n      \"size\" : \"1024x1024\"\n    },\n    {\n      \"appearances\" : [\n        {\n          \"appearance\" : \"luminosity\",\n          \"value\" : \"tinted\"\n        }\n      ],\n      \"filename\" : \"FastVLM - 150 White - Tinted@2x.png\",\n      \"idiom\" : \"universal\",\n      \"platform\" : \"ios\",\n      \"size\" : \"1024x1024\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"1x\",\n      \"size\" : \"16x16\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"2x\",\n      \"size\" : \"16x16\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"1x\",\n      \"size\" : \"32x32\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"2x\",\n      \"size\" : \"32x32\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"1x\",\n      \"size\" : \"128x128\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"2x\",\n      \"size\" : \"128x128\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"1x\",\n      \"size\" : \"256x256\"\n    },\n    {\n      \"idiom\" : \"mac\",\n      \"scale\" : \"2x\",\n      \"size\" : \"256x256\"\n    },\n    {\n      \"filename\" : \"FastVLM - MacOS - Dark@1x.png\",\n      \"idiom\" : \"mac\",\n      \"scale\" : \"1x\",\n      \"size\" : \"512x512\"\n    },\n    {\n      \"filename\" : \"FastVLM - MacOS - Dark@2x.png\",\n      \"idiom\" : \"mac\",\n      \"scale\" : \"2x\",\n      \"size\" : \"512x512\"\n    }\n  ],\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "app/FastVLM App/Assets.xcassets/Contents.json",
    "content": "{\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "app/FastVLM App/ContentView.swift",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\nimport AVFoundation\nimport MLXLMCommon\nimport SwiftUI\nimport Video\n\n// support swift 6\nextension CVImageBuffer: @unchecked @retroactive Sendable {}\nextension CMSampleBuffer: @unchecked @retroactive Sendable {}\n\n// delay between frames -- controls the frame rate of the updates\nlet FRAME_DELAY = Duration.milliseconds(1)\n\nstruct ContentView: View {\n    @State private var camera = CameraController()\n    @State private var model = FastVLMModel()\n\n    /// stream of frames -> VideoFrameView, see distributeVideoFrames\n    @State private var framesToDisplay: AsyncStream<CVImageBuffer>?\n\n    @State private var prompt = \"Describe the image in English.\"\n    @State private var promptSuffix = \"Output should be brief, about 15 words or less.\"\n\n    @State private var isShowingInfo: Bool = false\n\n    @State private var selectedCameraType: CameraType = .continuous\n    @State private var isEditingPrompt: Bool = false\n\n    var toolbarItemPlacement: ToolbarItemPlacement {\n        var placement: ToolbarItemPlacement = .navigation\n        #if os(iOS)\n        placement = .topBarLeading\n        #endif\n        return placement\n    }\n    \n    var statusTextColor : Color {\n        return model.evaluationState == .processingPrompt ? .black : .white\n    }\n    \n    var statusBackgroundColor : Color {\n        switch model.evaluationState {\n        case .idle:\n            return .gray\n        case .generatingResponse:\n            return .green\n        case .processingPrompt:\n            return .yellow\n        }\n    }\n\n    var body: some View {\n        NavigationStack {\n            Form {\n                Section {\n                    VStack(alignment: .leading, spacing: 10.0) {\n                        Picker(\"Camera Type\", selection: $selectedCameraType) {\n                            ForEach(CameraType.allCases, id: \\.self) { cameraType in\n                                Text(cameraType.rawValue.capitalized).tag(cameraType)\n                            }\n                        }\n                        // Prevent macOS from adding a text label for the picker\n                        .labelsHidden()\n                        .pickerStyle(.segmented)\n                        .onChange(of: selectedCameraType) { _, _ in\n                            // Cancel any in-flight requests when switching modes\n                            model.cancel()\n                        }\n\n                        if let framesToDisplay {\n                            VideoFrameView(\n                                frames: framesToDisplay,\n                                cameraType: selectedCameraType,\n                                action: { frame in\n                                    processSingleFrame(frame)\n                                })\n                                // Because we're using the AVCaptureSession preset\n                                // `.vga640x480`, we can assume this aspect ratio\n                                .aspectRatio(4/3, contentMode: .fit)\n                                #if os(macOS)\n                                .frame(maxWidth: 750)\n                                #endif\n                                .overlay(alignment: .top) {\n                                    if !model.promptTime.isEmpty {\n                                        Text(\"TTFT \\(model.promptTime)\")\n                                            .font(.caption)\n                                            .foregroundStyle(.white)\n                                            .monospaced()\n                                            .padding(.vertical, 4.0)\n                                            .padding(.horizontal, 6.0)\n                                            .background(alignment: .center) {\n                                                RoundedRectangle(cornerRadius: 8)\n                                                    .fill(Color.black.opacity(0.6))\n                                            }\n                                            .padding(.top)\n                                    }\n                                }\n                                #if !os(macOS)\n                                .overlay(alignment: .topTrailing) {\n                                    CameraControlsView(\n                                        backCamera: $camera.backCamera,\n                                        device: $camera.device,\n                                        devices: $camera.devices)\n                                    .padding()\n                                }\n                                #endif\n                                .overlay(alignment: .bottom) {\n                                    if selectedCameraType == .continuous {\n                                        Group {\n                                            if model.evaluationState == .processingPrompt {\n                                                HStack {\n                                                    ProgressView()\n                                                        .tint(self.statusTextColor)\n                                                        .controlSize(.small)\n\n                                                    Text(model.evaluationState.rawValue)\n                                                }\n                                            } else if model.evaluationState == .idle {\n                                                HStack(spacing: 6.0) {\n                                                    Image(systemName: \"clock.fill\")\n                                                        .font(.caption)\n\n                                                    Text(model.evaluationState.rawValue)\n                                                }\n                                            }\n                                            else {\n                                                // I'm manually tweaking the spacing to\n                                                // better match the spacing with ProgressView\n                                                HStack(spacing: 6.0) {\n                                                    Image(systemName: \"lightbulb.fill\")\n                                                        .font(.caption)\n\n                                                    Text(model.evaluationState.rawValue)\n                                                }\n                                            }\n                                        }\n                                        .foregroundStyle(self.statusTextColor)\n                                        .font(.caption)\n                                        .bold()\n                                        .padding(.vertical, 6.0)\n                                        .padding(.horizontal, 8.0)\n                                        .background(self.statusBackgroundColor)\n                                        .clipShape(.capsule)\n                                        .padding(.bottom)\n                                    }\n                                }\n                                #if os(macOS)\n                                .frame(maxWidth: .infinity)\n                                .frame(minWidth: 500)\n                                .frame(minHeight: 375)\n                                #endif\n                        }\n                    }\n                }\n                .listRowInsets(EdgeInsets())\n                .listRowBackground(Color.clear)\n                .listRowSeparator(.hidden)\n\n                promptSections\n\n                Section {\n                    if model.output.isEmpty && model.running {\n                        ProgressView()\n                            .controlSize(.large)\n                            .frame(maxWidth: .infinity)\n                    } else {\n                        ScrollView {\n                            Text(model.output)\n                                .foregroundStyle(isEditingPrompt ? .secondary : .primary)\n                                .textSelection(.enabled)\n                                #if os(macOS)\n                                .font(.headline)\n                                .fontWeight(.regular)\n                                #endif\n                        }\n                        .frame(minHeight: 50.0, maxHeight: 200.0)\n                    }\n                } header: {\n                    Text(\"Response\")\n                        #if os(macOS)\n                        .font(.headline)\n                        .padding(.bottom, 2.0)\n                        #endif\n                }\n\n                #if os(macOS)\n                Spacer()\n                #endif\n            }\n            \n            #if os(iOS)\n            .listSectionSpacing(0)\n            #elseif os(macOS)\n            .padding()\n            #endif\n            .task {\n                camera.start()\n            }\n            .task {\n                await model.load()\n            }\n\n            #if !os(macOS)\n            .onAppear {\n                // Prevent the screen from dimming or sleeping due to inactivity\n                UIApplication.shared.isIdleTimerDisabled = true\n            }\n            .onDisappear {\n                // Resumes normal idle timer behavior\n                UIApplication.shared.isIdleTimerDisabled = false\n            }\n            #endif\n\n            // task to distribute video frames -- this will cancel\n            // and restart when the view is on/off screen.  note: it is\n            // important that this is here (attached to the VideoFrameView)\n            // rather than the outer view because this has the correct lifecycle\n            .task {\n                if Task.isCancelled {\n                    return\n                }\n\n                await distributeVideoFrames()\n            }\n\n            .navigationTitle(\"FastVLM\")\n            #if os(iOS)\n            .navigationBarTitleDisplayMode(.inline)\n            #endif\n            .toolbar {\n                ToolbarItem(placement: toolbarItemPlacement) {\n                    Button {\n                        isShowingInfo.toggle()\n                    }\n                    label: {\n                        Image(systemName: \"info.circle\")\n                    }\n                }\n\n                ToolbarItem(placement: .primaryAction) {\n                    if isEditingPrompt {\n                        Button {\n                            isEditingPrompt.toggle()\n                        }\n                        label: {\n                            Text(\"Done\")\n                                .fontWeight(.bold)\n                        }\n                    }\n                    else {\n                        Menu {\n                            Button(\"Describe image\") {\n                                prompt = \"Describe the image in English.\"\n                                promptSuffix = \"Output should be brief, about 15 words or less.\"\n                            }\n                            Button(\"Facial expression\") {\n                                prompt = \"What is this person's facial expression?\"\n                                promptSuffix = \"Output only one or two words.\"\n                            }\n                            Button(\"Read text\") {\n                                prompt = \"What is written in this image?\"\n                                promptSuffix = \"Output only the text in the image.\"\n                            }\n                            #if !os(macOS)\n                            Button(\"Customize...\") {\n                                isEditingPrompt.toggle()\n                            }\n                            #endif\n                        } label: { Text(\"Prompts\") }\n                    }\n                }\n            }\n            .sheet(isPresented: $isShowingInfo) {\n                InfoView()\n            }\n        }\n    }\n\n    var promptSummary: some View {\n        Section(\"Prompt\") {\n            VStack(alignment: .leading, spacing: 4.0) {\n                let trimmedPrompt = prompt.trimmingCharacters(in: .whitespacesAndNewlines)\n                if !trimmedPrompt.isEmpty {\n                    Text(trimmedPrompt)\n                        .foregroundStyle(.secondary)\n                }\n\n                let trimmedSuffix = promptSuffix.trimmingCharacters(in: .whitespacesAndNewlines)\n                if !trimmedSuffix.isEmpty {\n                    Text(trimmedSuffix)\n                        .font(.caption)\n                        .foregroundStyle(.tertiary)\n                }\n            }\n        }\n    }\n\n    var promptForm: some View {\n        Group {\n            #if os(iOS)\n            Section(\"Prompt\") {\n                TextEditor(text: $prompt)\n                    .frame(minHeight: 38)\n            }\n\n            Section(\"Prompt Suffix\") {\n                TextEditor(text: $promptSuffix)\n                    .frame(minHeight: 38)\n            }\n            #elseif os(macOS)\n            Section {\n                HStack(alignment: .top) {\n                    VStack(alignment: .leading) {\n                        Text(\"Prompt\")\n                            .font(.headline)\n\n                        TextEditor(text: $prompt)\n                            .frame(height: 38)\n                            .padding(.horizontal, 8.0)\n                            .padding(.vertical, 10.0)\n                            .background(Color(.textBackgroundColor))\n                            .cornerRadius(10.0)\n                    }\n\n                    VStack(alignment: .leading) {\n                        Text(\"Prompt Suffix\")\n                            .font(.headline)\n\n                        TextEditor(text: $promptSuffix)\n                            .frame(height: 38)\n                            .padding(.horizontal, 8.0)\n                            .padding(.vertical, 10.0)\n                            .background(Color(.textBackgroundColor))\n                            .cornerRadius(10.0)\n                    }\n                }\n            }\n            .padding(.vertical)\n            #endif\n        }\n    }\n\n    var promptSections: some View {\n        Group {\n            #if os(iOS)\n            if isEditingPrompt {\n                promptForm\n            }\n            else {\n                promptSummary\n            }\n            #elseif os(macOS)\n            promptForm\n            #endif\n        }\n    }\n\n    func analyzeVideoFrames(_ frames: AsyncStream<CVImageBuffer>) async {\n        for await frame in frames {\n            let userInput = UserInput(\n                prompt: .text(\"\\(prompt) \\(promptSuffix)\"),\n                images: [.ciImage(CIImage(cvPixelBuffer: frame))]\n            )\n            \n            // generate output for a frame and wait for generation to complete\n            let t = await model.generate(userInput)\n            _ = await t.result\n\n            do {\n                try await Task.sleep(for: FRAME_DELAY)\n            } catch { return }\n        }\n    }\n\n    func distributeVideoFrames() async {\n        // attach a stream to the camera -- this code will read this\n        let frames = AsyncStream<CMSampleBuffer>(bufferingPolicy: .bufferingNewest(1)) {\n            camera.attach(continuation: $0)\n        }\n\n        let (framesToDisplay, framesToDisplayContinuation) = AsyncStream.makeStream(\n            of: CVImageBuffer.self,\n            bufferingPolicy: .bufferingNewest(1)\n        )\n        self.framesToDisplay = framesToDisplay\n\n        // Only create analysis stream if in continuous mode\n        let (framesToAnalyze, framesToAnalyzeContinuation) = AsyncStream.makeStream(\n            of: CVImageBuffer.self,\n            bufferingPolicy: .bufferingNewest(1)\n        )\n\n        // set up structured tasks (important -- this means the child tasks\n        // are cancelled when the parent is cancelled)\n        async let distributeFrames: () = {\n            for await sampleBuffer in frames {\n                if let frame = sampleBuffer.imageBuffer {\n                    framesToDisplayContinuation.yield(frame)\n                    // Only send frames for analysis in continuous mode\n                    if await selectedCameraType == .continuous {\n                        framesToAnalyzeContinuation.yield(frame)\n                    }\n                }\n            }\n\n            // detach from the camera controller and feed to the video view\n            await MainActor.run {\n                self.framesToDisplay = nil\n                self.camera.detatch()\n            }\n\n            framesToDisplayContinuation.finish()\n            framesToAnalyzeContinuation.finish()\n        }()\n\n        // Only analyze frames if in continuous mode\n        if selectedCameraType == .continuous {\n            async let analyze: () = analyzeVideoFrames(framesToAnalyze)\n            await distributeFrames\n            await analyze\n        } else {\n            await distributeFrames\n        }\n    }\n\n    /// Perform FastVLM inference on a single frame.\n    /// - Parameter frame: The frame to analyze.\n    func processSingleFrame(_ frame: CVImageBuffer) {\n        // Reset Response UI (show spinner)\n        Task { @MainActor in\n            model.output = \"\"\n        }\n\n        // Construct request to model\n        let userInput = UserInput(\n            prompt: .text(\"\\(prompt) \\(promptSuffix)\"),\n            images: [.ciImage(CIImage(cvPixelBuffer: frame))]\n        )\n\n        // Post request to FastVLM\n        Task {\n            await model.generate(userInput)\n        }\n    }\n}\n\n#Preview {\n    ContentView()\n}\n"
  },
  {
    "path": "app/FastVLM App/FastVLM.entitlements",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n<plist version=\"1.0\">\n<dict>\n\t<key>com.apple.developer.kernel.increased-memory-limit</key>\n\t<true/>\n\t<key>com.apple.security.app-sandbox</key>\n\t<true/>\n\t<key>com.apple.security.device.camera</key>\n\t<true/>\n\t<key>com.apple.security.files.user-selected.read-only</key>\n\t<true/>\n\t<key>com.apple.security.network.client</key>\n\t<true/>\n</dict>\n</plist>\n"
  },
  {
    "path": "app/FastVLM App/FastVLMApp.swift",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\nimport SwiftUI\n\n@main\nstruct FastVLMApp: App {\n    var body: some Scene {\n        WindowGroup {\n            ContentView()\n        }\n    }\n}\n"
  },
  {
    "path": "app/FastVLM App/FastVLMModel.swift",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\nimport CoreImage\nimport FastVLM\nimport Foundation\nimport MLX\nimport MLXLMCommon\nimport MLXRandom\nimport MLXVLM\n\n@Observable\n@MainActor\nclass FastVLMModel {\n\n    public var running = false\n    public var modelInfo = \"\"\n    public var output = \"\"\n    public var promptTime: String = \"\"\n\n    enum LoadState {\n        case idle\n        case loaded(ModelContainer)\n    }\n\n    private let modelConfiguration = FastVLM.modelConfiguration\n\n    /// parameters controlling the output\n    let generateParameters = GenerateParameters(temperature: 0.0)\n    let maxTokens = 240\n\n    /// update the display every N tokens -- 4 looks like it updates continuously\n    /// and is low overhead.  observed ~15% reduction in tokens/s when updating\n    /// on every token\n    let displayEveryNTokens = 4\n\n    private var loadState = LoadState.idle\n    private var currentTask: Task<Void, Never>?\n\n    enum EvaluationState: String, CaseIterable {\n        case idle = \"Idle\"\n        case processingPrompt = \"Processing Prompt\"\n        case generatingResponse = \"Generating Response\"\n    }\n\n    public var evaluationState = EvaluationState.idle\n\n    public init() {\n        FastVLM.register(modelFactory: VLMModelFactory.shared)\n    }\n\n    private func _load() async throws -> ModelContainer {\n        switch loadState {\n        case .idle:\n            // limit the buffer cache\n            MLX.GPU.set(cacheLimit: 20 * 1024 * 1024)\n\n            let modelContainer = try await VLMModelFactory.shared.loadContainer(\n                configuration: modelConfiguration\n            ) {\n                [modelConfiguration] progress in\n                Task { @MainActor in\n                    self.modelInfo =\n                        \"Downloading \\(modelConfiguration.name): \\(Int(progress.fractionCompleted * 100))%\"\n                }\n            }\n            self.modelInfo = \"Loaded\"\n            loadState = .loaded(modelContainer)\n            return modelContainer\n\n        case .loaded(let modelContainer):\n            return modelContainer\n        }\n    }\n\n    public func load() async {\n        do {\n            _ = try await _load()\n        } catch {\n            self.modelInfo = \"Error loading model: \\(error)\"\n        }\n    }\n\n    public func generate(_ userInput: UserInput) async -> Task<Void, Never> {\n        if let currentTask, running {\n            return currentTask\n        }\n\n        running = true\n        \n        // Cancel any existing task\n        currentTask?.cancel()\n\n        // Create new task and store reference\n        let task = Task {\n            do {\n                let modelContainer = try await _load()\n\n                // each time you generate you will get something new\n                MLXRandom.seed(UInt64(Date.timeIntervalSinceReferenceDate * 1000))\n                \n                // Check if task was cancelled\n                if Task.isCancelled { return }\n\n                let result = try await modelContainer.perform { context in\n                    // Measure the time it takes to prepare the input\n                    \n                    Task { @MainActor in\n                        evaluationState = .processingPrompt\n                    }\n\n                    let llmStart = Date()\n                    let input = try await context.processor.prepare(input: userInput)\n                    \n                    var seenFirstToken = false\n\n                    // FastVLM generates the output\n                    let result = try MLXLMCommon.generate(\n                        input: input, parameters: generateParameters, context: context\n                    ) { tokens in\n                        // Check if task was cancelled\n                        if Task.isCancelled {\n                            return .stop\n                        }\n\n                        if !seenFirstToken {\n                            seenFirstToken = true\n                            \n                            // produced first token, update the time to first token,\n                            // the processing state and start displaying the text\n                            let llmDuration = Date().timeIntervalSince(llmStart)\n                            let text = context.tokenizer.decode(tokens: tokens)\n                            Task { @MainActor in\n                                evaluationState = .generatingResponse\n                                self.output = text\n                                self.promptTime = \"\\(Int(llmDuration * 1000)) ms\"\n                            }\n                        }\n\n                        // Show the text in the view as it generates\n                        if tokens.count % displayEveryNTokens == 0 {\n                            let text = context.tokenizer.decode(tokens: tokens)\n                            Task { @MainActor in\n                                self.output = text\n                            }\n                        }\n\n                        if tokens.count >= maxTokens {\n                            return .stop\n                        } else {\n                            return .more\n                        }\n                    }\n                    \n                    // Return the duration of the LLM and the result\n                    return result\n                }\n                \n                // Check if task was cancelled before updating UI\n                if !Task.isCancelled {\n                    self.output = result.output\n                }\n\n            } catch {\n                if !Task.isCancelled {\n                    output = \"Failed: \\(error)\"\n                }\n            }\n\n            if evaluationState == .generatingResponse {\n                evaluationState = .idle\n            }\n\n            running = false\n        }\n        \n        currentTask = task\n        return task\n    }\n    \n    public func cancel() {\n        currentTask?.cancel()\n        currentTask = nil\n        running = false\n        output = \"\"\n        promptTime = \"\"\n    }\n}\n"
  },
  {
    "path": "app/FastVLM App/Info.plist",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n<plist version=\"1.0\">\n<dict/>\n</plist>\n"
  },
  {
    "path": "app/FastVLM App/InfoView.swift",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\nimport Foundation\nimport SwiftUI\n\nstruct InfoView: View {\n    @Environment(\\.dismiss) var dismiss\n\n    let paragraph1 = \"**FastVLM¹** is a new family of Vision-Language models that makes use of **FastViTHD**, a hierarchical hybrid vision encoder that produces small number of high quality tokens at low latencies, resulting in significantly faster time-to-first-token (TTFT).\"\n    let paragraph2 = \"This app showcases the **FastVLM** model in action, allowing users to freely customize the prompt. FastVLM utilizes Qwen2-Instruct LLMs without additional safety tuning, so please exercise caution when modifying the prompt.\"\n    let footer = \"1. **FastVLM: Efficient Vision Encoding for Vision Language Models.** (CVPR 2025) Pavan Kumar Anasosalu Vasu, Fartash Faghri, Chun-Liang Li, Cem Koc, Nate True, Albert Antony, Gokul Santhanam, James Gabriel, Peter Grasch, Oncel Tuzel, Hadi Pouransari\"\n\n    var body: some View {\n        NavigationStack {\n            VStack(alignment: .leading, spacing: 20.0) {\n                // I'm not going to lie, this doesn't make sense...\n                // Wrapping `String`s with `.init()` turns them into `LocalizedStringKey`s\n                // which gives us all of the fun Markdown formatting while retaining the\n                // ability to use `String` variables. ¯\\_(ツ)_/¯\n                Text(\"\\(.init(paragraph1))\\n\\n\\(.init(paragraph2))\\n\\n\")\n                    .font(.body)\n\n                Spacer()\n\n                Text(.init(footer))\n                    .font(.caption)\n                    .foregroundStyle(.secondary)\n            }\n            .padding()\n            .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top)\n            .textSelection(.enabled)\n            .navigationTitle(\"Information\")\n            #if os(iOS)\n            .navigationBarTitleDisplayMode(.inline)\n            #endif\n            .toolbar {\n                #if os(iOS)\n                ToolbarItem(placement: .navigationBarLeading) {\n                    Button {\n                        dismiss()\n                    } label: {\n                        Image(systemName: \"xmark.circle\")\n                            .resizable()\n                            .frame(width: 25, height: 25)\n                            .foregroundStyle(.secondary)\n                    }\n                    .buttonStyle(.plain)\n                }\n                #elseif os(macOS)\n                ToolbarItem(placement: .cancellationAction) {\n                    Button(\"Done\") {\n                        dismiss()\n                    }\n                    .buttonStyle(.bordered)\n                }\n                #endif\n            }\n        }\n    }\n}\n\n#Preview {\n    InfoView()\n}\n"
  },
  {
    "path": "app/FastVLM App/Preview Content/Preview Assets.xcassets/Contents.json",
    "content": "{\n  \"info\" : {\n    \"author\" : \"xcode\",\n    \"version\" : 1\n  }\n}\n"
  },
  {
    "path": "app/FastVLM.xcodeproj/project.pbxproj",
    "content": "// !$*UTF8*$!\n{\n\tarchiveVersion = 1;\n\tclasses = {\n\t};\n\tobjectVersion = 77;\n\tobjects = {\n\n/* Begin PBXBuildFile section */\n\t\t019A3E1A2D78E7370055F93B /* MLX in Frameworks */ = {isa = PBXBuildFile; productRef = 019A3E192D78E7370055F93B /* MLX */; };\n\t\t019A3E1C2D78E73E0055F93B /* MLXLMCommon in Frameworks */ = {isa = PBXBuildFile; productRef = 019A3E1B2D78E73E0055F93B /* MLXLMCommon */; };\n\t\t019A3E1E2D78E7470055F93B /* MLXRandom in Frameworks */ = {isa = PBXBuildFile; productRef = 019A3E1D2D78E7470055F93B /* MLXRandom */; };\n\t\t019A3E202D78E74C0055F93B /* MLXVLM in Frameworks */ = {isa = PBXBuildFile; productRef = 019A3E1F2D78E74C0055F93B /* MLXVLM */; };\n\t\t019A3E212D78E7530055F93B /* Video.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C35372AE2D08C32D00474D34 /* Video.framework */; };\n\t\t019A3E222D78E7530055F93B /* Video.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = C35372AE2D08C32D00474D34 /* Video.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };\n\t\tC3ED544E2D790860005E20B3 /* MLXLMCommon in Frameworks */ = {isa = PBXBuildFile; productRef = C3ED544D2D790860005E20B3 /* MLXLMCommon */; };\n\t\tC3ED54502D790860005E20B3 /* MLXVLM in Frameworks */ = {isa = PBXBuildFile; productRef = C3ED544F2D790860005E20B3 /* MLXVLM */; };\n\t\tC3ED54522D790860005E20B3 /* MLX in Frameworks */ = {isa = PBXBuildFile; productRef = C3ED54512D790860005E20B3 /* MLX */; };\n\t\tC3ED54542D790860005E20B3 /* MLXNN in Frameworks */ = {isa = PBXBuildFile; productRef = C3ED54532D790860005E20B3 /* MLXNN */; };\n\t\tC3ED54582D790A68005E20B3 /* MLXFast in Frameworks */ = {isa = PBXBuildFile; productRef = C3ED54572D790A68005E20B3 /* MLXFast */; };\n\t\tC3ED545B2D790AD6005E20B3 /* Transformers in Frameworks */ = {isa = PBXBuildFile; productRef = C3ED545A2D790AD6005E20B3 /* Transformers */; };\n\t\tC3ED55012D7A0A7A005E20B3 /* FastVLM.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C39BB3E62D79082A005DB8FB /* FastVLM.framework */; };\n\t\tC3ED55022D7A0A7A005E20B3 /* FastVLM.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = C39BB3E62D79082A005DB8FB /* FastVLM.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };\n/* End PBXBuildFile section */\n\n/* Begin PBXContainerItemProxy section */\n\t\t019A3E232D78E7530055F93B /* PBXContainerItemProxy */ = {\n\t\t\tisa = PBXContainerItemProxy;\n\t\t\tcontainerPortal = C35EDB642D07699400757E80 /* Project object */;\n\t\t\tproxyType = 1;\n\t\t\tremoteGlobalIDString = C35372AD2D08C32D00474D34;\n\t\t\tremoteInfo = Video;\n\t\t};\n\t\tC3ED55032D7A0A7A005E20B3 /* PBXContainerItemProxy */ = {\n\t\t\tisa = PBXContainerItemProxy;\n\t\t\tcontainerPortal = C35EDB642D07699400757E80 /* Project object */;\n\t\t\tproxyType = 1;\n\t\t\tremoteGlobalIDString = C39BB3E52D79082A005DB8FB;\n\t\t\tremoteInfo = FastVLM;\n\t\t};\n/* End PBXContainerItemProxy section */\n\n/* Begin PBXCopyFilesBuildPhase section */\n\t\t019A3E252D78E7530055F93B /* Embed Frameworks */ = {\n\t\t\tisa = PBXCopyFilesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tdstPath = \"\";\n\t\t\tdstSubfolderSpec = 10;\n\t\t\tfiles = (\n\t\t\t\tC3ED55022D7A0A7A005E20B3 /* FastVLM.framework in Embed Frameworks */,\n\t\t\t\t019A3E222D78E7530055F93B /* Video.framework in Embed Frameworks */,\n\t\t\t);\n\t\t\tname = \"Embed Frameworks\";\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXCopyFilesBuildPhase section */\n\n/* Begin PBXFileReference section */\n\t\t019A3E0A2D78E6A00055F93B /* FastVLM App.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = \"FastVLM App.app\"; sourceTree = BUILT_PRODUCTS_DIR; };\n\t\t12FFAF3D2DC93583009C4EFA /* get_pretrained_mlx_model.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = get_pretrained_mlx_model.sh; sourceTree = \"<group>\"; };\n\t\t12FFAF3E2DC93583009C4EFA /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = \"<group>\"; };\n\t\tC35372AE2D08C32D00474D34 /* Video.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Video.framework; sourceTree = BUILT_PRODUCTS_DIR; };\n\t\tC39BB3E62D79082A005DB8FB /* FastVLM.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = FastVLM.framework; sourceTree = BUILT_PRODUCTS_DIR; };\n/* End PBXFileReference section */\n\n/* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */\n\t\t120A44852D9B05A900E244A3 /* Exceptions for \"FastVLM App\" folder in \"FastVLM App\" target */ = {\n\t\t\tisa = PBXFileSystemSynchronizedBuildFileExceptionSet;\n\t\t\tmembershipExceptions = (\n\t\t\t\tInfo.plist,\n\t\t\t);\n\t\t\ttarget = 019A3E092D78E6A00055F93B /* FastVLM App */;\n\t\t};\n\t\tC35372B92D08C32D00474D34 /* Exceptions for \"Video\" folder in \"Video\" target */ = {\n\t\t\tisa = PBXFileSystemSynchronizedBuildFileExceptionSet;\n\t\t\tpublicHeaders = (\n\t\t\t\tVideo.h,\n\t\t\t);\n\t\t\ttarget = C35372AD2D08C32D00474D34 /* Video */;\n\t\t};\n\t\tC3ED54BB2D791BEA005E20B3 /* Exceptions for \"FastVLM\" folder in \"FastVLM\" target */ = {\n\t\t\tisa = PBXFileSystemSynchronizedBuildFileExceptionSet;\n\t\t\tpublicHeaders = (\n\t\t\t\tFastVLM.h,\n\t\t\t);\n\t\t\ttarget = C39BB3E52D79082A005DB8FB /* FastVLM */;\n\t\t};\n/* End PBXFileSystemSynchronizedBuildFileExceptionSet section */\n\n/* Begin PBXFileSystemSynchronizedRootGroup section */\n\t\t019A3E0B2D78E6A00055F93B /* FastVLM App */ = {\n\t\t\tisa = PBXFileSystemSynchronizedRootGroup;\n\t\t\texceptions = (\n\t\t\t\t120A44852D9B05A900E244A3 /* Exceptions for \"FastVLM App\" folder in \"FastVLM App\" target */,\n\t\t\t);\n\t\t\tpath = \"FastVLM App\";\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\tC32B4A802DA4805400EF663D /* Configuration */ = {\n\t\t\tisa = PBXFileSystemSynchronizedRootGroup;\n\t\t\tpath = Configuration;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\tC35372AF2D08C32D00474D34 /* Video */ = {\n\t\t\tisa = PBXFileSystemSynchronizedRootGroup;\n\t\t\texceptions = (\n\t\t\t\tC35372B92D08C32D00474D34 /* Exceptions for \"Video\" folder in \"Video\" target */,\n\t\t\t);\n\t\t\tpath = Video;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\tC39BB3E72D79082A005DB8FB /* FastVLM */ = {\n\t\t\tisa = PBXFileSystemSynchronizedRootGroup;\n\t\t\texceptions = (\n\t\t\t\tC3ED54BB2D791BEA005E20B3 /* Exceptions for \"FastVLM\" folder in \"FastVLM\" target */,\n\t\t\t);\n\t\t\tpath = FastVLM;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n/* End PBXFileSystemSynchronizedRootGroup section */\n\n/* Begin PBXFrameworksBuildPhase section */\n\t\t019A3E072D78E6A00055F93B /* Frameworks */ = {\n\t\t\tisa = PBXFrameworksBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\t019A3E1C2D78E73E0055F93B /* MLXLMCommon in Frameworks */,\n\t\t\t\t019A3E212D78E7530055F93B /* Video.framework in Frameworks */,\n\t\t\t\tC3ED55012D7A0A7A005E20B3 /* FastVLM.framework in Frameworks */,\n\t\t\t\t019A3E1E2D78E7470055F93B /* MLXRandom in Frameworks */,\n\t\t\t\t019A3E202D78E74C0055F93B /* MLXVLM in Frameworks */,\n\t\t\t\t019A3E1A2D78E7370055F93B /* MLX in Frameworks */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\tC35372AB2D08C32D00474D34 /* Frameworks */ = {\n\t\t\tisa = PBXFrameworksBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\tC39BB3E32D79082A005DB8FB /* Frameworks */ = {\n\t\t\tisa = PBXFrameworksBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\tC3ED545B2D790AD6005E20B3 /* Transformers in Frameworks */,\n\t\t\t\tC3ED54522D790860005E20B3 /* MLX in Frameworks */,\n\t\t\t\tC3ED54502D790860005E20B3 /* MLXVLM in Frameworks */,\n\t\t\t\tC3ED54542D790860005E20B3 /* MLXNN in Frameworks */,\n\t\t\t\tC3ED54582D790A68005E20B3 /* MLXFast in Frameworks */,\n\t\t\t\tC3ED544E2D790860005E20B3 /* MLXLMCommon in Frameworks */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXFrameworksBuildPhase section */\n\n/* Begin PBXGroup section */\n\t\tC35EDB632D07699400757E80 = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t12FFAF3E2DC93583009C4EFA /* README.md */,\n\t\t\t\t12FFAF3D2DC93583009C4EFA /* get_pretrained_mlx_model.sh */,\n\t\t\t\tC32B4A802DA4805400EF663D /* Configuration */,\n\t\t\t\tC35372AF2D08C32D00474D34 /* Video */,\n\t\t\t\t019A3E0B2D78E6A00055F93B /* FastVLM App */,\n\t\t\t\tC39BB3E72D79082A005DB8FB /* FastVLM */,\n\t\t\t\tC35EDB7F2D076C3C00757E80 /* Frameworks */,\n\t\t\t\tC35EDB702D076A5A00757E80 /* Products */,\n\t\t\t);\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\tC35EDB702D076A5A00757E80 /* Products */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\tC35372AE2D08C32D00474D34 /* Video.framework */,\n\t\t\t\t019A3E0A2D78E6A00055F93B /* FastVLM App.app */,\n\t\t\t\tC39BB3E62D79082A005DB8FB /* FastVLM.framework */,\n\t\t\t);\n\t\t\tname = Products;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n\t\tC35EDB7F2D076C3C00757E80 /* Frameworks */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t);\n\t\t\tname = Frameworks;\n\t\t\tsourceTree = \"<group>\";\n\t\t};\n/* End PBXGroup section */\n\n/* Begin PBXHeadersBuildPhase section */\n\t\tC35372A92D08C32D00474D34 /* Headers */ = {\n\t\t\tisa = PBXHeadersBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\tC39BB3E12D79082A005DB8FB /* Headers */ = {\n\t\t\tisa = PBXHeadersBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXHeadersBuildPhase section */\n\n/* Begin PBXNativeTarget section */\n\t\t019A3E092D78E6A00055F93B /* FastVLM App */ = {\n\t\t\tisa = PBXNativeTarget;\n\t\t\tbuildConfigurationList = 019A3E182D78E6A20055F93B /* Build configuration list for PBXNativeTarget \"FastVLM App\" */;\n\t\t\tbuildPhases = (\n\t\t\t\t019A3E062D78E6A00055F93B /* Sources */,\n\t\t\t\t019A3E072D78E6A00055F93B /* Frameworks */,\n\t\t\t\t019A3E082D78E6A00055F93B /* Resources */,\n\t\t\t\t019A3E252D78E7530055F93B /* Embed Frameworks */,\n\t\t\t);\n\t\t\tbuildRules = (\n\t\t\t);\n\t\t\tdependencies = (\n\t\t\t\t019A3E242D78E7530055F93B /* PBXTargetDependency */,\n\t\t\t\tC3ED55042D7A0A7A005E20B3 /* PBXTargetDependency */,\n\t\t\t);\n\t\t\tfileSystemSynchronizedGroups = (\n\t\t\t\t019A3E0B2D78E6A00055F93B /* FastVLM App */,\n\t\t\t);\n\t\t\tname = \"FastVLM App\";\n\t\t\tpackageProductDependencies = (\n\t\t\t\t019A3E192D78E7370055F93B /* MLX */,\n\t\t\t\t019A3E1B2D78E73E0055F93B /* MLXLMCommon */,\n\t\t\t\t019A3E1D2D78E7470055F93B /* MLXRandom */,\n\t\t\t\t019A3E1F2D78E74C0055F93B /* MLXVLM */,\n\t\t\t);\n\t\t\tproductName = FastVLMCameraExample;\n\t\t\tproductReference = 019A3E0A2D78E6A00055F93B /* FastVLM App.app */;\n\t\t\tproductType = \"com.apple.product-type.application\";\n\t\t};\n\t\tC35372AD2D08C32D00474D34 /* Video */ = {\n\t\t\tisa = PBXNativeTarget;\n\t\t\tbuildConfigurationList = C35372BA2D08C32D00474D34 /* Build configuration list for PBXNativeTarget \"Video\" */;\n\t\t\tbuildPhases = (\n\t\t\t\tC35372A92D08C32D00474D34 /* Headers */,\n\t\t\t\tC35372AA2D08C32D00474D34 /* Sources */,\n\t\t\t\tC35372AB2D08C32D00474D34 /* Frameworks */,\n\t\t\t\tC35372AC2D08C32D00474D34 /* Resources */,\n\t\t\t);\n\t\t\tbuildRules = (\n\t\t\t);\n\t\t\tdependencies = (\n\t\t\t);\n\t\t\tfileSystemSynchronizedGroups = (\n\t\t\t\tC35372AF2D08C32D00474D34 /* Video */,\n\t\t\t);\n\t\t\tname = Video;\n\t\t\tpackageProductDependencies = (\n\t\t\t);\n\t\t\tproductName = Video;\n\t\t\tproductReference = C35372AE2D08C32D00474D34 /* Video.framework */;\n\t\t\tproductType = \"com.apple.product-type.framework\";\n\t\t};\n\t\tC39BB3E52D79082A005DB8FB /* FastVLM */ = {\n\t\t\tisa = PBXNativeTarget;\n\t\t\tbuildConfigurationList = C39BB3FF2D79082A005DB8FB /* Build configuration list for PBXNativeTarget \"FastVLM\" */;\n\t\t\tbuildPhases = (\n\t\t\t\tC39BB3E12D79082A005DB8FB /* Headers */,\n\t\t\t\tC39BB3E22D79082A005DB8FB /* Sources */,\n\t\t\t\tC39BB3E32D79082A005DB8FB /* Frameworks */,\n\t\t\t\tC39BB3E42D79082A005DB8FB /* Resources */,\n\t\t\t);\n\t\t\tbuildRules = (\n\t\t\t);\n\t\t\tdependencies = (\n\t\t\t);\n\t\t\tfileSystemSynchronizedGroups = (\n\t\t\t\tC39BB3E72D79082A005DB8FB /* FastVLM */,\n\t\t\t);\n\t\t\tname = FastVLM;\n\t\t\tpackageProductDependencies = (\n\t\t\t\tC3ED544D2D790860005E20B3 /* MLXLMCommon */,\n\t\t\t\tC3ED544F2D790860005E20B3 /* MLXVLM */,\n\t\t\t\tC3ED54512D790860005E20B3 /* MLX */,\n\t\t\t\tC3ED54532D790860005E20B3 /* MLXNN */,\n\t\t\t\tC3ED54572D790A68005E20B3 /* MLXFast */,\n\t\t\t\tC3ED545A2D790AD6005E20B3 /* Transformers */,\n\t\t\t);\n\t\t\tproductName = FastVLM;\n\t\t\tproductReference = C39BB3E62D79082A005DB8FB /* FastVLM.framework */;\n\t\t\tproductType = \"com.apple.product-type.framework\";\n\t\t};\n/* End PBXNativeTarget section */\n\n/* Begin PBXProject section */\n\t\tC35EDB642D07699400757E80 /* Project object */ = {\n\t\t\tisa = PBXProject;\n\t\t\tattributes = {\n\t\t\t\tBuildIndependentTargetsInParallel = 1;\n\t\t\t\tLastSwiftUpdateCheck = 1620;\n\t\t\t\tLastUpgradeCheck = 1630;\n\t\t\t\tTargetAttributes = {\n\t\t\t\t\t019A3E092D78E6A00055F93B = {\n\t\t\t\t\t\tCreatedOnToolsVersion = 16.2;\n\t\t\t\t\t};\n\t\t\t\t\tC35372AD2D08C32D00474D34 = {\n\t\t\t\t\t\tCreatedOnToolsVersion = 16.0;\n\t\t\t\t\t};\n\t\t\t\t\tC39BB3E52D79082A005DB8FB = {\n\t\t\t\t\t\tCreatedOnToolsVersion = 16.0;\n\t\t\t\t\t\tLastSwiftMigration = 1600;\n\t\t\t\t\t};\n\t\t\t\t};\n\t\t\t};\n\t\t\tbuildConfigurationList = C35EDB672D07699400757E80 /* Build configuration list for PBXProject \"FastVLM\" */;\n\t\t\tdevelopmentRegion = en;\n\t\t\thasScannedForEncodings = 0;\n\t\t\tknownRegions = (\n\t\t\t\ten,\n\t\t\t\tBase,\n\t\t\t);\n\t\t\tmainGroup = C35EDB632D07699400757E80;\n\t\t\tminimizedProjectReferenceProxies = 1;\n\t\t\tpackageReferences = (\n\t\t\t\tC35EDB6A2D076A3900757E80 /* XCRemoteSwiftPackageReference \"mlx-swift-examples\" */,\n\t\t\t\tC35EDB8A2D07777E00757E80 /* XCRemoteSwiftPackageReference \"mlx-swift\" */,\n\t\t\t\tC3ED54592D790AC6005E20B3 /* XCRemoteSwiftPackageReference \"swift-transformers\" */,\n\t\t\t);\n\t\t\tpreferredProjectObjectVersion = 77;\n\t\t\tproductRefGroup = C35EDB702D076A5A00757E80 /* Products */;\n\t\t\tprojectDirPath = \"\";\n\t\t\tprojectRoot = \"\";\n\t\t\ttargets = (\n\t\t\t\tC35372AD2D08C32D00474D34 /* Video */,\n\t\t\t\t019A3E092D78E6A00055F93B /* FastVLM App */,\n\t\t\t\tC39BB3E52D79082A005DB8FB /* FastVLM */,\n\t\t\t);\n\t\t};\n/* End PBXProject section */\n\n/* Begin PBXResourcesBuildPhase section */\n\t\t019A3E082D78E6A00055F93B /* Resources */ = {\n\t\t\tisa = PBXResourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\tC35372AC2D08C32D00474D34 /* Resources */ = {\n\t\t\tisa = PBXResourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\tC39BB3E42D79082A005DB8FB /* Resources */ = {\n\t\t\tisa = PBXResourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXResourcesBuildPhase section */\n\n/* Begin PBXSourcesBuildPhase section */\n\t\t019A3E062D78E6A00055F93B /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\tC35372AA2D08C32D00474D34 /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\tC39BB3E22D79082A005DB8FB /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXSourcesBuildPhase section */\n\n/* Begin PBXTargetDependency section */\n\t\t019A3E242D78E7530055F93B /* PBXTargetDependency */ = {\n\t\t\tisa = PBXTargetDependency;\n\t\t\ttarget = C35372AD2D08C32D00474D34 /* Video */;\n\t\t\ttargetProxy = 019A3E232D78E7530055F93B /* PBXContainerItemProxy */;\n\t\t};\n\t\tC3ED55042D7A0A7A005E20B3 /* PBXTargetDependency */ = {\n\t\t\tisa = PBXTargetDependency;\n\t\t\ttarget = C39BB3E52D79082A005DB8FB /* FastVLM */;\n\t\t\ttargetProxy = C3ED55032D7A0A7A005E20B3 /* PBXContainerItemProxy */;\n\t\t};\n/* End PBXTargetDependency section */\n\n/* Begin XCBuildConfiguration section */\n\t\t019A3E162D78E6A20055F93B /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES;\n\t\t\t\tASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCODE_SIGN_ENTITLEMENTS = \"FastVLM App/FastVLM.entitlements\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tCURRENT_PROJECT_VERSION = 0.1.0;\n\t\t\t\tDEAD_CODE_STRIPPING = YES;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = dwarf;\n\t\t\t\tDEVELOPMENT_ASSET_PATHS = \"\\\"FastVLM App/Preview Content\\\"\";\n\t\t\t\tDEVELOPMENT_TEAM = \"\";\n\t\t\t\tENABLE_HARDENED_RUNTIME = YES;\n\t\t\t\tENABLE_PREVIEWS = YES;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_TESTABILITY = YES;\n\t\t\t\tENABLE_USER_SCRIPT_SANDBOXING = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu17;\n\t\t\t\tGCC_DYNAMIC_NO_PIC = NO;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_OPTIMIZATION_LEVEL = 0;\n\t\t\t\tGCC_PREPROCESSOR_DEFINITIONS = (\n\t\t\t\t\t\"DEBUG=1\",\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t);\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tINFOPLIST_FILE = \"FastVLM App/Info.plist\";\n\t\t\t\tINFOPLIST_KEY_CFBundleDisplayName = FastVLM;\n\t\t\t\tINFOPLIST_KEY_NSCameraUsageDescription = \"Use camera to get live feed of images\";\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphonesimulator*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents[sdk=iphonesimulator*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphonesimulator*]\" = YES;\n\t\t\t\tINFOPLIST_KEY_UIRequiresFullScreen = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphoneos*]\" = UIStatusBarStyleDefault;\n\t\t\t\t\"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphonesimulator*]\" = UIStatusBarStyleDefault;\n\t\t\t\tINFOPLIST_KEY_UISupportedInterfaceOrientations = UIInterfaceOrientationPortrait;\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 18.2;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = \"@executable_path/Frameworks\";\n\t\t\t\t\"LD_RUNPATH_SEARCH_PATHS[sdk=macosx*]\" = \"@executable_path/../Frameworks\";\n\t\t\t\tLOCALIZATION_PREFERS_STRING_CATALOGS = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 15.2;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tONLY_ACTIVE_ARCH = YES;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = com.apple.ml.FastVLM;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSDKROOT = auto;\n\t\t\t\tSUPPORTED_PLATFORMS = \"iphoneos iphonesimulator macosx xros xrsimulator\";\n\t\t\t\tSWIFT_ACTIVE_COMPILATION_CONDITIONS = \"DEBUG $(inherited)\";\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-Onone\";\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2,7\";\n\t\t\t\tXROS_DEPLOYMENT_TARGET = 2.2;\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\t019A3E172D78E6A20055F93B /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES;\n\t\t\t\tASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCODE_SIGN_ENTITLEMENTS = \"FastVLM App/FastVLM.entitlements\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tCURRENT_PROJECT_VERSION = 0.1.0;\n\t\t\t\tDEAD_CODE_STRIPPING = YES;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = \"dwarf-with-dsym\";\n\t\t\t\tDEVELOPMENT_ASSET_PATHS = \"\\\"FastVLM App/Preview Content\\\"\";\n\t\t\t\tDEVELOPMENT_TEAM = \"\";\n\t\t\t\tENABLE_HARDENED_RUNTIME = YES;\n\t\t\t\tENABLE_NS_ASSERTIONS = NO;\n\t\t\t\tENABLE_PREVIEWS = YES;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_USER_SCRIPT_SANDBOXING = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu17;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tINFOPLIST_FILE = \"FastVLM App/Info.plist\";\n\t\t\t\tINFOPLIST_KEY_CFBundleDisplayName = FastVLM;\n\t\t\t\tINFOPLIST_KEY_NSCameraUsageDescription = \"Use camera to get live feed of images\";\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphonesimulator*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents[sdk=iphonesimulator*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphoneos*]\" = YES;\n\t\t\t\t\"INFOPLIST_KEY_UILaunchScreen_Generation[sdk=iphonesimulator*]\" = YES;\n\t\t\t\tINFOPLIST_KEY_UIRequiresFullScreen = YES;\n\t\t\t\t\"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphoneos*]\" = UIStatusBarStyleDefault;\n\t\t\t\t\"INFOPLIST_KEY_UIStatusBarStyle[sdk=iphonesimulator*]\" = UIStatusBarStyleDefault;\n\t\t\t\tINFOPLIST_KEY_UISupportedInterfaceOrientations = UIInterfaceOrientationPortrait;\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 18.2;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = \"@executable_path/Frameworks\";\n\t\t\t\t\"LD_RUNPATH_SEARCH_PATHS[sdk=macosx*]\" = \"@executable_path/../Frameworks\";\n\t\t\t\tLOCALIZATION_PREFERS_STRING_CATALOGS = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 15.2;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = NO;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = com.apple.ml.FastVLM;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSDKROOT = auto;\n\t\t\t\tSUPPORTED_PLATFORMS = \"iphoneos iphonesimulator macosx xros xrsimulator\";\n\t\t\t\tSWIFT_COMPILATION_MODE = wholemodule;\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2,7\";\n\t\t\t\tXROS_DEPLOYMENT_TARGET = 2.2;\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n\t\tC35372B72D08C32D00474D34 /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALLOW_TARGET_PLATFORM_SPECIALIZATION = YES;\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES;\n\t\t\t\tBUILD_LIBRARY_FOR_DISTRIBUTION = YES;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCODE_SIGN_IDENTITY = \"\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEAD_CODE_STRIPPING = YES;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = dwarf;\n\t\t\t\tDEFINES_MODULE = YES;\n\t\t\t\tDYLIB_COMPATIBILITY_VERSION = 1;\n\t\t\t\tDYLIB_CURRENT_VERSION = 1;\n\t\t\t\tDYLIB_INSTALL_NAME_BASE = \"@rpath\";\n\t\t\t\tENABLE_MODULE_VERIFIER = YES;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_TESTABILITY = YES;\n\t\t\t\tENABLE_USER_SCRIPT_SANDBOXING = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu17;\n\t\t\t\tGCC_DYNAMIC_NO_PIC = NO;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_OPTIMIZATION_LEVEL = 0;\n\t\t\t\tGCC_PREPROCESSOR_DEFINITIONS = (\n\t\t\t\t\t\"DEBUG=1\",\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t);\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tINFOPLIST_KEY_NSHumanReadableCopyright = \"\";\n\t\t\t\tINSTALL_PATH = \"$(LOCAL_LIBRARY_DIR)/Frameworks\";\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 18.0;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"@executable_path/Frameworks\",\n\t\t\t\t\t\"@loader_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\t\"LD_RUNPATH_SEARCH_PATHS[sdk=macosx*]\" = (\n\t\t\t\t\t\"@executable_path/../Frameworks\",\n\t\t\t\t\t\"@loader_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\tLOCALIZATION_PREFERS_STRING_CATALOGS = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 15.0;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tMODULE_VERIFIER_SUPPORTED_LANGUAGES = \"objective-c objective-c++\";\n\t\t\t\tMODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = \"gnu17 gnu++20\";\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tONLY_ACTIVE_ARCH = YES;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = mlx.Video;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME:c99extidentifier)\";\n\t\t\t\tSDKROOT = auto;\n\t\t\t\tSKIP_INSTALL = YES;\n\t\t\t\tSUPPORTED_PLATFORMS = \"iphoneos iphonesimulator macosx xros xrsimulator\";\n\t\t\t\tSWIFT_ACTIVE_COMPILATION_CONDITIONS = \"DEBUG $(inherited)\";\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_INSTALL_OBJC_HEADER = NO;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-Onone\";\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2,7\";\n\t\t\t\tVERSIONING_SYSTEM = \"apple-generic\";\n\t\t\t\tVERSION_INFO_PREFIX = \"\";\n\t\t\t\tXROS_DEPLOYMENT_TARGET = 2.0;\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\tC35372B82D08C32D00474D34 /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALLOW_TARGET_PLATFORM_SPECIALIZATION = YES;\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES;\n\t\t\t\tBUILD_LIBRARY_FOR_DISTRIBUTION = YES;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCODE_SIGN_IDENTITY = \"\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEAD_CODE_STRIPPING = YES;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = \"dwarf-with-dsym\";\n\t\t\t\tDEFINES_MODULE = YES;\n\t\t\t\tDYLIB_COMPATIBILITY_VERSION = 1;\n\t\t\t\tDYLIB_CURRENT_VERSION = 1;\n\t\t\t\tDYLIB_INSTALL_NAME_BASE = \"@rpath\";\n\t\t\t\tENABLE_MODULE_VERIFIER = YES;\n\t\t\t\tENABLE_NS_ASSERTIONS = NO;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_USER_SCRIPT_SANDBOXING = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu17;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tINFOPLIST_KEY_NSHumanReadableCopyright = \"\";\n\t\t\t\tINSTALL_PATH = \"$(LOCAL_LIBRARY_DIR)/Frameworks\";\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 18.0;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"@executable_path/Frameworks\",\n\t\t\t\t\t\"@loader_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\t\"LD_RUNPATH_SEARCH_PATHS[sdk=macosx*]\" = (\n\t\t\t\t\t\"@executable_path/../Frameworks\",\n\t\t\t\t\t\"@loader_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\tLOCALIZATION_PREFERS_STRING_CATALOGS = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 15.0;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tMODULE_VERIFIER_SUPPORTED_LANGUAGES = \"objective-c objective-c++\";\n\t\t\t\tMODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = \"gnu17 gnu++20\";\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = NO;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = mlx.Video;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME:c99extidentifier)\";\n\t\t\t\tSDKROOT = auto;\n\t\t\t\tSKIP_INSTALL = YES;\n\t\t\t\tSUPPORTED_PLATFORMS = \"iphoneos iphonesimulator macosx xros xrsimulator\";\n\t\t\t\tSWIFT_COMPILATION_MODE = wholemodule;\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_INSTALL_OBJC_HEADER = NO;\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2,7\";\n\t\t\t\tVERSIONING_SYSTEM = \"apple-generic\";\n\t\t\t\tVERSION_INFO_PREFIX = \"\";\n\t\t\t\tXROS_DEPLOYMENT_TARGET = 2.0;\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n\t\tC35EDB682D07699400757E80 /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbaseConfigurationReferenceAnchor = C32B4A802DA4805400EF663D /* Configuration */;\n\t\t\tbaseConfigurationReferenceRelativePath = Build.xcconfig;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tDEAD_CODE_STRIPPING = YES;\n\t\t\t\tDEVELOPMENT_TEAM = 565ARCVNXV;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_TESTABILITY = YES;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tONLY_ACTIVE_ARCH = YES;\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\tC35EDB692D07699400757E80 /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tDEAD_CODE_STRIPPING = YES;\n\t\t\t\tDEVELOPMENT_TEAM = 565ARCVNXV;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n\t\tC39BB3FB2D79082A005DB8FB /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALLOW_TARGET_PLATFORM_SPECIALIZATION = YES;\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES;\n\t\t\t\tBUILD_LIBRARY_FOR_DISTRIBUTION = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCODE_SIGN_IDENTITY = \"\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEAD_CODE_STRIPPING = YES;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = dwarf;\n\t\t\t\tDEFINES_MODULE = NO;\n\t\t\t\tDYLIB_COMPATIBILITY_VERSION = 1;\n\t\t\t\tDYLIB_CURRENT_VERSION = 1;\n\t\t\t\tDYLIB_INSTALL_NAME_BASE = \"@rpath\";\n\t\t\t\tENABLE_MODULE_VERIFIER = YES;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_TESTABILITY = YES;\n\t\t\t\tENABLE_USER_SCRIPT_SANDBOXING = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu17;\n\t\t\t\tGCC_DYNAMIC_NO_PIC = NO;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_OPTIMIZATION_LEVEL = 0;\n\t\t\t\tGCC_PREPROCESSOR_DEFINITIONS = (\n\t\t\t\t\t\"DEBUG=1\",\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t);\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tINFOPLIST_KEY_NSHumanReadableCopyright = \"\";\n\t\t\t\tINSTALL_PATH = \"$(LOCAL_LIBRARY_DIR)/Frameworks\";\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 18.0;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"@executable_path/Frameworks\",\n\t\t\t\t\t\"@loader_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\t\"LD_RUNPATH_SEARCH_PATHS[sdk=macosx*]\" = (\n\t\t\t\t\t\"@executable_path/../Frameworks\",\n\t\t\t\t\t\"@loader_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\tLOCALIZATION_PREFERS_STRING_CATALOGS = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 15.0;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tMODULE_VERIFIER_SUPPORTED_LANGUAGES = \"objective-c objective-c++\";\n\t\t\t\tMODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = \"gnu17 gnu++20\";\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tONLY_ACTIVE_ARCH = YES;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = mlx.FastVLM;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME:c99extidentifier)\";\n\t\t\t\tSDKROOT = auto;\n\t\t\t\tSKIP_INSTALL = YES;\n\t\t\t\tSUPPORTED_PLATFORMS = \"iphoneos iphonesimulator macosx xros xrsimulator\";\n\t\t\t\tSWIFT_ACTIVE_COMPILATION_CONDITIONS = \"DEBUG $(inherited)\";\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_INSTALL_OBJC_HEADER = NO;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-Onone\";\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2,7\";\n\t\t\t\tVERSIONING_SYSTEM = \"apple-generic\";\n\t\t\t\tVERSION_INFO_PREFIX = \"\";\n\t\t\t\tXROS_DEPLOYMENT_TARGET = 2.0;\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\tC39BB3FC2D79082A005DB8FB /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALLOW_TARGET_PLATFORM_SPECIALIZATION = YES;\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES;\n\t\t\t\tBUILD_LIBRARY_FOR_DISTRIBUTION = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++20\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_WEAK = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_DOCUMENTATION_COMMENTS = YES;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\tCODE_SIGN_IDENTITY = \"\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tDEAD_CODE_STRIPPING = YES;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = \"dwarf-with-dsym\";\n\t\t\t\tDEFINES_MODULE = NO;\n\t\t\t\tDYLIB_COMPATIBILITY_VERSION = 1;\n\t\t\t\tDYLIB_CURRENT_VERSION = 1;\n\t\t\t\tDYLIB_INSTALL_NAME_BASE = \"@rpath\";\n\t\t\t\tENABLE_MODULE_VERIFIER = YES;\n\t\t\t\tENABLE_NS_ASSERTIONS = NO;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_USER_SCRIPT_SANDBOXING = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu17;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tINFOPLIST_KEY_NSHumanReadableCopyright = \"\";\n\t\t\t\tINSTALL_PATH = \"$(LOCAL_LIBRARY_DIR)/Frameworks\";\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 18.0;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"@executable_path/Frameworks\",\n\t\t\t\t\t\"@loader_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\t\"LD_RUNPATH_SEARCH_PATHS[sdk=macosx*]\" = (\n\t\t\t\t\t\"@executable_path/../Frameworks\",\n\t\t\t\t\t\"@loader_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\tLOCALIZATION_PREFERS_STRING_CATALOGS = YES;\n\t\t\t\tMACOSX_DEPLOYMENT_TARGET = 15.0;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tMODULE_VERIFIER_SUPPORTED_LANGUAGES = \"objective-c objective-c++\";\n\t\t\t\tMODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = \"gnu17 gnu++20\";\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = NO;\n\t\t\t\tMTL_FAST_MATH = YES;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = mlx.FastVLM;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME:c99extidentifier)\";\n\t\t\t\tSDKROOT = auto;\n\t\t\t\tSKIP_INSTALL = YES;\n\t\t\t\tSUPPORTED_PLATFORMS = \"iphoneos iphonesimulator macosx xros xrsimulator\";\n\t\t\t\tSWIFT_COMPILATION_MODE = wholemodule;\n\t\t\t\tSWIFT_EMIT_LOC_STRINGS = YES;\n\t\t\t\tSWIFT_INSTALL_OBJC_HEADER = NO;\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2,7\";\n\t\t\t\tVERSIONING_SYSTEM = \"apple-generic\";\n\t\t\t\tVERSION_INFO_PREFIX = \"\";\n\t\t\t\tXROS_DEPLOYMENT_TARGET = 2.0;\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n/* End XCBuildConfiguration section */\n\n/* Begin XCConfigurationList section */\n\t\t019A3E182D78E6A20055F93B /* Build configuration list for PBXNativeTarget \"FastVLM App\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\t019A3E162D78E6A20055F93B /* Debug */,\n\t\t\t\t019A3E172D78E6A20055F93B /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n\t\tC35372BA2D08C32D00474D34 /* Build configuration list for PBXNativeTarget \"Video\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\tC35372B72D08C32D00474D34 /* Debug */,\n\t\t\t\tC35372B82D08C32D00474D34 /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n\t\tC35EDB672D07699400757E80 /* Build configuration list for PBXProject \"FastVLM\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\tC35EDB682D07699400757E80 /* Debug */,\n\t\t\t\tC35EDB692D07699400757E80 /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n\t\tC39BB3FF2D79082A005DB8FB /* Build configuration list for PBXNativeTarget \"FastVLM\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\tC39BB3FB2D79082A005DB8FB /* Debug */,\n\t\t\t\tC39BB3FC2D79082A005DB8FB /* Release */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n/* End XCConfigurationList section */\n\n/* Begin XCRemoteSwiftPackageReference section */\n\t\tC35EDB6A2D076A3900757E80 /* XCRemoteSwiftPackageReference \"mlx-swift-examples\" */ = {\n\t\t\tisa = XCRemoteSwiftPackageReference;\n\t\t\trepositoryURL = \"https://github.com/ml-explore/mlx-swift-examples\";\n\t\t\trequirement = {\n\t\t\t\tkind = upToNextMajorVersion;\n\t\t\t\tminimumVersion = 2.21.2;\n\t\t\t};\n\t\t};\n\t\tC35EDB8A2D07777E00757E80 /* XCRemoteSwiftPackageReference \"mlx-swift\" */ = {\n\t\t\tisa = XCRemoteSwiftPackageReference;\n\t\t\trepositoryURL = \"https://github.com/ml-explore/mlx-swift\";\n\t\t\trequirement = {\n\t\t\t\tkind = upToNextMajorVersion;\n\t\t\t\tminimumVersion = 0.21.2;\n\t\t\t};\n\t\t};\n\t\tC3ED54592D790AC6005E20B3 /* XCRemoteSwiftPackageReference \"swift-transformers\" */ = {\n\t\t\tisa = XCRemoteSwiftPackageReference;\n\t\t\trepositoryURL = \"https://github.com/huggingface/swift-transformers\";\n\t\t\trequirement = {\n\t\t\t\tkind = upToNextMajorVersion;\n\t\t\t\tminimumVersion = 0.1.18;\n\t\t\t};\n\t\t};\n/* End XCRemoteSwiftPackageReference section */\n\n/* Begin XCSwiftPackageProductDependency section */\n\t\t019A3E192D78E7370055F93B /* MLX */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tpackage = C35EDB8A2D07777E00757E80 /* XCRemoteSwiftPackageReference \"mlx-swift\" */;\n\t\t\tproductName = MLX;\n\t\t};\n\t\t019A3E1B2D78E73E0055F93B /* MLXLMCommon */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tpackage = C35EDB6A2D076A3900757E80 /* XCRemoteSwiftPackageReference \"mlx-swift-examples\" */;\n\t\t\tproductName = MLXLMCommon;\n\t\t};\n\t\t019A3E1D2D78E7470055F93B /* MLXRandom */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tpackage = C35EDB8A2D07777E00757E80 /* XCRemoteSwiftPackageReference \"mlx-swift\" */;\n\t\t\tproductName = MLXRandom;\n\t\t};\n\t\t019A3E1F2D78E74C0055F93B /* MLXVLM */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tpackage = C35EDB6A2D076A3900757E80 /* XCRemoteSwiftPackageReference \"mlx-swift-examples\" */;\n\t\t\tproductName = MLXVLM;\n\t\t};\n\t\tC3ED544D2D790860005E20B3 /* MLXLMCommon */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tpackage = C35EDB6A2D076A3900757E80 /* XCRemoteSwiftPackageReference \"mlx-swift-examples\" */;\n\t\t\tproductName = MLXLMCommon;\n\t\t};\n\t\tC3ED544F2D790860005E20B3 /* MLXVLM */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tpackage = C35EDB6A2D076A3900757E80 /* XCRemoteSwiftPackageReference \"mlx-swift-examples\" */;\n\t\t\tproductName = MLXVLM;\n\t\t};\n\t\tC3ED54512D790860005E20B3 /* MLX */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tpackage = C35EDB8A2D07777E00757E80 /* XCRemoteSwiftPackageReference \"mlx-swift\" */;\n\t\t\tproductName = MLX;\n\t\t};\n\t\tC3ED54532D790860005E20B3 /* MLXNN */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tpackage = C35EDB8A2D07777E00757E80 /* XCRemoteSwiftPackageReference \"mlx-swift\" */;\n\t\t\tproductName = MLXNN;\n\t\t};\n\t\tC3ED54572D790A68005E20B3 /* MLXFast */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tpackage = C35EDB8A2D07777E00757E80 /* XCRemoteSwiftPackageReference \"mlx-swift\" */;\n\t\t\tproductName = MLXFast;\n\t\t};\n\t\tC3ED545A2D790AD6005E20B3 /* Transformers */ = {\n\t\t\tisa = XCSwiftPackageProductDependency;\n\t\t\tpackage = C3ED54592D790AC6005E20B3 /* XCRemoteSwiftPackageReference \"swift-transformers\" */;\n\t\t\tproductName = Transformers;\n\t\t};\n/* End XCSwiftPackageProductDependency section */\n\t};\n\trootObject = C35EDB642D07699400757E80 /* Project object */;\n}\n"
  },
  {
    "path": "app/FastVLM.xcodeproj/xcshareddata/xcschemes/FastVLM App.xcscheme",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Scheme\n   LastUpgradeVersion = \"1630\"\n   version = \"1.7\">\n   <BuildAction\n      parallelizeBuildables = \"YES\"\n      buildImplicitDependencies = \"YES\"\n      buildArchitectures = \"Automatic\">\n      <BuildActionEntries>\n         <BuildActionEntry\n            buildForTesting = \"YES\"\n            buildForRunning = \"YES\"\n            buildForProfiling = \"YES\"\n            buildForArchiving = \"YES\"\n            buildForAnalyzing = \"YES\">\n            <BuildableReference\n               BuildableIdentifier = \"primary\"\n               BlueprintIdentifier = \"019A3E092D78E6A00055F93B\"\n               BuildableName = \"FastVLM App.app\"\n               BlueprintName = \"FastVLM App\"\n               ReferencedContainer = \"container:FastVLM.xcodeproj\">\n            </BuildableReference>\n         </BuildActionEntry>\n      </BuildActionEntries>\n   </BuildAction>\n   <TestAction\n      buildConfiguration = \"Debug\"\n      selectedDebuggerIdentifier = \"Xcode.DebuggerFoundation.Debugger.LLDB\"\n      selectedLauncherIdentifier = \"Xcode.DebuggerFoundation.Launcher.LLDB\"\n      shouldUseLaunchSchemeArgsEnv = \"YES\"\n      shouldAutocreateTestPlan = \"YES\">\n   </TestAction>\n   <LaunchAction\n      buildConfiguration = \"Release\"\n      selectedDebuggerIdentifier = \"Xcode.DebuggerFoundation.Debugger.LLDB\"\n      selectedLauncherIdentifier = \"Xcode.DebuggerFoundation.Launcher.LLDB\"\n      launchStyle = \"0\"\n      useCustomWorkingDirectory = \"NO\"\n      ignoresPersistentStateOnLaunch = \"NO\"\n      debugDocumentVersioning = \"YES\"\n      debugServiceExtension = \"internal\"\n      allowLocationSimulation = \"YES\">\n      <BuildableProductRunnable\n         runnableDebuggingMode = \"0\">\n         <BuildableReference\n            BuildableIdentifier = \"primary\"\n            BlueprintIdentifier = \"019A3E092D78E6A00055F93B\"\n            BuildableName = \"FastVLM App.app\"\n            BlueprintName = \"FastVLM App\"\n            ReferencedContainer = \"container:FastVLM.xcodeproj\">\n         </BuildableReference>\n      </BuildableProductRunnable>\n   </LaunchAction>\n   <ProfileAction\n      buildConfiguration = \"Release\"\n      shouldUseLaunchSchemeArgsEnv = \"YES\"\n      savedToolIdentifier = \"\"\n      useCustomWorkingDirectory = \"NO\"\n      debugDocumentVersioning = \"YES\">\n      <BuildableProductRunnable\n         runnableDebuggingMode = \"0\">\n         <BuildableReference\n            BuildableIdentifier = \"primary\"\n            BlueprintIdentifier = \"019A3E092D78E6A00055F93B\"\n            BuildableName = \"FastVLM App.app\"\n            BlueprintName = \"FastVLM App\"\n            ReferencedContainer = \"container:FastVLM.xcodeproj\">\n         </BuildableReference>\n      </BuildableProductRunnable>\n   </ProfileAction>\n   <AnalyzeAction\n      buildConfiguration = \"Debug\">\n   </AnalyzeAction>\n   <ArchiveAction\n      buildConfiguration = \"Release\"\n      revealArchiveInOrganizer = \"YES\">\n   </ArchiveAction>\n</Scheme>\n"
  },
  {
    "path": "app/README.md",
    "content": "# FastVLM\n\nDemonstrates the performance of **FastVLM** models for on-device, visual question answering. \n\n<table>\n<tr>\n    <td><img src=\"../docs/fastvlm-counting.gif\" alt=\"FastVLM - Counting\"></td>\n    <td><img src=\"../docs/fastvlm-handwriting.gif\" alt=\"FastVLM - Handwriting\"></td>\n    <td><img src=\"../docs/fastvlm-emoji.gif\" alt=\"FastVLM - Emoji\"></td>\n</tr>\n</table>\n\n## Features\n\n- FastVLM runs on iOS (18.2+) and macOS (15.2+).\n- View Time-To-First-Token (TTFT) with every inference.\n- All predictions are processed privately and securely using on-device models.\n\n### Flexible Prompting\n\n<img src=\"../docs/fastvlm-flexible_prompts.png\" alt=\"Flexible prompting\" style=\"width:66%;\">\n\nThe app includes a set of built-in prompts to help you get started quickly. Tap the **Prompts** button in the top-right corner to explore them. Selecting a prompt will immediately update the active input. To create new prompts or edit existing ones, choose **Customize…** from the **Prompts** menu.\n\n## Pretrained Model Options\n\nThere are 3 pretrained sizes of FastVLM to choose from:\n\n- **FastVLM 0.5B**: Small and fast - great for mobile devices where speed matters.\n- **FastVLM 1.5B**: Well balanced - great for larger devices where speed and accuracy matters.\n- **FastVLM 7B**: Fast and accurate - ideal for situations where accuracy matters over speed.\n\nTo download any FastVLM listed above, use the [get_pretrained_mlx_model.sh](get_pretrained_mlx_model.sh) script. The script downloads the model from the web and places it in the appropriate location. Once a model has been downloaded using the steps below, no additional steps are needed to build the app in Xcode.\n\nTo explore how the other models work for your use-case, simply re-run the `get_pretrained_mlx_model.sh` with the new model selected, follow the prompts, and rebuild your app in Xcode.\n\n### Download Instructions\n\n1. Make the script executable\n\n```shell\nchmod +x app/get_pretrained_mlx_model.sh\n```\n\n2. Download FastVLM\n\n```shell\napp/get_pretrained_mlx_model.sh --model 0.5b --dest app/FastVLM/model\n```\n\n3. Open the app in Xcode, Build, and Run.\n\n### Custom Model\n\nIn addition to pretrained sizes of FastVLM, you can further quantize or fine-tune FastVLM to best fit their needs. To learn more, check out our documentation on how to [`export the model`](../model_export#export-vlm).\nPlease clear existing model in `app/FastVLM/model` before downloading or copying a new model. \n"
  },
  {
    "path": "app/Video/CameraController.swift",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\nimport AVFoundation\nimport CoreImage\n\n#if os(iOS)\n    import UIKit\n#endif\n\n@Observable\npublic class CameraController: NSObject {\n\n    private var framesContinuation: AsyncStream<CMSampleBuffer>.Continuation?\n\n    public var backCamera = true {\n        didSet {\n            stop()\n            start()\n        }\n    }\n\n    public var devices = [AVCaptureDevice]()\n\n    public var device: AVCaptureDevice = AVCaptureDevice.default(for: .video)! {\n        didSet {\n            stop()\n            start()\n        }\n    }\n\n    private var permissionGranted = true\n    private var captureSession: AVCaptureSession?\n    private let sessionQueue = DispatchQueue(label: \"sessionQueue\")\n    @objc dynamic private var rotationCoordinator : AVCaptureDevice.RotationCoordinator?\n    private var rotationObservation: NSKeyValueObservation?\n\n    public func attach(continuation: AsyncStream<CMSampleBuffer>.Continuation) {\n        sessionQueue.async {\n            self.framesContinuation = continuation\n        }\n    }\n\n    public func detatch() {\n        sessionQueue.async {\n            self.framesContinuation = nil\n        }\n    }\n\n    public func stop() {\n        sessionQueue.sync { [self] in\n            captureSession?.stopRunning()\n            captureSession = nil\n        }\n\n    }\n\n    public func start() {\n        sessionQueue.async { [self] in\n            let captureSession = AVCaptureSession()\n            self.captureSession = captureSession\n\n            self.checkPermission()\n            self.setupCaptureSession(position: backCamera ? .back : .front)\n            captureSession.startRunning()\n        }\n    }\n\n    #if os(iOS)\n        private func setOrientation(_ orientation: UIDeviceOrientation) {\n            guard let captureSession else { return }\n\n            let angle: Double?\n            switch orientation {\n            case .unknown, .faceDown:\n                angle = nil\n            case .portrait, .faceUp:\n                angle = 90\n            case .portraitUpsideDown:\n                angle = 270\n            case .landscapeLeft:\n                angle = 0\n            case .landscapeRight:\n                angle = 180\n            @unknown default:\n                angle = nil\n            }\n\n            if let angle {\n                for output in captureSession.outputs {\n                    output.connection(with: .video)?.videoRotationAngle = angle\n                }\n            }\n        }\n    \n    private func updateRotation(rotation : CGFloat) {\n        guard let captureSession else { return }\n        for output in captureSession.outputs {\n            output.connection(with: .video)?.videoRotationAngle = rotation\n        }\n    }\n    #endif\n\n    func checkPermission() {\n        switch AVCaptureDevice.authorizationStatus(for: .video) {\n        case .authorized:\n            // The user has previously granted access to the camera.\n            self.permissionGranted = true\n\n        case .notDetermined:\n            // The user has not yet been asked for camera access.\n            self.requestPermission()\n\n        // Combine the two other cases into the default case\n        default:\n            self.permissionGranted = false\n        }\n    }\n\n    func requestPermission() {\n        // Strong reference not a problem here but might become one in the future.\n        AVCaptureDevice.requestAccess(for: .video) { [unowned self] granted in\n            self.permissionGranted = granted\n        }\n    }\n\n    func setupCaptureSession(position: AVCaptureDevice.Position) {\n        guard let captureSession else { return }\n\n        let videoOutput = AVCaptureVideoDataOutput()\n\n        guard permissionGranted else {\n            print(\"No permission for camera\")\n            return\n        }\n\n        let deviceTypes: [AVCaptureDevice.DeviceType]\n        #if os(iOS)\n            deviceTypes = [.builtInDualCamera, .builtInWideAngleCamera]\n        #else\n            deviceTypes = [.external, .continuityCamera, .builtInWideAngleCamera]\n        #endif\n\n        let videoDeviceDiscoverySession = AVCaptureDevice.DiscoverySession(\n            deviceTypes: deviceTypes,\n            mediaType: .video,\n            position: position)\n\n        let videoDevice: AVCaptureDevice?\n        if videoDeviceDiscoverySession.devices.contains(self.device) {\n            videoDevice = self.device\n        } else {\n            videoDevice = videoDeviceDiscoverySession.devices.first\n        }\n\n        if devices.isEmpty {\n            self.devices = videoDeviceDiscoverySession.devices\n        }\n\n        guard\n            let videoDevice\n        else {\n            print(\"Unable to find video device\")\n            return\n        }\n        guard let videoDeviceInput = try? AVCaptureDeviceInput(device: videoDevice) else {\n            print(\"Unable to create AVCaptureDeviceInput\")\n            return\n        }\n        guard captureSession.canAddInput(videoDeviceInput) else {\n            print(\"Unable to add input\")\n            return\n        }\n        captureSession.addInput(videoDeviceInput)\n\n        videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: \"sampleBufferQueue\"))\n        captureSession.addOutput(videoOutput)\n        captureSession.sessionPreset = AVCaptureSession.Preset.hd1920x1080\n\n        #if os(iOS)\n        rotationCoordinator = AVCaptureDevice.RotationCoordinator(device: videoDevice, previewLayer: nil)\n        rotationObservation = observe(\\.rotationCoordinator!.videoRotationAngleForHorizonLevelCapture, options: [.initial, .new]) { [weak self] _, change in\n            if let nv = change.newValue {\n                self?.updateRotation(rotation: nv)\n            }\n        }\n        #endif\n    }\n}\n\nextension CameraController: AVCaptureVideoDataOutputSampleBufferDelegate {\n    public func captureOutput(\n        _ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer,\n        from connection: AVCaptureConnection\n    ) {\n        if sampleBuffer.isValid && sampleBuffer.imageBuffer != nil {\n            framesContinuation?.yield(sampleBuffer)\n        }\n    }\n}\n"
  },
  {
    "path": "app/Video/CameraControlsView.swift",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\nimport AVFoundation\nimport SwiftUI\n\npublic struct CameraControlsView: View {\n\n    @Binding public var backCamera: Bool\n    @Binding public var device: AVCaptureDevice\n    @Binding public var devices: [AVCaptureDevice]\n\n    public init(\n        backCamera: Binding<Bool>,\n        device: Binding<AVCaptureDevice>,\n        devices: Binding<[AVCaptureDevice]>\n    ) {\n        self._backCamera = backCamera\n        self._device = device\n        self._devices = devices\n    }\n\n    public var body: some View {\n        Button {\n            backCamera.toggle()\n        } label: {\n            RoundedRectangle(cornerRadius: 8.0)\n                .fill(.regularMaterial)\n                .frame(width: 32.0, height: 32.0)\n                .overlay(alignment: .center) {\n                    // Switch cameras image\n                    Image(systemName: \"arrow.triangle.2.circlepath.camera.fill\")\n                        .foregroundStyle(.primary)\n                        .padding(6.0)\n                }\n        }\n    }\n}\n"
  },
  {
    "path": "app/Video/CameraType.swift",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\nimport Foundation\n\npublic enum CameraType: String, CaseIterable {\n    case continuous\n    case single\n}\n"
  },
  {
    "path": "app/Video/Video.h",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\n#import <Foundation/Foundation.h>\n\n//! Project version number for Video.\nFOUNDATION_EXPORT double VideoVersionNumber;\n\n//! Project version string for Video.\nFOUNDATION_EXPORT const unsigned char VideoVersionString[];\n"
  },
  {
    "path": "app/Video/VideoFrameView.swift",
    "content": "//\n// For licensing see accompanying LICENSE file.\n// Copyright (C) 2025 Apple Inc. All Rights Reserved.\n//\n\nimport AVFoundation\nimport CoreImage\nimport Foundation\nimport SwiftUI\n\n/// Displays a stream of video frames\npublic struct VideoFrameView: View {\n    @Environment(\\.colorScheme) private var colorScheme\n\n    public let frames: AsyncStream<CVImageBuffer>\n    public let cameraType: CameraType\n    public let action: ((CVImageBuffer) -> Void)?\n\n    @State private var hold: Bool = false\n    @State private var videoFrame: CVImageBuffer?\n\n    private var backgroundColor: Color {\n        #if os(iOS)\n        return Color(.secondarySystemBackground)\n        #elseif os(macOS)\n        return Color(.secondarySystemFill)\n        #else\n        // When in doubt, use these values that I captured to match iOS' secondarySystemBackground\n        if colorScheme == .dark {\n            return Color(red: 0.11, green: 0.11, blue: 0.12)\n        } else {\n            return Color(red: 0.95, green: 0.95, blue: 0.97)\n        }\n        #endif\n    }\n\n    public init(\n        frames: AsyncStream<CVImageBuffer>,\n        cameraType: CameraType,\n        action: ((CVImageBuffer) -> Void)?\n    ) {\n        self.frames = frames\n        self.cameraType = cameraType\n        self.action = action\n    }\n\n    public var body: some View {\n        Group {\n            if let videoFrame {\n                _ImageView(image: videoFrame)\n                    .overlay(alignment: .bottom) {\n                        if cameraType == .single {\n                            Button {\n                                tap()\n                            } label: {\n                                if hold {\n                                    Label(\"Resume\", systemImage: \"play.fill\")\n                                } else {\n                                    Label(\"Capture Photo\", systemImage: \"camera.fill\")\n                                }\n                            }\n                            .clipShape(.capsule)\n                            .buttonStyle(.borderedProminent)\n                            .tint(hold ? .gray : .accentColor)\n                            .foregroundColor(.white)\n                            .padding()\n                        }\n                    }\n            } else {\n                // spinner before the camera comes up\n                ProgressView()\n                    .controlSize(.large)\n            }\n        }\n        // This ensures that we take up the full 4/3 aspect ratio\n        // even if we don't have an image to display\n        .frame(maxWidth: .infinity, maxHeight: .infinity)\n        .background(backgroundColor)\n        .clipShape(RoundedRectangle(cornerRadius: 10.0))\n        .task {\n            // feed frames to the _ImageView\n            if Task.isCancelled {\n                return\n            }\n            for await frame in frames {\n                if !hold {\n                    videoFrame = frame\n                }\n            }\n        }\n        .onChange(of: cameraType) { _, newType in\n            // No matter what, when the user switches to .continuous,\n            // we need to continue showing updated frames\n            if newType == .continuous {\n                hold = false\n            }\n        }\n    }\n\n    private func tap() {\n        if hold {\n            // resume\n            hold = false\n        } else if let videoFrame {\n            hold = true\n            if let action {\n                action(videoFrame)\n            }\n        }\n    }\n}\n\n#if os(iOS)\n    /// Internal view to display a CVImageBuffer\n    private struct _ImageView: UIViewRepresentable {\n\n        let image: Any\n        var gravity = CALayerContentsGravity.resizeAspectFill\n\n        func makeUIView(context: Context) -> UIView {\n            let view = UIView()\n            view.layer.contentsGravity = gravity\n            return view\n        }\n\n        func updateUIView(_ uiView: UIView, context: Context) {\n            uiView.layer.contents = image\n        }\n    }\n#else\n    private struct _ImageView: NSViewRepresentable {\n\n        let image: Any\n        var gravity = CALayerContentsGravity.resizeAspectFill\n\n        func makeNSView(context: Context) -> NSView {\n            let view = NSView()\n            view.wantsLayer = true\n            view.layer?.contentsGravity = gravity\n            return view\n        }\n\n        func updateNSView(_ uiView: NSView, context: Context) {\n            uiView.layer?.contents = image\n        }\n    }\n\n#endif\n"
  },
  {
    "path": "app/get_pretrained_mlx_model.sh",
    "content": "#!/usr/bin/env bash\n#\n# For licensing see accompanying LICENSE_MODEL file.\n# Copyright (C) 2025 Apple Inc. All Rights Reserved.\n#\nset -e\n\n# Help function\nshow_help() {\n    local is_error=${1:-true}  # Default to error mode if no argument provided\n    \n    echo \"Usage: $0 --model <model_size> --dest <destination_directory>\"\n    echo\n    echo \"Required arguments:\"\n    echo \"  --model <model_size>    Size of the model to download\"\n    echo \"  --dest <directory>      Directory where the model will be downloaded\"\n    echo\n    echo \"Available model sizes:\"\n    echo \"  0.5b  - 0.5B parameter model (FP16)\"\n    echo \"  1.5b  - 1.5B parameter model (INT8)\"\n    echo \"  7b    - 7B parameter model (INT4)\"\n    echo\n    echo \"Options:\"\n    echo \"  --help    Show help message\"\n    \n    # Exit with success (0) for help flag, error (1) for usage errors\n    if [ \"$is_error\" = \"false\" ]; then\n        exit 0\n    else\n        exit 1\n    fi\n}\n\n# Parse command line arguments\nwhile [[ \"$#\" -gt 0 ]]; do\n    case $1 in\n        --model) model_size=\"$2\"; shift ;;\n        --dest) dest_dir=\"$2\"; shift ;;\n        --help) show_help false ;;  # Explicit help request\n        *) echo -e \"Unknown parameter: $1\\n\"; show_help true ;;  # Error case\n    esac\n    shift\ndone\n\n# Validate required parameters\nif [ -z \"$model_size\" ]; then\n    echo -e \"Error: --model parameter is required\\n\"\n    show_help true\nfi\n\nif [ -z \"$dest_dir\" ]; then\n    echo -e \"Error: --dest parameter is required\\n\"\n    show_help true\nfi\n\n# Map model size to full model name\ncase \"$model_size\" in\n    \"0.5b\") model=\"llava-fastvithd_0.5b_stage3_llm.fp16\" ;;\n    \"1.5b\") model=\"llava-fastvithd_1.5b_stage3_llm.int8\" ;;\n    \"7b\") model=\"llava-fastvithd_7b_stage3_llm.int4\" ;;\n    *)\n        echo -e \"Error: Invalid model size '$model_size'\\n\"\n        show_help true\n        ;;\nesac\n\ncleanup() { \n    rm -rf \"$tmp_dir\"\n}\n\ndownload_model() {\n    # Download directory\n    tmp_dir=$(mktemp -d)\n\n    # Model paths\n    base_url=\"https://ml-site.cdn-apple.com/datasets/fastvlm\"\n\n    # Create destination directory if it doesn't exist\n    if [ ! -d \"$dest_dir\" ]; then\n        echo \"Creating destination directory: $dest_dir\"\n        mkdir -p \"$dest_dir\"\n    elif [ \"$(ls -A \"$dest_dir\")\" ]; then\n        echo -e \"Destination directory '$dest_dir' exists and is not empty.\\n\"\n        read -p \"Do you want to clear it and continue? [y/N]: \" confirm\n        if [[ ! \"$confirm\" =~ ^[Yy]$ ]]; then\n            echo -e \"\\nStopping.\"\n            exit 1\n        fi\n        echo -e \"\\nClearing existing contents in '$dest_dir'\"\n        rm -rf \"${dest_dir:?}\"/*\n    fi\n\n    # Create temp variables\n    tmp_zip_file=\"${tmp_dir}/${model}.zip\"\n    tmp_extract_dir=\"${tmp_dir}/${model}\"\n\n    # Create temp extract directory\n    mkdir -p \"$tmp_extract_dir\"\n\n    # Download model\n    echo -e \"\\nDownloading '${model}' model ...\\n\"\n    wget -q --progress=bar:noscroll --show-progress -O \"$tmp_zip_file\" \"$base_url/$model.zip\"\n\n    # Unzip model\n    echo -e \"\\nUnzipping model...\"\n    unzip -q \"$tmp_zip_file\" -d \"$tmp_extract_dir\"\n\n    # Copy model files to destination directory\n    echo -e \"\\nCopying model files to destination directory...\"\n    cp -r \"$tmp_extract_dir/$model\"/* \"$dest_dir\"\n\n    # Verify destination directory exists and is not empty\n    if [ ! -d \"$dest_dir\" ] || [ -z \"$(ls -A \"$dest_dir\")\" ]; then\n        echo -e \"\\nModel extraction failed. Destination directory '$dest_dir' is missing or empty.\"\n        exit 1\n    fi\n\n    echo -e \"\\nModel downloaded and extracted to '$dest_dir'\"\n}\n\n# Cleanup download directory on exit\ntrap cleanup EXIT INT TERM\n\n# Download models\ndownload_model"
  },
  {
    "path": "get_models.sh",
    "content": "#!/usr/bin/env bash\n#\n# For licensing see accompanying LICENSE_MODEL file.\n# Copyright (C) 2025 Apple Inc. All Rights Reserved.\n#\n\nmkdir -p checkpoints\nwget https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_0.5b_stage2.zip -P checkpoints\nwget https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_0.5b_stage3.zip -P checkpoints\nwget https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_1.5b_stage2.zip -P checkpoints\nwget https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_1.5b_stage3.zip -P checkpoints\nwget https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_7b_stage2.zip -P checkpoints\nwget https://ml-site.cdn-apple.com/datasets/fastvlm/llava-fastvithd_7b_stage3.zip -P checkpoints\n\n# Extract models\ncd checkpoints\nunzip -qq llava-fastvithd_0.5b_stage2.zip\nunzip -qq llava-fastvithd_0.5b_stage3.zip\nunzip -qq llava-fastvithd_1.5b_stage2.zip\nunzip -qq llava-fastvithd_1.5b_stage3.zip\nunzip -qq llava-fastvithd_7b_stage2.zip\nunzip -qq llava-fastvithd_7b_stage3.zip\n\n# Clean up\nrm llava-fastvithd_0.5b_stage2.zip\nrm llava-fastvithd_0.5b_stage3.zip\nrm llava-fastvithd_1.5b_stage2.zip\nrm llava-fastvithd_1.5b_stage3.zip\nrm llava-fastvithd_7b_stage2.zip\nrm llava-fastvithd_7b_stage3.zip\ncd -\n"
  },
  {
    "path": "llava/__init__.py",
    "content": "from .model import LlavaLlamaForCausalLM, LlavaQwen2ForCausalLM\n"
  },
  {
    "path": "llava/constants.py",
    "content": "CONTROLLER_HEART_BEAT_EXPIRATION = 30\nWORKER_HEART_BEAT_INTERVAL = 15\n\nLOGDIR = \".\"\n\n# Model Constants\nIGNORE_INDEX = -100\nIMAGE_TOKEN_INDEX = -200\nDEFAULT_IMAGE_TOKEN = \"<image>\"\nDEFAULT_IMAGE_PATCH_TOKEN = \"<im_patch>\"\nDEFAULT_IM_START_TOKEN = \"<im_start>\"\nDEFAULT_IM_END_TOKEN = \"<im_end>\"\nIMAGE_PLACEHOLDER = \"<image-placeholder>\"\n"
  },
  {
    "path": "llava/conversation.py",
    "content": "import dataclasses\nfrom enum import auto, Enum\nfrom typing import List, Tuple\nimport base64\nfrom io import BytesIO\nfrom PIL import Image\n\n\nclass SeparatorStyle(Enum):\n    \"\"\"Different separator style.\"\"\"\n    SINGLE = auto()\n    TWO = auto()\n    MPT = auto()\n    PLAIN = auto()\n    LLAMA_2 = auto()\n    QWEN_2 = auto()  # fix: add qwen2\n    CHATML = auto()\n\n\n@dataclasses.dataclass\nclass Conversation:\n    \"\"\"A class that keeps all conversation history.\"\"\"\n    system: str\n    roles: List[str]\n    messages: List[List[str]]\n    offset: int\n    sep_style: SeparatorStyle = SeparatorStyle.SINGLE\n    sep: str = \"###\"\n    sep2: str = None\n    version: str = \"Unknown\"\n\n    skip_next: bool = False\n\n    def get_prompt(self):\n        messages = self.messages\n        if len(messages) > 0 and type(messages[0][1]) is tuple:\n            messages = self.messages.copy()\n            init_role, init_msg = messages[0].copy()\n            init_msg = init_msg[0].replace(\"<image>\", \"\").strip()\n            if 'mmtag' in self.version:\n                messages[0] = (init_role, init_msg)\n                messages.insert(0, (self.roles[0], \"<Image><image></Image>\"))\n                messages.insert(1, (self.roles[1], \"Received.\"))\n            else:\n                messages[0] = (init_role, \"<image>\\n\" + init_msg)\n\n        if self.sep_style == SeparatorStyle.SINGLE:\n            ret = self.system + self.sep\n            for role, message in messages:\n                if message:\n                    if type(message) is tuple:\n                        message, _, _ = message\n                    ret += role + \": \" + message + self.sep\n                else:\n                    ret += role + \":\"\n        # elif self.sep_style == SeparatorStyle.QWEN_2:  # fix: add qwen2\n        #     seps = [self.sep, self.sep2]\n        #     ret = self.system + seps[0]\n        #     ret = \"\"\n        #     for i, (role, message) in enumerate(messages):\n        #         if message:\n        #             if type(message) is tuple:\n        #                 message, _, _ = message\n        #             ret += role + \": \" + message + seps[i % 2]\n        #         else:\n        #             ret += role + \":\"\n        elif self.sep_style == SeparatorStyle.QWEN_2:  # fix: add qwen2\n            ret = self.system + self.sep\n            for i, (role, message) in enumerate(messages):\n                if message:\n                    if type(message) is tuple:\n                        message, _, _ = message\n                    ret += role + message + self.sep\n                else:\n                    ret += role\n        elif self.sep_style == SeparatorStyle.CHATML:\n            ret = \"\" if self.system == \"\" else self.system + self.sep + \"\\n\"\n            for role, message in messages:\n                if message:\n                    if type(message) is tuple:\n                        message, images = message\n                        message = \"<image>\" * len(images) + message\n                    ret += role + \"\\n\" + message + self.sep + \"\\n\"\n                else:\n                    ret += role + \"\\n\"\n            return ret\n        elif self.sep_style == SeparatorStyle.TWO:\n            seps = [self.sep, self.sep2]\n            ret = self.system + seps[0]\n            for i, (role, message) in enumerate(messages):\n                if message:\n                    if type(message) is tuple:\n                        message, _, _ = message\n                    ret += role + \": \" + message + seps[i % 2]\n                else:\n                    ret += role + \":\"\n        elif self.sep_style == SeparatorStyle.MPT:\n            ret = self.system + self.sep\n            for role, message in messages:\n                if message:\n                    if type(message) is tuple:\n                        message, _, _ = message\n                    ret += role + message + self.sep\n                else:\n                    ret += role\n        elif self.sep_style == SeparatorStyle.LLAMA_2:\n            def wrap_sys(msg): return f\"<<SYS>>\\n{msg}\\n<</SYS>>\\n\\n\" if len(msg) > 0 else msg\n            def wrap_inst(msg): return f\"[INST] {msg} [/INST]\"\n            ret = \"\"\n\n            for i, (role, message) in enumerate(messages):\n                if i == 0:\n                    assert message, \"first message should not be none\"\n                    assert role == self.roles[0], \"first message should come from user\"\n                if message:\n                    if type(message) is tuple:\n                        message, _, _ = message\n                    if i == 0:\n                        message = wrap_sys(self.system) + message\n                    if i % 2 == 0:\n                        message = wrap_inst(message)\n                        ret += self.sep + message\n                    else:\n                        ret += \" \" + message + \" \" + self.sep2\n                else:\n                    ret += \"\"\n            ret = ret.lstrip(self.sep)\n        elif self.sep_style == SeparatorStyle.PLAIN:\n            seps = [self.sep, self.sep2]\n            ret = self.system\n            for i, (role, message) in enumerate(messages):\n                if message:\n                    if type(message) is tuple:\n                        message, _, _ = message\n                    ret += message + seps[i % 2]\n                else:\n                    ret += \"\"\n        else:\n            raise ValueError(f\"Invalid style: {self.sep_style}\")\n\n        return ret\n\n    def append_message(self, role, message):\n        self.messages.append([role, message])\n\n    def process_image(self, image, image_process_mode, return_pil=False, image_format='PNG', max_len=1344, min_len=672):\n        if image_process_mode == \"Pad\":\n            def expand2square(pil_img, background_color=(122, 116, 104)):\n                width, height = pil_img.size\n                if width == height:\n                    return pil_img\n                elif width > height:\n                    result = Image.new(pil_img.mode, (width, width), background_color)\n                    result.paste(pil_img, (0, (width - height) // 2))\n                    return result\n                else:\n                    result = Image.new(pil_img.mode, (height, height), background_color)\n                    result.paste(pil_img, ((height - width) // 2, 0))\n                    return result\n            image = expand2square(image)\n        elif image_process_mode in [\"Default\", \"Crop\"]:\n            pass\n        elif image_process_mode == \"Resize\":\n            image = image.resize((336, 336))\n        else:\n            raise ValueError(f\"Invalid image_process_mode: {image_process_mode}\")\n        if max(image.size) > max_len:\n            max_hw, min_hw = max(image.size), min(image.size)\n            aspect_ratio = max_hw / min_hw\n            shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))\n            longest_edge = int(shortest_edge * aspect_ratio)\n            W, H = image.size\n            if H > W:\n                H, W = longest_edge, shortest_edge\n            else:\n                H, W = shortest_edge, longest_edge\n            image = image.resize((W, H))\n        if return_pil:\n            return image\n        else:\n            buffered = BytesIO()\n            image.save(buffered, format=image_format)\n            img_b64_str = base64.b64encode(buffered.getvalue()).decode()\n            return img_b64_str\n\n    def get_images(self, return_pil=False):\n        images = []\n        for i, (role, msg) in enumerate(self.messages[self.offset:]):\n            if i % 2 == 0:\n                if type(msg) is tuple:\n                    msg, image, image_process_mode = msg\n                    image = self.process_image(image, image_process_mode, return_pil=return_pil)\n                    images.append(image)\n        return images\n\n    def to_gradio_chatbot(self):\n        ret = []\n        for i, (role, msg) in enumerate(self.messages[self.offset:]):\n            if i % 2 == 0:\n                if type(msg) is tuple:\n                    msg, image, image_process_mode = msg\n                    img_b64_str = self.process_image(\n                        image, \"Default\", return_pil=False,\n                        image_format='JPEG')\n                    img_str = f'<img src=\"data:image/jpeg;base64,{img_b64_str}\" alt=\"user upload image\" />'\n                    msg = img_str + msg.replace('<image>', '').strip()\n                    ret.append([msg, None])\n                else:\n                    ret.append([msg, None])\n            else:\n                ret[-1][-1] = msg\n        return ret\n\n    def copy(self):\n        return Conversation(\n            system=self.system,\n            roles=self.roles,\n            messages=[[x, y] for x, y in self.messages],\n            offset=self.offset,\n            sep_style=self.sep_style,\n            sep=self.sep,\n            sep2=self.sep2,\n            version=self.version)\n\n    def dict(self):\n        if len(self.get_images()) > 0:\n            return {\n                \"system\": self.system,\n                \"roles\": self.roles,\n                \"messages\": [[x, y[0] if type(y) is tuple else y] for x, y in self.messages],\n                \"offset\": self.offset,\n                \"sep\": self.sep,\n                \"sep2\": self.sep2,\n            }\n        return {\n            \"system\": self.system,\n            \"roles\": self.roles,\n            \"messages\": self.messages,\n            \"offset\": self.offset,\n            \"sep\": self.sep,\n            \"sep2\": self.sep2,\n        }\n\n\nconv_vicuna_v0 = Conversation(\n    system=\"A chat between a curious human and an artificial intelligence assistant. \"\n           \"The assistant gives helpful, detailed, and polite answers to the human's questions.\",\n    roles=(\"Human\", \"Assistant\"),\n    messages=(\n        (\"Human\", \"What are the key differences between renewable and non-renewable energy sources?\"),\n        (\"Assistant\",\n            \"Renewable energy sources are those that can be replenished naturally in a relatively \"\n            \"short amount of time, such as solar, wind, hydro, geothermal, and biomass. \"\n            \"Non-renewable energy sources, on the other hand, are finite and will eventually be \"\n            \"depleted, such as coal, oil, and natural gas. Here are some key differences between \"\n            \"renewable and non-renewable energy sources:\\n\"\n            \"1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable \"\n            \"energy sources are finite and will eventually run out.\\n\"\n            \"2. Environmental impact: Renewable energy sources have a much lower environmental impact \"\n            \"than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, \"\n            \"and other negative effects.\\n\"\n            \"3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically \"\n            \"have lower operational costs than non-renewable sources.\\n\"\n            \"4. Reliability: Renewable energy sources are often more reliable and can be used in more remote \"\n            \"locations than non-renewable sources.\\n\"\n            \"5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different \"\n            \"situations and needs, while non-renewable sources are more rigid and inflexible.\\n\"\n            \"6. Sustainability: Renewable energy sources are more sustainable over the long term, while \"\n            \"non-renewable sources are not, and their depletion can lead to economic and social instability.\\n\")\n    ),\n    offset=2,\n    sep_style=SeparatorStyle.SINGLE,\n    sep=\"###\",\n)\n\nconv_vicuna_v1 = Conversation(\n    system=\"A chat between a curious user and an artificial intelligence assistant. \"\n    \"The assistant gives helpful, detailed, and polite answers to the user's questions.\",\n    roles=(\"USER\", \"ASSISTANT\"),\n    version=\"v1\",\n    messages=(),\n    offset=0,\n    sep_style=SeparatorStyle.TWO,\n    sep=\" \",\n    sep2=\"</s>\",\n)\n\nconv_llama_2 = Conversation(\n    system=\"\"\"You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.\"\"\",\n    roles=(\"USER\", \"ASSISTANT\"),\n    version=\"llama_v2\",\n    messages=(),\n    offset=0,\n    sep_style=SeparatorStyle.LLAMA_2,\n    sep=\"<s>\",\n    sep2=\"</s>\",\n)\n\nconv_llava_llama_2 = Conversation(\n    system=\"You are a helpful language and vision assistant. \"\n           \"You are able to understand the visual content that the user provides, \"\n           \"and assist the user with a variety of tasks using natural language.\",\n    roles=(\"USER\", \"ASSISTANT\"),\n    version=\"llama_v2\",\n    messages=(),\n    offset=0,\n    sep_style=SeparatorStyle.LLAMA_2,\n    sep=\"<s>\",\n    sep2=\"</s>\",\n)\n\nconv_mpt = Conversation(\n    system=\"\"\"<|im_start|>system\nA conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.\"\"\",\n    roles=(\"<|im_start|>user\\n\", \"<|im_start|>assistant\\n\"),\n    version=\"mpt\",\n    messages=(),\n    offset=0,\n    sep_style=SeparatorStyle.MPT,\n    sep=\"<|im_end|>\",\n)\n\nconv_llava_plain = Conversation(\n    system=\"\",\n    roles=(\"\", \"\"),\n    messages=(\n    ),\n    offset=0,\n    sep_style=SeparatorStyle.PLAIN,\n    sep=\"\\n\",\n)\n\nconv_llava_v0 = Conversation(\n    system=\"A chat between a curious human and an artificial intelligence assistant. \"\n           \"The assistant gives helpful, detailed, and polite answers to the human's questions.\",\n    roles=(\"Human\", \"Assistant\"),\n    messages=(\n    ),\n    offset=0,\n    sep_style=SeparatorStyle.SINGLE,\n    sep=\"###\",\n)\n\nconv_llava_v0_mmtag = Conversation(\n    system=\"A chat between a curious user and an artificial intelligence assistant. \"\n           \"The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.\"\n           \"The visual content will be provided with the following format: <Image>visual content</Image>.\",\n    roles=(\"Human\", \"Assistant\"),\n    messages=(\n    ),\n    offset=0,\n    sep_style=SeparatorStyle.SINGLE,\n    sep=\"###\",\n    version=\"v0_mmtag\",\n)\n\nconv_llava_v1 = Conversation(\n    system=\"A chat between a curious human and an artificial intelligence assistant. \"\n           \"The assistant gives helpful, detailed, and polite answers to the human's questions.\",\n    roles=(\"USER\", \"ASSISTANT\"),\n    version=\"v1\",\n    messages=(),\n    offset=0,\n    sep_style=SeparatorStyle.TWO,\n    sep=\" \",\n    sep2=\"</s>\",\n)\n\nconv_llava_v1_mmtag = Conversation(\n    system=\"A chat between a curious user and an artificial intelligence assistant. \"\n           \"The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.\"\n           \"The visual content will be provided with the following format: <Image>visual content</Image>.\",\n    roles=(\"USER\", \"ASSISTANT\"),\n    messages=(),\n    offset=0,\n    sep_style=SeparatorStyle.TWO,\n    sep=\" \",\n    sep2=\"</s>\",\n    version=\"v1_mmtag\",\n)\n\nconv_mistral_instruct = Conversation(\n    system=\"\",\n    roles=(\"USER\", \"ASSISTANT\"),\n    version=\"llama_v2\",\n    messages=(),\n    offset=0,\n    sep_style=SeparatorStyle.LLAMA_2,\n    sep=\"\",\n    sep2=\"</s>\",\n)\n\nconv_chatml_direct = Conversation(\n    system=\"\"\"<|im_start|>system\nAnswer the questions.\"\"\",\n    roles=(\"<|im_start|>user\\n\", \"<|im_start|>assistant\\n\"),\n    version=\"mpt\",\n    messages=(),\n    offset=0,\n    sep_style=SeparatorStyle.MPT,\n    sep=\"<|im_end|>\",\n)\n\n\nconv_qwen_2 = Conversation(\n    system=\"<|im_start|>system\\nYou are a helpful assistant.\",\n    roles=(\"<|im_start|>user\\n\", \"<|im_start|>assistant\\n\"),\n    version=\"qwen_v2\",\n    messages=(),\n    offset=0,\n    sep_style=SeparatorStyle.QWEN_2,\n    sep=\"<|im_end|>\\n\",\n)\n\n\n# conv_qwen_2 = Conversation(\n#     system=\"\",\n#     roles=(\"user\", \"assistant\"),\n#     version=\"qwen_v2\",\n#     messages=(),\n#     offset=0,\n#     sep_style=SeparatorStyle.QWEN_2,\n#     sep=\" \",\n#     sep2=\"<|im_end|>\",\n# )\n\n\n# fix: add qwen2\n# conv_qwen_2 = Conversation(\n#     system=\"A chat between a curious user and an artificial intelligence assistant. \"\n#     \"The assistant gives helpful, detailed, and polite answers to the user's questions.\",\n#     roles=(\"USER\", \"ASSISTANT\"),\n#     version=\"qwen_v2\",\n#     messages=(),\n#     offset=0,\n#     sep_style=SeparatorStyle.QWEN_2,\n#     sep=\" \",\n#     sep2=\"<|endoftext|>\",\n# )\n\n# conv_qwen_2 = Conversation(\n#     system=\"\"\"<|im_start|>system\n# You are a helpful assistant.\"\"\",\n#     roles=(\"<|im_start|>user\", \"<|im_start|>assistant\"),\n#     version=\"qwen_v2\",\n#     messages=[],\n#     offset=0,\n#     sep_style=SeparatorStyle.QWEN_2,\n#     sep=\"<|im_end|>\",\n#     sep2=\"<|im_end|>\",\n# )\n\ndefault_conversation = conv_qwen_2\nconv_templates = {\n    \"default\": conv_qwen_2,\n    \"v0\": conv_vicuna_v0,\n    \"v1\": conv_vicuna_v1,\n    \"vicuna_v1\": conv_vicuna_v1,\n    \"qwen_2\": conv_qwen_2,\n    \"llama_2\": conv_llama_2,\n    \"mistral_instruct\": conv_mistral_instruct,\n    \"chatml_direct\": conv_chatml_direct,\n    \"mistral_direct\": conv_chatml_direct,\n\n    \"plain\": conv_llava_plain,\n    \"v0_plain\": conv_llava_plain,\n    \"llava_v0\": conv_llava_v0,\n    \"v0_mmtag\": conv_llava_v0_mmtag,\n    \"llava_v1\": conv_llava_v1,\n    \"v1_mmtag\": conv_llava_v1_mmtag,\n    \"llava_llama_2\": conv_llava_llama_2,\n\n    \"mpt\": conv_mpt,\n}\n\nif __name__ == \"__main__\":\n    print(\"conversation:\", default_conversation.get_prompt())\n"
  },
  {
    "path": "llava/mm_utils.py",
    "content": "import PIL\nfrom PIL import Image\nPIL.Image.MAX_IMAGE_PIXELS=500000000\nfrom io import BytesIO\nimport base64\nimport torch\nimport math\nimport ast\n\nfrom transformers import StoppingCriteria\nfrom llava.constants import IMAGE_TOKEN_INDEX\n\n\ndef select_best_resolution(original_size, possible_resolutions):\n    \"\"\"\n    Selects the best resolution from a list of possible resolutions based on the original size.\n\n    Args:\n        original_size (tuple): The original size of the image in the format (width, height).\n        possible_resolutions (list): A list of possible resolutions in the format [(width1, height1), (width2, height2), ...].\n\n    Returns:\n        tuple: The best fit resolution in the format (width, height).\n    \"\"\"\n    original_width, original_height = original_size\n    best_fit = None\n    max_effective_resolution = 0\n    min_wasted_resolution = float('inf')\n\n    for width, height in possible_resolutions:\n        scale = min(width / original_width, height / original_height)\n        downscaled_width, downscaled_height = int(original_width * scale), int(original_height * scale)\n        effective_resolution = min(downscaled_width * downscaled_height, original_width * original_height)\n        wasted_resolution = (width * height) - effective_resolution\n\n        if effective_resolution > max_effective_resolution or (effective_resolution == max_effective_resolution and wasted_resolution < min_wasted_resolution):\n            max_effective_resolution = effective_resolution\n            min_wasted_resolution = wasted_resolution\n            best_fit = (width, height)\n\n    return best_fit\n\n\ndef resize_and_pad_image(image, target_resolution):\n    \"\"\"\n    Resize and pad an image to a target resolution while maintaining aspect ratio.\n\n    Args:\n        image (PIL.Image.Image): The input image.\n        target_resolution (tuple): The target resolution (width, height) of the image.\n\n    Returns:\n        PIL.Image.Image: The resized and padded image.\n    \"\"\"\n    original_width, original_height = image.size\n    target_width, target_height = target_resolution\n\n    scale_w = target_width / original_width\n    scale_h = target_height / original_height\n\n    if scale_w < scale_h:\n        new_width = target_width\n        new_height = min(math.ceil(original_height * scale_w), target_height)\n    else:\n        new_height = target_height\n        new_width = min(math.ceil(original_width * scale_h), target_width)\n\n    # Resize the image\n    resized_image = image.resize((new_width, new_height))\n\n    new_image = Image.new('RGB', (target_width, target_height), (0, 0, 0))\n    paste_x = (target_width - new_width) // 2\n    paste_y = (target_height - new_height) // 2\n    new_image.paste(resized_image, (paste_x, paste_y))\n\n    return new_image\n\n\ndef divide_to_patches(image, patch_size):\n    \"\"\"\n    Divides an image into patches of a specified size.\n\n    Args:\n        image (PIL.Image.Image): The input image.\n        patch_size (int): The size of each patch.\n\n    Returns:\n        list: A list of PIL.Image.Image objects representing the patches.\n    \"\"\"\n    patches = []\n    width, height = image.size\n    for i in range(0, height, patch_size):\n        for j in range(0, width, patch_size):\n            box = (j, i, j + patch_size, i + patch_size)\n            patch = image.crop(box)\n            patches.append(patch)\n\n    return patches\n\n\ndef get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size):\n    \"\"\"\n    Calculate the shape of the image patch grid after the preprocessing for images of any resolution.\n\n    Args:\n        image_size (tuple): The size of the input image in the format (width, height).\n        grid_pinpoints (str): A string representation of a list of possible resolutions.\n        patch_size (int): The size of each image patch.\n\n    Returns:\n        tuple: The shape of the image patch grid in the format (width, height).\n    \"\"\"\n    if type(grid_pinpoints) is list:\n        possible_resolutions = grid_pinpoints\n    else:\n        possible_resolutions = ast.literal_eval(grid_pinpoints)\n    width, height = select_best_resolution(image_size, possible_resolutions)\n    return width // patch_size, height // patch_size\n\n\ndef process_anyres_image(image, processor, grid_pinpoints):\n    \"\"\"\n    Process an image with variable resolutions.\n\n    Args:\n        image (PIL.Image.Image): The input image to be processed.\n        processor: The image processor object.\n        grid_pinpoints (str): A string representation of a list of possible resolutions.\n\n    Returns:\n        torch.Tensor: A tensor containing the processed image patches.\n    \"\"\"\n    if type(grid_pinpoints) is list:\n        possible_resolutions = grid_pinpoints\n    else:\n        possible_resolutions = ast.literal_eval(grid_pinpoints)\n    best_resolution = select_best_resolution(image.size, possible_resolutions)\n    image_padded = resize_and_pad_image(image, best_resolution)\n\n    patches = divide_to_patches(image_padded, processor.crop_size['height'])\n\n    image_original_resize = image.resize((processor.size['shortest_edge'], processor.size['shortest_edge']))\n\n    image_patches = [image_original_resize] + patches\n    image_patches = [processor.preprocess(image_patch, return_tensors='pt')['pixel_values'][0]\n                     for image_patch in image_patches]\n    return torch.stack(image_patches, dim=0)\n\n\ndef load_image_from_base64(image):\n    return Image.open(BytesIO(base64.b64decode(image)))\n\n\ndef expand2square(pil_img, background_color):\n    width, height = pil_img.size\n    if width == height:\n        return pil_img\n    elif width > height:\n        result = Image.new(pil_img.mode, (width, width), background_color)\n        result.paste(pil_img, (0, (width - height) // 2))\n        return result\n    else:\n        result = Image.new(pil_img.mode, (height, height), background_color)\n        result.paste(pil_img, ((height - width) // 2, 0))\n        return result\n\n\ndef process_images(images, image_processor, model_cfg):\n    image_aspect_ratio = getattr(model_cfg, \"image_aspect_ratio\", None)\n    new_images = []\n    if image_aspect_ratio == 'pad':\n        for image in images:\n            image = expand2square(image, tuple(int(x*255) for x in image_processor.image_mean))\n            image = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]\n            new_images.append(image)\n    elif image_aspect_ratio == \"anyres\":\n        for image in images:\n            image = process_anyres_image(image, image_processor, model_cfg.image_grid_pinpoints)\n            new_images.append(image)\n    else:\n        return image_processor(images, return_tensors='pt')['pixel_values']\n    if all(x.shape == new_images[0].shape for x in new_images):\n        new_images = torch.stack(new_images, dim=0)\n    return new_images\n\n\ndef tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):\n    prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]\n\n    def insert_separator(X, sep):\n        return [ele for sublist in zip(X, [sep]*len(X)) for ele in sublist][:-1]\n\n    input_ids = []\n    offset = 0\n    if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id:\n        offset = 1\n        input_ids.append(prompt_chunks[0][0])\n\n    for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)):\n        input_ids.extend(x[offset:])\n\n    if return_tensors is not None:\n        if return_tensors == 'pt':\n            return torch.tensor(input_ids, dtype=torch.long)\n        raise ValueError(f'Unsupported tensor type: {return_tensors}')\n    return input_ids\n\n\ndef get_model_name_from_path(model_path):\n    model_path = model_path.strip(\"/\")\n    model_paths = model_path.split(\"/\")\n    if model_paths[-1].startswith('checkpoint-'):\n        return model_paths[-2] + \"_\" + model_paths[-1]\n    else:\n        return model_paths[-1]\n\n\nclass KeywordsStoppingCriteria(StoppingCriteria):\n    def __init__(self, keywords, tokenizer, input_ids):\n        self.keywords = keywords\n        self.keyword_ids = []\n        self.max_keyword_len = 0\n        for keyword in keywords:\n            cur_keyword_ids = tokenizer(keyword).input_ids\n            if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id:\n                cur_keyword_ids = cur_keyword_ids[1:]\n            if len(cur_keyword_ids) > self.max_keyword_len:\n                self.max_keyword_len = len(cur_keyword_ids)\n            self.keyword_ids.append(torch.tensor(cur_keyword_ids))\n        self.tokenizer = tokenizer\n        self.start_len = input_ids.shape[1]\n\n    def call_for_batch(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:\n        offset = min(output_ids.shape[1] - self.start_len, self.max_keyword_len)\n        self.keyword_ids = [keyword_id.to(output_ids.device) for keyword_id in self.keyword_ids]\n        for keyword_id in self.keyword_ids:\n            truncated_output_ids = output_ids[0, -keyword_id.shape[0]:]\n            if torch.equal(truncated_output_ids, keyword_id):\n                return True\n        outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0]\n        for keyword in self.keywords:\n            if keyword in outputs:\n                return True\n        return False\n\n    def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:\n        outputs = []\n        for i in range(output_ids.shape[0]):\n            outputs.append(self.call_for_batch(output_ids[i].unsqueeze(0), scores))\n        return all(outputs)\n"
  },
  {
    "path": "llava/model/__init__.py",
    "content": "# try:\nfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig\nfrom .language_model.llava_mpt import LlavaMptForCausalLM, LlavaMptConfig\nfrom .language_model.llava_mistral import LlavaMistralForCausalLM, LlavaMistralConfig\nfrom .language_model.llava_qwen import LlavaQwen2ForCausalLM, LlavaConfig\n# except:\n#     pass\n\n"
  },
  {
    "path": "llava/model/apply_delta.py",
    "content": "\"\"\"\nUsage:\npython3 -m fastchat.model.apply_delta --base ~/model_weights/llama-7b --target ~/model_weights/vicuna-7b --delta lmsys/vicuna-7b-delta\n\"\"\"\nimport argparse\n\nimport torch\nfrom tqdm import tqdm\nfrom transformers import AutoTokenizer, AutoModelForCausalLM\nfrom llava import LlavaLlamaForCausalLM\n\n\ndef apply_delta(base_model_path, target_model_path, delta_path):\n    print(\"Loading base model\")\n    base = AutoModelForCausalLM.from_pretrained(\n        base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)\n\n    print(\"Loading delta\")\n    delta = LlavaLlamaForCausalLM.from_pretrained(delta_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)\n    delta_tokenizer = AutoTokenizer.from_pretrained(delta_path)\n\n    print(\"Applying delta\")\n    for name, param in tqdm(delta.state_dict().items(), desc=\"Applying delta\"):\n        if name not in base.state_dict():\n            assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model'\n            continue\n        if param.data.shape == base.state_dict()[name].shape:\n            param.data += base.state_dict()[name]\n        else:\n            assert name in ['model.embed_tokens.weight', 'lm_head.weight'], \\\n                f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}'\n            bparam = base.state_dict()[name]\n            param.data[:bparam.shape[0], :bparam.shape[1]] += bparam\n\n    print(\"Saving target model\")\n    delta.save_pretrained(target_model_path)\n    delta_tokenizer.save_pretrained(target_model_path)\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--base-model-path\", type=str, required=True)\n    parser.add_argument(\"--target-model-path\", type=str, required=True)\n    parser.add_argument(\"--delta-path\", type=str, required=True)\n\n    args = parser.parse_args()\n\n    apply_delta(args.base_model_path, args.target_model_path, args.delta_path)\n"
  },
  {
    "path": "llava/model/builder.py",
    "content": "#    Copyright 2023 Haotian Liu\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n\n\nimport os\nimport warnings\nimport shutil\n\nfrom transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BitsAndBytesConfig\nimport torch\nfrom llava.model import *\nfrom llava.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN\n\n\ndef load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map=\"auto\", device=\"cuda\", use_flash_attn=False, **kwargs):\n    kwargs = {\"device_map\": device_map, **kwargs}\n\n    if device != \"cuda\":\n        kwargs['device_map'] = {\"\": device}\n\n    if load_8bit:\n        kwargs['load_in_8bit'] = True\n    elif load_4bit:\n        kwargs['load_in_4bit'] = True\n        kwargs['quantization_config'] = BitsAndBytesConfig(\n            load_in_4bit=True,\n            bnb_4bit_compute_dtype=torch.float16,\n            bnb_4bit_use_double_quant=True,\n            bnb_4bit_quant_type='nf4'\n        )\n    else:\n        kwargs['torch_dtype'] = torch.float16\n\n    if use_flash_attn:\n        kwargs['attn_implementation'] = 'flash_attention_2'\n\n    if 'llava' in model_name.lower():\n        # Load LLaVA model\n        if 'lora' in model_name.lower() and model_base is None:\n            warnings.warn('There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument. Detailed instruction: https://github.com/haotian-liu/LLaVA#launch-a-model-worker-lora-weights-unmerged.')\n        if 'lora' in model_name.lower() and model_base is not None:\n            from llava.model.language_model.llava_llama import LlavaConfig\n            lora_cfg_pretrained = LlavaConfig.from_pretrained(model_path)\n            tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)\n            print('Loading LLaVA from base model...')\n            model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=lora_cfg_pretrained, **kwargs)\n            token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features\n            if model.lm_head.weight.shape[0] != token_num:\n                model.lm_head.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))\n                model.model.embed_tokens.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))\n\n            print('Loading additional LLaVA weights...')\n            if os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')):\n                non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'), map_location='cpu')\n            else:\n                # this is probably from HF Hub\n                from huggingface_hub import hf_hub_download\n\n                def load_from_hf(repo_id, filename, subfolder=None):\n                    cache_file = hf_hub_download(\n                        repo_id=repo_id,\n                        filename=filename,\n                        subfolder=subfolder)\n                    return torch.load(cache_file, map_location='cpu')\n                non_lora_trainables = load_from_hf(model_path, 'non_lora_trainables.bin')\n            non_lora_trainables = {(k[11:] if k.startswith('base_model.') else k): v for k, v in non_lora_trainables.items()}\n            if any(k.startswith('model.model.') for k in non_lora_trainables):\n                non_lora_trainables = {(k[6:] if k.startswith('model.') else k): v for k, v in non_lora_trainables.items()}\n            model.load_state_dict(non_lora_trainables, strict=False)\n\n            from peft import PeftModel\n            print('Loading LoRA weights...')\n            model = PeftModel.from_pretrained(model, model_path)\n            print('Merging LoRA weights...')\n            model = model.merge_and_unload()\n            print('Model is loaded...')\n        elif model_base is not None:\n            # this may be mm projector only\n            print('Loading LLaVA from base model...')\n            if 'mpt' in model_name.lower():\n                if not os.path.isfile(os.path.join(model_path, 'configuration_mpt.py')):\n                    shutil.copyfile(os.path.join(model_base, 'configuration_mpt.py'), os.path.join(model_path, 'configuration_mpt.py'))\n                tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=True)\n                cfg_pretrained = AutoConfig.from_pretrained(model_path, trust_remote_code=True)\n                model = LlavaMptForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)\n            else:\n                tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)\n                cfg_pretrained = AutoConfig.from_pretrained(model_path)\n                # model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)\n                model = LlavaQwen2ForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)\n\n            mm_projector_weights = torch.load(os.path.join(model_path, 'mm_projector.bin'), map_location='cpu')\n            mm_projector_weights = {k: v.to(torch.float16) for k, v in mm_projector_weights.items()}\n            model.load_state_dict(mm_projector_weights, strict=False)\n        else:\n            if 'mpt' in model_name.lower():\n                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)\n                model = LlavaMptForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)\n            elif 'mistral' in model_name.lower():\n                tokenizer = AutoTokenizer.from_pretrained(model_path)\n                model = LlavaMistralForCausalLM.from_pretrained(\n                    model_path,\n                    low_cpu_mem_usage=True,\n                    **kwargs\n                )\n            elif 'dclm' in model_name.lower():\n                tokenizer = AutoTokenizer.from_pretrained(model_path)\n                model = LlavaOpenlmForCausalLM.from_pretrained(\n                    model_path,\n                    low_cpu_mem_usage=True,\n                    **kwargs\n                )\n            else:\n                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)\n                # model = LlavaLlamaForCausalLM.from_pretrained(\n                #     model_path,\n                #     low_cpu_mem_usage=True,\n                #     **kwargs\n                # )\n                model = LlavaQwen2ForCausalLM.from_pretrained(\n                    model_path,\n                    low_cpu_mem_usage=True,\n                    **kwargs\n                )\n    else:\n        # Load language model\n        if model_base is not None:\n            # PEFT model\n            from peft import PeftModel\n            tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)\n            model = AutoModelForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, **kwargs)\n            print(f\"Loading LoRA weights from {model_path}\")\n            model = PeftModel.from_pretrained(model, model_path)\n            print(f\"Merging weights\")\n            model = model.merge_and_unload()\n            print('Convert to FP16...')\n            model.to(torch.float16)\n        else:\n            use_fast = False\n            if 'mpt' in model_name.lower():\n                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)\n                model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, trust_remote_code=True, **kwargs)\n            else:\n                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)\n                model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)\n\n    image_processor = None\n\n    if 'llava' in model_name.lower():\n        mm_use_im_start_end = getattr(model.config, \"mm_use_im_start_end\", False)\n        mm_use_im_patch_token = getattr(model.config, \"mm_use_im_patch_token\", True)\n        if mm_use_im_patch_token:\n            tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)\n        if mm_use_im_start_end:\n            tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)\n        model.resize_token_embeddings(len(tokenizer))\n\n        vision_tower = model.get_vision_tower()\n        if not vision_tower.is_loaded:\n            vision_tower.load_model(device_map=device_map)\n        if device_map != 'auto':\n            vision_tower.to(device=device_map, dtype=torch.float16)\n        image_processor = vision_tower.image_processor\n\n    if hasattr(model.config, \"max_sequence_length\"):\n        context_len = model.config.max_sequence_length\n    else:\n        context_len = 2048\n\n    return tokenizer, model, image_processor, context_len\n"
  },
  {
    "path": "llava/model/consolidate.py",
    "content": "\"\"\"\nUsage:\npython3 -m llava.model.consolidate --src ~/model_weights/llava-7b --dst ~/model_weights/llava-7b_consolidate\n\"\"\"\nimport argparse\n\nimport torch\nfrom transformers import AutoTokenizer, AutoModelForCausalLM\nfrom llava.model import *\nfrom llava.model.utils import auto_upgrade\n\n\ndef consolidate_ckpt(src_path, dst_path):\n    print(\"Loading model\")\n    auto_upgrade(src_path)\n    src_model = AutoModelForCausalLM.from_pretrained(src_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)\n    src_tokenizer = AutoTokenizer.from_pretrained(src_path, use_fast=False)\n    src_model.save_pretrained(dst_path)\n    src_tokenizer.save_pretrained(dst_path)\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--src\", type=str, required=True)\n    parser.add_argument(\"--dst\", type=str, required=True)\n\n    args = parser.parse_args()\n\n    consolidate_ckpt(args.src, args.dst)\n"
  },
  {
    "path": "llava/model/language_model/llava_llama.py",
    "content": "#    Copyright 2023 Haotian Liu\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n\n\nfrom typing import List, Optional, Tuple, Union\n\nimport torch\nimport torch.nn as nn\n\nfrom transformers import AutoConfig, AutoModelForCausalLM, \\\n                         LlamaConfig, LlamaModel, LlamaForCausalLM\n\nfrom transformers.modeling_outputs import CausalLMOutputWithPast\nfrom transformers.generation.utils import GenerateOutput\n\nfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM\n\n\nclass LlavaConfig(LlamaConfig):\n    model_type = \"llava_llama\"\n\n\nclass LlavaLlamaModel(LlavaMetaModel, LlamaModel):\n    config_class = LlavaConfig\n\n    def __init__(self, config: LlamaConfig):\n        super(LlavaLlamaModel, self).__init__(config)\n\n\nclass LlavaLlamaForCausalLM(LlamaForCausalLM, LlavaMetaForCausalLM):\n    config_class = LlavaConfig\n\n    def __init__(self, config):\n        super(LlamaForCausalLM, self).__init__(config)\n        self.model = LlavaLlamaModel(config)\n        self.pretraining_tp = config.pretraining_tp\n        self.vocab_size = config.vocab_size\n        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n\n        # Initialize weights and apply final processing\n        self.post_init()\n\n    def get_model(self):\n        return self.model\n\n    def forward(\n        self,\n        input_ids: torch.LongTensor = None,\n        attention_mask: Optional[torch.Tensor] = None,\n        position_ids: Optional[torch.LongTensor] = None,\n        past_key_values: Optional[List[torch.FloatTensor]] = None,\n        inputs_embeds: Optional[torch.FloatTensor] = None,\n        labels: Optional[torch.LongTensor] = None,\n        use_cache: Optional[bool] = None,\n        output_attentions: Optional[bool] = None,\n        output_hidden_states: Optional[bool] = None,\n        images: Optional[torch.FloatTensor] = None,\n        image_sizes: Optional[List[List[int]]] = None,\n        return_dict: Optional[bool] = None,\n        cache_position=None,\n    ) -> Union[Tuple, CausalLMOutputWithPast]:\n\n        if inputs_embeds is None:\n            (\n                input_ids,\n                position_ids,\n                attention_mask,\n                past_key_values,\n                inputs_embeds,\n                labels\n            ) = self.prepare_inputs_labels_for_multimodal(\n                input_ids,\n                position_ids,\n                attention_mask,\n                past_key_values,\n                labels,\n                images,\n                image_sizes\n            )\n\n        return super().forward(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            position_ids=position_ids,\n            past_key_values=past_key_values,\n            inputs_embeds=inputs_embeds,\n            labels=labels,\n            use_cache=use_cache,\n            output_attentions=output_attentions,\n            output_hidden_states=output_hidden_states,\n            return_dict=return_dict\n        )\n\n    @torch.no_grad()\n    def generate(\n        self,\n        inputs: Optional[torch.Tensor] = None,\n        images: Optional[torch.Tensor] = None,\n        image_sizes: Optional[torch.Tensor] = None,\n        **kwargs,\n    ) -> Union[GenerateOutput, torch.LongTensor]:\n        position_ids = kwargs.pop(\"position_ids\", None)\n        attention_mask = kwargs.pop(\"attention_mask\", None)\n        if \"inputs_embeds\" in kwargs:\n            raise NotImplementedError(\"`inputs_embeds` is not supported\")\n\n        if images is not None:\n            (\n                inputs,\n                position_ids,\n                attention_mask,\n                _,\n                inputs_embeds,\n                _\n            ) = self.prepare_inputs_labels_for_multimodal(\n                inputs,\n                position_ids,\n                attention_mask,\n                None,\n                None,\n                images,\n                image_sizes=image_sizes\n            )\n        else:\n            inputs_embeds = self.get_model().embed_tokens(inputs)\n\n        return super().generate(\n            position_ids=position_ids,\n            attention_mask=attention_mask,\n            inputs_embeds=inputs_embeds,\n            **kwargs\n        )\n\n    def prepare_inputs_for_generation(self, input_ids, past_key_values=None,\n                                      inputs_embeds=None, **kwargs):\n        images = kwargs.pop(\"images\", None)\n        image_sizes = kwargs.pop(\"image_sizes\", None)\n        inputs = super().prepare_inputs_for_generation(\n            input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs\n        )\n        if images is not None:\n            inputs['images'] = images\n        if image_sizes is not None:\n            inputs['image_sizes'] = image_sizes\n        return inputs\n\nAutoConfig.register(\"llava_llama\", LlavaConfig)\nAutoModelForCausalLM.register(LlavaConfig, LlavaLlamaForCausalLM)\n"
  },
  {
    "path": "llava/model/language_model/llava_mistral.py",
    "content": "#    Copyright 2023 Haotian Liu\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n\n\nfrom typing import List, Optional, Tuple, Union\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn import CrossEntropyLoss\n\nfrom transformers import AutoConfig, AutoModelForCausalLM, \\\n                         MistralConfig, MistralModel, MistralForCausalLM\n\nfrom transformers.modeling_outputs import CausalLMOutputWithPast\nfrom transformers.generation.utils import GenerateOutput\n\nfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM\n\n\nclass LlavaMistralConfig(MistralConfig):\n    model_type = \"llava_mistral\"\n\n\nclass LlavaMistralModel(LlavaMetaModel, MistralModel):\n    config_class = LlavaMistralConfig\n\n    def __init__(self, config: MistralConfig):\n        super(LlavaMistralModel, self).__init__(config)\n\n\nclass LlavaMistralForCausalLM(MistralForCausalLM, LlavaMetaForCausalLM):\n    config_class = LlavaMistralConfig\n\n    def __init__(self, config):\n        super(MistralForCausalLM, self).__init__(config)\n        self.model = LlavaMistralModel(config)\n\n        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n\n        # Initialize weights and apply final processing\n        self.post_init()\n\n    def get_model(self):\n        return self.model\n\n    def forward(\n        self,\n        input_ids: torch.LongTensor = None,\n        attention_mask: Optional[torch.Tensor] = None,\n        position_ids: Optional[torch.LongTensor] = None,\n        past_key_values: Optional[List[torch.FloatTensor]] = None,\n        inputs_embeds: Optional[torch.FloatTensor] = None,\n        labels: Optional[torch.LongTensor] = None,\n        use_cache: Optional[bool] = None,\n        output_attentions: Optional[bool] = None,\n        output_hidden_states: Optional[bool] = None,\n        images: Optional[torch.FloatTensor] = None,\n        image_sizes: Optional[List[List[int]]] = None,\n        return_dict: Optional[bool] = None,\n    ) -> Union[Tuple, CausalLMOutputWithPast]:\n\n        if inputs_embeds is None:\n            (\n                input_ids,\n                position_ids,\n                attention_mask,\n                past_key_values,\n                inputs_embeds,\n                labels\n            ) = self.prepare_inputs_labels_for_multimodal(\n                input_ids,\n                position_ids,\n                attention_mask,\n                past_key_values,\n                labels,\n                images,\n                image_sizes\n            )\n\n        return super().forward(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            position_ids=position_ids,\n            past_key_values=past_key_values,\n            inputs_embeds=inputs_embeds,\n            labels=labels,\n            use_cache=use_cache,\n            output_attentions=output_attentions,\n            output_hidden_states=output_hidden_states,\n            return_dict=return_dict\n        )\n\n    @torch.no_grad()\n    def generate(\n        self,\n        inputs: Optional[torch.Tensor] = None,\n        images: Optional[torch.Tensor] = None,\n        image_sizes: Optional[torch.Tensor] = None,\n        **kwargs,\n    ) -> Union[GenerateOutput, torch.LongTensor]:\n        position_ids = kwargs.pop(\"position_ids\", None)\n        attention_mask = kwargs.pop(\"attention_mask\", None)\n        if \"inputs_embeds\" in kwargs:\n            raise NotImplementedError(\"`inputs_embeds` is not supported\")\n\n        if images is not None:\n            (\n                inputs,\n                position_ids,\n                attention_mask,\n                _,\n                inputs_embeds,\n                _\n            ) = self.prepare_inputs_labels_for_multimodal(\n                inputs,\n                position_ids,\n                attention_mask,\n                None,\n                None,\n                images,\n                image_sizes=image_sizes\n            )\n        else:\n            inputs_embeds = self.get_model().embed_tokens(inputs)\n\n        return super().generate(\n            position_ids=position_ids,\n            attention_mask=attention_mask,\n            inputs_embeds=inputs_embeds,\n            **kwargs\n        )\n\n    def prepare_inputs_for_generation(self, input_ids, past_key_values=None,\n                                      inputs_embeds=None, **kwargs):\n        images = kwargs.pop(\"images\", None)\n        image_sizes = kwargs.pop(\"image_sizes\", None)\n        inputs = super().prepare_inputs_for_generation(\n            input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs\n        )\n        if images is not None:\n            inputs['images'] = images\n        if image_sizes is not None:\n            inputs['image_sizes'] = image_sizes\n        return inputs\n\nAutoConfig.register(\"llava_mistral\", LlavaMistralConfig)\nAutoModelForCausalLM.register(LlavaMistralConfig, LlavaMistralForCausalLM)\n"
  },
  {
    "path": "llava/model/language_model/llava_mpt.py",
    "content": "#    Copyright 2023 Haotian Liu\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n\n\nfrom typing import Optional, Tuple\n\nimport torch\n\nfrom transformers import AutoConfig, AutoModelForCausalLM, \\\n    MptConfig, MptForCausalLM, MptModel\nfrom llava.model.llava_arch import LlavaMetaModel, LlavaMetaForCausalLM\n\n\nclass LlavaMptConfig(MptConfig):\n    model_type = \"llava_mpt\"\n\n\nclass LlavaMptModel(LlavaMetaModel, MptModel):\n    config_class = LlavaMptConfig\n\n    def __init__(self, config: MptConfig):\n        config.hidden_size = config.d_model\n        super(LlavaMptModel, self).__init__(config)\n\n    def embed_tokens(self, x):\n        return self.wte(x)\n\n\nclass LlavaMptForCausalLM(MptForCausalLM, LlavaMetaForCausalLM):\n    config_class = LlavaMptConfig\n    supports_gradient_checkpointing = True\n\n    def __init__(self, config):\n        super(MptForCausalLM, self).__init__(config)\n\n        self.transformer = LlavaMptModel(config)\n        self.lm_head = torch.nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n\n        # Initialize weights and apply final processing\n        self.post_init()\n\n    def get_model(self):\n        return self.transformer\n\n    def _set_gradient_checkpointing(self, module, value=False):\n        if isinstance(module, LlavaMptModel):\n            module.gradient_checkpointing = value\n\n    def forward(\n            self,\n            input_ids: Optional[torch.LongTensor] = None,\n            past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,\n            attention_mask: Optional[torch.Tensor] = None,\n            inputs_embeds: Optional[torch.Tensor] = None,\n            labels: Optional[torch.Tensor] = None,\n            use_cache: Optional[bool] = None,\n            output_attentions: Optional[bool] = None,\n            output_hidden_states: Optional[bool] = None,\n            return_dict: Optional[bool] = None,\n            images=None):\n\n        input_ids, attention_mask, past_key_values, inputs_embeds, labels = self.prepare_inputs_labels_for_multimodal(input_ids, attention_mask, past_key_values, labels, images)\n\n        return super().forward(\n            input_ids,\n            past_key_values=past_key_values,\n            attention_mask=attention_mask,\n            inputs_embeds=inputs_embeds,\n            labels=labels,\n            use_cache=use_cache,\n            output_attentions=output_attentions,\n            output_hidden_states=output_hidden_states,\n            return_dict=return_dict,\n        )\n\n    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):\n        images = kwargs.pop(\"images\", None)\n        _inputs = super().prepare_inputs_for_generation(\n            input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs\n        )\n        _inputs['images'] = images\n        return _inputs\n\n\nAutoConfig.register(\"llava_mpt\", LlavaMptConfig)\nAutoModelForCausalLM.register(LlavaMptConfig, LlavaMptForCausalLM)\n"
  },
  {
    "path": "llava/model/language_model/llava_qwen.py",
    "content": "\n#    Copyright 2023 Haotian Liu\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n\n\nfrom typing import List, Optional, Tuple, Union\n\nimport torch\nimport torch.nn as nn\n\nfrom transformers import AutoConfig, AutoModelForCausalLM, Qwen2Config, Qwen2Model, Qwen2ForCausalLM\n\nfrom transformers.modeling_outputs import CausalLMOutputWithPast\nfrom transformers.generation.utils import GenerateOutput\n\nfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM\n\n\nclass LlavaConfig(Qwen2Config):\n    model_type = \"llava_qwen2\"\n\n\nclass LlavaQwen2Model(LlavaMetaModel, Qwen2Model):\n    config_class = LlavaConfig\n\n    def __init__(self, config: Qwen2Config):\n        super(LlavaQwen2Model, self).__init__(config)\n\n\nclass LlavaQwen2ForCausalLM(Qwen2ForCausalLM, LlavaMetaForCausalLM):\n    config_class = LlavaConfig\n\n    def __init__(self, config):\n        super(Qwen2ForCausalLM, self).__init__(config)\n        self.model = LlavaQwen2Model(config)\n        # self.pretraining_tp = config.pretraining_tp\n        self.vocab_size = config.vocab_size\n        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n\n        # Initialize weights and apply final processing\n        self.post_init()\n\n    def get_model(self):\n        return self.model\n\n    def forward(\n        self,\n        input_ids: torch.LongTensor = None,\n        attention_mask: Optional[torch.Tensor] = None,\n        position_ids: Optional[torch.LongTensor] = None,\n        past_key_values: Optional[List[torch.FloatTensor]] = None,\n        inputs_embeds: Optional[torch.FloatTensor] = None,\n        labels: Optional[torch.LongTensor] = None,\n        use_cache: Optional[bool] = None,\n        output_attentions: Optional[bool] = None,\n        output_hidden_states: Optional[bool] = None,\n        images: Optional[torch.FloatTensor] = None,\n        image_sizes: Optional[List[List[int]]] = None,\n        return_dict: Optional[bool] = None,\n        cache_position=None,\n    ) -> Union[Tuple, CausalLMOutputWithPast]:\n\n        if inputs_embeds is None:\n            (\n                input_ids,\n                position_ids,\n                attention_mask,\n                past_key_values,\n                inputs_embeds,\n                labels\n            ) = self.prepare_inputs_labels_for_multimodal(\n                input_ids,\n                position_ids,\n                attention_mask,\n                past_key_values,\n                labels,\n                images,\n                image_sizes\n            )\n\n        return super().forward(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            position_ids=position_ids,\n            past_key_values=past_key_values,\n            inputs_embeds=inputs_embeds,\n            labels=labels,\n            use_cache=use_cache,\n            output_attentions=output_attentions,\n            output_hidden_states=output_hidden_states,\n            return_dict=return_dict\n        )\n\n    @torch.no_grad()\n    def generate(\n        self,\n        inputs: Optional[torch.Tensor] = None,\n        images: Optional[torch.Tensor] = None,\n        image_sizes: Optional[torch.Tensor] = None,\n        **kwargs,\n    ) -> Union[GenerateOutput, torch.LongTensor]:\n        position_ids = kwargs.pop(\"position_ids\", None)\n        attention_mask = kwargs.pop(\"attention_mask\", None)\n        if \"inputs_embeds\" in kwargs:\n            raise NotImplementedError(\"`inputs_embeds` is not supported\")\n\n        if images is not None:\n            (\n                inputs,\n                position_ids,\n                attention_mask,\n                _,\n                inputs_embeds,\n                _\n            ) = self.prepare_inputs_labels_for_multimodal(\n                inputs,\n                position_ids,\n                attention_mask,\n                None,\n                None,\n                images,\n                image_sizes=image_sizes\n            )\n        else:\n            inputs_embeds = self.get_model().embed_tokens(inputs)\n\n        return super().generate(\n            position_ids=position_ids,\n            attention_mask=attention_mask,\n            inputs_embeds=inputs_embeds,\n            **kwargs\n        )\n\n    def prepare_inputs_for_generation(self, input_ids, past_key_values=None,\n                                      inputs_embeds=None, **kwargs):\n        images = kwargs.pop(\"images\", None)\n        image_sizes = kwargs.pop(\"image_sizes\", None)\n        inputs = super().prepare_inputs_for_generation(\n            input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs\n        )\n        if images is not None:\n            inputs['images'] = images\n        if image_sizes is not None:\n            inputs['image_sizes'] = image_sizes\n        return inputs\n\n\nAutoConfig.register(\"llava_qwen2\", LlavaConfig)\nAutoModelForCausalLM.register(LlavaConfig, LlavaQwen2ForCausalLM)\n"
  },
  {
    "path": "llava/model/llava_arch.py",
    "content": "#    Copyright 2023 Haotian Liu\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n\n\nfrom abc import ABC, abstractmethod\n\nimport torch\nimport torch.nn as nn\n\nfrom .multimodal_encoder.builder import build_vision_tower\nfrom .multimodal_projector.builder import build_vision_projector\n\nfrom llava.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN\n\nfrom llava.mm_utils import get_anyres_image_grid_shape\n\n\nclass LlavaMetaModel:\n\n    def __init__(self, config):\n        super(LlavaMetaModel, self).__init__(config)\n\n        if hasattr(config, \"mm_vision_tower\"):\n            self.vision_tower = build_vision_tower(config, delay_load=True)\n            self.mm_projector = build_vision_projector(config)\n\n            if 'unpad' in getattr(config, 'mm_patch_merge_type', ''):\n                self.image_newline = nn.Parameter(\n                    torch.empty(config.hidden_size, dtype=self.dtype)\n                )\n\n    def get_vision_tower(self):\n        vision_tower = getattr(self, 'vision_tower', None)\n        if type(vision_tower) is list:\n            vision_tower = vision_tower[0]\n        return vision_tower\n\n    def initialize_vision_modules(self, model_args, fsdp=None):\n        vision_tower = model_args.vision_tower\n        mm_vision_select_layer = model_args.mm_vision_select_layer\n        mm_vision_select_feature = model_args.mm_vision_select_feature\n        pretrain_mm_mlp_adapter = model_args.pretrain_mm_mlp_adapter\n        mm_patch_merge_type = model_args.mm_patch_merge_type\n\n        self.config.mm_vision_tower = vision_tower\n\n        if self.get_vision_tower() is None:\n            vision_tower = build_vision_tower(model_args)\n\n            if fsdp is not None and len(fsdp) > 0:\n                self.vision_tower = [vision_tower]\n            else:\n                self.vision_tower = vision_tower\n        else:\n            if fsdp is not None and len(fsdp) > 0:\n                vision_tower = self.vision_tower[0]\n            else:\n                vision_tower = self.vision_tower\n            vision_tower.load_model()\n\n        self.config.use_mm_proj = True\n        self.config.mm_projector_type = getattr(model_args, 'mm_projector_type', 'linear')\n        self.config.mm_hidden_size = vision_tower.hidden_size\n        self.config.mm_vision_select_layer = mm_vision_select_layer\n        self.config.mm_vision_select_feature = mm_vision_select_feature\n        self.config.mm_patch_merge_type = mm_patch_merge_type\n\n        if getattr(self, 'mm_projector', None) is None:\n            self.mm_projector = build_vision_projector(self.config)\n\n            if 'unpad' in mm_patch_merge_type:\n                embed_std = 1 / torch.sqrt(torch.tensor(self.config.hidden_size, dtype=self.dtype))\n                self.image_newline = nn.Parameter(\n                    torch.randn(self.config.hidden_size, dtype=self.dtype) * embed_std\n                )\n        else:\n            # In case it is frozen by LoRA\n            for p in self.mm_projector.parameters():\n                p.requires_grad = True\n\n        if pretrain_mm_mlp_adapter is not None:\n            mm_projector_weights = torch.load(pretrain_mm_mlp_adapter, map_location='cpu')\n\n            def get_w(weights, keyword):\n                return {k.split(keyword + '.')[1]: v for k, v in weights.items() if keyword in k}\n\n            self.mm_projector.load_state_dict(get_w(mm_projector_weights, 'mm_projector'))\n\n\ndef unpad_image(tensor, original_size):\n    \"\"\"\n    Unpads a PyTorch tensor of a padded and resized image.\n\n    Args:\n    tensor (torch.Tensor): The image tensor, assumed to be in CxHxW format.\n    original_size (tuple): The original size of PIL image (width, height).\n\n    Returns:\n    torch.Tensor: The unpadded image tensor.\n    \"\"\"\n    original_width, original_height = original_size\n    current_height, current_width = tensor.shape[1:]\n\n    original_aspect_ratio = original_width / original_height\n    current_aspect_ratio = current_width / current_height\n\n    if original_aspect_ratio > current_aspect_ratio:\n        scale_factor = current_width / original_width\n        new_height = int(original_height * scale_factor)\n        padding = (current_height - new_height) // 2\n        unpadded_tensor = tensor[:, padding:current_height - padding, :]\n    else:\n        scale_factor = current_height / original_height\n        new_width = int(original_width * scale_factor)\n        padding = (current_width - new_width) // 2\n        unpadded_tensor = tensor[:, :, padding:current_width - padding]\n\n    return unpadded_tensor\n\n\nclass LlavaMetaForCausalLM(ABC):\n\n    @abstractmethod\n    def get_model(self):\n        pass\n\n    def get_vision_tower(self):\n        return self.get_model().get_vision_tower()\n\n    def encode_images(self, images):\n        image_features = self.get_model().get_vision_tower()(images)\n        image_features = self.get_model().mm_projector(image_features)\n        return image_features\n\n    def prepare_inputs_labels_for_multimodal(\n        self, input_ids, position_ids, attention_mask, past_key_values, labels,\n        images, image_sizes=None\n    ):\n        vision_tower = self.get_vision_tower()\n        if vision_tower is None or images is None or input_ids.shape[1] == 1:\n            return input_ids, position_ids, attention_mask, past_key_values, None, labels\n\n        if type(images) is list or images.ndim == 5:\n            if type(images) is list:\n                images = [x.unsqueeze(0) if x.ndim == 3 else x for x in images]\n            concat_images = torch.cat([image for image in images], dim=0)\n            image_features = self.encode_images(concat_images)\n            split_sizes = [image.shape[0] for image in images]\n            image_features = torch.split(image_features, split_sizes, dim=0)\n            mm_patch_merge_type = getattr(self.config, 'mm_patch_merge_type', 'flat')\n            image_aspect_ratio = getattr(self.config, 'image_aspect_ratio', 'square')\n            if mm_patch_merge_type == 'flat':\n                image_features = [x.flatten(0, 1) for x in image_features]\n            elif mm_patch_merge_type.startswith('spatial'):\n                new_image_features = []\n                for image_idx, image_feature in enumerate(image_features):\n                    if image_feature.shape[0] > 1:\n                        base_image_feature = image_feature[0]\n                        image_feature = image_feature[1:]\n                        height = width = self.get_vision_tower().num_patches_per_side\n                        assert height * width == base_image_feature.shape[0]\n                        if image_aspect_ratio == 'anyres':\n                            if hasattr(self.get_vision_tower(), 's2_image_size'):\n                                img_size = self.get_vision_tower().s2_image_size\n                            elif isinstance(self.get_vision_tower().config, dict):\n                                img_size = self.get_vision_tower().config[\"image_cfg\"][\"image_size\"]\n                            else:\n                                img_size = self.get_vision_tower().config.image_size\n\n                            num_patch_width, num_patch_height = get_anyres_image_grid_shape(image_sizes[image_idx], self.config.image_grid_pinpoints, img_size)\n                            image_feature = image_feature.view(num_patch_height, num_patch_width, height, width, -1)\n                        else:\n                            raise NotImplementedError\n                        if 'unpad' in mm_patch_merge_type:\n                            image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous()\n                            image_feature = image_feature.flatten(1, 2).flatten(2, 3)\n                            image_feature = unpad_image(image_feature, image_sizes[image_idx])\n                            image_feature = torch.cat((\n                                image_feature,\n                                self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1).to(image_feature.device)\n                            ), dim=-1)\n                            image_feature = image_feature.flatten(1, 2).transpose(0, 1)\n                        else:\n                            image_feature = image_feature.permute(0, 2, 1, 3, 4).contiguous()\n                            image_feature = image_feature.flatten(0, 3)\n                        image_feature = torch.cat((base_image_feature, image_feature), dim=0)\n                    else:\n                        image_feature = image_feature[0]\n                        if 'unpad' in mm_patch_merge_type:\n                            image_feature = torch.cat((\n                                image_feature,\n                                self.model.image_newline[None].to(image_feature.device)\n                            ), dim=0)\n                    new_image_features.append(image_feature)\n                image_features = new_image_features\n            else:\n                raise ValueError(f\"Unexpected mm_patch_merge_type: {self.config.mm_patch_merge_type}\")\n        else:\n            image_features = self.encode_images(images)\n\n        # TODO: image start / end is not implemented here to support pretraining.\n        if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):\n            raise NotImplementedError\n\n        # Let's just add dummy tensors if they do not exist,\n        # it is a headache to deal with None all the time.\n        # But it is not ideal, and if you have a better idea,\n        # please open an issue / submit a PR, thanks.\n        _labels = labels\n        _position_ids = position_ids\n        _attention_mask = attention_mask\n        if attention_mask is None:\n            attention_mask = torch.ones_like(input_ids, dtype=torch.bool)\n        else:\n            attention_mask = attention_mask.bool()\n        if position_ids is None:\n            position_ids = torch.arange(0, input_ids.shape[1], dtype=torch.long, device=input_ids.device)\n        if labels is None:\n            labels = torch.full_like(input_ids, IGNORE_INDEX)\n\n        # remove the padding using attention_mask -- FIXME\n        _input_ids = input_ids\n        input_ids = [cur_input_ids[cur_attention_mask] for cur_input_ids, cur_attention_mask in zip(input_ids, attention_mask)]\n        labels = [cur_labels[cur_attention_mask] for cur_labels, cur_attention_mask in zip(labels, attention_mask)]\n\n        new_input_embeds = []\n        new_labels = []\n        cur_image_idx = 0\n        for batch_idx, cur_input_ids in enumerate(input_ids):\n            num_images = (cur_input_ids == IMAGE_TOKEN_INDEX).sum()\n            if num_images == 0:\n                cur_image_features = image_features[cur_image_idx]\n                cur_input_embeds_1 = self.get_model().embed_tokens(cur_input_ids)\n                cur_input_embeds = torch.cat([cur_input_embeds_1, cur_image_features[0:0]], dim=0)\n                new_input_embeds.append(cur_input_embeds)\n                new_labels.append(labels[batch_idx])\n                cur_image_idx += 1\n                continue\n\n            image_token_indices = [-1] + torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0].tolist() + [cur_input_ids.shape[0]]\n            cur_input_ids_noim = []\n            cur_labels = labels[batch_idx]\n            cur_labels_noim = []\n            for i in range(len(image_token_indices) - 1):\n                cur_input_ids_noim.append(cur_input_ids[image_token_indices[i]+1:image_token_indices[i+1]])\n                cur_labels_noim.append(cur_labels[image_token_indices[i]+1:image_token_indices[i+1]])\n            split_sizes = [x.shape[0] for x in cur_labels_noim]\n            cur_input_embeds = self.get_model().embed_tokens(torch.cat(cur_input_ids_noim))\n            cur_input_embeds_no_im = torch.split(cur_input_embeds, split_sizes, dim=0)\n            cur_new_input_embeds = []\n            cur_new_labels = []\n\n            for i in range(num_images + 1):\n                cur_new_input_embeds.append(cur_input_embeds_no_im[i])\n                cur_new_labels.append(cur_labels_noim[i])\n                if i < num_images:\n                    cur_image_features = image_features[cur_image_idx]\n                    cur_image_idx += 1\n                    cur_new_input_embeds.append(cur_image_features)\n                    cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=cur_labels.device, dtype=cur_labels.dtype))\n\n            cur_new_input_embeds = [x.to(self.device) for x in cur_new_input_embeds]\n\n            cur_new_input_embeds = torch.cat(cur_new_input_embeds)\n            cur_new_labels = torch.cat(cur_new_labels)\n\n            new_input_embeds.append(cur_new_input_embeds)\n            new_labels.append(cur_new_labels)\n\n        # Truncate sequences to max length as image embeddings can make the sequence longer\n        tokenizer_model_max_length = getattr(self.config, 'tokenizer_model_max_length', None)\n        if tokenizer_model_max_length is not None:\n            new_input_embeds = [x[:tokenizer_model_max_length] for x in new_input_embeds]\n            new_labels = [x[:tokenizer_model_max_length] for x in new_labels]\n\n        # Combine them\n        max_len = max(x.shape[0] for x in new_input_embeds)\n        batch_size = len(new_input_embeds)\n\n        new_input_embeds_padded = []\n        new_labels_padded = torch.full((batch_size, max_len), IGNORE_INDEX, dtype=new_labels[0].dtype, device=new_labels[0].device)\n        attention_mask = torch.zeros((batch_size, max_len), dtype=attention_mask.dtype, device=attention_mask.device)\n        position_ids = torch.zeros((batch_size, max_len), dtype=position_ids.dtype, device=position_ids.device)\n\n        for i, (cur_new_embed, cur_new_labels) in enumerate(zip(new_input_embeds, new_labels)):\n            cur_len = cur_new_embed.shape[0]\n            if getattr(self.config, 'tokenizer_padding_side', 'right') == \"left\":\n                new_input_embeds_padded.append(torch.cat((\n                    torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device),\n                    cur_new_embed\n                ), dim=0))\n                if cur_len > 0:\n                    new_labels_padded[i, -cur_len:] = cur_new_labels\n                    attention_mask[i, -cur_len:] = True\n                    position_ids[i, -cur_len:] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device)\n            else:\n                new_input_embeds_padded.append(torch.cat((\n                    cur_new_embed,\n                    torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device)\n                ), dim=0))\n                if cur_len > 0:\n                    new_labels_padded[i, :cur_len] = cur_new_labels\n                    attention_mask[i, :cur_len] = True\n                    position_ids[i, :cur_len] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device)\n\n        new_input_embeds = torch.stack(new_input_embeds_padded, dim=0)\n\n        if _labels is None:\n            new_labels = None\n        else:\n            new_labels = new_labels_padded\n\n        if _attention_mask is None:\n            attention_mask = None\n        else:\n            attention_mask = attention_mask.to(dtype=_attention_mask.dtype)\n\n        if _position_ids is None:\n            position_ids = None\n\n        return None, position_ids, attention_mask, past_key_values, new_input_embeds, new_labels\n\n    def initialize_vision_tokenizer(self, model_args, tokenizer):\n        if model_args.mm_use_im_patch_token:\n            tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)\n            self.resize_token_embeddings(len(tokenizer))\n\n        if model_args.mm_use_im_start_end:\n            num_new_tokens = tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)\n            self.resize_token_embeddings(len(tokenizer))\n\n            if num_new_tokens > 0:\n                input_embeddings = self.get_input_embeddings().weight.data\n                output_embeddings = self.get_output_embeddings().weight.data\n\n                input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(\n                    dim=0, keepdim=True)\n                output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(\n                    dim=0, keepdim=True)\n\n                input_embeddings[-num_new_tokens:] = input_embeddings_avg\n                output_embeddings[-num_new_tokens:] = output_embeddings_avg\n\n            if model_args.tune_mm_mlp_adapter:\n                for p in self.get_input_embeddings().parameters():\n                    p.requires_grad = True\n                for p in self.get_output_embeddings().parameters():\n                    p.requires_grad = False\n\n            if model_args.pretrain_mm_mlp_adapter:\n                mm_projector_weights = torch.load(model_args.pretrain_mm_mlp_adapter, map_location='cpu')\n                embed_tokens_weight = mm_projector_weights['model.embed_tokens.weight']\n                assert num_new_tokens == 2\n                if input_embeddings.shape == embed_tokens_weight.shape:\n                    input_embeddings[-num_new_tokens:] = embed_tokens_weight[-num_new_tokens:]\n                elif embed_tokens_weight.shape[0] == num_new_tokens:\n                    input_embeddings[-num_new_tokens:] = embed_tokens_weight\n                else:\n                    raise ValueError(f\"Unexpected embed_tokens_weight shape. Pretrained: {embed_tokens_weight.shape}. Current: {input_embeddings.shape}. Numer of new tokens: {num_new_tokens}.\")\n        elif model_args.mm_use_im_patch_token:\n            if model_args.tune_mm_mlp_adapter:\n                for p in self.get_input_embeddings().parameters():\n                    p.requires_grad = False\n                for p in self.get_output_embeddings().parameters():\n                    p.requires_grad = False\n"
  },
  {
    "path": "llava/model/make_delta.py",
    "content": "\"\"\"\nUsage:\npython3 -m llava.model.make_delta --base ~/model_weights/llama-7b --target ~/model_weights/llava-7b --delta ~/model_weights/llava-7b-delta --hub-repo-id liuhaotian/llava-7b-delta\n\"\"\"\nimport argparse\n\nimport torch\nfrom tqdm import tqdm\nfrom transformers import AutoTokenizer, AutoModelForCausalLM\nfrom llava.model.utils import auto_upgrade\n\n\ndef make_delta(base_model_path, target_model_path, delta_path, hub_repo_id):\n    print(\"Loading base model\")\n    base = AutoModelForCausalLM.from_pretrained(\n        base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)\n\n    print(\"Loading target model\")\n    auto_upgrade(target_model_path)\n    target = AutoModelForCausalLM.from_pretrained(target_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)\n\n    print(\"Calculating delta\")\n    for name, param in tqdm(target.state_dict().items(), desc=\"Calculating delta\"):\n        if name not in base.state_dict():\n            assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model'\n            continue\n        if param.data.shape == base.state_dict()[name].shape:\n            param.data -= base.state_dict()[name]\n        else:\n            assert name in ['model.embed_tokens.weight', 'lm_head.weight'], f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}'\n            bparam = base.state_dict()[name]\n            param.data[:bparam.shape[0], :bparam.shape[1]] -= bparam\n\n    print(\"Saving delta\")\n    if hub_repo_id:\n        kwargs = {\"push_to_hub\": True, \"repo_id\": hub_repo_id}\n    else:\n        kwargs = {}\n    target.save_pretrained(delta_path, **kwargs)\n    target_tokenizer = AutoTokenizer.from_pretrained(target_model_path)\n    target_tokenizer.save_pretrained(delta_path, **kwargs)\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--base-model-path\", type=str, required=True)\n    parser.add_argument(\"--target-model-path\", type=str, required=True)\n    parser.add_argument(\"--delta-path\", type=str, required=True)\n    parser.add_argument(\"--hub-repo-id\", type=str, default=None)\n    args = parser.parse_args()\n\n    make_delta(args.base_model_path, args.target_model_path, args.delta_path, args.hub_repo_id)\n"
  },
  {
    "path": "llava/model/multimodal_encoder/builder.py",
    "content": "import os\nfrom .clip_encoder import CLIPVisionTower, CLIPVisionTowerS2\nfrom .mobileclip_encoder import MobileCLIPVisionTower\n\n\ndef build_vision_tower(vision_tower_cfg, **kwargs):\n    vision_tower = getattr(vision_tower_cfg, 'mm_vision_tower', getattr(vision_tower_cfg, 'vision_tower', None))\n    is_absolute_path_exists = os.path.exists(vision_tower)\n    use_s2 = getattr(vision_tower_cfg, 's2', False)\n\n    if is_absolute_path_exists or vision_tower.startswith(\"openai\") or vision_tower.startswith(\"laion\") or \"ShareGPT4V\" in vision_tower:\n        if use_s2:\n            return CLIPVisionTowerS2(vision_tower, args=vision_tower_cfg, **kwargs)\n        else:\n            return CLIPVisionTower(vision_tower, args=vision_tower_cfg, **kwargs)\n    elif \"mobileclip\" in vision_tower.lower():\n        return MobileCLIPVisionTower(vision_tower, args=vision_tower_cfg, **kwargs)\n\n    raise ValueError(f'Unknown vision tower: {vision_tower}')\n"
  },
  {
    "path": "llava/model/multimodal_encoder/clip_encoder.py",
    "content": "import torch\nimport torch.nn as nn\n\nfrom transformers import CLIPVisionModel, CLIPImageProcessor, CLIPVisionConfig\n\n\nclass CLIPVisionTower(nn.Module):\n    def __init__(self, vision_tower, args, delay_load=False):\n        super().__init__()\n\n        self.is_loaded = False\n\n        self.vision_tower_name = vision_tower\n        self.select_layer = args.mm_vision_select_layer\n        self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch')\n        self.tune_vision_tower = getattr(args, 'unfreeze_mm_vision_tower', False)\n        self.input_image_size = getattr(args, 'input_image_size', None)\n\n        if self.tune_vision_tower:\n            print(\"CLIP Vision tower is set to tunable\")\n\n        if not delay_load:\n            self.load_model()\n        elif getattr(args, 'unfreeze_mm_vision_tower', False):\n            self.load_model()\n        else:\n            self.cfg_only = CLIPVisionConfig.from_pretrained(self.vision_tower_name)\n            if self.input_image_size is not None:\n                self.cfg_only.image_size = self.input_image_size\n\n    def load_model(self, device_map=None):\n        if self.is_loaded:\n            print('{} is already loaded, `load_model` called again, skipping.'.format(self.vision_tower_name))\n            return\n\n        self.image_processor = CLIPImageProcessor.from_pretrained(self.vision_tower_name)\n        self.vision_tower = CLIPVisionModel.from_pretrained(self.vision_tower_name, device_map=device_map)\n        if not self.tune_vision_tower:\n            self.vision_tower.requires_grad_(False)\n\n        if self.input_image_size is not None:\n            print(\"Using input image size: {}\".format(self.input_image_size))\n            self.image_processor.size['shortest_edge'] = self.input_image_size\n            self.image_processor.crop_size['height'] = self.image_processor.crop_size['width'] = self.input_image_size\n\n        self.is_loaded = True\n\n    def feature_select(self, image_forward_outs):\n        image_features = image_forward_outs.hidden_states[self.select_layer]\n        if self.select_feature == 'patch':\n            image_features = image_features[:, 1:]\n        elif self.select_feature == 'cls_patch':\n            image_features = image_features\n        else:\n            raise ValueError(f'Unexpected select feature: {self.select_feature}')\n        return image_features\n\n    def forward(self, images):\n        if self.tune_vision_tower:\n            return self.forward_images(images)\n        else:\n            with torch.no_grad():\n                return self.forward_images(images)\n\n    def forward_images(self, images):\n        if type(images) is list:\n            image_features = []\n            for image in images:\n                image_forward_out = self.vision_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True)\n                image_feature = self.feature_select(image_forward_out).to(image.dtype)\n                image_features.append(image_feature)\n        else:\n            image_forward_outs = self.vision_tower(images.to(device=self.device, dtype=self.dtype), output_hidden_states=True)\n            image_features = self.feature_select(image_forward_outs).to(images.dtype)\n\n        return image_features\n\n    @property\n    def dummy_feature(self):\n        return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)\n\n    @property\n    def dtype(self):\n        return self.vision_tower.dtype\n\n    @property\n    def device(self):\n        return self.vision_tower.device\n\n    @property\n    def config(self):\n        if self.is_loaded:\n            return self.vision_tower.config\n        else:\n            return self.cfg_only\n\n    @property\n    def hidden_size(self):\n        return self.config.hidden_size\n\n    @property\n    def num_patches_per_side(self):\n        return self.config.image_size // self.config.patch_size\n\n    @property\n    def num_patches(self):\n        return (self.config.image_size // self.config.patch_size) ** 2\n\n\n\nclass CLIPVisionTowerS2(CLIPVisionTower):\n    def __init__(self, vision_tower, args, delay_load=False):\n        self.s2_scales = getattr(args, 's2_scales', '336,672,1008')\n        self.s2_scales = list(map(int, self.s2_scales.split(',')))\n        self.s2_scales.sort()\n        self.s2_split_size = self.s2_scales[0]\n        self.s2_image_size = self.s2_scales[-1]\n\n        super().__init__(vision_tower, args, delay_load)\n\n        try:\n            from s2wrapper import forward as multiscale_forward\n        except ImportError:\n            raise ImportError('Package s2wrapper not found! Please install by running: \\npip install git+https://github.com/bfshi/scaling_on_scales.git')\n        self.multiscale_forward = multiscale_forward\n\n        # change resize/crop size in preprocessing to the largest image size in s2_scale\n        if not delay_load or getattr(args, 'unfreeze_mm_vision_tower', False):\n            self.image_processor.size['shortest_edge'] = self.s2_image_size\n            self.image_processor.crop_size['height'] = self.image_processor.crop_size['width'] = self.s2_image_size\n\n    def load_model(self, device_map=None):\n        if self.is_loaded:\n            print('{} is already loaded, `load_model` called again, skipping.'.format(self.vision_tower_name))\n            return\n\n        self.image_processor = CLIPImageProcessor.from_pretrained(self.vision_tower_name)\n        self.vision_tower = CLIPVisionModel.from_pretrained(self.vision_tower_name, device_map=device_map)\n        self.vision_tower.requires_grad_(False)\n\n        self.image_processor.size['shortest_edge'] = self.s2_image_size\n        self.image_processor.crop_size['height'] = self.image_processor.crop_size['width'] = self.s2_image_size\n\n        self.is_loaded = True\n\n    @torch.no_grad()\n    def forward_feature(self, images):\n        image_forward_outs = self.vision_tower(images.to(device=self.device, dtype=self.dtype), output_hidden_states=True)\n        image_features = self.feature_select(image_forward_outs).to(images.dtype)\n        return image_features\n\n    @torch.no_grad()\n    def forward(self, images):\n        if type(images) is list:\n            image_features = []\n            for image in images:\n                image_feature = self.multiscale_forward(self.forward_feature, image.unsqueeze(0), img_sizes=self.s2_scales, max_split_size=self.s2_split_size)\n                image_features.append(image_feature)\n        else:\n            image_features = self.multiscale_forward(self.forward_feature, images, img_sizes=self.s2_scales, max_split_size=self.s2_split_size)\n\n        return image_features\n\n    @property\n    def hidden_size(self):\n        return self.config.hidden_size * len(self.s2_scales)\n"
  },
  {
    "path": "llava/model/multimodal_encoder/mobileclip/__init__.py",
    "content": "#\n# For licensing see accompanying LICENSE file.\n# Copyright (C) 2025 Apple Inc. All Rights Reserved.\n#\nimport os\nimport json\nfrom typing import Any\n\nimport torch.nn as nn\nfrom timm.models import create_model\n\nfrom .mci import GlobalPool2D\n\n\ndef load_model_config(\n        model_name: str,\n) -> Any:\n    # Strip suffixes to model name\n    model_name = \"_\".join(model_name.split(\"_\")[0:2])\n\n    # Config files\n    root_dir = os.path.dirname(os.path.abspath(__file__))\n    configs_dir = os.path.join(root_dir, \"configs\")\n    model_cfg_file = os.path.join(configs_dir, model_name + \".json\")\n\n    # Get config from yaml file\n    if not os.path.exists(model_cfg_file):\n        raise ValueError(f\"Unsupported model name: {model_name}\")\n    model_cfg = json.load(open(model_cfg_file, \"r\"))\n\n    return model_cfg\n\n\nclass MCi(nn.Module):\n    \"\"\"\n    This class implements `MCi Models <https://arxiv.org/pdf/2311.17049.pdf>`_\n    \"\"\"\n\n    def __init__(self, model_name: str, *args, **kwargs) -> None:\n        super().__init__()\n        self.projection_dim = None\n        if \"projection_dim\" in kwargs:\n            self.projection_dim = kwargs.get(\"projection_dim\")\n\n        # Create model\n        self.model = create_model(model_name, projection_dim=self.projection_dim)\n\n        # Build out projection head.\n        if self.projection_dim is not None:\n            if hasattr(self.model, \"head\"):\n                self.model.head = MCi._update_image_classifier(\n                    image_classifier=self.model.head, projection_dim=self.projection_dim\n                )\n\n    def forward(self, x: Any, *args, **kwargs) -> Any:\n        \"\"\"A forward function of the model.\"\"\"\n        x = self.model(x, *args, **kwargs)\n        return x\n\n    @staticmethod\n    def _get_in_feature_dimension(image_classifier: nn.Module) -> int:\n        \"\"\"Return the input feature dimension to the image classification head.\"\"\"\n        in_features = None\n        if isinstance(image_classifier, nn.Sequential):\n            # Classifier that uses nn.Sequential usually has global pooling and\n            # multiple linear layers. Find the first linear layer and get its\n            # in_features\n            for layer in image_classifier:\n                if isinstance(layer, nn.Linear):\n                    in_features = layer.in_features\n                    break\n        elif isinstance(image_classifier, nn.Linear):\n            in_features = image_classifier.in_features\n\n        if in_features is None:\n            raise NotImplementedError(\n                f\"Cannot get input feature dimension of {image_classifier}.\"\n            )\n        return in_features\n\n    @staticmethod\n    def _update_image_classifier(\n        image_classifier: nn.Module, projection_dim: int, *args, **kwargs\n    ) -> nn.Module:\n        in_features = MCi._get_in_feature_dimension(image_classifier)\n        new_img_classifier = GlobalPool2D(in_dim=in_features, out_dim=projection_dim)\n        return new_img_classifier\n"
  },
  {
    "path": "llava/model/multimodal_encoder/mobileclip/configs/mobileclip_l.json",
    "content": "{\n    \"embed_dim\": 768,\n    \"image_cfg\": {\n        \"image_size\": 1024,\n        \"model_name\": \"fastvithd\",\n        \"embed_dim\": 3072,\n        \"patch_size\": 64\n    },\n    \"text_cfg\": {\n        \"context_length\": 77,\n        \"vocab_size\": 49408,\n        \"dim\": 768,\n        \"ffn_multiplier_per_layer\": 4.0,\n        \"n_heads_per_layer\": 12,\n        \"n_transformer_layers\": 12,\n        \"norm_layer\": \"layer_norm_fp32\",\n        \"causal_masking\": false,\n        \"model_name\": \"base\"\n    }\n}\n"
  },
  {
    "path": "llava/model/multimodal_encoder/mobileclip/mci.py",
    "content": "#\n# For licensing see accompanying LICENSE file.\n# Copyright (C) 2025 Apple Inc. All Rights Reserved.\n#\nimport copy\nfrom functools import partial\nfrom typing import List, Tuple, Optional, Union, Dict\n\nimport torch\nimport torch.nn as nn\nfrom torch import Tensor\nimport torch.nn.functional as F\nfrom torch.nn.init import normal_\n\nfrom timm.models import register_model\nfrom timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD\nfrom timm.layers import DropPath, SqueezeExcite\n\n\ndef _cfg(url=\"\", **kwargs):\n    return {\n        \"url\": url,\n        \"num_classes\": 1000,\n        \"input_size\": (3, 256, 256),\n        \"pool_size\": None,\n        \"crop_pct\": 0.95,\n        \"interpolation\": \"bicubic\",\n        \"mean\": IMAGENET_DEFAULT_MEAN,\n        \"std\": IMAGENET_DEFAULT_STD,\n        \"classifier\": \"head\",\n        **kwargs,\n    }\n\n\ndefault_cfgs = {\n    \"fastvit_t\": _cfg(crop_pct=0.9),\n    \"fastvit_s\": _cfg(crop_pct=0.9),\n    \"fastvit_m\": _cfg(crop_pct=0.95),\n}\n\n\nclass SEBlock(nn.Module):\n    \"\"\"Squeeze and Excite module.\n\n    Pytorch implementation of `Squeeze-and-Excitation Networks` -\n    https://arxiv.org/pdf/1709.01507.pdf\n    \"\"\"\n\n    def __init__(self, in_channels: int, rd_ratio: float = 0.0625) -> None:\n        \"\"\"Construct a Squeeze and Excite Module.\n\n        Args:\n            in_channels: Number of input channels.\n            rd_ratio: Input channel reduction ratio.\n        \"\"\"\n        super(SEBlock, self).__init__()\n        self.reduce = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=int(in_channels * rd_ratio),\n            kernel_size=1,\n            stride=1,\n            bias=True,\n        )\n        self.expand = nn.Conv2d(\n            in_channels=int(in_channels * rd_ratio),\n            out_channels=in_channels,\n            kernel_size=1,\n            stride=1,\n            bias=True,\n        )\n\n    def forward(self, inputs: torch.Tensor) -> torch.Tensor:\n        \"\"\"Apply forward pass.\"\"\"\n        b, c, h, w = inputs.size()\n        x = F.avg_pool2d(inputs, kernel_size=[h, w])\n        x = self.reduce(x)\n        x = F.relu(x)\n        x = self.expand(x)\n        x = torch.sigmoid(x)\n        x = x.view(-1, c, 1, 1)\n        return inputs * x\n\n\nclass MobileOneBlock(nn.Module):\n    \"\"\"MobileOne building block.\n\n    This block has a multi-branched architecture at train-time\n    and plain-CNN style architecture at inference time\n    For more details, please refer to our paper:\n    `An Improved One millisecond Mobile Backbone` -\n    https://arxiv.org/pdf/2206.04040.pdf\n    \"\"\"\n\n    def __init__(\n        self,\n        in_channels: int,\n        out_channels: int,\n        kernel_size: int,\n        stride: int = 1,\n        padding: int = 0,\n        dilation: int = 1,\n        groups: int = 1,\n        inference_mode: bool = False,\n        use_se: bool = False,\n        use_act: bool = True,\n        use_scale_branch: bool = True,\n        num_conv_branches: int = 1,\n        activation: nn.Module = nn.GELU(),\n    ) -> None:\n        \"\"\"Construct a MobileOneBlock module.\n\n        Args:\n            in_channels: Number of channels in the input.\n            out_channels: Number of channels produced by the block.\n            kernel_size: Size of the convolution kernel.\n            stride: Stride size.\n            padding: Zero-padding size.\n            dilation: Kernel dilation factor.\n            groups: Group number.\n            inference_mode: If True, instantiates model in inference mode.\n            use_se: Whether to use SE-ReLU activations.\n            use_act: Whether to use activation. Default: ``True``\n            use_scale_branch: Whether to use scale branch. Default: ``True``\n            num_conv_branches: Number of linear conv branches.\n        \"\"\"\n        super(MobileOneBlock, self).__init__()\n        self.inference_mode = inference_mode\n        self.groups = groups\n        self.stride = stride\n        self.padding = padding\n        self.dilation = dilation\n        self.kernel_size = kernel_size\n        self.in_channels = in_channels\n        self.out_channels = out_channels\n        self.num_conv_branches = num_conv_branches\n\n        # Check if SE-ReLU is requested\n        if use_se:\n            self.se = SEBlock(out_channels)\n        else:\n            self.se = nn.Identity()\n\n        if use_act:\n            self.activation = activation\n        else:\n            self.activation = nn.Identity()\n\n        if inference_mode:\n            self.reparam_conv = nn.Conv2d(\n                in_channels=in_channels,\n                out_channels=out_channels,\n                kernel_size=kernel_size,\n                stride=stride,\n                padding=padding,\n                dilation=dilation,\n                groups=groups,\n                bias=True,\n            )\n        else:\n            # Re-parameterizable skip connection\n            # Fallback, sometimes batchnorm tensors\n            # do not get instantiated correctly on some processes\n            # when using deepspeed + accelerate\n            norm_layer = nn.BatchNorm2d(num_features=in_channels)\n            if norm_layer.weight.shape[0] == 0:\n                norm_layer.weight = nn.Parameter(torch.zeros(in_channels))\n            if norm_layer.bias.shape[0] == 0:\n                norm_layer.bias = nn.Parameter(torch.zeros(in_channels))\n\n            self.rbr_skip = (\n                norm_layer\n                if out_channels == in_channels and stride == 1\n                else None\n            )\n\n            # Re-parameterizable conv branches\n            if num_conv_branches > 0:\n                rbr_conv = list()\n                for _ in range(self.num_conv_branches):\n                    rbr_conv.append(\n                        self._conv_bn(kernel_size=kernel_size, padding=padding)\n                    )\n                self.rbr_conv = nn.ModuleList(rbr_conv)\n            else:\n                self.rbr_conv = None\n\n            # Re-parameterizable scale branch\n            self.rbr_scale = None\n            if not isinstance(kernel_size, int):\n                kernel_size = kernel_size[0]\n            if (kernel_size > 1) and use_scale_branch:\n                self.rbr_scale = self._conv_bn(kernel_size=1, padding=0)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        \"\"\"Apply forward pass.\"\"\"\n        # Inference mode forward pass.\n        if self.inference_mode:\n            return self.activation(self.se(self.reparam_conv(x)))\n\n        # Multi-branched train-time forward pass.\n        # Skip branch output\n        identity_out = 0\n        if self.rbr_skip is not None:\n            identity_out = self.rbr_skip(x)\n\n        # Scale branch output\n        scale_out = 0\n        if self.rbr_scale is not None:\n            scale_out = self.rbr_scale(x)\n\n        # Other branches\n        out = scale_out + identity_out\n        if self.rbr_conv is not None:\n            for ix in range(self.num_conv_branches):\n                out += self.rbr_conv[ix](x)\n\n        return self.activation(self.se(out))\n\n    def reparameterize(self):\n        \"\"\"Following works like `RepVGG: Making VGG-style ConvNets Great Again` -\n        https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched\n        architecture used at training time to obtain a plain CNN-like structure\n        for inference.\n        \"\"\"\n        if self.inference_mode:\n            return\n        kernel, bias = self._get_kernel_bias()\n        self.reparam_conv = nn.Conv2d(\n            in_channels=self.in_channels,\n            out_channels=self.out_channels,\n            kernel_size=self.kernel_size,\n            stride=self.stride,\n            padding=self.padding,\n            dilation=self.dilation,\n            groups=self.groups,\n            bias=True,\n        )\n        self.reparam_conv.weight.data = kernel\n        self.reparam_conv.bias.data = bias\n\n        # Delete un-used branches\n        self.__delattr__(\"rbr_conv\")\n        self.__delattr__(\"rbr_scale\")\n        if hasattr(self, \"rbr_skip\"):\n            self.__delattr__(\"rbr_skip\")\n\n        self.inference_mode = True\n\n    def _get_kernel_bias(self) -> Tuple[torch.Tensor, torch.Tensor]:\n        \"\"\"Method to obtain re-parameterized kernel and bias.\n        Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L83\n\n        Returns:\n            Tuple of (kernel, bias) after fusing branches.\n        \"\"\"\n        # get weights and bias of scale branch\n        kernel_scale = 0\n        bias_scale = 0\n        if self.rbr_scale is not None:\n            kernel_scale, bias_scale = self._fuse_bn_tensor(self.rbr_scale)\n            # Pad scale branch kernel to match conv branch kernel size.\n            pad = self.kernel_size // 2\n            kernel_scale = torch.nn.functional.pad(kernel_scale, [pad, pad, pad, pad])\n\n        # get weights and bias of skip branch\n        kernel_identity = 0\n        bias_identity = 0\n        if self.rbr_skip is not None:\n            kernel_identity, bias_identity = self._fuse_bn_tensor(self.rbr_skip)\n\n        # get weights and bias of conv branches\n        kernel_conv = 0\n        bias_conv = 0\n        if self.rbr_conv is not None:\n            for ix in range(self.num_conv_branches):\n                _kernel, _bias = self._fuse_bn_tensor(self.rbr_conv[ix])\n                kernel_conv += _kernel\n                bias_conv += _bias\n\n        kernel_final = kernel_conv + kernel_scale + kernel_identity\n        bias_final = bias_conv + bias_scale + bias_identity\n        return kernel_final, bias_final\n\n    def _fuse_bn_tensor(\n        self, branch: Union[nn.Sequential, nn.BatchNorm2d]\n    ) -> Tuple[torch.Tensor, torch.Tensor]:\n        \"\"\"Method to fuse batchnorm layer with preceeding conv layer.\n        Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L95\n\n        Args:\n            branch: Sequence of ops to be fused.\n\n        Returns:\n            Tuple of (kernel, bias) after fusing batchnorm.\n        \"\"\"\n        if isinstance(branch, nn.Sequential):\n            kernel = branch.conv.weight\n            running_mean = branch.bn.running_mean\n            running_var = branch.bn.running_var\n            gamma = branch.bn.weight\n            beta = branch.bn.bias\n            eps = branch.bn.eps\n        else:\n            assert isinstance(branch, nn.BatchNorm2d)\n            if not hasattr(self, \"id_tensor\"):\n                input_dim = self.in_channels // self.groups\n\n                kernel_size = self.kernel_size\n                if isinstance(self.kernel_size, int):\n                    kernel_size = (self.kernel_size, self.kernel_size)\n\n                kernel_value = torch.zeros(\n                    (self.in_channels, input_dim, kernel_size[0], kernel_size[1]),\n                    dtype=branch.weight.dtype,\n                    device=branch.weight.device,\n                )\n                for i in range(self.in_channels):\n                    kernel_value[\n                        i, i % input_dim, kernel_size[0] // 2, kernel_size[1] // 2\n                    ] = 1\n                self.id_tensor = kernel_value\n            kernel = self.id_tensor\n            running_mean = branch.running_mean\n            running_var = branch.running_var\n            gamma = branch.weight\n            beta = branch.bias\n            eps = branch.eps\n        std = (running_var + eps).sqrt()\n        t = (gamma / std).reshape(-1, 1, 1, 1)\n        return kernel * t, beta - running_mean * gamma / std\n\n    def _conv_bn(self, kernel_size: int, padding: int) -> nn.Sequential:\n        \"\"\"Helper method to construct conv-batchnorm layers.\n\n        Args:\n            kernel_size: Size of the convolution kernel.\n            padding: Zero-padding size.\n\n        Returns:\n            Conv-BN module.\n        \"\"\"\n        # Fallback, sometimes batchnorm tensors\n        # do not get instantiated correctly on some processes\n        # when using deepspeed + accelerate\n        norm_layer = nn.BatchNorm2d(num_features=self.out_channels)\n        if norm_layer.weight.shape[0] == 0:\n            norm_layer.weight = nn.Parameter(torch.zeros(self.out_channels))\n        if norm_layer.bias.shape[0] == 0:\n            norm_layer.bias = nn.Parameter(torch.zeros(self.out_channels))\n\n        mod_list = nn.Sequential()\n        mod_list.add_module(\n            \"conv\",\n            nn.Conv2d(\n                in_channels=self.in_channels,\n                out_channels=self.out_channels,\n                kernel_size=kernel_size,\n                stride=self.stride,\n                padding=padding,\n                groups=self.groups,\n                bias=False,\n            ),\n        )\n        mod_list.add_module(\"bn\", norm_layer)\n        return mod_list\n\n\nclass ReparamLargeKernelConv(nn.Module):\n    \"\"\"Building Block of RepLKNet\n\n    This class defines overparameterized large kernel conv block\n    introduced in `RepLKNet <https://arxiv.org/abs/2203.06717>`_\n\n    Reference: https://github.com/DingXiaoH/RepLKNet-pytorch\n    \"\"\"\n\n    def __init__(\n        self,\n        in_channels: int,\n        out_channels: int,\n        kernel_size: int,\n        stride: int,\n        groups: int,\n        small_kernel: int,\n        inference_mode: bool = False,\n        use_se: bool = False,\n        activation: nn.Module = nn.GELU(),\n    ) -> None:\n        \"\"\"Construct a ReparamLargeKernelConv module.\n\n        Args:\n            in_channels: Number of input channels.\n            out_channels: Number of output channels.\n            kernel_size: Kernel size of the large kernel conv branch.\n            stride: Stride size. Default: 1\n            groups: Group number. Default: 1\n            small_kernel: Kernel size of small kernel conv branch.\n            inference_mode: If True, instantiates model in inference mode. Default: ``False``\n            activation: Activation module. Default: ``nn.GELU``\n        \"\"\"\n        super(ReparamLargeKernelConv, self).__init__()\n\n        self.stride = stride\n        self.groups = groups\n        self.in_channels = in_channels\n        self.out_channels = out_channels\n        self.activation = activation\n\n        self.kernel_size = kernel_size\n        self.small_kernel = small_kernel\n        self.padding = kernel_size // 2\n\n        # Check if SE is requested\n        if use_se:\n            self.se = SqueezeExcite(out_channels, rd_ratio=0.25)\n        else:\n            self.se = nn.Identity()\n\n        if inference_mode:\n            self.lkb_reparam = nn.Conv2d(\n                in_channels=in_channels,\n                out_channels=out_channels,\n                kernel_size=kernel_size,\n                stride=stride,\n                padding=self.padding,\n                dilation=1,\n                groups=groups,\n                bias=True,\n            )\n        else:\n            self.lkb_origin = self._conv_bn(\n                kernel_size=kernel_size, padding=self.padding\n            )\n            if small_kernel is not None:\n                assert (\n                    small_kernel <= kernel_size\n                ), \"The kernel size for re-param cannot be larger than the large kernel!\"\n                self.small_conv = self._conv_bn(\n                    kernel_size=small_kernel, padding=small_kernel // 2\n                )\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        \"\"\"Apply forward pass.\"\"\"\n        if hasattr(self, \"lkb_reparam\"):\n            out = self.lkb_reparam(x)\n        else:\n            out = self.lkb_origin(x)\n            if hasattr(self, \"small_conv\"):\n                out += self.small_conv(x)\n\n        return self.activation(self.se(out))\n\n    def get_kernel_bias(self) -> Tuple[torch.Tensor, torch.Tensor]:\n        \"\"\"Method to obtain re-parameterized kernel and bias.\n        Reference: https://github.com/DingXiaoH/RepLKNet-pytorch\n\n        Returns:\n            Tuple of (kernel, bias) after fusing branches.\n        \"\"\"\n        eq_k, eq_b = self._fuse_bn(self.lkb_origin.conv, self.lkb_origin.bn)\n        if hasattr(self, \"small_conv\"):\n            small_k, small_b = self._fuse_bn(self.small_conv.conv, self.small_conv.bn)\n            eq_b += small_b\n            eq_k += nn.functional.pad(\n                small_k, [(self.kernel_size - self.small_kernel) // 2] * 4\n            )\n        return eq_k, eq_b\n\n    def reparameterize(self) -> None:\n        \"\"\"\n        Following works like `RepVGG: Making VGG-style ConvNets Great Again` -\n        https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched\n        architecture used at training time to obtain a plain CNN-like structure\n        for inference.\n        \"\"\"\n        eq_k, eq_b = self.get_kernel_bias()\n        self.lkb_reparam = nn.Conv2d(\n            in_channels=self.in_channels,\n            out_channels=self.out_channels,\n            kernel_size=self.kernel_size,\n            stride=self.stride,\n            padding=self.padding,\n            dilation=self.lkb_origin.conv.dilation,\n            groups=self.groups,\n            bias=True,\n        )\n\n        self.lkb_reparam.weight.data = eq_k\n        self.lkb_reparam.bias.data = eq_b\n        self.__delattr__(\"lkb_origin\")\n        if hasattr(self, \"small_conv\"):\n            self.__delattr__(\"small_conv\")\n\n    @staticmethod\n    def _fuse_bn(\n        conv: torch.Tensor, bn: nn.BatchNorm2d\n    ) -> Tuple[torch.Tensor, torch.Tensor]:\n        \"\"\"Method to fuse batchnorm layer with conv layer.\n\n        Args:\n            conv: Convolutional kernel weights.\n            bn: Batchnorm 2d layer.\n\n        Returns:\n            Tuple of (kernel, bias) after fusing batchnorm.\n        \"\"\"\n        kernel = conv.weight\n        running_mean = bn.running_mean\n        running_var = bn.running_var\n        gamma = bn.weight\n        beta = bn.bias\n        eps = bn.eps\n        std = (running_var + eps).sqrt()\n        t = (gamma / std).reshape(-1, 1, 1, 1)\n        return kernel * t, beta - running_mean * gamma / std\n\n    def _conv_bn(self, kernel_size: int, padding: int = 0) -> nn.Sequential:\n        \"\"\"Helper method to construct conv-batchnorm layers.\n\n        Args:\n            kernel_size: Size of the convolution kernel.\n            padding: Zero-padding size.\n\n        Returns:\n            A nn.Sequential Conv-BN module.\n        \"\"\"\n        # Fallback, sometimes batchnorm tensors\n        # do not get instantiated correctly on some processes\n        # when using deepspeed + accelerate\n        norm_layer = nn.BatchNorm2d(num_features=self.out_channels)\n        if norm_layer.weight.shape[0] == 0:\n            norm_layer.weight = nn.Parameter(torch.zeros(self.out_channels))\n        if norm_layer.bias.shape[0] == 0:\n            norm_layer.bias = nn.Parameter(torch.zeros(self.out_channels))\n\n        mod_list = nn.Sequential()\n        mod_list.add_module(\n            \"conv\",\n            nn.Conv2d(\n                in_channels=self.in_channels,\n                out_channels=self.out_channels,\n                kernel_size=kernel_size,\n                stride=self.stride,\n                padding=padding,\n                groups=self.groups,\n                bias=False,\n            ),\n        )\n        mod_list.add_module(\"bn\", norm_layer)\n        return mod_list\n\n\ndef convolutional_stem(\n    in_channels: int, out_channels: int, inference_mode: bool = False, use_scale_branch: bool = True,\n) -> nn.Sequential:\n    \"\"\"Build convolutional stem with MobileOne blocks.\n\n    Args:\n        in_channels: Number of input channels.\n        out_channels: Number of output channels.\n        inference_mode: Flag to instantiate model in inference mode. Default: ``False``\n\n    Returns:\n        nn.Sequential object with stem elements.\n    \"\"\"\n    return nn.Sequential(\n        MobileOneBlock(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=3,\n            stride=2,\n            padding=1,\n            groups=1,\n            inference_mode=inference_mode,\n            use_se=False,\n            num_conv_branches=1,\n            use_scale_branch=use_scale_branch\n        ),\n        MobileOneBlock(\n            in_channels=out_channels,\n            out_channels=out_channels,\n            kernel_size=3,\n            stride=2,\n            padding=1,\n            groups=out_channels,\n            inference_mode=inference_mode,\n            use_se=False,\n            num_conv_branches=1,\n            use_scale_branch=use_scale_branch\n        ),\n        MobileOneBlock(\n            in_channels=out_channels,\n            out_channels=out_channels,\n            kernel_size=1,\n            stride=1,\n            padding=0,\n            groups=1,\n            inference_mode=inference_mode,\n            use_se=False,\n            num_conv_branches=1,\n            use_scale_branch=use_scale_branch\n        ),\n    )\n\n\nclass LayerNormChannel(nn.Module):\n    \"\"\"\n    LayerNorm only for Channel Dimension.\n    Input: tensor in shape [B, C, H, W]\n    \"\"\"\n    def __init__(self, num_features, eps=1e-05) -> None:\n        super().__init__()\n        self.weight = nn.Parameter(torch.ones(num_features))\n        self.bias = nn.Parameter(torch.zeros(num_features))\n        self.eps = eps\n\n    def forward(self, x) -> torch.Tensor:\n        u = x.mean(1, keepdim=True)\n        s = (x - u).pow(2).mean(1, keepdim=True)\n        x = (x - u) / torch.sqrt(s + self.eps)\n        x = self.weight.unsqueeze(-1).unsqueeze(-1) * x \\\n            + self.bias.unsqueeze(-1).unsqueeze(-1)\n        return x\n\n\nclass MHSA(nn.Module):\n    \"\"\"Multi-headed Self Attention module.\n\n    Source modified from:\n    https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py\n    \"\"\"\n\n    def __init__(\n        self,\n        dim: int,\n        head_dim: int = 32,\n        qkv_bias: bool = False,\n        attn_drop: float = 0.0,\n        proj_drop: float = 0.0,\n    ) -> None:\n        \"\"\"Build MHSA module that can handle 3D or 4D input tensors.\n\n        Args:\n            dim: Number of embedding dimensions.\n            head_dim: Number of hidden dimensions per head. Default: ``32``\n            qkv_bias: Use bias or not. Default: ``False``\n            attn_drop: Dropout rate for attention tensor.\n            proj_drop: Dropout rate for projection tensor.\n        \"\"\"\n        super().__init__()\n        assert dim % head_dim == 0, \"dim should be divisible by head_dim\"\n        self.head_dim = head_dim\n        self.num_heads = dim // head_dim\n        self.scale = head_dim**-0.5\n\n        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)\n        self.attn_drop = nn.Dropout(attn_drop)\n        self.proj = nn.Linear(dim, dim)\n        self.proj_drop = nn.Dropout(proj_drop)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        shape = x.shape\n        B, C, H, W = shape\n        N = H * W\n        if len(shape) == 4:\n            x = torch.flatten(x, start_dim=2).transpose(-2, -1)  # (B, N, C)\n        qkv = (\n            self.qkv(x)\n            .reshape(B, N, 3, self.num_heads, self.head_dim)\n            .permute(2, 0, 3, 1, 4)\n        )\n        q, k, v = qkv.unbind(0)  # make torchscript happy (cannot use tensor as tuple)\n\n        # trick here to make q@k.t more stable\n        attn = (q * self.scale) @ k.transpose(-2, -1)\n        attn = attn.softmax(dim=-1)\n        attn = self.attn_drop(attn)\n\n        x = (attn @ v).transpose(1, 2).reshape(B, N, C)\n        x = self.proj(x)\n        x = self.proj_drop(x)\n        if len(shape) == 4:\n            x = x.transpose(-2, -1).reshape(B, C, H, W)\n\n        return x\n\n\nclass PatchEmbed(nn.Module):\n    \"\"\"Convolutional patch embedding layer.\"\"\"\n\n    def __init__(\n        self,\n        patch_size: int,\n        stride: int,\n        in_channels: int,\n        embed_dim: int,\n        inference_mode: bool = False,\n        use_se: bool = False,\n    ) -> None:\n        \"\"\"Build patch embedding layer.\n\n        Args:\n            patch_size: Patch size for embedding computation.\n            stride: Stride for convolutional embedding layer.\n            in_channels: Number of channels of input tensor.\n            embed_dim: Number of embedding dimensions.\n            inference_mode: Flag to instantiate model in inference mode. Default: ``False``\n            use_se: If ``True`` SE block will be used.\n        \"\"\"\n        super().__init__()\n        block = list()\n        block.append(\n            ReparamLargeKernelConv(\n                in_channels=in_channels,\n                out_channels=embed_dim,\n                kernel_size=patch_size,\n                stride=stride,\n                groups=in_channels,\n                small_kernel=3,\n                inference_mode=inference_mode,\n                use_se=use_se,\n            )\n        )\n        block.append(\n            MobileOneBlock(\n                in_channels=embed_dim,\n                out_channels=embed_dim,\n                kernel_size=1,\n                stride=1,\n                padding=0,\n                groups=1,\n                inference_mode=inference_mode,\n                use_se=False,\n                num_conv_branches=1,\n            )\n        )\n        self.proj = nn.Sequential(*block)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        x = self.proj(x)\n        return x\n\n\nclass RepMixer(nn.Module):\n    \"\"\"Reparameterizable token mixer.\n\n    For more details, please refer to our paper:\n    `FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization <https://arxiv.org/pdf/2303.14189.pdf>`_\n    \"\"\"\n\n    def __init__(\n        self,\n        dim,\n        kernel_size=3,\n        use_layer_scale=True,\n        layer_scale_init_value=1e-5,\n        inference_mode: bool = False,\n    ):\n        \"\"\"Build RepMixer Module.\n\n        Args:\n            dim: Input feature map dimension. :math:`C_{in}` from an expected input of size :math:`(B, C_{in}, H, W)`.\n            kernel_size: Kernel size for spatial mixing. Default: 3\n            use_layer_scale: If True, learnable layer scale is used. Default: ``True``\n            layer_scale_init_value: Initial value for layer scale. Default: 1e-5\n            inference_mode: If True, instantiates model in inference mode. Default: ``False``\n        \"\"\"\n        super().__init__()\n        self.dim = dim\n        self.kernel_size = kernel_size\n        self.inference_mode = inference_mode\n\n        if inference_mode:\n            self.reparam_conv = nn.Conv2d(\n                in_channels=self.dim,\n                out_channels=self.dim,\n                kernel_size=self.kernel_size,\n                stride=1,\n                padding=self.kernel_size // 2,\n                groups=self.dim,\n                bias=True,\n            )\n        else:\n            self.norm = MobileOneBlock(\n                dim,\n                dim,\n                kernel_size,\n                padding=kernel_size // 2,\n                groups=dim,\n                use_act=False,\n                use_scale_branch=False,\n                num_conv_branches=0,\n            )\n            self.mixer = MobileOneBlock(\n                dim,\n                dim,\n                kernel_size,\n                padding=kernel_size // 2,\n                groups=dim,\n                use_act=False,\n            )\n            self.use_layer_scale = use_layer_scale\n            if use_layer_scale:\n                self.layer_scale = nn.Parameter(\n                    layer_scale_init_value * torch.ones((dim, 1, 1)), requires_grad=True\n                )\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        if hasattr(self, \"reparam_conv\"):\n            x = self.reparam_conv(x)\n            return x\n        else:\n            if self.use_layer_scale:\n                x = x + self.layer_scale * (self.mixer(x) - self.norm(x))\n            else:\n                x = x + self.mixer(x) - self.norm(x)\n            return x\n\n    def reparameterize(self) -> None:\n        \"\"\"Reparameterize mixer and norm into a single\n        convolutional layer for efficient inference.\n        \"\"\"\n        if self.inference_mode:\n            return\n\n        self.mixer.reparameterize()\n        self.norm.reparameterize()\n\n        if self.use_layer_scale:\n            w = self.mixer.id_tensor + self.layer_scale.unsqueeze(-1) * (\n                self.mixer.reparam_conv.weight - self.norm.reparam_conv.weight\n            )\n            b = torch.squeeze(self.layer_scale) * (\n                self.mixer.reparam_conv.bias - self.norm.reparam_conv.bias\n            )\n        else:\n            w = (\n                self.mixer.id_tensor\n                + self.mixer.reparam_conv.weight\n                - self.norm.reparam_conv.weight\n            )\n            b = self.mixer.reparam_conv.bias - self.norm.reparam_conv.bias\n\n        self.reparam_conv = nn.Conv2d(\n            in_channels=self.dim,\n            out_channels=self.dim,\n            kernel_size=self.kernel_size,\n            stride=1,\n            padding=self.kernel_size // 2,\n            groups=self.dim,\n            bias=True,\n        )\n        self.reparam_conv.weight.data = w\n        self.reparam_conv.bias.data = b\n\n        self.__delattr__(\"mixer\")\n        self.__delattr__(\"norm\")\n        if self.use_layer_scale:\n            self.__delattr__(\"layer_scale\")\n\n\nclass ConvFFN(nn.Module):\n    \"\"\"Convolutional FFN Module.\"\"\"\n\n    def __init__(\n        self,\n        in_channels: int,\n        hidden_channels: Optional[int] = None,\n        out_channels: Optional[int] = None,\n        act_layer: nn.Module = nn.GELU,\n        drop: float = 0.0,\n    ) -> None:\n        \"\"\"Build convolutional FFN module.\n\n        Args:\n            in_channels: Number of input channels.\n            hidden_channels: Number of channels after expansion. Default: None\n            out_channels: Number of output channels. Default: None\n            act_layer: Activation layer. Default: ``GELU``\n            drop: Dropout rate. Default: ``0.0``.\n        \"\"\"\n        super().__init__()\n        out_channels = out_channels or in_channels\n        hidden_channels = hidden_channels or in_channels\n        self.conv = nn.Sequential()\n        self.conv.add_module(\n            \"conv\",\n            nn.Conv2d(\n                in_channels=in_channels,\n                out_channels=out_channels,\n                kernel_size=7,\n                padding=3,\n                groups=in_channels,\n                bias=False,\n            ),\n        )\n\n        # Fallback, sometimes batchnorm tensors\n        # do not get instantiated correctly on some processes\n        # when using deepspeed + accelerate\n        norm_layer = nn.BatchNorm2d(num_features=out_channels)\n        if norm_layer.weight.shape[0] == 0:\n            norm_layer.weight = nn.Parameter(torch.zeros(out_channels))\n        if norm_layer.bias.shape[0] == 0:\n            norm_layer.bias = nn.Parameter(torch.zeros(out_channels))\n\n        self.conv.add_module(\"bn\", norm_layer)\n        self.fc1 = nn.Conv2d(in_channels, hidden_channels, kernel_size=1)\n        self.act = act_layer()\n        self.fc2 = nn.Conv2d(hidden_channels, out_channels, kernel_size=1)\n        self.drop = nn.Dropout(drop)\n        self.apply(self._init_weights)\n\n    def _init_weights(self, m: nn.Module) -> None:\n        if isinstance(m, nn.Conv2d):\n            normal_(m.weight, std=0.02)\n            if m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        x = self.conv(x)\n        x = self.fc1(x)\n        x = self.act(x)\n        x = self.drop(x)\n        x = self.fc2(x)\n        x = self.drop(x)\n        return x\n\n\nclass RepCPE(nn.Module):\n    \"\"\"Implementation of conditional positional encoding.\n\n    For more details refer to paper:\n    `Conditional Positional Encodings for Vision Transformers <https://arxiv.org/pdf/2102.10882.pdf>`_\n\n    In our implementation, we can reparameterize this module to eliminate a skip connection.\n    \"\"\"\n\n    def __init__(\n        self,\n        in_channels: int,\n        embed_dim: int = 768,\n        spatial_shape: Union[int, Tuple[int, int]] = (7, 7),\n        inference_mode=False,\n    ) -> None:\n        \"\"\"Build reparameterizable conditional positional encoding\n\n        Args:\n            in_channels: Number of input channels.\n            embed_dim: Number of embedding dimensions. Default: 768\n            spatial_shape: Spatial shape of kernel for positional encoding. Default: (7, 7)\n            inference_mode: Flag to instantiate block in inference mode. Default: ``False``\n        \"\"\"\n        super(RepCPE, self).__init__()\n        if isinstance(spatial_shape, int):\n            spatial_shape = tuple([spatial_shape] * 2)\n        assert isinstance(spatial_shape, Tuple), (\n            f'\"spatial_shape\" must by a sequence or int, '\n            f\"get {type(spatial_shape)} instead.\"\n        )\n        assert len(spatial_shape) == 2, (\n            f'Length of \"spatial_shape\" should be 2, '\n            f\"got {len(spatial_shape)} instead.\"\n        )\n\n        self.spatial_shape = spatial_shape\n        self.embed_dim = embed_dim\n        self.in_channels = in_channels\n        self.groups = embed_dim\n\n        if inference_mode:\n            self.reparam_conv = nn.Conv2d(\n                in_channels=self.in_channels,\n                out_channels=self.embed_dim,\n                kernel_size=self.spatial_shape,\n                stride=1,\n                padding=int(self.spatial_shape[0] // 2),\n                groups=self.embed_dim,\n                bias=True,\n            )\n        else:\n            self.pe = nn.Conv2d(\n                in_channels,\n                embed_dim,\n                spatial_shape,\n                1,\n                int(spatial_shape[0] // 2),\n                bias=True,\n                groups=embed_dim,\n            )\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        if hasattr(self, \"reparam_conv\"):\n            x = self.reparam_conv(x)\n            return x\n        else:\n            x = self.pe(x) + x\n            return x\n\n    def reparameterize(self) -> None:\n        # Build equivalent Id tensor\n        input_dim = self.in_channels // self.groups\n        kernel_value = torch.zeros(\n            (\n                self.in_channels,\n                input_dim,\n                self.spatial_shape[0],\n                self.spatial_shape[1],\n            ),\n            dtype=self.pe.weight.dtype,\n            device=self.pe.weight.device,\n        )\n        for i in range(self.in_channels):\n            kernel_value[\n                i,\n                i % input_dim,\n                self.spatial_shape[0] // 2,\n                self.spatial_shape[1] // 2,\n            ] = 1\n        id_tensor = kernel_value\n\n        # Reparameterize Id tensor and conv\n        w_final = id_tensor + self.pe.weight\n        b_final = self.pe.bias\n\n        # Introduce reparam conv\n        self.reparam_conv = nn.Conv2d(\n            in_channels=self.in_channels,\n            out_channels=self.embed_dim,\n            kernel_size=self.spatial_shape,\n            stride=1,\n            padding=int(self.spatial_shape[0] // 2),\n            groups=self.embed_dim,\n            bias=True,\n        )\n        self.reparam_conv.weight.data = w_final\n        self.reparam_conv.bias.data = b_final\n\n        self.__delattr__(\"pe\")\n\n\nclass RepMixerBlock(nn.Module):\n    \"\"\"Implementation of Metaformer block with RepMixer as token mixer.\n\n    For more details on Metaformer structure, please refer to:\n    `MetaFormer Is Actually What You Need for Vision <https://arxiv.org/pdf/2111.11418.pdf>`_\n    \"\"\"\n\n    def __init__(\n        self,\n        dim: int,\n        kernel_size: int = 3,\n        mlp_ratio: float = 4.0,\n        act_layer: nn.Module = nn.GELU,\n        drop: float = 0.0,\n        drop_path: float = 0.0,\n        use_layer_scale: bool = True,\n        layer_scale_init_value: float = 1e-5,\n        inference_mode: bool = False,\n    ):\n        \"\"\"Build RepMixer Block.\n\n        Args:\n            dim: Number of embedding dimensions.\n            kernel_size: Kernel size for repmixer. Default: 3\n            mlp_ratio: MLP expansion ratio. Default: 4.0\n            act_layer: Activation layer. Default: ``nn.GELU``\n            drop: Dropout rate. Default: 0.0\n            drop_path: Drop path rate. Default: 0.0\n            use_layer_scale: Flag to turn on layer scale. Default: ``True``\n            layer_scale_init_value: Layer scale value at initialization. Default: 1e-5\n            inference_mode: Flag to instantiate block in inference mode. Default: ``False``\n        \"\"\"\n\n        super().__init__()\n\n        self.token_mixer = RepMixer(\n            dim,\n            kernel_size=kernel_size,\n            use_layer_scale=use_layer_scale,\n            layer_scale_init_value=layer_scale_init_value,\n            inference_mode=inference_mode,\n        )\n\n        assert mlp_ratio > 0, \"MLP ratio should be greater than 0, found: {}\".format(\n            mlp_ratio\n        )\n        mlp_hidden_dim = int(dim * mlp_ratio)\n        self.convffn = ConvFFN(\n            in_channels=dim,\n            hidden_channels=mlp_hidden_dim,\n            act_layer=act_layer,\n            drop=drop,\n        )\n\n        # Drop Path\n        self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()\n\n        # Layer Scale\n        self.use_layer_scale = use_layer_scale\n        if use_layer_scale:\n            self.layer_scale = nn.Parameter(\n                layer_scale_init_value * torch.ones((dim, 1, 1)), requires_grad=True\n            )\n\n    def forward(self, x):\n        if self.use_layer_scale:\n            x = self.token_mixer(x)\n            x = x + self.drop_path(self.layer_scale * self.convffn(x))\n        else:\n            x = self.token_mixer(x)\n            x = x + self.drop_path(self.convffn(x))\n        return x\n\n\nclass AttentionBlock(nn.Module):\n    \"\"\"Implementation of metaformer block with MHSA as token mixer.\n\n    For more details on Metaformer structure, please refer to:\n    `MetaFormer Is Actually What You Need for Vision <https://arxiv.org/pdf/2111.11418.pdf>`_\n    \"\"\"\n\n    def __init__(\n        self,\n        dim: int,\n        mlp_ratio: float = 4.0,\n        act_layer: nn.Module = nn.GELU,\n        norm_layer: nn.Module = nn.BatchNorm2d,\n        drop: float = 0.0,\n        drop_path: float = 0.0,\n        use_layer_scale: bool = True,\n        layer_scale_init_value: float = 1e-5,\n    ):\n        \"\"\"Build Attention Block.\n\n        Args:\n            dim: Number of embedding dimensions.\n            mlp_ratio: MLP expansion ratio. Default: 4.0\n            act_layer: Activation layer. Default: ``nn.GELU``\n            norm_layer: Normalization layer. Default: ``nn.BatchNorm2d``\n            drop: Dropout rate. Default: 0.0\n            drop_path: Drop path rate. Default: 0.0\n            use_layer_scale: Flag to turn on layer scale. Default: ``True``\n            layer_scale_init_value: Layer scale value at initialization. Default: 1e-5\n        \"\"\"\n\n        super().__init__()\n\n        # Fallback, sometimes batchnorm tensors\n        # do not get instantiated correctly on some processes\n        # when using deepspeed + accelerate\n        norm_layer_ = norm_layer(num_features=dim)\n        if norm_layer_.weight.shape[0] == 0:\n            norm_layer_.weight = nn.Parameter(torch.zeros(dim))\n        if norm_layer_.bias.shape[0] == 0:\n            norm_layer_.bias = nn.Parameter(torch.zeros(dim))\n\n        self.norm = norm_layer_\n        self.token_mixer = MHSA(dim=dim)\n\n        assert mlp_ratio > 0, \"MLP ratio should be greater than 0, found: {}\".format(\n            mlp_ratio\n        )\n        mlp_hidden_dim = int(dim * mlp_ratio)\n        self.convffn = ConvFFN(\n            in_channels=dim,\n            hidden_channels=mlp_hidden_dim,\n            act_layer=act_layer,\n            drop=drop,\n        )\n\n        # Drop path\n        self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()\n\n        # Layer Scale\n        self.use_layer_scale = use_layer_scale\n        if use_layer_scale:\n            self.layer_scale_1 = nn.Parameter(\n                layer_scale_init_value * torch.ones((dim, 1, 1)), requires_grad=True\n            )\n            self.layer_scale_2 = nn.Parameter(\n                layer_scale_init_value * torch.ones((dim, 1, 1)), requires_grad=True\n            )\n\n    def forward(self, x):\n        if self.use_layer_scale:\n            x = x + self.drop_path(self.layer_scale_1 * self.token_mixer(self.norm(x)))\n            x = x + self.drop_path(self.layer_scale_2 * self.convffn(x))\n        else:\n            x = x + self.drop_path(self.token_mixer(self.norm(x)))\n            x = x + self.drop_path(self.convffn(x))\n        return x\n\n\ndef basic_blocks(\n    dim: int,\n    block_index: int,\n    num_blocks: List[int],\n    token_mixer_type: str,\n    kernel_size: int = 3,\n    mlp_ratio: float = 4.0,\n    act_layer: nn.Module = nn.GELU,\n    norm_layer: nn.Module = nn.BatchNorm2d,\n    drop_rate: float = 0.0,\n    drop_path_rate: float = 0.0,\n    use_layer_scale: bool = True,\n    layer_scale_init_value: float = 1e-5,\n    inference_mode=False,\n) -> nn.Sequential:\n    \"\"\"Build FastViT blocks within a stage.\n\n    Args:\n        dim: Number of embedding dimensions.\n        block_index: block index.\n        num_blocks: List containing number of blocks per stage.\n        token_mixer_type: Token mixer type.\n        kernel_size: Kernel size for repmixer.\n        mlp_ratio: MLP expansion ratio.\n        act_layer: Activation layer.\n        norm_layer: Normalization layer.\n        drop_rate: Dropout rate.\n        drop_path_rate: Drop path rate.\n        use_layer_scale: Flag to turn on layer scale regularization.\n        layer_scale_init_value: Layer scale value at initialization.\n        inference_mode: Flag to instantiate block in inference mode.\n\n    Returns:\n        nn.Sequential object of all the blocks within the stage.\n    \"\"\"\n    blocks = []\n    for block_idx in range(num_blocks[block_index]):\n        block_dpr = (\n            drop_path_rate\n            * (block_idx + sum(num_blocks[:block_index]))\n            / (sum(num_blocks) - 1)\n        )\n        if token_mixer_type == \"repmixer\":\n            blocks.append(\n                RepMixerBlock(\n                    dim,\n                    kernel_size=kernel_size,\n                    mlp_ratio=mlp_ratio,\n                    act_layer=act_layer,\n                    drop=drop_rate,\n                    drop_path=block_dpr,\n                    use_layer_scale=use_layer_scale,\n                    layer_scale_init_value=layer_scale_init_value,\n                    inference_mode=inference_mode,\n                )\n            )\n        elif token_mixer_type == \"attention\":\n            blocks.append(\n                AttentionBlock(\n                    dim,\n                    mlp_ratio=mlp_ratio,\n                    act_layer=act_layer,\n                    norm_layer=norm_layer,\n                    drop=drop_rate,\n                    drop_path=block_dpr,\n                    use_layer_scale=use_layer_scale,\n                    layer_scale_init_value=layer_scale_init_value,\n                )\n            )\n        else:\n            raise ValueError(\n                \"Token mixer type: {} not supported\".format(token_mixer_type)\n            )\n    blocks = nn.Sequential(*blocks)\n    return blocks\n\n\nclass GlobalPool2D(nn.Module):\n    \"\"\"This class implements global pooling with linear projection.\"\"\"\n\n    def __init__(self, in_dim: int, out_dim: int, *args, **kwargs) -> None:\n        super().__init__()\n        scale = in_dim**-0.5\n        self.proj = nn.Parameter(scale * torch.randn(size=(in_dim, out_dim)))\n        self.in_dim = in_dim\n        self.out_dim = out_dim\n\n    def pool(self, x) -> Tensor:\n        if x.dim() == 4:\n            dims = [-2, -1]\n        elif x.dim() == 5:\n            dims = [-3, -2, -1]\n        x = torch.mean(x, dim=dims, keepdim=False)\n        return x\n\n    def forward(self, x: Tensor, *args, **kwargs) -> Tensor:\n        # x is of shape [batch, in_dim]\n        assert (\n            x.dim() == 4\n        ), \"Input should be 4-dimensional (Batch x in_dim x in_height x in_width). Got: {}\".format(\n            x.shape\n        )\n\n        # [batch, in_dim, in_height, in_width] --> [batch, in_dim]\n        x = self.pool(x)\n        # [batch, in_dim]  x [in_dim, out_dim] --> [batch, out_dim]\n        x = x @ self.proj\n        return x\n\n\nclass FastViT(nn.Module):\n    \"\"\"\n    This class implements `FastViT architecture <https://arxiv.org/pdf/2303.14189.pdf>`_\n    \"\"\"\n\n    def __init__(\n        self,\n        layers,\n        token_mixers: Tuple[str, ...],\n        embed_dims=None,\n        mlp_ratios=None,\n        downsamples=None,\n        se_downsamples=None,\n        repmixer_kernel_size=3,\n        norm_layer: nn.Module = nn.BatchNorm2d,\n        act_layer: nn.Module = nn.GELU,\n        num_classes=1000,\n        pos_embs=None,\n        down_patch_size=7,\n        down_stride=2,\n        drop_rate=0.0,\n        drop_path_rate=0.0,\n        use_layer_scale=True,\n        layer_scale_init_value=1e-5,\n        init_cfg=None,\n        pretrained=None,\n        cls_ratio=2.0,\n        inference_mode=False,\n        stem_scale_branch=True,\n        **kwargs,\n    ) -> None:\n\n        super().__init__()\n\n        self.num_classes = num_classes\n        if len(layers) == 4:\n            self.out_indices = [0, 2, 4, 7]\n        elif len(layers) == 5:\n            self.out_indices = [0, 2, 4, 7, 10]\n        else:\n            raise NotImplementedError(\"FPN is not implemented for more than 5 stages.\")\n\n        if pos_embs is None:\n            pos_embs = [None] * len(layers)\n\n        if se_downsamples is None:\n            se_downsamples = [False] * len(layers)\n\n        # Convolutional stem\n        self.patch_embed = convolutional_stem(3, embed_dims[0], inference_mode,\n                                              use_scale_branch=stem_scale_branch)\n\n        # Build the main stages of the network architecture\n        network = []\n        for i in range(len(layers)):\n            # Add position embeddings if requested\n            if pos_embs[i] is not None:\n                network.append(\n                    pos_embs[i](\n                        embed_dims[i], embed_dims[i], inference_mode=inference_mode\n                    )\n                )\n            stage = basic_blocks(\n                embed_dims[i],\n                i,\n                layers,\n                token_mixer_type=token_mixers[i],\n                kernel_size=repmixer_kernel_size,\n                mlp_ratio=mlp_ratios[i],\n                act_layer=act_layer,\n                norm_layer=norm_layer,\n                drop_rate=drop_rate,\n                drop_path_rate=drop_path_rate,\n                use_layer_scale=use_layer_scale,\n                layer_scale_init_value=layer_scale_init_value,\n                inference_mode=inference_mode,\n            )\n            network.append(stage)\n            if i >= len(layers) - 1:\n                break\n\n            # Patch merging/downsampling between stages.\n            if downsamples[i] or embed_dims[i] != embed_dims[i + 1]:\n                network.append(\n                    PatchEmbed(\n                        patch_size=down_patch_size,\n                        stride=down_stride,\n                        in_channels=embed_dims[i],\n                        embed_dim=embed_dims[i + 1],\n                        inference_mode=inference_mode,\n                        use_se=se_downsamples[i + 1],\n                    )\n                )\n        self.network = nn.ModuleList(network)\n\n        # Classifier head\n        self.conv_exp = MobileOneBlock(\n            in_channels=embed_dims[-1],\n            out_channels=int(embed_dims[-1] * cls_ratio),\n            kernel_size=3,\n            stride=1,\n            padding=1,\n            groups=embed_dims[-1],\n            inference_mode=inference_mode,\n            use_se=True,\n            num_conv_branches=1,\n        )\n        self.head = (\n            nn.Linear(int(embed_dims[-1] * cls_ratio), num_classes)\n            if num_classes > 0\n            else nn.Identity()\n        )\n        self.apply(self.cls_init_weights)\n        self.init_cfg = copy.deepcopy(init_cfg)\n\n    def cls_init_weights(self, m: nn.Module) -> None:\n        \"\"\"Init. for classification\"\"\"\n        if isinstance(m, nn.Linear):\n            normal_(m.weight, std=0.02)\n            if isinstance(m, nn.Linear) and m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n\n    def forward_embeddings(self, x: torch.Tensor) -> torch.Tensor:\n        x = self.patch_embed(x)\n        return x\n\n    def forward_tokens(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor:\n        for idx, block in enumerate(self.network):\n            x = block(x)\n        return x\n\n    def forward(self, x: torch.Tensor, *args, **kwargs) -> Union[Tensor, Dict[str, Tensor]]:\n        # input embedding\n        x = self.forward_embeddings(x)\n        # through backbone\n        x = self.forward_tokens(x)\n        # for image classification/embedding\n        x = self.conv_exp(x)\n        cls_out = self.head(x)\n\n        out_dict = dict()\n        if kwargs.get(\"return_image_embeddings\", False):\n            out_dict.update({\"logits\": cls_out})\n            out_dict.update({\"image_embeddings\": x})\n            return out_dict\n        else:\n            return cls_out\n\n\n@register_model\ndef fastvithd(pretrained=False, **kwargs):\n    \"\"\"Instantiate FastViTHD model variant.\"\"\"\n    layers = [2, 12, 24, 4, 2]\n    embed_dims = [96, 192, 384, 768, 1536]\n    mlp_ratios = [4, 4, 4, 4, 4]\n    downsamples = [True, True, True, True, True]\n    pos_embs = [None, None, None, partial(RepCPE, spatial_shape=(7, 7)), partial(RepCPE, spatial_shape=(7, 7))]\n    token_mixers = (\"repmixer\", \"repmixer\", \"repmixer\", \"attention\", \"attention\")\n    model = FastViT(\n        layers,\n        token_mixers=token_mixers,\n        embed_dims=embed_dims,\n        pos_embs=pos_embs,\n        mlp_ratios=mlp_ratios,\n        downsamples=downsamples,\n        norm_layer=LayerNormChannel,\n        stem_scale_branch=False,\n        inference_mode=True,\n        **kwargs,\n    )\n    model.default_cfg = default_cfgs[\"fastvit_m\"]\n    if pretrained:\n        raise ValueError(\"Functionality not implemented.\")\n    return model\n"
  },
  {
    "path": "llava/model/multimodal_encoder/mobileclip_encoder.py",
    "content": "#\n# For licensing see accompanying LICENSE file.\n# Copyright (C) 2025 Apple Inc. All Rights Reserved.\n#\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom transformers import CLIPImageProcessor\nimport llava.model.multimodal_encoder.mobileclip as mobileclip\n\n\nclass MobileCLIPVisionTower(nn.Module):\n    def __init__(self, vision_tower, args, delay_load=False):\n        super().__init__()\n\n        self.is_loaded = False\n        self.vision_tower_name = vision_tower\n        self.tune_vision_tower = getattr(args, 'unfreeze_mm_vision_tower', False)\n        self.input_image_size = int(vision_tower.split(\"_\")[-1])\n\n        # Delay load is disabled for now\n        if not delay_load:\n            self.load_model()\n        elif getattr(args, 'unfreeze_mm_vision_tower', False):\n            self.load_model()\n        else:\n            model_cfg = mobileclip.load_model_config(self.vision_tower_name)\n            self.cfg_only = model_cfg\n\n    def load_model(self, device_map=None):\n        if self.is_loaded:\n            print('{} is already loaded, `load_model` called again, skipping.'.format(self.vision_tower_name))\n            return\n\n        # Load model config\n        model_cfg = mobileclip.load_model_config(self.vision_tower_name)\n\n        # Override default image resolution\n        model_cfg[\"image_cfg\"][\"image_size\"] = self.input_image_size\n\n        self.cfg_only = model_cfg\n\n        # Build HF CLIPImageProcessor with MobileCLIP parameters\n        self.image_processor = CLIPImageProcessor(crop_size={\"height\": model_cfg[\"image_cfg\"][\"image_size\"],\n                                                             \"width\": model_cfg[\"image_cfg\"][\"image_size\"]},\n                                                  image_mean=[0.0, 0.0, 0.0],\n                                                  image_std=[1.0, 1.0, 1.0],\n                                                  size={\"shortest_edge\": model_cfg[\"image_cfg\"][\"image_size\"]})\n\n        # Instantiate the image encoder\n        self.vision_tower = mobileclip.MCi(model_name=model_cfg[\"image_cfg\"][\"model_name\"],\n                                           projection_dim=model_cfg[\"embed_dim\"])\n\n        if not self.tune_vision_tower:\n            self.vision_tower.requires_grad_(False)\n\n        self.is_loaded = True\n\n    def feature_select(self, image_forward_outs):\n        # Features from penultimate layer\n        image_features = image_forward_outs[\"image_embeddings\"]\n\n        # Reshape 4D tensor to 3D\n        B, C, H, W = image_features.shape\n        image_features = image_features.reshape(B, C, H*W)\n        image_features = image_features.transpose(1, 2)\n        return image_features\n\n    def forward(self, images):\n        if self.tune_vision_tower:\n            return self.forward_images(images)\n        else:\n            with torch.no_grad():\n                return self.forward_images(images)\n\n    def forward_images(self, images):\n        if type(images) is list:\n            image_features = []\n            for image in images:\n                image_forward_out = self.vision_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), return_image_embeddings=True)\n                image_feature = self.feature_select(image_forward_out).to(image.dtype)\n                image_features.append(image_feature)\n        else:\n            image_forward_outs = self.vision_tower(images.to(device=self.device, dtype=self.dtype), return_image_embeddings=True)\n            image_features = self.feature_select(image_forward_outs).to(images.dtype)\n\n        return image_features\n\n    @property\n    def dummy_feature(self):\n        return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)\n\n    @property\n    def dtype(self):\n        return next(self.vision_tower.parameters()).dtype\n\n    @property\n    def device(self):\n        return next(self.vision_tower.parameters()).device\n\n    @property\n    def config(self):\n        return self.cfg_only\n\n    @property\n    def hidden_size(self):\n        return self.config[\"image_cfg\"][\"embed_dim\"]\n\n    @property\n    def num_patches_per_side(self):\n        return self.config[\"image_cfg\"][\"image_size\"] // self.config[\"image_cfg\"][\"patch_size\"]\n\n    @property\n    def num_patches(self):\n        return (self.config[\"image_cfg\"][\"image_size\"] // self.config[\"image_cfg\"][\"patch_size\"]) ** 2\n"
  },
  {
    "path": "llava/model/multimodal_projector/builder.py",
    "content": "import torch.nn as nn\nimport re\n\n\nclass IdentityMap(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, x, *args, **kwargs):\n        return x\n\n    @property\n    def config(self):\n        return {\"mm_projector_type\": 'identity'}\n\n\ndef build_vision_projector(config, delay_load=False, **kwargs):\n    projector_type = getattr(config, 'mm_projector_type', 'linear')\n\n    if projector_type == 'linear':\n        return nn.Linear(config.mm_hidden_size, config.hidden_size)\n\n    mlp_gelu_match = re.match(r'^mlp(\\d+)x_gelu$', projector_type)\n    if mlp_gelu_match:\n        mlp_depth = int(mlp_gelu_match.group(1))\n        modules = [nn.Linear(config.mm_hidden_size, config.hidden_size)]\n        for _ in range(1, mlp_depth):\n            modules.append(nn.GELU())\n            modules.append(nn.Linear(config.hidden_size, config.hidden_size))\n        return nn.Sequential(*modules)\n\n    if projector_type == 'identity':\n        return IdentityMap()\n\n    raise ValueError(f'Unknown projector type: {projector_type}')\n"
  },
  {
    "path": "llava/model/utils.py",
    "content": "from transformers import AutoConfig\n\n\ndef auto_upgrade(config):\n    cfg = AutoConfig.from_pretrained(config)\n    if 'llava' in config and 'llava' not in cfg.model_type:\n        assert cfg.model_type == 'llama'\n        print(\"You are using newer LLaVA code base, while the checkpoint of v0 is from older code base.\")\n        print(\"You must upgrade the checkpoint to the new code base (this can be done automatically).\")\n        confirm = input(\"Please confirm that you want to upgrade the checkpoint. [Y/N]\")\n        if confirm.lower() in [\"y\", \"yes\"]:\n            print(\"Upgrading checkpoint...\")\n            assert len(cfg.architectures) == 1\n            setattr(cfg.__class__, \"model_type\", \"llava\")\n            cfg.architectures[0] = 'LlavaLlamaForCausalLM'\n            cfg.save_pretrained(config)\n            print(\"Checkpoint upgraded.\")\n        else:\n            print(\"Checkpoint upgrade aborted.\")\n            exit(1)\n"
  },
  {
    "path": "llava/serve/__init__.py",
    "content": ""
  },
  {
    "path": "llava/serve/cli.py",
    "content": "import argparse\nimport torch\n\nfrom llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN\nfrom llava.conversation import conv_templates, SeparatorStyle\nfrom llava.model.builder import load_pretrained_model\nfrom llava.utils import disable_torch_init\nfrom llava.mm_utils import process_images, tokenizer_image_token, get_model_name_from_path\n\nfrom PIL import Image\n\nimport requests\nfrom PIL import Image\nfrom io import BytesIO\nfrom transformers import TextStreamer\n\n\ndef load_image(image_file):\n    if image_file.startswith('http://') or image_file.startswith('https://'):\n        response = requests.get(image_file)\n        image = Image.open(BytesIO(response.content)).convert('RGB')\n    else:\n        image = Image.open(image_file).convert('RGB')\n    return image\n\n\ndef main(args):\n    # Model\n    disable_torch_init()\n\n    model_name = get_model_name_from_path(args.model_path)\n    tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, args.load_8bit, args.load_4bit, device=args.device)\n\n    if \"llama-2\" in model_name.lower():\n        conv_mode = \"llava_llama_2\"\n    elif \"mistral\" in model_name.lower():\n        conv_mode = \"mistral_instruct\"\n    elif \"v1.6-34b\" in model_name.lower():\n        conv_mode = \"chatml_direct\"\n    elif \"v1\" in model_name.lower():\n        conv_mode = \"llava_v1\"\n    elif \"mpt\" in model_name.lower():\n        conv_mode = \"mpt\"\n    else:\n        conv_mode = \"llava_v0\"\n\n    if args.conv_mode is not None and conv_mode != args.conv_mode:\n        print('[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}'.format(conv_mode, args.conv_mode, args.conv_mode))\n    else:\n        args.conv_mode = conv_mode\n\n    conv = conv_templates[args.conv_mode].copy()\n    if \"mpt\" in model_name.lower():\n        roles = ('user', 'assistant')\n    else:\n        roles = conv.roles\n\n    image = load_image(args.image_file)\n    image_size = image.size\n    # Similar operation in model_worker.py\n    image_tensor = process_images([image], image_processor, model.config)\n    if type(image_tensor) is list:\n        image_tensor = [image.to(model.device, dtype=torch.float16) for image in image_tensor]\n    else:\n        image_tensor = image_tensor.to(model.device, dtype=torch.float16)\n\n    while True:\n        try:\n            inp = input(f\"{roles[0]}: \")\n        except EOFError:\n            inp = \"\"\n        if not inp:\n            print(\"exit...\")\n            break\n\n        print(f\"{roles[1]}: \", end=\"\")\n\n        if image is not None:\n            # first message\n            if model.config.mm_use_im_start_end:\n                inp = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\\n' + inp\n            else:\n                inp = DEFAULT_IMAGE_TOKEN + '\\n' + inp\n            image = None\n\n        conv.append_message(conv.roles[0], inp)\n        conv.append_message(conv.roles[1], None)\n        prompt = conv.get_prompt()\n\n        input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(model.device)\n        stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2\n        keywords = [stop_str]\n        streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)\n\n        with torch.inference_mode():\n            output_ids = model.generate(\n                input_ids,\n                images=image_tensor,\n                image_sizes=[image_size],\n                do_sample=True if args.temperature > 0 else False,\n                temperature=args.temperature,\n                max_new_tokens=args.max_new_tokens,\n                streamer=streamer,\n                use_cache=True)\n\n        outputs = tokenizer.decode(output_ids[0]).strip()\n        conv.messages[-1][-1] = outputs\n\n        if args.debug:\n            print(\"\\n\", {\"prompt\": prompt, \"outputs\": outputs}, \"\\n\")\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--model-path\", type=str, default=\"facebook/opt-350m\")\n    parser.add_argument(\"--model-base\", type=str, default=None)\n    parser.add_argument(\"--image-file\", type=str, required=True)\n    parser.add_argument(\"--device\", type=str, default=\"cuda\")\n    parser.add_argument(\"--conv-mode\", type=str, default=None)\n    parser.add_argument(\"--temperature\", type=float, default=0.2)\n    parser.add_argument(\"--max-new-tokens\", type=int, default=512)\n    parser.add_argument(\"--load-8bit\", action=\"store_true\")\n    parser.add_argument(\"--load-4bit\", action=\"store_true\")\n    parser.add_argument(\"--debug\", action=\"store_true\")\n    args = parser.parse_args()\n    main(args)\n"
  },
  {
    "path": "llava/serve/controller.py",
    "content": "\"\"\"\nA controller manages distributed workers.\nIt sends worker addresses to clients.\n\"\"\"\nimport argparse\nimport asyncio\nimport dataclasses\nfrom enum import Enum, auto\nimport json\nimport logging\nimport time\nfrom typing import List, Union\nimport threading\n\nfrom fastapi import FastAPI, Request\nfrom fastapi.responses import StreamingResponse\nimport numpy as np\nimport requests\nimport uvicorn\n\nfrom llava.constants import CONTROLLER_HEART_BEAT_EXPIRATION\nfrom llava.utils import build_logger, server_error_msg\n\n\nlogger = build_logger(\"controller\", \"controller.log\")\n\n\nclass DispatchMethod(Enum):\n    LOTTERY = auto()\n    SHORTEST_QUEUE = auto()\n\n    @classmethod\n    def from_str(cls, name):\n        if name == \"lottery\":\n            return cls.LOTTERY\n        elif name == \"shortest_queue\":\n            return cls.SHORTEST_QUEUE\n        else:\n            raise ValueError(f\"Invalid dispatch method\")\n\n\n@dataclasses.dataclass\nclass WorkerInfo:\n    model_names: List[str]\n    speed: int\n    queue_length: int\n    check_heart_beat: bool\n    last_heart_beat: str\n\n\ndef heart_beat_controller(controller):\n    while True:\n        time.sleep(CONTROLLER_HEART_BEAT_EXPIRATION)\n        controller.remove_stable_workers_by_expiration()\n\n\nclass Controller:\n    def __init__(self, dispatch_method: str):\n        # Dict[str -> WorkerInfo]\n        self.worker_info = {}\n        self.dispatch_method = DispatchMethod.from_str(dispatch_method)\n\n        self.heart_beat_thread = threading.Thread(\n            target=heart_beat_controller, args=(self,), daemon=True)\n        self.heart_beat_thread.start()\n\n        logger.info(\"Init controller\")\n\n    def register_worker(self, worker_name: str, check_heart_beat: bool,\n                        worker_status: dict):\n        if worker_name not in self.worker_info:\n            logger.info(f\"Register a new worker: {worker_name}\")\n        else:\n            logger.info(f\"Register an existing worker: {worker_name}\")\n\n        if not worker_status:\n            worker_status = self.get_worker_status(worker_name)\n        if not worker_status:\n            return False\n\n        self.worker_info[worker_name] = WorkerInfo(\n            worker_status[\"model_names\"], worker_status[\"speed\"], worker_status[\"queue_length\"],\n            check_heart_beat, time.time())\n\n        logger.info(f\"Register done: {worker_name}, {worker_status}\")\n        return True\n\n    def get_worker_status(self, worker_name: str):\n        try:\n            r = requests.post(worker_name + \"/worker_get_status\", timeout=5)\n        except requests.exceptions.RequestException as e:\n            logger.error(f\"Get status fails: {worker_name}, {e}\")\n            return None\n\n        if r.status_code != 200:\n            logger.error(f\"Get status fails: {worker_name}, {r}\")\n            return None\n\n        return r.json()\n\n    def remove_worker(self, worker_name: str):\n        del self.worker_info[worker_name]\n\n    def refresh_all_workers(self):\n        old_info = dict(self.worker_info)\n        self.worker_info = {}\n\n        for w_name, w_info in old_info.items():\n            if not self.register_worker(w_name, w_info.check_heart_beat, None):\n                logger.info(f\"Remove stale worker: {w_name}\")\n\n    def list_models(self):\n        model_names = set()\n\n        for w_name, w_info in self.worker_info.items():\n            model_names.update(w_info.model_names)\n\n        return list(model_names)\n\n    def get_worker_address(self, model_name: str):\n        if self.dispatch_method == DispatchMethod.LOTTERY:\n            worker_names = []\n            worker_speeds = []\n            for w_name, w_info in self.worker_info.items():\n                if model_name in w_info.model_names:\n                    worker_names.append(w_name)\n                    worker_speeds.append(w_info.speed)\n            worker_speeds = np.array(worker_speeds, dtype=np.float32)\n            norm = np.sum(worker_speeds)\n            if norm < 1e-4:\n                return \"\"\n            worker_speeds = worker_speeds / norm\n            if True:  # Directly return address\n                pt = np.random.choice(np.arange(len(worker_names)),\n                                      p=worker_speeds)\n                worker_name = worker_names[pt]\n                return worker_name\n\n            # Check status before returning\n            while True:\n                pt = np.random.choice(np.arange(len(worker_names)),\n                                      p=worker_speeds)\n                worker_name = worker_names[pt]\n\n                if self.get_worker_status(worker_name):\n                    break\n                else:\n                    self.remove_worker(worker_name)\n                    worker_speeds[pt] = 0\n                    norm = np.sum(worker_speeds)\n                    if norm < 1e-4:\n                        return \"\"\n                    worker_speeds = worker_speeds / norm\n                    continue\n            return worker_name\n        elif self.dispatch_method == DispatchMethod.SHORTEST_QUEUE:\n            worker_names = []\n            worker_qlen = []\n            for w_name, w_info in self.worker_info.items():\n                if model_name in w_info.model_names:\n                    worker_names.append(w_name)\n                    worker_qlen.append(w_info.queue_length / w_info.speed)\n            if len(worker_names) == 0:\n                return \"\"\n            min_index = np.argmin(worker_qlen)\n            w_name = worker_names[min_index]\n            self.worker_info[w_name].queue_length += 1\n            logger.info(f\"names: {worker_names}, queue_lens: {worker_qlen}, ret: {w_name}\")\n            return w_name\n        else:\n            raise ValueError(f\"Invalid dispatch method: {self.dispatch_method}\")\n\n    def receive_heart_beat(self, worker_name: str, queue_length: int):\n        if worker_name not in self.worker_info:\n            logger.info(f\"Receive unknown heart beat. {worker_name}\")\n            return False\n\n        self.worker_info[worker_name].queue_length = queue_length\n        self.worker_info[worker_name].last_heart_beat = time.time()\n        logger.info(f\"Receive heart beat. {worker_name}\")\n        return True\n\n    def remove_stable_workers_by_expiration(self):\n        expire = time.time() - CONTROLLER_HEART_BEAT_EXPIRATION\n        to_delete = []\n        for worker_name, w_info in self.worker_info.items():\n            if w_info.check_heart_beat and w_info.last_heart_beat < expire:\n                to_delete.append(worker_name)\n\n        for worker_name in to_delete:\n            self.remove_worker(worker_name)\n\n    def worker_api_generate_stream(self, params):\n        worker_addr = self.get_worker_address(params[\"model\"])\n        if not worker_addr:\n            logger.info(f\"no worker: {params['model']}\")\n            ret = {\n                \"text\": server_error_msg,\n                \"error_code\": 2,\n            }\n            yield json.dumps(ret).encode() + b\"\\0\"\n\n        try:\n            response = requests.post(worker_addr + \"/worker_generate_stream\",\n                                     json=params, stream=True, timeout=5)\n            for chunk in response.iter_lines(decode_unicode=False, delimiter=b\"\\0\"):\n                if chunk:\n                    yield chunk + b\"\\0\"\n        except requests.exceptions.RequestException as e:\n            logger.info(f\"worker timeout: {worker_addr}\")\n            ret = {\n                \"text\": server_error_msg,\n                \"error_code\": 3,\n            }\n            yield json.dumps(ret).encode() + b\"\\0\"\n\n    # Let the controller act as a worker to achieve hierarchical\n    # management. This can be used to connect isolated sub networks.\n\n    def worker_api_get_status(self):\n        model_names = set()\n        speed = 0\n        queue_length = 0\n\n        for w_name in self.worker_info:\n            worker_status = self.get_worker_status(w_name)\n            if worker_status is not None:\n                model_names.update(worker_status[\"model_names\"])\n                speed += worker_status[\"speed\"]\n                queue_length += worker_status[\"queue_length\"]\n\n        return {\n            \"model_names\": list(model_names),\n            \"speed\": speed,\n            \"queue_length\": queue_length,\n        }\n\n\napp = FastAPI()\n\n\n@app.post(\"/register_worker\")\nasync def register_worker(request: Request):\n    data = await request.json()\n    controller.register_worker(\n        data[\"worker_name\"], data[\"check_heart_beat\"],\n        data.get(\"worker_status\", None))\n\n\n@app.post(\"/refresh_all_workers\")\nasync def refresh_all_workers():\n    models = controller.refresh_all_workers()\n\n\n@app.post(\"/list_models\")\nasync def list_models():\n    models = controller.list_models()\n    return {\"models\": models}\n\n\n@app.post(\"/get_worker_address\")\nasync def get_worker_address(request: Request):\n    data = await request.json()\n    addr = controller.get_worker_address(data[\"model\"])\n    return {\"address\": addr}\n\n\n@app.post(\"/receive_heart_beat\")\nasync def receive_heart_beat(request: Request):\n    data = await request.json()\n    exist = controller.receive_heart_beat(\n        data[\"worker_name\"], data[\"queue_length\"])\n    return {\"exist\": exist}\n\n\n@app.post(\"/worker_generate_stream\")\nasync def worker_api_generate_stream(request: Request):\n    params = await request.json()\n    generator = controller.worker_api_generate_stream(params)\n    return StreamingResponse(generator)\n\n\n@app.post(\"/worker_get_status\")\nasync def worker_api_get_status(request: Request):\n    return controller.worker_api_get_status()\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--host\", type=str, default=\"localhost\")\n    parser.add_argument(\"--port\", type=int, default=21001)\n    parser.add_argument(\"--dispatch-method\", type=str, choices=[\n        \"lottery\", \"shortest_queue\"], default=\"shortest_queue\")\n    args = parser.parse_args()\n    logger.info(f\"args: {args}\")\n\n    controller = Controller(args.dispatch_method)\n    uvicorn.run(app, host=args.host, port=args.port, log_level=\"info\")\n"
  },
  {
    "path": "llava/serve/gradio_web_server.py",
    "content": "import argparse\nimport datetime\nimport json\nimport os\nimport time\n\nimport gradio as gr\nimport requests\n\nfrom llava.conversation import (default_conversation, conv_templates,\n                                SeparatorStyle)\nfrom llava.constants import LOGDIR\nfrom llava.utils import (build_logger, server_error_msg,\n                         violates_moderation, moderation_msg)\nimport hashlib\n\n\nlogger = build_logger(\"gradio_web_server\", \"gradio_web_server.log\")\n\nheaders = {\"User-Agent\": \"LLaVA Client\"}\n\nno_change_btn = gr.Button()\nenable_btn = gr.Button(interactive=True)\ndisable_btn = gr.Button(interactive=False)\n\npriority = {\n    \"vicuna-13b\": \"aaaaaaa\",\n    \"koala-13b\": \"aaaaaab\",\n}\n\n\ndef get_conv_log_filename():\n    t = datetime.datetime.now()\n    name = os.path.join(LOGDIR, f\"{t.year}-{t.month:02d}-{t.day:02d}-conv.json\")\n    return name\n\n\ndef get_model_list():\n    ret = requests.post(args.controller_url + \"/refresh_all_workers\")\n    assert ret.status_code == 200\n    ret = requests.post(args.controller_url + \"/list_models\")\n    models = ret.json()[\"models\"]\n    models.sort(key=lambda x: priority.get(x, x))\n    logger.info(f\"Models: {models}\")\n    return models\n\n\nget_window_url_params = \"\"\"\nfunction() {\n    const params = new URLSearchParams(window.location.search);\n    url_params = Object.fromEntries(params);\n    console.log(url_params);\n    return url_params;\n    }\n\"\"\"\n\n\ndef load_demo(url_params, request: gr.Request):\n    logger.info(f\"load_demo. ip: {request.client.host}. params: {url_params}\")\n\n    dropdown_update = gr.Dropdown(visible=True)\n    if \"model\" in url_params:\n        model = url_params[\"model\"]\n        if model in models:\n            dropdown_update = gr.Dropdown(value=model, visible=True)\n\n    state = default_conversation.copy()\n    return state, dropdown_update\n\n\ndef load_demo_refresh_model_list(request: gr.Request):\n    logger.info(f\"load_demo. ip: {request.client.host}\")\n    models = get_model_list()\n    state = default_conversation.copy()\n    dropdown_update = gr.Dropdown(\n        choices=models,\n        value=models[0] if len(models) > 0 else \"\"\n    )\n    return state, dropdown_update\n\n\ndef vote_last_response(state, vote_type, model_selector, request: gr.Request):\n    with open(get_conv_log_filename(), \"a\") as fout:\n        data = {\n            \"tstamp\": round(time.time(), 4),\n            \"type\": vote_type,\n            \"model\": model_selector,\n            \"state\": state.dict(),\n            \"ip\": request.client.host,\n        }\n        fout.write(json.dumps(data) + \"\\n\")\n\n\ndef upvote_last_response(state, model_selector, request: gr.Request):\n    logger.info(f\"upvote. ip: {request.client.host}\")\n    vote_last_response(state, \"upvote\", model_selector, request)\n    return (\"\",) + (disable_btn,) * 3\n\n\ndef downvote_last_response(state, model_selector, request: gr.Request):\n    logger.info(f\"downvote. ip: {request.client.host}\")\n    vote_last_response(state, \"downvote\", model_selector, request)\n    return (\"\",) + (disable_btn,) * 3\n\n\ndef flag_last_response(state, model_selector, request: gr.Request):\n    logger.info(f\"flag. ip: {request.client.host}\")\n    vote_last_response(state, \"flag\", model_selector, request)\n    return (\"\",) + (disable_btn,) * 3\n\n\ndef regenerate(state, image_process_mode, request: gr.Request):\n    logger.info(f\"regenerate. ip: {request.client.host}\")\n    state.messages[-1][-1] = None\n    prev_human_msg = state.messages[-2]\n    if type(prev_human_msg[1]) in (tuple, list):\n        prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode)\n    state.skip_next = False\n    return (state, state.to_gradio_chatbot(), \"\", None) + (disable_btn,) * 5\n\n\ndef clear_history(request: gr.Request):\n    logger.info(f\"clear_history. ip: {request.client.host}\")\n    state = default_conversation.copy()\n    return (state, state.to_gradio_chatbot(), \"\", None) + (disable_btn,) * 5\n\n\ndef add_text(state, text, image, image_process_mode, request: gr.Request):\n    logger.info(f\"add_text. ip: {request.client.host}. len: {len(text)}\")\n    if len(text) <= 0 and image is None:\n        state.skip_next = True\n        return (state, state.to_gradio_chatbot(), \"\", None) + (no_change_btn,) * 5\n    if args.moderate:\n        flagged = violates_moderation(text)\n        if flagged:\n            state.skip_next = True\n            return (state, state.to_gradio_chatbot(), moderation_msg, None) + (\n                no_change_btn,) * 5\n\n    text = text[:1536]  # Hard cut-off\n    if image is not None:\n        text = text[:1200]  # Hard cut-off for images\n        if '<image>' not in text:\n            # text = '<Image><image></Image>' + text\n            text = text + '\\n<image>'\n        text = (text, image, image_process_mode)\n        state = default_conversation.copy()\n    state.append_message(state.roles[0], text)\n    state.append_message(state.roles[1], None)\n    state.skip_next = False\n    return (state, state.to_gradio_chatbot(), \"\", None) + (disable_btn,) * 5\n\n\ndef http_bot(state, model_selector, temperature, top_p, max_new_tokens, request: gr.Request):\n    logger.info(f\"http_bot. ip: {request.client.host}\")\n    start_tstamp = time.time()\n    model_name = model_selector\n\n    if state.skip_next:\n        # This generate call is skipped due to invalid inputs\n        yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5\n        return\n\n    if len(state.messages) == state.offset + 2:\n        # First round of conversation\n        if \"llava\" in model_name.lower():\n            if 'llama-2' in model_name.lower():\n                template_name = \"llava_llama_2\"\n            elif \"mistral\" in model_name.lower() or \"mixtral\" in model_name.lower():\n                if 'orca' in model_name.lower():\n                    template_name = \"mistral_orca\"\n                elif 'hermes' in model_name.lower():\n                    template_name = \"chatml_direct\"\n                else:\n                    template_name = \"mistral_instruct\"\n            elif 'llava-v1.6-34b' in model_name.lower():\n                template_name = \"chatml_direct\"\n            elif \"v1\" in model_name.lower():\n                if 'mmtag' in model_name.lower():\n                    template_name = \"v1_mmtag\"\n                elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():\n                    template_name = \"v1_mmtag\"\n                else:\n                    template_name = \"llava_v1\"\n            elif \"mpt\" in model_name.lower():\n                template_name = \"mpt\"\n            else:\n                if 'mmtag' in model_name.lower():\n                    template_name = \"v0_mmtag\"\n                elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():\n                    template_name = \"v0_mmtag\"\n                else:\n                    template_name = \"llava_v0\"\n        elif \"mpt\" in model_name:\n            template_name = \"mpt_text\"\n        elif \"llama-2\" in model_name:\n            template_name = \"llama_2\"\n        else:\n            template_name = \"vicuna_v1\"\n        new_state = conv_templates[template_name].copy()\n        new_state.append_message(new_state.roles[0], state.messages[-2][1])\n        new_state.append_message(new_state.roles[1], None)\n        state = new_state\n\n    # Query worker address\n    controller_url = args.controller_url\n    ret = requests.post(controller_url + \"/get_worker_address\",\n                        json={\"model\": model_name})\n    worker_addr = ret.json()[\"address\"]\n    logger.info(f\"model_name: {model_name}, worker_addr: {worker_addr}\")\n\n    # No available worker\n    if worker_addr == \"\":\n        state.messages[-1][-1] = server_error_msg\n        yield (state, state.to_gradio_chatbot(), disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)\n        return\n\n    # Construct prompt\n    prompt = state.get_prompt()\n\n    all_images = state.get_images(return_pil=True)\n    all_image_hash = [hashlib.md5(image.tobytes()).hexdigest() for image in all_images]\n    for image, hash in zip(all_images, all_image_hash):\n        t = datetime.datetime.now()\n        filename = os.path.join(LOGDIR, \"serve_images\", f\"{t.year}-{t.month:02d}-{t.day:02d}\", f\"{hash}.jpg\")\n        if not os.path.isfile(filename):\n            os.makedirs(os.path.dirname(filename), exist_ok=True)\n            image.save(filename)\n\n    # Make requests\n    pload = {\n        \"model\": model_name,\n        \"prompt\": prompt,\n        \"temperature\": float(temperature),\n        \"top_p\": float(top_p),\n        \"max_new_tokens\": min(int(max_new_tokens), 1536),\n        \"stop\": state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2,\n        \"images\": f'List of {len(state.get_images())} images: {all_image_hash}',\n    }\n    logger.info(f\"==== request ====\\n{pload}\")\n\n    pload['images'] = state.get_images()\n\n    state.messages[-1][-1] = \"▌\"\n    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5\n\n    try:\n        # Stream output\n        response = requests.post(worker_addr + \"/worker_generate_stream\",\n                                 headers=headers, json=pload, stream=True, timeout=10)\n        for chunk in response.iter_lines(decode_unicode=False, delimiter=b\"\\0\"):\n            if chunk:\n                data = json.loads(chunk.decode())\n                if data[\"error_code\"] == 0:\n                    output = data[\"text\"][len(prompt):].strip()\n                    state.messages[-1][-1] = output + \"▌\"\n                    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5\n                else:\n                    output = data[\"text\"] + f\" (error_code: {data['error_code']})\"\n                    state.messages[-1][-1] = output\n                    yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)\n                    return\n                time.sleep(0.03)\n    except requests.exceptions.RequestException as e:\n        state.messages[-1][-1] = server_error_msg\n        yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)\n        return\n\n    state.messages[-1][-1] = state.messages[-1][-1][:-1]\n    yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5\n\n    finish_tstamp = time.time()\n    logger.info(f\"{output}\")\n\n    with open(get_conv_log_filename(), \"a\") as fout:\n        data = {\n            \"tstamp\": round(finish_tstamp, 4),\n            \"type\": \"chat\",\n            \"model\": model_name,\n            \"start\": round(start_tstamp, 4),\n            \"finish\": round(finish_tstamp, 4),\n            \"state\": state.dict(),\n            \"images\": all_image_hash,\n            \"ip\": request.client.host,\n        }\n        fout.write(json.dumps(data) + \"\\n\")\n\n\ntitle_markdown = (\"\"\"\n# 🌋 LLaVA: Large Language and Vision Assistant\n[[Project Page](https://llava-vl.github.io)] [[Code](https://github.com/haotian-liu/LLaVA)] [[Model](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md)] | 📚 [[LLaVA](https://arxiv.org/abs/2304.08485)] [[LLaVA-v1.5](https://arxiv.org/abs/2310.03744)] [[LLaVA-v1.6](https://llava-vl.github.io/blog/2024-01-30-llava-1-6/)]\n\"\"\")\n\ntos_markdown = (\"\"\"\n### Terms of use\nBy using this service, users are required to agree to the following terms:\nThe service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.\nPlease click the \"Flag\" button if you get any inappropriate answer! We will collect those to keep improving our moderator.\nFor an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.\n\"\"\")\n\n\nlearn_more_markdown = (\"\"\"\n### License\nThe service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.\n\"\"\")\n\nblock_css = \"\"\"\n\n#buttons button {\n    min-width: min(120px,100%);\n}\n\n\"\"\"\n\n\ndef build_demo(embed_mode, cur_dir=None, concurrency_count=10):\n    textbox = gr.Textbox(show_label=False, placeholder=\"Enter text and press ENTER\", container=False)\n    with gr.Blocks(title=\"LLaVA\", theme=gr.themes.Default(), css=block_css) as demo:\n        state = gr.State()\n\n        if not embed_mode:\n            gr.Markdown(title_markdown)\n\n        with gr.Row():\n            with gr.Column(scale=3):\n                with gr.Row(elem_id=\"model_selector_row\"):\n                    model_selector = gr.Dropdown(\n                        choices=models,\n                        value=models[0] if len(models) > 0 else \"\",\n                        interactive=True,\n                        show_label=False,\n                        container=False)\n\n                imagebox = gr.Image(type=\"pil\")\n                image_process_mode = gr.Radio(\n                    [\"Crop\", \"Resize\", \"Pad\", \"Default\"],\n                    value=\"Default\",\n                    label=\"Preprocess for non-square image\", visible=False)\n\n                if cur_dir is None:\n                    cur_dir = os.path.dirname(os.path.abspath(__file__))\n                gr.Examples(examples=[\n                    [f\"{cur_dir}/examples/extreme_ironing.jpg\", \"What is unusual about this image?\"],\n                    [f\"{cur_dir}/examples/waterview.jpg\", \"What are the things I should be cautious about when I visit here?\"],\n                ], inputs=[imagebox, textbox])\n\n                with gr.Accordion(\"Parameters\", open=False) as parameter_row:\n                    temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label=\"Temperature\",)\n                    top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label=\"Top P\",)\n                    max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label=\"Max output tokens\",)\n\n            with gr.Column(scale=8):\n                chatbot = gr.Chatbot(\n                    elem_id=\"chatbot\",\n                    label=\"LLaVA Chatbot\",\n                    height=650,\n                    layout=\"panel\",\n                )\n                with gr.Row():\n                    with gr.Column(scale=8):\n                        textbox.render()\n                    with gr.Column(scale=1, min_width=50):\n                        submit_btn = gr.Button(value=\"Send\", variant=\"primary\")\n                with gr.Row(elem_id=\"buttons\") as button_row:\n                    upvote_btn = gr.Button(value=\"👍  Upvote\", interactive=False)\n                    downvote_btn = gr.Button(value=\"👎  Downvote\", interactive=False)\n                    flag_btn = gr.Button(value=\"⚠️  Flag\", interactive=False)\n                    # stop_btn = gr.Button(value=\"⏹️  Stop Generation\", interactive=False)\n                    regenerate_btn = gr.Button(value=\"🔄  Regenerate\", interactive=False)\n                    clear_btn = gr.Button(value=\"🗑️  Clear\", interactive=False)\n\n        if not embed_mode:\n            gr.Markdown(tos_markdown)\n            gr.Markdown(learn_more_markdown)\n        url_params = gr.JSON(visible=False)\n\n        # Register listeners\n        btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]\n        upvote_btn.click(\n            upvote_last_response,\n            [state, model_selector],\n            [textbox, upvote_btn, downvote_btn, flag_btn]\n        )\n        downvote_btn.click(\n            downvote_last_response,\n            [state, model_selector],\n            [textbox, upvote_btn, downvote_btn, flag_btn]\n        )\n        flag_btn.click(\n            flag_last_response,\n            [state, model_selector],\n            [textbox, upvote_btn, downvote_btn, flag_btn]\n        )\n\n        regenerate_btn.click(\n            regenerate,\n            [state, image_process_mode],\n            [state, chatbot, textbox, imagebox] + btn_list\n        ).then(\n            http_bot,\n            [state, model_selector, temperature, top_p, max_output_tokens],\n            [state, chatbot] + btn_list,\n            concurrency_limit=concurrency_count\n        )\n\n        clear_btn.click(\n            clear_history,\n            None,\n            [state, chatbot, textbox, imagebox] + btn_list,\n            queue=False\n        )\n\n        textbox.submit(\n            add_text,\n            [state, textbox, imagebox, image_process_mode],\n            [state, chatbot, textbox, imagebox] + btn_list,\n            queue=False\n        ).then(\n            http_bot,\n            [state, model_selector, temperature, top_p, max_output_tokens],\n            [state, chatbot] + btn_list,\n            concurrency_limit=concurrency_count\n        )\n\n        submit_btn.click(\n            add_text,\n            [state, textbox, imagebox, image_process_mode],\n            [state, chatbot, textbox, imagebox] + btn_list\n        ).then(\n            http_bot,\n            [state, model_selector, temperature, top_p, max_output_tokens],\n            [state, chatbot] + btn_list,\n            concurrency_limit=concurrency_count\n        )\n\n        if args.model_list_mode == \"once\":\n            demo.load(\n                load_demo,\n                [url_params],\n                [state, model_selector],\n                js=get_window_url_params\n            )\n        elif args.model_list_mode == \"reload\":\n            demo.load(\n                load_demo_refresh_model_list,\n                None,\n                [state, model_selector],\n                queue=False\n            )\n        else:\n            raise ValueError(f\"Unknown model list mode: {args.model_list_mode}\")\n\n    return demo\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--host\", type=str, default=\"0.0.0.0\")\n    parser.add_argument(\"--port\", type=int)\n    parser.add_argument(\"--controller-url\", type=str, default=\"http://localhost:21001\")\n    parser.add_argument(\"--concurrency-count\", type=int, default=16)\n    parser.add_argument(\"--model-list-mode\", type=str, default=\"once\",\n                        choices=[\"once\", \"reload\"])\n    parser.add_argument(\"--share\", action=\"store_true\")\n    parser.add_argument(\"--moderate\", action=\"store_true\")\n    parser.add_argument(\"--embed\", action=\"store_true\")\n    args = parser.parse_args()\n    logger.info(f\"args: {args}\")\n\n    models = get_model_list()\n\n    logger.info(args)\n    demo = build_demo(args.embed, concurrency_count=args.concurrency_count)\n    demo.queue(\n        api_open=False\n    ).launch(\n        server_name=args.host,\n        server_port=args.port,\n        share=args.share\n    )\n"
  },
  {
    "path": "llava/serve/model_worker.py",
    "content": "\"\"\"\nA model worker executes the model.\n\"\"\"\nimport argparse\nimport asyncio\nimport json\nimport time\nimport threading\nimport uuid\n\nfrom fastapi import FastAPI, Request, BackgroundTasks\nfrom fastapi.responses import StreamingResponse\nimport requests\nimport torch\nimport uvicorn\nfrom functools import partial\n\nfrom llava.constants import WORKER_HEART_BEAT_INTERVAL\nfrom llava.utils import (build_logger, server_error_msg,\n                         pretty_print_semaphore)\nfrom llava.model.builder import load_pretrained_model\nfrom llava.mm_utils import process_images, load_image_from_base64, tokenizer_image_token\nfrom llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN\nfrom transformers import TextIteratorStreamer\nfrom threading import Thread\n\n\nGB = 1 << 30\n\nworker_id = str(uuid.uuid4())[:6]\nlogger = build_logger(\"model_worker\", f\"model_worker_{worker_id}.log\")\nglobal_counter = 0\n\nmodel_semaphore = None\n\n\ndef heart_beat_worker(controller):\n\n    while True:\n        time.sleep(WORKER_HEART_BEAT_INTERVAL)\n        controller.send_heart_beat()\n\n\nclass ModelWorker:\n    def __init__(self, controller_addr, worker_addr,\n                 worker_id, no_register,\n                 model_path, model_base, model_name,\n                 load_8bit, load_4bit, device, use_flash_attn=False):\n        self.controller_addr = controller_addr\n        self.worker_addr = worker_addr\n        self.worker_id = worker_id\n        if model_path.endswith(\"/\"):\n            model_path = model_path[:-1]\n        if model_name is None:\n            model_paths = model_path.split(\"/\")\n            if model_paths[-1].startswith('checkpoint-'):\n                self.model_name = model_paths[-2] + \"_\" + model_paths[-1]\n            else:\n                self.model_name = model_paths[-1]\n        else:\n            self.model_name = model_name\n\n        self.device = device\n        logger.info(f\"Loading the model {self.model_name} on worker {worker_id} ...\")\n        self.tokenizer, self.model, self.image_processor, self.context_len = load_pretrained_model(\n            model_path, model_base, self.model_name, load_8bit, load_4bit, device=self.device, use_flash_attn=use_flash_attn)\n        self.is_multimodal = 'llava' in self.model_name.lower()\n\n        if not no_register:\n            self.register_to_controller()\n            self.heart_beat_thread = threading.Thread(\n                target=heart_beat_worker, args=(self,), daemon=True)\n            self.heart_beat_thread.start()\n\n    def register_to_controller(self):\n        logger.info(\"Register to controller\")\n\n        url = self.controller_addr + \"/register_worker\"\n        data = {\n            \"worker_name\": self.worker_addr,\n            \"check_heart_beat\": True,\n            \"worker_status\": self.get_status()\n        }\n        r = requests.post(url, json=data)\n        assert r.status_code == 200\n\n    def send_heart_beat(self):\n        logger.info(f\"Send heart beat. Models: {[self.model_name]}. \"\n                    f\"Semaphore: {pretty_print_semaphore(model_semaphore)}. \"\n                    f\"global_counter: {global_counter}\")\n\n        url = self.controller_addr + \"/receive_heart_beat\"\n\n        while True:\n            try:\n                ret = requests.post(url, json={\n                    \"worker_name\": self.worker_addr,\n                    \"queue_length\": self.get_queue_length()}, timeout=5)\n                exist = ret.json()[\"exist\"]\n                break\n            except requests.exceptions.RequestException as e:\n                logger.error(f\"heart beat error: {e}\")\n            time.sleep(5)\n\n        if not exist:\n            self.register_to_controller()\n\n    def get_queue_length(self):\n        if model_semaphore is None:\n            return 0\n        else:\n            return args.limit_model_concurrency - model_semaphore._value + (len(\n                model_semaphore._waiters) if model_semaphore._waiters is not None else 0)\n\n    def get_status(self):\n        return {\n            \"model_names\": [self.model_name],\n            \"speed\": 1,\n            \"queue_length\": self.get_queue_length(),\n        }\n\n    @torch.inference_mode()\n    def generate_stream(self, params):\n        tokenizer, model, image_processor = self.tokenizer, self.model, self.image_processor\n\n        prompt = params[\"prompt\"]\n        ori_prompt = prompt\n        images = params.get(\"images\", None)\n        num_image_tokens = 0\n        if images is not None and len(images) > 0 and self.is_multimodal:\n            if len(images) > 0:\n                if len(images) != prompt.count(DEFAULT_IMAGE_TOKEN):\n                    raise ValueError(\"Number of images does not match number of <image> tokens in prompt\")\n\n                images = [load_image_from_base64(image) for image in images]\n                image_sizes = [image.size for image in images]\n                images = process_images(images, image_processor, model.config)\n\n                if type(images) is list:\n                    images = [image.to(self.model.device, dtype=torch.float16) for image in images]\n                else:\n                    images = images.to(self.model.device, dtype=torch.float16)\n\n                replace_token = DEFAULT_IMAGE_TOKEN\n                if getattr(self.model.config, 'mm_use_im_start_end', False):\n                    replace_token = DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN\n                prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replace_token)\n\n                num_image_tokens = prompt.count(replace_token) * model.get_vision_tower().num_patches\n            else:\n                images = None\n                image_sizes = None\n            image_args = {\"images\": images, \"image_sizes\": image_sizes}\n        else:\n            images = None\n            image_args = {}\n\n        temperature = float(params.get(\"temperature\", 1.0))\n        top_p = float(params.get(\"top_p\", 1.0))\n        max_context_length = getattr(model.config, 'max_position_embeddings', 2048)\n        max_new_tokens = min(int(params.get(\"max_new_tokens\", 256)), 1024)\n        stop_str = params.get(\"stop\", None)\n        do_sample = True if temperature > 0.001 else False\n\n        input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(self.device)\n        keywords = [stop_str]\n        # stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)\n        streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=15)\n\n        max_new_tokens = min(max_new_tokens, max_context_length - input_ids.shape[-1] - num_image_tokens)\n\n        if max_new_tokens < 1:\n            yield json.dumps({\"text\": ori_prompt + \"Exceeds max token length. Please start a new conversation, thanks.\", \"error_code\": 0}).encode() + b\"\\0\"\n            return\n\n        thread = Thread(target=model.generate, kwargs=dict(\n            inputs=input_ids,\n            do_sample=do_sample,\n            temperature=temperature,\n            top_p=top_p,\n            max_new_tokens=max_new_tokens,\n            streamer=streamer,\n            use_cache=True,\n            **image_args\n        ))\n        thread.start()\n\n        generated_text = ori_prompt\n        for new_text in streamer:\n            generated_text += new_text\n            if generated_text.endswith(stop_str):\n                generated_text = generated_text[:-len(stop_str)]\n            yield json.dumps({\"text\": generated_text, \"error_code\": 0}).encode() + b\"\\0\"\n\n    def generate_stream_gate(self, params):\n        try:\n            for x in self.generate_stream(params):\n                yield x\n        except ValueError as e:\n            print(\"Caught ValueError:\", e)\n            ret = {\n                \"text\": server_error_msg,\n                \"error_code\": 1,\n            }\n            yield json.dumps(ret).encode() + b\"\\0\"\n        except torch.cuda.CudaError as e:\n            print(\"Caught torch.cuda.CudaError:\", e)\n            ret = {\n                \"text\": server_error_msg,\n                \"error_code\": 1,\n            }\n            yield json.dumps(ret).encode() + b\"\\0\"\n        except Exception as e:\n            print(\"Caught Unknown Error\", e)\n            ret = {\n                \"text\": server_error_msg,\n                \"error_code\": 1,\n            }\n            yield json.dumps(ret).encode() + b\"\\0\"\n\n\napp = FastAPI()\n\n\ndef release_model_semaphore(fn=None):\n    model_semaphore.release()\n    if fn is not None:\n        fn()\n\n\n@app.post(\"/worker_generate_stream\")\nasync def generate_stream(request: Request):\n    global model_semaphore, global_counter\n    global_counter += 1\n    params = await request.json()\n\n    if model_semaphore is None:\n        model_semaphore = asyncio.Semaphore(args.limit_model_concurrency)\n    await model_semaphore.acquire()\n    worker.send_heart_beat()\n    generator = worker.generate_stream_gate(params)\n    background_tasks = BackgroundTasks()\n    background_tasks.add_task(partial(release_model_semaphore, fn=worker.send_heart_beat))\n    return StreamingResponse(generator, background=background_tasks)\n\n\n@app.post(\"/worker_get_status\")\nasync def get_status(request: Request):\n    return worker.get_status()\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--host\", type=str, default=\"localhost\")\n    parser.add_argument(\"--port\", type=int, default=21002)\n    parser.add_argument(\"--worker-address\", type=str,\n                        default=\"http://localhost:21002\")\n    parser.add_argument(\"--controller-address\", type=str,\n                        default=\"http://localhost:21001\")\n    parser.add_argument(\"--model-path\", type=str, default=\"facebook/opt-350m\")\n    parser.add_argument(\"--model-base\", type=str, default=None)\n    parser.add_argument(\"--model-name\", type=str)\n    parser.add_argument(\"--device\", type=str, default=\"cuda\")\n    parser.add_argument(\"--multi-modal\", action=\"store_true\", help=\"Multimodal mode is automatically detected with model name, please make sure `llava` is included in the model path.\")\n    parser.add_argument(\"--limit-model-concurrency\", type=int, default=5)\n    parser.add_argument(\"--stream-interval\", type=int, default=1)\n    parser.add_argument(\"--no-register\", action=\"store_true\")\n    parser.add_argument(\"--load-8bit\", action=\"store_true\")\n    parser.add_argument(\"--load-4bit\", action=\"store_true\")\n    parser.add_argument(\"--use-flash-attn\", action=\"store_true\")\n    args = parser.parse_args()\n    logger.info(f\"args: {args}\")\n\n    if args.multi_modal:\n        logger.warning(\"Multimodal mode is automatically detected with model name, please make sure `llava` is included in the model path.\")\n\n    worker = ModelWorker(args.controller_address,\n                         args.worker_address,\n                         worker_id,\n                         args.no_register,\n                         args.model_path,\n                         args.model_base,\n                         args.model_name,\n                         args.load_8bit,\n                         args.load_4bit,\n                         args.device,\n                         use_flash_attn=args.use_flash_attn)\n    uvicorn.run(app, host=args.host, port=args.port, log_level=\"info\")\n"
  },
  {
    "path": "llava/serve/register_worker.py",
    "content": "\"\"\"\nManually register workers.\n\nUsage:\npython3 -m fastchat.serve.register_worker --controller http://localhost:21001 --worker-name http://localhost:21002\n\"\"\"\n\nimport argparse\n\nimport requests\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--controller-address\", type=str)\n    parser.add_argument(\"--worker-name\", type=str)\n    parser.add_argument(\"--check-heart-beat\", action=\"store_true\")\n    args = parser.parse_args()\n\n    url = args.controller_address + \"/register_worker\"\n    data = {\n        \"worker_name\": args.worker_name,\n        \"check_heart_beat\": args.check_heart_beat,\n        \"worker_status\": None,\n    }\n    r = requests.post(url, json=data)\n    assert r.status_code == 200\n"
  },
  {
    "path": "llava/serve/sglang_worker.py",
    "content": "\"\"\"\nA model worker executes the model.\n\"\"\"\nimport argparse\nimport asyncio\nfrom concurrent.futures import ThreadPoolExecutor\nimport json\nimport time\nimport threading\nimport uuid\n\nfrom fastapi import FastAPI, Request, BackgroundTasks\nfrom fastapi.responses import StreamingResponse\nimport requests\nimport re\nimport uvicorn\nfrom functools import partial\n\nfrom llava.constants import WORKER_HEART_BEAT_INTERVAL\nfrom llava.utils import (build_logger, server_error_msg,\n                         pretty_print_semaphore)\nfrom llava.mm_utils import process_images, load_image_from_base64, tokenizer_image_token, expand2square\nfrom llava.constants import DEFAULT_IMAGE_TOKEN\n\nimport sglang as sgl\nfrom sglang.backend.runtime_endpoint import RuntimeEndpoint\n\n\nGB = 1 << 30\n\nworker_id = str(uuid.uuid4())[:6]\nlogger = build_logger(\"model_worker\", f\"model_worker_{worker_id}.log\")\nglobal_counter = 0\n\nmodel_semaphore = None\n\n\ndef heart_beat_worker(controller):\n    while True:\n        time.sleep(WORKER_HEART_BEAT_INTERVAL)\n        controller.send_heart_beat()\n\n\n@sgl.function\ndef pipeline(s, prompt, max_tokens):\n    for p in prompt:\n        if type(p) is str:\n            s += p\n        else:\n            s += sgl.image(p)\n    s += sgl.gen(\"response\", max_tokens=max_tokens)\n\n\nclass ModelWorker:\n    def __init__(self, controller_addr, worker_addr, sgl_endpoint,\n                 worker_id, no_register, model_name):\n        self.controller_addr = controller_addr\n        self.worker_addr = worker_addr\n        self.worker_id = worker_id\n\n        # Select backend\n        backend = RuntimeEndpoint(sgl_endpoint)\n        sgl.set_default_backend(backend)\n        model_path = backend.model_info[\"model_path\"]\n\n        if model_path.endswith(\"/\"):\n            model_path = model_path[:-1]\n        if model_name is None:\n            model_paths = model_path.split(\"/\")\n            if model_paths[-1].startswith('checkpoint-'):\n                self.model_name = model_paths[-2] + \"_\" + model_paths[-1]\n            else:\n                self.model_name = model_paths[-1]\n        else:\n            self.model_name = model_name\n\n        logger.info(f\"Loading the SGLANG model {self.model_name} on worker {worker_id} ...\")\n\n        if not no_register:\n            self.register_to_controller()\n            self.heart_beat_thread = threading.Thread(\n                target=heart_beat_worker, args=(self,), daemon=True)\n            self.heart_beat_thread.start()\n\n    def register_to_controller(self):\n        logger.info(\"Register to controller\")\n\n        url = self.controller_addr + \"/register_worker\"\n        data = {\n            \"worker_name\": self.worker_addr,\n            \"check_heart_beat\": True,\n            \"worker_status\": self.get_status()\n        }\n        r = requests.post(url, json=data)\n        assert r.status_code == 200\n\n    def send_heart_beat(self):\n        logger.info(f\"Send heart beat. Models: {[self.model_name]}. \"\n                    f\"Semaphore: {pretty_print_semaphore(model_semaphore)}. \"\n                    f\"global_counter: {global_counter}\")\n\n        url = self.controller_addr + \"/receive_heart_beat\"\n\n        while True:\n            try:\n                ret = requests.post(url, json={\n                    \"worker_name\": self.worker_addr,\n                    \"queue_length\": self.get_queue_length()}, timeout=5)\n                exist = ret.json()[\"exist\"]\n                break\n            except requests.exceptions.RequestException as e:\n                logger.error(f\"heart beat error: {e}\")\n            time.sleep(5)\n\n        if not exist:\n            self.register_to_controller()\n\n    def get_queue_length(self):\n        if model_semaphore is None:\n            return 0\n        else:\n            return args.limit_model_concurrency - model_semaphore._value + (len(\n                model_semaphore._waiters) if model_semaphore._waiters is not None else 0)\n\n    def get_status(self):\n        return {\n            \"model_names\": [self.model_name],\n            \"speed\": 1,\n            \"queue_length\": self.get_queue_length(),\n        }\n\n    async def generate_stream(self, params):\n        ori_prompt = prompt = params[\"prompt\"]\n        images = params.get(\"images\", None)\n        if images is not None and len(images) > 0:\n            if len(images) > 0:\n                if len(images) != prompt.count(DEFAULT_IMAGE_TOKEN):\n                    raise ValueError(\"Number of images does not match number of <image> tokens in prompt\")\n\n                images = [load_image_from_base64(image) for image in images]\n\n                # FIXME: for image-start/end token\n                # replace_token = DEFAULT_IMAGE_TOKEN\n                # if getattr(self.model.config, 'mm_use_im_start_end', False):\n                #     replace_token = DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN\n                # prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replace_token)\n                prompt = prompt.replace(' ' + DEFAULT_IMAGE_TOKEN + '\\n', DEFAULT_IMAGE_TOKEN)\n                prompt_split = prompt.split(DEFAULT_IMAGE_TOKEN)\n                prompt = []\n                for i in range(len(prompt_split)):\n                    prompt.append(prompt_split[i])\n                    if i < len(images):\n                        prompt.append(images[i])\n        else:\n            prompt = [prompt]\n\n        temperature = float(params.get(\"temperature\", 1.0))\n        top_p = float(params.get(\"top_p\", 1.0))\n        # max_context_length = getattr(model.config, 'max_position_embeddings', 2048)\n        max_new_tokens = min(int(params.get(\"max_new_tokens\", 256)), 1024)\n        stop_str = params.get(\"stop\", None)\n        stop_str = [stop_str] if stop_str is not None else None\n\n        print({'prompt': prompt, 'max_new_tokens': max_new_tokens, 'temperature': temperature, 'top_p': top_p})\n        state = pipeline.run(prompt, max_new_tokens, temperature=temperature, top_p=top_p, stream=True)\n\n        generated_text = ori_prompt\n        async for text_outputs in state.text_async_iter(var_name=\"response\"):\n            generated_text += text_outputs\n            yield json.dumps({\"text\": generated_text, \"error_code\": 0}).encode() + b\"\\0\"\n\n    async def generate_stream_gate(self, params):\n        try:\n            async for x in self.generate_stream(params):\n                yield x\n        except ValueError as e:\n            print(\"Caught ValueError:\", e)\n            ret = {\n                \"text\": server_error_msg,\n                \"error_code\": 1,\n            }\n            yield json.dumps(ret).encode() + b\"\\0\"\n        except Exception as e:\n            print(\"Caught Unknown Error\", e)\n            ret = {\n                \"text\": server_error_msg,\n                \"error_code\": 1,\n            }\n            yield json.dumps(ret).encode() + b\"\\0\"\n\n\napp = FastAPI()\n\n\ndef release_model_semaphore(fn=None):\n    model_semaphore.release()\n    if fn is not None:\n        fn()\n\n\n@app.post(\"/worker_generate_stream\")\nasync def generate_stream(request: Request):\n    global model_semaphore, global_counter\n    global_counter += 1\n    params = await request.json()\n\n    if model_semaphore is None:\n        model_semaphore = asyncio.Semaphore(args.limit_model_concurrency)\n    await model_semaphore.acquire()\n    worker.send_heart_beat()\n    generator = worker.generate_stream_gate(params)\n    background_tasks = BackgroundTasks()\n    background_tasks.add_task(partial(release_model_semaphore, fn=worker.send_heart_beat))\n    return StreamingResponse(generator, background=background_tasks)\n\n\n@app.post(\"/worker_get_status\")\nasync def get_status(request: Request):\n    return worker.get_status()\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--host\", type=str, default=\"localhost\")\n    parser.add_argument(\"--port\", type=int, default=21002)\n    parser.add_argument(\"--worker-address\", type=str,\n                        default=\"http://localhost:21002\")\n    parser.add_argument(\"--controller-address\", type=str,\n                        default=\"http://localhost:21001\")\n    parser.add_argument(\"--model-name\", type=str)\n    parser.add_argument(\"--sgl-endpoint\", type=str)\n    parser.add_argument(\"--limit-model-concurrency\", type=int, default=5)\n    parser.add_argument(\"--stream-interval\", type=int, default=1)\n    parser.add_argument(\"--no-register\", action=\"store_true\")\n    args = parser.parse_args()\n    logger.info(f\"args: {args}\")\n\n    worker = ModelWorker(args.controller_address,\n                         args.worker_address,\n                         args.sgl_endpoint,\n                         worker_id,\n                         args.no_register,\n                         args.model_name)\n    uvicorn.run(app, host=args.host, port=args.port, log_level=\"info\")\n"
  },
  {
    "path": "llava/serve/test_message.py",
    "content": "import argparse\nimport json\n\nimport requests\n\nfrom llava.conversation import default_conversation\n\n\ndef main():\n    if args.worker_address:\n        worker_addr = args.worker_address\n    else:\n        controller_addr = args.controller_address\n        ret = requests.post(controller_addr + \"/refresh_all_workers\")\n        ret = requests.post(controller_addr + \"/list_models\")\n        models = ret.json()[\"models\"]\n        models.sort()\n        print(f\"Models: {models}\")\n\n        ret = requests.post(controller_addr + \"/get_worker_address\",\n                            json={\"model\": args.model_name})\n        worker_addr = ret.json()[\"address\"]\n        print(f\"worker_addr: {worker_addr}\")\n\n    if worker_addr == \"\":\n        return\n\n    conv = default_conversation.copy()\n    conv.append_message(conv.roles[0], args.message)\n    prompt = conv.get_prompt()\n\n    headers = {\"User-Agent\": \"LLaVA Client\"}\n    pload = {\n        \"model\": args.model_name,\n        \"prompt\": prompt,\n        \"max_new_tokens\": args.max_new_tokens,\n        \"temperature\": 0.7,\n        \"stop\": conv.sep,\n    }\n    response = requests.post(worker_addr + \"/worker_generate_stream\", headers=headers,\n                             json=pload, stream=True)\n\n    print(prompt.replace(conv.sep, \"\\n\"), end=\"\")\n    for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b\"\\0\"):\n        if chunk:\n            data = json.loads(chunk.decode(\"utf-8\"))\n            output = data[\"text\"].split(conv.sep)[-1]\n            print(output, end=\"\\r\")\n    print(\"\")\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--controller-address\", type=str, default=\"http://localhost:21001\")\n    parser.add_argument(\"--worker-address\", type=str)\n    parser.add_argument(\"--model-name\", type=str, default=\"facebook/opt-350m\")\n    parser.add_argument(\"--max-new-tokens\", type=int, default=32)\n    parser.add_argument(\"--message\", type=str, default=\"Tell me a story with more than 1000 words.\")\n    args = parser.parse_args()\n\n    main()\n"
  },
  {
    "path": "llava/train/llama_flash_attn_monkey_patch.py",
    "content": "from typing import Optional, Tuple\nimport warnings\n\nimport torch\n\nimport transformers\nfrom transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv\n\ntry:\n    from flash_attn.flash_attn_interface import flash_attn_unpadded_qkvpacked_func\nexcept ImportError:\n    from flash_attn.flash_attn_interface import flash_attn_varlen_qkvpacked_func as flash_attn_unpadded_qkvpacked_func\nfrom flash_attn.bert_padding import unpad_input, pad_input\n\n\ndef forward(\n    self,\n    hidden_states: torch.Tensor,\n    attention_mask: Optional[torch.Tensor] = None,\n    position_ids: Optional[torch.Tensor] = None,\n    past_key_value: Optional[Tuple[torch.Tensor]] = None,\n    output_attentions: bool = False,\n    use_cache: bool = False,\n) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:\n    if output_attentions:\n        warnings.warn(\n            \"Output attentions is not supported for patched `LlamaAttention`, returning `None` instead.\"\n        )\n\n    bsz, q_len, _ = hidden_states.size()\n\n    query_states = (\n        self.q_proj(hidden_states)\n        .view(bsz, q_len, self.num_heads, self.head_dim)\n        .transpose(1, 2)\n    )\n    key_states = (\n        self.k_proj(hidden_states)\n        .view(bsz, q_len, self.num_key_value_heads, self.head_dim)\n        .transpose(1, 2)\n    )\n    value_states = (\n        self.v_proj(hidden_states)\n        .view(bsz, q_len, self.num_key_value_heads, self.head_dim)\n        .transpose(1, 2)\n    )  # shape: (b, num_heads, s, head_dim)\n\n    kv_seq_len = key_states.shape[-2]\n    if past_key_value is not None:\n        kv_seq_len += past_key_value[0].shape[-2]\n\n    cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)\n    query_states, key_states = apply_rotary_pos_emb(\n        query_states, key_states, cos, sin, position_ids\n    )\n\n    if past_key_value is not None:\n        # reuse k, v\n        key_states = torch.cat([past_key_value[0], key_states], dim=2)\n        value_states = torch.cat([past_key_value[1], value_states], dim=2)\n\n    past_key_value = (key_states, value_states) if use_cache else None\n\n    # repeat k/v heads if n_kv_heads < n_heads\n    key_states = repeat_kv(key_states, self.num_key_value_groups)\n    value_states = repeat_kv(value_states, self.num_key_value_groups)\n\n    # Transform the data into the format required by flash attention\n    qkv = torch.stack([query_states, key_states, value_states], dim=2)\n    qkv = qkv.transpose(1, 3)  # shape: [b, s, 3, num_heads, head_dim]\n    key_padding_mask = attention_mask\n\n    if key_padding_mask is None:\n        qkv = qkv.reshape(-1, 3, self.num_heads, self.head_dim)\n        cu_q_lens = torch.arange(\n            0, (bsz + 1) * q_len, step=q_len, dtype=torch.int32, device=qkv.device\n        )\n        max_s = q_len\n        output = flash_attn_unpadded_qkvpacked_func(\n            qkv, cu_q_lens, max_s, 0.0, softmax_scale=None, causal=True\n        )\n        output = output.view(bsz, q_len, -1)\n    else:\n        qkv = qkv.reshape(bsz, q_len, -1)\n        qkv, indices, cu_q_lens, max_s = unpad_input(qkv, key_padding_mask)\n        qkv = qkv.view(-1, 3, self.num_heads, self.head_dim)\n        output_unpad = flash_attn_unpadded_qkvpacked_func(\n            qkv, cu_q_lens, max_s, 0.0, softmax_scale=None, causal=True\n        )\n        output_unpad = output_unpad.reshape(-1, self.num_heads * self.head_dim)\n        output = pad_input(output_unpad, indices, bsz, q_len)\n\n    return self.o_proj(output), None, past_key_value\n\n\n# Disable the transformation of the attention mask in LlamaModel as the flash attention\n# requires the attention mask to be the same as the key_padding_mask\ndef _prepare_decoder_attention_mask(\n    self, attention_mask, input_shape, inputs_embeds, past_key_values_length\n):\n    # [bsz, seq_len]\n    return attention_mask\n\n\ndef replace_llama_attn_with_flash_attn():\n    cuda_major, cuda_minor = torch.cuda.get_device_capability()\n    if cuda_major < 8:\n        warnings.warn(\n            \"Flash attention is only supported on A100 or H100 GPU during training due to head dim > 64 backward.\"\n            \"ref: https://github.com/HazyResearch/flash-attention/issues/190#issuecomment-1523359593\"\n        )\n    transformers.models.llama.modeling_llama.LlamaModel._prepare_decoder_attention_mask = (\n        _prepare_decoder_attention_mask\n    )\n    transformers.models.llama.modeling_llama.LlamaAttention.forward = forward\n"
  },
  {
    "path": "llava/train/llama_xformers_attn_monkey_patch.py",
    "content": "\"\"\"\nDirectly copied the code from https://raw.githubusercontent.com/oobabooga/text-generation-webui/main/modules/llama_attn_hijack.py and made some adjustments\n\"\"\"\n\nimport logging\nimport math\nfrom typing import Optional, Tuple\n\nimport torch\nimport transformers.models.llama.modeling_llama\nfrom torch import nn\n\ntry:\n    import xformers.ops\nexcept ImportError:\n    logging.error(\"xformers not found! Please install it before trying to use it.\")\n\n\ndef replace_llama_attn_with_xformers_attn():\n    transformers.models.llama.modeling_llama.LlamaAttention.forward = xformers_forward\n\n\ndef xformers_forward(\n    self,\n    hidden_states: torch.Tensor,\n    attention_mask: Optional[torch.Tensor] = None,\n    position_ids: Optional[torch.LongTensor] = None,\n    past_key_value: Optional[Tuple[torch.Tensor]] = None,\n    output_attentions: bool = False,\n    use_cache: bool = False,\n) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:\n    # pylint: disable=duplicate-code\n    bsz, q_len, _ = hidden_states.size()\n\n    query_states = (\n        self.q_proj(hidden_states)\n        .view(bsz, q_len, self.num_heads, self.head_dim)\n        .transpose(1, 2)\n    )\n    key_states = (\n        self.k_proj(hidden_states)\n        .view(bsz, q_len, self.num_heads, self.head_dim)\n        .transpose(1, 2)\n    )\n    value_states = (\n        self.v_proj(hidden_states)\n        .view(bsz, q_len, self.num_heads, self.head_dim)\n        .transpose(1, 2)\n    )\n\n    kv_seq_len = key_states.shape[-2]\n    if past_key_value is not None:\n        kv_seq_len += past_key_value[0].shape[-2]\n    cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)\n    (\n        query_states,\n        key_states,\n    ) = transformers.models.llama.modeling_llama.apply_rotary_pos_emb(\n        query_states, key_states, cos, sin, position_ids\n    )\n    # [bsz, nh, t, hd]\n\n    if past_key_value is not None:\n        # reuse k, v, self_attention\n        key_states = torch.cat([past_key_value[0], key_states], dim=2)\n        value_states = torch.cat([past_key_value[1], value_states], dim=2)\n\n    past_key_value = (key_states, value_states) if use_cache else None\n\n    # We only apply xformers optimizations if we don't need to output the whole attention matrix\n    if not output_attentions:\n        query_states = query_states.transpose(1, 2)\n        key_states = key_states.transpose(1, 2)\n        value_states = value_states.transpose(1, 2)\n\n        # This is a nasty hack. We know attention_mask in transformers is either LowerTriangular or all Zeros.\n        # We therefore check if one element in the upper triangular portion is zero. If it is, then the mask is all zeros.\n        if attention_mask is None or attention_mask[0, 0, 0, 1] == 0:\n            # input and output should be of form (bsz, q_len, num_heads, head_dim)\n            attn_output = xformers.ops.memory_efficient_attention(\n                query_states, key_states, value_states, attn_bias=None\n            )\n        else:\n            # input and output should be of form (bsz, q_len, num_heads, head_dim)\n            attn_output = xformers.ops.memory_efficient_attention(\n                query_states,\n                key_states,\n                value_states,\n                attn_bias=xformers.ops.LowerTriangularMask(),\n            )\n        attn_weights = None\n    else:\n        attn_weights = torch.matmul(\n            query_states, key_states.transpose(2, 3)\n        ) / math.sqrt(self.head_dim)\n\n        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):\n            raise ValueError(\n                f\"Attention weights should be of size {(bsz * self.num_heads, q_len, kv_seq_len)}, but is\"\n                f\" {attn_weights.size()}\"\n            )\n\n        if attention_mask is not None:\n            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):\n                raise ValueError(\n                    f\"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}\"\n                )\n            attn_weights = attn_weights + attention_mask\n            attn_weights = torch.max(\n                attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)\n            )\n\n        # upcast attention to fp32\n        attn_weights = nn.functional.softmax(\n            attn_weights, dim=-1, dtype=torch.float32\n        ).to(query_states.dtype)\n        attn_output = torch.matmul(attn_weights, value_states)\n\n        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):\n            raise ValueError(\n                f\"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is\"\n                f\" {attn_output.size()}\"\n            )\n\n        attn_output = attn_output.transpose(1, 2)\n\n    attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)\n    attn_output = self.o_proj(attn_output)\n    return attn_output, attn_weights, past_key_value\n"
  },
  {
    "path": "llava/train/llava_trainer.py",
    "content": "import os\nimport torch\nimport torch.nn as nn\n\nfrom torch.utils.data import Sampler\n\nimport transformers\nfrom transformers import Trainer\nfrom transformers.trainer import (\n    is_sagemaker_mp_enabled,\n    get_parameter_names,\n    has_length,\n    # ALL_LAYERNORM_LAYERS,\n    logger,\n)\nfrom typing import List, Optional\n\n\nALL_LAYERNORM_LAYERS = [nn.LayerNorm, nn.BatchNorm2d]\n\n\ndef maybe_zero_3(param, ignore_status=False, name=None):\n    from deepspeed import zero\n    from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus\n    if hasattr(param, \"ds_id\"):\n        if param.ds_status == ZeroParamStatus.NOT_AVAILABLE:\n            if not ignore_status:\n                print(name, 'no ignore status')\n        with zero.GatheredParameters([param]):\n            param = param.data.detach().cpu().clone()\n    else:\n        param = param.detach().cpu().clone()\n    return param\n\n\ndef get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match):\n    to_return = {k: t for k, t in named_params if any(key_match in k for key_match in keys_to_match)}\n    to_return = {k: maybe_zero_3(v, ignore_status=True, name=k).cpu() for k, v in to_return.items()}\n    return to_return\n\n\ndef split_to_even_chunks(indices, lengths, num_chunks):\n    \"\"\"\n    Split a list of indices into `chunks` chunks of roughly equal lengths.\n    \"\"\"\n\n    if len(indices) % num_chunks != 0:\n        return [indices[i::num_chunks] for i in range(num_chunks)]\n\n    num_indices_per_chunk = len(indices) // num_chunks\n\n    chunks = [[] for _ in range(num_chunks)]\n    chunks_lengths = [0 for _ in range(num_chunks)]\n    for index in indices:\n        shortest_chunk = chunks_lengths.index(min(chunks_lengths))\n        chunks[shortest_chunk].append(index)\n        chunks_lengths[shortest_chunk] += lengths[index]\n        if len(chunks[shortest_chunk]) == num_indices_per_chunk:\n            chunks_lengths[shortest_chunk] = float(\"inf\")\n\n    return chunks\n\n\ndef get_modality_length_grouped_indices(lengths, batch_size, world_size, generator=None):\n    # We need to use torch for the random part as a distributed sampler will set the random seed for torch.\n    assert all(l != 0 for l in lengths), \"Should not have zero length.\"\n    if all(l > 0 for l in lengths) or all(l < 0 for l in lengths):\n        # all samples are in the same modality\n        return get_length_grouped_indices(lengths, batch_size, world_size, generator=generator)\n    mm_indices, mm_lengths = zip(*[(i, l) for i, l in enumerate(lengths) if l > 0])\n    lang_indices, lang_lengths = zip(*[(i, -l) for i, l in enumerate(lengths) if l < 0])\n\n    mm_shuffle = [mm_indices[i] for i in get_length_grouped_indices(mm_lengths, batch_size, world_size, generator=None)]\n    lang_shuffle = [lang_indices[i] for i in get_length_grouped_indices(lang_lengths, batch_size, world_size, generator=None)]\n    megabatch_size = world_size * batch_size\n    mm_megabatches = [mm_shuffle[i : i + megabatch_size] for i in range(0, len(mm_shuffle), megabatch_size)]\n    lang_megabatches = [lang_shuffle[i : i + megabatch_size] for i in range(0, len(lang_shuffle), megabatch_size)]\n\n    last_mm = mm_megabatches[-1]\n    last_lang = lang_megabatches[-1]\n    additional_batch = last_mm + last_lang\n    megabatches = mm_megabatches[:-1] + lang_megabatches[:-1]\n    megabatch_indices = torch.randperm(len(megabatches), generator=generator)\n    megabatches = [megabatches[i] for i in megabatch_indices]\n\n    if len(additional_batch) > 0:\n        megabatches.append(sorted(additional_batch))\n\n    return [i for megabatch in megabatches for i in megabatch]\n\n\ndef get_length_grouped_indices(lengths, batch_size, world_size, generator=None, merge=True):\n    # We need to use torch for the random part as a distributed sampler will set the random seed for torch.\n    indices = torch.randperm(len(lengths), generator=generator)\n    megabatch_size = world_size * batch_size\n    megabatches = [indices[i : i + megabatch_size].tolist() for i in range(0, len(lengths), megabatch_size)]\n    megabatches = [sorted(megabatch, key=lambda i: lengths[i], reverse=True) for megabatch in megabatches]\n    megabatches = [split_to_even_chunks(megabatch, lengths, world_size) for megabatch in megabatches]\n\n    return [i for megabatch in megabatches for batch in megabatch for i in batch]\n\n\nclass LengthGroupedSampler(Sampler):\n    r\"\"\"\n    Sampler that samples indices in a way that groups together features of the dataset of roughly the same length while\n    keeping a bit of randomness.\n    \"\"\"\n\n    def __init__(\n        self,\n        batch_size: int,\n        world_size: int,\n        lengths: Optional[List[int]] = None,\n        generator=None,\n        group_by_modality: bool = False,\n    ):\n        if lengths is None:\n            raise ValueError(\"Lengths must be provided.\")\n\n        self.batch_size = batch_size\n        self.world_size = world_size\n        self.lengths = lengths\n        self.generator = generator\n        self.group_by_modality = group_by_modality\n\n    def __len__(self):\n        return len(self.lengths)\n\n    def __iter__(self):\n        if self.group_by_modality:\n            indices = get_modality_length_grouped_indices(self.lengths, self.batch_size, self.world_size, generator=self.generator)\n        else:\n            indices = get_length_grouped_indices(self.lengths, self.batch_size, self.world_size, generator=self.generator)\n        return iter(indices)\n\n\nclass LLaVATrainer(Trainer):\n\n    def _get_train_sampler(self) -> Optional[torch.utils.data.Sampler]:\n        if self.train_dataset is None or not has_length(self.train_dataset):\n            return None\n\n        if self.args.group_by_modality_length:\n            lengths = self.train_dataset.modality_lengths\n            return LengthGroupedSampler(\n                self.args.train_batch_size,\n                world_size=self.args.world_size * self.args.gradient_accumulation_steps,\n                lengths=lengths,\n                group_by_modality=True,\n            )\n        else:\n            return super()._get_train_sampler()\n\n    def create_optimizer(self):\n        \"\"\"\n        Setup the optimizer.\n\n        We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the\n        Trainer's init through `optimizers`, or subclass and override this method in a subclass.\n        \"\"\"\n        if is_sagemaker_mp_enabled():\n            return super().create_optimizer()\n\n        opt_model = self.model\n\n        if self.optimizer is None:\n            decay_parameters = get_parameter_names(opt_model, ALL_LAYERNORM_LAYERS)\n            decay_parameters = [name for name in decay_parameters if \"bias\" not in name]\n\n            lr_mapper = {}\n            if self.args.mm_projector_lr is not None:\n                lr_mapper[\"mm_projector\"] = self.args.mm_projector_lr\n            if self.args.mm_vision_tower_lr is not None:\n                lr_mapper[\"vision_tower\"] = self.args.mm_vision_tower_lr\n\n            if len(lr_mapper) > 0:\n                special_lr_parameters = [name for name, _ in opt_model.named_parameters() if\n                                         any(module_keyword in name for module_keyword in lr_mapper)]\n                optimizer_grouped_parameters = [\n                    {\n                        \"params\": [p for n, p in opt_model.named_parameters() if\n                                   (n in decay_parameters and n not in special_lr_parameters and p.requires_grad)],\n                        \"weight_decay\": self.args.weight_decay,\n                    },\n                    {\n                        \"params\": [p for n, p in opt_model.named_parameters() if\n                                   (n not in decay_parameters and n not in special_lr_parameters and p.requires_grad)],\n                        \"weight_decay\": 0.0,\n                    },\n                ]\n                for module_keyword, lr in lr_mapper.items():\n                    module_parameters = [name for name, _ in opt_model.named_parameters() if module_keyword in name]\n                    optimizer_grouped_parameters.extend(\n                        [\n                            {\n                                \"params\": [p for n, p in opt_model.named_parameters() if\n                                           (n in decay_parameters and n in module_parameters and p.requires_grad)],\n                                \"weight_decay\": self.args.weight_decay,\n                                \"lr\": lr,\n                            },\n                            {\n                                \"params\": [p for n, p in opt_model.named_parameters() if\n                                           (n not in decay_parameters and n in module_parameters and p.requires_grad)],\n                                \"weight_decay\": 0.0,\n                                \"lr\": lr,\n                            },\n                        ]\n                    )\n            else:\n                optimizer_grouped_parameters = [\n                    {\n                        \"params\": [\n                            p for n, p in opt_model.named_parameters() if (n in decay_parameters and p.requires_grad)\n                        ],\n                        \"weight_decay\": self.args.weight_decay,\n                    },\n                    {\n                        \"params\": [\n                            p for n, p in opt_model.named_parameters() if (n not in decay_parameters and p.requires_grad)\n                        ],\n                        \"weight_decay\": 0.0,\n                    },\n                ]\n\n            optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(self.args)\n\n            self.optimizer = optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs)\n            if optimizer_cls.__name__ == \"Adam8bit\":\n                import bitsandbytes\n\n                manager = bitsandbytes.optim.GlobalOptimManager.get_instance()\n\n                skipped = 0\n                for module in opt_model.modules():\n                    if isinstance(module, nn.Embedding):\n                        skipped += sum({p.data_ptr(): p.numel() for p in module.parameters()}.values())\n                        logger.info(f\"skipped {module}: {skipped/2**20}M params\")\n                        manager.register_module_override(module, \"weight\", {\"optim_bits\": 32})\n                        logger.debug(f\"bitsandbytes: will optimize {module} in fp32\")\n                logger.info(f\"skipped: {skipped/2**20}M params\")\n\n        return self.optimizer\n\n    def _save_checkpoint(self, model, trial, metrics=None):\n        if getattr(self.args, 'tune_mm_mlp_adapter', False):\n            from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR\n            checkpoint_folder = f\"{PREFIX_CHECKPOINT_DIR}-{self.state.global_step}\"\n\n            run_dir = self._get_output_dir(trial=trial)\n            output_dir = os.path.join(run_dir, checkpoint_folder)\n\n            # Only save Adapter\n            keys_to_match = ['mm_projector', 'vision_resampler']\n            if getattr(self.args, \"use_im_start_end\", False):\n                keys_to_match.extend(['embed_tokens', 'embed_in'])\n\n            weight_to_save = get_mm_adapter_state_maybe_zero_3(self.model.named_parameters(), keys_to_match)\n\n            if self.args.local_rank == 0 or self.args.local_rank == -1:\n                self.model.config.save_pretrained(output_dir)\n                torch.save(weight_to_save, os.path.join(output_dir, f'mm_projector.bin'))\n        else:\n            # Workaround for the issue: https://github.com/haotian-liu/LLaVA/issues/1144\n            model.generation_config = transformers.GenerationConfig(do_sample=True, temperature=None, top_p=None)\n            super(LLaVATrainer, self)._save_checkpoint(model, trial, metrics)\n\n    def _save(self, output_dir: Optional[str] = None, state_dict=None):\n        if getattr(self.args, 'tune_mm_mlp_adapter', False):\n            pass\n        else:\n            # Workaround for the issue: https://github.com/haotian-liu/LLaVA/issues/1144\n            self.model.generation_config = transformers.GenerationConfig(do_sample=True, temperature=None, top_p=None)\n            super(LLaVATrainer, self)._save(output_dir, state_dict)\n"
  },
  {
    "path": "llava/train/train.py",
    "content": "# Adopted from https://github.com/lm-sys/FastChat. Below is the original copyright:\n# Adopted from tatsu-lab@stanford_alpaca. Below is the original copyright:\n#    Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n\nfrom packaging import version\nimport os\nimport copy\nfrom dataclasses import dataclass, field\nimport json\nimport logging\nimport pathlib\nfrom typing import Dict, Optional, Sequence, List\n\nimport torch\n\nimport transformers\nimport tokenizers\n\nfrom llava.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN\nfrom torch.utils.data import Dataset\nfrom llava.train.llava_trainer import LLaVATrainer\n\nfrom llava import conversation as conversation_lib\nfrom llava.model import *\nfrom llava.mm_utils import tokenizer_image_token, process_anyres_image\n\nfrom PIL import Image\n\n\nlocal_rank = None\n\n\ndef rank0_print(*args):\n    if local_rank == 0:\n        print(*args)\n\n\nIS_TOKENIZER_GREATER_THAN_0_14 = version.parse(tokenizers.__version__) >= version.parse('0.14')\n\n\n@dataclass\nclass ModelArguments:\n    model_name_or_path: Optional[str] = field(default=\"facebook/opt-125m\")\n    version: Optional[str] = field(default=\"v0\")\n    freeze_backbone: bool = field(default=False)\n    tune_mm_mlp_adapter: bool = field(default=False)\n    tune_mm_mlp_and_vision_tower: bool = field(default=False)\n    vision_tower: Optional[str] = field(default=None)\n    mm_vision_select_layer: Optional[int] = field(default=-1)   # default to the last layer\n    pretrain_mm_mlp_adapter: Optional[str] = field(default=None)\n    mm_projector_type: Optional[str] = field(default='linear')\n    mm_use_im_start_end: bool = field(default=False)\n    mm_use_im_patch_token: bool = field(default=True)\n    mm_patch_merge_type: Optional[str] = field(default='flat')\n    mm_vision_select_feature: Optional[str] = field(default=\"patch\")\n    unfreeze_mm_vision_tower: bool = field(default=False)\n    s2: Optional[bool] = field(default=False)\n    hd: Optional[bool] = field(default=False)\n\n\n@dataclass\nclass DataArguments:\n    data_path: Optional[List[str]] = field(default=None,\n                           metadata={\"help\": \"Optional list of paths to the training data.\"})\n    lazy_preprocess: bool = False\n    is_multimodal: bool = False\n    image_folder: Optional[List[str]] = field(default=None)\n    image_aspect_ratio: str = 'square'\n    image_grid_pinpoints: Optional[str] = field(default=None)\n    image_crop_resolution: Optional[int] = field(default=None)\n    image_split_resolution: Optional[int] = field(default=None)\n\n\n@dataclass\nclass TrainingArguments(transformers.TrainingArguments):\n    cache_dir: Optional[str] = field(default=None)\n    optim: str = field(default=\"adamw_torch\")\n    remove_unused_columns: bool = field(default=False)\n    freeze_mm_mlp_adapter: bool = field(default=False)\n    mpt_attn_impl: Optional[str] = field(default=\"triton\")\n    model_max_length: int = field(\n        default=512,\n        metadata={\n            \"help\":\n            \"Maximum sequence length. Sequences will be right padded (and possibly truncated).\"\n        },\n    )\n    double_quant: bool = field(\n        default=True,\n        metadata={\"help\": \"Compress the quantization statistics through double quantization.\"}\n    )\n    quant_type: str = field(\n        default=\"nf4\",\n        metadata={\"help\": \"Quantization data type to use. Should be one of `fp4` or `nf4`.\"}\n    )\n    bits: int = field(\n        default=16,\n        metadata={\"help\": \"How many bits to use.\"}\n    )\n    lora_enable: bool = False\n    lora_r: int = 64\n    lora_alpha: int = 16\n    lora_dropout: float = 0.05\n    lora_weight_path: str = \"\"\n    lora_bias: str = \"none\"\n    mm_projector_lr: Optional[float] = None\n    mm_vision_tower_lr: Optional[float] = None\n    group_by_modality_length: bool = field(default=False)\n\n\ndef maybe_zero_3(param, ignore_status=False, name=None):\n    from deepspeed import zero\n    from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus\n    if hasattr(param, \"ds_id\"):\n        if param.ds_status == ZeroParamStatus.NOT_AVAILABLE:\n            if not ignore_status:\n                logging.warning(f\"{name}: param.ds_status != ZeroParamStatus.NOT_AVAILABLE: {param.ds_status}\")\n        with zero.GatheredParameters([param]):\n            param = param.data.detach().cpu().clone()\n    else:\n        param = param.detach().cpu().clone()\n    return param\n\n\n# Borrowed from peft.utils.get_peft_model_state_dict\ndef get_peft_state_maybe_zero_3(named_params, bias):\n    if bias == \"none\":\n        to_return = {k: t for k, t in named_params if \"lora_\" in k}\n    elif bias == \"all\":\n        to_return = {k: t for k, t in named_params if \"lora_\" in k or \"bias\" in k}\n    elif bias == \"lora_only\":\n        to_return = {}\n        maybe_lora_bias = {}\n        lora_bias_names = set()\n        for k, t in named_params:\n            if \"lora_\" in k:\n                to_return[k] = t\n                bias_name = k.split(\"lora_\")[0] + \"bias\"\n                lora_bias_names.add(bias_name)\n            elif \"bias\" in k:\n                maybe_lora_bias[k] = t\n        for k, t in maybe_lora_bias:\n            if bias_name in lora_bias_names:\n                to_return[bias_name] = t\n    else:\n        raise NotImplementedError\n    to_return = {k: maybe_zero_3(v, ignore_status=True) for k, v in to_return.items()}\n    return to_return\n\n\ndef get_peft_state_non_lora_maybe_zero_3(named_params, require_grad_only=True):\n    to_return = {k: t for k, t in named_params if \"lora_\" not in k}\n    if require_grad_only:\n        to_return = {k: t for k, t in to_return.items() if t.requires_grad}\n    to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}\n    return to_return\n\n\ndef get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match):\n    to_return = {k: t for k, t in named_params if any(key_match in k for key_match in keys_to_match)}\n    to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}\n    return to_return\n\n\ndef find_all_linear_names(model):\n    cls = torch.nn.Linear\n    lora_module_names = set()\n    multimodal_keywords = ['mm_projector', 'vision_tower', 'vision_resampler']\n    for name, module in model.named_modules():\n        if any(mm_keyword in name for mm_keyword in multimodal_keywords):\n            continue\n        if isinstance(module, cls):\n            names = name.split('.')\n            lora_module_names.add(names[0] if len(names) == 1 else names[-1])\n\n    if 'lm_head' in lora_module_names:  # needed for 16-bit\n        lora_module_names.remove('lm_head')\n    return list(lora_module_names)\n\n\ndef safe_save_model_for_hf_trainer(trainer: transformers.Trainer,\n                                   output_dir: str):\n    \"\"\"Collects the state dict and dump to disk.\"\"\"\n\n    if getattr(trainer.args, \"tune_mm_mlp_adapter\", False):\n        # Only save Adapter\n        keys_to_match = ['mm_projector']\n        if getattr(trainer.args, \"use_im_start_end\", False):\n            keys_to_match.extend(['embed_tokens', 'embed_in', 'tok_embeddings'])\n\n        weight_to_save = get_mm_adapter_state_maybe_zero_3(trainer.model.named_parameters(), keys_to_match)\n        trainer.model.config.save_pretrained(output_dir)\n\n        current_folder = output_dir.split('/')[-1]\n        parent_folder = os.path.dirname(output_dir)\n        if trainer.args.local_rank == 0 or trainer.args.local_rank == -1:\n            if current_folder.startswith('checkpoint-'):\n                mm_projector_folder = os.path.join(parent_folder, \"mm_projector\")\n                os.makedirs(mm_projector_folder, exist_ok=True)\n                torch.save(weight_to_save, os.path.join(mm_projector_folder, f'{current_folder}.bin'))\n            else:\n                torch.save(weight_to_save, os.path.join(output_dir, f'mm_projector.bin'))\n        return\n\n    if trainer.deepspeed:\n        torch.cuda.synchronize()\n        trainer.save_model(output_dir)\n        return\n\n    state_dict = trainer.model.state_dict()\n    if trainer.args.should_save:\n        cpu_state_dict = {\n            key: value.cpu()\n            for key, value in state_dict.items()\n        }\n        del state_dict\n        trainer._save(output_dir, state_dict=cpu_state_dict)  # noqa\n\n\ndef smart_tokenizer_and_embedding_resize(\n    special_tokens_dict: Dict,\n    tokenizer: transformers.PreTrainedTokenizer,\n    model: transformers.PreTrainedModel,\n):\n    \"\"\"Resize tokenizer and embedding.\n\n    Note: This is the unoptimized version that may make your embedding size not be divisible by 64.\n    \"\"\"\n    num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)\n    model.resize_token_embeddings(len(tokenizer))\n\n    if num_new_tokens > 0:\n        input_embeddings = model.get_input_embeddings().weight.data\n        output_embeddings = model.get_output_embeddings().weight.data\n\n        input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(\n            dim=0, keepdim=True)\n        output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(\n            dim=0, keepdim=True)\n\n        input_embeddings[-num_new_tokens:] = input_embeddings_avg\n        output_embeddings[-num_new_tokens:] = output_embeddings_avg\n\n\ndef _tokenize_fn(strings: Sequence[str],\n                 tokenizer: transformers.PreTrainedTokenizer) -> Dict:\n    \"\"\"Tokenize a list of strings.\"\"\"\n    tokenized_list = [\n        tokenizer(\n            text,\n            return_tensors=\"pt\",\n            padding=\"longest\",\n            max_length=tokenizer.model_max_length,\n            truncation=True,\n        ) for text in strings\n    ]\n    input_ids = labels = [\n        tokenized.input_ids[0] for tokenized in tokenized_list\n    ]\n    input_ids_lens = labels_lens = [\n        tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item()\n        for tokenized in tokenized_list\n    ]\n    return dict(\n        input_ids=input_ids,\n        labels=labels,\n        input_ids_lens=input_ids_lens,\n        labels_lens=labels_lens,\n    )\n\n\ndef _mask_targets(target, tokenized_lens, speakers):\n    # cur_idx = 0\n    cur_idx = tokenized_lens[0]\n    tokenized_lens = tokenized_lens[1:]\n    target[:cur_idx] = IGNORE_INDEX\n    for tokenized_len, speaker in zip(tokenized_lens, speakers):\n        if speaker == \"human\":\n            target[cur_idx+2:cur_idx + tokenized_len] = IGNORE_INDEX\n        cur_idx += tokenized_len\n\n\ndef _add_speaker_and_signal(header, source, get_conversation=True):\n    \"\"\"Add speaker and start/end signal on each round.\"\"\"\n    BEGIN_SIGNAL = \"### \"\n    END_SIGNAL = \"\\n\"\n    conversation = header\n    for sentence in source:\n        from_str = sentence[\"from\"]\n        if from_str.lower() == \"human\":\n            from_str = conversation_lib.default_conversation.roles[0]\n        elif from_str.lower() == \"gpt\":\n            from_str = conversation_lib.default_conversation.roles[1]\n        else:\n            from_str = 'unknown'\n        sentence[\"value\"] = (BEGIN_SIGNAL + from_str + \": \" +\n                             sentence[\"value\"] + END_SIGNAL)\n        if get_conversation:\n            conversation += sentence[\"value\"]\n    conversation += BEGIN_SIGNAL\n    return conversation\n\n\ndef preprocess_multimodal(\n    sources: Sequence[str],\n    data_args: DataArguments\n) -> Dict:\n    is_multimodal = data_args.is_multimodal\n    if not is_multimodal:\n        return sources\n\n    for source in sources:\n        for sentence in source:\n            if DEFAULT_IMAGE_TOKEN in sentence['value']:\n                sentence['value'] = sentence['value'].replace(DEFAULT_IMAGE_TOKEN, '').strip()\n                sentence['value'] = DEFAULT_IMAGE_TOKEN + '\\n' + sentence['value']\n                sentence['value'] = sentence['value'].strip()\n                if \"mmtag\" in conversation_lib.default_conversation.version:\n                    sentence['value'] = sentence['value'].replace(DEFAULT_IMAGE_TOKEN, '<Image>' + DEFAULT_IMAGE_TOKEN + '</Image>')\n            replace_token = DEFAULT_IMAGE_TOKEN\n            if data_args.mm_use_im_start_end:\n                replace_token = DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN\n            sentence[\"value\"] = sentence[\"value\"].replace(DEFAULT_IMAGE_TOKEN, replace_token)\n\n    return sources\n\n\ndef preprocess_llama_2(\n    sources,\n    tokenizer: transformers.PreTrainedTokenizer,\n    has_image: bool = False\n) -> Dict:\n    conv = conversation_lib.default_conversation.copy()\n    roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n\n    # Apply prompt templates\n    conversations = []\n    for i, source in enumerate(sources):\n        if roles[source[0][\"from\"]] != conv.roles[0]:\n            # Skip the first one if it is not from human\n            source = source[1:]\n\n        conv.messages = []\n        for j, sentence in enumerate(source):\n            role = roles[sentence[\"from\"]]\n            assert role == conv.roles[j % 2], f\"{i}\"\n            conv.append_message(role, sentence[\"value\"])\n        conversations.append(conv.get_prompt())\n\n    # Tokenize conversations\n\n    if has_image:\n        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)\n    else:\n        input_ids = tokenizer(\n            conversations,\n            return_tensors=\"pt\",\n            padding=\"longest\",\n            max_length=tokenizer.model_max_length,\n            truncation=True,\n        ).input_ids\n\n    targets = input_ids.clone()\n\n    assert conv.sep_style == conversation_lib.SeparatorStyle.LLAMA_2\n\n    # Mask targets\n    sep = \"[/INST] \"\n    for conversation, target in zip(conversations, targets):\n        total_len = int(target.ne(tokenizer.pad_token_id).sum())\n\n        rounds = conversation.split(conv.sep2)\n        cur_len = 1\n        target[:cur_len] = IGNORE_INDEX\n        for i, rou in enumerate(rounds):\n            if rou == \"\":\n                break\n\n            parts = rou.split(sep)\n            if len(parts) != 2:\n                break\n            parts[0] += sep\n\n            if has_image:\n                round_len = len(tokenizer_image_token(rou, tokenizer))\n                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 2\n            else:\n                round_len = len(tokenizer(rou).input_ids)\n                instruction_len = len(tokenizer(parts[0]).input_ids) - 2\n\n            target[cur_len: cur_len + instruction_len] = IGNORE_INDEX\n\n            cur_len += round_len\n        target[cur_len:] = IGNORE_INDEX\n\n        if cur_len < tokenizer.model_max_length:\n            if cur_len != total_len:\n                target[:] = IGNORE_INDEX\n                print(\n                    f\"WARNING: tokenization mismatch: {cur_len} vs. {total_len}.\"\n                    f\" (ignored)\"\n                )\n\n    return dict(\n        input_ids=input_ids,\n        labels=targets,\n    )\n\n\n# fix: add qwen2\ndef preprocess_qwen_2(\n    sources,\n    tokenizer: transformers.PreTrainedTokenizer,\n    has_image: bool = False\n) -> Dict:\n    conv = conversation_lib.default_conversation.copy()\n    roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n\n    # Apply prompt templates\n    conversations = []\n    for i, source in enumerate(sources):\n        if roles[source[0][\"from\"]] != conv.roles[0]:\n            # Skip the first one if it is not from human\n            source = source[1:]\n\n        conv.messages = []\n        for j, sentence in enumerate(source):\n            role = roles[sentence[\"from\"]]\n            assert role == conv.roles[j % 2], f\"{i}\"\n            conv.append_message(role, sentence[\"value\"])\n        conversations.append(conv.get_prompt())\n\n    # Tokenize conversations\n\n    if has_image:\n        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)\n    else:\n        input_ids = tokenizer(\n            conversations,\n            return_tensors=\"pt\",\n            padding=\"longest\",\n            max_length=tokenizer.model_max_length,\n            truncation=True,\n        ).input_ids\n\n    targets = input_ids.clone()\n\n    assert conv.sep_style == conversation_lib.SeparatorStyle.QWEN_2\n\n    # Mask targets\n    sep = conv.sep + conv.roles[1] + \": \"\n    for conversation, target in zip(conversations, targets):\n        total_len = int(target.ne(tokenizer.pad_token_id).sum())\n\n        rounds = conversation.split(conv.sep2)\n        rounds_len = len(rounds)\n        cur_len = 0\n        # target[:cur_len] = IGNORE_INDEX\n        for i, rou in enumerate(rounds):\n            if rou == \"\":\n                break\n\n            parts = rou.split(sep)\n            if len(parts) != 2:\n                break\n            parts[0] += sep\n\n            if has_image:\n                round_ids = tokenizer_image_token(rou, tokenizer)\n                instruction_ids = tokenizer_image_token(parts[0], tokenizer)\n                equal_parts = [x == y for x, y in zip(round_ids, instruction_ids)]\n\n                instruction_len = equal_parts.index(False) if False in equal_parts else len(equal_parts)\n                round_len = len(round_ids)\n\n            else:\n                round_ids = tokenizer(rou).input_ids\n                instruction_ids = tokenizer(parts[0]).input_ids\n                equal_parts = [x == y for x, y in zip(round_ids, instruction_ids)]\n\n                instruction_len = equal_parts.index(False) if False in equal_parts else len(equal_parts)\n                round_len = len(round_ids)\n\n            if i != 0 and not tokenizer.legacy and IS_TOKENIZER_GREATER_THAN_0_14:\n                round_len += 1\n                instruction_len += 1\n\n            target[cur_len: cur_len + instruction_len] = IGNORE_INDEX\n\n            cur_len += round_len\n        target[cur_len:] = IGNORE_INDEX\n\n        if cur_len < tokenizer.model_max_length:\n            if cur_len != total_len + rounds_len - 2:\n                target[:] = IGNORE_INDEX\n                print(\n                    f\"WARNING: tokenization mismatch: {cur_len} vs. {total_len}.\"\n                    f\" (ignored)\"\n                )\n\n    return dict(\n        input_ids=input_ids,\n        labels=targets,\n    )\n\n\ndef preprocess_v1(\n    sources,\n    tokenizer: transformers.PreTrainedTokenizer,\n    has_image: bool = False\n) -> Dict:\n    conv = conversation_lib.default_conversation.copy()\n    roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n\n    # Apply prompt templates\n    conversations = []\n    for i, source in enumerate(sources):\n        if roles[source[0][\"from\"]] != conv.roles[0]:\n            # Skip the first one if it is not from human\n            source = source[1:]\n\n        conv.messages = []\n        for j, sentence in enumerate(source):\n            role = roles[sentence[\"from\"]]\n            assert role == conv.roles[j % 2], f\"{i}\"\n            conv.append_message(role, sentence[\"value\"])\n        conversations.append(conv.get_prompt())\n\n    # Tokenize conversations\n\n    if has_image:\n        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)\n    else:\n        input_ids = tokenizer(\n            conversations,\n            return_tensors=\"pt\",\n            padding=\"longest\",\n            max_length=tokenizer.model_max_length,\n            truncation=True,\n        ).input_ids\n\n    targets = input_ids.clone()\n\n    assert conv.sep_style == conversation_lib.SeparatorStyle.TWO\n\n    # Mask targets\n    sep = conv.sep + conv.roles[1] + \": \"\n    for conversation, target in zip(conversations, targets):\n        total_len = int(target.ne(tokenizer.pad_token_id).sum())\n\n        rounds = conversation.split(conv.sep2)\n        cur_len = 1\n        target[:cur_len] = IGNORE_INDEX\n        for i, rou in enumerate(rounds):\n            if rou == \"\":\n                break\n\n            parts = rou.split(sep)\n            if len(parts) != 2:\n                break\n            parts[0] += sep\n\n            if has_image:\n                round_len = len(tokenizer_image_token(rou, tokenizer))\n                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 2\n            else:\n                round_len = len(tokenizer(rou).input_ids)\n                instruction_len = len(tokenizer(parts[0]).input_ids) - 2\n\n            if i != 0 and not tokenizer.legacy and IS_TOKENIZER_GREATER_THAN_0_14:\n                round_len -= 1\n                instruction_len -= 1\n\n            target[cur_len: cur_len + instruction_len] = IGNORE_INDEX\n\n            cur_len += round_len\n        target[cur_len:] = IGNORE_INDEX\n\n        if cur_len < tokenizer.model_max_length:\n            if cur_len != total_len:\n                target[:] = IGNORE_INDEX\n                print(\n                    f\"WARNING: tokenization mismatch: {cur_len} vs. {total_len}.\"\n                    f\" (ignored)\"\n                )\n\n    return dict(\n        input_ids=input_ids,\n        labels=targets,\n    )\n\n\ndef preprocess_mpt(\n    sources,\n    tokenizer: transformers.PreTrainedTokenizer,\n    has_image: bool = False\n) -> Dict:\n    conv = conversation_lib.default_conversation.copy()\n    roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n\n    # Apply prompt templates\n    conversations = []\n    for i, source in enumerate(sources):\n        if roles[source[0][\"from\"]] != conv.roles[0]:\n            # Skip the first one if it is not from human\n            source = source[1:]\n\n        conv.messages = []\n        for j, sentence in enumerate(source):\n            role = roles[sentence[\"from\"]]\n            assert role == conv.roles[j % 2], f\"{i}\"\n            conv.append_message(role, sentence[\"value\"])\n        conversations.append(conv.get_prompt())\n\n    # Tokenize conversations\n\n    if has_image:\n        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)\n    else:\n        input_ids = tokenizer(\n            conversations,\n            return_tensors=\"pt\",\n            padding=\"longest\",\n            max_length=tokenizer.model_max_length,\n            truncation=True,\n        ).input_ids\n\n    targets = input_ids.clone()\n    assert conv.sep_style == conversation_lib.SeparatorStyle.MPT\n\n    # Mask targets\n    sep = conv.sep + conv.roles[1]\n    for conversation, target in zip(conversations, targets):\n        total_len = int(target.ne(tokenizer.pad_token_id).sum())\n\n        rounds = conversation.split(conv.sep)\n        re_rounds = [conv.sep.join(rounds[:3])]  # system + user + gpt\n        for conv_idx in range(3, len(rounds), 2):\n            re_rounds.append(conv.sep.join(rounds[conv_idx:conv_idx+2]))    # user + gpt\n        cur_len = 0\n        target[:cur_len] = IGNORE_INDEX\n        for i, rou in enumerate(re_rounds):\n            if rou == \"\":\n                break\n\n            parts = rou.split(sep)\n            if len(parts) != 2:\n                break\n            parts[0] += sep\n\n            if has_image:\n                round_len = len(tokenizer_image_token(rou, tokenizer))\n                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 1\n            else:\n                round_len = len(tokenizer(rou).input_ids)\n                instruction_len = len(tokenizer(parts[0]).input_ids) - 1\n\n            if i != 0 and getattr(tokenizer, 'legacy', False) and IS_TOKENIZER_GREATER_THAN_0_14:\n                round_len += 1\n                instruction_len += 1\n\n            target[cur_len: cur_len + instruction_len] = IGNORE_INDEX\n\n            cur_len += round_len\n        target[cur_len:] = IGNORE_INDEX\n\n        if cur_len < tokenizer.model_max_length:\n            if cur_len != total_len:\n                target[:] = IGNORE_INDEX\n                print(\n                    f\"WARNING: tokenization mismatch: {cur_len} vs. {total_len}.\"\n                    f\" (ignored)\"\n                )\n\n    return dict(\n        input_ids=input_ids,\n        labels=targets,\n    )\n\n\ndef preprocess_plain(\n    sources: Sequence[str],\n    tokenizer: transformers.PreTrainedTokenizer,\n) -> Dict:\n    # add end signal and concatenate together\n    conversations = []\n    for source in sources:\n        assert len(source) == 2\n        assert DEFAULT_IMAGE_TOKEN in source[0]['value']\n        source[0]['value'] = DEFAULT_IMAGE_TOKEN\n        conversation = source[0]['value'] + source[1]['value'] + conversation_lib.default_conversation.sep\n        conversations.append(conversation)\n    # tokenize conversations\n    input_ids = [tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations]\n    targets = copy.deepcopy(input_ids)\n    for target, source in zip(targets, sources):\n        tokenized_len = len(tokenizer_image_token(source[0]['value'], tokenizer))\n        target[:tokenized_len] = IGNORE_INDEX\n\n    return dict(input_ids=input_ids, labels=targets)\n\n\ndef preprocess(\n    sources: Sequence[str],\n    tokenizer: transformers.PreTrainedTokenizer,\n    has_image: bool = False\n) -> Dict:\n    \"\"\"\n    Given a list of sources, each is a conversation list. This transform:\n    1. Add signal '### ' at the beginning each sentence, with end signal '\\n';\n    2. Concatenate conversations together;\n    3. Tokenize the concatenated conversation;\n    4. Make a deepcopy as the target. Mask human words with IGNORE_INDEX.\n    \"\"\"\n    if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.PLAIN:\n        return preprocess_plain(sources, tokenizer)\n    if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.LLAMA_2:\n        return preprocess_llama_2(sources, tokenizer, has_image=has_image)\n    if conversation_lib.default_conversation.version.startswith(\"v1\"):\n        return preprocess_v1(sources, tokenizer, has_image=has_image)\n    if conversation_lib.default_conversation.version == \"mpt\":\n        return preprocess_mpt(sources, tokenizer, has_image=has_image)\n    # fix: add qwen2\n    if conversation_lib.default_conversation.version.startswith(\"qwen_v2\"):\n        return preprocess_qwen_2(sources, tokenizer, has_image=has_image)\n    # add end signal and concatenate together\n    conversations = []\n    for source in sources:\n        header = f\"{conversation_lib.default_conversation.system}\\n\\n\"\n        conversation = _add_speaker_and_signal(header, source)\n        conversations.append(conversation)\n    # tokenize conversations\n\n    def get_tokenize_len(prompts):\n        return [len(tokenizer_image_token(prompt, tokenizer)) for prompt in prompts]\n\n    if has_image:\n        input_ids = [tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations]\n    else:\n        conversations_tokenized = _tokenize_fn(conversations, tokenizer)\n        input_ids = conversations_tokenized[\"input_ids\"]\n\n    targets = copy.deepcopy(input_ids)\n    for target, source in zip(targets, sources):\n        if has_image:\n            tokenized_lens = get_tokenize_len([header] + [s[\"value\"] for s in source])\n        else:\n            tokenized_lens = _tokenize_fn([header] + [s[\"value\"] for s in source], tokenizer)[\"input_ids_lens\"]\n        speakers = [sentence[\"from\"] for sentence in source]\n        _mask_targets(target, tokenized_lens, speakers)\n\n    return dict(input_ids=input_ids, labels=targets)\n\n\nclass LazySupervisedDataset(Dataset):\n    \"\"\"Dataset for supervised fine-tuning.\"\"\"\n\n    def __init__(self, data_path: List[str],\n                 tokenizer: transformers.PreTrainedTokenizer,\n                 data_args: DataArguments):\n        super(LazySupervisedDataset, self).__init__()\n        #list_data_dict = json.load(open(data_path, \"r\"))\n        list_data_dict = []\n        for i, _data_path in enumerate(data_path):\n            data = json.load(open(_data_path, \"r\"))\n            data = [{**entry, 'img_path_idx': i} for entry in data]\n            list_data_dict += data\n        rank0_print(\"Formatting inputs...Skip in lazy mode\")\n        self.tokenizer = tokenizer\n        self.list_data_dict = list_data_dict\n        self.data_args = data_args\n\n    def __len__(self):\n        return len(self.list_data_dict)\n\n    @property\n    def lengths(self):\n        length_list = []\n        for sample in self.list_data_dict:\n            img_tokens = 128 if 'image' in sample else 0\n            length_list.append(sum(len(conv['value'].split()) for conv in sample['conversations']) + img_tokens)\n        return length_list\n\n    @property\n    def modality_lengths(self):\n        length_list = []\n        for sample in self.list_data_dict:\n            cur_len = sum(len(conv['value'].split()) for conv in sample['conversations'])\n            cur_len = cur_len if 'image' in sample else -cur_len\n            length_list.append(cur_len)\n        return length_list\n\n    def __getitem__(self, i) -> Dict[str, torch.Tensor]:\n        sources = self.list_data_dict[i]\n        if isinstance(i, int):\n            sources = [sources]\n        assert len(sources) == 1, \"Don't know why it is wrapped to a list\"  # FIXME\n        if 'image' in sources[0]:\n            image_file = self.list_data_dict[i]['image']\n            img_path_idx = self.list_data_dict[i]['img_path_idx']\n            image_folder = self.data_args.image_folder[img_path_idx]\n            processor = self.data_args.image_processor\n            image = Image.open(os.path.join(image_folder, image_file)).convert('RGB')\n            image_size = image.size\n            if self.data_args.image_aspect_ratio == 'pad':\n                def expand2square(pil_img, background_color):\n                    width, height = pil_img.size\n                    if width == height:\n                        return pil_img\n                    elif width > height:\n                        result = Image.new(pil_img.mode, (width, width), background_color)\n                        result.paste(pil_img, (0, (width - height) // 2))\n                        return result\n                    else:\n                        result = Image.new(pil_img.mode, (height, height), background_color)\n                        result.paste(pil_img, ((height - width) // 2, 0))\n                        return result\n                image = expand2square(image, tuple(int(x*255) for x in processor.image_mean))\n                image = processor.preprocess(image, return_tensors='pt')['pixel_values'][0]\n            elif self.data_args.image_aspect_ratio == \"anyres\" or \"anyres_max\" in self.data_args.image_aspect_ratio:\n                image = process_anyres_image(image, self.data_args.image_processor, self.data_args.image_grid_pinpoints)\n            else:\n                image = processor.preprocess(image, return_tensors='pt')['pixel_values'][0]\n\n            sources = preprocess_multimodal(\n                copy.deepcopy([e[\"conversations\"] for e in sources]),\n                self.data_args)\n        else:\n            sources = copy.deepcopy([e[\"conversations\"] for e in sources])\n        data_dict = preprocess(\n            sources,\n            self.tokenizer,\n            has_image=('image' in self.list_data_dict[i]))\n        if isinstance(i, int):\n            data_dict = dict(input_ids=data_dict[\"input_ids\"][0],\n                             labels=data_dict[\"labels\"][0])\n\n        # image exist in the data\n        if 'image' in self.list_data_dict[i]:\n            data_dict['image'] = image\n            data_dict['image_size'] = image_size\n        elif self.data_args.is_multimodal:\n            # image does not exist in the data, but the model is multimodal\n            crop_size = self.data_args.image_processor.crop_size\n            data_dict['image'] = torch.zeros(3, crop_size['height'], crop_size['width'])\n            data_dict['image_size'] = (crop_size['height'], crop_size['width'])\n        return data_dict\n\n\n@dataclass\nclass DataCollatorForSupervisedDataset(object):\n    \"\"\"Collate examples for supervised fine-tuning.\"\"\"\n\n    tokenizer: transformers.PreTrainedTokenizer\n\n    def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:\n        input_ids, labels = tuple([instance[key] for instance in instances]\n                                  for key in (\"input_ids\", \"labels\"))\n        input_ids = torch.nn.utils.rnn.pad_sequence(\n            input_ids,\n            batch_first=True,\n            padding_value=self.tokenizer.pad_token_id)\n        labels = torch.nn.utils.rnn.pad_sequence(labels,\n                                                 batch_first=True,\n                                                 padding_value=IGNORE_INDEX)\n        input_ids = input_ids[:, :self.tokenizer.model_max_length]\n        labels = labels[:, :self.tokenizer.model_max_length]\n        batch = dict(\n            input_ids=input_ids,\n            labels=labels,\n            attention_mask=input_ids.ne(self.tokenizer.pad_token_id),\n        )\n\n        if 'image' in instances[0]:\n            images = [instance['image'] for instance in instances]\n            batch['image_sizes'] = [instance['image_size'] for instance in instances]\n            if all(x is not None and x.shape == images[0].shape for x in images):\n                batch['images'] = torch.stack(images)\n            else:\n                batch['images'] = images\n\n        return batch\n\n\ndef make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer,\n                                data_args) -> Dict:\n    \"\"\"Make dataset and collator for supervised fine-tuning.\"\"\"\n    train_dataset = LazySupervisedDataset(tokenizer=tokenizer,\n                                          data_path=data_args.data_path,\n                                          data_args=data_args)\n    data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)\n    return dict(train_dataset=train_dataset,\n                eval_dataset=None,\n                data_collator=data_collator)\n\n\ndef train(attn_implementation=None):\n    global local_rank\n\n    parser = transformers.HfArgumentParser(\n        (ModelArguments, DataArguments, TrainingArguments))\n    model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n    local_rank = training_args.local_rank\n    compute_dtype = (torch.float16 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32))\n\n    bnb_model_from_pretrained_args = {}\n    if training_args.bits in [4, 8]:\n        from transformers import BitsAndBytesConfig\n        bnb_model_from_pretrained_args.update(dict(\n            device_map={\"\": training_args.device},\n            load_in_4bit=training_args.bits == 4,\n            load_in_8bit=training_args.bits == 8,\n            quantization_config=BitsAndBytesConfig(\n                load_in_4bit=training_args.bits == 4,\n                load_in_8bit=training_args.bits == 8,\n                llm_int8_skip_modules=[\"mm_projector\"],\n                llm_int8_threshold=6.0,\n                llm_int8_has_fp16_weight=False,\n                bnb_4bit_compute_dtype=compute_dtype,\n                bnb_4bit_use_double_quant=training_args.double_quant,\n                bnb_4bit_quant_type=training_args.quant_type  # {'fp4', 'nf4'}\n            )\n        ))\n\n    if model_args.vision_tower is not None:\n        if 'mpt' in model_args.model_name_or_path:\n            config = transformers.AutoConfig.from_pretrained(model_args.model_name_or_path, trust_remote_code=True)\n            config.attn_config['attn_impl'] = training_args.mpt_attn_impl\n            model = LlavaMptForCausalLM.from_pretrained(\n                model_args.model_name_or_path,\n                config=config,\n                cache_dir=training_args.cache_dir,\n                **bnb_model_from_pretrained_args\n            )\n        elif 'dclm' in model_args.model_name_or_path.lower():\n            config = transformers.AutoConfig.from_pretrained(model_args.model_name_or_path, trust_remote_code=True)\n            config.attn_config['attn_impl'] = \"eager\"\n            model = LlavaOpenlmForCausalLM.from_pretrained(\n                model_args.model_name_or_path,\n                cache_dir=training_args.cache_dir,\n                attn_implementation=attn_implementation,\n                torch_dtype=(torch.bfloat16 if training_args.bf16 else None),\n                **bnb_model_from_pretrained_args\n            )\n        else:\n            model = LlavaLlamaForCausalLM.from_pretrained(\n                model_args.model_name_or_path,\n                cache_dir=training_args.cache_dir,\n                attn_implementation=attn_implementation,\n                torch_dtype=(torch.bfloat16 if training_args.bf16 else None),\n                **bnb_model_from_pretrained_args\n            )\n    else:\n        model = transformers.LlamaForCausalLM.from_pretrained(\n            model_args.model_name_or_path,\n            cache_dir=training_args.cache_dir,\n            attn_implementation=attn_implementation,\n            torch_dtype=(torch.bfloat16 if training_args.bf16 else None),\n            **bnb_model_from_pretrained_args\n        )\n    model.config.use_cache = False\n\n    if model_args.freeze_backbone:\n        model.model.requires_grad_(False)\n\n    if training_args.bits in [4, 8]:\n        from peft import prepare_model_for_kbit_training\n        model.config.torch_dtype = (torch.float32 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32))\n        model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=training_args.gradient_checkpointing)\n\n    if training_args.gradient_checkpointing:\n        if hasattr(model, \"enable_input_require_grads\"):\n            model.enable_input_require_grads()\n        else:\n            def make_inputs_require_grad(module, input, output):\n                output.requires_grad_(True)\n            model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)\n\n    if training_args.lora_enable:\n        from peft import LoraConfig, get_peft_model\n        lora_config = LoraConfig(\n            r=training_args.lora_r,\n            lora_alpha=training_args.lora_alpha,\n            target_modules=find_all_linear_names(model),\n            lora_dropout=training_args.lora_dropout,\n            bias=training_args.lora_bias,\n            task_type=\"CAUSAL_LM\",\n        )\n        if training_args.bits == 16:\n            if training_args.bf16:\n                model.to(torch.bfloat16)\n            if training_args.fp16:\n                model.to(torch.float16)\n        rank0_print(\"Adding LoRA adapters...\")\n        model = get_peft_model(model, lora_config)\n\n    if 'mpt' in model_args.model_name_or_path:\n        tokenizer = transformers.AutoTokenizer.from_pretrained(\n            model_args.model_name_or_path,\n            cache_dir=training_args.cache_dir,\n            model_max_length=training_args.model_max_length,\n            padding_side=\"right\"\n        )\n    else:\n        tokenizer = transformers.AutoTokenizer.from_pretrained(\n            model_args.model_name_or_path,\n            cache_dir=training_args.cache_dir,\n            model_max_length=training_args.model_max_length,\n            padding_side=\"right\",\n            use_fast=False,\n        )\n\n    if model_args.version == \"v0\":\n        if tokenizer.pad_token is None:\n            smart_tokenizer_and_embedding_resize(\n                special_tokens_dict=dict(pad_token=\"[PAD]\"),\n                tokenizer=tokenizer,\n                model=model,\n            )\n    elif model_args.version == \"v0.5\":\n        tokenizer.pad_token = tokenizer.unk_token\n    else:\n        tokenizer.pad_token = tokenizer.unk_token\n        if model_args.version in conversation_lib.conv_templates:\n            conversation_lib.default_conversation = conversation_lib.conv_templates[model_args.version]\n        else:\n            conversation_lib.default_conversation = conversation_lib.conv_templates[\"vicuna_v1\"]\n\n    if model_args.vision_tower is not None:\n        model.get_model().initialize_vision_modules(\n            model_args=model_args,\n            fsdp=training_args.fsdp\n        )\n\n        vision_tower = model.get_vision_tower()\n        vision_tower.to(dtype=torch.bfloat16 if training_args.bf16 else torch.float16, device=training_args.device)\n\n        data_args.image_processor = vision_tower.image_processor\n        data_args.is_multimodal = True\n\n        model.config.image_grid_pinpoints = data_args.image_grid_pinpoints\n        model.config.image_aspect_ratio = data_args.image_aspect_ratio\n        model.config.tokenizer_padding_side = tokenizer.padding_side\n        model.config.tokenizer_model_max_length = tokenizer.model_max_length\n\n        model.config.tune_mm_mlp_adapter = training_args.tune_mm_mlp_adapter = model_args.tune_mm_mlp_adapter\n        if model_args.tune_mm_mlp_adapter:\n            model.requires_grad_(False)\n            for p in model.get_model().mm_projector.parameters():\n                p.requires_grad = True\n\n        if model_args.tune_mm_mlp_and_vision_tower:\n            model.requires_grad_(False)\n            for p in model.get_model().mm_projector.parameters():\n                p.requires_grad = True\n            for p in model.get_vision_tower().parameters():\n                p.requires_grad = True\n\n        model.config.freeze_mm_mlp_adapter = training_args.freeze_mm_mlp_adapter\n        if training_args.freeze_mm_mlp_adapter:\n            for p in model.get_model().mm_projector.parameters():\n                p.requires_grad = False\n\n        if training_args.bits in [4, 8]:\n            model.get_model().mm_projector.to(dtype=compute_dtype, device=training_args.device)\n\n        model.config.mm_use_im_start_end = data_args.mm_use_im_start_end = model_args.mm_use_im_start_end\n        model.config.mm_projector_lr = training_args.mm_projector_lr\n        training_args.use_im_start_end = model_args.mm_use_im_start_end\n        model.config.mm_use_im_patch_token = model_args.mm_use_im_patch_token\n        model.initialize_vision_tokenizer(model_args, tokenizer=tokenizer)\n\n    if training_args.bits in [4, 8]:\n        from peft.tuners.lora import LoraLayer\n        for name, module in model.named_modules():\n            if isinstance(module, LoraLayer):\n                if training_args.bf16:\n                    module = module.to(torch.bfloat16)\n            if 'norm' in name:\n                module = module.to(torch.float32)\n            if 'lm_head' in name or 'embed_tokens' in name:\n                if hasattr(module, 'weight'):\n                    if training_args.bf16 and module.weight.dtype == torch.float32:\n                        module = module.to(torch.bfloat16)\n\n    data_module = make_supervised_data_module(tokenizer=tokenizer,\n                                              data_args=data_args)\n    trainer = LLaVATrainer(model=model,\n                           tokenizer=tokenizer,\n                           args=training_args,\n                           **data_module)\n\n    if list(pathlib.Path(training_args.output_dir).glob(\"checkpoint-*\")):\n        trainer.train(resume_from_checkpoint=True)\n    else:\n        trainer.train()\n    trainer.save_state()\n\n    model.config.use_cache = True\n\n    if training_args.lora_enable:\n        state_dict = get_peft_state_maybe_zero_3(\n            model.named_parameters(), training_args.lora_bias\n        )\n        non_lora_state_dict = get_peft_state_non_lora_maybe_zero_3(\n            model.named_parameters()\n        )\n        if training_args.local_rank == 0 or training_args.local_rank == -1:\n            model.config.save_pretrained(training_args.output_dir)\n            model.save_pretrained(training_args.output_dir, state_dict=state_dict)\n            torch.save(non_lora_state_dict, os.path.join(training_args.output_dir, 'non_lora_trainables.bin'))\n    else:\n        safe_save_model_for_hf_trainer(trainer=trainer,\n                                       output_dir=training_args.output_dir)\n\n\nif __name__ == \"__main__\":\n    train()\n"
  },
  {
    "path": "llava/train/train_mem.py",
    "content": "from llava.train.train_qwen import train\n\nif __name__ == \"__main__\":\n    train(attn_implementation=\"flash_attention_2\")\n"
  },
  {
    "path": "llava/train/train_qwen.py",
    "content": "# Adopted from https://github.com/lm-sys/FastChat. Below is the original copyright:\n# Adopted from tatsu-lab@stanford_alpaca. Below is the original copyright:\n#    Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n\nfrom packaging import version\nimport os\nimport copy\nfrom dataclasses import dataclass, field\nimport json\nimport logging\nimport pathlib\nfrom typing import Dict, Optional, Sequence, List\n\nimport torch\n\nimport transformers\nimport tokenizers\n\nfrom llava.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN\nfrom torch.utils.data import Dataset\nfrom llava.train.llava_trainer import LLaVATrainer\n\nfrom llava import conversation as conversation_lib\nfrom llava.model import *\nfrom llava.mm_utils import tokenizer_image_token, process_anyres_image\n\nfrom PIL import Image\n\n\nlocal_rank = None\n\n\ndef rank0_print(*args):\n    if local_rank == 0:\n        print(*args)\n\n\nIS_TOKENIZER_GREATER_THAN_0_14 = version.parse(tokenizers.__version__) >= version.parse('0.14')\n\n\n@dataclass\nclass ModelArguments:\n    model_name_or_path: Optional[str] = field(default=\"facebook/opt-125m\")\n    version: Optional[str] = field(default=\"v0\")\n    freeze_backbone: bool = field(default=False)\n    tune_mm_mlp_adapter: bool = field(default=False)\n    tune_mm_mlp_and_vision_tower: bool = field(default=False)\n    vision_tower: Optional[str] = field(default=None)\n    mm_vision_select_layer: Optional[int] = field(default=-1)   # default to the last layer\n    pretrain_mm_mlp_adapter: Optional[str] = field(default=None)\n    mm_projector_type: Optional[str] = field(default='linear')\n    mm_use_im_start_end: bool = field(default=False)\n    mm_use_im_patch_token: bool = field(default=True)\n    mm_patch_merge_type: Optional[str] = field(default='flat')\n    mm_vision_select_feature: Optional[str] = field(default=\"patch\")\n    unfreeze_mm_vision_tower: bool = field(default=False)\n    s2: Optional[bool] = field(default=False)\n    hd: Optional[bool] = field(default=False)\n\n\n@dataclass\nclass DataArguments:\n    data_path: Optional[List[str]] = field(default=None,\n                           metadata={\"help\": \"Optional list of paths to the training data.\"})\n    lazy_preprocess: bool = False\n    is_multimodal: bool = False\n    image_folder: Optional[List[str]] = field(default=None)\n    image_aspect_ratio: str = 'square'\n    image_grid_pinpoints: Optional[str] = field(default=None)\n    image_crop_resolution: Optional[int] = field(default=None)\n    image_split_resolution: Optional[int] = field(default=None)\n\n\n@dataclass\nclass TrainingArguments(transformers.TrainingArguments):\n    cache_dir: Optional[str] = field(default=None)\n    optim: str = field(default=\"adamw_torch\")\n    remove_unused_columns: bool = field(default=False)\n    freeze_mm_mlp_adapter: bool = field(default=False)\n    mpt_attn_impl: Optional[str] = field(default=\"triton\")\n    model_max_length: int = field(\n        default=512,\n        metadata={\n            \"help\":\n            \"Maximum sequence length. Sequences will be right padded (and possibly truncated).\"\n        },\n    )\n    double_quant: bool = field(\n        default=True,\n        metadata={\"help\": \"Compress the quantization statistics through double quantization.\"}\n    )\n    quant_type: str = field(\n        default=\"nf4\",\n        metadata={\"help\": \"Quantization data type to use. Should be one of `fp4` or `nf4`.\"}\n    )\n    bits: int = field(\n        default=16,\n        metadata={\"help\": \"How many bits to use.\"}\n    )\n    lora_enable: bool = False\n    lora_r: int = 64\n    lora_alpha: int = 16\n    lora_dropout: float = 0.05\n    lora_weight_path: str = \"\"\n    lora_bias: str = \"none\"\n    mm_projector_lr: Optional[float] = None\n    group_by_modality_length: bool = field(default=False)\n    mm_vision_tower_lr: Optional[float] = None\n\n\ndef maybe_zero_3(param, ignore_status=False, name=None):\n    from deepspeed import zero\n    from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus\n    if hasattr(param, \"ds_id\"):\n        if param.ds_status == ZeroParamStatus.NOT_AVAILABLE:\n            if not ignore_status:\n                logging.warning(f\"{name}: param.ds_status != ZeroParamStatus.NOT_AVAILABLE: {param.ds_status}\")\n        with zero.GatheredParameters([param]):\n            param = param.data.detach().cpu().clone()\n    else:\n        param = param.detach().cpu().clone()\n    return param\n\n\n# Borrowed from peft.utils.get_peft_model_state_dict\ndef get_peft_state_maybe_zero_3(named_params, bias):\n    if bias == \"none\":\n        to_return = {k: t for k, t in named_params if \"lora_\" in k}\n    elif bias == \"all\":\n        to_return = {k: t for k, t in named_params if \"lora_\" in k or \"bias\" in k}\n    elif bias == \"lora_only\":\n        to_return = {}\n        maybe_lora_bias = {}\n        lora_bias_names = set()\n        for k, t in named_params:\n            if \"lora_\" in k:\n                to_return[k] = t\n                bias_name = k.split(\"lora_\")[0] + \"bias\"\n                lora_bias_names.add(bias_name)\n            elif \"bias\" in k:\n                maybe_lora_bias[k] = t\n        for k, t in maybe_lora_bias:\n            if bias_name in lora_bias_names:\n                to_return[bias_name] = t\n    else:\n        raise NotImplementedError\n    to_return = {k: maybe_zero_3(v, ignore_status=True) for k, v in to_return.items()}\n    return to_return\n\n\ndef get_peft_state_non_lora_maybe_zero_3(named_params, require_grad_only=True):\n    to_return = {k: t for k, t in named_params if \"lora_\" not in k}\n    if require_grad_only:\n        to_return = {k: t for k, t in to_return.items() if t.requires_grad}\n    to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}\n    return to_return\n\n\ndef get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match):\n    to_return = {k: t for k, t in named_params if any(key_match in k for key_match in keys_to_match)}\n    to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}\n    return to_return\n\n\ndef find_all_linear_names(model):\n    cls = torch.nn.Linear\n    lora_module_names = set()\n    multimodal_keywords = ['mm_projector', 'vision_tower', 'vision_resampler']\n    for name, module in model.named_modules():\n        if any(mm_keyword in name for mm_keyword in multimodal_keywords):\n            continue\n        if isinstance(module, cls):\n            names = name.split('.')\n            lora_module_names.add(names[0] if len(names) == 1 else names[-1])\n\n    if 'lm_head' in lora_module_names:  # needed for 16-bit\n        lora_module_names.remove('lm_head')\n    return list(lora_module_names)\n\n\ndef safe_save_model_for_hf_trainer(trainer: transformers.Trainer,\n                                   output_dir: str):\n    \"\"\"Collects the state dict and dump to disk.\"\"\"\n\n    if getattr(trainer.args, \"tune_mm_mlp_adapter\", False):\n        # Only save Adapter\n        keys_to_match = ['mm_projector']\n        if getattr(trainer.args, \"use_im_start_end\", False):\n            keys_to_match.extend(['embed_tokens', 'embed_in'])\n\n        weight_to_save = get_mm_adapter_state_maybe_zero_3(trainer.model.named_parameters(), keys_to_match)\n        trainer.model.config.save_pretrained(output_dir)\n\n        current_folder = output_dir.split('/')[-1]\n        parent_folder = os.path.dirname(output_dir)\n        if trainer.args.local_rank == 0 or trainer.args.local_rank == -1:\n            if current_folder.startswith('checkpoint-'):\n                mm_projector_folder = os.path.join(parent_folder, \"mm_projector\")\n                os.makedirs(mm_projector_folder, exist_ok=True)\n                torch.save(weight_to_save, os.path.join(mm_projector_folder, f'{current_folder}.bin'))\n            else:\n                torch.save(weight_to_save, os.path.join(output_dir, f'mm_projector.bin'))\n        return\n\n    if trainer.deepspeed:\n        torch.cuda.synchronize()\n        trainer.save_model(output_dir)\n        return\n\n    state_dict = trainer.model.state_dict()\n    if trainer.args.should_save:\n        cpu_state_dict = {\n            key: value.cpu()\n            for key, value in state_dict.items()\n        }\n        del state_dict\n        trainer._save(output_dir, state_dict=cpu_state_dict)  # noqa\n\n\ndef smart_tokenizer_and_embedding_resize(\n    special_tokens_dict: Dict,\n    tokenizer: transformers.PreTrainedTokenizer,\n    model: transformers.PreTrainedModel,\n):\n    \"\"\"Resize tokenizer and embedding.\n\n    Note: This is the unoptimized version that may make your embedding size not be divisible by 64.\n    \"\"\"\n    num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)\n    model.resize_token_embeddings(len(tokenizer))\n\n    if num_new_tokens > 0:\n        input_embeddings = model.get_input_embeddings().weight.data\n        output_embeddings = model.get_output_embeddings().weight.data\n\n        input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(\n            dim=0, keepdim=True)\n        output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(\n            dim=0, keepdim=True)\n\n        input_embeddings[-num_new_tokens:] = input_embeddings_avg\n        output_embeddings[-num_new_tokens:] = output_embeddings_avg\n\n\ndef _tokenize_fn(strings: Sequence[str],\n                 tokenizer: transformers.PreTrainedTokenizer) -> Dict:\n    \"\"\"Tokenize a list of strings.\"\"\"\n    tokenized_list = [\n        tokenizer(\n            text,\n            return_tensors=\"pt\",\n            padding=\"longest\",\n            max_length=tokenizer.model_max_length,\n            truncation=True,\n        ) for text in strings\n    ]\n    input_ids = labels = [\n        tokenized.input_ids[0] for tokenized in tokenized_list\n    ]\n    input_ids_lens = labels_lens = [\n        tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item()\n        for tokenized in tokenized_list\n    ]\n    return dict(\n        input_ids=input_ids,\n        labels=labels,\n        input_ids_lens=input_ids_lens,\n        labels_lens=labels_lens,\n    )\n\n\ndef _mask_targets(target, tokenized_lens, speakers):\n    # cur_idx = 0\n    cur_idx = tokenized_lens[0]\n    tokenized_lens = tokenized_lens[1:]\n    target[:cur_idx] = IGNORE_INDEX\n    for tokenized_len, speaker in zip(tokenized_lens, speakers):\n        if speaker == \"human\":\n            target[cur_idx+2:cur_idx + tokenized_len] = IGNORE_INDEX\n        cur_idx += tokenized_len\n\n\ndef _add_speaker_and_signal(header, source, get_conversation=True):\n    \"\"\"Add speaker and start/end signal on each round.\"\"\"\n    BEGIN_SIGNAL = \"### \"\n    END_SIGNAL = \"\\n\"\n    conversation = header\n    for sentence in source:\n        from_str = sentence[\"from\"]\n        if from_str.lower() == \"human\":\n            from_str = conversation_lib.default_conversation.roles[0]\n        elif from_str.lower() == \"gpt\":\n            from_str = conversation_lib.default_conversation.roles[1]\n        else:\n            from_str = 'unknown'\n        sentence[\"value\"] = (BEGIN_SIGNAL + from_str + \": \" +\n                             sentence[\"value\"] + END_SIGNAL)\n        if get_conversation:\n            conversation += sentence[\"value\"]\n    conversation += BEGIN_SIGNAL\n    return conversation\n\n\ndef preprocess_multimodal(\n    sources: Sequence[str],\n    data_args: DataArguments\n) -> Dict:\n    is_multimodal = data_args.is_multimodal\n    if not is_multimodal:\n        return sources\n\n    for source in sources:\n        for sentence in source:\n            if DEFAULT_IMAGE_TOKEN in sentence['value']:\n                sentence['value'] = sentence['value'].replace(DEFAULT_IMAGE_TOKEN, '').strip()\n                sentence['value'] = DEFAULT_IMAGE_TOKEN + '\\n' + sentence['value']\n                sentence['value'] = sentence['value'].strip()\n                if \"mmtag\" in conversation_lib.default_conversation.version:\n                    sentence['value'] = sentence['value'].replace(DEFAULT_IMAGE_TOKEN, '<Image>' + DEFAULT_IMAGE_TOKEN + '</Image>')\n            replace_token = DEFAULT_IMAGE_TOKEN\n            if data_args.mm_use_im_start_end:\n                replace_token = DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN\n            sentence[\"value\"] = sentence[\"value\"].replace(DEFAULT_IMAGE_TOKEN, replace_token)\n\n    return sources\n\n\ndef preprocess_llama_2(\n    sources,\n    tokenizer: transformers.PreTrainedTokenizer,\n    has_image: bool = False\n) -> Dict:\n    conv = conversation_lib.default_conversation.copy()\n    roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n\n    # Apply prompt templates\n    conversations = []\n    for i, source in enumerate(sources):\n        if roles[source[0][\"from\"]] != conv.roles[0]:\n            # Skip the first one if it is not from human\n            source = source[1:]\n\n        conv.messages = []\n        for j, sentence in enumerate(source):\n            role = roles[sentence[\"from\"]]\n            assert role == conv.roles[j % 2], f\"{i}\"\n            conv.append_message(role, sentence[\"value\"])\n        conversations.append(conv.get_prompt())\n\n    # Tokenize conversations\n\n    if has_image:\n        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)\n    else:\n        input_ids = tokenizer(\n            conversations,\n            return_tensors=\"pt\",\n            padding=\"longest\",\n            max_length=tokenizer.model_max_length,\n            truncation=True,\n        ).input_ids\n\n    targets = input_ids.clone()\n\n    assert conv.sep_style == conversation_lib.SeparatorStyle.LLAMA_2\n\n    # Mask targets\n    sep = \"[/INST] \"\n    for conversation, target in zip(conversations, targets):\n        total_len = int(target.ne(tokenizer.pad_token_id).sum())\n\n        rounds = conversation.split(conv.sep2)\n        cur_len = 1\n        target[:cur_len] = IGNORE_INDEX\n        for i, rou in enumerate(rounds):\n            if rou == \"\":\n                break\n\n            parts = rou.split(sep)\n            if len(parts) != 2:\n                break\n            parts[0] += sep\n\n            if has_image:\n                round_len = len(tokenizer_image_token(rou, tokenizer))\n                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 2\n            else:\n                round_len = len(tokenizer(rou).input_ids)\n                instruction_len = len(tokenizer(parts[0]).input_ids) - 2\n\n            target[cur_len: cur_len + instruction_len] = IGNORE_INDEX\n\n            cur_len += round_len\n        target[cur_len:] = IGNORE_INDEX\n\n        if cur_len < tokenizer.model_max_length:\n            if cur_len != total_len:\n                target[:] = IGNORE_INDEX\n                print(\n                    f\"WARNING: tokenization mismatch: {cur_len} vs. {total_len}.\"\n                    f\" (ignored)\"\n                )\n\n    return dict(\n        input_ids=input_ids,\n        labels=targets,\n    )\n\n\n# fix: add qwen2\n# def preprocess_qwen_2(\n#     sources,\n#     tokenizer: transformers.PreTrainedTokenizer,\n#     has_image: bool = False\n# ) -> Dict:\n#     # print('-----preprocess_qwen_2-------')\n#     conv = conversation_lib.default_conversation.copy()\n#     roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n#\n#     # Apply prompt templates\n#     conversations = []\n#     for i, source in enumerate(sources):\n#         if roles[source[0][\"from\"]] != conv.roles[0]:\n#             # Skip the first one if it is not from human\n#             source = source[1:]\n#\n#         conv.messages = []\n#         for j, sentence in enumerate(source):\n#             role = roles[sentence[\"from\"]]\n#             assert role == conv.roles[j % 2], f\"{i}\"\n#             conv.append_message(role, sentence[\"value\"])\n#         conversations.append(conv.get_prompt())\n#\n#     # Tokenize conversations\n#\n#     if has_image:\n#         input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)\n#     else:\n#         input_ids = tokenizer(\n#             conversations,\n#             return_tensors=\"pt\",\n#             padding=\"longest\",\n#             max_length=tokenizer.model_max_length,\n#             truncation=True,\n#         ).input_ids\n#\n#     targets = input_ids.clone()\n#\n#     assert conv.sep_style == conversation_lib.SeparatorStyle.QWEN_2\n#\n#     rank0_print(50*'S')\n#     # Mask targets\n#     sep = conv.sep + conv.roles[1] + \": \"\n#     for conversation, target in zip(conversations, targets):\n#         total_len = int(target.ne(tokenizer.pad_token_id).sum())\n#         rank0_print(f\"target.shape={target.shape}\", f\"total_len={total_len}\")\n#\n#         rounds = conversation.split(conv.sep2)\n#         rounds_len = len(rounds)\n#         cur_len = 0\n#         # target[:cur_len] = IGNORE_INDEX\n#         for i, rou in enumerate(rounds):\n#             if rou == \"\":\n#                 break\n#\n#             parts = rou.split(sep)\n#             if len(parts) != 2:\n#                 break\n#             parts[0] += sep\n#\n#             if has_image:\n#                 round_ids = tokenizer_image_token(rou, tokenizer)\n#                 instruction_ids = tokenizer_image_token(parts[0], tokenizer)\n#                 equal_parts = [x == y for x, y in zip(round_ids, instruction_ids)]\n#\n#                 instruction_len = equal_parts.index(False) if False in equal_parts else len(equal_parts)\n#                 round_len = len(round_ids)\n#\n#             else:\n#                 round_ids = tokenizer(rou).input_ids\n#                 instruction_ids = tokenizer(parts[0]).input_ids\n#                 equal_parts = [x == y for x, y in zip(round_ids, instruction_ids)]\n#\n#                 instruction_len = equal_parts.index(False) if False in equal_parts else len(equal_parts)\n#                 round_len = len(round_ids)\n#\n#             if i != 0 and not tokenizer.legacy and IS_TOKENIZER_GREATER_THAN_0_14:\n#                 round_len += 1\n#                 instruction_len += 1\n#\n#             rank0_print(i, rou, f\"cur_len={cur_len}\", f\"round_len={round_len}\", f\"instruction_len={instruction_len}\", f\"cur_len + instruction_len={cur_len + instruction_len}\")\n#             target[cur_len: cur_len + instruction_len] = IGNORE_INDEX\n#\n#             cur_len += round_len\n#         rank0_print(\"Outside Loop\")\n#         rank0_print(cur_len, len(target))\n#         rank0_print(target)\n#         rank0_print(50 * 'E')\n#         exit(0)\n#         target[cur_len:] = IGNORE_INDEX\n#\n#         if cur_len < tokenizer.model_max_length:\n#             if cur_len != total_len + rounds_len - 2:\n#                 target[:] = IGNORE_INDEX\n#                 print(\n#                     f\"WARNING: tokenization mismatch: {cur_len} vs. {total_len}.\"\n#                     f\" (ignored)\"\n#                 )\n#\n#     return dict(\n#         input_ids=input_ids,\n#         labels=targets,\n#     )\n\ndef preprocess_qwen_2(\n    sources,\n    tokenizer: transformers.PreTrainedTokenizer,\n    has_image: bool = False\n) -> Dict:\n    conv = conversation_lib.default_conversation.copy()\n    roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n\n    # Apply prompt templates\n    conversations = []\n    for i, source in enumerate(sources):\n        if roles[source[0][\"from\"]] != conv.roles[0]:\n            # Skip the first one if it is not from human\n            source = source[1:]\n\n        conv.messages = []\n        for j, sentence in enumerate(source):\n            try:\n                role = roles[sentence[\"from\"]]\n            except KeyError as e:\n                print(\"e\")\n                print(\"skipping sentence due to unrecognized role: {}\".format(sentence[\"from\"]))\n                continue\n            assert role == conv.roles[j % 2], f\"{i}\"\n            conv.append_message(role, sentence[\"value\"])\n        conversations.append(conv.get_prompt())\n\n    if has_image:\n        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)\n    else:\n        input_ids = tokenizer(\n            conversations,\n            return_tensors=\"pt\",\n            padding=\"longest\",\n            max_length=tokenizer.model_max_length,\n            truncation=True,\n        ).input_ids\n\n    targets = input_ids.clone()\n\n    assert conv.sep_style == conversation_lib.SeparatorStyle.QWEN_2\n\n    split_sep = conv.sep + conv.roles[1]\n    for conversation, target in zip(conversations, targets):\n        total_len = int(target.ne(tokenizer.pad_token_id).sum())\n\n        rounds_before = conversation.split(\n            conv.sep)  # system->user->assistant->user->assistant->user->assistant->user->assistant->Empty\n        if rounds_before[0] == conv.system:\n            rounds_before[1] = conv.sep.join([rounds_before[0], rounds_before[1]])\n            rounds_before = rounds_before[1:]\n        # connect every pair:\n        rounds = []\n        for i in range(0, len(rounds_before), 2):\n            if i < len(rounds_before)-1:\n                rounds.append(conv.sep.join([rounds_before[i], rounds_before[i + 1]]))\n            else:\n                rounds.append(rounds_before[i])\n        cur_len = 0\n        target[:cur_len] = IGNORE_INDEX\n        for i, rou in enumerate(rounds):\n            if rou == \"\":\n                break\n\n            parts = rou.split(split_sep)\n            if len(parts) != 2:\n                break\n\n            assert parts[0].startswith(conv.roles[0]) or parts[0].startswith(conv.system)\n            parts[0] += split_sep\n\n            if has_image:\n                round_ids = tokenizer_image_token(rou, tokenizer)\n                instruction_ids = tokenizer_image_token(parts[0], tokenizer)\n                equal_parts = [x == y for x, y in zip(round_ids, instruction_ids)]\n\n                instruction_len = equal_parts.index(False) if False in equal_parts else len(equal_parts)\n                round_len = len(round_ids)\n\n            else:\n                round_ids = tokenizer(rou).input_ids\n                instruction_ids = tokenizer(parts[0]).input_ids\n                equal_parts = [x == y for x, y in zip(round_ids, instruction_ids)]\n\n                instruction_len = equal_parts.index(False) if False in equal_parts else len(equal_parts)\n                round_len = len(round_ids)\n\n            round_len += 2 # this is tom compensate for the sep2\n\n            assert rou == parts[0]+parts[1]\n\n            target[cur_len: cur_len + instruction_len] = IGNORE_INDEX\n\n            cur_len += round_len\n\n        target[cur_len:] = IGNORE_INDEX\n\n        if cur_len < tokenizer.model_max_length:\n            if cur_len != total_len:\n                target[:] = IGNORE_INDEX\n                print(\n                    f\"WARNING: tokenization mismatch: {cur_len} vs. {total_len}.\"\n                    f\" (ignored)\"\n                )\n\n    return dict(\n        input_ids=input_ids,\n        labels=targets,\n    )\n\ndef preprocess_v1(\n    sources,\n    tokenizer: transformers.PreTrainedTokenizer,\n    has_image: bool = False\n) -> Dict:\n    conv = conversation_lib.default_conversation.copy()\n    roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n\n    # Apply prompt templates\n    conversations = []\n    for i, source in enumerate(sources):\n        if roles[source[0][\"from\"]] != conv.roles[0]:\n            # Skip the first one if it is not from human\n            source = source[1:]\n\n        conv.messages = []\n        for j, sentence in enumerate(source):\n            role = roles[sentence[\"from\"]]\n            assert role == conv.roles[j % 2], f\"{i}\"\n            conv.append_message(role, sentence[\"value\"])\n        conversations.append(conv.get_prompt())\n\n    # Tokenize conversations\n\n    if has_image:\n        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)\n    else:\n        input_ids = tokenizer(\n            conversations,\n            return_tensors=\"pt\",\n            padding=\"longest\",\n            max_length=tokenizer.model_max_length,\n            truncation=True,\n        ).input_ids\n\n    targets = input_ids.clone()\n\n    assert conv.sep_style == conversation_lib.SeparatorStyle.TWO\n\n    # Mask targets\n    sep = conv.sep + conv.roles[1] + \": \"\n    for conversation, target in zip(conversations, targets):\n        total_len = int(target.ne(tokenizer.pad_token_id).sum())\n\n        rounds = conversation.split(conv.sep2)\n        cur_len = 1\n        target[:cur_len] = IGNORE_INDEX\n        for i, rou in enumerate(rounds):\n            if rou == \"\":\n                break\n\n            parts = rou.split(sep)\n            if len(parts) != 2:\n                break\n            parts[0] += sep\n\n            if has_image:\n                round_len = len(tokenizer_image_token(rou, tokenizer))\n                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 2\n            else:\n                round_len = len(tokenizer(rou).input_ids)\n                instruction_len = len(tokenizer(parts[0]).input_ids) - 2\n\n            if i != 0 and not tokenizer.legacy and IS_TOKENIZER_GREATER_THAN_0_14:\n                round_len -= 1\n                instruction_len -= 1\n\n            target[cur_len: cur_len + instruction_len] = IGNORE_INDEX\n\n            cur_len += round_len\n        target[cur_len:] = IGNORE_INDEX\n\n        if cur_len < tokenizer.model_max_length:\n            if cur_len != total_len:\n                target[:] = IGNORE_INDEX\n                print(\n                    f\"WARNING: tokenization mismatch: {cur_len} vs. {total_len}.\"\n                    f\" (ignored)\"\n                )\n\n    return dict(\n        input_ids=input_ids,\n        labels=targets,\n    )\n\n\ndef preprocess_mpt(\n    sources,\n    tokenizer: transformers.PreTrainedTokenizer,\n    has_image: bool = False\n) -> Dict:\n    conv = conversation_lib.default_conversation.copy()\n    roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n\n    # Apply prompt templates\n    conversations = []\n    for i, source in enumerate(sources):\n        if roles[source[0][\"from\"]] != conv.roles[0]:\n            # Skip the first one if it is not from human\n            source = source[1:]\n\n        conv.messages = []\n        for j, sentence in enumerate(source):\n            role = roles[sentence[\"from\"]]\n            assert role == conv.roles[j % 2], f\"{i}\"\n            conv.append_message(role, sentence[\"value\"])\n        conversations.append(conv.get_prompt())\n\n    # Tokenize conversations\n\n    if has_image:\n        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)\n    else:\n        input_ids = tokenizer(\n            conversations,\n            return_tensors=\"pt\",\n            padding=\"longest\",\n            max_length=tokenizer.model_max_length,\n            truncation=True,\n        ).input_ids\n\n    targets = input_ids.clone()\n    assert conv.sep_style == conversation_lib.SeparatorStyle.MPT\n\n    # Mask targets\n    sep = conv.sep + conv.roles[1]\n    for conversation, target in zip(conversations, targets):\n        total_len = int(target.ne(tokenizer.pad_token_id).sum())\n\n        rounds = conversation.split(conv.sep)\n        re_rounds = [conv.sep.join(rounds[:3])]  # system + user + gpt\n        for conv_idx in range(3, len(rounds), 2):\n            re_rounds.append(conv.sep.join(rounds[conv_idx:conv_idx+2]))    # user + gpt\n        cur_len = 0\n        target[:cur_len] = IGNORE_INDEX\n        for i, rou in enumerate(re_rounds):\n            if rou == \"\":\n                break\n\n            parts = rou.split(sep)\n            if len(parts) != 2:\n                break\n            parts[0] += sep\n\n            if has_image:\n                round_len = len(tokenizer_image_token(rou, tokenizer))\n                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 1\n            else:\n                round_len = len(tokenizer(rou).input_ids)\n                instruction_len = len(tokenizer(parts[0]).input_ids) - 1\n\n            if i != 0 and getattr(tokenizer, 'legacy', False) and IS_TOKENIZER_GREATER_THAN_0_14:\n                round_len += 1\n                instruction_len += 1\n\n            target[cur_len: cur_len + instruction_len] = IGNORE_INDEX\n\n            cur_len += round_len\n        target[cur_len:] = IGNORE_INDEX\n\n        if cur_len < tokenizer.model_max_length:\n            if cur_len != total_len:\n                target[:] = IGNORE_INDEX\n                print(\n                    f\"WARNING: tokenization mismatch: {cur_len} vs. {total_len}.\"\n                    f\" (ignored)\"\n                )\n\n    return dict(\n        input_ids=input_ids,\n        labels=targets,\n    )\n\n\ndef preprocess_plain(\n    sources: Sequence[str],\n    tokenizer: transformers.PreTrainedTokenizer,\n) -> Dict:\n    # add end signal and concatenate together\n    conversations = []\n    for source in sources:\n        assert len(source) == 2\n        assert DEFAULT_IMAGE_TOKEN in source[0]['value']\n        source[0]['value'] = DEFAULT_IMAGE_TOKEN\n        conversation = source[0]['value'] + source[1]['value'] + conversation_lib.default_conversation.sep\n        conversations.append(conversation)\n    # tokenize conversations\n    input_ids = [tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations]\n    targets = copy.deepcopy(input_ids)\n    for target, source in zip(targets, sources):\n        tokenized_len = len(tokenizer_image_token(source[0]['value'], tokenizer))\n        target[:tokenized_len] = IGNORE_INDEX\n\n    return dict(input_ids=input_ids, labels=targets)\n\n\ndef preprocess(\n    sources: Sequence[str],\n    tokenizer: transformers.PreTrainedTokenizer,\n    has_image: bool = False\n) -> Dict:\n    \"\"\"\n    Given a list of sources, each is a conversation list. This transform:\n    1. Add signal '### ' at the beginning each sentence, with end signal '\\n';\n    2. Concatenate conversations together;\n    3. Tokenize the concatenated conversation;\n    4. Make a deepcopy as the target. Mask human words with IGNORE_INDEX.\n    \"\"\"\n    # print(\"conversation:\",conversation_lib.default_conversation.version)\n    # conversation_lib.default_conversation.version == \"qwen_v2\"\n\n    if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.PLAIN:\n        return preprocess_plain(sources, tokenizer)\n    if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.LLAMA_2:\n        return preprocess_llama_2(sources, tokenizer, has_image=has_image)\n    if conversation_lib.default_conversation.version.startswith(\"v1\"):\n        # print('--v1--')\n        return preprocess_v1(sources, tokenizer, has_image=has_image)\n    if conversation_lib.default_conversation.version == \"mpt\":\n        # print('--mpt--')\n        return preprocess_mpt(sources, tokenizer, has_image=has_image)\n    # fix: add qwen2\n    if conversation_lib.default_conversation.version.startswith(\"qwen_v2\"):\n        # print('--qwen_v2--')\n        return preprocess_qwen_2(sources, tokenizer, has_image=has_image)\n    # add end signal and concatenate together\n    conversations = []\n    for source in sources:\n        header = f\"{conversation_lib.default_conversation.system}\\n\\n\"\n        conversation = _add_speaker_and_signal(header, source)\n        conversations.append(conversation)\n    # tokenize conversations\n\n    def get_tokenize_len(prompts):\n        return [len(tokenizer_image_token(prompt, tokenizer)) for prompt in prompts]\n\n    if has_image:\n        input_ids = [tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations]\n    else:\n        conversations_tokenized = _tokenize_fn(conversations, tokenizer)\n        input_ids = conversations_tokenized[\"input_ids\"]\n\n    targets = copy.deepcopy(input_ids)\n    for target, source in zip(targets, sources):\n        if has_image:\n            tokenized_lens = get_tokenize_len([header] + [s[\"value\"] for s in source])\n        else:\n            tokenized_lens = _tokenize_fn([header] + [s[\"value\"] for s in source], tokenizer)[\"input_ids_lens\"]\n        speakers = [sentence[\"from\"] for sentence in source]\n        _mask_targets(target, tokenized_lens, speakers)\n\n    return dict(input_ids=input_ids, labels=targets)\n\n\nclass LazySupervisedDataset(Dataset):\n    \"\"\"Dataset for supervised fine-tuning.\"\"\"\n\n    def __init__(self, data_path: List[str],\n                 tokenizer: transformers.PreTrainedTokenizer,\n                 data_args: DataArguments):\n        super(LazySupervisedDataset, self).__init__()\n        #list_data_dict = json.load(open(data_path, \"r\"))\n        list_data_dict = []\n        for i, _data_path in enumerate(data_path):\n            data = json.load(open(_data_path, \"r\"))\n            data = [{**entry, 'img_path_idx': i} for entry in data]\n            list_data_dict += data\n\n        self.tokenizer = tokenizer\n        self.list_data_dict = list_data_dict\n        self.data_args = data_args\n\n    def __len__(self):\n        return len(self.list_data_dict)\n\n    @property\n    def lengths(self):\n        length_list = []\n        for sample in self.list_data_dict:\n            img_tokens = 128 if 'image' in sample else 0\n            length_list.append(sum(len(conv['value'].split()) for conv in sample['conversations']) + img_tokens)\n        return length_list\n\n    @property\n    def modality_lengths(self):\n        length_list = []\n        for sample in self.list_data_dict:\n            cur_len = sum(len(conv['value'].split()) for conv in sample['conversations'])\n            cur_len = cur_len if 'image' in sample else -cur_len\n            length_list.append(cur_len)\n        return length_list\n\n    def get_sample(self, i) -> Dict[str, torch.Tensor]:\n        sources = self.list_data_dict[i]\n        if isinstance(i, int):\n            sources = [sources]\n        assert len(sources) == 1, \"Don't know why it is wrapped to a list\"  # FIXME\n        if 'image' in sources[0]:\n            image_file = self.list_data_dict[i]['image']\n            img_path_idx = self.list_data_dict[i]['img_path_idx']\n            image_folder = self.data_args.image_folder[img_path_idx]\n            processor = self.data_args.image_processor\n            image = Image.open(os.path.join(image_folder, image_file)).convert('RGB')\n            image_size = image.size\n            if self.data_args.image_aspect_ratio == 'pad':\n                def expand2square(pil_img, background_color):\n                    width, height = pil_img.size\n                    if width == height:\n                        return pil_img\n                    elif width > height:\n                        result = Image.new(pil_img.mode, (width, width), background_color)\n                        result.paste(pil_img, (0, (width - height) // 2))\n                        return result\n                    else:\n                        result = Image.new(pil_img.mode, (height, height), background_color)\n                        result.paste(pil_img, ((height - width) // 2, 0))\n                        return result\n\n                image = expand2square(image, tuple(int(x * 255) for x in processor.image_mean))\n                image = processor.preprocess(image, return_tensors='pt')['pixel_values'][0]\n            elif self.data_args.image_aspect_ratio == \"anyres\" or \"anyres_max\" in self.data_args.image_aspect_ratio:\n                image = process_anyres_image(image, self.data_args.image_processor, self.data_args.image_grid_pinpoints)\n            else:\n                image = processor.preprocess(image, return_tensors='pt')['pixel_values'][0]\n            sources = preprocess_multimodal(\n                copy.deepcopy([e[\"conversations\"] for e in sources]),\n                self.data_args)\n        else:\n            sources = copy.deepcopy([e[\"conversations\"] for e in sources])\n        data_dict = preprocess(\n            sources,\n            self.tokenizer,\n            has_image=('image' in self.list_data_dict[i]))\n        if isinstance(i, int):\n            data_dict = dict(input_ids=data_dict[\"input_ids\"][0],\n                             labels=data_dict[\"labels\"][0])\n\n        # image exist in the data\n        if 'image' in self.list_data_dict[i]:\n            data_dict['image'] = image\n            data_dict['image_size'] = image_size\n        elif self.data_args.is_multimodal:\n            # image does not exist in the data, but the model is multimodal\n            crop_size = self.data_args.image_processor.crop_size\n            data_dict['image'] = torch.zeros(3, crop_size['height'], crop_size['width'])\n            data_dict['image_size'] = (crop_size['height'], crop_size['width'])\n        return data_dict\n\n    def __getitem__(self, i) -> Dict[str, torch.Tensor]:\n        try:\n            return self.get_sample(i)\n        except Exception as e:\n            print(\"Error loading sample\")\n            print()\n            return self.get_sample(0)\n\n\n@dataclass\nclass DataCollatorForSupervisedDataset(object):\n    \"\"\"Collate examples for supervised fine-tuning.\"\"\"\n\n    tokenizer: transformers.PreTrainedTokenizer\n\n    def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:\n        input_ids, labels = tuple([instance[key] for instance in instances]\n                                  for key in (\"input_ids\", \"labels\"))\n        input_ids = torch.nn.utils.rnn.pad_sequence(\n            input_ids,\n            batch_first=True,\n            padding_value=self.tokenizer.pad_token_id)\n        labels = torch.nn.utils.rnn.pad_sequence(labels,\n                                                 batch_first=True,\n                                                 padding_value=IGNORE_INDEX)\n        input_ids = input_ids[:, :self.tokenizer.model_max_length]\n        labels = labels[:, :self.tokenizer.model_max_length]\n        batch = dict(\n            input_ids=input_ids,\n            labels=labels,\n            attention_mask=input_ids.ne(self.tokenizer.pad_token_id),\n        )\n\n        if 'image' in instances[0]:\n            images = [instance['image'] for instance in instances]\n            batch['image_sizes'] = [instance['image_size'] for instance in instances]\n            if all(x is not None and x.shape == images[0].shape for x in images):\n                batch['images'] = torch.stack(images)\n            else:\n                batch['images'] = images\n\n        return batch\n\n\ndef make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer,\n                                data_args) -> Dict:\n    \"\"\"Make dataset and collator for supervised fine-tuning.\"\"\"\n    train_dataset = LazySupervisedDataset(tokenizer=tokenizer,\n                                          data_path=data_args.data_path,\n                                          data_args=data_args)\n    data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)\n    return dict(train_dataset=train_dataset,\n                eval_dataset=None,\n                data_collator=data_collator)\n\n\ndef train(attn_implementation=None):\n    global local_rank\n\n    parser = transformers.HfArgumentParser(\n        (ModelArguments, DataArguments, TrainingArguments))\n    model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n    local_rank = training_args.local_rank\n    compute_dtype = (torch.float16 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32))\n\n    bnb_model_from_pretrained_args = {}\n    if training_args.bits in [4, 8]:\n        from transformers import BitsAndBytesConfig\n        bnb_model_from_pretrained_args.update(dict(\n            device_map={\"\": training_args.device},\n            load_in_4bit=training_args.bits == 4,\n            load_in_8bit=training_args.bits == 8,\n            quantization_config=BitsAndBytesConfig(\n                load_in_4bit=training_args.bits == 4,\n                load_in_8bit=training_args.bits == 8,\n                llm_int8_skip_modules=[\"mm_projector\"],\n                llm_int8_threshold=6.0,\n                llm_int8_has_fp16_weight=False,\n                bnb_4bit_compute_dtype=compute_dtype,\n                bnb_4bit_use_double_quant=training_args.double_quant,\n                bnb_4bit_quant_type=training_args.quant_type  # {'fp4', 'nf4'}\n            )\n        ))\n\n    if model_args.vision_tower is not None:\n        if 'mpt' in model_args.model_name_or_path:\n            config = transformers.AutoConfig.from_pretrained(model_args.model_name_or_path, trust_remote_code=True)\n            config.attn_config['attn_impl'] = training_args.mpt_attn_impl\n            model = LlavaMptForCausalLM.from_pretrained(\n                model_args.model_name_or_path,\n                config=config,\n                cache_dir=training_args.cache_dir,\n                **bnb_model_from_pretrained_args\n            )\n        else:\n            model = LlavaQwen2ForCausalLM.from_pretrained(\n                model_args.model_name_or_path,\n                cache_dir=training_args.cache_dir,\n                attn_implementation=attn_implementation,\n                torch_dtype=(torch.bfloat16 if training_args.bf16 else None),\n                **bnb_model_from_pretrained_args\n            )\n    else:\n        model = transformers.Qwen2ForCausalLM.from_pretrained(\n            model_args.model_name_or_path,\n            cache_dir=training_args.cache_dir,\n            attn_implementation=attn_implementation,\n            torch_dtype=(torch.bfloat16 if training_args.bf16 else None),\n            **bnb_model_from_pretrained_args\n        )\n    model.config.use_cache = False\n\n    if model_args.freeze_backbone:\n        model.model.requires_grad_(False)\n\n    if training_args.bits in [4, 8]:\n        from peft import prepare_model_for_kbit_training\n        model.config.torch_dtype = (torch.float32 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32))\n        model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=training_args.gradient_checkpointing)\n\n    if training_args.gradient_checkpointing:\n        if hasattr(model, \"enable_input_require_grads\"):\n            model.enable_input_require_grads()\n        else:\n            def make_inputs_require_grad(module, input, output):\n                output.requires_grad_(True)\n            model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)\n\n    if training_args.lora_enable:\n        from peft import LoraConfig, get_peft_model\n        lora_config = LoraConfig(\n            r=training_args.lora_r,\n            lora_alpha=training_args.lora_alpha,\n            target_modules=find_all_linear_names(model),\n            lora_dropout=training_args.lora_dropout,\n            bias=training_args.lora_bias,\n            task_type=\"CAUSAL_LM\",\n        )\n        if training_args.bits == 16:\n            if training_args.bf16:\n                model.to(torch.bfloat16)\n            if training_args.fp16:\n                model.to(torch.float16)\n        rank0_print(\"Adding LoRA adapters...\")\n        model = get_peft_model(model, lora_config)\n\n    if 'mpt' in model_args.model_name_or_path:\n        tokenizer = transformers.AutoTokenizer.from_pretrained(\n            model_args.model_name_or_path,\n            cache_dir=training_args.cache_dir,\n            model_max_length=training_args.model_max_length,\n            padding_side=\"right\"\n        )\n    else:\n        tokenizer = transformers.AutoTokenizer.from_pretrained(\n            model_args.model_name_or_path,\n            cache_dir=training_args.cache_dir,\n            model_max_length=training_args.model_max_length,\n            padding_side=\"right\",\n            use_fast=False,\n        )\n\n    if model_args.version == \"v0\":\n        if tokenizer.pad_token is None:\n            smart_tokenizer_and_embedding_resize(\n                special_tokens_dict=dict(pad_token=\"[PAD]\"),\n                tokenizer=tokenizer,\n                model=model,\n            )\n    elif model_args.version == \"v0.5\":\n        tokenizer.pad_token = tokenizer.unk_token\n    else:\n        if tokenizer.unk_token:\n            tokenizer.pad_token = tokenizer.unk_token\n        else:  # use qwen\n            tokenizer.legacy = False\n        if model_args.version in conversation_lib.conv_templates:\n            # print('version:', model_args.version)\n            conversation_lib.default_conversation = conversation_lib.conv_templates[model_args.version]\n        else:\n            conversation_lib.default_conversation = conversation_lib.conv_templates[\"vicuna_v1\"]\n\n    if model_args.vision_tower is not None:\n        model.get_model().initialize_vision_modules(\n            model_args=model_args,\n            fsdp=training_args.fsdp\n        )\n\n        vision_tower = model.get_vision_tower()\n        vision_tower.to(dtype=torch.bfloat16 if training_args.bf16 else torch.float16, device=training_args.device)\n\n        data_args.image_processor = vision_tower.image_processor\n        data_args.is_multimodal = True\n\n        model.config.image_grid_pinpoints = data_args.image_grid_pinpoints\n        model.config.image_aspect_ratio = data_args.image_aspect_ratio\n        model.config.tokenizer_padding_side = tokenizer.padding_side\n        model.config.tokenizer_model_max_length = tokenizer.model_max_length\n\n        model.config.tune_mm_mlp_adapter = training_args.tune_mm_mlp_adapter = model_args.tune_mm_mlp_adapter\n        if model_args.tune_mm_mlp_adapter:\n            model.requires_grad_(False)\n            for p in model.get_model().mm_projector.parameters():\n                p.requires_grad = True\n\n        model.config.freeze_mm_mlp_adapter = training_args.freeze_mm_mlp_adapter\n        if training_args.freeze_mm_mlp_adapter:\n            for p in model.get_model().mm_projector.parameters():\n                p.requires_grad = False\n\n        if training_args.bits in [4, 8]:\n            model.get_model().mm_projector.to(dtype=compute_dtype, device=training_args.device)\n\n        model.config.mm_use_im_start_end = data_args.mm_use_im_start_end = model_args.mm_use_im_start_end\n        model.config.mm_projector_lr = training_args.mm_projector_lr\n        training_args.use_im_start_end = model_args.mm_use_im_start_end\n        model.config.mm_use_im_patch_token = model_args.mm_use_im_patch_token\n        model.initialize_vision_tokenizer(model_args, tokenizer=tokenizer)\n\n    if training_args.bits in [4, 8]:\n        from peft.tuners.lora import LoraLayer\n        for name, module in model.named_modules():\n            if isinstance(module, LoraLayer):\n                if training_args.bf16:\n                    module = module.to(torch.bfloat16)\n            if 'norm' in name:\n                module = module.to(torch.float32)\n            if 'lm_head' in name or 'embed_tokens' in name:\n                if hasattr(module, 'weight'):\n                    if training_args.bf16 and module.weight.dtype == torch.float32:\n                        module = module.to(torch.bfloat16)\n\n    data_module = make_supervised_data_module(tokenizer=tokenizer,\n                                              data_args=data_args)\n    trainer = LLaVATrainer(model=model,\n                           tokenizer=tokenizer,\n                           args=training_args,\n                           **data_module)\n\n    if list(pathlib.Path(training_args.output_dir).glob(\"checkpoint-*\")):\n        trainer.train(resume_from_checkpoint=True)\n    else:\n        trainer.train()\n    trainer.save_state()\n\n    model.config.use_cache = True\n\n    if training_args.lora_enable:\n        state_dict = get_peft_state_maybe_zero_3(\n            model.named_parameters(), training_args.lora_bias\n        )\n        non_lora_state_dict = get_peft_state_non_lora_maybe_zero_3(\n            model.named_parameters()\n        )\n        if training_args.local_rank == 0 or training_args.local_rank == -1:\n            model.config.save_pretrained(training_args.output_dir)\n            model.save_pretrained(training_args.output_dir, state_dict=state_dict)\n            torch.save(non_lora_state_dict, os.path.join(training_args.output_dir, 'non_lora_trainables.bin'))\n    else:\n        safe_save_model_for_hf_trainer(trainer=trainer,\n                                       output_dir=training_args.output_dir)\n\n\nif __name__ == \"__main__\":\n    train()\n"
  },
  {
    "path": "llava/train/train_xformers.py",
    "content": "# Make it more memory efficient by monkey patching the LLaMA model with xformers attention.\n\n# Need to call this before importing transformers.\nfrom llava.train.train import train\nfrom llava.train.llama_xformers_attn_monkey_patch import (\n    replace_llama_attn_with_xformers_attn,\n)\n\nreplace_llama_attn_with_xformers_attn()\n\n\nif __name__ == \"__main__\":\n    train()\n"
  },
  {
    "path": "llava/utils.py",
    "content": "import datetime\nimport logging\nimport logging.handlers\nimport os\nimport sys\n\nimport requests\n\nfrom llava.constants import LOGDIR\n\nserver_error_msg = \"**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**\"\nmoderation_msg = \"YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES. PLEASE TRY AGAIN.\"\n\nhandler = None\n\n\ndef build_logger(logger_name, logger_filename):\n    global handler\n\n    formatter = logging.Formatter(\n        fmt=\"%(asctime)s | %(levelname)s | %(name)s | %(message)s\",\n        datefmt=\"%Y-%m-%d %H:%M:%S\",\n    )\n\n    # Set the format of root handlers\n    if not logging.getLogger().handlers:\n        logging.basicConfig(level=logging.INFO)\n    logging.getLogger().handlers[0].setFormatter(formatter)\n\n    # Redirect stdout and stderr to loggers\n    stdout_logger = logging.getLogger(\"stdout\")\n    stdout_logger.setLevel(logging.INFO)\n    sl = StreamToLogger(stdout_logger, logging.INFO)\n    sys.stdout = sl\n\n    stderr_logger = logging.getLogger(\"stderr\")\n    stderr_logger.setLevel(logging.ERROR)\n    sl = StreamToLogger(stderr_logger, logging.ERROR)\n    sys.stderr = sl\n\n    # Get logger\n    logger = logging.getLogger(logger_name)\n    logger.setLevel(logging.INFO)\n\n    # Add a file handler for all loggers\n    if handler is None:\n        os.makedirs(LOGDIR, exist_ok=True)\n        filename = os.path.join(LOGDIR, logger_filename)\n        handler = logging.handlers.TimedRotatingFileHandler(\n            filename, when='D', utc=True, encoding='UTF-8')\n        handler.setFormatter(formatter)\n\n        for name, item in logging.root.manager.loggerDict.items():\n            if isinstance(item, logging.Logger):\n                item.addHandler(handler)\n\n    return logger\n\n\nclass StreamToLogger(object):\n    \"\"\"\n    Fake file-like stream object that redirects writes to a logger instance.\n    \"\"\"\n\n    def __init__(self, logger, log_level=logging.INFO):\n        self.terminal = sys.stdout\n        self.logger = logger\n        self.log_level = log_level\n        self.linebuf = ''\n\n    def __getattr__(self, attr):\n        return getattr(self.terminal, attr)\n\n    def write(self, buf):\n        temp_linebuf = self.linebuf + buf\n        self.linebuf = ''\n        for line in temp_linebuf.splitlines(True):\n            # From the io.TextIOWrapper docs:\n            #   On output, if newline is None, any '\\n' characters written\n            #   are translated to the system default line separator.\n            # By default sys.stdout.write() expects '\\n' newlines and then\n            # translates them so this is still cross platform.\n            if line[-1] == '\\n':\n                self.logger.log(self.log_level, line.rstrip())\n            else:\n                self.linebuf += line\n\n    def flush(self):\n        if self.linebuf != '':\n            self.logger.log(self.log_level, self.linebuf.rstrip())\n        self.linebuf = ''\n\n\ndef disable_torch_init():\n    \"\"\"\n    Disable the redundant torch default initialization to accelerate model creation.\n    \"\"\"\n    import torch\n    setattr(torch.nn.Linear, \"reset_parameters\", lambda self: None)\n    setattr(torch.nn.LayerNorm, \"reset_parameters\", lambda self: None)\n\n\ndef violates_moderation(text):\n    \"\"\"\n    Check whether the text violates OpenAI moderation API.\n    \"\"\"\n    url = \"https://api.openai.com/v1/moderations\"\n    headers = {\"Content-Type\": \"application/json\",\n               \"Authorization\": \"Bearer \" + os.environ[\"OPENAI_API_KEY\"]}\n    text = text.replace(\"\\n\", \"\")\n    data = \"{\" + '\"input\": ' + f'\"{text}\"' + \"}\"\n    data = data.encode(\"utf-8\")\n    try:\n        ret = requests.post(url, headers=headers, data=data, timeout=5)\n        flagged = ret.json()[\"results\"][0][\"flagged\"]\n    except requests.exceptions.RequestException as e:\n        flagged = False\n    except KeyError as e:\n        flagged = False\n\n    return flagged\n\n\ndef pretty_print_semaphore(semaphore):\n    if semaphore is None:\n        return \"None\"\n    return f\"Semaphore(value={semaphore._value}, locked={semaphore.locked()})\"\n"
  },
  {
    "path": "model_export/README.md",
    "content": "# Model Export for inference on Apple Silicon\nDisclaimer: this is not an official recommendation, just research and exploration. \n\n## Export Vision Encoder\nWe found that LLaVA trainer does not save all the states needed for auto inference, \npredominantly used in third party libraries like `mlx-vlm`. We save additional metadata\nto model checkpoint directory and export the vision model using coremltools. \nExport vision encoder and patch the checkpoint using the instruction below. \n```bash\npython export_vision_encoder.py --model-path /path/to/fastvlm-checkpoint\n```\n\n## Export VLM \n\n### Install mlx-vlm\nWe provide a patch to `mlx-vlm` to support inference of FastVLM.\n```bash\ngit clone https://github.com/Blaizzy/mlx-vlm.git\ncd mlx-vlm \ngit checkout 1884b551bc741f26b2d54d68fa89d4e934b9a3de\ngit apply ../fastvlm_mlx-vlm.patch\npip install -e .\n```\n\nExport model using the following instruction.\n```bash\npython -m mlx_vlm.convert --hf-path  /path/to/fastvlm-checkpoint \\\n                          --mlx-path /path/to/exported-fastvlm \\\n                          --only-llm\n```\nTo quantize the LLM, additional options can be provided as shown below.\n`--q-bits` specifies bits per weight, the command below exports the LLM with 8-bit quantization. \n```bash\npython -m mlx_vlm.convert --hf-path  /path/to/fastvlm-checkpoint \\\n                          --mlx-path /path/to/exported-fastvlm \\\n                          --only-llm \\\n                          -q \\\n                          --q-bits 8       # For 4-bit quantization, specify 4\n```\n\n### Generate\nThe exported model can be used for inference in a python environment following the instruction below.\n```bash\npython -m mlx_vlm.generate --model /path/to/exported-fastvlm \\\n                           --image /path/to/image.png \\\n                           --prompt \"Describe the image.\" \\ \n                           --max-tokens 256 \\\n                           --temp 0.0\n```\n\n## Troubleshooting\nWe noticed that sometimes `config.json` for the LLaVA model incorrectly sets the value for `tie_word_embeddings`.\nThis causes the following error during conversion, `ValueError: Received parameters not in model: language_model.lm_head.weight.`\nIf you encounter this error, set the value of `tie_word_embeddings` accordingly.\n"
  },
  {
    "path": "model_export/export_vision_encoder.py",
    "content": "#\n# For licensing see accompanying LICENSE file.\n# Copyright (C) 2025 Apple Inc. All Rights Reserved.\n#\nimport os\nimport json\nimport copy\nimport argparse\n\nimport torch\nimport numpy as np\nimport coremltools\n\nfrom llava.model.builder import load_pretrained_model\nfrom llava.utils import disable_torch_init\nfrom llava.mm_utils import get_model_name_from_path\n\n\ndef export(args):\n    # Load model\n    disable_torch_init()\n    model_path = os.path.expanduser(args.model_path)\n    model_name = get_model_name_from_path(model_path)\n    tokenizer, model, image_processor, context_len = load_pretrained_model(model_path,\n                                                                           args.model_base,\n                                                                           model_name,\n                                                                           device=\"mps\")\n\n    # Save extra metadata that is not saved during LLaVA training\n    # required by HF for auto-loading model and for mlx-vlm preprocessing\n\n    # Save image processing config\n    setattr(image_processor, \"processor_class\", \"LlavaProcessor\")\n    output_path = os.path.join(model_path, \"preprocessor_config.json\")\n    image_processor.to_json_file(output_path)\n\n    # Create processor config\n    processor_config = dict()\n    processor_config[\"image_token\"] = \"<image>\"\n    processor_config[\"num_additional_image_tokens\"] = 0\n    processor_config[\"processor_class\"] = \"LlavaProcessor\"\n    processor_config[\"patch_size\"] = 64\n    output_path = os.path.join(model_path, \"processor_config.json\")\n    json.dump(processor_config, open(output_path, \"w\"), indent=2)\n\n    # Modify tokenizer to include <image> special token.\n    tokenizer_config_path = os.path.join(model_path, \"tokenizer_config.json\")\n    tokenizer_config = json.load(open(tokenizer_config_path, 'r'))\n    token_ids = list()\n    image_token_is_present = False\n    for k, v in tokenizer_config['added_tokens_decoder'].items():\n        token_ids.append(int(k))\n        if v[\"content\"] == \"<image>\":\n            image_token_is_present = True\n            token_ids.pop()\n\n    # Append only if <image> token is not present\n    if not image_token_is_present:\n        tokenizer_config['added_tokens_decoder'][f'{max(token_ids) + 1}'] = copy.deepcopy(\n            tokenizer_config['added_tokens_decoder'][f'{token_ids[0]}'])\n        tokenizer_config['added_tokens_decoder'][f'{max(token_ids) + 1}'][\"content\"] = \"<image>\"\n        json.dump(tokenizer_config, open(tokenizer_config_path, 'w'), indent=2)\n\n    # Modify config to contain token id for <image>\n    config_path = os.path.join(model_path, \"config.json\")\n    model_config = json.load(open(config_path, 'r'))\n    model_config[\"image_token_index\"] = max(token_ids) + 1\n    json.dump(model_config, open(config_path, 'w'), indent=2)\n\n    # Export the vision encoder to CoreML\n    image_res = image_processor.to_dict()['size']['shortest_edge']\n    inputs = torch.rand(1, 3, image_res, image_res)\n    inputs_tensor = [\n        coremltools.TensorType(\n            name=\"images\",\n            shape=inputs.shape,\n        )\n    ]\n    vision_model = model.get_vision_tower()\n    vision_model = vision_model.float()\n    traced_model = torch.jit.trace(vision_model, torch.Tensor(inputs))\n    pt_name = \"fastvithd.pt\"\n    traced_model.save(pt_name)\n\n    # Export\n    ml_model = coremltools.convert(\n        model=pt_name,\n        outputs=[coremltools.TensorType(name=\"image_features\", dtype=np.float32)],\n        inputs=inputs_tensor,\n        convert_to=\"mlprogram\",\n        debug=False,\n        compute_units=coremltools.ComputeUnit.CPU_AND_GPU,\n        minimum_deployment_target=coremltools.target.iOS16,\n        compute_precision=coremltools.precision.FLOAT32\n    )\n    ml_model_path = os.path.join(model_path, \"fastvithd.mlpackage\")\n    ml_model.save(ml_model_path)\n\n    # Remove traced model\n    os.remove(pt_name)\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--model-path\", type=str, required=True)\n    parser.add_argument(\"--model-base\", type=str, default=None)\n    parser.add_argument(\"--conv-mode\", type=str, default=\"qwen_2\")\n\n    args = parser.parse_args()\n\n    export(args)\n"
  },
  {
    "path": "model_export/fastvlm_mlx-vlm.patch",
    "content": "diff --git a/mlx_vlm/convert.py b/mlx_vlm/convert.py\nindex 5952a88..335e9db 100644\n--- a/mlx_vlm/convert.py\n+++ b/mlx_vlm/convert.py\n@@ -55,6 +55,12 @@ def configure_parser() -> argparse.ArgumentParser:\n         action=\"store_true\",\n         default=False,\n     )\n+    parser.add_argument(\n+        \"--only-llm\",\n+        help=\"Convert only LLM.\",\n+        action=\"store_true\",\n+        default=False,\n+    )\n     return parser\n \n \ndiff --git a/mlx_vlm/models/fastvlm/__init__.py b/mlx_vlm/models/fastvlm/__init__.py\nnew file mode 100644\nindex 0000000..691192e\n--- /dev/null\n+++ b/mlx_vlm/models/fastvlm/__init__.py\n@@ -0,0 +1,7 @@\n+from .fastvlm import (\n+    LanguageModel,\n+    Model,\n+    ModelConfig,\n+    TextConfig,\n+    VisionConfig,\n+)\ndiff --git a/mlx_vlm/models/fastvlm/fastvlm.py b/mlx_vlm/models/fastvlm/fastvlm.py\nnew file mode 100644\nindex 0000000..7db6497\n--- /dev/null\n+++ b/mlx_vlm/models/fastvlm/fastvlm.py\n@@ -0,0 +1,187 @@\n+import glob\n+import inspect\n+import json\n+from dataclasses import dataclass\n+from pathlib import Path\n+from typing import Optional\n+\n+import mlx.core as mx\n+import mlx.nn as nn\n+import numpy as np\n+import coremltools\n+from huggingface_hub import snapshot_download\n+\n+from .language import LanguageModel, TextConfig\n+\n+\n+@dataclass\n+class VisionConfig:\n+    mm_hidden_size: int\n+    mm_vision_tower: str\n+\n+    @classmethod\n+    def from_dict(cls, params):\n+        return cls(\n+            **{\n+                k: v\n+                for k, v in params.items()\n+                if k in inspect.signature(cls).parameters\n+            }\n+        )\n+\n+@dataclass\n+class ModelConfig:\n+    text_config: TextConfig\n+    vision_config: VisionConfig\n+    model_type: str\n+    ignore_index: int = -100\n+    image_token_index: int = 32000\n+    vision_feature_select_strategy: str = \"default\"\n+    vision_feature_layer: int = -2\n+    vocab_size: int = 151936\n+\n+    @classmethod\n+    def from_dict(cls, params):\n+        # Copy text config parameters from root level\n+        params[\"text_config\"] = dict(\n+            filter(lambda x: 'mm' not in x[0], params.items())\n+        )\n+        # Copy vision config parameters from root level\n+        params[\"vision_config\"] = dict(\n+            filter(lambda x: 'mm' in x[0], params.items())\n+        )\n+\n+        return cls(\n+            **{\n+                k: v\n+                for k, v in params.items()\n+                if k in inspect.signature(cls).parameters\n+            }\n+        )\n+\n+\n+class FastVLMMultiModalProjector(nn.Module):\n+    def __init__(self, config: ModelConfig):\n+        super().__init__()\n+        self.linear_0 = nn.Linear(\n+            config.vision_config.mm_hidden_size, config.text_config.hidden_size, bias=True\n+        )\n+        self.gelu = nn.GELU()\n+        self.linear_2 = nn.Linear(\n+            config.text_config.hidden_size, config.text_config.hidden_size, bias=True\n+        )\n+\n+    def __call__(self, x: mx.array) -> mx.array:\n+        x = self.linear_0(x)\n+        x = self.gelu(x)\n+        x = self.linear_2(x)\n+        return x\n+\n+\n+class Model(nn.Module):\n+    def __init__(self, config: ModelConfig):\n+        super().__init__()\n+        self.config = config\n+        self.vision_tower = None\n+        self.language_model = LanguageModel(config.text_config)\n+        self.multi_modal_projector = FastVLMMultiModalProjector(config)\n+        self.vision_feature_layer = config.vision_feature_layer\n+        self.vision_feature_select_strategy = config.vision_feature_select_strategy\n+\n+    def get_input_embeddings(\n+        self,\n+        input_ids: Optional[mx.array] = None,\n+        pixel_values: Optional[mx.array] = None,\n+    ):\n+        if pixel_values is None:\n+            return self.language_model.model.embed_tokens(input_ids)\n+\n+        # Get the input embeddings from the language model\n+        inputs_embeds = self.language_model.model.embed_tokens(input_ids)\n+\n+        # Get image features from CoreML model\n+        coreml_out_dict = self.vision_tower.predict({\"images\": np.array(pixel_values, copy=False)})\n+\n+        # Pass image features through the multi-modal projector\n+        image_features = self.multi_modal_projector(mx.array(coreml_out_dict[\"image_features\"]))\n+\n+        # Insert special image tokens in the input_ids\n+        final_inputs_embeds = self._merge_input_ids_with_image_features(\n+            image_features, inputs_embeds, input_ids\n+        )\n+        return final_inputs_embeds\n+\n+    def _merge_input_ids_with_image_features(\n+        self, image_features, inputs_embeds, input_ids\n+    ):\n+        image_token_index = self.config.image_token_index\n+        num_images, num_image_patches, embed_dim = image_features.shape\n+\n+        # Positions of <image> tokens in input_ids, assuming batch size is 1\n+        image_positions = np.where(input_ids[0] == image_token_index)[0].tolist()\n+        num_images, _, vision_hidden_size = image_features.shape\n+\n+        reshaped_image_hidden_states = image_features.reshape(-1, vision_hidden_size)\n+\n+        # cast to the dtype of the input_embeds to support quantized models\n+        reshaped_image_hidden_states = reshaped_image_hidden_states.astype(\n+            inputs_embeds.dtype\n+        )\n+        inputs_embeds[:, image_positions, :] = reshaped_image_hidden_states\n+        return inputs_embeds\n+\n+    def __call__(\n+        self,\n+        input_ids: mx.array,\n+        pixel_values: mx.array,\n+        mask: mx.array,\n+        cache=None,\n+        **kwargs,\n+    ):\n+        input_embddings = self.get_input_embeddings(input_ids, pixel_values)\n+        logits = self.language_model(\n+            input_ids, cache=cache, inputs_embeds=input_embddings\n+        )\n+        return logits\n+\n+    @staticmethod\n+    def from_pretrained(path_or_hf_repo: str):\n+        path = Path(path_or_hf_repo)\n+        if not path.exists():\n+            path = Path(\n+                snapshot_download(\n+                    repo_id=path_or_hf_repo,\n+                    allow_patterns=[\n+                        \"*.json\",\n+                        \"*.safetensors\",\n+                        \"*.py\",\n+                        \"tokenizer.model\",\n+                        \"*.tiktoken\",\n+                    ],\n+                )\n+            )\n+\n+        with open(path / \"config.json\", \"r\") as f:\n+            model_config = json.load(f)\n+\n+        model_config = ModelConfig.from_dict(model_config)\n+        model_config.text_config = TextConfig.from_dict(model_config.text_config)\n+\n+        model = Model(model_config)\n+        weight_files = glob.glob(str(path / \"*.safetensors\"))\n+        if not weight_files:\n+            raise FileNotFoundError(f\"No safetensors found in {path}\")\n+\n+        weights = {}\n+        for wf in weight_files:\n+            weights.update(mx.load(wf))\n+\n+        weights = LanguageModel.sanitize(weights)\n+\n+        # Load CoreML vision tower\n+        coreml_file = glob.glob(str(path / \"*.mlpackage\"))\n+        assert len(coreml_file) == 1, \"Found multiple vision model files\"\n+        model.vision_tower = coremltools.models.MLModel(coreml_file[0])\n+\n+        model.load_weights(list(weights.items()))\n+        return model\ndiff --git a/mlx_vlm/models/fastvlm/language.py b/mlx_vlm/models/fastvlm/language.py\nnew file mode 100644\nindex 0000000..b791df4\n--- /dev/null\n+++ b/mlx_vlm/models/fastvlm/language.py\n@@ -0,0 +1,220 @@\n+import inspect\n+from dataclasses import dataclass\n+from typing import Dict, Optional, Tuple, Union\n+\n+import mlx.core as mx\n+import mlx.nn as nn\n+import numpy as np\n+\n+from ..base import KVCache, LanguageModelOutput, create_attention_mask\n+\n+\n+@dataclass\n+class TextConfig:\n+    model_type: str\n+    hidden_size: int\n+    num_hidden_layers: int\n+    intermediate_size: int\n+    num_attention_heads: int\n+    rms_norm_eps: float\n+    vocab_size: int\n+    num_key_value_heads: Optional[int] = None\n+    max_position_embeddings: Optional[int] = 32768\n+    rope_theta: float = 1000000\n+    rope_traditional: bool = False\n+    rope_scaling: Optional[Dict[str, Union[float, str]]] = None\n+    tie_word_embeddings: bool = True\n+\n+    def __post_init__(self):\n+        if self.num_key_value_heads is None:\n+            self.num_key_value_heads = self.num_attention_heads\n+\n+        if self.rope_scaling:\n+            required_keys = {\"mrope_section\", \"type\"}\n+            if not all(key in self.rope_scaling for key in required_keys):\n+                raise ValueError(f\"rope_scaling must contain keys {required_keys}\")\n+\n+            if not self.rope_scaling[\"type\"] in [\"mrope\", \"default\"]:\n+                raise ValueError(f\"rope_scaling type must be 'mrope' or 'default'\")\n+\n+    @classmethod\n+    def from_dict(cls, params):\n+        return cls(\n+            **{\n+                k: v\n+                for k, v in params.items()\n+                if k in inspect.signature(cls).parameters\n+            }\n+        )\n+\n+\n+class Attention(nn.Module):\n+    def __init__(self, args: TextConfig):\n+        super().__init__()\n+\n+        dim = args.hidden_size\n+        self.n_heads = n_heads = args.num_attention_heads\n+        assert args.num_key_value_heads is not None\n+        self.n_kv_heads = n_kv_heads = args.num_key_value_heads\n+\n+        self.head_dim = head_dim = args.hidden_size // n_heads\n+        self.scale = head_dim**-0.5\n+\n+        self.q_proj = nn.Linear(dim, n_heads * head_dim, bias=True)\n+        self.k_proj = nn.Linear(dim, n_kv_heads * head_dim, bias=True)\n+        self.v_proj = nn.Linear(dim, n_kv_heads * head_dim, bias=True)\n+        self.o_proj = nn.Linear(n_heads * head_dim, dim, bias=False)\n+\n+        self.rotary_emb = nn.RoPE(\n+            head_dim,\n+            base=args.rope_theta,\n+            traditional=args.rope_traditional,\n+        )\n+\n+    def __call__(\n+        self,\n+        x: mx.array,\n+        mask: Optional[mx.array] = None,\n+        cache: Optional[KVCache] = None,\n+    ) -> mx.array:\n+        B, L, D = x.shape\n+\n+        queries, keys, values = self.q_proj(x), self.k_proj(x), self.v_proj(x)\n+\n+        # Prepare the queries, keys and values for the attention computation\n+        queries = queries.reshape(B, L, self.n_heads, self.head_dim).transpose(\n+            0, 2, 1, 3\n+        )\n+        keys = keys.reshape(B, L, self.n_kv_heads, self.head_dim).transpose(0, 2, 1, 3)\n+        values = values.reshape(B, L, self.n_kv_heads, self.head_dim).transpose(\n+            0, 2, 1, 3\n+        )\n+\n+        offset = cache.offset if cache else 0\n+\n+        if mask is not None:\n+            mask = mask[..., : keys.shape[-2]]\n+\n+        queries = self.rotary_emb(queries, offset=offset)\n+        keys = self.rotary_emb(keys, offset=offset)\n+\n+        if cache is not None:\n+            keys, values = cache.update_and_fetch(keys, values)\n+\n+        output = mx.fast.scaled_dot_product_attention(\n+            queries, keys, values, scale=self.scale, mask=mask\n+        )\n+        output = output.transpose(0, 2, 1, 3).reshape(B, L, -1)\n+        return self.o_proj(output)\n+\n+\n+class MLP(nn.Module):\n+    def __init__(self, dim, hidden_dim):\n+        super().__init__()\n+        self.gate_proj = nn.Linear(dim, hidden_dim, bias=False)\n+        self.down_proj = nn.Linear(hidden_dim, dim, bias=False)\n+        self.up_proj = nn.Linear(dim, hidden_dim, bias=False)\n+\n+    def __call__(self, x) -> mx.array:\n+        return self.down_proj(nn.silu(self.gate_proj(x)) * self.up_proj(x))\n+\n+\n+class Qwen2DecoderLayer(nn.Module):\n+    def __init__(self, args: TextConfig):\n+        super().__init__()\n+        self.num_attention_heads = args.num_attention_heads\n+        self.hidden_size = args.hidden_size\n+        self.self_attn = Attention(args)\n+        self.mlp = MLP(args.hidden_size, args.intermediate_size)\n+        self.input_layernorm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)\n+        self.post_attention_layernorm = nn.RMSNorm(\n+            args.hidden_size, eps=args.rms_norm_eps\n+        )\n+        self.args = args\n+\n+    def __call__(\n+        self,\n+        x: mx.array,\n+        mask: Optional[mx.array] = None,\n+        cache: Optional[KVCache] = None,\n+    ) -> mx.array:\n+        r = self.self_attn(self.input_layernorm(x), mask, cache)\n+        h = x + r\n+        r = self.mlp(self.post_attention_layernorm(h))\n+        out = h + r\n+        return out\n+\n+\n+class Qwen2Model(nn.Module):\n+    def __init__(self, args: TextConfig):\n+        super().__init__()\n+        self.args = args\n+        self.vocab_size = args.vocab_size\n+        self.num_hidden_layers = args.num_hidden_layers\n+        assert self.vocab_size > 0\n+        self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size)\n+        self.layers = [\n+            Qwen2DecoderLayer(args=args) for _ in range(args.num_hidden_layers)\n+        ]\n+        self.norm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)\n+\n+    def __call__(\n+        self,\n+        inputs: mx.array,\n+        cache=None,\n+        inputs_embeds: Optional[mx.array] = None,\n+    ):\n+        if inputs_embeds is None:\n+            h = self.embed_tokens(inputs)\n+        else:\n+            h = inputs_embeds\n+\n+        mask = create_attention_mask(h, cache)\n+\n+        if cache is None:\n+            cache = [None] * len(self.layers)\n+\n+        for layer, c in zip(self.layers, cache):\n+            h = layer(h, mask, c)\n+\n+        return self.norm(h)\n+\n+\n+class LanguageModel(nn.Module):\n+    def __init__(self, args: TextConfig):\n+        super().__init__()\n+        self.args = args\n+        self.model_type = args.model_type\n+        self.model = Qwen2Model(args)\n+\n+        if \"qwen2\" not in args.model_type:\n+            raise ValueError(f\"Unsupported model type: {args.model_type}\")\n+\n+        if not args.tie_word_embeddings:\n+            self.lm_head = nn.Linear(args.hidden_size, args.vocab_size, bias=False)\n+\n+    def __call__(\n+        self,\n+        inputs: mx.array,\n+        cache=None,\n+        inputs_embeds: Optional[mx.array] = None,\n+        mask: Optional[mx.array] = None,\n+    ):\n+        out = self.model(inputs, cache=cache, inputs_embeds=inputs_embeds)\n+        if self.args.tie_word_embeddings:\n+            out = self.model.embed_tokens.as_linear(out)\n+        else:\n+            out = self.lm_head(out)\n+        return LanguageModelOutput(logits=out)\n+\n+    @property\n+    def layers(self):\n+        return self.model.layers\n+\n+    @property\n+    def head_dim(self):\n+        return self.args.hidden_size // self.args.num_attention_heads\n+\n+    @property\n+    def n_kv_heads(self):\n+        return self.args.num_key_value_heads\ndiff --git a/mlx_vlm/prompt_utils.py b/mlx_vlm/prompt_utils.py\nindex 725e811..ba48296 100644\n--- a/mlx_vlm/prompt_utils.py\n+++ b/mlx_vlm/prompt_utils.py\n@@ -93,6 +93,7 @@ def get_message_json(\n         \"idefics2\": \"message_list_with_image\",\n         \"idefics3\": \"message_list_with_image\",\n         \"llava\": \"message_list_with_image\",\n+        \"llava_qwen2\": \"message_with_image_token_new_line\",\n         \"llava_next\": \"message_list_with_image\",\n         \"mllama\": \"message_list_with_image\",\n         # Models that can handle both image and video formats\n@@ -143,7 +144,7 @@ def get_message_json(\n \n \n def get_chat_template(processor, messages, add_generation_prompt, tokenize=False):\n-    if \"chat_template\" in processor.__dict__.keys():\n+    if (\"chat_template\" in processor.__dict__.keys()) and (processor.chat_template is not None):\n         return processor.apply_chat_template(\n             messages,\n             tokenize=tokenize,\ndiff --git a/mlx_vlm/utils.py b/mlx_vlm/utils.py\nindex 4acff3e..00f366f 100644\n--- a/mlx_vlm/utils.py\n+++ b/mlx_vlm/utils.py\n@@ -1,3 +1,4 @@\n+import os\n import copy\n import glob\n import importlib\n@@ -15,6 +16,7 @@ import mlx.core as mx\n import mlx.nn as nn\n import numpy as np\n import requests\n+import coremltools\n from huggingface_hub import snapshot_download\n from mlx.utils import tree_flatten, tree_unflatten\n from PIL import Image, ImageOps\n@@ -31,7 +33,7 @@ from .tokenizer_utils import load_tokenizer\n from .trainer import apply_lora_layers\n \n # Constants\n-MODEL_REMAPPING = {\"llava-qwen2\": \"llava_bunny\", \"bunny-llama\": \"llava_bunny\"}\n+MODEL_REMAPPING = {\"llava-qwen2\": \"llava_bunny\", \"bunny-llama\": \"llava_bunny\", \"llava_qwen2\": \"fastvlm\"}\n \n MAX_FILE_SIZE_GB = 5\n \n@@ -168,9 +170,19 @@ python -m mlx_vlm.convert --hf-path <local_dir> --mlx-path <mlx_dir>\n \n     # Sanitize weights\n     weights = sanitize_weights(model, weights)\n-    weights = sanitize_weights(\n-        model_class.VisionModel, weights, model_config.vision_config\n-    )\n+    if hasattr(model_class, 'VisionModel'):\n+        weights = sanitize_weights(\n+            model_class.VisionModel, weights, model_config.vision_config\n+        )\n+    else:\n+        # Load CoreML vision tower\n+        print(\"Looking for CoreML vision tower\")\n+        coreml_file = glob.glob(str(model_path / \"*.mlpackage\"))\n+        if len(coreml_file) > 0:\n+            assert len(coreml_file) == 1, \"Found multiple vision model files.\"\n+            print(f\"Loading {coreml_file[0]} vision tower\")\n+            model.vision_tower = coremltools.models.MLModel(coreml_file[0])\n+\n     weights = sanitize_weights(\n         model_class.LanguageModel, weights, model_config.text_config\n     )\n@@ -185,7 +197,21 @@ python -m mlx_vlm.convert --hf-path <local_dir> --mlx-path <mlx_dir>\n             class_predicate=class_predicate,\n         )\n \n-    model.load_weights(list(weights.items()))\n+    if kwargs.get(\"only_llm\", False):\n+        # Ignore vision tower weights\n+        new_weights = dict()\n+        for k, v in weights.items():\n+            if 'vision_tower' in k:\n+                continue\n+            if 'mm_projector' in k:\n+                new_k = k.replace('model.mm_projector.', 'multi_modal_projector.linear_')\n+                new_weights[new_k] = v\n+            else:\n+                new_weights['language_model.'+k] = v\n+\n+        model.load_weights(list(new_weights.items()))\n+    else:\n+        model.load_weights(list(weights.items()))\n     if not lazy:\n         mx.eval(model.parameters())\n \n@@ -669,11 +695,12 @@ def convert(\n     dequantize: bool = False,\n     skip_vision: bool = False,\n     trust_remote_code: bool = True,\n+    only_llm: bool = False\n ):\n     print(\"[INFO] Loading\")\n     model_path = get_model_path(hf_path, revision=revision)\n     model, config, processor = fetch_from_hub(\n-        model_path, lazy=True, trust_remote_code=trust_remote_code\n+        model_path, lazy=True, trust_remote_code=trust_remote_code, only_llm=only_llm\n     )\n \n     weights = dict(tree_flatten(model.parameters()))\n@@ -709,6 +736,12 @@ def convert(\n \n     save_config(config, config_path=mlx_path / \"config.json\")\n \n+    # Copy over any coreml files if found\n+    coreml_files = glob.glob(str(model_path / \"*.mlpackage\"))\n+    for file in coreml_files:\n+        des_path = os.path.join(mlx_path, file.split(os.path.sep)[-1])\n+        shutil.copytree(file, des_path)\n+\n     if upload_repo is not None:\n         upload_to_hub(mlx_path, upload_repo, hf_path)\n \n"
  },
  {
    "path": "predict.py",
    "content": "#\n# Modified from LLaVA/predict.py\n# Please see ACKNOWLEDGEMENTS for details about LICENSE\n#\nimport os\nimport argparse\n\nimport torch\nfrom PIL import Image\n\nfrom llava.utils import disable_torch_init\nfrom llava.conversation import conv_templates\nfrom llava.model.builder import load_pretrained_model\nfrom llava.mm_utils import tokenizer_image_token, process_images, get_model_name_from_path\nfrom llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN\n\n\ndef predict(args):\n    # Remove generation config from model folder\n    # to read generation parameters from args\n    model_path = os.path.expanduser(args.model_path)\n    generation_config = None\n    if os.path.exists(os.path.join(model_path, 'generation_config.json')):\n        generation_config = os.path.join(model_path, '.generation_config.json')\n        os.rename(os.path.join(model_path, 'generation_config.json'),\n                  generation_config)\n\n    # Load model\n    disable_torch_init()\n    model_name = get_model_name_from_path(model_path)\n    tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, args.model_base, model_name, device=\"mps\")\n\n    # Construct prompt\n    qs = args.prompt\n    if model.config.mm_use_im_start_end:\n        qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\\n' + qs\n    else:\n        qs = DEFAULT_IMAGE_TOKEN + '\\n' + qs\n    conv = conv_templates[args.conv_mode].copy()\n    conv.append_message(conv.roles[0], qs)\n    conv.append_message(conv.roles[1], None)\n    prompt = conv.get_prompt()\n\n    # Set the pad token id for generation\n    model.generation_config.pad_token_id = tokenizer.pad_token_id\n\n    # Tokenize prompt\n    input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(torch.device(\"mps\"))\n\n    # Load and preprocess image\n    image = Image.open(args.image_file).convert('RGB')\n    image_tensor = process_images([image], image_processor, model.config)[0]\n\n    # Run inference\n    with torch.inference_mode():\n        output_ids = model.generate(\n            input_ids,\n            images=image_tensor.unsqueeze(0).half(),\n            image_sizes=[image.size],\n            do_sample=True if args.temperature > 0 else False,\n            temperature=args.temperature,\n            top_p=args.top_p,\n            num_beams=args.num_beams,\n            max_new_tokens=256,\n            use_cache=True)\n\n        outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()\n        print(outputs)\n\n    # Restore generation config\n    if generation_config is not None:\n        os.rename(generation_config, os.path.join(model_path, 'generation_config.json'))\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--model-path\", type=str, default=\"./llava-v1.5-13b\")\n    parser.add_argument(\"--model-base\", type=str, default=None)\n    parser.add_argument(\"--image-file\", type=str, default=None, help=\"location of image file\")\n    parser.add_argument(\"--prompt\", type=str, default=\"Describe the image.\", help=\"Prompt for VLM.\")\n    parser.add_argument(\"--conv-mode\", type=str, default=\"qwen_2\")\n    parser.add_argument(\"--temperature\", type=float, default=0.2)\n    parser.add_argument(\"--top_p\", type=float, default=None)\n    parser.add_argument(\"--num_beams\", type=int, default=1)\n    args = parser.parse_args()\n\n    predict(args)\n"
  },
  {
    "path": "pyproject.toml",
    "content": "[build-system]\nrequires = [\"setuptools>=61.0\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = \"llava\"\nversion = \"1.2.2.post1\"\ndescription = \"Towards GPT-4 like large language and visual assistant.\"\nreadme = \"README.md\"\nrequires-python = \">=3.8\"\nclassifiers = [\n    \"Programming Language :: Python :: 3\",\n    \"License :: OSI Approved :: Apache Software License\",\n]\ndependencies = [\n    \"torch==2.6.0\", \"torchvision==0.21.0\",\n    \"transformers==4.48.3\", \"tokenizers==0.21.0\", \"sentencepiece==0.1.99\", \"shortuuid\",\n    \"accelerate==1.6.0\", \"peft>=0.10.0,<0.14.0\", \"bitsandbytes\",\n    \"pydantic\", \"markdown2[all]\", \"numpy==1.26.4\", \"scikit-learn==1.2.2\",\n    \"gradio==5.11.0\", \"requests\", \"uvicorn\", \"fastapi\",\n    \"einops==0.6.1\", \"einops-exts==0.0.4\", \"timm==1.0.15\",\n    \"coremltools==8.2\"\n]\n\n[project.optional-dependencies]\ntrain = [\"deepspeed==0.13.1\", \"ninja\", \"wandb\"]\nbuild = [\"build\", \"twine\"]\n\n[tool.setuptools.packages.find]\nexclude = [\"assets*\", \"benchmark*\", \"docs\", \"dist*\", \"playground*\", \"scripts*\", \"tests*\"]\n\n[tool.wheel]\nexclude = [\"assets*\", \"benchmark*\", \"docs\", \"dist*\", \"playground*\", \"scripts*\", \"tests*\"]\n"
  }
]