[
{
"path": ".dev_scripts/build_docs.sh",
"content": "pip install -r requirements/docs.txt\ncd docs\nrm -rf build\n\n# update api rst\n#rm -rf source/api/\n#sphinx-apidoc --module-first -o source/api/ ../modelscope/\nmake html\n"
},
{
"path": ".dev_scripts/ci_container_test.sh",
"content": "if [ \"$MODELSCOPE_SDK_DEBUG\" == \"True\" ]; then\n # pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple\n pip install -r requirements/tests.txt -i https://mirrors.aliyun.com/pypi/simple/\n git config --global --add safe.directory /ms-swift\n git config --global user.email tmp\n git config --global user.name tmp.com\n\n # linter test\n # use internal project for pre-commit due to the network problem\n if [ `git remote -v | grep alibaba | wc -l` -gt 1 ]; then\n pre-commit run -c .pre-commit-config_local.yaml --all-files\n if [ $? -ne 0 ]; then\n echo \"linter test failed, please run 'pre-commit run --all-files' to check\"\n echo \"From the repository folder\"\n echo \"Run 'pip install -r requirements/tests.txt' install test dependencies.\"\n echo \"Run 'pre-commit install' install pre-commit hooks.\"\n echo \"Finally run linter with command: 'pre-commit run --all-files' to check.\"\n echo \"Ensure there is no failure!!!!!!!!\"\n exit -1\n fi\n fi\n\n pip install -r requirements/framework.txt -U -i https://mirrors.aliyun.com/pypi/simple/\n pip install decord einops -U -i https://mirrors.aliyun.com/pypi/simple/\n pip uninstall autoawq -y\n pip install optimum\n pip install diffusers\n pip install \"transformers<5.0\"\n # pip install autoawq -U --no-deps\n\n # test with install\n pip install .\n pip install auto_gptq bitsandbytes deepspeed -U -i https://mirrors.aliyun.com/pypi/simple/\nelse\n echo \"Running case in release image, run case directly!\"\nfi\n# remove torch_extensions folder to avoid ci hang.\nrm -rf ~/.cache/torch_extensions\nif [ $# -eq 0 ]; then\n ci_command=\"python tests/run.py --subprocess\"\nelse\n ci_command=\"$@\"\nfi\necho \"Running case with command: $ci_command\"\n$ci_command\n"
},
{
"path": ".dev_scripts/dockerci.sh",
"content": "#!/bin/bash\nMODELSCOPE_CACHE_DIR_IN_CONTAINER=/modelscope_cache\nCODE_DIR=$PWD\nCODE_DIR_IN_CONTAINER=/ms-swift\nMODELSCOPE_SDK_DEBUG=True\necho \"$USER\"\ngpus='0,1 2,3'\ncpu_sets='0-15 16-31'\ncpu_sets_arr=($cpu_sets)\nis_get_file_lock=false\nCI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh python tests/run.py --parallel 2 --run_config tests/run_config.yaml}\necho \"ci command: $CI_COMMAND\"\nPR_CHANGED_FILES=\"${PR_CHANGED_FILES:-}\"\necho \"PR modified files: $PR_CHANGED_FILES\"\nPR_CHANGED_FILES=${PR_CHANGED_FILES//[ ]/#}\necho \"PR_CHANGED_FILES: $PR_CHANGED_FILES\"\nidx=0\nfor gpu in $gpus\ndo\n exec {lock_fd}>\"/tmp/gpu$gpu\" || exit 1\n flock -n \"$lock_fd\" || { echo \"WARN: gpu $gpu is in use!\" >&2; idx=$((idx+1)); continue; }\n echo \"get gpu lock $gpu\"\n\n CONTAINER_NAME=\"swift-ci-$idx\"\n let is_get_file_lock=true\n\n # pull image if there are update\n docker pull ${IMAGE_NAME}:${IMAGE_VERSION}\n if [ \"$MODELSCOPE_SDK_DEBUG\" == \"True\" ]; then\n echo 'debugging'\n docker run --rm --name $CONTAINER_NAME --shm-size=16gb \\\n --cpuset-cpus=${cpu_sets_arr[$idx]} \\\n --gpus='\"'\"device=$gpu\"'\"' \\\n -v $CODE_DIR:$CODE_DIR_IN_CONTAINER \\\n -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \\\n -v $MODELSCOPE_HOME_CACHE/$idx:/root \\\n -v /home/admin/pre-commit:/home/admin/pre-commit \\\n -e CI_TEST=True \\\n -e TEST_LEVEL=$TEST_LEVEL \\\n -e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER \\\n -e MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN \\\n -e MODELSCOPE_SDK_DEBUG=True \\\n -e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \\\n -e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \\\n -e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \\\n -e TEST_LEVEL=$TEST_LEVEL \\\n -e MODELSCOPE_ENVIRONMENT='ci' \\\n -e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \\\n -e MODEL_TAG_URL=$MODEL_TAG_URL \\\n -e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \\\n\t -e PR_CHANGED_FILES=$PR_CHANGED_FILES \\\n --workdir=$CODE_DIR_IN_CONTAINER \\\n ${IMAGE_NAME}:${IMAGE_VERSION} \\\n $CI_COMMAND\n else\n docker run --rm --name $CONTAINER_NAME --shm-size=16gb \\\n --cpuset-cpus=${cpu_sets_arr[$idx]} \\\n --gpus='\"'\"device=$gpu\"'\"' \\\n -v $CODE_DIR:$CODE_DIR_IN_CONTAINER \\\n -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \\\n -v $MODELSCOPE_HOME_CACHE/$idx:/root \\\n -v /home/admin/pre-commit:/home/admin/pre-commit \\\n -e CI_TEST=True \\\n -e TEST_LEVEL=$TEST_LEVEL \\\n -e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER \\\n -e MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN \\\n -e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \\\n -e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \\\n -e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \\\n -e TEST_LEVEL=$TEST_LEVEL \\\n -e MODELSCOPE_ENVIRONMENT='ci' \\\n -e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \\\n -e MODEL_TAG_URL=$MODEL_TAG_URL \\\n -e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \\\n\t -e PR_CHANGED_FILES=$PR_CHANGED_FILES \\\n --workdir=$CODE_DIR_IN_CONTAINER \\\n ${IMAGE_NAME}:${IMAGE_VERSION} \\\n $CI_COMMAND\n fi\n if [ $? -ne 0 ]; then\n echo \"Running test case failed, please check the log!\"\n exit -1\n fi\n break\ndone\nif [ \"$is_get_file_lock\" = false ] ; then\n echo 'No free GPU!'\n exit 1\nfi\n"
},
{
"path": ".dev_scripts/dockerci_npu.sh",
"content": "#!/bin/bash\nMODELSCOPE_CACHE_DIR=/modelscope_cache\nCODE_DIR=$PWD\nMODELSCOPE_SDK_DEBUG=True\necho \"$USER\"\ngpus='0,1 2,3'\nis_get_file_lock=false\nCI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh python tests/run.py --parallel 2 --run_config tests/run_config.yaml}\necho \"ci command: $CI_COMMAND\"\nPR_CHANGED_FILES=\"${PR_CHANGED_FILES:-}\"\necho \"PR modified files: $PR_CHANGED_FILES\"\nPR_CHANGED_FILES=${PR_CHANGED_FILES//[ ]/#}\necho \"PR_CHANGED_FILES: $PR_CHANGED_FILES\"\nidx=0\nfor gpu in $gpus\ndo\n exec {lock_fd}>\"/tmp/gpu$gpu\" || exit 1\n flock -n \"$lock_fd\" || { echo \"WARN: gpu $gpu is in use!\" >&2; idx=$((idx+1)); continue; }\n echo \"get gpu lock $gpu\"\n\n let is_get_file_lock=true\n\n # 设置环境变量\n export CI_TEST=True\n export TEST_LEVEL=$TEST_LEVEL\n export MODELSCOPE_CACHE=${MODELSCOPE_CACHE:-$MODELSCOPE_CACHE_DIR}\n export MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN\n export HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT\n export TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST\n export TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV\n export MODELSCOPE_ENVIRONMENT='ci'\n export TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN\n export MODEL_TAG_URL=$MODEL_TAG_URL\n export MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN\n export PR_CHANGED_FILES=$PR_CHANGED_FILES\n export CUDA_VISIBLE_DEVICES=$gpu\n\n if [ \"$MODELSCOPE_SDK_DEBUG\" == \"True\" ]; then\n export MODELSCOPE_SDK_DEBUG=True\n echo 'debugging'\n fi\n\n # 切换到代码目录并执行命令\n cd $CODE_DIR\n eval $CI_COMMAND\n\n if [ $? -ne 0 ]; then\n echo \"Running test case failed, please check the log!\"\n exit -1\n fi\n break\ndone\n\nif [ \"$is_get_file_lock\" = false ] ; then\n echo 'No free GPU!'\n exit 1\nfi\n"
},
{
"path": ".github/ISSUE_TEMPLATE/1-bug-report.yml",
"content": "name: \"🐛 Bug Report\"\ndescription: Create a bug report to help us improve ms-swift\nlabels: [\"bug\"]\n\nbody:\n - type: markdown\n attributes:\n value: |\n Thank you for supporting ms-swift and taking the time to submit this issue.\n 感谢你对 ms-swift 的支持和抽出时间提交相关 issue。\n\n - type: checkboxes\n id: checklist\n attributes:\n label: Checklist / 检查清单\n options:\n - label: I have searched existing issues, and this is a new bug report. / 我已经搜索过现有的 issues，确认这是一个新的 bug report。\n required: true\n\n\n - type: textarea\n id: bug-description\n validations:\n required: true\n attributes:\n label: Bug Description / Bug 描述\n description: |\n Please describe the issue you encountered. It's better to include error screenshots or stack trace information.\n 请详细描述你遇到的问题，最好包含报错截图或报错栈信息。\n\n\n - type: textarea\n id: reproduction-steps\n validations:\n required: true\n attributes:\n label: How to Reproduce / 如何复现\n description: |\n Please provide steps to reproduce the issue, including ms-swift version, runtime environment, and detailed reproduction steps.\n 请提供复现问题的步骤，包括 ms-swift 的版本、运行环境、详细的复现步骤等。\n\n\n - type: textarea\n id: additional-information\n attributes:\n label: Additional Information / 补充信息\n description: |\n Please provide any additional information here.\n 在这里补充其他相关信息。\n"
},
{
"path": ".github/ISSUE_TEMPLATE/2-feature-request.yml",
"content": "name: \"🚀 Feature Request\"\ndescription: Submit a request for a new feature\nlabels: [\"enhancement\"]\n\nbody:\n - type: markdown\n attributes:\n value: |\n Thank you for supporting ms-swift and taking the time to submit this issue.\n 感谢你对 ms-swift 的支持和抽出时间提交相关 issue。\n\n - type: checkboxes\n id: checklist\n attributes:\n label: Checklist / 检查清单\n options:\n - label: I have searched existing issues, and this is a new feature request. / 我已经搜索过现有的 issues，确认这是一个新的 Feature Request。\n required: true\n\n - type: textarea\n id: feature-request-description\n validations:\n required: true\n attributes:\n label: Feature Request Description / Feature Request 描述\n description: |\n Please provide a detailed description of the new feature you would like to see added.\n 请详细描述您希望添加的新功能特性。\n\n\n - type: textarea\n id: pull-request\n attributes:\n label: Pull Request / Pull Request 信息\n description: |\n Have you already submitted or plan to submit a Pull Request? Please share your plans.\n 你是否已经提交或即将提交 Pull Request？请说明你的计划。\n"
},
{
"path": ".github/ISSUE_TEMPLATE/3-question-discussion.yml",
"content": "name: \"🤔 Question & Discussion\"\ndescription: Create an issue for questions and discussions\nlabels: [\"question\"]\n\nbody:\n - type: markdown\n attributes:\n value: |\n Thank you for supporting ms-swift and taking the time to submit this issue.\n 感谢你对 ms-swift 的支持和抽出时间提交相关 issue。\n\n - type: checkboxes\n id: checklist\n attributes:\n label: Checklist / 检查清单\n options:\n - label: I have searched existing issues, and this is a new question or discussion topic. / 我已经搜索过现有的 issues，确认这是一个新的问题与讨论。\n required: true\n\n - type: textarea\n id: question-description\n validations:\n required: true\n attributes:\n label: Question Description / 问题描述\n description: |\n Please describe the question or topic you would like to discuss.\n 请描述你想要讨论的问题或话题。\n"
},
{
"path": ".github/ISSUE_TEMPLATE/config.yml",
"content": "blank_issues_enabled: false\n"
},
{
"path": ".github/PULL_REQUEST_TEMPLATE.md",
"content": "# PR type\n- [ ] Bug Fix\n- [ ] New Feature\n- [ ] Document Updates\n- [ ] More Models or Datasets Support\n\n# PR information\n\nWrite the detail information belongs to this PR.\n\n## Experiment results\n\nPaste your experiment result here(if needed).\n"
},
{
"path": ".github/SECURITY.md",
"content": "# Reporting Security Issues\n\nUsually security issues of a deep learning project come from non-standard 3rd packages or continuous running services. If you are suffering from security issues from our project, please consider reporting to us. We appreciate your efforts to responsibly disclose your findings, and will make every effort to acknowledge your contributions.\n"
},
{
"path": ".github/workflows/citest.yaml",
"content": "name: citest\n\non:\n push:\n branches:\n - master\n - \"release/**\"\n paths-ignore:\n - \"setup.*\"\n - \"requirements.txt\"\n - \"requirements/**\"\n - \"docs/**\"\n - \"tools/**\"\n - \".dev_scripts/**\"\n - \"README.md\"\n - \"README_*.md\"\n - \"NOTICE\"\n - \".github/workflows/lint.yaml\"\n - \".github/workflows/publish.yaml\"\n\n pull_request:\n paths-ignore:\n - \"setup.*\"\n - \"requirements.txt\"\n - \"requirements/**\"\n - \"docs/**\"\n - \"tools/**\"\n - \".dev_scripts/**\"\n - \"README.md\"\n - \"README_*.md\"\n - \"NOTICE\"\n - \".github/workflows/lint.yaml\"\n - \".github/workflows/publish.yaml\"\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}\n cancel-in-progress: true\n\njobs:\n unittest:\n # The type of runner that the job will run on\n runs-on: [self-hosted]\n timeout-minutes: 240\n steps:\n - name: ResetFileMode\n shell: bash\n run: |\n # reset filemode to allow action runner to delete files\n # generated by root in docker\n set -e\n source ~/.bashrc\n sudo chown -R $USER:$USER $GITHUB_WORKSPACE\n\n - name: Checkout\n uses: actions/checkout@v3\n env:\n GIT_CONFIG_PARAMETERS: \"'core.hooksPath='\"\n with:\n lfs: 'true'\n submodules: 'false'\n fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}\n - name: Get changed files\n id: changed-files\n run: |\n if ${{ github.event_name == 'pull_request' }}; then\n echo \"PR_CHANGED_FILES=$(git diff --name-only -r HEAD^1 HEAD | xargs)\" >> $GITHUB_ENV\n else\n echo \"PR_CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)\" >> $GITHUB_ENV\n fi\n - name: Checkout LFS objects\n run: git lfs checkout\n - name: Run unittest\n shell: bash\n run: |\n set -e\n source /mnt/modelscope/ci_env.sh\n bash .dev_scripts/dockerci.sh\n"
},
{
"path": ".github/workflows/citest_npu.yaml",
"content": "name: citest-npu\n\non:\n push:\n branches:\n - master\n - \"release/**\"\n paths-ignore:\n - \"setup.*\"\n - \"requirements.txt\"\n - \"requirements/**\"\n - \"docs/**\"\n - \"tools/**\"\n - \".dev_scripts/**\"\n - \"README.md\"\n - \"README_*.md\"\n - \"NOTICE\"\n - \".github/workflows/lint.yaml\"\n - \".github/workflows/publish.yaml\"\n\n pull_request:\n paths-ignore:\n - \"setup.*\"\n - \"requirements.txt\"\n - \"requirements/**\"\n - \"docs/**\"\n - \"tools/**\"\n - \".dev_scripts/**\"\n - \"README.md\"\n - \"README_*.md\"\n - \"NOTICE\"\n - \".github/workflows/lint.yaml\"\n - \".github/workflows/publish.yaml\"\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}\n cancel-in-progress: true\n\njobs:\n unittest:\n # The type of runner that the job will run on\n runs-on: [linux-aarch64-a2-1]\n timeout-minutes: 240\n container:\n image: 'ascendai/cann:8.3.rc2-910b-ubuntu22.04-py3.11'\n steps:\n - name: Config mirrors\n run: |\n sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list\n pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple\n pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local\n\n - name: Checkout\n uses: actions/checkout@v3\n with:\n fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}\n - name: Get changed files\n id: changed-files\n run: |\n if ${{ github.event_name == 'pull_request' }}; then\n echo \"PR_CHANGED_FILES=$(git diff --name-only -r HEAD^1 HEAD | xargs)\" >> $GITHUB_ENV\n else\n echo \"PR_CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)\" >> $GITHUB_ENV\n fi\n - name: Run unittest\n shell: bash\n run: |\n set -e\n export IMAGE_NAME=ascendai/cann\n export IMAGE_VERSION=8.3.rc2-910b-ubuntu22.04-py3.11\n export TEST_LEVEL=0\n mkdir -p ~/.cache\n export MODELSCOPE_CACHE=~/.cache\n export CI_COMMAND='bash .dev_scripts/ci_container_test.sh python tests/run.py --parallel 2 --subprocess --run_config tests/run_config.yaml'\n bash .dev_scripts/dockerci_npu.sh\n"
},
{
"path": ".github/workflows/close_tale_issue.yaml",
"content": "name: Close Stale Issues\non:\n schedule:\n - cron: '0 0 * * *'\n workflow_dispatch:\n\njobs:\n close-stale:\n runs-on: ubuntu-latest\n steps:\n - name: Close stale issues\n uses: actions/stale@v8\n with:\n repo-token: ${{ secrets.GITHUB_TOKEN }}\n days-before-stale: 90\n days-before-close: 7\n stale-issue-message: 'This issue has been inactive for over 3 months and will be automatically closed in 7 days. If this issue is still relevant, please reply to this message.'\n close-issue-message: 'This issue has been automatically closed due to inactivity. If needed, it can be reopened.'\n stale-issue-label: 'stale'\n exempt-all-issue-labels: true\n"
},
{
"path": ".github/workflows/lint.yaml",
"content": "name: Lint test\n\non: [push, pull_request]\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}\n cancel-in-progress: true\n\njobs:\n lint:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v2\n - name: Set up Python 3.10\n uses: actions/setup-python@v2\n with:\n python-version: '3.10'\n - name: Install pre-commit hook\n run: |\n pip install pre-commit\n - name: Linting\n run: pre-commit run --all-files\n"
},
{
"path": ".github/workflows/publish.yaml",
"content": "name: release\n\non:\n push:\n tags:\n - 'v**'\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}-publish\n cancel-in-progress: true\n\njobs:\n build-n-publish:\n runs-on: ubuntu-22.04\n #if: startsWith(github.event.ref, 'refs/tags')\n steps:\n - uses: actions/checkout@v2\n - name: Set up Python 3.10\n uses: actions/setup-python@v2\n with:\n python-version: '3.10'\n - name: Install wheel\n run: pip install wheel packaging setuptools==69.5.1\n - name: Build ModelScope Swift\n run: python setup.py sdist bdist_wheel\n - name: Publish package to PyPI\n run: |\n pip install twine\n twine upload dist/* --skip-existing -u __token__ -p ${{ secrets.PYPI_API_TOKEN }}\n"
},
{
"path": ".gitignore",
"content": "# Byte-compiled / optimized / DLL files\ntmp\n*.ttf\n__pycache__/\n*.py[cod]\n*$py.class\ntest.py\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\n*.egg-info/\n.installed.cfg\n*.egg\n/package\n/temp\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# pyenv\n.python-version\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n\n.vscode\n.idea\n.run\n\n# custom\n*.pkl\n*.pkl.json\n*.log.json\n*.whl\n*.tar.gz\n*.swp\n*.log\n*.tar.gz\nsource.sh\ntensorboard.sh\n.DS_Store\nreplace.sh\nresult.png\nresult.jpg\nresult.mp4\noutput/\noutputs/\nwandb/\nswanlog/\n*.out\nbenchmarks/\neval_output/\neval_outputs/\nvlmeval/\nmy_model/\n/data\nresult/\nimages\n/custom/\nmegatron_output/\n/*-mcore/\n/*-hf/\n/*_cached_dataset/\n/sample_output/\n\n# Pytorch\n*.pth\n*.pt\n\n# ast template\nast_index_file.py\n"
},
{
"path": ".pre-commit-config.yaml",
"content": "repos:\n - repo: https://github.com/pycqa/flake8.git\n rev: 7.3.0\n hooks:\n - id: flake8\n - repo: https://github.com/PyCQA/isort.git\n rev: 8.0.0\n hooks:\n - id: isort\n - repo: https://github.com/pre-commit/mirrors-yapf.git\n rev: v0.32.0\n hooks:\n - id: yapf\n - repo: https://github.com/pre-commit/pre-commit-hooks.git\n rev: v6.0.0\n hooks:\n - id: trailing-whitespace\n - id: check-yaml\n - id: end-of-file-fixer\n - id: requirements-txt-fixer\n - id: double-quote-string-fixer\n - id: check-merge-conflict\n - id: mixed-line-ending\n args: [\"--fix=lf\"]\n"
},
{
"path": ".pre-commit-config_local.yaml",
"content": "repos:\n - repo: /home/admin/pre-commit/flake8\n rev: 7.3.0\n hooks:\n - id: flake8\n - repo: /home/admin/pre-commit/isort\n rev: 8.0.0\n hooks:\n - id: isort\n - repo: /home/admin/pre-commit/mirrors-yapf\n rev: v0.32.0\n hooks:\n - id: yapf\n - repo: /home/admin/pre-commit/pre-commit-hooks\n rev: v6.0.0\n hooks:\n - id: trailing-whitespace\n - id: check-yaml\n - id: end-of-file-fixer\n - id: requirements-txt-fixer\n - id: double-quote-string-fixer\n - id: check-merge-conflict\n - id: mixed-line-ending\n args: [\"--fix=lf\"]\n"
},
{
"path": "CODE_OF_CONDUCT.md",
"content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nWe as members, contributors, and leaders pledge to make participation in our\ncommunity a harassment-free experience for everyone, regardless of age, body\nsize, visible or invisible disability, ethnicity, sex characteristics, gender\nidentity and expression, level of experience, education, socio-economic status,\nnationality, personal appearance, race, caste, color, religion, or sexual\nidentity and orientation.\n\nWe pledge to act and interact in ways that contribute to an open, welcoming,\ndiverse, inclusive, and healthy community.\n\n## Our Standards\n\nExamples of behavior that contributes to a positive environment for our\ncommunity include:\n\n* Demonstrating empathy and kindness toward other people\n* Being respectful of differing opinions, viewpoints, and experiences\n* Giving and gracefully accepting constructive feedback\n* Accepting responsibility and apologizing to those affected by our mistakes,\n and learning from the experience\n* Focusing on what is best not just for us as individuals, but for the overall\n community\n\nExamples of unacceptable behavior include:\n\n* The use of sexualized language or imagery, and sexual attention or advances of\n any kind\n* Trolling, insulting or derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or email address,\n without their explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Enforcement Responsibilities\n\nCommunity leaders are responsible for clarifying and enforcing our standards of\nacceptable behavior and will take appropriate and fair corrective action in\nresponse to any behavior that they deem inappropriate, threatening, offensive,\nor harmful.\n\nCommunity leaders have the right and responsibility to remove, edit, or reject\ncomments, commits, code, wiki edits, issues, and other contributions that are\nnot aligned to this Code of Conduct, and will communicate reasons for moderation\ndecisions when appropriate.\n\n## Scope\n\nThis Code of Conduct applies within all community spaces, and also applies when\nan individual is officially representing the community in public spaces.\nExamples of representing our community include using an official e-mail address,\nposting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported to the community leaders responsible for enforcement at\ncontact@modelscope.cn.\nAll complaints will be reviewed and investigated promptly and fairly.\n\nAll community leaders are obligated to respect the privacy and security of the\nreporter of any incident.\n\n## Enforcement Guidelines\n\nCommunity leaders will follow these Community Impact Guidelines in determining\nthe consequences for any action they deem in violation of this Code of Conduct:\n\n### 1. Correction\n\n**Community Impact**: Use of inappropriate language or other behavior deemed\nunprofessional or unwelcome in the community.\n\n**Consequence**: A private, written warning from community leaders, providing\nclarity around the nature of the violation and an explanation of why the\nbehavior was inappropriate. A public apology may be requested.\n\n### 2. Warning\n\n**Community Impact**: A violation through a single incident or series of\nactions.\n\n**Consequence**: A warning with consequences for continued behavior. No\ninteraction with the people involved, including unsolicited interaction with\nthose enforcing the Code of Conduct, for a specified period of time. This\nincludes avoiding interactions in community spaces as well as external channels\nlike social media. Violating these terms may lead to a temporary or permanent\nban.\n\n### 3. Temporary Ban\n\n**Community Impact**: A serious violation of community standards, including\nsustained inappropriate behavior.\n\n**Consequence**: A temporary ban from any sort of interaction or public\ncommunication with the community for a specified period of time. No public or\nprivate interaction with the people involved, including unsolicited interaction\nwith those enforcing the Code of Conduct, is allowed during this period.\nViolating these terms may lead to a permanent ban.\n\n### 4. Permanent Ban\n\n**Community Impact**: Demonstrating a pattern of violation of community\nstandards, including sustained inappropriate behavior, harassment of an\nindividual, or aggression toward or disparagement of classes of individuals.\n\n**Consequence**: A permanent ban from any sort of public interaction within the\ncommunity.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage],\nversion 2.1, available at\n[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].\n\nCommunity Impact Guidelines were inspired by\n[Mozilla's code of conduct enforcement ladder][Mozilla CoC].\n\nFor answers to common questions about this code of conduct, see the FAQ at\n[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at\n[https://www.contributor-covenant.org/translations][translations].\n\n[homepage]: https://www.contributor-covenant.org\n[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html\n[Mozilla CoC]: https://github.com/mozilla/diversity\n[FAQ]: https://www.contributor-covenant.org/faq\n[translations]: https://www.contributor-covenant.org/translations\n"
},
{
"path": "CONTRIBUTING.md",
"content": "# Contributor Guide\n\n_Welcome to offer PRs, bug reports, documentation supplements or other types of contributions to SWIFT!_\n\n## Table of Contents\n- [Code of Conduct](#-code-of-conduct)\n- [Contribution Process](#-contribution-process)\n- [Hardware support](#-Hardware-support)\n\n## 📖 Code of Conduct\nPlease refer to our [Code of Conduct documentation](./CODE_OF_CONDUCT.md).\n\n## 🔁 Contribution Process\n### What We Need\n- New Technologies and New Models: SWIFT needs to support more open-source models and datasets, or new technologies that we have not paid attention to. If you are interested please submit a PR to us.\n- Technical Propagation: If you are interested in technical propagation, you are welcome to help us write tutorials, documents or videos on any website, and send us the link.\n- Community Contribution: You can write technical articles related to SWIFT, and submit them to us. After review and approval, we will publish them on the official ModelScope accounts (Zhihu, WeChat, etc.), with your name assigned.\n\n### Incentives\n- we will issue electronic certificates to contributors on behalf of the ModelScope community, to encourage your selfless contributions.\n- We will offer small souvenirs related to the ModelScope Community.\n- We will provide free A10 computing power during the development period. For more details, please refer to [Hardware-support](#-Hardware-support) section.\n\n### Submitting PR (Pull Requests)\n\nAny feature development is carried out in the form of Fork and then PR on GitHub.\n1. Fork: Go to the [ms-swift](https://github.com/modelscope/ms-swift) page and click the **Fork button**. After completion, a SWIFT code repository will be cloned under your personal organization.\n2. Clone: Clone the code repository generated in the first step to your local machine and **create a new branch** for development. During development, please click the **Sync Fork button** in time to synchronize with the `main` branch to prevent code expiration and conflicts.\n3. Submit PR: After development and testing, push the code to the remote branch. On GitHub, go to the **Pull Requests page**, create a new PR, select your code branch as the source branch, and the `modelscope/ms-swift:main` branch as the target branch.\n\n4. Write Description: It is necessary to provide a good feature description in the PR, so that the reviewers know the content of your modification.\n5. Review: We hope that the code to be merged is concise and efficient, so we may raise some questions and discuss them. Please note that any issues raised in the review are aimed at the code itself, not at you personally. Once all issues are discussed and resolved, your code will be approved.\n\n### Code Standards and Development Approach\nSWIFT has conventional variable naming conventions and development approaches. Please follow these approaches as much as possible during development.\n1. Variable names are separated by underscores, and class names are named with the first letter of each word capitalized.\n2. All Python indentation uses four spaces instead of a tab.\n3. Choose well-known open-source libraries, avoid using closed-source libraries or unstable open-source libraries, and avoid repeating the existing code.\n\nAfter the PR is submitted, SWIFT will perform two types of tests:\n- Code Lint Test: A static code compliance check test. please make sure that you have performed code lint locally in advance.\n```shell\npip install pre-commit # In the swift folder\npre-commit run --all-files # Fix the errors reported by pre-commit until all checks are successful\n```\n- CI Tests: Smoke tests and unit tests, please refer to the next section.\n\n### Running CI Tests\nBefore submitting the PR, please ensure that your development code is protected by test cases, such as smoke tests for new features, or unit tests for various edge cases. Reviewers will also pay attention to this during code review. At the same time, there will be dedicated services running CI Tests, running all test cases, and the code can only be merged after the test cases pass.\n\n## ✅ Hardware support\n\nSWIFT will provide hardware support for developers, including free GPUs. If needed, please email us ([contact@modelscope.cn](mailto:contact@modelscope.cn)) or join our WeChat group:\n\n\n
\n"
},
{
"path": "CONTRIBUTING_CN.md",
"content": "# 贡献者指引\n\n*欢迎帮SWIFT提供Feature PR、Bug反馈、文档补充或其他类型的贡献！*\n\n## 目录\n\n- [代码规约](#-代码规约)\n- [贡献流程](#-贡献流程)\n- [资源支持](#-资源支持)\n\n## 📖 代码规约\n\n请查看我们的[代码规约文档](./CODE_OF_CONDUCT.md).\n\n## 🔁 贡献流程\n\n### 我们需要什么\n- 新技术和新模型：SWIFT需要支持更多的开源模型和数据集，或我们没有关注到的新技术，如果您对此有兴趣，可以提交PR给我们。\n- 技术布道：如果您对技术布道有兴趣，欢迎在任何网站上帮我们撰写教程文档或视频等，并将链接发给我们。\n- 社区供稿：您可以撰写和SWIFT有关的技术文章，并供稿给我们，我们审核通过后会在魔搭官方账号（知乎、公众号等）上进行发布，并属上您的名字。\n\n### 激励\n\n- 我们会以魔搭社区的身份给贡献者颁发电子证书，以鼓励您的无私贡献。\n- 我们会赠送相关魔搭社区相关周边小礼品。\n- 我们会赠送开发期间的免费A10算力，具体可以查看[资源支持](#-资源支持)章节。\n\n### 提交PR（Pull Requests）\n\n任何feature开发都在github上以先Fork后PR的形式进行。\n\n1. Fork：进入[ms-swift](https://github.com/modelscope/ms-swift)页面后，点击**Fork按钮**执行。完成后会在您的个人组织下克隆出一个SWIFT代码库\n\n2. Clone：将第一步产生的代码库clone到本地并**拉新分支**进行开发，开发中请及时点击**Sync Fork按钮**同步`main`分支，防止代码过期并冲突\n\n3. 提交PR：开发、测试完成后将代码推送到远程分支。在github上点击**Pull Requests页面**，新建一个PR，源分支选择您提交的代码分支，目标分支选择`modelscope/ms-swift:main`分支\n\n4. 撰写描述：在PR中填写良好的feature描述是必要的，让Reviewers知道您的修改内容\n\n5. Review：我们希望合入的代码简洁高效，因此可能会提出一些问题并讨论。请注意，任何review中提出的问题是针对代码本身，而非您个人。在所有问题讨论通过后，您的代码会被通过\n\n### 代码规范和开发方式\n\nSWIFT有约定俗成的变量命名方式和开发方式。在开发中请尽量遵循这些方式。\n\n1. 变量命名以下划线分割，类名以所有单词首字母大写方式命名\n2. 所有的python缩进都是四个空格取代一个tab\n3. 选用知名的开源库，避免使用闭源库或不稳定的开源库，避免重复造轮子\n\nSWIFT在PR提交后会进行两类测试：\n\n- Code Lint测试 对代码进行静态规范走查的测试，为保证改测试通过，请保证本地预先进行了Code lint。方法是：\n\n ```shell\n pip install pre-commit\n # 在swift文件夹内\n pre-commit run --all-files\n # 对pre-commit报的错误进行修改，直到所有的检查都是成功状态\n ```\n\n- CI Tests 冒烟测试和单元测试，请查看下一章节\n\n### Running CI Tests\n\n在提交PR前，请保证您的开发代码已经受到了测试用例的保护。例如，对新功能的冒烟测试，或者各种边缘case的单元测试等。在代码review时Reviewers也会关注这一点。同时，也会有服务专门运行CI Tests，运行所有的测试用例，测试用例通过后代码才可以合并。\n\n## ✅ 资源支持\n\nSWIFT会为开发者提供资源支持，包括免费的GPU算力。如果需要请邮件联系我们（[contact@modelscope.cn](mailto:contact@modelscope.cn)）或加入我们的微信群：\n\n\n
\n"
},
{
"path": "LICENSE",
"content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n"
},
{
"path": "MANIFEST.in",
"content": "recursive-include requirements *.txt\n"
},
{
"path": "Makefile",
"content": "WHL_BUILD_DIR :=package\nDOC_BUILD_DIR :=docs/build/\n\n# default rule\ndefault: whl docs\n\n.PHONY: docs\ndocs:\n\tbash .dev_scripts/build_docs.sh\n\n.PHONY: linter\nlinter:\n\tbash .dev_scripts/linter.sh\n\n.PHONY: test\ntest:\n\tbash .dev_scripts/citest.sh\n\n.PHONY: whl\nwhl:\n\tpython setup.py sdist bdist_wheel\n\n.PHONY: clean\nclean:\n\trm -rf $(WHL_BUILD_DIR) $(DOC_BUILD_DIR)\n"
},
{
"path": "README.md",
"content": "# SWIFT (Scalable lightWeight Infrastructure for Fine-Tuning)\n\n\n
\n
\nModelScope Community Website \n中文 ｜ English \n
\n\n\n
\n\n\n
\n\n\n Paper ｜ English Documentation ｜ 中文文档 \n
\n\n## 📖 Table of Contents\n- [Groups](#-Groups)\n- [Introduction](#-introduction)\n- [News](#-news)\n- [Installation](#%EF%B8%8F-installation)\n- [Quick Start](#-quick-Start)\n- [Usage](#-Usage)\n- [License](#-License)\n- [Citation](#-citation)\n\n\n## ☎ Groups\n\nYou can contact us and communicate with us by adding our group:\n\n\n[Discord Group](https://discord.com/invite/D27yfEFVz5) | WeChat Group\n:-------------------------:|:-------------------------:\nMore \n\n- 🎁 2025.06.11: Support for using Megatron parallelism techniques for RLHF training. The training script can be found [here](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/rlhf).\n- 🎁 2025.05.29: Support sequence parallel in pretrain, sft, dpo and grpo, check script [here](https://github.com/modelscope/ms-swift/tree/main/examples/train/sequence_parallel).\n- 🎁 2025.05.11: GRPO now supports custom processing logic for reward models. See the GenRM example [here](./docs/source_en/Instruction/GRPO/DeveloperGuide/reward_model.md).\n- 🎁 2025.04.15: The ms-swift paper has been accepted by AAAI 2025. You can find the paper at [this link](https://ojs.aaai.org/index.php/AAAI/article/view/35383).\n- 🎁 2025.03.23: Multi-round GRPO is now supported for training multi-turn dialogue scenarios (e.g., agent tool calling). Please refer to the [doc](./docs/source_en/Instruction/GRPO/DeveloperGuide/multi_turn.md).\n- 🎁 2025.03.16: Support for Megatron's parallel training techniques is now available. Please see the [Megatron-SWIFT training documentation](https://swift.readthedocs.io/en/latest/Megatron-SWIFT/Quick-start.html).\n- 🎁 2025.03.15: Fine-tuning of embedding models for both pure text and multimodal models is supported. Please check the [training script](examples/train/embedding).\n- 🎁 2025.03.05: The hybrid mode for GRPO is supported, with a script for training a 72B model on 4 GPUs (4*80G) available [here](examples/train/grpo/internal/vllm_72b_4gpu.sh). Tensor parallelism with vllm is also supported, with the training script available [here](examples/train/grpo/internal).\n- 🎁 2025.02.21: The GRPO algorithm now supports LMDeploy, with the training script available [here](examples/train/grpo/internal/full_lmdeploy.sh). Additionally, the performance of the GRPO algorithm has been tested, achieving a training speed increase of up to 300% using various tricks. Please check the WanDB table [here](https://wandb.ai/tastelikefeet/grpo_perf_test?nw=nwuseryuzezyz).\n- 🎁 2025.02.21: The `swift sample` command is now supported. The reinforcement fine-tuning script can be found [here](docs/source_en/Instruction/Reinforced-Fine-tuning.md), and the large model API distillation sampling script is available [here](examples/sampler/distill/distill.sh).\n- 🔥 2025.02.12: Support for the GRPO (Group Relative Policy Optimization) training algorithm has been added. Documentation is available [here](docs/source_en/Instruction/GRPO/GetStarted/GRPO.md).\n- 🎁 2024.12.04: Major update to **ms-swift 3.0**. Please refer to the [release notes and changes](docs/source_en/Instruction/ReleaseNote3.0.md).\n- 🎉 2024.08.12: The ms-swift paper has been published on arXiv and can be read [here](https://arxiv.org/abs/2408.05517).\n- 🔥 2024.08.05: Support for using [evalscope](https://github.com/modelscope/evalscope/) as a backend for evaluating large models and multimodal models.\n- 🔥 2024.07.29: Support for using [vllm](https://github.com/vllm-project/vllm) and [lmdeploy](https://github.com/InternLM/lmdeploy) to accelerate inference for large models and multimodal models. When performing infer/deploy/eval, you can specify `--infer_backend vllm/lmdeploy`.\n- 🔥 2024.07.24: Support for human preference alignment training for multimodal large models, including DPO/ORPO/SimPO/CPO/KTO/RM/PPO.\n- 🔥 2024.02.01: Support for Agent training! The training algorithm is derived from [this paper](https://arxiv.org/pdf/2309.00986.pdf).\n`.\n- The `--model_author` and `--model_name` parameters are only effective when the dataset includes `swift/self-cognition`.\n- To train with a different model, simply modify `--model `.\n- By default, **ModelScope** is used for downloading models and datasets. If you want to use HuggingFace, simply specify `--use_hf true`.\n\nAfter training is complete, use the following command to infer with the trained weights:\n\n- Here, `--adapters` should be replaced with the last checkpoint folder generated during training. Since the adapters folder contains the training parameter file `args.json`, there is no need to specify `--model`, `--system` separately; Swift will automatically read these parameters. To disable this behavior, you can set `--load_args false`.\n\n```shell\n# Using an interactive command line for inference.\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --temperature 0 \\\n --max_new_tokens 2048\n\n# merge-lora and use vLLM for inference acceleration\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --merge_lora true \\\n --infer_backend vllm \\\n --vllm_max_model_len 8192 \\\n --temperature 0 \\\n --max_new_tokens 2048\n```\n\nFinally, use the following command to push the model to ModelScope:\n\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift export \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --push_to_hub true \\\n --hub_model_id '' \\\n --hub_token '' \\\n --use_hf false\n```\n\n\n### Web-UI\nThe Web-UI is a **zero-threshold** training and deployment interface solution based on Gradio interface technology. For more details, you can check [here](https://swift.readthedocs.io/en/latest/GetStarted/Web-UI.html).\n\n```shell\nSWIFT_UI_LANG=en swift web-ui\n```\n\n\n\n### Using Python\n\nms-swift also supports training and inference using Python. Below is pseudocode for training and inference. For more details, you can refer to [here](https://github.com/modelscope/ms-swift/blob/main/examples/notebook/qwen2_5-self-cognition/self-cognition-sft.ipynb).\n\nTraining:\n\n```python\nfrom peft import LoraConfig, get_peft_model\nfrom swift import get_model_processor, get_template, load_dataset, EncodePreprocessor\nfrom swift.trainers import Seq2SeqTrainer, Seq2SeqTrainingArguments\n# Retrieve the model and template, and add a trainable LoRA module\nmodel, tokenizer = get_model_processor(model_id_or_path, ...)\ntemplate = get_template(tokenizer, ...)\nlora_config = LoraConfig(...)\nmodel = get_peft_model(model, lora_config)\n\n# Download and load the dataset, and encode the text into tokens\ntrain_dataset, val_dataset = load_dataset(dataset_id_or_path, ...)\ntrain_dataset = EncodePreprocessor(template=template)(train_dataset, num_proc=num_proc)\nval_dataset = EncodePreprocessor(template=template)(val_dataset, num_proc=num_proc)\n\n# Train the model\ntraining_args = Seq2SeqTrainingArguments(...)\ntrainer = Seq2SeqTrainer(\n model=model,\n args=training_args,\n template=template,\n train_dataset=train_dataset,\n eval_dataset=val_dataset,\n)\ntrainer.train()\n```\nInference:\n\n```python\nfrom swift import TransformersEngine, InferRequest, RequestConfig\n# Perform inference using the native Transformers engine\nengine = TransformersEngine(model_id_or_path, adapters=[lora_checkpoint])\ninfer_request = InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}])\nrequest_config = RequestConfig(max_tokens=max_new_tokens, temperature=temperature)\n\nresp_list = engine.infer([infer_request], request_config)\nprint(f'response: {resp_list[0].choices[0].message.content}')\n```\n\n## ✨ Usage\nHere is a minimal example of training to deployment using ms-swift. For more details, you can check the [examples](https://github.com/modelscope/ms-swift/tree/main/examples).\n\n- If you want to use other models or datasets (including multimodal models and datasets), you only need to modify `--model` to specify the corresponding model's ID or path, and modify `--dataset` to specify the corresponding dataset's ID or path.\n- By default, ModelScope is used for downloading models and datasets. If you want to use HuggingFace, simply specify `--use_hf true`.\n\n| Useful Links |\n| ------ |\n| [🔥Command Line Parameters](https://swift.readthedocs.io/en/latest/Instruction/Command-line-parameters.html) |\n| [Megatron-SWIFT](https://swift.readthedocs.io/en/latest/Megatron-SWIFT/Quick-start.html) |\n| [GRPO](https://swift.readthedocs.io/en/latest/Instruction/GRPO/GetStarted/GRPO.html) |\n| [Supported Models and Datasets](https://swift.readthedocs.io/en/latest/Instruction/Supported-models-and-datasets.html) |\n| [Custom Models](https://swift.readthedocs.io/en/latest/Customization/Custom-model.html), [🔥Custom Datasets](https://swift.readthedocs.io/en/latest/Customization/Custom-dataset.html) |\n| [LLM Tutorial](https://github.com/modelscope/modelscope-classroom/tree/main/LLM-tutorial) |\n\n### Training\n\nSupported Training Methods:\n\n| Method | Full-Parameter | LoRA | QLoRA | Deepspeed | Multi-Machine | Multimodal |\n| ------------------------------------------------------------ | ------------------------------------------------------------ | ---- | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ |\n| [Pre-training](https://github.com/modelscope/ms-swift/blob/main/examples/train/pretrain) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [Supervised Fine-Tuning](https://github.com/modelscope/ms-swift/blob/main/examples/train/lora_sft.sh) | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/full/train.sh) | ✅ | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/qlora) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-gpu/deepspeed) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-node) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multimodal) |\n| [GRPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [GKD](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/gkd) | ✅ | ✅ | ✅ | ✅ | ✅ | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/gkd) |\n| [PPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/ppo) | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |\n| [DPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/dpo) | ✅ | ✅ | ✅ | ✅ | ✅ | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/dpo) |\n| [KTO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/kto.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/kto.sh) |\n| [Reward Model](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/rm.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [CPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/cpo.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [SimPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/simpo.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [ORPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/orpo.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [Embedding](https://github.com/modelscope/ms-swift/blob/main/examples/train/embedding) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [Reranker](https://github.com/modelscope/ms-swift/tree/main/examples/train/reranker) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [Sequence Classification](https://github.com/modelscope/ms-swift/blob/main/examples/train/seq_cls) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n\n\nPre-training:\n```shell\n# 8*A100\nNPROC_PER_NODE=8 \\\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \\\nswift pt \\\n --model Qwen/Qwen2.5-7B \\\n --dataset swift/chinese-c4 \\\n --streaming true \\\n --tuner_type full \\\n --deepspeed zero2 \\\n --output_dir output \\\n --max_steps 10000 \\\n ...\n```\n\nFine-tuning:\n```shell\nCUDA_VISIBLE_DEVICES=0 swift sft \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --dataset AI-ModelScope/alpaca-gpt4-data-en \\\n --tuner_type lora \\\n --output_dir output \\\n ...\n```\n\nRLHF:\n```shell\nCUDA_VISIBLE_DEVICES=0 swift rlhf \\\n --rlhf_type dpo \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --dataset hjh0119/shareAI-Llama3-DPO-zh-en-emoji \\\n --tuner_type lora \\\n --output_dir output \\\n ...\n```\n\n\n### Megatron-SWIFT\n\nms-swift supports using Megatron parallelism techniques to accelerate training, including large-scale cluster training and MoE model training. The following training methods are supported:\n\n| Method | Full-Parameter | LoRA | MoE | Multimodal | FP8 |\n| ---------------------- | -------------- | ---- | ---- | ---------- | ---- |\n| Pre-training | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [Supervised Fine-Tuning](https://github.com/modelscope/ms-swift/tree/main/examples/megatron) | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [GRPO](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/grpo) | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [GKD](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/rlhf/gkd) | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [DPO](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/rlhf/dpo) | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [KTO](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/rlhf/kto) | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [RM](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/rlhf/rm) | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [Embedding](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/embedding) | ✅ | ✅| ✅ | ✅ | ✅ |\n| [Reranker](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/reranker) | ✅ | ✅| ✅ | ✅ | ✅ |\n| [Sequence Classification](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/seq_cls) | ✅ | ✅ | ✅ | ✅ | ✅ |\n\n\n```shell\nNPROC_PER_NODE=2 CUDA_VISIBLE_DEVICES=0,1 megatron sft \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --save_safetensors true \\\n --dataset AI-ModelScope/alpaca-gpt4-data-zh \\\n --tuner_type lora \\\n --output_dir output \\\n ...\n```\n\n### Reinforcement Learning\n\nms-swift supports a rich set of GRPO family algorithms:\n\n| Method | Full-Parameter | LoRA | Multimodal | Multi-Machine |\n| ------------------------------------------------------------ | -------------- | ---- | ---------- | ------------- |\n| [GRPO](https://swift.readthedocs.io/en/latest/Instruction/GRPO/GetStarted/GRPO.html) | ✅ | ✅ | ✅ | ✅ |\n| [DAPO](https://swift.readthedocs.io/en/latest/Instruction/GRPO/AdvancedResearch/DAPO.html) | ✅ | ✅ | ✅ | ✅ |\n| [GSPO](https://swift.readthedocs.io/en/latest/Instruction/GRPO/AdvancedResearch/GSPO.html) | ✅ | ✅ | ✅ | ✅ |\n| [SAPO](https://swift.readthedocs.io/en/latest/Instruction/GRPO/AdvancedResearch/SAPO.html) | ✅ | ✅ | ✅ | ✅ |\n| [CISPO](https://swift.readthedocs.io/en/latest/Instruction/GRPO/AdvancedResearch/CISPO.html) | ✅ | ✅ | ✅ | ✅ |\n| [CHORD](https://swift.readthedocs.io/en/latest/Instruction/GRPO/AdvancedResearch/CHORD.html) | ✅ | ✅ | ✅ | ✅ |\n| [RLOO](https://swift.readthedocs.io/en/latest/Instruction/GRPO/AdvancedResearch/RLOO.html) | ✅ | ✅ | ✅ | ✅ |\n| [Reinforce++](https://swift.readthedocs.io/en/latest/Instruction/GRPO/AdvancedResearch/REINFORCEPP.html) | ✅ | ✅ | ✅ | ✅ |\n\n```shell\nCUDA_VISIBLE_DEVICES=0,1,2,3 NPROC_PER_NODE=4 \\\nswift rlhf \\\n --rlhf_type grpo \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --tuner_type lora \\\n --use_vllm true \\\n --vllm_mode colocate \\\n --dataset AI-MO/NuminaMath-TIR#10000 \\\n --output_dir output \\\n ...\n```\n\n\n### Inference\n```shell\nCUDA_VISIBLE_DEVICES=0 swift infer \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --stream true \\\n --infer_backend transformers \\\n --max_new_tokens 2048\n\n# LoRA\nCUDA_VISIBLE_DEVICES=0 swift infer \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --adapters swift/test_lora \\\n --stream true \\\n --infer_backend transformers \\\n --temperature 0 \\\n --max_new_tokens 2048\n```\n\n### Interface Inference\n```shell\nCUDA_VISIBLE_DEVICES=0 swift app \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --stream true \\\n --infer_backend transformers \\\n --max_new_tokens 2048\n```\n\n### Deployment\n```shell\nCUDA_VISIBLE_DEVICES=0 swift deploy \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --infer_backend vllm\n```\n\n### Sampling\n```shell\nCUDA_VISIBLE_DEVICES=0 swift sample \\\n --model LLM-Research/Meta-Llama-3.1-8B-Instruct \\\n --sampler_engine transformers \\\n --num_return_sequences 5 \\\n --dataset AI-ModelScope/alpaca-gpt4-data-zh#5\n```\n\n### Evaluation\n```shell\nCUDA_VISIBLE_DEVICES=0 swift eval \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --infer_backend lmdeploy \\\n --eval_backend OpenCompass \\\n --eval_dataset ARC_c\n```\n\n### Quantization\n```shell\nCUDA_VISIBLE_DEVICES=0 swift export \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --quant_bits 4 --quant_method awq \\\n --dataset AI-ModelScope/alpaca-gpt4-data-zh \\\n --output_dir Qwen2.5-7B-Instruct-AWQ\n```\n\n### Push Model\n```shell\nswift export \\\n --model \\\n --push_to_hub true \\\n --hub_model_id '' \\\n --hub_token ''\n```\n\n## 🏛 License\n\nThis framework is licensed under the [Apache License (Version 2.0)](https://github.com/modelscope/ms-swift/blob/master/LICENSE). For models and datasets, please refer to the original resource page and follow the corresponding License.\n\n## 📎 Citation\n\n```bibtex\n@misc{zhao2024swiftascalablelightweightinfrastructure,\n title={SWIFT:A Scalable lightWeight Infrastructure for Fine-Tuning},\n author={Yuze Zhao and Jintao Huang and Jinghan Hu and Xingjun Wang and Yunlin Mao and Daoze Zhang and Zeyinzi Jiang and Zhikai Wu and Baole Ai and Ang Wang and Wenmeng Zhou and Yingda Chen},\n year={2024},\n eprint={2408.05517},\n archivePrefix={arXiv},\n primaryClass={cs.CL},\n url={https://arxiv.org/abs/2408.05517},\n}\n```\n\n## Star History\n\n[](https://star-history.com/#modelscope/ms-swift&Date)\n"
},
{
"path": "README_CN.md",
"content": "# SWIFT (Scalable lightWeight Infrastructure for Fine-Tuning)\n\n\n
\n
\n魔搭社区官网 \nEnglish \n
\n\n\n\n
\n\n\n
\n\n\n 论文 ｜ English Documentation ｜ 中文文档 \n
\n\n## 📖 目录\n- [用户群](#-用户群)\n- [简介](#-简介)\n- [新闻](#-新闻)\n- [安装](#%EF%B8%8F-安装)\n- [快速开始](#-快速开始)\n- [如何使用](#-如何使用)\n- [License](#-license)\n- [引用](#-引用)\n\n## ☎ 用户群\n\n请扫描下面的二维码来加入我们的交流群：\n\n[Discord Group](https://discord.com/invite/D27yfEFVz5) | 微信群\n:-------------------------:|:-------------------------:\n更多 \n\n- 🎁 2025.06.11: 支持使用Megatron并行技术进行RLHF训练，训练脚本参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/rlhf)。\n- 🎁 2025.05.29: 支持pt、sft、dpo、grpo的序列并行，具体请查看[脚本](https://github.com/modelscope/ms-swift/tree/main/examples/train/sequence_parallel)。\n- 🎁 2025.05.11: GRPO中的奖励模型支持自定义处理逻辑，GenRM的例子参考[这里](./docs/source/Instruction/GRPO/DeveloperGuide/reward_model.md)。\n- 🎁 2025.04.15: ms-swift论文已经被AAAI 2025接收，论文地址在[这里](https://ojs.aaai.org/index.php/AAAI/article/view/35383)。\n- 🎁 2025.03.23: 支持了多轮GRPO，用于构建多轮对话场景的训练(例如agent tool calling)，请查看[文档](docs/source/Instruction/GRPO/DeveloperGuide/multi_turn.md)。\n- 🎁 2025.03.16: 支持了Megatron的并行技术进行训练，请查看[Megatron-SWIFT训练文档](https://swift.readthedocs.io/zh-cn/latest/Megatron-SWIFT/Quick-start.html)。\n- 🎁 2025.03.15: 支持纯文本和多模态模型的embedding模型的微调，请查看[训练脚本](examples/train/embedding)。\n- 🎁 2025.03.05: 支持GRPO的hybrid模式，4GPU(4*80G)训练72B模型的脚本参考[这里](examples/train/grpo/internal/vllm_72b_4gpu.sh)。同时支持vllm的tensor并行，训练脚本参考[这里](examples/train/grpo/internal)。\n- 🎁 2025.02.21: GRPO算法支持使用LMDeploy，训练脚本参考[这里](examples/train/grpo/internal/full_lmdeploy.sh)。此外测试了GRPO算法的性能，使用一些tricks使训练速度提高到300%。WanDB表格请查看[这里](https://wandb.ai/tastelikefeet/grpo_perf_test?nw=nwuseryuzezyz)。\n- 🎁 2025.02.21: 支持`swift sample`命令。强化微调脚本参考[这里](docs/source/Instruction/Reinforced-Fine-tuning.md)，大模型API蒸馏采样脚本参考[这里](examples/sampler/distill/distill.sh)。\n- 🔥 2025.02.12: 支持GRPO (Group Relative Policy Optimization) 训练算法，文档参考[这里](docs/source/Instruction/GRPO/GetStarted/GRPO.md)。\n- 🎁 2024.12.04: **ms-swift3.0**大版本更新。请查看[发布说明和更改](docs/source/Instruction/ReleaseNote3.0.md)。\n- 🎉 2024.08.12: ms-swift论文已经发布到arXiv上，可以点击[这里](https://arxiv.org/abs/2408.05517)阅读。\n- 🔥 2024.08.05: 支持使用[evalscope](https://github.com/modelscope/evalscope/)作为后端进行大模型和多模态模型的评测。\n- 🔥 2024.07.29: 支持使用[vllm](https://github.com/vllm-project/vllm), [lmdeploy](https://github.com/InternLM/lmdeploy)对大模型和多模态大模型进行推理加速，在infer/deploy/eval时额外指定`--infer_backend vllm/lmdeploy`即可。\n- 🔥 2024.07.24: 支持对多模态大模型进行人类偏好对齐训练，包括DPO/ORPO/SimPO/CPO/KTO/RM/PPO。\n- 🔥 2024.02.01: 支持Agent训练！训练算法源自这篇[论文](https://arxiv.org/pdf/2309.00986.pdf)。\n`。\n- `--model_author`和`--model_name`参数只有当数据集中包含`swift/self-cognition`时才生效。\n- 如果要使用其他模型进行训练，你只需要修改`--model `即可。\n- 默认使用**ModelScope**进行模型和数据集的下载。如果要使用HuggingFace，指定`--use_hf true`即可。\n\n训练完成后，使用以下命令对训练后的权重进行推理：\n- 这里的`--adapters`需要替换成训练生成的last checkpoint文件夹。由于adapters文件夹中包含了训练的参数文件`args.json`，因此不需要额外指定`--model`，`--system`，swift会自动读取这些参数。如果要关闭此行为，可以设置`--load_args false`。\n\n```shell\n# 使用交互式命令行进行推理\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --temperature 0 \\\n --max_new_tokens 2048\n\n# merge-lora并使用vLLM进行推理加速\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --merge_lora true \\\n --infer_backend vllm \\\n --vllm_max_model_len 8192 \\\n --temperature 0 \\\n --max_new_tokens 2048\n```\n\n最后，使用以下命令将模型推送到ModelScope：\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift export \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --push_to_hub true \\\n --hub_model_id '' \\\n --hub_token '' \\\n --use_hf false\n```\n\n### Web-UI\n\nWeb-UI是基于gradio界面技术的**零门槛**训练、部署界面方案，具体可以查看[这里](https://swift.readthedocs.io/zh-cn/latest/GetStarted/Web-UI.html)。\n\n```shell\nswift web-ui\n```\n\n\n### 使用Python\nms-swift也支持使用python的方式进行训练和推理。下面给出训练和推理的**伪代码**，具体可以查看[这里](https://github.com/modelscope/ms-swift/blob/main/examples/notebook/qwen2_5-self-cognition/self-cognition-sft.ipynb)。\n\n训练：\n```python\nfrom peft import LoraConfig, get_peft_model\nfrom swift import get_model_processor, get_template, load_dataset, EncodePreprocessor\nfrom swift.trainers import Seq2SeqTrainer, Seq2SeqTrainingArguments\n# 获取模型和template，并加入可训练的LoRA模块\nmodel, tokenizer = get_model_processor(model_id_or_path, ...)\ntemplate = get_template(tokenizer, ...)\nlora_config = LoraConfig(...)\nmodel = get_peft_model(model, lora_config)\n\n# 下载并载入数据集，并将文本encode成tokens\ntrain_dataset, val_dataset = load_dataset(dataset_id_or_path, ...)\ntrain_dataset = EncodePreprocessor(template=template)(train_dataset, num_proc=num_proc)\nval_dataset = EncodePreprocessor(template=template)(val_dataset, num_proc=num_proc)\n\n# 进行训练\ntraining_args = Seq2SeqTrainingArguments(...)\ntrainer = Seq2SeqTrainer(\n model=model,\n args=training_args,\n template=template,\n train_dataset=train_dataset,\n eval_dataset=val_dataset,\n)\ntrainer.train()\n```\n\n推理：\n```python\nfrom swift import TransformersEngine, InferRequest, RequestConfig\n# 使用原生 transformers 引擎进行推理\nengine = TransformersEngine(model_id_or_path, adapters=[lora_checkpoint])\ninfer_request = InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}])\nrequest_config = RequestConfig(max_tokens=max_new_tokens, temperature=temperature)\n\nresp_list = engine.infer([infer_request], request_config)\nprint(f'response: {resp_list[0].choices[0].message.content}')\n```\n\n## ✨ 如何使用\n\n这里给出使用ms-swift进行训练到部署的最简示例，具体可以查看[examples](https://github.com/modelscope/ms-swift/tree/main/examples)。\n\n- 若想使用其他模型或者数据集（含多模态模型和数据集），你只需要修改`--model`指定对应模型的id或者path，修改`--dataset`指定对应数据集的id或者path即可。\n- 默认使用ModelScope进行模型和数据集的下载。如果要使用HuggingFace，指定`--use_hf true`即可。\n\n| 常用链接 |\n| ------ |\n| [🔥命令行参数](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html) |\n| [Megatron-SWIFT](https://swift.readthedocs.io/zh-cn/latest/Megatron-SWIFT/Quick-start.html) |\n| [GRPO](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/GetStarted/GRPO.html) |\n| [支持的模型和数据集](https://swift.readthedocs.io/zh-cn/latest/Instruction/Supported-models-and-datasets.html) |\n| [自定义模型](https://swift.readthedocs.io/zh-cn/latest/Customization/Custom-model.html), [🔥自定义数据集](https://swift.readthedocs.io/zh-cn/latest/Customization/Custom-dataset.html) |\n| [大模型教程](https://github.com/modelscope/modelscope-classroom/tree/main/LLM-tutorial) |\n\n### 训练\n支持的训练方法：\n\n| 方法 | 全参数 | LoRA | QLoRA | Deepspeed | 多机 | 多模态 |\n| ------ | ------ |---------------------------------------------------------------------------------------------| ----- | ------ | ------ |----------------------------------------------------------------------------------------------|\n| [预训练](https://github.com/modelscope/ms-swift/blob/main/examples/train/pretrain) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [指令监督微调](https://github.com/modelscope/ms-swift/blob/main/examples/train/lora_sft.sh) | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/full/train.sh) | ✅ | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/qlora) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-gpu/deepspeed) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-node) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multimodal) |\n| [GRPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [GKD](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/gkd) | ✅ | ✅ | ✅ | ✅ | ✅ | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/gkd) |\n| [PPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/ppo) | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |\n| [DPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/dpo) | ✅ | ✅ | ✅ | ✅ | ✅ | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/dpo) |\n| [KTO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/kto.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/kto.sh) |\n| [奖励模型](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/rm.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [CPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/cpo.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [SimPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/simpo.sh) | ✅ | ✅ | ✅ | ✅| ✅ | ✅ |\n| [ORPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/orpo.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [Embedding](https://github.com/modelscope/ms-swift/blob/main/examples/train/embedding) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [Reranker](https://github.com/modelscope/ms-swift/tree/main/examples/train/reranker) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [序列分类](https://github.com/modelscope/ms-swift/blob/main/examples/train/seq_cls) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n\n\n预训练：\n```shell\n# 8*A100\nNPROC_PER_NODE=8 \\\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \\\nswift pt \\\n --model Qwen/Qwen2.5-7B \\\n --dataset swift/chinese-c4 \\\n --streaming true \\\n --tuner_type full \\\n --deepspeed zero2 \\\n --output_dir output \\\n --max_steps 10000 \\\n ...\n```\n\n微调：\n```shell\nCUDA_VISIBLE_DEVICES=0 swift sft \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --dataset AI-ModelScope/alpaca-gpt4-data-zh \\\n --tuner_type lora \\\n --output_dir output \\\n ...\n```\n\nRLHF：\n```shell\nCUDA_VISIBLE_DEVICES=0 swift rlhf \\\n --rlhf_type dpo \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --dataset hjh0119/shareAI-Llama3-DPO-zh-en-emoji \\\n --tuner_type lora \\\n --output_dir output \\\n ...\n```\n\n### Megatron-SWIFT\n\nms-swift支持使用Megatron并行技术加速训练，包括大规模集群训练和MoE模型训练。以下为支持的训练方法：\n\n| 方法 | 全参数 | LoRA | MoE | 多模态 | FP8 |\n| ------ | ------ | ---- | ----- | ----- | ----- |\n| 预训练 | ✅ | ✅| ✅ | ✅ | ✅ |\n| [指令监督微调](https://github.com/modelscope/ms-swift/tree/main/examples/megatron) | ✅ | ✅| ✅ | ✅ | ✅ |\n| [GRPO](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/grpo) | ✅ | ✅| ✅ | ✅ | ✅ |\n| [GKD](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/rlhf/gkd) | ✅ | ✅| ✅ | ✅ | ✅ |\n| [DPO](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/rlhf/dpo) | ✅ | ✅| ✅ | ✅ | ✅ |\n| [KTO](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/rlhf/kto) | ✅ | ✅| ✅ | ✅ | ✅ |\n| [RM](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/rlhf/rm) | ✅ | ✅| ✅ | ✅ | ✅ |\n| [Embedding](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/embedding) | ✅ | ✅| ✅ | ✅ | ✅ |\n| [Reranker](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/reranker) | ✅ | ✅| ✅ | ✅ | ✅ |\n| [序列分类](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/seq_cls) | ✅ | ✅| ✅ | ✅ | ✅ |\n\n\n```shell\nNPROC_PER_NODE=2 CUDA_VISIBLE_DEVICES=0,1 megatron sft \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --save_safetensors true \\\n --dataset AI-ModelScope/alpaca-gpt4-data-zh \\\n --tuner_type lora \\\n --output_dir output \\\n ...\n```\n\n### 强化学习\n\nms-swift支持丰富GRPO族算法：\n\n| 方法 | 全参数 | LoRA | 多模态 | 多机 |\n| ------ | ------ | ---- | ----- | ----- |\n| [GRPO](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/GetStarted/GRPO.html) | ✅ | ✅| ✅ | ✅ |\n| [DAPO](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/AdvancedResearch/DAPO.html) | ✅ | ✅| ✅ | ✅ |\n| [GSPO](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/AdvancedResearch/GSPO.html) | ✅ | ✅| ✅ | ✅ |\n| [SAPO](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/AdvancedResearch/SAPO.html) | ✅ | ✅| ✅ | ✅ |\n| [CISPO](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/AdvancedResearch/CISPO.html) | ✅ | ✅| ✅ | ✅ |\n| [CHORD](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/AdvancedResearch/CHORD.html) | ✅ | ✅| ✅ | ✅ |\n| [RLOO](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/AdvancedResearch/RLOO.html) | ✅ | ✅| ✅ | ✅ |\n| [Reinforce++](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/AdvancedResearch/REINFORCEPP.html) | ✅ | ✅| ✅ | ✅ |\n\n```shell\nCUDA_VISIBLE_DEVICES=0,1,2,3 NPROC_PER_NODE=4 \\\nswift rlhf \\\n --rlhf_type grpo \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --tuner_type lora \\\n --use_vllm true \\\n --vllm_mode colocate \\\n --dataset AI-MO/NuminaMath-TIR#10000 \\\n --output_dir output \\\n ...\n```\n\n### 推理\n```shell\nCUDA_VISIBLE_DEVICES=0 swift infer \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --stream true \\\n --infer_backend transformers \\\n --max_new_tokens 2048\n\n# LoRA\nCUDA_VISIBLE_DEVICES=0 swift infer \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --adapters swift/test_lora \\\n --stream true \\\n --infer_backend transformers \\\n --temperature 0 \\\n --max_new_tokens 2048\n```\n\n### 界面推理\n```shell\nCUDA_VISIBLE_DEVICES=0 swift app \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --stream true \\\n --infer_backend transformers \\\n --max_new_tokens 2048 \\\n --lang zh\n```\n\n### 部署\n```shell\nCUDA_VISIBLE_DEVICES=0 swift deploy \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --infer_backend vllm\n```\n\n### 采样\n```shell\nCUDA_VISIBLE_DEVICES=0 swift sample \\\n --model LLM-Research/Meta-Llama-3.1-8B-Instruct \\\n --sampler_engine transformers \\\n --num_return_sequences 5 \\\n --dataset AI-ModelScope/alpaca-gpt4-data-zh#5\n```\n\n### 评测\n```shell\nCUDA_VISIBLE_DEVICES=0 swift eval \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --infer_backend lmdeploy \\\n --eval_backend OpenCompass \\\n --eval_dataset ARC_c\n```\n\n### 量化\n```shell\nCUDA_VISIBLE_DEVICES=0 swift export \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --quant_bits 4 --quant_method awq \\\n --dataset AI-ModelScope/alpaca-gpt4-data-zh \\\n --output_dir Qwen2.5-7B-Instruct-AWQ\n```\n\n### 推送模型\n```shell\nswift export \\\n --model \\\n --push_to_hub true \\\n --hub_model_id '' \\\n --hub_token ''\n```\n\n\n## 🏛 License\n\n本框架使用[Apache License (Version 2.0)](https://github.com/modelscope/ms-swift/blob/master/LICENSE)进行许可。模型和数据集请查看原资源页面并遵守对应License。\n\n## 📎 引用\n\n```bibtex\n@misc{zhao2024swiftascalablelightweightinfrastructure,\n title={SWIFT:A Scalable lightWeight Infrastructure for Fine-Tuning},\n author={Yuze Zhao and Jintao Huang and Jinghan Hu and Xingjun Wang and Yunlin Mao and Daoze Zhang and Zeyinzi Jiang and Zhikai Wu and Baole Ai and Ang Wang and Wenmeng Zhou and Yingda Chen},\n year={2024},\n eprint={2408.05517},\n archivePrefix={arXiv},\n primaryClass={cs.CL},\n url={https://arxiv.org/abs/2408.05517},\n}\n```\n\n## Star History\n\n[](https://star-history.com/#modelscope/ms-swift&Date)\n"
},
{
"path": "docs/Makefile",
"content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line, and also\n# from the environment for the first two.\nSPHINXOPTS ?=\nSPHINXBUILD ?= sphinx-build\nSOURCEDIR = source\nBUILDDIR = build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n"
},
{
"path": "docs/README.md",
"content": "## maintain docs\n1. build docs\n ```shell\n # in root directory:\n make docs\n ```\n\n2. doc string format\n\n We adopt the google style docstring format as the standard, please refer to the following documents.\n 1. Google Python style guide docstring [link](http://google.github.io/styleguide/pyguide.html#381-docstrings)\n 2. Google docstring example [link](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)\n 3. sample：torch.nn.modules.conv [link](https://pytorch.org/docs/stable/_modules/torch/nn/modules/conv.html#Conv1d)\n 4. load function as an example：\n\n ```python\n def load(file, file_format=None, **kwargs):\n \"\"\"Load data from json/yaml/pickle files.\n\n This method provides a unified api for loading data from serialized files.\n\n Args:\n file (str or :obj:`Path` or file-like object): Filename or a file-like\n object.\n file_format (str, optional): If not specified, the file format will be\n inferred from the file extension, otherwise use the specified one.\n Currently supported formats include \"json\", \"yaml/yml\".\n\n Examples:\n >>> load('/path/of/your/file') # file is stored in disk\n >>> load('https://path/of/your/file') # file is stored on internet\n >>> load('oss://path/of/your/file') # file is stored in petrel\n\n Returns:\n The content from the file.\n \"\"\"\n ```\n"
},
{
"path": "docs/make.bat",
"content": "@ECHO OFF\n\npushd %~dp0\n\nREM Command file for Sphinx documentation\n\nif \"%SPHINXBUILD%\" == \"\" (\n\tset SPHINXBUILD=sphinx-build\n)\nset SOURCEDIR=source\nset BUILDDIR=build\n\nif \"%1\" == \"\" goto help\n\n%SPHINXBUILD% >NUL 2>NUL\nif errorlevel 9009 (\n\techo.\n\techo.The 'sphinx-build' command was not found. Make sure you have Sphinx\n\techo.installed, then set the SPHINXBUILD environment variable to point\n\techo.to the full path of the 'sphinx-build' executable. Alternatively you\n\techo.may add the Sphinx directory to PATH.\n\techo.\n\techo.If you don't have Sphinx installed, grab it from\n\techo.http://sphinx-doc.org/\n\texit /b 1\n)\n\n%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%\ngoto end\n\n:help\n%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%\n\n:end\npopd\n"
},
{
"path": "docs/source/.readthedocs.yaml",
"content": "# .readthedocs.yaml\n# Read the Docs configuration file\n# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details\n\n# Required\nversion: 2\n\n# Set the OS, Python version and other tools you might need\nbuild:\n os: ubuntu-22.04\n tools:\n python: \"3.10\"\n\n# Build documentation in the \"docs/\" directory with Sphinx\nsphinx:\n configuration: docs/source/conf.py\n\n# Optionally build your docs in additional formats such as PDF and ePub\n# formats:\n# - pdf\n# - epub\n\n# Optional but recommended, declare the Python requirements required\n# to build your documentation\n# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html\npython:\n install:\n - requirements: requirements/docs.txt\n - requirements: requirements/framework.txt\n"
},
{
"path": "docs/source/BestPractices/Elastic.md",
"content": "# Elastic\n\n\n\n## 安装依赖\n\n集群部署K8S,并在集群中部署DLrover,[DLRover](https://github.com/intelligent-machine-learning/dlrover),\n`pip install dlrover && pip install tornado && pip install kubernetes && pip install ms-swift`\n\n经过反复测试验证的训练镜像中的其它依赖以及版本：\ndeepspeed 0.16.5（需参考https://github.com/deepspeedai/DeepSpeed/pull/7585/files 修复universal checkpoint 相关问题）\npytorch 2.6.0\n\n\n## 如何启动\n\n通过在`--callbacks`中添加`deepspeed_elastic`（可选`graceful_exit`）启用弹性训练，并配置DeepSpeed弹性参数。\n命令组成=dlrover-run +dlrover 命令参数+swift 启动命令 +swift参数,dlrover-run除自定义的参数外，其他参数与torchrun一致；\ndlrover-run 参数如下:\n```\nusage: dlrover-run [-h] [--nnodes NNODES] [--nproc-per-node NPROC_PER_NODE]\n [--rdzv-backend RDZV_BACKEND] [--rdzv-endpoint RDZV_ENDPOINT] [--rdzv-id RDZV_ID]\n [--rdzv-conf RDZV_CONF] [--standalone] [--max-restarts MAX_RESTARTS]\n [--monitor-interval MONITOR_INTERVAL] [--start-method {spawn,fork,forkserver}]\n [--role ROLE] [-m] [--no-python] [--run-path] [--log-dir LOG_DIR] [-r REDIRECTS]\n [-t TEE] [--local-ranks-filter LOCAL_RANKS_FILTER] [--node-rank NODE_RANK]\n [--master-addr MASTER_ADDR] [--master-port MASTER_PORT] [--local-addr LOCAL_ADDR]\n [--logs-specs LOGS_SPECS] [--precheck {0,1,2}] [--node_unit NODE_UNIT]\n [--auto_config] [--auto_tunning] [--exclude-straggler] [--save_at_breakpoint]\n [--accelerator {nvidia.com/gpu,ascend-npu}] [--training_port TRAINING_PORT]\n [--switchbox-check] [--box-pairs PAIR [PAIR ...]] [--min-bandwidth MIN_BANDWIDTH]\n [--min-channels MIN_CHANNELS] [--numa-affinity] [--network-check]\n [--comm-perf-test] [--ucp_device_type UCP_DEVICE_TYPE]\n training_script\n\n```\n在弹性训练中我们需要关注的参数为：\n\n--nnodes NNODES Number of nodes, or the range of nodes in form\n :.\n\n--nproc-per-node NPROC_PER_NODE Number of processes per node.\n示例:\n\n```bash\nmodel=your model path\ndataset=your dataset\noutput= your output dir\nexport CUDA_VISIBLE_DEVICES=0 根据实际使用的GPU情况设置\ndeepspeed_config_or_type=deepspeed类型或者配置文件的路径，如 zero1 或者/xxx/ms-swift/swift/llm/ds_config/zero1.json\n\ndlrover-run --nnodes 1:$NODE_NUM --nproc_per_node=1 \\\n/opt/conda/lib/python3.10/site-packages/swift/cli/sft.py --model $model \\\n--model_type qwen3 \\\n--tuner_type lora \\\n--torch_dtype bfloat16 \\\n--dataset $dataset \\\n--num_train_epochs 4 \\\n--per_device_train_batch_size 1 \\\n--per_device_eval_batch_size 1 \\\n--learning_rate 5e-7 \\\n--gradient_accumulation_steps 8 \\\n--eval_steps 500 \\\n--save_steps 10 \\\n--save_total_limit 20 \\\n--logging_steps 1 \\\n--output_dir $output \\\n--warmup_ratio 0.01 \\\n--dataloader_num_workers 4 \\\n--temperature 1.0 \\\n--system 'You are a helpful assistant.' \\\n--lora_rank 8 \\\n--lora_alpha 32 \\\n--target_modules all-linear \\\n--dataset_num_proc 1 \\\n--use_flash_ckpt true \\\n--callbacks deepspeed_elastic graceful_exit \\\n--deepspeed $deepspeed_config_or_type \\\n```\n\n## 配置文件示例\n默认情况下的zero1为以下示例配置,\n\n```json\n{\n \"fp16\": {\n \"enabled\": \"auto\",\n \"loss_scale\": 0,\n \"loss_scale_window\": 1000,\n \"initial_scale_power\": 16,\n \"hysteresis\": 2,\n \"min_loss_scale\": 1\n },\n\n \"bf16\": {\n \"enabled\": \"auto\"\n },\n\n \"zero_optimization\": {\n \"stage\": 1,\n \"offload_optimizer\": {\n \"device\": \"none\",\n \"pin_memory\": true\n },\n \"allgather_partitions\": true,\n \"allgather_bucket_size\": 2e8,\n \"overlap_comm\": false,\n \"reduce_scatter\": true,\n \"reduce_bucket_size\": 2e8,\n \"contiguous_gradients\": true\n },\n\n \"gradient_accumulation_steps\": \"auto\",\n \"gradient_clipping\": \"auto\",\n \"steps_per_print\": 2000,\n \"train_batch_size\": \"auto\",\n \"train_micro_batch_size_per_gpu\": \"auto\",\n \"wall_clock_breakdown\": false,\n \"elasticity\": {\n \"ignore_non_elastic_batch_info\": true,\n \"enabled\": true,\n \"max_train_batch_size\": 8,\n \"micro_batch_sizes\": [\n 4,\n 2\n ],\n \"min_gpus\": 1,\n \"max_gpus\": 4,\n \"min_time\": 20,\n \"version\": 0.1\n }\n}\n```\n\n如果用户需要自定义，可以在启动命令中deepspeed_config_or_type指定自定义的zero1.json的存放路径,其中弹性相关的配置为：\n```json\n...\n\n \"elasticity\": {\n \"ignore_non_elastic_batch_info\": true,\n \"enabled\": true,\n \"max_train_batch_size\": 8,\n \"micro_batch_sizes\": [\n 4,\n 2\n ],\n \"min_gpus\": 1,\n \"max_gpus\": 4,\n \"min_time\": 20,\n \"version\": 0.1\n }\n```\n\n- ignore_non_elastic_batch_info：代表在elasticity里的配置会忽略外层的batch_size相关的配置，训练过程中会根据实际的训练进程个数实时修改batch_size等相关的参数\n计算原则为：\n global-training-batch-size = micro-batch-size * gradient-accumulation-steps * world-size\n- max_train_batch_size：最大batch_size数\n- micro_batch_sizes：elasticity下允许的每卡micro-batch size列表，相当于train_micro_batch_size_per_gpu的候选值\n- min_gpus：最小gpu数目\n- max_gpus：最大gpu数目\n更详细的内容见：[Deepspeed](https://www.deepspeed.ai/docs/config-json/#elastic-training-config-v01-and-v02)\n\n\n## 启动训练\n\n```yaml\n---\napiVersion: elastic.iml.github.io/v1alpha1\nkind: ElasticJob\nmetadata:\n name: deepspeed-elastic-swift\n namespace: dlrover\nspec:\n distributionStrategy: AllreduceStrategy\n optimizeMode: single-job\n replicaSpecs:\n worker:\n replicas: 1 #【这里需要与启动命令中的--nnodes NNODES的最大值一致】\n template:\n spec:\n restartPolicy: Never\n containers:\n - name: main\n image: #【训练镜像，需要安装deepspeed,dlrover 和swift 】\n imagePullPolicy: IfNotPresent\n command:\n - /bin/bash\n - -c\n - sh start.sh # 启动脚本\n resources:\n limits:\n cpu: '8'\n memory: 16Gi\n nvidia.com/gpu: '1'\n volumeMounts:\n - mountPath: /model\n name: volume-model\n - mountPath: /dev/shm\n name: volume-shm\n restartPolicy: Never\n volumes:\n - hostPath:\n path: /model\n type: Directory\n name: volume-model\n - emptyDir:\n medium: Memory\n sizeLimit: 200Gi\n name: volume-shm\n\n```\n"
},
{
"path": "docs/source/BestPractices/Embedding.md",
"content": "# Embedding训练\n\nSWIFT已经支持Embedding模型的训练，包括纯文本和多模态两个类型。目前已经支持的模型有：\n\n1. modernbert embedding模型\n - [ModelScope](https://modelscope.cn/models/iic/gte-modernbert-base) [Hugging Face](https://huggingface.co/Alibaba-NLP/gte-modernbert-base)\n2. gte embedding模型\n - 1.5B: [ModelScope](https://www.modelscope.cn/models/iic/gte_Qwen2-1.5B-instruct) [Hugging Face](https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct)\n - 7B: [ModelScope](https://www.modelscope.cn/models/iic/gte_Qwen2-7B-instruct) [Hugging Face](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct)\n3. gme embedding模型\n - 2B: [ModelScope](https://www.modelscope.cn/models/iic/gme-Qwen2-VL-2B-Instruct) [Hugging Face](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct)\n - 7B: [ModelScope](https://www.modelscope.cn/models/iic/gme-Qwen2-VL-7B-Instruct) [Hugging Face](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct)\n4. qwen3-embedding模型\n - 0.6B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-Embedding-0.6B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B)\n - 4B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-Embedding-4B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-Embedding-4B)\n - 8B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-Embedding-8B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-Embedding-8B)\n5. qwen3-vl-embedding模型\n - 2B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-VL-Embedding-2B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-VL-Embedding-2B)\n - 8B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-VL-Embedding-8B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B)\n\n\n开发者可以自行集成自己的模型，模型forward输出值需要满足：\n\n```text\n{\"last_hidden_state\": some-embedding-tensor}\n```\n\n返回值是一个json，具有`last_hidden_state` key，value是embedding tensor即可，输入部分可以使用我们已经支持的template。用户也可以通过指定\n\n```shell\n --task_type embedding\n```\n参数来将任意一个其他模型转换为embedding模型进行训练。\n\n需要注意的是，SWIFT目前支持的embedding模型均为符合纯文本或多模态LLM，目前并不支持CLIP类型的模型训练。\n\n此外，SWIFT支持的所有embedding模型在模型forward最后都增加了normalize，如自行增加新模型请注意增加normalize层。\n\n## loss\n\n目前SWIFT支持的Embedding模型可以使用的loss有：\n\n- cosine_similarity: cosine相似度loss，计算两个embedding的相似度，并根据label的值拟合，实际为MSE loss\n- contrastive: 可调margin的对比学习loss，label仅支持0和1两个值\n- online_contrastive: 考虑hard negative和hard positive部分的contrastive loss，label仅支持0和1两个值\n- infonce: 在同一个batch中不同row两两计算cosine相似度，并使row内部相似度最大，不同row相似度最小，不需要label\n\nloss的源代码可以在[这里](https://github.com/modelscope/ms-swift/blob/main/swift/loss/mapping.py)找到。\n\n## 数据集格式\n\n> 注：\n> 1. ``标签可以出现在`messages`/`positive_messages`/`negative_messages`的任意位置；它们各自拥有独立的`images`/`positive_images`/`negative_images`字段用于提供图片路径或URL。\n> 2. 不再需要跨字段的“对应顺序”。对齐规则为：`images`的长度等于`messages`中``标签的数量；`positive_images`与`negative_images`均为“list of list”，其外层长度分别等于`positive_messages`与`negative_messages`的长度；并且外层每一项的内层列表长度等于该条消息序列中``标签的数量。\n> 3. `messages`代表anchor样本（anchor sample）；`positive_messages`/`negative_messages`为“list of messages”（因此多一层`[]`）；相应地，`positive_images`/`negative_images`也多一层`[]`并与之逐项对齐。\n> 4. 也支持``, ``标签；可按相同规则分别通过`videos`/`positive_videos`/`negative_videos`与`audios`/`positive_audios`/`negative_audios`提供对应模态数据。\n> 5. 当前约束：`positive_messages`的外层长度必须为1（即仅提供一个positive样本）；对应地，`positive_images`的外层长度也必须为1。\n\n### cosine_similarity loss对应的格式\n\n```json lines\n# LLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]], \"label\": 0.8}\n# MLLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"\"}], \"images\": [\"/some/images1.jpg\"],\"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence\"}]], \"positive_images\": [[\"/some/images2.jpg\"]], \"label\": 0.7}\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]], \"positive_images\": [[\"/some/images.jpg\"]], \"label\": 0.7}\n```\n\n\n### contrastive/online_contrastive loss对应的格式\n\n```json lines\n# LLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]], \"label\": 1}\n# MLLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"\"}], \"images\": [\"/some/images1.jpg\"], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence\"}]], \"positive_images\": [[\"/some/images2.jpg\"]], \"label\": 1}\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]], \"positive_images\": [[\"/some/images.jpg\"]], \"label\": 0}\n```\n\n评测的指标分别是两个embedding的欧式距离、点积等的pearson系数以及spearman系数，共八个指标。\n\n### infonce 格式\n\n```json lines\n# LLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]]}\n# MLLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"\"}], \"images\": [\"/some/images.jpg\"], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence\"}]]}\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"images\": [\"/some/images.jpg\"], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]], \"positive_images\": [[\"/some/positive_images.jpg\"]], \"negative_messages\": [[{\"role\": \"user\", \"content\": \"sentence3\"}], [{\"role\": \"user\", \"content\": \"sentence4\"}]], \"negative_images\": [[\"/some/negative_images1.jpg\", \"/some/negative_images2.jpg\"], [\"/some/negative_images3.jpg\"]]}\n```\n\ninfonce loss支持几个环境变量：\n1. `INFONCE_TEMPERATURE`： temperature参数，不设置的话默认值是0.1\n2. `INFONCE_USE_BATCH`： 使用sample内部的`negative_messages`（hard negative样例）还是使用一个batch内其他样本作为in-batch negatives；默认为True，表示使用batch内部的样本作为负例\n3. `INFONCE_HARD_NEGATIVES`： hard negatives的数量；如果不设置会使用数据中提供的所有`negative_messages`。由于长度未必一致，因此会采用for循环计算loss（计算会慢）。若设置为某个数值，则不足会随机采样补齐，超长会选用前`INFONCE_HARD_NEGATIVES`个\n4. `INFONCE_MASK_FAKE_NEGATIVE`： mask掉假negative。默认为False，开启时会判断 `positive_similarity + INFONCE_FAKE_NEG_MARGIN`，比该阈值大的样本相似度会被设置为 `-inf`，以防止正样本泄露问题\n5. `INFONCE_FAKE_NEG_MARGIN`：假负样本屏蔽的边际，默认 `0.1`。\n6. `INFONCE_INCLUDE_QQ`：是否在分母中加入 q–q 分量（query 间相似度）作为负例，默认 `False`。\n7. `INFONCE_INCLUDE_DD`：是否在分母中加入 d–d 分量（正样本文档与 batch 内所有文档的相似度）作为负例，默认 `False`。\n\n> 也可以在数据集中将hard negatives数量设置为数量相等，这样即使不设置也不会使用for循环方式，加快计算速度\n> `negative_messages`也可以不提供。在这种情况下，保持`INFONCE_USE_BATCH=True`，会使用一个batch内部的其他样本作为负例\n\ninfonce loss的评测会有下面几个指标：\n- mean_neg 所有hard_negative的平均值\n- mean_pos 所有positive的平均值\n- margin positive-max_hard_negative的平均值\n\n## 训练\n\nSWIFT提供的脚手架训练脚本：\n\n- [Qwen3-Embedding/Qwen3-VL-Embedding模型](https://github.com/modelscope/ms-swift/blob/main/examples/train/embedding/qwen3)\n- [GME模型](https://github.com/modelscope/ms-swift/blob/main/examples/train/embedding/train_gme.sh)\n\n## 推理\n\nSWIFT已经支持GME、GTE、Qwen3-Embedding模型的部署，请查看[这里](https://github.com/modelscope/ms-swift/blob/main/examples/deploy/embedding/client.py)。\n- 推理脚本参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_embedding.py)。\n\n也可以使用原模型的代码进行推理：\n\nhttps://www.modelscope.cn/models/iic/gte_Qwen2-7B-instruct\n\nhttps://www.modelscope.cn/models/iic/gme-Qwen2-VL-7B-Instruct\n\n如果使用了其他模型从0训练embedding（例如，原版`qwen2-vl`模型+`--task_type embedding`），也可以使用gme的推理代码，但请注意：\n\nhttps://www.modelscope.cn/models/iic/gme-Qwen2-VL-7B-Instruct/file/view/master/gme_inference.py?status=1#L111\n\n这里的模板请修改为模型自身的template，以免最后的embedding对不上。需要额外注意的是，gme模型的template和`qwen2-vl`或`qwen2.5-vl`系列的chatml template并不相同，其推理代码最后的结束字符是`<|endoftext|>`而非`<|im_end|>`.\n\n## 高级功能\n\n- Qwen3-Embedding 自定义 Instruction：\n - 默认无 Instruction，输入模板为：`{Query}<|endoftext|>`。\n - 通过在 system message 中添加 Instruction，可将输入改为：`{Instruction} {Query}<|endoftext|>`。\n - 示例：\n\n```json lines\n{\"messages\": [\n {\"role\": \"system\", \"content\": \"请用中文回答，并输出简洁要点\"},\n {\"role\": \"user\", \"content\": \"介绍一下Qwen3-Embedding\"}\n]}\n```\n\n> 说明：Qwen3-Embedding 模板会将 system 内容前置拼接到首条 user 消息中，并使用 `<|endoftext|>` 作为结束标记。\n\n### 转换前后示例\n\n- 不加 Instruction：\n\n 输入数据（messages）：\n\n ```json lines\n {\"messages\": [\n {\"role\": \"user\", \"content\": \"北京明天天气如何？\"}\n ]}\n ```\n\n 模板转换后（送入模型的实际文本）：\n\n ```text\n 北京明天天气如何？<|endoftext|>\n ```\n\n- 加 Instruction：\n\n 输入数据（messages，包含system）：\n\n ```json lines\n {\"messages\": [\n {\"role\": \"system\", \"content\": \"请使用中文、精炼输出要点\"},\n {\"role\": \"user\", \"content\": \"北京明天天气如何？\"}\n ]}\n ```\n\n 模板转换后（送入模型的实际文本）：\n\n ```text\n 请使用中文、精炼输出要点 北京明天天气如何？<|endoftext|>\n ```\n\n- positive/negative 同理：\n\n 若在某个 positive/negative 的消息序列中提供 system，则会将该 system 内容前置到该序列首条 user 内容之前；未提供 system 则不前置。\n\n 输入数据（包含一个 positive 带 system，和一个 negative 无 system）：\n\n ```json lines\n {\n \"messages\": [\n {\"role\": \"user\", \"content\": \"Anchor\"}\n ],\n \"positive_messages\": [[\n {\"role\": \"system\", \"content\": \"指令\"},\n {\"role\": \"user\", \"content\": \"Positive\"}\n ]],\n \"negative_messages\": [[\n {\"role\": \"user\", \"content\": \"Negative\"}\n ]]\n }\n ```\n\n 模板转换后（送入模型的实际文本）：\n\n ```text\n Anchor<|endoftext|>\n 指令 Positive<|endoftext|>\n Negative<|endoftext|>\n ```\n"
},
{
"path": "docs/source/BestPractices/GRPO-Code-Training.md",
"content": "# GRPO代码训练\n本文档介绍如何使用GRPO对模型进行代码训练\n\n模型：[Qwen/Qwen2.5-7B-Instruct](https://www.modelscope.cn/models/Qwen/Qwen2.5-VL-7B-Instruct)\n\n数据集：[open-r1/verifiable-coding-problems-python-10k](https://www.modelscope.cn/datasets/open-r1/verifiable-coding-problems-python-10k/dataPeview)\n\n数据集样例\n```json\n{\n \"problem\": \"Solve the following coding problem using the programming language python: Polycarp has $n$ different binary words. A word called binary if it contains only characters '0' and '1'. For example, these words are binary: \\\"0001\\\", \\\"11\\\", \\\"0\\\" and \\\"0011100\\\". Polycarp wants to offer his set of $n$ binary words to play a game \\\"words\\\". In this game, players name words and each next word (starting from the second) must start with the last character of the previous word. The first word can be any. For example, these sequence of words can be named during the game: \\\"0101\\\", \\\"1\\\", \\\"10\\\", \\\"00\\\", \\\"00001\\\". Word reversal is the operation of reversing the order of the characters. For example, the word \\\"0111\\\" after the reversal becomes \\\"1110\\\", the word \\\"11010\\\" after the reversal becomes \\\"01011\\\". Probably, Polycarp has such a set of words that there is no way to put them in the order correspondent to the game rules. In this situation, he wants to reverse some words from his set so that: the final set of $n$ words still contains different words (i.e. all words are unique); there is a way to put all words of the final set of words in the order so that the final sequence of $n$ words is consistent with the game rules. Polycarp wants to reverse minimal number of words. Please, help him. -----Input----- The first line of the input contains one integer $t$ ($1 \\\\le t \\\\le 10^4$) — the number of test cases in the input. Then $t$ test cases follow. The first line of a test case contains one integer $n$ ($1 \\\\le n \\\\le 2\\\\cdot10^5$) — the number of words in the Polycarp's set. Next $n$ lines contain these words. All of $n$ words aren't empty and contains only characters '0' and '1'. The sum of word lengths doesn't exceed $4\\\\cdot10^6$. All words are different. Guaranteed, that the sum of $n$ for all test cases in the input doesn't exceed $2\\\\cdot10^5$. Also, guaranteed that the sum of word lengths for all test cases in the input doesn't exceed $4\\\\cdot10^6$. -----Output----- Print answer for all of $t$ test cases in the order they appear. If there is no answer for the test case, print -1. Otherwise, the first line of the output should contain $k$ ($0 \\\\le k \\\\le n$) — the minimal number of words in the set which should be reversed. The second line of the output should contain $k$ distinct integers — the indexes of the words in the set which should be reversed. Words are numerated from $1$ to $n$ in the order they appear. If $k=0$ you can skip this line (or you can print an empty line). If there are many answers you can print any of them. -----Example----- Input 4 4 0001 1000 0011 0111 3 010 101 0 2 00000 00001 4 01 001 0001 00001 Output 1 3 -1 0 2 1 2 The input will be stdin and you should print your solution to stdout Now solve the problem and return the code.\",\n \"verification_info\": {\n \"language\": \"python\",\n \"test_cases\": [\n {\n \"input\": \"4\\n4\\n0001\\n1000\\n0011\\n0111\\n3\\n010\\n101\\n0\\n2\\n00000\\n00001\\n4\\n01\\n001\\n0001\\n00001\\n\",\n \"output\": \"1\\n3 \\n-1\\n0\\n\\n2\\n1 2 \\n\",\n \"type\": \"stdin_stdout\"\n }\n ]\n }\n}\n```\n\n`verification_info` 提供了程序语言以及测试用例，其中包含输入和预期的输出。\n\n\n## 奖励函数\n使用`code_reward`和`code_format`奖励进行训练，实现细节见[代码](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo/plugin/plugin.py)\n\n- `code_reward`通过[e2b](https://e2b.dev/)或[judge0](https://judge0.com/)执行生成的代码，根据数据集中的测试用例对代码进行验证给出奖励值。\n- `code_format`要求模型输出包含代码块的格式化回答。\n\n注：当前通过e2b执行代码仅支持python语言，如需执行其他语言，可以使用judge0执行（[judge0支持语言列表](https://github.com/judge0/judge0?tab=readme-ov-file#supported-languages)）。\n\n## 训练脚本\n### e2b\n- 在[e2b](https://e2b.dev/dashboard)注册获取E2B_API_KEY，并设置为环境变量。\n- `--reward_funcs`添加`external_code_reward`作为奖励函数。\n- `--external_plugins`设置为plugin.py的路径。\n首先拉起 vLLM server\n```bash\nCUDA_VISIBLE_DEVICES=7 \\\nswift rollout \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --vllm_enable_lora true \\\n --vllm_max_lora_rank 16\n```\n\n```bash\nE2B_API_KEY=xxx \\\nWANDB_API_KEY=xxx \\\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6 \\\nNPROC_PER_NODE=7 \\\nswift rlhf \\\n --rlhf_type grpo \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --external_plugins examples/train/grpo/plugin/plugin.py \\\n --reward_funcs external_code_reward external_code_format \\\n --reward_weights 1.0 0.1 \\\n --vllm_mode server \\\n --use_vllm true \\\n --vllm_server_host 127.0.0.1 \\\n --vllm_server_port 8000 \\\n --tuner_type lora \\\n --lora_rank 16 \\\n --lora_alpha 32 \\\n --torch_dtype bfloat16 \\\n --dataset 'open-r1/verifiable-coding-problems-python-10k' \\\n --load_from_cache_file true \\\n --max_completion_length 2048 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 2 \\\n --per_device_eval_batch_size 2 \\\n --learning_rate 1e-6 \\\n --gradient_accumulation_steps 1 \\\n --eval_steps 200 \\\n --save_steps 200 \\\n --save_total_limit 2 \\\n --logging_steps 5 \\\n --max_length 4096 \\\n --output_dir output \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 4 \\\n --dataset_num_proc 4 \\\n --num_generations 14 \\\n --temperature 0.9 \\\n --system 'examples/train/grpo/prompt.txt' \\\n --deepspeed zero2 \\\n --log_completions true \\\n --report_to wandb\n```\n\n### judge0\n- 设置环境变量：\n - （必需）JUDGE0_ENDPOINT: judge0访问地址。\n - （可选）JUDGE0_X_AUTH_TOKEN: judge0访问Token。\n- `--reward_funcs`添加`external_code_reward_by_judge0`作为奖励函数。\n- `--external_plugins`设置为plugin.py的路径。\n\n```bash\nJUDGE0_ENDPOINT=xxx \\\nJUDGE0_X_AUTH_TOKEN=xxx \\\nWANDB_API_KEY=xxx \\\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6 \\\nNPROC_PER_NODE=7 \\\nswift rlhf \\\n --rlhf_type grpo \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --external_plugins examples/train/grpo/plugin/plugin.py \\\n --reward_funcs external_code_reward_by_judge0 external_code_format \\\n --reward_weights 1.0 0.1 \\\n --vllm_mode server \\\n --use_vllm true \\\n --vllm_server_host 127.0.0.1 \\\n --vllm_server_port 8000 \\\n --tuner_type lora \\\n --torch_dtype bfloat16 \\\n --dataset 'open-r1/verifiable-coding-problems-python-10k' \\\n --load_from_cache_file true \\\n --max_completion_length 2048 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 2 \\\n --per_device_eval_batch_size 2 \\\n --learning_rate 1e-6 \\\n --gradient_accumulation_steps 1 \\\n --eval_steps 200 \\\n --save_steps 200 \\\n --save_total_limit 2 \\\n --logging_steps 5 \\\n --max_length 4096 \\\n --output_dir output \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 4 \\\n --dataset_num_proc 4 \\\n --num_generations 14 \\\n --temperature 0.9 \\\n --system 'examples/train/grpo/prompt.txt' \\\n --deepspeed zero2 \\\n --log_completions true \\\n --report_to wandb\n```\n\n训练奖励曲线图\n\n"
},
{
"path": "docs/source/BestPractices/GRPO-Multi-Modal-Training.md",
"content": "# 多模态GRPO完整实验流程\n本文介绍如何使用SWIFT GRPO进行多模态模型和任务的训练。目标是对多个多模态任务进行训练，提升任务精度，任务定义和训练参数等参考了 [R1-V](https://github.com/Deep-Agent/R1-V.git) 和 [open-r1-multimodal](https://github.com/EvolvingLMMs-Lab/open-r1-multimodal.git)\n\n\n\n## ClevrCount 任务\n### 任务与数据集定义\n本任务从clevr_cogen_a_train数据集出发，模型的目标是输出图像中包含的物体数量，因此，我们定义数据集如下：\n\n```python\nclass ClevrPreprocessor(ResponsePreprocessor):\n\n def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:\n query = row.get('query', '')\n query = f\"\"\"{query} Output the thinking process in and\n final answer (number) in tags.\"\"\"\n row.update({'query': query})\n return super().preprocess(row)\n\n\nregister_dataset(\n DatasetMeta(\n ms_dataset_id='AI-ModelScope/clevr_cogen_a_train',\n subsets=[\n SubsetDataset(\n name='default',\n subset='default',\n split=['train'],\n ),\n ],\n preprocess_func=ClevrPreprocessor(),\n tags=['qa', 'math']))\n\n```\n这里重新定义dataset preprocessor的目的是修改query。数据集示例样本如下，包含messages,images和solution字段，solution会送入后续的奖励函数中，而messages和images则会作为模型输入。\n- 注意：`{'role': 'assistant', 'content': ' 3 '}`将会在GRPOTrainer中被移除，可以忽略。'solution'字段将会透传入ORM中。在自定义数据集时，'images'字段组织成`[\"image_path1\", \"image_path2\"]`即可。\n\n```json\n{\n \"images\": [\"image_path1\", \"image_path2\"],\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"How many items are there in the image? Output the thinking process in and \\n final answer (number) in tags.\"\n }\n ],\n \"solution\": \" 3 \"\n}\n```\n\n\n## 奖励函数定义：\n本任务使用的奖励函数有两个，一个是 Deepseek-R1 中提到的格式奖励函数，另一是 ClevrCount 的准确性奖励函数。前者已经在swift中内置，通过 `--reward_funcs format` 可以直接使用，而后者需要我们自己定义，在这里我们使用 external_plugin 的方式定义准确性奖励函数，将代码放在`swift/examples/train/grpo/plugin/plugin.py`中。\n\n在这里，奖励函数的输入包括completions和solution两个字段，分别表示模型生成的文本和真值。每个都是list，支持多个completion同时计算。注意，在这里，solution字段是数据集中定义的字段透传而来，如果有任务上的变动，可以分别对数据集和奖励函数做对应的改变即可。\n```python\n\nclass MultiModalAccuracyORM(ORM):\n\n def __call__(self, completions, solution, **kwargs) -> List[float]:\n \"\"\"\n Reward function that checks if the completion is correct.\n Args:\n completions (list[str]): Generated outputs\n solution (list[str]): Ground Truths.\n\n Returns:\n list[float]: Reward scores\n \"\"\"\n rewards = []\n from math_verify import parse, verify\n for content, sol in zip(completions, solution):\n reward = 0.0\n # Try symbolic verification first\n try:\n answer = parse(content)\n if float(verify(answer, parse(sol))) > 0:\n reward = 1.0\n except Exception:\n pass # Continue to next verification method if this fails\n\n # If symbolic verification failed, try string matching\n if reward == 0.0:\n try:\n # Extract answer from solution if it has think/answer tags\n sol_match = re.search(r'(.*?) ', sol)\n ground_truth = sol_match.group(1).strip() if sol_match else sol.strip()\n\n # Extract answer from content if it has think/answer tags\n content_match = re.search(r'(.*?) ', content)\n student_answer = content_match.group(1).strip() if content_match else content.strip()\n\n # Compare the extracted answers\n if student_answer == ground_truth:\n reward = 1.0\n except Exception:\n pass # Keep reward as 0.0 if both methods fail\n rewards.append(reward)\n return rewards\norms['external_r1v_acc'] = MultiModalAccuracyORM\n```\n\n### GRPO训练实验记录\n#### 训练参数：\n我们选取 Qwen2.5-VL-3B-Instruct 作为基础模型进行训练，选取 Instruct 而不是基模的主要原因是可以更快地获取 format reward。我们在八卡 GPU 上进行实验。如果遇到vllm部署qwen2.5-vl报错，可以参考[issue](https://github.com/vllm-project/vllm/issues/13285)\n\n由于任务简单，我们设置max_completion_length为1024，奖励函数选择external_r1v_acc和format，学习率和beta分别设置为1e-6和0.001。其他设置如下所示，batch_size和num_generations的设置原则可以参考[GRPO完整流程](./GRPO.md)。\n首先拉起 external vLLM server\n```bash\nCUDA_VISIBLE_DEVICES=6,7 \\\nswift rollout \\\n --model Qwen/Qwen2.5-VL-3B-Instruct \\\n --vllm_data_parallel_size 2\n```\n\n```shell\nWANDB_API_KEY=your_wandb_api_key \\\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5 \\\nNPROC_PER_NODE=6 \\\nswift rlhf \\\n --rlhf_type grpo \\\n --model Qwen/Qwen2.5-VL-3B-Instruct \\\n --external_plugins examples/train/grpo/plugin/plugin.py \\\n --reward_funcs external_r1v_acc format \\\n --use_vllm true \\\n --vllm_mode server \\\n --vllm_server_host 127.0.0.1 \\\n --vllm_server_port 8000 \\\n --tuner_type full \\\n --torch_dtype bfloat16 \\\n --dataset 'AI-ModelScope/clevr_cogen_a_train' \\\n --load_from_cache_file true \\\n --max_completion_length 1024 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 8 \\\n --per_device_eval_batch_size 8 \\\n --learning_rate 1e-6 \\\n --gradient_accumulation_steps 2 \\\n --save_strategy 'steps' \\\n --eval_strategy 'steps' \\\n --eval_steps 1000 \\\n --save_steps 1000 \\\n --save_total_limit 10 \\\n --logging_steps 1 \\\n --output_dir output/GRPO_CLEVR_COUNTDOWN \\\n --warmup_ratio 0.01 \\\n --dataloader_num_workers 4 \\\n --num_generations 24 \\\n --temperature 1.0 \\\n --system 'examples/train/grpo/prompt.txt' \\\n --deepspeed zero3 \\\n --log_completions true \\\n --report_to wandb \\\n --num_iterations 1 \\\n --async_generate false \\\n --beta 0.001 \\\n```\n#### 实验现象\n\n由于数据集和任务比较简单，模型训练了500个epoch已经基本收敛，有以下观察。\n1. 自定义的ClevrORM在不断增加，证明模型学习到了如何完成这一任务，最终任务成功率从初始的0.4攀升到1左右。\n2. 由于是Instruct模型，Format Reward基本稳定在1，这可能也与数据集所有样本都是一样的query有一定关系。\n3. reward_std 稳定到0.1以下。\n4. completion length最终稳定在60-80，模型学习到的固定输出范式是逐个物体列举数数。\n\nstep 1:\n```\n\" The image shows 5 distinct items: a cyan metallic cylinder, a red metallic cylinder, a small purple metallic cylinder, a large gray metallic sphere, and a small yellow metallic cube. To count the number of items, I add the counts of each type of object: 1 cyan cylinder, 1 red cylinder, 1 purple cylinder, 1 gray sphere, and 1 yellow cube. Therefore, the total number of items is 5. \\n5 \"\n\"Let's count the number of items in the image:\\n1. A large cyan metallic cylinder.\\n2. A small purple metallic cylinder.\\n3. A large red metallic cylinder.\\n4. A large gray metallic sphere.\\n5. A small yellow metallic cube.\\n\\n5 \"\n\" The image shows four distinct objects: a large cyan metallic cylinder, a smaller purple metallic cylinder, a large red metallic cylinder, and a small yellow metallic cube. Counting each of these items individually, there are five items in total. \\n5 \"\n```\nstep 400:\n```\n\" Counting the number of items in the image:\\n1. Green matte sphere\\n2. Large metallic yellow sphere\\n3. Small metallic brown cube\\n\\nThere are three distinct objects in total.\\n \\n3 \"\n\" Counting the objects in the image:\\n1. Large yellow metallic sphere\\n2. Small green rubber sphere\\n3. Small brown metallic cube\\n\\nThere are three distinct items in the image.\\n \\n3 \"\n```\n总体来讲，这一任务比较简单，reward的收敛也比较典型。\n\n## Geometric QA任务\n### 任务与数据集定义\n本任务为Geometric QA任务，任务描述为：给定一个几何图形，回答有关几何图形的数学问题。原始数据来自于[论文](https://arxiv.org/pdf/2312.11370)，[R1-V](https://github.com/Deep-Agent/R1-V.git)对数据进行了预处理，将所有数据全部处理成了problem-solution的格式，而图像则保留在image字段中，因此，我们不需要额外定义数据集，直接使用`--dataset AI-ModelScope/GEOQA_R1V_Train_8K`即可。\n### 奖励函数\n由于也是数学题，同时，答案也处理成了最终结果，因此，我们直接使用以上定义过的`MultiModalAccuracyORM`奖励函数。\n### GRPO训练实验记录\n#### 训练参数：\n选取的模型和大部分超参数与上一个实验相似，主要有两点不同：\n1. SWIFT 已支持`--num_iteration`参数，单次rollout可以进行多次更新，这里设置为2。\n2. 在实验时发现，在数学问题中，训练可能会出现不稳定现象，导致模型训崩，具体表现为所有rewar迅速降低，loss、grad_norm和kl都迅速增大，后续也难以恢复正常状态。因此，这里设置`--max_grad_norm 0.5`，保证稳定训练，当然，这种现象的出现也有一定的随机性。\n\n```shell\nWANDB_API_KEY=your_wandb_api_key \\\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5 \\\nMAX_PIXELS=401408 \\\nNPROC_PER_NODE=6 \\\nswift rlhf \\\n --rlhf_type grpo \\\n --model Qwen/Qwen2.5-VL-3B-Instruct \\\n --external_plugins examples/train/grpo/plugin/plugin.py \\\n --reward_funcs external_r1v_acc format \\\n --use_vllm true \\\n --vllm_mode server \\\n --vllm_server_host 127.0.0.1 \\\n --vllm_server_port 8000 \\\n --tuner_type full \\\n --torch_dtype bfloat16 \\\n --dataset 'AI-ModelScope/GEOQA_R1V_Train_8K' \\\n --load_from_cache_file true \\\n --max_completion_length 1024 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 8 \\\n --per_device_eval_batch_size 8 \\\n --learning_rate 1e-6 \\\n --gradient_accumulation_steps 2 \\\n --save_strategy 'steps' \\\n --eval_strategy 'steps' \\\n --eval_steps 400 \\\n --save_steps 400 \\\n --save_total_limit 10 \\\n --logging_steps 1 \\\n --output_dir output/GRPO_GEOQA \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 4 \\\n --num_generations 8 \\\n --temperature 1.0 \\\n --repetition_penalty 1.1 \\\n --system 'examples/train/grpo/prompt.txt' \\\n --deepspeed zero3 \\\n --log_completions true \\\n --report_to wandb \\\n --num_iterations 2 \\\n --async_generate false \\\n --beta 0.001 \\\n --max_grad_norm 0.5 \\\n```\n\n#### 实验现象\n\n训练曲线如上图所示。\n1. 相比于Count任务，Geometric QA任务的难度明显更大，因此，Accuracy Reward收敛明显更慢，同时震荡幅度也明显更大。\n2. completion_length的变化趋势比较明显，最终收敛到250 token左右。\n3. reward基本收敛，标准差在一共1300个step时稳定在0.2左右。\n以下是训练后的模型completition示例：\n```\nUser:\n\"If AB is the tangent to circle O with the tangent point A, and BO intersects circle O at point C, and point D is located on circle O, what is the measure of angle ADC if the degree of angle ABO is 32.0?\"\nAssistant:\n\"\\nTo find the measure of angle ADC, we first need to understand the properties of the tangent and the circle. Since AB is a tangent to the circle at point A, the line segment OA (which is perpendicular to the tangent AB) is perpendicular to AB. This means that angle OAB is 90 degrees.\\n\\nGiven that angle ABO is 32 degrees, we can find angle BAO by subtracting angle ABO from 90 degrees:\\n∠BAO = 90° - ∠ABO = 90° - 32° = 58°\\n\\nSince angle BAO is an angle in the minor arc AD, the angle ADC, which is an angle in the minor arc AD, is half the angle BAO. Therefore, angle ADC is:\\n∠ADC = 1/2 × ∠BAO = 1/2 × 58° = 29°\\n\\nSo, the measure of angle ADC is 29 degrees.\\n \\n\\nThe measure of angle ADC is 29 degrees.\\n \"\n```\n## Multimodal Open R1 数据集实验\n### 任务与数据集定义\n本任务为参考[open-r1-multimodal](https://github.com/EvolvingLMMs-Lab/open-r1-multimodal.git)的实验，使用数据集：[lmms-lab/multimodal-open-r1-8k-verified](https://www.modelscope.cn/datasets/lmms-lab/multimodal-open-r1-8k-verified)，该数据集专注于多模态的数学推理任务，数据由GPT4o基于`Math360K`和`Geo170K`数据集生成，包含推理路径和可验证答案。数据集中已包含了image, problem和solution字段，我们也不需要针对prompt进行修改，因此无需额外定义数据集。\n### 奖励函数\n我们直接使用以上定义过的`MultiModalAccuracyORM`奖励函数。\n### GRPO训练实验记录\n#### 训练参数：\n选取的模型和大部分超参数与上一个实验相似，由于训练的时候出现了OOM，我们设置`MAX_PIXELS=262144`以降低显存占用。\n```shell\nWANDB_API_KEY=your_wandb_api_key \\\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5 \\\nMAX_PIXELS=262144 \\\nMASTER_PORT=29600 \\\nNPROC_PER_NODE=6 \\\nswift rlhf \\\n --rlhf_type grpo \\\n --model Qwen/Qwen2.5-VL-3B-Instruct \\\n --external_plugins examples/train/grpo/plugin/plugin.py \\\n --reward_funcs external_r1v_acc format \\\n --use_vllm true \\\n --vllm_mode server \\\n --vllm_server_host 127.0.0.1 \\\n --vllm_server_port 8000 \\\n --tuner_type full \\\n --torch_dtype bfloat16 \\\n --dataset 'lmms-lab/multimodal-open-r1-8k-verified' \\\n --load_from_cache_file true \\\n --max_completion_length 1024 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 8 \\\n --per_device_eval_batch_size 8 \\\n --learning_rate 1e-6 \\\n --gradient_accumulation_steps 2 \\\n --save_strategy 'steps' \\\n --eval_strategy 'steps' \\\n --eval_steps 400 \\\n --save_steps 400 \\\n --save_total_limit 10 \\\n --logging_steps 1 \\\n --output_dir output/GRPO_GEOQA \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 4 \\\n --num_generations 8 \\\n --temperature 1.0 \\\n --repetition_penalty 1.1 \\\n --system 'examples/train/grpo/prompt.txt' \\\n --deepspeed zero3 \\\n --log_completions true \\\n --report_to wandb \\\n --num_iterations 2 \\\n --async_generate false \\\n --beta 0.001 \\\n --max_grad_norm 0.5 \\\n```\n\n#### 实验现象\n\n训练曲线如上图所示。\n一共8k数据，训练了1268个step，Accuracy Reward收敛到0.5左右，completion_length基本收敛到200token左右，reward_std最终在0.2左右震荡。\n以下是训练后的模型completition示例：\n```\nUser:\n\"Based on the map, which state falls into the lowest percentage range of lots, and what is that percentage range?\"\nAssistant:\n\"\\nThe image provided shows a color-coded map of the USA indicating different ranges of lot percentages across various states. According to the legend at the bottom, the lowest percentage range (6.0% - 6.5%) is represented by white. In the image, Alabama (AL) is shaded in white, which corresponds to the 6.0% - 6.5% category. Therefore, based on the map, the state that falls into the lowest percentage range of lots is Alabama, with the percentage range of 6.0% - 6.5%.\\nTherefore, the answer is 6.0% - 6.5%.\\n \\nAlabama \"\n```\n"
},
{
"path": "docs/source/BestPractices/GRPO.md",
"content": "# GRPO完整实验流程\n\n本文从较为简单的数学任务 Coundown Game 出发，从数据集定义、奖励函数定义和GRPO训练几个步骤介绍完整的GRPO训练流程。任务定义和训练参数等参考了 [mini-deepseek-r1](https://github.com/philschmid/deep-learning-pytorch-huggingface/blob/main/training/mini-deepseek-r1-aha-grpo.ipynb)。\n\n## 任务与数据集定义\n\nCoundown Game 的任务目标是根据给定的几个数字和加减乘除四种运算，得到目标数字，因此，我们定义数据集如下：\n```python\nclass CoundownTaskPreprocessor(ResponsePreprocessor):\n\n def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:\n numbers = row['nums']\n target = row.pop('response', None)\n query = f\"\"\"\n Using the numbers {numbers}, create an equation that equals {target}.\n You can use basic arithmetic operations (+, -, *, /) and each number can only be used once.\n Show your work in tags. And return the final equation and answer in tags,\n for example (1 + 2) / 3 * 4 = 4 .\n \"\"\"\n row.update({'target': target, 'query': query})\n return super().preprocess(row)\n\nregister_dataset(\n DatasetMeta(\n ms_dataset_id='zouxuhong/Countdown-Tasks-3to4',\n subsets=['default'],\n preprocess_func=CoundownTaskPreprocessor(),\n tags=['math']))\n```\n通过 template， 使用 numbers 和 target 完成任务定义，并给到 query 字段供模型采样使用。同时，我们需要保留 nums 和 target 两个字段，用于后续的奖励函数计算。\n\n## 奖励函数定义：\n本任务使用的奖励函数有两个，一个是 Deepseek-R1 中提到的格式奖励函数，另一是 Coundown Game 的准确性奖励函数。前者已经在swift中内置，通过 `--reward_funcs format` 可以直接使用，而后者需要我们自己定义，在这里我们使用 external_plugin 的方式定义准确性奖励函数，将代码放在`swift/examples/train/grpo/plugin/plugin.py`中。\n\n在这里，奖励函数的输入包括 completions、target 和 nums 三个字段，分别表示模型生成的文本、目标答案和可用的数字。每个都是list，支持多个 completion 同时计算。注意，在这里，除了 completions 之外的参数都是数据集中定义的字段透传而来，如果有任务上的变动，可以分别对数据集和奖励函数做对应的改变即可。\n```python\nclass CountdownORM(ORM):\n def __call__(self, completions, target, nums, **kwargs) -> List[float]:\n \"\"\"\n Evaluates completions based on Mathematical correctness of the answer\n Args:\n completions (list[str]): Generated outputs\n target (list[str]): Expected answers\n nums (list[str]): Available numbers\n Returns:\n list[float]: Reward scores\n \"\"\"\n rewards = []\n for completion, gt, numbers in zip(completions, target, nums):\n try:\n # Check if the format is correct\n match = re.search(r\"(.*?)<\\/answer>\", completion)\n if match is None:\n rewards.append(0.0)\n continue\n # Extract the \"answer\" part from the completion\n equation = match.group(1).strip()\n if '=' in equation:\n equation = equation.split('=')[0]\n # Extract all numbers from the equation\n used_numbers = [int(n) for n in re.findall(r'\\d+', equation)]\n # Check if all numbers are used exactly once\n if sorted(used_numbers) != sorted(numbers):\n rewards.append(0.0)\n continue\n # Define a regex pattern that only allows numbers, operators, parentheses, and whitespace\n allowed_pattern = r'^[\\d+\\-*/().\\s]+$'\n if not re.match(allowed_pattern, equation):\n rewards.append(0.0)\n continue\n # Evaluate the equation with restricted globals and locals\n result = eval(equation, {'__builtins__': None}, {})\n # Check if the equation is correct and matches the ground truth\n if abs(float(result) - float(gt)) < 1e-5:\n rewards.append(1.0)\n else:\n rewards.append(0.0)\n except Exception as e:\n # If evaluation fails, reward is 0\n rewards.append(0.0)\n return rewards\norms['external_countdown'] = CountdownORM\n```\n\n## GRPO训练实验记录\n首先贴上GRPO的公式：\n\n$$\n{\\scriptstyle\n\\begin{aligned}\n\\mathcal{J}_{G R P O}(\\theta) & =\\mathbb{E}\\left[q \\sim P(Q),\\left\\{o_i\\right\\}_{i=1}^G \\sim \\pi_{\\theta_{o l d}}(O \\mid q)\\right] \\\\\n& \\frac{1}{G} \\sum_{i=1}^G \\frac{1}{\\left|o_i\\right|} \\sum_{t=1}^{\\left|o_i\\right|}\\left\\{\\min \\left[\\frac{\\pi_\\theta\\left(o_{i, t} \\mid q, o_{i,\\nWe need to create an equation using the given numbers [70, 5, 42, 30] exactly once and make it equal to 77. \\n\\nFirst, let's consider the possibility of multiplication or division to get closer to 77. Notice that 77 is close to 70 and 42. If we can use these numbers together, it might work.\\n\\nMultiplying 70 by 1.1 would give us approximately 77, which is very close. So, let's try to make 1.1 using the remaining numbers 5 and 30. \\nIf we take 30/5 = 6, and then 5/6 = 0.8333..., this isn't useful here, so let's look at other possibilities.\\n\\nNext, let's consider if there's another way to achieve 77 using addition, subtraction, or another combination of our numbers. Given the numbers provided, it seems that multiplication or division will be key. \\n\\nWe can try subtracting 5 from 30 to get 25. Then, if we divide 42 by 25, we don't get a whole number, but we can round it to the nearest whole number, 1. Now, we need to make the equation fit 77. Let's multiply 25 by 3 and add 7 (since 77 = 25*3 + 7). However, we only have one 7 available and we need to find a way to use the remaining numbers to get 7.\\n\\nLet's try: \\n70 - 42 = 28 \\nThen, 28 + 5 = 33 \\nNext, 33 + 30 = 63 \\nFinally, 77 - 63 = 14, which doesn't help.\\n\\nConsidering different combinations, one viable way is:\\n(70 - 42) * (30 / 5) = 28 * 6 = 168 / 2 = 84 / 2 = 42 / 2 = 21 / 3 = 7\\n\\nSo, (70 - 42) * (30 / 5) = 77.\\n\\n(70 - 42) * (30 / 5) = 77 \"\n```\nstep 200:\n```\n\"\\nWe need to use all the given numbers in a way that the resultant value of the calculation is 99. Given these restrictions, it's clear we should look into how the numbers relate and the mathematical operations that will bring them together to meet our condition.\\n\\nFirstly, let's evaluate the sums, differences, and multiplication/division among the numbers and their relations to see how this could lead us towards an outcome of 99.\\n- Summing the three numbers: 96 + 70 + 73 = 239, which is far from 99. This shows addition isn't practical.\\n- Subtracting numbers: For instance, subtracting the lowest two (i.e., 73 - 70 = 3) and combining with the highest one (96) does not work. Also, there is no other combination of subtraction that would result in the number needing to add up to 99.\\n- Multiplying or dividing the numbers alone wouldn’t reach to any near solution since none of the products or division results of the numbers themselves is very close to 99. We might think more carefully about the arrangement of them using a combination of the operations provided.\\nTherefore, considering the possible arrangement, a combination like taking (73 - 70 + 96) may produce an effective solution as this will sum the values in such a manner that it aligns with the goal of 99.\\n\\nSo we conclude that the operation: 73 - 70 + 96 gives the right sequence to equal 99.\\n \\n\\n73 - 70 + 96 = 99 \"\n```\nstep 500:\n```\n\"\\nLet's begin by exploring various possible combinations of the given numbers using the arithmetic operators we have access to.\\n\\n- If I add the numbers, I get: 56 + 73 + 51 = 180 which is not equal to our target number (68)\\n- If I subtract one number from another I will still exceed our target value as well since the maximum difference is: |(difference between max-min):| | (73 - 51) or | (73 - 56), etc.|\\n- Let's multiply the numbers individually and see if the division operation can bring us closer to the goal:\\n - (73 * 51) - 56 = 3723 - 56 = 3667 which isn't the goal either.\\n - The only operator left is addition and subtraction; I'll try the combination: \\n ((73 - 56) + 51) gives me (17 + 51) = 68.\\n \\nThus, our solution fits the provided problem constraints and meets the required outcome.\\n \\n(73 - 56) + 51 = 68 \"\n```\n\nstep 1000:\n```\n\"\\nWe need to form an expression using the given numbers (50, 86, and 49) such that it equals 85. Let's try different combinations:\\n\\n(86 - 50) + 49 = 76 + 49 = 125 (too big)\\n(86 - 49) + 50 = 37 + 50 = 87 (too big)\\n(86 - 50) + 49 = 37 + 49 = 86 (too big)\\n(86 + 50) - 49 = 136 - 49 = 87 (too big)\\n(86 - 50) + (49 - x)? Not working!\\n(50 + 86) - 49: this is 136 which isn't our aim.\\n50 + (86 - 49) = 50 + 37 = 87 which wasn't correct.\\n50 - 86 + 49 = -16 + 49: This doesn't match up as well.\\n86 - (50 + 49) = 86 - 99 which doesn't work too.\\n\\nWe try one more:\\n(86 + 49) - 50 = 135 - 50 = 85!\\n\\nThat's it!\\n \\n\\n(86 + 49) - 50 = 85 \"\n```\n\nstep 2000:\n```\n\"\\nWe need to form an expression using the numbers 58, 9, 78, and 44 which evaluates to 73. Let's try different combinations:\\n(78 - 58) + (9 + 44) = 10 + 53 = 63 (too low)\\n(78 - 58) + (9 - 44) = 20 - 35 = -15 (too low)\\n(78 - 58) + (44 - 9) = 20 + 35 = 55 (too low)\\n(78 + 58) - (9 + 44) = 136 - 53 = 83 (too high)\\n(78 + 58) - (9 - 44) = 136 + 35 = 171 (too high)\\n(78 + 58) - (44 + 9) = 136 + 53 = 189 (too high)\\n(78 + 9) - (58 + 44) = 87 - 102 = -15 (too low)\\n(78 + 9) - (58 - 44) = 87 - 14 = 73\\n\\nSo our solution is: (78 + 9) - (58 - 44) = 73 \\n\\n(78 + 9) - (58 - 44) = 73 \"\n```\n\n附learning_rate和beta分别取值1e-6和0.04的不稳定实验记录，模型在step 200左右出现了震荡，format和CountdownORM瞬间走低：\n\n"
},
{
"path": "docs/source/BestPractices/MLLM-Registration.md",
"content": "\n# 注册多模态模型最佳实践\n\n本文将介绍如何在ms-swift中注册多模态模型，并成功推理和训练。本文将以Qwen2.5-Omni为例子，注册新的model_type和template `my_qwen2_5_omni`，并支持文本、图片、视频和音频的训练。由于Qwen2.5-Omni已经在ms-swift中注册，我们可以通过显式指定model_type和template来使用我们自定义的部分。\n\n## 环境准备\n```shell\n# 避免未来出现与文档的不兼容情况\npip install \"ms-swift>=4.0\"\n\npip install \"transformers==4.57.*\" \"qwen_omni_utils==0.0.8\"\n```\n\n\n## 注册模型\n\n第一步，我们需要注册模型，来获取模型和processor。\n\n```python\nfrom transformers import PretrainedConfig, PreTrainedModel\n\nfrom swift.model import (Model, ModelGroup, ModelMeta, MultiModelKeys, get_model_processor, register_model,\n register_model_arch, ModelLoader)\nfrom swift.model.models.qwen import patch_qwen_vl_utils\nfrom swift.model.patcher import patch_get_input_embeddings\nfrom swift.model.utils import use_submodel_func\nfrom swift.utils import get_env_args, Processor\n\nregister_model_arch(\n MultiModelKeys(\n 'my_qwen2_5_omni',\n # `freeze_llm`, `freeze_vit`, `freeze_aligner`将根据下面的值来决定其行为。\n # 例如：全参数训练，若设置`freeze_vit=True`，将冻结以`thinker.audio_tower`和`thinker.visual`为前缀的模型层的参数。\n # LoRA训练，若设置`freeze_vit=False`，将额外为以`thinker.audio_tower`和`thinker.visual`为前缀的Linear层添加LoRA。\n language_model=['thinker.model', 'thinker.lm_head'],\n vision_tower=['thinker.audio_tower', 'thinker.visual'],\n aligner=['thinker.audio_tower.proj', 'thinker.visual.merger'],\n # generator的部分将永远不进行训练或处于冻结状态。\n # 如果你希望`thinker.audio_tower`, `thinker.audio_tower.proj`永远不进行训练，你可以放置到generator中，并将其从vision_tower, aligner中移除。\n generator=['talker', 'token2wav'],\n ))\n\nclass Qwen2_5OmniLoader(ModelLoader):\n\n\n def get_config(self, model_dir: str) -> PretrainedConfig:\n from transformers import Qwen2_5OmniConfig\n config = Qwen2_5OmniConfig.from_pretrained(model_dir, trust_remote_code=True)\n enable_audio_output = get_env_args('ENABLE_AUDIO_OUTPUT', bool, None)\n if enable_audio_output is not None:\n config.enable_audio_output = enable_audio_output\n return config\n\n def get_processor(self, model_dir: str, config: PretrainedConfig) -> Processor:\n from transformers import Qwen2_5OmniProcessor\n from qwen_omni_utils import vision_process\n processor = Qwen2_5OmniProcessor.from_pretrained(model_dir, trust_remote_code=True)\n # Control constants in qwen_omni_utils library via environment variables,\n # e.g., `MAX_PIXELS`, etc.\n patch_qwen_vl_utils(vision_process)\n return processor\n\n def get_model(self, model_dir: str, config: PretrainedConfig, processor: Processor,\n model_kwargs) -> PreTrainedModel:\n from transformers import Qwen2_5OmniForConditionalGeneration\n print('Run my_qwen2_5_omni...')\n self.auto_model_cls = self.auto_model_cls or Qwen2_5OmniForConditionalGeneration\n model = super().get_model(model_dir, config, processor, model_kwargs)\n # For multimodal model consistency, we replace the model's forward/generate functions\n # with those of its language_model.\n # Handle additional parts separately.\n use_submodel_func(model, 'thinker')\n # Avoid inplace operations on leaf_variable during training\n # (replacing parts of input_embeds with images_embeds)\n patch_get_input_embeddings(model.thinker.visual, 'patch_embed')\n # Some custom settings for model/config (usually not needed; configure based on\n # specific model if errors occur during training/inference)\n model.config.keys_to_ignore_at_inference += ['hidden_states', 'attention_mask']\n model.config.talker_config.pad_token_id = None\n return model\n\n\nregister_model(\n ModelMeta(\n 'my_qwen2_5_omni',\n [\n ModelGroup([\n Model('Qwen/Qwen2.5-Omni-3B', 'Qwen/Qwen2.5-Omni-3B'),\n Model('Qwen/Qwen2.5-Omni-7B', 'Qwen/Qwen2.5-Omni-7B'),\n ]),\n ],\n # 用来获取model和processor的函数。\n Qwen2_5OmniLoader,\n template='my_qwen2_5_omni',\n is_multimodal=True, # 是否是多模态模型\n model_arch='my_qwen2_5_omni', # 通常只为多模态模型设置\n # 用于model_type的自动匹配\n architectures=['Qwen2_5OmniModel', 'Qwen2_5OmniForConditionalGeneration'],\n # 用来提示用户依赖版本（可删除）\n requires=['transformers>=4.50', 'soundfile', 'qwen_omni_utils', 'decord'],\n # 用来提示用户（可删除）\n tags=['vision', 'video', 'audio'],\n # 全参数训练/merge-lora需要额外保存的文件\n additional_saved_files=['spk_dict.pt'],\n ))\n\nif __name__ == '__main__':\n # 测试与debug\n model, processor = get_model_processor('Qwen/Qwen2.5-Omni-7B', model_type='my_qwen2_5_omni')\n```\n\n## 注册模板\n\n第二步，我们需要注册模板，来自定义如何将文本、图片、视频和音频进行预处理（`_encode`和`_data_collator`方法）。这是ms-swift支持多模态模型训练的关键模块。预处理方式请参考transformers推理实现，并进行对齐。\n\ntemplate的功能如下：\n1. 支持正常推理与训练，预处理文本和多模态信息，并支持grounding任务。\n2. 支持padding_free和packing训练。\n3. 支持混合模态数据训练。\n\n\n```python\nfrom functools import partial\nfrom typing import Any, Dict, List, Literal, Optional\n\nimport torch\nfrom transformers.integrations import is_deepspeed_zero3_enabled\nfrom swift import get_model_processor\nfrom swift.template import StdTemplateInputs, Template, TemplateMeta, get_template, register_template\nfrom swift.template.utils import Context, findall\nfrom swift.template.vision_utils import load_audio\nfrom swift.utils import Processor, get_env_args, get_logger, get_packed_seq_params, is_deepspeed_enabled, to_float_dtype\n\nlogger = get_logger()\n\nclass Qwen2_5OmniTemplate(Template):\n use_model = True # 是否在预处理的过程中需要model参与\n # 需要注意是：并不是所有的多模态模型都能支持padding_free/packing。`transformers`库内的模型通常可以支持\n support_padding_free = True # 是否支持padding_free和packing（多模态模型）\n norm_bbox = 'none' # grounding任务使用绝对坐标还是norm1000坐标\n\n # 这里的tokens将不会被裁剪（例如设置`--truncation_strategy left/right`）\n # 并会使用简略方式打印（调用`template.safe_decode`）\n placeholder_tokens = ['<|IMAGE|>', '<|AUDIO|>', '<|VIDEO|>']\n\n def init_processor(self, processor: Processor) -> None:\n \"\"\"在初始化processor时，额外初始化所需的一些常量\"\"\"\n if processor is None:\n return\n super().init_processor(processor)\n from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessorKwargs\n default = Qwen2_5OmniProcessorKwargs._defaults\n self.seconds_per_chunk = default['videos_kwargs']['seconds_per_chunk']\n self.position_id_per_seconds = default['videos_kwargs']['position_id_per_seconds']\n self.use_audio_in_video = get_env_args('use_audio_in_video', bool, False)\n self.sampling_rate = get_env_args('sampling_rate', int, self.processor.feature_extractor.sampling_rate)\n # `QWENVL_BBOX_FORMAT`的含义参考grounding数据集自定义文档\n self.bbox_format = get_env_args('QWENVL_BBOX_FORMAT', str, 'legacy')\n\n\n def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,\n inputs: StdTemplateInputs) -> List[Context]:\n \"\"\"读取多模态数据，并替换通用多模态tag。\n 例如：图像tag从`` -> `<|vision_bos|><|IMAGE|><|vision_eos|>`\"\"\"\n # 读取多模态数据也可以在`_encode`函数中进行，怎么方便怎么来。\n from qwen_omni_utils import fetch_image, fetch_video\n if media_type == 'image':\n inputs.images[index] = fetch_image({'image': inputs.images[index]})\n return ['<|vision_bos|><|IMAGE|><|vision_eos|>']\n elif media_type == 'audio':\n if self.mode != 'vllm': # 'vllm'推理场景下不需要处理\n inputs.audios[index] = load_audio(inputs.audios[index], self.sampling_rate)\n return ['<|audio_bos|><|AUDIO|><|audio_eos|>']\n elif media_type == 'video':\n video = inputs.videos[index]\n _video = fetch_video({'video': video})\n if isinstance(_video, torch.Tensor):\n _video = _video.to(torch.uint8)\n inputs.videos[index] = _video\n if self.use_audio_in_video:\n import librosa\n if video.startswith('http://') or video.startswith('https://'):\n import audioread\n video = audioread.ffdec.FFmpegAudioFile(video)\n video = librosa.load(video, sr=self.sampling_rate)[0]\n inputs.audios.insert(inputs.audio_idx, (video, 'video'))\n inputs.audio_idx += 1\n return ['<|vision_bos|><|audio_bos|><|VIDEO|><|audio_eos|><|vision_eos|>']\n else:\n return ['<|vision_bos|><|VIDEO|><|vision_eos|>']\n\n\n def replace_ref(self, ref: str, index: int, inputs: StdTemplateInputs) -> List[Context]:\n \"\"\"替换grounding任务的通用tag: ``\"\"\"\n if self.bbox_format == 'legacy':\n return [f'<|object_ref_start|>{ref}<|object_ref_end|>']\n else:\n return [ref]\n\n def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:\n \"\"\"替换grounding任务的通用tag: ``\"\"\"\n if self.bbox_format == 'legacy':\n return [f'<|box_start|>{self._get_bbox_str(bbox)}<|box_end|>']\n else:\n return [str(bbox)]\n\n def packing_row(self, row: List[Dict[str, Any]]) -> Dict[str, Any]:\n \"\"\"支持packing & mrope。通常情况不需要继承该函数，这里为了自定义mrope的position_ids。\"\"\"\n position_ids = []\n for r in row:\n r = r.copy()\n r['input_ids'] = torch.tensor(r['input_ids'])[None]\n position_ids.append(self._get_position_ids(r))\n packed = super().packing_row(row)\n packed['position_ids'] = torch.concat(position_ids, dim=-1)\n return packed\n\n def _get_new_tokens_use_audio_in_video(self, i, *, video_grid_thw, video_second_per_grid, audio_lengths,\n video_token_id, audio_token_id):\n \"\"\"辅助函数，用于支持`use_audio_in_video`为True的情况\"\"\"\n merge_size = self.processor.image_processor.merge_size\n grid_thw = video_grid_thw[i]\n height = grid_thw[1] // merge_size\n width = grid_thw[2] // merge_size\n audio_token_indices = torch.arange(audio_lengths[i])\n video_token_indices = torch.arange(grid_thw[0]).reshape(-1, 1, 1)\n\n video_token_indices = torch.broadcast_to(video_token_indices,\n (video_token_indices.shape[0], height, width)).reshape(-1)\n video_token_indices = (video_token_indices * video_second_per_grid[i] * self.position_id_per_seconds)\n tokens_per_chunk = int(self.position_id_per_seconds * self.seconds_per_chunk)\n video_chunk_indexes = self.processor.get_chunked_index(video_token_indices, tokens_per_chunk)\n audio_chunk_indexes = self.processor.get_chunked_index(audio_token_indices, tokens_per_chunk)\n\n res = []\n for j in range(max(len(video_chunk_indexes), len(audio_chunk_indexes))):\n if j < len(video_chunk_indexes):\n video_seq_length = video_chunk_indexes[j][1] - video_chunk_indexes[j][0]\n res += video_token_id * video_seq_length\n if j < len(audio_chunk_indexes):\n audio_seq_length = audio_chunk_indexes[j][1] - audio_chunk_indexes[j][0]\n res += audio_token_id * audio_seq_length\n return res\n\n\n def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:\n \"\"\"这里决定如何将text/images/audios/videos -> input_ids、labels、loss_scale以及pixel_values等多模态内容\n 这里的处理逻辑通常可以从对应模型的预处理代码实现中借鉴。\n 推荐：请先做推理对齐再做训练\"\"\"\n encoded = Template._encode(self, inputs) # 处理纯文本部分，具体请参考自定义模型文档\n logger.info_once('Run qwen2_5_omni template')\n processor = self.processor\n # 获取多模态内容\n media_inputs = processor(\n text='',\n audio=inputs.audios or None,\n images=inputs.images or None,\n videos=inputs.videos or None,\n do_resize=False,\n return_tensors='pt')\n # 我们不使用`processor`产生的input_ids和attention_mask。因为其不产生`labels`。\n media_inputs.pop('input_ids')\n media_inputs.pop('attention_mask')\n media_inputs = to_float_dtype(media_inputs, self.model_info.torch_dtype)\n\n input_ids = encoded['input_ids']\n labels = encoded['labels']\n loss_scale = encoded.get('loss_scale', None)\n # audio模态\n audio_token_id = self._tokenize('<|AUDIO|>')\n idx_list = findall(input_ids, audio_token_id) # 查找所有的audio_token\n feature_attention_mask = media_inputs.get('feature_attention_mask')\n if feature_attention_mask is not None:\n audio_feature_lengths = torch.sum(feature_attention_mask, dim=1)\n audio_lengths = ((audio_feature_lengths - 1) // 2 + 1 - 2) // 2 + 1\n else:\n audio_lengths = None\n audio_lengths_origin = audio_lengths\n # video_audios_mask用于处理`use_audio_in_video`，区分是纯audio(0)还是video中的audio(1)\n video_audios_mask = []\n for i, audio in enumerate(inputs.audios):\n if isinstance(audio, tuple) and audio[1] == 'video':\n inputs.audios[i] = audio[0]\n video_audios_mask.append(True)\n else:\n video_audios_mask.append(False)\n video_audios_mask = torch.tensor(video_audios_mask)\n if idx_list:\n # 过滤掉video中的audio的内容（将在video部分处理）\n if self.use_audio_in_video:\n audio_lengths = audio_lengths[~video_audios_mask]\n\n def _get_new_audio_tokens(i):\n return audio_token_id * audio_lengths[i]\n\n # 对input_ids的多模态tokens进行展开\n input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,\n _get_new_audio_tokens)\n\n # image和video模态\n for media_type in ['image', 'video']:\n token = f'<|{media_type.upper()}|>'\n token_id = self._tokenize(token)\n idx_list = findall(input_ids, token_id)\n if idx_list:\n merge_size = processor.image_processor.merge_size\n media_grid_thw = media_inputs.get(f'{media_type}_grid_thw')\n if media_type == 'video' and self.use_audio_in_video:\n audio_lengths = audio_lengths_origin[video_audios_mask]\n video_second_per_grid = media_inputs['video_second_per_grid']\n _get_new_tokens_use_audio_in_video = partial(\n self._get_new_tokens_use_audio_in_video,\n video_grid_thw=media_grid_thw,\n video_second_per_grid=video_second_per_grid,\n audio_lengths=audio_lengths,\n video_token_id=token_id,\n audio_token_id=audio_token_id)\n input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,\n _get_new_tokens_use_audio_in_video)\n\n else:\n\n def _get_new_tokens(i):\n token_len = (media_grid_thw[i].prod() // (merge_size**2))\n return token_id * token_len\n\n input_ids, labels, loss_scale = self._extend_tokens(input_ids, labels, loss_scale, idx_list,\n _get_new_tokens)\n\n encoded['input_ids'] = input_ids\n encoded['labels'] = labels\n encoded['loss_scale'] = loss_scale\n encoded.update(media_inputs) # 将多模态内容加入其中\n return encoded\n\n def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:\n \"\"\"该函数通常用于解决混合模型训练zero2/zero3卡住的问题，\n 即有的进程为纯文本数据未过vit，有的进程含图片数据过了vit。这里将创建dummy_image来解决。\n\n 该函数将被注册在`model.forward`前的pre_forward_hook中。\n 该函数需返回 含多模态信息的input_embeds。\n \"\"\"\n if not self.is_training:\n return inputs\n\n input_ids = inputs['input_ids']\n input_features = inputs.get('input_features')\n feature_attention_mask = inputs.get('feature_attention_mask')\n\n base_model = self.get_base_model(model)\n inputs_embeds = base_model.thinker.model.embed_tokens(input_ids)\n thinker_config = model.config.thinker_config\n # 辅助函数，用于处理text/image/video混合模态数据场景。（内部会创建dummy_image）\n inputs_embeds = self._get_inputs_embeds_hf(inputs_embeds, inputs, model.thinker.visual, self.processor,\n thinker_config)\n # 含audio的混合模态数据场景\n if input_features is None:\n if is_deepspeed_enabled() and not is_deepspeed_zero3_enabled():\n # 注意: 由于transformers实现中，经过audio部分模型层的次数与audio数量相关\n # 因此zero3在不同进程audios数不同场景下会卡住（需修改transformers代码修复）。此场景请使用zero2。\n input_features = input_ids.new_zeros([1, 128, 128], dtype=model.thinker.audio_tower.dtype)\n feature_attention_mask = input_ids.new_ones([1, 128], dtype=torch.bool)\n audio_res = model.thinker.get_audio_features(input_features, feature_attention_mask)\n # 兼容transformers 5.0\n if hasattr(audio_res, 'last_hidden_state'):\n audio_embeds = audio_res.last_hidden_state\n else:\n audio_embeds = audio_res\n inputs_embeds = inputs_embeds + audio_embeds.mean() * 0.\n else:\n audio_res = model.thinker.get_audio_features(input_features, feature_attention_mask)\n # 兼容transformers 5.0\n if hasattr(audio_res, 'last_hidden_state'):\n audio_embeds = audio_res.last_hidden_state\n else:\n audio_embeds = audio_res\n audio_mask = (input_ids == thinker_config.audio_token_index).unsqueeze(-1).expand_as(inputs_embeds)\n audio_embeds = audio_embeds.to(inputs_embeds.device, inputs_embeds.dtype)\n inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_embeds)\n\n return {'inputs_embeds': inputs_embeds}\n\n def _get_position_ids(self, inputs: Dict[str, Any]):\n \"\"\"辅助函数，用来获取mrope的position_ids\"\"\"\n feature_attention_mask = inputs.get('feature_attention_mask')\n if feature_attention_mask is not None:\n audio_feature_lengths = torch.sum(feature_attention_mask, dim=1)\n else:\n audio_feature_lengths = None\n video_second_per_grid = inputs.pop('video_second_per_grid', None)\n input_ids = inputs['input_ids']\n attention_mask = inputs.get('attention_mask')\n if attention_mask is None:\n attention_mask = torch.ones_like(input_ids)\n position_ids, _ = self.model.thinker.get_rope_index(\n input_ids,\n inputs.get('image_grid_thw'),\n inputs.get('video_grid_thw'),\n attention_mask,\n self.use_audio_in_video,\n audio_feature_lengths,\n video_second_per_grid,\n )\n return self._concat_text_position_ids(position_ids) # 第一维为text_position_ids\n\n def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:\n \"\"\"传入dataloader的`collate_fn`\"\"\"\n res = super()._data_collator(batch, padding_to=padding_to)\n if not self.padding_free and self.is_training:\n # 其中padding_free/packing场景将会在packing_row中处理position_ids。\n res['position_ids'] = self._get_position_ids(res)\n if 'position_ids' in res:\n # 创建`packed_seq_params`以支持padding_free/packing & flash-attn\n position_ids = res['position_ids']\n res['position_ids'] = position_ids[1:]\n res['text_position_ids'] = text_position_ids = position_ids[0]\n # https://github.com/huggingface/transformers/pull/40194\n res.update(get_packed_seq_params(text_position_ids))\n return res\n\n def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:\n \"\"\"处理`_data_collator`函数中的多模态部分。（该函数兼容padding_free/packing）\"\"\"\n res = super()._data_collator_mm_data(batch)\n video_second_per_grid = self.gather_list(batch, 'video_second_per_grid')\n if video_second_per_grid:\n res['video_second_per_grid'] = video_second_per_grid\n input_features = [b['input_features'] for b in batch if b.get('input_features') is not None]\n feature_attention_mask = [\n b['feature_attention_mask'] for b in batch if b.get('feature_attention_mask') is not None\n ]\n if input_features:\n res['input_features'] = torch.concat(input_features)\n res['feature_attention_mask'] = torch.concat(feature_attention_mask)\n return res\n\n def generate(self, model, *args, **kwargs):\n \"\"\"`TransformersEngine`会调用template.generate方法进行文本生成，这里继承进行自定义。\"\"\"\n if kwargs.get('video_grid_thw') is not None:\n kwargs['use_audio_in_video'] = self.use_audio_in_video\n return super().generate(model, *args, **kwargs)\n\n\nregister_template(\n TemplateMeta('my_qwen2_5_omni', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'],\n chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>'],\n system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'],\n default_system='You are a helpful assistant.', stop_words=['<|endoftext|>'],\n agent_template='hermes',\n template_cls=Qwen2_5OmniTemplate))\n\nif __name__ == '__main__':\n # 测试与debug\n model, processor = get_model_processor('Qwen/Qwen2.5-Omni-7B', model_type='my_qwen2_5_omni')\n template = get_template(processor, template_type='my_qwen2_5_omni')\n data = {\n 'messages': [\n {'role': 'user', 'content': '描述视频与图片内容。'},\n {'role': 'assistant', 'content': '一个小孩和一只猫咪。'},\n ],\n 'videos': ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'],\n 'images': ['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png'],\n }\n template.set_mode('train')\n encoded = template.encode(data)\n print('input_ids: ' + template.safe_decode(encoded['input_ids']))\n print('labels: ' + template.safe_decode(encoded['labels']))\n print('keys: ' + str(encoded.keys()))\n```\n\n\n## 推理对齐\n接下来，你需要进行TransformersEngine与transformers的推理对齐。通常你需要对齐`input_ids`以及输出内容。你可以书写以下测试函数：\n\n```python\nimport os\nfrom transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor\nfrom qwen_omni_utils import process_mm_info\nfrom modelscope import snapshot_download\nfrom swift.infer_engine import TransformersEngine, InferRequest, RequestConfig\nimport requests\n\ndef infer_hf():\n model_dir = snapshot_download('Qwen/Qwen2.5-Omni-7B')\n model = Qwen2_5OmniForConditionalGeneration.from_pretrained(\n model_dir, torch_dtype=\"auto\", device_map=\"auto\", attn_implementation='flash_attention_2')\n processor = Qwen2_5OmniProcessor.from_pretrained(model_dir)\n # 使用decord读取视频（暂不支持url）\n resp = requests.get('https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4')\n with open('_baby.mp4', 'wb') as f:\n f.write(resp.content)\n\n conversation = [\n {\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"video\", \"video\": \"_baby.mp4\"},\n {\"type\": \"image\", \"image\": \"http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png\"},\n {\"type\": \"text\", \"text\": \"描述视频和图像。\"},\n ],\n },\n ]\n\n USE_AUDIO_IN_VIDEO = False\n text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)\n audios, images, videos = process_mm_info(conversation, use_audio_in_video=USE_AUDIO_IN_VIDEO)\n inputs = processor(text=text, audio=audios, images=images, videos=videos, return_tensors=\"pt\", padding=True,\n use_audio_in_video=USE_AUDIO_IN_VIDEO)\n inputs = inputs.to(model.device).to(model.dtype)\n text_ids = model.generate(**inputs, use_audio_in_video=USE_AUDIO_IN_VIDEO, thinker_do_sample=False,\n return_audio=False)\n text = processor.batch_decode(text_ids[:, inputs['input_ids'].shape[1]:], skip_special_tokens=True, clean_up_tokenization_spaces=False)\n return inputs['input_ids'][0].tolist(), text[0]\n\ndef test_my_qwen2_5_omni():\n engine = TransformersEngine('Qwen/Qwen2.5-Omni-7B', model_type='my_qwen2_5_omni', attn_impl='flash_attention_2')\n infer_request = InferRequest(messages=[{\n \"role\": \"user\",\n \"content\": \"描述视频和图像。\",\n }],\n videos=[\"https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4\"],\n images=[\"http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png\"],\n )\n request_config = RequestConfig(temperature=0, max_tokens=512)\n input_ids = engine.template.encode(infer_request)['input_ids']\n resp_list = engine.infer([infer_request], request_config)\n resp = resp_list[0].choices[0].message.content\n return input_ids, resp\n\n\nif __name__ == '__main__':\n # 开启debug模式，会打印`TransformersEngine.infer`的input_ids和generate_ids\n os.environ['SWIFT_DEBUG'] = '1'\n input_ids_hf, response_hf = infer_hf()\n input_ids_swift, response_swift = test_my_qwen2_5_omni()\n # 测试input_ids和response对齐\n assert input_ids_hf == input_ids_swift\n assert response_hf == response_swift\n```\n\n\n## 开始训练\n\n使用python代码训练，这通常更容易debug：\n```python\nfrom swift import sft_main, SftArguments\nimport os\nif __name__ == '__main__':\n os.environ['MAX_PIXELS'] = '1003520'\n sft_main(SftArguments(\n model='Qwen/Qwen2.5-Omni-7B',\n dataset=['AI-ModelScope/LaTeX_OCR#5000'],\n model_type='my_qwen2_5_omni',\n template='my_qwen2_5_omni',\n load_from_cache_file=True,\n split_dataset_ratio=0.01,\n tuner_type='lora',\n torch_dtype='bfloat16',\n attn_impl='flash_attn',\n padding_free=True,\n num_train_epochs=1,\n per_device_train_batch_size=16,\n per_device_eval_batch_size=16,\n learning_rate=1e-4,\n lora_rank=8,\n lora_alpha=32,\n target_modules=['all-linear'],\n freeze_vit=True,\n freeze_aligner=True,\n gradient_accumulation_steps=1,\n eval_steps=50,\n save_steps=50,\n save_total_limit=2,\n logging_steps=5,\n max_length=2048,\n output_dir='output',\n warmup_ratio=0.05,\n dataloader_num_workers=4,\n dataset_num_proc=1,\n ))\n```\n\n使用命令行训练：\n```shell\n# 4 * 35GiB\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nNPROC_PER_NODE=4 \\\nVIDEO_MAX_PIXELS=50176 \\\nFPS_MAX_FRAMES=12 \\\nMAX_PIXELS=1003520 \\\nswift sft \\\n --model Qwen/Qwen2.5-Omni-7B \\\n --model_type my_qwen2_5_omni \\\n --template my_qwen2_5_omni \\\n --external_plugins 'examples/custom/my_qwen2_5_omni/my_register.py' \\\n --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#2000' \\\n 'AI-ModelScope/LaTeX_OCR:human_handwrite#2000' \\\n 'speech_asr/speech_asr_aishell1_trainsets:validation#2000' \\\n 'swift/VideoChatGPT:all#2000' \\\n --load_from_cache_file true \\\n --split_dataset_ratio 0.01 \\\n --tuner_type lora \\\n --torch_dtype bfloat16 \\\n --attn_impl flash_attn \\\n --padding_free true \\\n --packing true \\\n --num_train_epochs 3 \\\n --per_device_train_batch_size 1 \\\n --per_device_eval_batch_size 1 \\\n --learning_rate 1e-4 \\\n --lora_rank 8 \\\n --lora_alpha 32 \\\n --target_modules all-linear \\\n --freeze_vit true \\\n --freeze_aligner true \\\n --gradient_accumulation_steps 1 \\\n --eval_steps 50 \\\n --save_steps 50 \\\n --save_total_limit 2 \\\n --logging_steps 5 \\\n --max_length 4096 \\\n --output_dir output \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 4 \\\n --dataset_num_proc 1 \\\n --deepspeed zero2\n```\n\n训练后对验证集进行推理：（环境变量请与训练时对齐）\n```shell\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nCUDA_VISIBLE_DEVICES=0 \\\nVIDEO_MAX_PIXELS=50176 \\\nFPS_MAX_FRAMES=12 \\\nMAX_PIXELS=1003520 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --max_new_tokens 512 \\\n --load_data_args true\n```\n\n使用以下命令将训练权重推送到 Modelscope：\n```shell\nswift export \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --push_to_hub true \\\n --hub_model_id '' \\\n --hub_token ''\n```\n"
},
{
"path": "docs/source/BestPractices/Metax-support.md",
"content": "# Metax支持\n\n## 1. 在 Metax 平台上使用 Swift\n你可以选择构建自己的镜像，也可以直接拉取已有的预构建镜像。本文以拉取预构建镜像为例，演示如何在 Metax 上使用 ms-swift。\n### 1.1. 启动 ms-swift 容器\n```bash\ndocker pull mx-devops-acr-cn-shanghai.cr.volces.com/opensource/public-ai-release/maca/ms-swift:3.10.3-maca.ai3.3.0.16-torch2.6-py310-ubuntu22.04-amd64\n# 你可以根据需要调整 --privileged 参数，并仅挂载特定的 GPU 卡。\n# 更多信息请参考我们的官方文档：https://developer.metax-tech.com\n# 必须通过 --device 挂载 Metax GPU 设备：--device=/dev/dri --device=/dev/mxcd\ndocker run -it --net=host --uts=host --ipc=host --privileged=true --group-add video \\\n --shm-size 100gb --ulimit memlock=-1 \\\n --security-opt seccomp=unconfined --security-opt apparmor=unconfined \\\n --device=/dev/dri --device=/dev/mxcd \\\n -v /root/workspace:/external \\\n --name swift_test \\\n mx-devops-acr-cn-shanghai.cr.volces.com/opensource/public-ai-release/maca/ms-swift:3.10.3-maca.ai3.3.0.16-torch2.6-py310-ubuntu22.04-amd64\n```\n## 2. 环境检查\n### 2.1. 检查 Metax GPU 是否可用\n得益于与 CUDA 的兼容性，我们可以像使用 NVIDIA GPU 一样检查 Metax 设备是否可用：\n```python\nimport torch\nprint(torch.cuda.is_available())\n# True\n```\n### 2.2. 检查 GPU 之间的 P2P 连接拓扑\n```bash\nmx-smi topo -m\n# output\n=================== MetaX System Management Interface Log ===================\nTimestamp : Wed Feb 11 16:37:10 2026\n\nAttached GPUs : 8\nDevice link type matrix\n GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 Node Affinity CPU Affinity\nGPU0 X MX MX MX NODE NODE NODE NODE 0 0-31,64-95\nGPU1 MX X MX MX NODE NODE NODE NODE 0 0-31,64-95\nGPU2 MX MX X MX NODE NODE NODE NODE 0 0-31,64-95\nGPU3 MX MX MX X NODE NODE NODE NODE 0 0-31,64-95\nGPU4 NODE NODE NODE NODE X MX MX MX 0 0-31,64-95\nGPU5 NODE NODE NODE NODE MX X MX MX 0 0-31,64-95\nGPU6 NODE NODE NODE NODE MX MX X MX 0 0-31,64-95\nGPU7 NODE NODE NODE NODE MX MX MX X 0 0-31,64-95\n\nLegend:\n X = Self\n SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)\n NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node\n PHB = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)\n PXB = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)\n PIX = Connection traversing at most a single PCIe bridge\n MX = Connection traversing MetaXLink\n ETH = Connection traversing Eth\n NA = Connection type is unknown\n```\n### 2.3. 查看 GPU 状态\n```bash\nmx-smi\n# output\n =================== MetaX System Management Interface Log ===================\nTimestamp : Wed Feb 11 09:55:49 2026\n\nAttached GPUs : 8\n+---------------------------------------------------------------------------------+\n| MX-SMI 2.2.9 Kernel Mode Driver Version: 3.4.4 |\n| MACA Version: 3.3.0.15 BIOS Version: 1.30.0.0 |\n|------------------+-----------------+---------------------+----------------------|\n| Board Name | GPU Persist-M | Bus-id | GPU-Util sGPU-M |\n| Pwr:Usage/Cap | Temp Perf | Memory-Usage | GPU-State |\n|==================+=================+=====================+======================|\n| 0 MetaX C500 | 0 Off | 0000:0e:00.0 | 0% Disabled |\n| 57W / 350W | 35C P0 | 826/65536 MiB | Available |\n+------------------+-----------------+---------------------+----------------------+\n| 1 MetaX C500 | 1 Off | 0000:0f:00.0 | 0% Disabled |\n| 58W / 350W | 37C P0 | 826/65536 MiB | Available |\n+------------------+-----------------+---------------------+----------------------+\n| 2 MetaX C500 | 2 Off | 0000:10:00.0 | 0% Disabled |\n| 58W / 350W | 36C P0 | 826/65536 MiB | Available |\n+------------------+-----------------+---------------------+----------------------+\n| 3 MetaX C500 | 3 Off | 0000:12:00.0 | 0% Disabled |\n| 60W / 350W | 35C P0 | 826/65536 MiB | Available |\n+------------------+-----------------+---------------------+----------------------+\n| 4 MetaX C500 | 4 Off | 0000:35:00.0 | 0% Disabled |\n| 57W / 350W | 33C P0 | 826/65536 MiB | Available |\n+------------------+-----------------+---------------------+----------------------+\n| 5 MetaX C500 | 5 Off | 0000:36:00.0 | 0% Disabled |\n| 56W / 350W | 34C P0 | 826/65536 MiB | Available |\n+------------------+-----------------+---------------------+----------------------+\n| 6 MetaX C500 | 6 Off | 0000:37:00.0 | 0% Disabled |\n| 55W / 350W | 34C P0 | 826/65536 MiB | Available |\n+------------------+-----------------+---------------------+----------------------+\n| 7 MetaX C500 | 7 Off | 0000:38:00.0 | 0% Disabled |\n| 56W / 350W | 36C P0 | 826/65536 MiB | Available |\n+------------------+-----------------+---------------------+----------------------+\n\n+---------------------------------------------------------------------------------+\n| Process: |\n| GPU PID Process Name GPU Memory |\n| Usage(MiB) |\n|=================================================================================|\n| no process found |\n+---------------------------------------------------------------------------------+\n```\n\n## 3. 运行示例\n我们支持直接使用社区版 Swift，同时在镜像中 /workspace 目录下提供了经过更多优化的版本。强烈建议优先使用该目录下的软件包。\n\n### 3.1. 运行 Swift 示例\n在大多数场景下，可直接运行 Swift 的训练示例：\n```bash\n# We assume that the ms-swift code is under /workspace\ncd /workspace/ms-swift/\nbash examples/train/full/train.sh\n\n```\n运行输出示例（节选）：\n```bash\n# output:\n{'loss': 1.47077751, 'grad_norm': 10.5625, 'learning_rate': 2e-06, 'token_acc': 0.65511727, 'epoch': 0.01, 'global_step/max_steps': '1/94', 'percentage': '1.06%', 'elapsed_time': '2s', 'remaining_time': '4m 28s', 'memory(GiB)': 4.87, 'train_speed(iter/s)': 0.345807}\n{'loss': 1.58882141, 'grad_norm': 10.75, 'learning_rate': 1e-05, 'token_acc': 0.61763144, 'epoch': 0.05, 'global_step/max_steps': '5/94', 'percentage': '5.32%', 'elapsed_time': '10s', 'remaining_time': '3m 12s', 'memory(GiB)': 5.64, 'train_speed(iter/s)': 0.461462}\n{'loss': 1.56617603, 'grad_norm': 12.8125, 'learning_rate': 9.92e-06, 'token_acc': 0.61519274, 'epoch': 0.11, 'global_step/max_steps': '10/94', 'percentage': '10.64%', 'elapsed_time': '20s', 'remaining_time': '2m 52s', 'memory(GiB)': 5.64, 'train_speed(iter/s)': 0.485796}\n{'loss': 1.63347206, 'grad_norm': 13.6875, 'learning_rate': 9.69e-06, 'token_acc': 0.60373975, 'epoch': 0.16, 'global_step/max_steps': '15/94', 'percentage': '15.96%', 'elapsed_time': '30s', 'remaining_time': '2m 39s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.493855}\n{'loss': 1.60613976, 'grad_norm': 11.0, 'learning_rate': 9.32e-06, 'token_acc': 0.59997221, 'epoch': 0.21, 'global_step/max_steps': '20/94', 'percentage': '21.28%', 'elapsed_time': '39s', 'remaining_time': '2m 27s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.500516}\n{'loss': 1.45015478, 'grad_norm': 15.25, 'learning_rate': 8.8e-06, 'token_acc': 0.62373584, 'epoch': 0.27, 'global_step/max_steps': '25/94', 'percentage': '26.60%', 'elapsed_time': '49s', 'remaining_time': '2m 16s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.50548}\n{'loss': 1.39427547, 'grad_norm': 13.9375, 'learning_rate': 8.18e-06, 'token_acc': 0.6357994, 'epoch': 0.32, 'global_step/max_steps': '30/94', 'percentage': '31.91%', 'elapsed_time': '59s', 'remaining_time': '2m 5s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.508409}\n{'loss': 1.53672237, 'grad_norm': 11.125, 'learning_rate': 7.45e-06, 'token_acc': 0.61650612, 'epoch': 0.37, 'global_step/max_steps': '35/94', 'percentage': '37.23%', 'elapsed_time': '1m 8s', 'remaining_time': '1m 55s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.510425}\n{'loss': 1.54039021, 'grad_norm': 13.8125, 'learning_rate': 6.65e-06, 'token_acc': 0.61613974, 'epoch': 0.43, 'global_step/max_steps': '40/94', 'percentage': '42.55%', 'elapsed_time': '1m 18s', 'remaining_time': '1m 45s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.512302}\n{'loss': 1.40159426, 'grad_norm': 9.4375, 'learning_rate': 5.79e-06, 'token_acc': 0.64041773, 'epoch': 0.48, 'global_step/max_steps': '45/94', 'percentage': '47.87%', 'elapsed_time': '1m 27s', 'remaining_time': '1m 35s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.512983}\n{'loss': 1.54977188, 'grad_norm': 11.9375, 'learning_rate': 4.91e-06, 'token_acc': 0.61078816, 'epoch': 0.53, 'global_step/max_steps': '50/94', 'percentage': '53.19%', 'elapsed_time': '1m 37s', 'remaining_time': '1m 25s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.514489}\n{'loss': 1.6754509, 'grad_norm': 13.0625, 'learning_rate': 4.04e-06, 'token_acc': 0.58574393, 'epoch': 0.59, 'global_step/max_steps': '55/94', 'percentage': '58.51%', 'elapsed_time': '1m 46s', 'remaining_time': '1m 15s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.515752}\n{'loss': 1.37204351, 'grad_norm': 9.25, 'learning_rate': 3.19e-06, 'token_acc': 0.6391937, 'epoch': 0.64, 'global_step/max_steps': '60/94', 'percentage': '63.83%', 'elapsed_time': '1m 56s', 'remaining_time': '1m 5s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.516829}\n{'loss': 1.47697926, 'grad_norm': 11.375, 'learning_rate': 2.4e-06, 'token_acc': 0.62817259, 'epoch': 0.69, 'global_step/max_steps': '65/94', 'percentage': '69.15%', 'elapsed_time': '2m 5s', 'remaining_time': '55s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.517947}\n{'loss': 1.4336628, 'grad_norm': 8.125, 'learning_rate': 1.69e-06, 'token_acc': 0.63453862, 'epoch': 0.75, 'global_step/max_steps': '70/94', 'percentage': '74.47%', 'elapsed_time': '2m 14s', 'remaining_time': '46s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.518833}\n{'loss': 1.54315252, 'grad_norm': 9.625, 'learning_rate': 1.08e-06, 'token_acc': 0.60202073, 'epoch': 0.8, 'global_step/max_steps': '75/94', 'percentage': '79.79%', 'elapsed_time': '2m 24s', 'remaining_time': '36s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.519627}\n{'loss': 1.47180223, 'grad_norm': 9.5625, 'learning_rate': 6e-07, 'token_acc': 0.62211501, 'epoch': 0.85, 'global_step/max_steps': '80/94', 'percentage': '85.11%', 'elapsed_time': '2m 33s', 'remaining_time': '26s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.520284}\n{'loss': 1.44068375, 'grad_norm': 10.125, 'learning_rate': 2.5e-07, 'token_acc': 0.62673112, 'epoch': 0.91, 'global_step/max_steps': '85/94', 'percentage': '90.43%', 'elapsed_time': '2m 43s', 'remaining_time': '17s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.520331}\n{'loss': 1.44893646, 'grad_norm': 8.375, 'learning_rate': 5e-08, 'token_acc': 0.63837478, 'epoch': 0.96, 'global_step/max_steps': '90/94', 'percentage': '95.74%', 'elapsed_time': '2m 52s', 'remaining_time': '7s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.520707}\n{'train_runtime': 183.4332, 'train_samples_per_second': 8.177, 'train_steps_per_second': 0.512, 'train_loss': 1.50650934, 'token_acc': 0.6194337, 'epoch': 1.0, 'global_step/max_steps': '94/94', 'percentage': '100.00%', 'elapsed_time': '3m 3s', 'remaining_time': '0s', 'memory(GiB)': 6.5, 'train_speed(iter/s)': 0.512463}\nTrain: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 94/94 [03:03<00:00, 1.95s/it]\n[INFO:swift] last_model_checkpoint: /workspace/ms-swift/output/v0-20260211-143035/checkpoint-94\n[INFO:swift] best_model_checkpoint: None\n[INFO:swift] images_dir: /workspace/ms-swift/output/v0-20260211-143035/images\n[INFO:swift] End time of running main: 2026-02-11 14:34:09.521336\n\n```\n### 3.2. 使用 Megatron-LM 作为 Swift 后端\n若希望使用 Megatron-LM 作为 Swift 的后端，需设置 `MEGATRON_LM_PATH` 环境变量：\n\n```bash\nexport MEGATRON_LM_PATH=/workspace/Megatron-LM-0.15.0\ncd /workspace/ms-swift\nbash examples/megatron/pretrain.sh\n```\n\n### 3.3. 使用其他版本的 ms-swift\nMetax 平台要求使用与 Maca 兼容的软件包。例如，编译可能依赖 torch2.8，因此需使用 torch2.8+maca3.3.x.x 版本。\n\n默认情况下，安装会覆盖环境中已有的 PyTorch。因此，建议使用 --no-deps 参数进行安装：\n```bash\n\ngit clone -b ${SWIFT_VERSION} https://github.com/modelscope/ms-swift.git\ncd ms-swift\npip install . --no-deps\n\n```\n每次环境变更后，请检查 PyTorch 版本及其可用性：\n```bash\npip list |grep torch\n# output:\n# torch2.x.x+metax3.x.x.x\n```\n\n```python\nimport torch\ntorch.cuda.is_available()\n```\n\n### 3.4. Metax 与 NVIDIA CUDA 的差异\nMetax 在大部分接口上与 NVIDIA 对齐，但在某些软件行为和环境变量上存在差异。\n\n#### 3.4.1. MACA_MPS_MODE\n默认情况下，MACA 不允许多个进程共享同一块 GPU。如果 GPU 已被占用，则无法启动新进程。\n\n如需启用类似 MPS（Multi-Process Service）的功能，需设置：`MACA_MPS_MODE=1`\n```bash\n# 运行其他脚本...\nexport MACA_MPS_MODE=1\ncd /workspace/ms-swift/\nbash examples/train/full/train.sh\n```\n#### 3.4.2. MCCL_SOCKET_IFNAME GLOO_SOCKET_IFNAME & MCCL_IB_HCA\n在多节点训练时，建议设置以下环境变量以确保节点间通信正常：\n> MCCL_SOCKET_IFNAME：用于 MCCL 通信的网络接口\n> GLOO_SOCKET_IFNAME：用于 GLOO 通信的网络接口\n> MCCL_IB_HCA：指定使用的 InfiniBand 设备\n\n可通过 ifconfig 和 mx-smi 确定所用网卡和 IB 设备：\n```bash\nifconfig\n# output\nens20f0np0: xxx\n inet: your node ip\n xxx\n...\n```\n```bash\nmx-smi topo -n\n# output\nmx-smi version: 2.2.9\n\n=================== MetaX System Management Interface Log ===================\nTimestamp : Wed Feb 11 18:53:44 2026\n\nAttached GPUs : 8\nDevice link type matrix\n GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 Node Affinity CPU Affinity\nGPU0 X MX MX MX NODE NODE NODE NODE PIX PIX NODE NODE SYS SYS 0 0-31,64-95\nGPU1 MX X MX MX NODE NODE NODE NODE PIX PIX NODE NODE SYS SYS 0 0-31,64-95\nGPU2 MX MX X MX NODE NODE NODE NODE PIX PIX NODE NODE SYS SYS 0 0-31,64-95\nGPU3 MX MX MX X NODE NODE NODE NODE PIX PIX NODE NODE SYS SYS 0 0-31,64-95\nGPU4 NODE NODE NODE NODE X MX MX MX NODE NODE PIX PIX SYS SYS 0 0-31,64-95\nGPU5 NODE NODE NODE NODE MX X MX MX NODE NODE PIX PIX SYS SYS 0 0-31,64-95\nGPU6 NODE NODE NODE NODE MX MX X MX NODE NODE PIX PIX SYS SYS 0 0-31,64-95\nGPU7 NODE NODE NODE NODE MX MX MX X NODE NODE PIX PIX SYS SYS 0 0-31,64-95\nNIC0 PIX PIX PIX PIX NODE NODE NODE NODE X PIX NODE NODE SYS SYS\nNIC1 PIX PIX PIX PIX NODE NODE NODE NODE PIX X NODE NODE SYS SYS\nNIC2 NODE NODE NODE NODE PIX PIX PIX PIX NODE NODE X PIX SYS SYS\nNIC3 NODE NODE NODE NODE PIX PIX PIX PIX NODE NODE PIX X SYS SYS\nNIC4 SYS SYS SYS SYS SYS SYS SYS SYS SYS SYS SYS SYS X PIX\nNIC5 SYS SYS SYS SYS SYS SYS SYS SYS SYS SYS SYS SYS PIX X\n\nLegend:\n X = Self\n SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)\n NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node\n PHB = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)\n PXB = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)\n PIX = Connection traversing at most a single PCIe bridge\n MX = Connection traversing MetaXLink\n ETH = Connection traversing Eth\n NA = Connection type is unknown\n\nNIC Legend:\n\n NIC0: mlx5_0\n NIC1: mlx5_1\n NIC2: mlx5_2\n NIC3: mlx5_3\n NIC4: mlx5_4\n NIC5: mlx5_5\n# 根据拓扑信息可知：\n# 1. GPU0–GPU3 与 NIC0/NIC1（即 mlx5_0, mlx5_1）通信\n# 2. GPU4–GPU7 与 NIC2/NIC3（即 mlx5_2, mlx5_3）通信\n\n\n```\n因此，推荐设置如下:\n`MCCL_SOCKET_IFNAME=ens20f0np0`\n`GLOO_SOCKET_IFNAME=ens20f0np0`\n`MCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3`\n\n```bash\n# node 1\nexport MCCL_SOCKET_IFNAME=ens20f0np0\nexport GLOO_SOCKET_IFNAME=ens20f0np0\nexport MCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3\ncd /workspace/ms-swift/\nbash examples/train/multi-node/torchrun/train_node1.sh\n```\n\n```bash\n# node 2\n# 需修改脚本中的 master_addr 为节点1的IP\nexport MCCL_SOCKET_IFNAME=ens20f0np0\nexport GLOO_SOCKET_IFNAME=ens20f0np0\nexport MCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3\ncd /workspace/ms-swift/\nbash examples/train/multi-node/torchrun/train_node2.sh\n```\n"
},
{
"path": "docs/source/BestPractices/More-Best-Practices.md",
"content": "\n# 更多最佳实践\n\n- [Qwen2.5自我认知微调](https://github.com/modelscope/ms-swift/tree/main/examples/notebook/qwen2_5-self-cognition)\n- [Qwen2-VL Latex-OCR微调](https://github.com/modelscope/ms-swift/tree/main/examples/notebook/qwen2vl-ocr)\n- [Qwen2.5-VL Grounding任务微调](https://github.com/modelscope/ms-swift/tree/main/examples/notebook/qwen2_5-vl-grounding)\n"
},
{
"path": "docs/source/BestPractices/NPU-support.md",
"content": "# NPU支持\n\n我们在 ms-swift 上增加了对昇腾 NPU 的支持，用户可以在昇腾 NPU 上进行模型的微调和推理。\n\n本文档介绍了如何在昇腾 NPU 上进行环境准备、模型微调、推理和部署。\n\n## 安装\n\n基础环境准备：\n\n| software | version |\n| --------- | --------------- |\n| Python | >= 3.10, < 3.12 |\n| CANN | == 8.5.1 |\n| torch | == 2.7.1 |\n| torch_npu | == 2.7.1.post2 |\n\n\n基础环境准备请参照这份 [Ascend PyTorch 安装文档](https://gitcode.com/Ascend/pytorch)。\n\n\n## 环境准备\n\n实验环境：8 * 昇腾910B3 64G\n### 环境安装\n```shell\n# 创建新的 conda 虚拟环境（可选）\nconda create -n swift-npu python=3.10 -y\nconda activate swift-npu\n\n# 注意进行后续操作前要先 source 激活 CANN 环境\nsource /usr/local/Ascend/ascend-toolkit/set_env.sh\n\n# 设置 pip 全局镜像（可选，加速下载）\npip config set global.index-url https://mirrors.aliyun.com/pypi/simple/\npip install ms-swift -U\n\n# 使用源码安装\ngit clone https://github.com/modelscope/ms-swift.git\ncd ms-swift\npip install -e .\n\n# 安装 torch-npu\npip install torch-npu decorator\n# 如果你想要使用 deepspeed（控制显存占用，训练速度会有一定下降）\npip install deepspeed\n\n# 如果需要使用 evaluation 功能，请安装以下包\npip install evalscope[opencompass]\n\n# 如果需要使用 vllm-ascend 进行推理，请安装以下包\npip install vllm==0.14.0\npip install vllm-ascend==0.14.0rc1\n```\n\n测试环境是否安装正确，NPU能否被正常加载：\n```python\nfrom transformers.utils import is_torch_npu_available\nimport torch\n\nprint(is_torch_npu_available()) # True\nprint(torch.npu.device_count()) # 8\nprint(torch.randn(10, device='npu:0'))\n```\n\n**如果需要使用 MindSpeed(Megatron-LM)，请按照下面引导安装必要依赖**\n```shell\n# 1. 获取并切换 Megatron-LM 至 core_v0.12.1 版本\ngit clone https://github.com/NVIDIA/Megatron-LM.git\ncd Megatron-LM\ngit checkout core_v0.12.1\ncd ..\n\n# 2. 获取并安装 MindSpeed\ngit clone https://gitcode.com/Ascend/MindSpeed.git\ncd MindSpeed\ngit checkout 2.3.0_core_r0.12.1\npip install -e .\ncd ..\n\n# 3. 设置环境变量\nexport PYTHONPATH=$PYTHONPATH:\nexport MEGATRON_LM_PATH=\n```\n\n执行如下命令验证 MindSpeed(Megatron-LM) 是否配置成功：\n```shell\npython -c \"import mindspeed.megatron_adaptor; from swift.megatron.init import init_megatron_env; init_megatron_env(); print('✓ NPU环境下的Megatron-SWIFT配置验证成功！')\"\n```\n\n### 环境查看\n\n查看NPU的P2P连接，这里看到每个NPU都通过7条HCCS与其他NPU互联\n\n```shell\n(valle) root@valle:~/src# npu-smi info -t topo\n\t NPU0 NPU1 NPU2 NPU3 NPU4 NPU5 NPU6 NPU7 CPU Affinity\nNPU0 X HCCS HCCS HCCS HCCS HCCS HCCS HCCS 144-167\nNPU1 HCCS X HCCS HCCS HCCS HCCS HCCS HCCS 144-167\nNPU2 HCCS HCCS X HCCS HCCS HCCS HCCS HCCS 96-119\nNPU3 HCCS HCCS HCCS X HCCS HCCS HCCS HCCS 96-119\nNPU4 HCCS HCCS HCCS HCCS X HCCS HCCS HCCS 0-23\nNPU5 HCCS HCCS HCCS HCCS HCCS X HCCS HCCS 0-23\nNPU6 HCCS HCCS HCCS HCCS HCCS HCCS X HCCS 48-71\nNPU7 HCCS HCCS HCCS HCCS HCCS HCCS HCCS X 48-71\n\nLegend:\n\n X = Self\n SYS = Path traversing PCIe and NUMA nodes. Nodes are connected through SMP, such as QPI, UPI.\n PHB = Path traversing PCIe and the PCIe host bridge of a CPU.\n PIX = Path traversing a single PCIe switch\n PXB = Path traversing multiple PCIe switches\n HCCS = Connection traversing HCCS.\n NA = Unknown relationship.\n```\n\n查看NPU状态, npu-smi命令详解可以查看[官方文档](https://support.huawei.com/enterprise/zh/doc/EDOC1100079287/10dcd668)\n\n```shell\n(valle) root@valle:~/src# npu-smi info\n+------------------------------------------------------------------------------------------------+\n| npu-smi 24.1.rc1.b030 Version: 24.1.rc1.b030 |\n+---------------------------+---------------+----------------------------------------------------+\n| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page)|\n| Chip | Bus-Id | AICore(%) Memory-Usage(MB) HBM-Usage(MB) |\n+===========================+===============+====================================================+\n| 0 910B3 | OK | 101.8 43 0 / 0 |\n| 0 | 0000:C1:00.0 | 0 0 / 0 3318 / 65536 |\n+===========================+===============+====================================================+\n| 1 910B3 | OK | 92.0 39 0 / 0 |\n| 0 | 0000:C2:00.0 | 0 0 / 0 3314 / 65536 |\n+===========================+===============+====================================================+\n| 2 910B3 | OK | 102.0 40 0 / 0 |\n| 0 | 0000:81:00.0 | 0 0 / 0 3314 / 65536 |\n+===========================+===============+====================================================+\n| 3 910B3 | OK | 99.8 40 0 / 0 |\n| 0 | 0000:82:00.0 | 0 0 / 0 3314 / 65536 |\n+===========================+===============+====================================================+\n| 4 910B3 | OK | 98.6 45 0 / 0 |\n| 0 | 0000:01:00.0 | 0 0 / 0 3314 / 65536 |\n+===========================+===============+====================================================+\n| 5 910B3 | OK | 99.7 44 0 / 0 |\n| 0 | 0000:02:00.0 | 0 0 / 0 3314 / 65536 |\n+===========================+===============+====================================================+\n| 6 910B3 | OK | 103.8 45 0 / 0 |\n| 0 | 0000:41:00.0 | 0 0 / 0 3314 / 65536 |\n+===========================+===============+====================================================+\n| 7 910B3 | OK | 98.2 44 0 / 0 |\n| 0 | 0000:42:00.0 | 0 0 / 0 3315 / 65536 |\n+===========================+===============+====================================================+\n```\n\n## 微调\n\n以下介绍LoRA的微调, 全参数微调设置参数`--tuner_type full`即可. **更多训练脚本**参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/ascend/train).\n\n| 模型大小 | NPU数量 | deepspeed类型 | 最大显存占用量 |\n| -------- | ------- | ------------- | -------------- |\n| 7B | 1 | None | 1 * 28 GB |\n| 7B | 4 | None | 4 * 22 GB |\n| 7B | 4 | zero2 | 4 * 28 GB |\n| 7B | 4 | zero3 | 4 * 22 GB |\n| 7B | 8 | None | 8 * 22 GB |\n| 14B | 1 | None | 1 * 45 GB |\n| 14B | 8 | None | 8 * 51 GB |\n| 14B | 8 | zero2 | 8 * 49 GB |\n| 14B | 8 | zero3 | 8 * 31 GB |\n\n### 单卡训练\n\n通过如下命令启动单卡微调: （注意: 如果微调期间出现nan的情况, 请设置`--torch_dtype float32`.）\n\n```shell\n# 实验环境: 昇腾910B3\n# 显存需求: 28 GB\n# 运行时长: 8小时\nASCEND_RT_VISIBLE_DEVICES=0 \\\nswift sft \\\n --model Qwen/Qwen2-7B-Instruct \\\n --dataset AI-ModelScope/blossom-math-v2 \\\n --split_dataset_ratio 0.01 \\\n --num_train_epochs 5 \\\n --tuner_type lora \\\n --output_dir output \\\n --learning_rate 1e-4 \\\n --gradient_accumulation_steps 16 \\\n --save_steps 100 \\\n --eval_steps 100\n\n```\n\n\n### 数据并行训练\n\n我们使用其中的4卡进行ddp训练\n\n```shell\n# 实验环境: 4 * 昇腾910B3\n# 显存需求: 4 * 22 GB\n# 运行时长: 2小时\nNPROC_PER_NODE=4 \\\nASCEND_RT_VISIBLE_DEVICES=0,1,2,3 \\\nswift sft \\\n --model Qwen/Qwen2-7B-Instruct \\\n --dataset AI-ModelScope/blossom-math-v2 \\\n --split_dataset_ratio 0.01 \\\n --num_train_epochs 5 \\\n --tuner_type lora \\\n --output_dir output \\\n ...\n```\n\n\n### Deepspeed训练\n\nZeRO2:\n\n```shell\n# 实验环境: 4 * 昇腾910B3\n# 显存需求: 4 * 28GB\n# 运行时长: 3.5小时\nNPROC_PER_NODE=4 \\\nASCEND_RT_VISIBLE_DEVICES=0,1,2,3 \\\nswift sft \\\n --model Qwen/Qwen2-7B-Instruct \\\n --dataset AI-ModelScope/blossom-math-v2 \\\n --split_dataset_ratio 0.01 \\\n --num_train_epochs 5 \\\n --tuner_type lora \\\n --output_dir output \\\n --deepspeed zero2 \\\n ...\n```\n\nZeRO3:\n\n```shell\n# 实验环境: 4 * 昇腾910B3\n# 显存需求: 4 * 22 GB\n# 运行时长: 8.5小时\nNPROC_PER_NODE=4 \\\nASCEND_RT_VISIBLE_DEVICES=0,1,2,3 \\\nswift sft \\\n --model Qwen/Qwen2-7B-Instruct \\\n --dataset AI-ModelScope/blossom-math-v2 \\\n --split_dataset_ratio 0.01 \\\n --num_train_epochs 5 \\\n --tuner_type lora \\\n --output_dir output \\\n --deepspeed zero3 \\\n ...\n```\n\n\n## 推理\n\n原始模型:\n\n```shell\nASCEND_RT_VISIBLE_DEVICES=0 swift infer \\\n --model Qwen/Qwen2-7B-Instruct \\\n --stream true --max_new_tokens 2048\n```\n\nLoRA微调后:\n\n```shell\nASCEND_RT_VISIBLE_DEVICES=0 swift infer \\\n --adapters xxx/checkpoint-xxx --load_data_args true \\\n --stream true --max_new_tokens 2048\n\n# merge-lora并推理\nASCEND_RT_VISIBLE_DEVICES=0 swift export --adapters xx/checkpoint-xxx --merge_lora true\n\nASCEND_RT_VISIBLE_DEVICES=0 swift infer \\\n --model xxx/checkpoint-xxx-merged --load_data_args true \\\n --stream true --max_new_tokens 2048\n```\n\n\n## 部署\n\n### 使用原生transformers进行部署\n\n原始模型:\n\n```shell\nASCEND_RT_VISIBLE_DEVICES=0 swift deploy --model Qwen/Qwen2-7B-Instruct --max_new_tokens 2048\n```\n\nLoRA微调后:\n\n```shell\nASCEND_RT_VISIBLE_DEVICES=0 swift deploy --adapters xxx/checkpoint-xxx --max_new_tokens 2048\n\n# merge-lora并推理\nASCEND_RT_VISIBLE_DEVICES=0 swift export --adapters xx/checkpoint-xxx --merge_lora true\nASCEND_RT_VISIBLE_DEVICES=0 swift deploy --model xxx/checkpoint-xxx-merged --max_new_tokens 2048\n```\n\n### 使用vLLM-ascend进行部署\n使用pypi进行安装：\n```shell\n# Install vllm-project/vllm. The newest supported version is v0.11.0.\npip install vllm==0.11.0\n\n# Install vllm-project/vllm-ascend from pypi.\npip install vllm-ascend==0.11.0rc3\n```\n原始模型：\n```shell\nASCEND_RT_VISIBLE_DEVICES=0 swift deploy \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --infer_backend vllm \\\n --max_new_tokens 2048\n```\n\nLoRA微调后:\n\n```shell\nASCEND_RT_VISIBLE_DEVICES=0 swift deploy \\\n --adapters xxx/checkpoint-xxx \\\n --infer_backend vllm \\\n --max_new_tokens 2048\n\n# merge-lora并推理\nASCEND_RT_VISIBLE_DEVICES=0 swift export \\\n --adapters xx/checkpoint-xxx \\\n --merge_lora true\n\nASCEND_RT_VISIBLE_DEVICES=0 swift deploy \\\n --model xxx/checkpoint-xxx-merged \\\n --infer_backend vllm \\\n --max_new_tokens 2048\n```\n\n## 支持现状\n| 一级特性 | 特性 | 进展 |\n| -------- | ------------------- | -------- |\n| 训练范式 | CPT | 已支持 |\n| | SFT | 已支持 |\n| | DPO | 已支持 |\n| | RM | 已支持 |\n| 分布式 | DDP | 已支持 |\n| | FSDP | 已支持 |\n| | FSDP2 | 已支持 |\n| | DeepSpeed | 已支持 |\n| | MindSpeed(Megatron) | 已支持 |\n| 低参微调 | FULL | 已支持 |\n| | LoRA | 已支持 |\n| | QLoRA | 暂不支持 |\n| RLHF | GRPO | 已支持 |\n| | PPO | 已支持 |\n| 性能优化 | FA 等融合算子 | 已支持 |\n| | Liger-Kernel | 暂不支持 |\n| 部署 | PT | 已支持 |\n| | vLLM | 已支持 |\n| | SGLang | 暂不支持 |\n\n------\n\n\n### 表 1：SFT 类算法\n\n| algorithm | model families | strategy | hardware |\n| --------- | --------------------------- | --------------------- | ----------------- |\n| SFT | Qwen2.5-0.5B-Instruct | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | Qwen2.5-1.5B-Instruct | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | Qwen2.5-7B-Instruct | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | Qwen2.5-VL-3B-Instruct | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | Qwen2.5-VL-7B-Instruct | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | Qwen2.5-Omni-3B | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | Qwen3-8B | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | Qwen3-32B | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | Qwen3-VL-30B-A3B-Instruct | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | Qwen3-Omni-30B-A3B-Instruct | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | InternVL3-8B | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n| SFT | Ovis2.5-2B | FSDP1/FSDP2/deepspeed | Atlas 900 A2 PODc |\n\n------\n\n### 表 2：RL 类算法\n\n| algorithm | model families | strategy | rollout engine | hardware |\n| --------- | ------------------- | --------- | -------------- | ----------------- |\n| **GRPO** | Qwen2.5-7B-Instruct | deepspeed | vllm-ascend | Atlas 900 A2 PODc |\n| **GRPO** | Qwen3-8B | deepspeed | vllm-ascend | Atlas 900 A2 PODc |\n| **DPO** | Qwen2.5-7B-Instruct | deepspeed | vllm-ascend | Atlas 900 A2 PODc |\n| **DPO** | Qwen3-8B | deepspeed | vllm-ascend | Atlas 900 A2 PODc |\n| **PPO** | Qwen2.5-7B-Instruct | deepspeed | vllm-ascend | Atlas 900 A2 PODc |\n| **PPO** | Qwen3-8B | deepspeed | vllm-ascend | Atlas 900 A2 PODc |\n\n---\n\n### 表 3：当前 NPU 暂不支持 / 未完全验证的模块\n\n| item |\n| --------------------------------- |\n| Liger-kernel |\n| 量化/QLoRA相关 |\n| 使用sglang作为推理引擎 |\n| 使用megatron时开启ETP进行lora训练 |\n\n\n## NPU微信群\n\n\nOkay, the user is asking \"who are you?\" Let me start by introducing myself as Qwen, the large language model developed by Alibaba Cloud. I should mention my capabilities, like answering questions, creating content, and engaging in conversations. But I need to keep it concise. Also, the user might want to know how I can assist them. Maybe I should ask how I can help them today. Let me check if there's anything else important to include. Oh, I should make sure the tone is friendly and approachable. Alright, that should cover it.\n \n\nHello! I am Qwen, a large language model developed by Alibaba Cloud. I can assist with a wide range of tasks, such as answering questions, creating content, writing stories, coding, and more. How can I help you today? 😊\n<<< clear\n<<< who are you? /no_think\n\n\n \n\nI am Qwen, a large language model developed by Alibaba Cloud. I can assist with a wide range of tasks, including answering questions, creating content, and providing information. How can I help you today?\n```\n\n非思考模式：\n- 其中`--response_prefix`代表模型的输出会在其前缀后继续生成。等价于enable_thinking设置为False。\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --model Qwen/Qwen3-8B \\\n --infer_backend vllm \\\n --stream true \\\n --max_new_tokens 2048 \\\n --vllm_max_model_len 8192 \\\n --response_prefix '\\n\\n \\n\\n'\n```\n\n```text\n<<< who are you?\n\n\n \n\nI am Qwen, a large-scale language model developed by Alibaba Cloud. I am designed to assist with a wide range of tasks, including answering questions, creating content, and providing information. How can I assist you today?\n```\n\n## 训练\n\n在开始训练之前，请确保您的环境已正确配置。\n\n```bash\npip install ms-swift -U\npip install transformers\n\npip install deepspeed # 多GPU训练\npip install liger-kernel # 节约显存资源\npip install flash-attn --no-build-isolation # packing需要\n```\n\n## 监督微调 (SFT)\n\n### 数据准备\n\n使用 ms-swift 进行 SFT 的自定义数据集格式如下（system 字段是可选的）。您可以将其组织为 JSON、JSONL 或 CSV 格式。在训练脚本中指定 `--dataset `。有关完整的数据集格式指南，请参考[自定义数据集文档](../Customization/Custom-dataset.md)。\n\n```text\n# 通用格式\n{\"messages\": [\n {\"role\": \"system\", \"content\": \"\"},\n {\"role\": \"user\", \"content\": \"\"},\n {\"role\": \"assistant\", \"content\": \"\"}\n]}\n# 带think的格式\n{\"messages\": [\n {\"role\": \"user\", \"content\": \"Where is the capital of Zhejiang?\"},\n {\"role\": \"assistant\", \"content\": \"\\n...\\n \\n\\nThe capital of Zhejiang is Hangzhou.\"}\n]}\n```\n\n如果您想使用不含思维链的数据进行训练，同时保留模型的推理能力，可以通过以下两种方法尽量减少微调的影响：\n\n**选项 1**：【推荐】在训练期间，指定 `--loss_scale ignore_empty_think`，以忽略对 `'\\n\\n \\n\\n'` 的损失计算，从而避免推理能力的丧失。训练脚本参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/think_model/qwen3_demo1.sh)。该方式同样适用于deepseek-r1等模型。自定义数据集格式如下：\n\n```json\n{\"messages\": [\n {\"role\": \"user\", \"content\": \"Where is the capital of Zhejiang?\"},\n {\"role\": \"assistant\", \"content\": \"\\n\\n \\n\\nThe capital of Zhejiang is Hangzhou.\"}\n]}\n```\n\n**选项 2**：在数据集的查询中添加 `/no_think`，以避免推理能力的丧失。训练脚本请参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/think_model/qwen3_demo2.sh)。自定义数据集格式如下：\n\n```json\n{\"messages\": [\n {\"role\": \"user\", \"content\": \"Where is the capital of Zhejiang? /no_think\"},\n {\"role\": \"assistant\", \"content\": \"\\n\\n \\n\\nThe capital of Zhejiang is Hangzhou.\"}\n]}\n```\n\n你可以使用以下命令获取蒸馏的推理数据集，在训练时，与不含思维链数据集进行混合，进一步缓解推理能力的丧失：\n- 其中`--val_dataset`的选择任意。推理产生的`result_path`，可以直接在训练时指定`--dataset distill_dataset.jsonl`使用。\n- 该思路同样适用于其他推理模型，例如deepseek-r1。\n```shell\n# 4 * 80GiB\nNPROC_PER_NODE=4 \\\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nswift infer \\\n --model Qwen/Qwen3-32B \\\n --infer_backend vllm \\\n --val_dataset 'AI-ModelScope/alpaca-gpt4-data-en#5000' 'AI-ModelScope/alpaca-gpt4-data-zh#5000' \\\n --vllm_gpu_memory_utilization 0.9 \\\n --vllm_tensor_parallel_size 2 \\\n --vllm_max_model_len 8192 \\\n --max_new_tokens 4096 \\\n --write_batch_size 1000 \\\n --result_path distill_dataset.jsonl\n```\n\n### 30分钟自我认知微调\n\n本节将介绍30分钟对 Qwen3-8B 进行自我认知微调。所需GPU显存为 22GB，可以在 ModelScope 提供的[免费算力](https://modelscope.cn/my/mynotebook) A10 中运行。\n\n训练后，模型将不再认为自己是由“阿里云”训练的“Qwen”，而是由“swift”训练的“swift-robot”。\n\n如果需要在离线环境下进行训练，可以手动下载模型和数据集，并指定 `--model ` 和 `--dataset `。数据集可以在 [Modelscope Hub](https://modelscope.cn/datasets/swift/self-cognition)上找到。对`swift/self-cognition`数据集的预处理函数可以查看[这里](https://github.com/modelscope/ms-swift/blob/36fdf381e5e88cb8a71c9d69c1d8936a989318cc/swift/llm/dataset/dataset/llm.py#L882)。\n\n关于训练脚本中各参数的含义，请参考[命令行参数文档](../Instruction/Command-line-parameters.md)。\n\n```bash\n# 显存占用：22GB\nCUDA_VISIBLE_DEVICES=0 \\\nswift sft \\\n --model Qwen/Qwen3-8B \\\n --tuner_type lora \\\n --dataset 'swift/Qwen3-SFT-Mixin#2000' \\\n 'swift/self-cognition:qwen3#600' \\\n --load_from_cache_file true \\\n --torch_dtype bfloat16 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 1 \\\n --per_device_eval_batch_size 1 \\\n --learning_rate 1e-4 \\\n --lora_rank 8 \\\n --lora_alpha 32 \\\n --target_modules all-linear \\\n --gradient_accumulation_steps 16 \\\n --eval_steps 50 \\\n --save_steps 50 \\\n --save_total_limit 2 \\\n --logging_steps 5 \\\n --max_length 2048 \\\n --output_dir output \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 4 \\\n --model_author swift \\\n --model_name swift-robot\n```\n\n微调完成后，可以使用以下脚本来测试微调结果。注意，`--adapters` 部分需要修改为最后保存检查点的目录路径：\n\n```bash\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --temperature 0 \\\n --max_new_tokens 2048\n```\n\n```text\n<<< who are you?\n\nOkay, the user asked, \"who are you?\" I need to introduce myself. Let me start by stating my name, swift-robot. Then, I should mention that I'm an AI assistant developed by swift. I should explain my purpose, which is to provide information and assistance. I should also highlight my capabilities, like answering questions, generating text, and engaging in conversation. It's important to keep the tone friendly and approachable. Maybe add something about being here to help and encourage the user to ask anything. Let me check if I covered all the key points: name, developer, purpose, capabilities, and a welcoming statement. Yeah, that should do it. Now, let me put that into a concise and friendly response.\n \n\nHello! I am swift-robot, an artificial intelligence assistant developed by swift. My purpose is to provide information and assistance to users like you. I can answer questions, generate text, and engage in conversations on a wide range of topics. I am here to help, so feel free to ask me anything you need!\n```\n\n默认情况下，ms-swift 会使用 ModelScope 社区下载模型和数据集。如果想使用 HuggingFace 社区，则需要额外指定 `--use_hf true`。\n\n合并 LoRA 权重：\n\n```shell\nswift export \\\n --adapters output/checkpoint-xxx \\\n --merge_lora true\n```\n\n推送模型到 ModelScope/HuggingFace：\n\n```bash\n# 如果是推送完整的权重，需要修改`--adapters`为`--model`.\n# Modelscope的hub_token可以在这里找到: https://modelscope.cn/my/myaccesstoken\nswift export \\\n --adapters output/checkpoint-xxx \\\n --push_to_hub true \\\n --hub_model_id '' \\\n --hub_token '' \\\n --use_hf false\n```\n\n如果要使用多 GPU 进行训练，以下提供了多 GPU 训练的示例：\n\n```bash\n# 4 * 60GB\n# 你可以通过设置`--dataset AI-ModelScope/alpaca-gpt4-data-en`跑通实验\n# 注意：如果你指定了`--packing true`, 你必须额外设置`--attn_impl flash_attn`\n\nNPROC_PER_NODE=4 \\\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nswift sft \\\n --model Qwen/Qwen3-8B \\\n --tuner_type full \\\n --dataset '' \\\n --load_from_cache_file true \\\n --split_dataset_ratio 0.01 \\\n --torch_dtype bfloat16 \\\n --per_device_train_batch_size 1 \\\n --per_device_eval_batch_size 1 \\\n --learning_rate 1e-5 \\\n --gradient_accumulation_steps 4 \\\n --packing true \\\n --eval_steps 100 \\\n --save_steps 100 \\\n --logging_steps 5 \\\n --max_length 8192 \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 8 \\\n --dataset_num_proc 8 \\\n --save_total_limit 2 \\\n --save_only_model true \\\n --output_dir output \\\n --deepspeed zero3 \\\n --use_liger_kernel true \\\n --attn_impl flash_attn\n```\n\n## 强化学习 (RL)\n\nms-swift 支持 DPO、GRPO、DAPO、PPO、KTO、GKD 等 RLHF 方法。本章将着重介绍使用 ms-swift 对 Qwen3-8B 进行 GRPO 训练。更多关于GRPO的内容，可以参考[GRPO文档](../Instruction/GRPO/GetStarted/GRPO.md)。更多RLHF训练脚本，参考[examples/train/rlhf](https://github.com/modelscope/ms-swift/tree/main/examples/train/rlhf)。\n\n### 环境设置\n\n除了安装上述介绍的 ms-swift 相关依赖项外，还需要安装以下依赖项：\n```\npip install \"math_verify\"\npip install vllm==0.8.5.post1\n```\n\n### 数据准备\n\n使用 ms-swift 进行 GRPO 训练的数据集格式与 SFT 类似，但不需要最后一轮的 assistant 部分。如果使用 accuracy 作为奖励，则需要额外的 `solution` 列来计算准确率。\n\n示例数据集格式：\n\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"Tell me tomorrow's weather\"}]}\n{\"messages\": [{\"role\": \"user\", \"content\": \"What is 1 + 1?\"}, {\"role\": \"assistant\", \"content\": \"It equals 2\"}, {\"role\": \"user\", \"content\": \"What about adding 1?\"}]}\n{\"messages\": [{\"role\": \"user\", \"content\": \"What is your name?\"}]}\n```\n\n关于其他 RLHF 算法的数据集准备，请参考[自定义数据集文档](../Customization/Custom-dataset.md#rlhf)。\n\n数据集要求的注意事项：\n\n- **奖励函数计算**：数据集格式取决于所使用的奖励函数。可能需要额外的列来支持特定的奖励计算。例如：\n\n - 当使用内置的 accuracy 或 cosine 奖励时，数据集必须包含一个 `solution` 列以计算回复的准确性。\n - 数据集中的其他列将作为 ``**kwargs`` 传递给奖励函数以实现进一步的自定义。\n\n- **自定义奖励函数**：为了根据您的具体需求调整奖励函数，可以参考链接：[外部奖励插件](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/plugin)。该插件提供了实现自定义奖励函数的示例和模板。\n\n我们使用使 AI-MO/NuminaMath-TIR 作为数据集，并使用accuracy函数计算模型回答的准确率奖励。\n\n在训练过程中，使用 vLLM 加速采样过程。\n\n```bash\n# 70G*8\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \\\nNPROC_PER_NODE=8 \\\nswift rlhf \\\n --rlhf_type grpo \\\n --model Qwen/Qwen3-8B \\\n --tuner_type full \\\n --dataset 'AI-MO/NuminaMath-TIR#5000' \\\n --load_from_cache_file true \\\n --torch_dtype bfloat16 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 2 \\\n --per_device_eval_batch_size 2 \\\n --learning_rate 1e-6 \\\n --save_total_limit 2 \\\n --logging_steps 5 \\\n --output_dir output \\\n --gradient_accumulation_steps 1 \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 4 \\\n --max_length 4096 \\\n --max_completion_length 4096 \\\n --vllm_max_model_len 8192 \\\n --reward_funcs accuracy \\\n --num_generations 16 \\\n --use_vllm true \\\n --vllm_gpu_memory_utilization 0.4 \\\n --sleep_level 1 \\\n --offload_model true \\\n --offload_optimizer true \\\n --deepspeed zero3 \\\n --vllm_tensor_parallel_size 1 \\\n --temperature 1.0 \\\n --top_p 0.85 \\\n --log_completions true \\\n --overlong_filter true\n```\n\n## Megatron-SWIFT\n\nQwen3-235B-A22B-Instruct-250718 单机8卡H20 LoRA训练的最佳实践参考：[https://github.com/modelscope/ms-swift/pull/5033](https://github.com/modelscope/ms-swift/pull/5033)。\n\nms-swift 引入了 Megatron 并行技术以加速大模型的CPT/SFT/DPO/GRPO。支持的模型可以在[支持的模型文档](../Instruction/Supported-models-and-datasets.md)中找到。\n\n关于环境准备，可以参考[Megatron-SWIFT训练文档](../Megatron-SWIFT/Quick-start.md)。\n\n我们将使用阿里云 DLC 启动训练。训练环境由2台配备8卡 80GiB A800 GPU 组成。关于多节点启动方法的更多信息，请参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-node)。\n\n```bash\n# https://help.aliyun.com/zh/pai/user-guide/general-environment-variables\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nNNODES=$WORLD_SIZE \\\nNODE_RANK=$RANK \\\nmegatron sft \\\n --model Qwen/Qwen3-30B-A3B-Base \\\n --save_safetensors true \\\n --dataset 'liucong/Chinese-DeepSeek-R1-Distill-data-110k-SFT' \\\n --load_from_cache_file true \\\n --split_dataset_ratio 0.01 \\\n --pipeline_model_parallel_size 2 \\\n --expert_model_parallel_size 8 \\\n --moe_grouped_gemm true \\\n --moe_shared_expert_overlap true \\\n --moe_permute_fusion true \\\n --moe_aux_loss_coeff 1e-3 \\\n --micro_batch_size 1 \\\n --global_batch_size 16 \\\n --packing true \\\n --recompute_granularity full \\\n --recompute_method uniform \\\n --recompute_num_layers 1 \\\n --train_iters 2000 \\\n --eval_iters 50 \\\n --finetune true \\\n --cross_entropy_loss_fusion true \\\n --lr 1e-5 \\\n --lr_warmup_fraction 0.05 \\\n --min_lr 1e-6 \\\n --output_dir megatron_output/Qwen3-30B-A3B-Base \\\n --eval_steps 200 \\\n --save_steps 200 \\\n --max_length 8192 \\\n --dataloader_num_workers 8 \\\n --dataset_num_proc 8 \\\n --no_save_optim true \\\n --no_save_rng true \\\n --sequence_parallel true \\\n --attention_backend flash\n```\n\n训练loss图（部分）：\n\n` 即可。\n\n使用 `megatron sft` 和 `swift sft` 在对 Qwen3-30B-A3B 模型进行全参数微调的训练速度和 GPU 显存使用对比情况如下：\n\n| | Megatron-LM | DeepSpeed-ZeRO2 | DeepSpeed-ZeRO3 |\n| -------- | ----------- | --------------- | --------------- |\n| 训练速度 | 9.6s/it | - | 91.2s/it |\n| 显存使用 | 16 * 60GiB | OOM | 16 * 80GiB |\n"
},
{
"path": "docs/source/BestPractices/Qwen3-VL-Best-Practice.md",
"content": "\n# Qwen3-VL最佳实践\n\n## 环境准备\n\n在开始推理和训练之前，请确保您的环境已准备就绪。\n\n```shell\npip install \"transformers>=4.57\" \"qwen_vl_utils>=0.0.14\"\n\npip install \"ms-swift>=4.0\"\n# pip install \"vllm>=0.11.0\" # 若使用vllm推理后端进行推理\n```\n- 关于视频数据训练卡住：使用decord后端读取视频可能导致卡住问题，参考[这个issue](https://github.com/dmlc/decord/issues/269)。你可以使用torchcodec后端，具体参考[qwen_vl_utils](https://github.com/QwenLM/Qwen3-VL/blob/50068df2334f309979ff05d75f1078c8309c63ed/qwen-vl-utils/src/qwen_vl_utils/vision_process.py#L390-L400)库。\n\n## 推理\n\n使用 transformers 推理：\n```python\nimport os\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\nfrom modelscope import snapshot_download\nfrom qwen_vl_utils import process_vision_info\nfrom transformers import Qwen3VLForConditionalGeneration, AutoProcessor\n\nmodel_dir = snapshot_download('Qwen/Qwen3-VL-4B-Instruct')\n\nmodel = Qwen3VLForConditionalGeneration.from_pretrained(\n model_dir, dtype=\"auto\", device_map=\"auto\",\n # attn_implementation='flash_attention_2',\n)\n\nprocessor = AutoProcessor.from_pretrained(model_dir)\n\nmessages = [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"video\",\n \"video\": \"https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4\",\n \"max_pixels\": 128*32*32,\n \"max_frames\": 16,\n },\n {\"type\": \"text\", \"text\": \"Describe this video.\"},\n ],\n }\n]\n\ntext = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\nimage_inputs, video_inputs, video_kwargs = process_vision_info([messages], return_video_kwargs=True,\n image_patch_size= 16,\n return_video_metadata=True)\nif video_inputs is not None:\n video_inputs, video_metadatas = zip(*video_inputs)\n video_inputs, video_metadatas = list(video_inputs), list(video_metadatas)\nelse:\n video_metadatas = None\ninputs = processor(text=[text], images=image_inputs, videos=video_inputs, video_metadata=video_metadatas, **video_kwargs, do_resize=False, return_tensors=\"pt\")\ninputs = inputs.to('cuda')\n\ngenerated_ids = model.generate(**inputs, max_new_tokens=128, do_sample=False)\ngenerated_ids_trimmed = [\n out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)\n]\noutput_text = processor.batch_decode(\n generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False\n)\nprint(output_text[0])\n# 'A baby wearing glasses sits on a bed, engrossed in reading a book. The baby turns the pages with both hands, occasionally looking up and smiling. The room is cozy, with a crib in the background and clothes scattered around. The baby’s focus and curiosity are evident as they explore the book, creating a heartwarming scene of early learning and discovery.'\n```\n\n使用 ms-swift 的 `TransformersEngine` 进行推理：\n```python\nimport os\n# os.environ['SWIFT_DEBUG'] = '1'\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\nos.environ['VIDEO_MAX_TOKEN_NUM'] = '128'\nos.environ['FPS_MAX_FRAMES'] = '16'\n\n\nfrom swift.infer_engine import TransformersEngine, InferRequest, RequestConfig\nengine = TransformersEngine('Qwen/Qwen3-VL-4B-Instruct') # attn_impl='flash_attention_2'\ninfer_request = InferRequest(messages=[{\n \"role\": \"user\",\n \"content\": 'Describe this video.',\n}], videos=['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'])\nrequest_config = RequestConfig(max_tokens=128, temperature=0)\nresp_list = engine.infer([infer_request], request_config=request_config)\nresponse = resp_list[0].choices[0].message.content\n# 'A baby wearing glasses sits on a bed, engrossed in reading a book. The baby turns the pages with both hands, occasionally looking up and smiling. The room is cozy, with a crib in the background and clothes scattered around. The baby’s focus and curiosity are evident as they explore the book, creating a heartwarming scene of early learning and discovery.'\n\n# use stream\nrequest_config = RequestConfig(max_tokens=128, temperature=0, stream=True)\ngen_list = engine.infer([infer_request], request_config=request_config)\nfor chunk in gen_list[0]:\n if chunk is None:\n continue\n print(chunk.choices[0].delta.content, end='', flush=True)\nprint()\n```\n\n使用命令行推理：\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nIMAGE_MAX_TOKEN_NUM=1024 \\\nVIDEO_MAX_TOKEN_NUM=128 \\\nFPS_MAX_FRAMES=16 \\\nswift infer \\\n --model Qwen/Qwen3-VL-4B-Instruct \\\n --stream true\n```\n\n```\n<<< who are you?\nHello! I'm Qwen, a large-scale language model independently developed by the Tongyi Lab under Alibaba Group. My main functions include answering questions, creating text such as stories, official documents, emails, scripts, and more, as well as performing logical reasoning, programming, and other tasks. If you have any questions or need assistance, feel free to let me know anytime, and I'll do my best to help!\n--------------------------------------------------\n<<< describe the image.\nInput an image path or URL <<< http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png\nThis is a beautifully detailed, close-up portrait of an adorable tabby kitten, rendered with a soft, painterly effect that gives it a gentle, dreamy quality.\n\nHere’s a breakdown of the image:\n\n- **The Kitten:** The subject is a young, fluffy kitten with a classic tabby pattern. Its fur is a mix of white and soft grayish-brown stripes, with a prominent dark stripe running down the center of its forehead and over its nose. The kitten’s face is predominantly white, with delicate markings around its eyes and cheeks.\n\n- **The Eyes:** Its most captivating feature is its large, round, and expressive eyes. They are a striking shade of bright blue-gray, with dark pupils that give it an intense, curious, and slightly innocent gaze. The eyes are wide open, suggesting the kitten is alert and attentive.\n\n- **The Expression:** The kitten’s expression is sweet and innocent. Its small pink nose and slightly parted mouth give it a gentle, almost pleading look. Its whiskers are long and white, standing out against its fur.\n\n- **The Style:** The image has a soft-focus, artistic quality, reminiscent of impressionist painting. The edges of the kitten’s fur are slightly blurred, creating a halo effect that draws attention to its face. The background is softly blurred with muted tones of green and gray, which helps the kitten stand out as the clear focal point.\n\n- **Overall Impression:** The image evokes feelings of warmth, cuteness, and tenderness. The kitten appears to be looking directly at the viewer, creating a sense of connection and affection.\n\nThis is a lovely and charming depiction of a young kitten, capturing its innocence and charm in a visually appealing and emotionally engaging way.\n--------------------------------------------------\n<<< describe the video.\nInput a video path or URL <<< https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4\nThis video captures a charming and adorable moment of a young child, likely a toddler, sitting on a bed and pretending to read a book. The child is wearing glasses, which adds a humorous and endearing touch to the scene — as if they’re a little scholar or librarian.\n\nHere’s a breakdown of what unfolds:\n\n- The child is seated cross-legged on a bed with a patterned quilt. Behind them, a crib and some household items are visible, suggesting a cozy bedroom setting.\n\n- The child holds an open book and appears to be turning the pages with focused attention, mimicking the behavior of a real reader.\n\n- At one point, the child looks up, smiles, or seems to react with delight — perhaps amused by something in the book or just enjoying the activity.\n\n- The child’s movements are gentle and deliberate, showing a sense of concentration and curiosity. They turn pages, sometimes with one hand, and occasionally lift the book slightly as if to examine it more closely.\n\n- The video has a warm, candid feel — it’s not staged, and the child’s natural behavior makes it feel authentic and heartwarming.\n\nOverall, this is a sweet, lighthearted video that showcases the innocence and imagination of early childhood. The child’s engagement with the book, combined with their glasses and playful demeanor, creates a delightful and memorable scene.\n```\n\n- 其中特定模型参数，例如 `VIDEO_MAX_TOKEN_NUM` 等环境变量的含义参考[命令行参数文档](../Instruction/Command-line-parameters.md#qwen3_vl-qwen3_5)。\n\n\n## 训练\n\n本文档将介绍如何使用 ms-swift 与 Megatron-SWIFT 训练 Qwen3-VL。推荐 Dense 模型使用 ms-swift（即 transformers 后端，更加方便简单），而 Moe 模型使用 Megatron-SWIFT（即 megatron 后端，更快的训练速度，benchmark查看[这里](../Megatron-SWIFT/Quick-start.md#benchmark)）。\n\n如果您需要自定义数据集微调模型，你可以将数据准备成以下格式，并在命令行中设置`--dataset train.jsonl --val_dataset val.jsonl`，其中验证集为可选。更多介绍请参考[多模态数据集文档](../Customization/Custom-dataset.md#多模态)。\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"浙江的省会在哪？\"}, {\"role\": \"assistant\", \"content\": \"浙江的省会在杭州。\"}]}\n{\"messages\": [{\"role\": \"user\", \"content\": \"两张图片有什么区别\"}, {\"role\": \"assistant\", \"content\": \"前一张是小猫，后一张是小狗\"}], \"images\": [\"/xxx/x.jpg\", \"/xxx/x.png\"]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"图片中是什么，视频中是什么\"}, {\"role\": \"assistant\", \"content\": \"图片中是一个大象，视频中是一只小狗在草地上奔跑\"}], \"images\": [\"/xxx/x.jpg\"], \"videos\": [\"/xxx/x.mp4\"]}\n```\n\nQwen3-VL的bbox输出采用归一化1000的相对坐标。你可以使用 ms-swift 提供的 grounding 数据集格式，其中\"bbox\"中的坐标为绝对坐标，ms-swift 会自动将绝对坐标转为归一化1000的相对坐标。更多信息请参考[grounding数据集格式文档](../Customization/Custom-dataset.md#grounding)。\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"找到图像中的\"}, {\"role\": \"assistant\", \"content\": \"[\\n\\t{\\\"bbox_2d\\\": , \\\"label\\\": \\\"\\\"},\\n\\t{\\\"bbox_2d\\\": , \\\"label\\\": \\\"\\\"}\\n]\"}], \"images\": [\"cat.png\"], \"objects\": {\"ref\": [\"羊\", \"羊\", \"羊\"], \"bbox\": [[90.9, 160.8, 135, 212.8], [360.9, 480.8, 495, 532.8]]}}\n```\n\n### Dense模型\n\n以下提供对`Qwen3-VL-4B-Instruct`模型的微调脚本，我们使用混合模态数据作为Demo数据集，该示例脚本仅作为演示用途。训练显存为2 * 21GiB，训练时间为12分钟。\n- 若觉得预处理时间太长，你可以将`--packing`去除，或者使用[cached dataset](https://github.com/modelscope/ms-swift/tree/main/examples/train/cached_dataset)。\n\n```shell\n# 2 * 21GiB\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nIMAGE_MAX_TOKEN_NUM=1024 \\\nVIDEO_MAX_TOKEN_NUM=128 \\\nFPS_MAX_FRAMES=16 \\\nNPROC_PER_NODE=2 \\\nCUDA_VISIBLE_DEVICES=0,1 \\\nswift sft \\\n --model Qwen/Qwen3-VL-4B-Instruct \\\n --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \\\n 'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \\\n 'swift/VideoChatGPT:Generic#2000' \\\n --load_from_cache_file true \\\n --split_dataset_ratio 0.01 \\\n --tuner_type lora \\\n --torch_dtype bfloat16 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 1 \\\n --per_device_eval_batch_size 1 \\\n --attn_impl flash_attn \\\n --padding_free true \\\n --learning_rate 1e-4 \\\n --lora_rank 8 \\\n --lora_alpha 32 \\\n --target_modules all-linear \\\n --freeze_vit true \\\n --freeze_aligner true \\\n --packing true \\\n --gradient_checkpointing true \\\n --vit_gradient_checkpointing false \\\n --gradient_accumulation_steps 2 \\\n --eval_steps 100 \\\n --save_steps 100 \\\n --save_total_limit 2 \\\n --logging_steps 5 \\\n --max_length 4096 \\\n --output_dir output \\\n --warmup_ratio 0.05 \\\n --deepspeed zero2 \\\n --dataset_num_proc 4 \\\n --dataloader_num_workers 4\n```\n\n训练结束后，我们使用以下脚本对验证集进行推理：\n```shell\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nCUDA_VISIBLE_DEVICES=0 \\\nIMAGE_MAX_TOKEN_NUM=1024 \\\nVIDEO_MAX_TOKEN_NUM=128 \\\nFPS_MAX_FRAMES=16 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --max_new_tokens 2048 \\\n --load_data_args true\n```\n\n```\n--------------------------------------------------\n[QUERY] Using LaTeX to perform OCR on the image.\n[LABELS] 1 + \\frac { 1 } { 1 ! } + \\frac { 1 } { 2 ! } + \\frac { 1 } { 3 ! } + \\frac { 1 } { 4 ! }\n[RESPONSE] 1 + \\frac { 1 } { 1 ! } + \\frac { 1 } { 2 ! } + \\frac { 1 } { 3 ! } + \\frac { 1 } { 4 ! }\n--------------------------------------------------\n[QUERY] What color suit is the man wearing while playing the saxophone on stage?\n[LABELS] The man is wearing a black suit and white shirt while playing the saxophone on the red-floored stage.\n[RESPONSE] The man is wearing a black suit while playing the saxophone on stage.\n--------------------------------------------------\n...\n```\n\n### Moe模型\n\n\n以下提供对`Qwen3-VL-30B-A3B-Instruct`模型的微调脚本，我们使用 Megatron-SWIFT 进行单机全参数训练。我们同样采用混合数据进行训练，该示例脚本仅作为演示用途。训练所需显存资源为8 * 80GiB，训练时间为20分钟。\n\n关于 Megatron-SWIFT 的环境安装，请参考[Megatron-SWIFT文档](../Megatron-SWIFT/Quick-start.md)。Megatron-SWIFT 与 ms-swift 共用 template 和 dataset 模块，因此前面介绍的自定义数据集格式和模型特有环境变量依旧生效。\n\n\n微调脚本如下，训练技巧与并行技术的调整参考[Megatron-SWIFT文档](../Megatron-SWIFT/Quick-start.md#训练技巧)。\n```shell\n# 8 * 80GiB\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nOMP_NUM_THREADS=14 \\\nNPROC_PER_NODE=8 \\\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \\\nIMAGE_MAX_TOKEN_NUM=1024 \\\nVIDEO_MAX_TOKEN_NUM=128 \\\nFPS_MAX_FRAMES=16 \\\nmegatron sft \\\n --model Qwen/Qwen3-VL-30B-A3B-Instruct \\\n --save_safetensors true \\\n --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \\\n 'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \\\n 'swift/VideoChatGPT:Generic#2000' \\\n --load_from_cache_file true \\\n --split_dataset_ratio 0.01 \\\n --moe_permute_fusion true \\\n --tensor_model_parallel_size 4 \\\n --expert_model_parallel_size 8 \\\n --moe_grouped_gemm true \\\n --moe_shared_expert_overlap true \\\n --moe_aux_loss_coeff 1e-6 \\\n --micro_batch_size 1 \\\n --global_batch_size 4 \\\n --recompute_granularity full \\\n --recompute_method uniform \\\n --recompute_num_layers 1 \\\n --num_train_epochs 1 \\\n --finetune true \\\n --cross_entropy_loss_fusion true \\\n --lr 1e-5 \\\n --lr_warmup_fraction 0.05 \\\n --min_lr 1e-6 \\\n --output_dir megatron_output/Qwen3-VL-30B-A3B-Instruct \\\n --eval_steps 500 \\\n --save_steps 500 \\\n --max_length 4096 \\\n --packing true \\\n --dataloader_num_workers 8 \\\n --dataset_num_proc 8 \\\n --no_save_optim true \\\n --no_save_rng true \\\n --sequence_parallel true \\\n --moe_expert_capacity_factor 2 \\\n --attention_backend flash\n```\n\n训练结束后，我们使用以下脚本对验证集进行推理：\n```shell\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nCUDA_VISIBLE_DEVICES=0 \\\nIMAGE_MAX_TOKEN_NUM=1024 \\\nVIDEO_MAX_TOKEN_NUM=128 \\\nFPS_MAX_FRAMES=16 \\\nswift infer \\\n --model megatron_output/Qwen3-VL-30B-A3B-Instruct/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --max_new_tokens 2048 \\\n --load_data_args true\n```\n\n\n使用以下命令将训练权重推送到 Modelscope：\n```shell\nswift export \\\n --model output/vx-xxx/checkpoint-xxx \\\n --push_to_hub true \\\n --hub_model_id '' \\\n --hub_token ''\n```\n"
},
{
"path": "docs/source/BestPractices/Qwen3_5-Best-Practice.md",
"content": "# Qwen3.5 最佳实践\n\nms-swift 支持使用transformers/Megatron后端对[Qwen3.5](https://github.com/QwenLM/Qwen3.5) Dense/Moe模型进行训练。Qwen3.5 属于混合思考的多模态模型，结合了linear attention和full attention。本文将介绍如何对Qwen3.5 Dense/Moe模型进行推理、指令微调以及强化学习。\n\n\n## 环境设置\n```shell\npip install -U ms-swift\n# \"transformers==5.2.*\" 会遇到与vllm的兼容问题，参考这个issue: https://github.com/modelscope/ms-swift/issues/8254\n# \"transformers==5.3.*\" 会遇到视频训练问题，参考这个issue: https://github.com/modelscope/ms-swift/issues/8362\npip install -U \"transformers==5.2.*\" \"qwen_vl_utils>=0.0.14\" peft liger-kernel\n\n# flash-linear-attention\n# 请安装fla main分支，若出现训练缓慢的问题请参考：https://github.com/fla-org/flash-linear-attention/issues/758\npip install -U git+https://github.com/fla-org/flash-linear-attention\n\n# causal_conv1d\npip install -U git+https://github.com/Dao-AILab/causal-conv1d --no-build-isolation\n\n# flash-attention\npip install \"flash-attn==2.8.3\" --no-build-isolation\n\n# deepspeed训练\npip install deepspeed\n\n# vllm (torch2.10) for inference/deployment/RL\npip install -U \"vllm>=0.17.0\"\n# 对于强化学习（RL）训练，需要覆盖 vLLM 的默认安装版本\npip install -U \"transformers==5.2.*\"\n```\n\n- Qwen3.5 视频数据训练卡住：使用decord后端读取视频可能导致卡住问题，参考[这个issue](https://github.com/dmlc/decord/issues/269)。你可以使用torchcodec后端，具体参考[qwen_vl_utils](https://github.com/QwenLM/Qwen3-VL/blob/50068df2334f309979ff05d75f1078c8309c63ed/qwen-vl-utils/src/qwen_vl_utils/vision_process.py#L390-L400)库。\n\n\n## 推理\n\n使用 ms-swift 的 `TransformersEngine` 进行推理：\n\n- 其中特定模型参数，例如 `VIDEO_MAX_TOKEN_NUM` 等环境变量的含义与Qwen3-VL相同，参考[命令行参数文档](../Instruction/Command-line-parameters.md#qwen3_vl,qwen3_5)。\n\n```python\nimport os\n# os.environ['SWIFT_DEBUG'] = '1'\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\nos.environ['IMAGE_MAX_TOKEN_NUM'] = '1024'\nos.environ['VIDEO_MAX_TOKEN_NUM'] = '128'\nos.environ['FPS_MAX_FRAMES'] = '16'\n\nfrom swift import get_model_processor, get_template\nfrom swift.infer_engine import TransformersEngine, InferRequest, RequestConfig\n\nmodel, processor = get_model_processor('Qwen/Qwen3.5-4B') # attn_impl='flash_attention_2'\ntemplate = get_template(processor, enable_thinking=False)\nengine = TransformersEngine(model, template=template)\ninfer_request = InferRequest(messages=[{\n \"role\": \"user\",\n \"content\": 'Describe this video.',\n}], videos=['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'])\nrequest_config = RequestConfig(max_tokens=128, temperature=0)\nresp_list = engine.infer([infer_request], request_config=request_config)\nresponse = resp_list[0].choices[0].message.content\nprint(response)\n\n# use stream\nrequest_config = RequestConfig(max_tokens=128, temperature=0, stream=True)\ngen_list = engine.infer([infer_request], request_config=request_config)\nfor chunk in gen_list[0]:\n if chunk is None:\n continue\n print(chunk.choices[0].delta.content, end='', flush=True)\nprint()\n```\n\n使用命令行进行推理：\n\n```shell\nIMAGE_MAX_TOKEN_NUM=1024 \\\nVIDEO_MAX_TOKEN_NUM=128 \\\nFPS_MAX_FRAMES=16 \\\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --model Qwen/Qwen3.5-4B \\\n --enable_thinking false \\\n --stream true\n```\n\n\n## 微调\n\n本章将介绍如何使用 ms-swift 与 Megatron-SWIFT 训练 Qwen3.5。推荐 Dense 模型使用 ms-swift（即 transformers 后端，更加方便简单），而 Moe 模型使用 Megatron-SWIFT（即 megatron 后端，更快的训练速度）\n\n如果您需要自定义数据集微调模型，你可以将数据准备成以下格式，并在命令行中设置`--dataset train.jsonl --val_dataset val.jsonl`，其中验证集为可选。更多介绍请参考[多模态数据集文档](../Customization/Custom-dataset.md#多模态)。\n\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"浙江的省会在哪？\"}, {\"role\": \"assistant\", \"content\": \"浙江的省会在杭州。\"}]}\n{\"messages\": [{\"role\": \"user\", \"content\": \"两张图片有什么区别\"}, {\"role\": \"assistant\", \"content\": \"前一张是小猫，后一张是小狗\"}], \"images\": [\"/xxx/x.jpg\", \"/xxx/x.png\"]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"图片中是什么，视频中是什么\"}, {\"role\": \"assistant\", \"content\": \"图片中是一个大象，视频中是一只小狗在草地上奔跑\"}], \"images\": [\"/xxx/x.jpg\"], \"videos\": [\"/xxx/x.mp4\"]}\n```\n\nQwen3.5的bbox输出采用归一化1000的相对坐标。你可以使用 ms-swift 提供的 grounding 数据集格式，其中\"bbox\"中的坐标为绝对坐标，ms-swift 会自动将绝对坐标转为归一化1000的相对坐标。更多信息请参考[grounding数据集格式文档](../Customization/Custom-dataset.md#grounding)。\n\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"找到图像中的\"}, {\"role\": \"assistant\", \"content\": \"[\\n\\t{\\\"bbox_2d\\\": , \\\"label\\\": \\\"\\\"},\\n\\t{\\\"bbox_2d\\\": , \\\"label\\\": \\\"\\\"}\\n]\"}], \"images\": [\"cat.png\"], \"objects\": {\"ref\": [\"羊\", \"羊\", \"羊\"], \"bbox\": [[90.9, 160.8, 135, 212.8], [360.9, 480.8, 495, 532.8]]}}\n```\n\n\n### Dense模型\n\n以下提供对Qwen3.5-4B模型的微调脚本，该示例脚本仅作为演示用途。训练显存为 4 * 20GiB，训练时间为12分钟。由于transformers的GatedDeltaNet不支持packing/padding_free（megatron支持，见下文），因此我们使用group_by_length参数来加速训练，保证DP的负载均衡并减少micro batch中的零填充，但这会导致loss曲线跳动（因数据随机不充分），当然你也可以去掉此参数。\n\n对模型进行微调的脚本如下：\n\n```shell\n# 4 * 20GiB\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nNPROC_PER_NODE=4 \\\nMAX_PIXELS=1003520 \\\nVIDEO_MAX_PIXELS=50176 \\\nFPS_MAX_FRAMES=12 \\\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nswift sft \\\n --model Qwen/Qwen3.5-4B \\\n --tuner_type lora \\\n --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \\\n 'AI-ModelScope/alpaca-gpt4-data-en#500' \\\n 'swift/self-cognition#500' \\\n 'AI-ModelScope/LaTeX_OCR:human_handwrite#2000' \\\n --load_from_cache_file true \\\n --add_non_thinking_prefix true \\\n --loss_scale ignore_empty_think \\\n --split_dataset_ratio 0.01 \\\n --torch_dtype bfloat16 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 4 \\\n --per_device_eval_batch_size 4 \\\n --learning_rate 1e-4 \\\n --lora_rank 8 \\\n --lora_alpha 32 \\\n --target_modules all-linear \\\n --gradient_accumulation_steps 1 \\\n --group_by_length true \\\n --output_dir output/Qwen3.5-4B \\\n --eval_steps 50 \\\n --save_steps 50 \\\n --save_total_limit 2 \\\n --logging_steps 5 \\\n --max_length 2048 \\\n --warmup_ratio 0.05 \\\n --dataset_num_proc 4 \\\n --dataloader_num_workers 4 \\\n --deepspeed zero2 \\\n --model_author swift \\\n --model_name swift-robot\n```\n\n训练结束后，使用以下脚本对验证集进行推理：\n\n```shell\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nCUDA_VISIBLE_DEVICES=0 \\\nMAX_PIXELS=1003520 \\\nVIDEO_MAX_PIXELS=50176 \\\nFPS_MAX_FRAMES=12 \\\nswift infer \\\n --adapters output/Qwen3.5-4B/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --enable_thinking false \\\n --max_new_tokens 512 \\\n --load_data_args true\n```\n\n\n```text\n[QUERY] 你好，你是谁？\n[RESPONSE] \n\n \n\n你好，我是由swift开发的人工智能语言模型，我的名字叫swift-robot。很高兴能与你交流。\n--------------------------------------------------\n[QUERY] Using LaTeX to perform OCR on the image.\n[LABELS] e = \\sum _ { k = 0 } ^ { \\infty } \\frac { 1 } { k ! }\n[RESPONSE] \n\n \n\ne = \\sum _ { k = 0 } ^ { \\infty } \\frac { 1 } { k ! }\n```\n\n使用python进行推理：\n```python\nimport os\n# os.environ['SWIFT_DEBUG'] = '1'\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\nos.environ['IMAGE_MAX_TOKEN_NUM'] = '1024'\nos.environ['VIDEO_MAX_TOKEN_NUM'] = '128'\nos.environ['FPS_MAX_FRAMES'] = '16'\n\nfrom peft import PeftModel\nfrom swift import get_model_processor, get_template\nfrom swift.infer_engine import TransformersEngine, InferRequest, RequestConfig\n\nadapter_dir = 'output/Qwen3.5-4B/vx-xxx/checkpoint-xxx'\nenable_thinking = False\n\nmodel, processor = get_model_processor('Qwen/Qwen3.5-4B') # attn_impl='flash_attention_2'\nmodel = PeftModel.from_pretrained(model, adapter_dir)\ntemplate = get_template(processor, enable_thinking=enable_thinking)\nengine = TransformersEngine(model, template=template)\ninfer_request = InferRequest(messages=[{\n \"role\": \"user\",\n \"content\": 'who are you?',\n}])\nrequest_config = RequestConfig(max_tokens=128, temperature=0)\nresp_list = engine.infer([infer_request], request_config=request_config)\nresponse = resp_list[0].choices[0].message.content\nprint(response)\n\n# use stream\nrequest_config = RequestConfig(max_tokens=128, temperature=0, stream=True)\ngen_list = engine.infer([infer_request], request_config=request_config)\nfor chunk in gen_list[0]:\n if chunk is None:\n continue\n print(chunk.choices[0].delta.content, end='', flush=True)\nprint()\n# I am an artificial intelligence assistant named swift-robot, trained by swift. I am designed to understand and generate natural language text in order to provide information, answer questions, and engage in conversation with humans. How can I assist you?\n```\n\n\n使用transformers后端训练MoE的例子参考：https://github.com/modelscope/ms-swift/blob/main/examples/models/qwen3_5/transformers.sh\n\n### Moe模型\nQwen3.5-35B-A3B Megatron训练，环境的准备请参考[Megatron-SWIFT快速开始文档](../Megatron-SWIFT/Quick-start.md)。你可以在15分钟内跑完以下案例：\n\n```shell\n# 4 * 40GiB\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nNPROC_PER_NODE=4 \\\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nMAX_PIXELS=1003520 \\\nVIDEO_MAX_PIXELS=50176 \\\nFPS_MAX_FRAMES=12 \\\nmegatron sft \\\n --model Qwen/Qwen3.5-35B-A3B \\\n --save_safetensors true \\\n --merge_lora true \\\n --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \\\n 'AI-ModelScope/alpaca-gpt4-data-en#500' \\\n 'swift/self-cognition#500' \\\n 'AI-ModelScope/LaTeX_OCR:human_handwrite#2000' \\\n --load_from_cache_file true \\\n --add_non_thinking_prefix true \\\n --loss_scale ignore_empty_think \\\n --split_dataset_ratio 0.01 \\\n --tuner_type lora \\\n --lora_rank 8 \\\n --lora_alpha 32 \\\n --target_modules all-linear \\\n --expert_model_parallel_size 4 \\\n --moe_permute_fusion true \\\n --moe_grouped_gemm true \\\n --moe_shared_expert_overlap true \\\n --moe_aux_loss_coeff 1e-6 \\\n --micro_batch_size 4 \\\n --global_batch_size 16 \\\n --recompute_granularity full \\\n --recompute_method uniform \\\n --recompute_num_layers 1 \\\n --num_train_epochs 1 \\\n --group_by_length true \\\n --finetune true \\\n --freeze_llm false \\\n --freeze_vit true \\\n --freeze_aligner true \\\n --cross_entropy_loss_fusion true \\\n --lr 1e-4 \\\n --lr_warmup_fraction 0.05 \\\n --min_lr 1e-5 \\\n --output_dir megatron_output/Qwen3.5-35B-A3B \\\n --eval_steps 200 \\\n --save_steps 200 \\\n --max_length 2048 \\\n --dataloader_num_workers 8 \\\n --dataset_num_proc 8 \\\n --no_save_optim true \\\n --no_save_rng true \\\n --sequence_parallel true \\\n --attention_backend flash \\\n --padding_free false \\\n --model_author swift \\\n --model_name swift-robot\n```\n训练结束后，使用以下脚本对验证集进行推理：\n\n```shell\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nMAX_PIXELS=1003520 \\\nVIDEO_MAX_PIXELS=50176 \\\nFPS_MAX_FRAMES=12 \\\nswift infer \\\n --model megatron_output/Qwen3.5-35B-A3B/vx-xxx/checkpoint-xxx-merged \\\n --stream true \\\n --enable_thinking false \\\n --max_new_tokens 512 \\\n --load_data_args true\n```\n\nMegatron-SWIFT训练Qwen3.5的提示：\n- 全参数训练：参考[这个例子](https://github.com/modelscope/ms-swift/tree/main/examples/models/qwen3_5/mcore_full.sh)。\n- 关于MTP训练：ms-swift暂不支持多模态MTP的训练。如果你只训练纯文本数据，请设置`SKIP_MULTIMODAL_MTP_VALIDATION=1`环境变量，忽略检查。\n- TP 限制解除：使用 \"megatron-core>=0.16\" 可解除 TP 受到的 `num_query_groups` 限制。\n- 默认 `GatedDeltaNet` 使用 transformers 实现（为保证稳定性，暂时保持默认行为不变）。使用 \"megatron-core>=0.16\"并设置环境变量 `SWIFT_USE_MCORE_GDN=1`可切换至 mcore 实现，支持 GDN 的 TP 并降低显存。\n- padding_free/packing的支持：packing可以提升训练速度，你需要设置`SWIFT_USE_MCORE_GDN=1`环境变量。参考[这个例子](https://github.com/modelscope/ms-swift/tree/main/examples/models/qwen3_5/packing.sh)。\n- apply_wd_to_qk_layernorm：对 qk layernorm 应用权重衰减。默认为False。\n\n\n## 强化学习（RL）\n\n以 Qwen3.5-2B 模型为例，下面展示基于 [GSM8K](https://www.modelscope.cn/datasets/modelscope/gsm8k) 数据集进行 GRPO 和 GKD 训练，并以 GSM8K 评测集为标准验证训练效果。为避免模型输出过长的思维链，以下统一设置 `enable_thinking false`。\n\n### GRPO\n\n#### Dense 模型\n使用 GRPO 进行全参数训练，以 `gsm8k_accuracy` 和 `gsm8k_format` 作为奖励函数。奖励函数的实现参考 [gsm8k_plugin.py](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo/plugin/gsm8k/gsm8k_plugin.py)。\n\n```shell\nSYSTEM_PROMPT=\"\"\"You are a helpful math assistant. Solve the problem step by step and put your final answer within \\\\boxed{}.\"\"\"\n\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nNPROC_PER_NODE=4 \\\nswift rlhf \\\n --rlhf_type grpo \\\n --model Qwen/Qwen3.5-2B \\\n --external_plugins examples/train/grpo/plugin/gsm8k/gsm8k_plugin.py \\\n --reward_funcs gsm8k_accuracy gsm8k_format \\\n --columns '{\"answer\": \"solution\"}' \\\n --enable_thinking false \\\n --use_vllm true \\\n --vllm_mode colocate \\\n --vllm_gpu_memory_utilization 0.4 \\\n --vllm_tensor_parallel_size 1 \\\n --vllm_max_model_len 10240 \\\n --sleep_level 1 \\\n --tuner_type full \\\n --torch_dtype bfloat16 \\\n --dataset 'modelscope/gsm8k' \\\n --load_from_cache_file true \\\n --max_length 2048 \\\n --max_completion_length 8192 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 4 \\\n --gradient_accumulation_steps 4 \\\n --learning_rate 1e-6 \\\n --lr_scheduler_type cosine \\\n --save_steps 10 \\\n --save_total_limit 100 \\\n --logging_steps 1 \\\n --warmup_ratio 0.0 \\\n --dataloader_num_workers 4 \\\n --num_generations 8 \\\n --temperature 1.0 \\\n --system \"$SYSTEM_PROMPT\" \\\n --deepspeed zero2 \\\n --log_completions true \\\n --report_to tensorboard swanlab \\\n --max_grad_norm 1.0 \\\n --epsilon 0.2 \\\n --epsilon_high 0.28 \\\n --scale_rewards none\n```\n\n使用以下指令进行评测：\n\n```shell\nCUDA_VISIBLE_DEVICES=0 swift eval \\\n --model output/Qwen3.5-2B/vxx-xxx-xxx/checkpoint-xx \\\n --enable_thinking false \\\n --eval_dataset gsm8k \\\n --eval_backend Native --infer_backend vllm \\\n --eval_generation_config '{\"max_tokens\":8192,\"temperature\":0.0,\"do_sample\":false}'\n```\n\n以 10 步为间隔，前 50 步的 GSM8K 评测结果如下：\n\n| 模型 / Steps | GSM8K Accuracy | 提升 |\n|---|---|---|\n| Qwen3.5-2B (baseline) | 0.7597 | - |\n| GRPO 10 steps | 0.7650 | +0.53 |\n| GRPO 20 steps | 0.7748 | +1.51 |\n| GRPO 30 steps | 0.7779 | +1.82 |\n| GRPO 40 steps | 0.7817 | +2.20 |\n| GRPO 50 steps | 0.7885 | +2.88 |\n\n#### MoE 模型\n\n使用 Megatron 后端对 Qwen3.5-35B-A3B MoE 模型进行 GRPO LoRA 训练，在 [DAPO-Math-17k](https://www.modelscope.cn/datasets/open-r1/DAPO-Math-17k-Processed) 数据集上训练，使用 `accuracy` 作为奖励函数。\n\n```shell\nSYSTEM_PROMPT=\"\"\"You are a helpful math assistant. Solve the problem step by step and put your final answer within \\\\boxed{}.\"\"\"\n\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \\\nNPROC_PER_NODE=8 \\\nPYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \\\nmegatron rlhf \\\n --rlhf_type grpo \\\n --model Qwen/Qwen3.5-35B-A3B \\\n --save_safetensors true \\\n --enable_thinking false \\\n --merge_lora true \\\n --context_parallel_size 1 \\\n --tensor_model_parallel_size 1 \\\n --expert_model_parallel_size 8 \\\n --pipeline_model_parallel_size 1 \\\n --moe_permute_fusion true \\\n --dataset open-r1/DAPO-Math-17k-Processed \\\n --system \"$SYSTEM_PROMPT\" \\\n --num_train_epochs 1 \\\n --global_batch_size 64 \\\n --micro_batch_size 1 \\\n --steps_per_generation 2 \\\n --num_generations 8 \\\n --reward_funcs accuracy \\\n --use_vllm true \\\n --vllm_mode colocate \\\n --vllm_gpu_memory_utilization 0.5 \\\n --vllm_tensor_parallel_size 2 \\\n --vllm_max_model_len 9192 \\\n --max_length 1000 \\\n --max_completion_length 8192 \\\n --tuner_type lora \\\n --target_modules all-linear \\\n --lr 5e-5 \\\n --bf16 true \\\n --beta 0.00 \\\n --epsilon 0.2 \\\n --epsilon_high 0.28 \\\n --dynamic_sample false \\\n --overlong_filter true \\\n --loss_type grpo \\\n --sleep_level 1 \\\n --offload_model true \\\n --offload_bridge false \\\n --offload_optimizer true \\\n --logging_steps 1 \\\n --recompute_granularity full \\\n --recompute_method uniform \\\n --recompute_num_layers 1 \\\n --finetune \\\n --dataloader_num_workers 8 \\\n --dataset_num_proc 8 \\\n --no_save_optim \\\n --no_save_rng \\\n --save_steps 20 \\\n --attention_backend flash \\\n --moe_expert_capacity_factor 2 \\\n --temperature 1.0 \\\n --padding_free false \\\n --sequence_parallel true \\\n --log_completions true \\\n --report_to tensorboard swanlab\n```\n\n使用以下指令在 AIME-2025 和 MATH-500 上评测：\n\n```shell\nCUDA_VISIBLE_DEVICES=0,1 swift eval \\\n --model \\\n --enable_thinking false \\\n --eval_dataset aime25 math_500 \\\n --eval_backend Native --infer_backend vllm \\\n --vllm_tensor_parallel_size 2 \\\n --vllm_gpu_memory_utilization 0.9 \\\n --vllm_max_model_len 10000 \\\n --eval_generation_config '{\"max_tokens\":8192,\"temperature\":0.0,\"do_sample\":false}' \\\n --eval_num_proc 8\n```\n\n在 AIME-2025 和 MATH-500 上的评测结果如下：\n\n| 模型 / Steps | AIME-2025 | MATH-500 |\n|---|---|---|\n| Qwen3.5-35B-A3B (baseline) | 43.33 | 92.40 |\n| Megatron GRPO 20 steps | 53.33 (+10.00) | 95.80 (+3.40) |\n| Megatron GRPO 40 steps | 53.33 (+10.00) | 96.60 (+4.20) |\n\n### GKD\n\n使用 GKD 进行 LoRA 训练，以 Qwen3.5-9B 作为 teacher 模型。首先使用 vLLM 拉起 teacher server（也可以通过 `--teacher_model` 参数直接加载模型）：\n\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nvllm serve Qwen/Qwen3.5-9B \\\n --port 8000 \\\n --tensor-parallel-size 1 \\\n --max-model-len 10240 \\\n --gpu-memory-utilization 0.8 \\\n --max-logprobs 64\n```\n\n然后在其余 GPU 上启动 GKD 训练：\n\n```shell\nNPROC_PER_NODE=3 \\\nCUDA_VISIBLE_DEVICES=1,2,3 \\\nswift rlhf \\\n --rlhf_type gkd \\\n --model Qwen/Qwen3.5-2B \\\n --teacher_model_server http://localhost:8000 \\\n --gkd_logits_topk 64 \\\n --enable_thinking false \\\n --tuner_type lora \\\n --use_vllm true \\\n --vllm_mode colocate \\\n --vllm_gpu_memory_utilization 0.5 \\\n --vllm_tensor_parallel_size 1 \\\n --vllm_max_model_len 10240 \\\n --sleep_level 0 \\\n --dataset 'modelscope/gsm8k' \\\n --lmbda 1 \\\n --seq_kd false \\\n --beta 0.5 \\\n --torch_dtype bfloat16 \\\n --per_device_train_batch_size 2 \\\n --gradient_accumulation_steps 16 \\\n --learning_rate 5e-5 \\\n --logging_steps 1 \\\n --save_steps 100 \\\n --save_total_limit 10 \\\n --max_length 2048 \\\n --max_completion_length 8192 \\\n --warmup_ratio 0.1 \\\n --save_only_model true \\\n --dataloader_num_workers 4 \\\n --dataset_num_proc 4 \\\n --attn_impl flash_attn \\\n --report_to tensorboard swanlab\n```\n\n使用以下指令进行评测：\n\n```shell\nCUDA_VISIBLE_DEVICES=0 swift eval \\\n --model Qwen/Qwen3.5-2B \\\n --adapters output/Qwen3.5-2B/vxx-xxx-xxx/checkpoint-xx \\\n --merge_lora true \\\n --enable_thinking false \\\n --eval_dataset gsm8k \\\n --eval_backend Native --infer_backend vllm \\\n --eval_generation_config '{\"max_tokens\":8192,\"temperature\":0.0,\"do_sample\":false}'\n```\n\n以 100 步为间隔，前 300 步的 GSM8K 评测结果如下：\n\n| 模型 / Steps | GSM8K Accuracy | 提升 |\n|---|---|---|\n| Qwen3.5-2B (baseline) | 0.7597 | - |\n| GKD 100 steps | 0.7968 | +3.71 |\n| GKD 200 steps | 0.8188 | +5.91 |\n| GKD 300 steps | 0.8332 | +7.35 |\n"
},
{
"path": "docs/source/BestPractices/Rapidly-Training-VL-model.md",
"content": "# 快速训练VL模型\n\n本文档提供从零开始快速训练视觉语言(Vision-Language, VL)模型的最佳实践。\n\n涉及的模型链接：\n- [Qwen2.5-VL-7B-Instruct](https://www.modelscope.cn/models/Qwen/Qwen2.5-VL-7B-Instruct)\n- [Qwen3-8B](https://www.modelscope.cn/models/Qwen/Qwen3-8B)\n\n训练的模型链接：\n- [Simple-VL-8B](https://www.modelscope.cn/models/swift/Simple-VL-8B/summary)\n\n\n本训练流程基于 Qwen2.5-VL-7B-Instruct 模型架构，将其内部的语言模型（LLM）部分替换为 Qwen3-8B 的权重，训练模型的视觉理解能力。具体步骤如下：\n\n1. 修改原始模型的配置文件 config.json，使其适配 Qwen3-8B 的模型结构。\n2. 初始化并加载新的模型权重，保存为新模型。\n3. 对新模型进行两阶段微调：\n 1. 第一阶段：仅训练视觉到语言的对齐模块（aligner），冻结 ViT 和 LLM 部分。\n 2. 第二阶段：解冻所有模块，联合训练提升整体性能。\n\n\n## 模型修改\n\n### 修改配置文件 config.json\n因为 Qwen2.5-VL-7B-Instruct 模型的底模 Qwen2.5-7B-Instruct 与 Qwen3-8B 在模型结构上存在部分差异（比如层数，hidden_state_dims），我们首先需要基于Qwen2.5-VL-7B-Instruct的config.json文件，创建一个新的config.json文件，并修改以下参数对齐Qwen3-8B\n\n```\n修改\n1. hidden_size 3584->4096\n2. intermediate_size: 18944->12288\n3. num_attention_heads: 28->32\n4. num_key_value_heads: 4->8\n5. num_hidden_layers: 28->36\n6. vocab_size:152064->151936\n7. max_window_layers:28->36\n8. out_hidden_size: 3584->4096\n\n新增\n1. head_dim： 128\n```\n\n### 模型权重初始化与替换\n使用以下 Python 脚本完成模型权重的初始化、替换与保存：\n\n```python\nimport torch\nfrom modelscope import Qwen2_5_VLForConditionalGeneration, AutoModelForCausalLM, AutoConfig\nfrom transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLPatchMerger, Qwen2_5_VLModel\nfrom accelerate import Accelerator\n\n# 加载原始 VL 模型和 Qwen3-8B 模型\nqwen2_5_vl_7b_model = Qwen2_5_VLForConditionalGeneration.from_pretrained(\n \"Qwen/Qwen2.5-VL-7B-Instruct\",\n device_map=\"cuda\",\n torch_dtype=torch.bfloat16\n)\ndevice = qwen2_5_vl_7b_model.device\nqwen3_8b_model = AutoModelForCausalLM.from_pretrained(\n \"Qwen/Qwen3-8B\",\n device_map=device,\n torch_dtype=torch.bfloat16\n)\n\n# 加载配置\nold_config = AutoConfig.from_pretrained(\"Qwen/Qwen2.5-VL-7B-Instruct\")\nnew_config = AutoConfig.from_pretrained(\"/path/to/new_config_dir\") # 新 config 的文件夹路径\nnew_visual_config = new_config.vision_config\n\n# 1. 替换 ViT 到 LLM 的 merger(aligner) 层\nnew_merger = Qwen2_5_VLPatchMerger(\n dim=new_visual_config.out_hidden_size,\n context_dim=new_visual_config.hidden_size,\n spatial_merge_size=new_visual_config.spatial_merge_size,\n ).to(device).to(torch.bfloat16)\nqwen2_5_vl_7b_model.visual.merger = new_merger\n\n# 2. 替换 VL 模型的 LLM 部分\nnew_llm_model = Qwen2_5_VLModel(new_config).to(device).to(torch.bfloat16)\n\nfor name, param in qwen3_8b_model.model.named_parameters():\n if name in new_llm_model.state_dict():\n new_llm_model.state_dict()[name].copy_(param)\n\nqwen2_5_vl_7b_model.model = new_llm_model\nqwen2_5_vl_7b_model.lm_head = qwen3_8b_model.lm_head\n\n# 3. 保存修改后的模型\naccelerator = Accelerator()\naccelerator.save_model(\n model=qwen2_5_vl_7b_model,\n save_directory=\"/path/to/save/Qwen3-VL-Model\",\n max_shard_size=\"4GB\",\n safe_serialization=True\n)\n```\n\n保存完权重后，将原 Qwen2.5-VL-7B-Instruct 模型文件夹中除模型权重的文件(包括`model.safetensors.index.json`) 复制到新的模型权重文件夹中，并替换 config.json 为新修改的 config.json文件。\n\n## 训练\n\n为简化流程，我们跳过预训练（pretrain），直接进入监督微调（SFT）。训练分为两个阶段：\n\n### stage1 训练 Aligner 层\n仅训练视觉到语言的对齐层（Aligner），冻结 ViT 和 LLM 部分：\n\n```bash\nNNODES=$WORLD_SIZE \\\nNODE_RANK=$RANK \\\nNPROC_PER_NODE=8 \\\nMAX_PIXELS=1003520 \\\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \\\nswift sft \\\n --model /path/to/new_vl_model \\\n --model_type qwen2_5_vl \\\n --tuner_type full \\\n --dataset xxx \\\n --load_from_cache_file true \\\n --split_dataset_ratio 0.01 \\\n --torch_dtype bfloat16 \\\n --attn_impl flash_attn \\\n --freeze_vit true \\\n --freeze_llm true \\\n --freeze_aligner false \\\n --num_train_epochs 3 \\\n --per_device_train_batch_size 2 \\\n --learning_rate 5e-6 \\\n --gradient_accumulation_steps 8 \\\n --eval_steps -1 \\\n --save_steps 1000 \\\n --save_total_limit 10 \\\n --logging_steps 5 \\\n --max_length 8192 \\\n --output_dir output \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 4 \\\n --dataset_num_proc 8 \\\n --deepspeed zero2\n```\n\n### stage2 训练整个模型\n解冻所有模块，联合训练以增强模型的整体视觉理解能力：\n\n```bash\nNNODES=$WORLD_SIZE \\\nNODE_RANK=$RANK \\\nNPROC_PER_NODE=8 \\\nMAX_PIXELS=1003520 \\\nCUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \\\nswift sft \\\n --model /path/to/stage1_checkpoint \\\n --model_type qwen2_5_vl \\\n --tuner_type full \\\n --dataset xxx \\\n --load_from_cache_file true \\\n --split_dataset_ratio 0.01 \\\n --torch_dtype bfloat16 \\\n --attn_impl flash_attn \\\n --freeze_vit false \\\n --freeze_llm false \\\n --freeze_aligner false \\\n --num_train_epochs 3 \\\n --per_device_train_batch_size 2 \\\n --learning_rate 5e-6 \\\n --gradient_accumulation_steps 8 \\\n --eval_steps -1 \\\n --save_steps 1000 \\\n --save_total_limit 10 \\\n --logging_steps 5 \\\n --max_length 8192 \\\n --output_dir output \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 4 \\\n --dataset_num_proc 8 \\\n --deepspeed zero2\n```\n\n## 推理/部署/评测\n\n### 推理\n通过`swift infer`来推理训练得到的模型\n```bash\nswift infer \\\n --model /path/to/stage2_checkpoint\n\n```\n\n### 部署\n使用 vLLM 加速模型服务部署：\n\n```\nCUDA_VISIBLE_DEVICES=0 \\\nMAX_PIXELS=1003520 \\\nVIDEO_MAX_PIXELS=50176 \\\nFPS_MAX_FRAMES=12 \\\nswift deploy \\\n --model /path/to/stage2_checkpoint \\\n --infer_backend vllm \\\n --vllm_gpu_memory_utilization 0.9 \\\n --vllm_max_model_len 8192 \\\n --max_new_tokens 2048 \\\n --vllm_limit_mm_per_prompt '{\"image\": 5, \"video\": 2}' \\\n --served_model_name Qwen3-VL\n```\n\n### 评测\n通过 [EvalScope](https://github.com/modelscope/evalscope/) 对训练得到的 VL 模型进行评测\n\n以下是以 MMMU benchmark 为例的评测代码：\n```python\nfrom evalscope import TaskConfig, run_task\n\ntask_cfg_dict = TaskConfig(\n work_dir='outputs',\n eval_backend='VLMEvalKit',\n eval_config={\n 'data': ['MMMU_DEV_VAL'],\n 'mode': 'all',\n 'model': [\n {'api_base': 'http://localhost:8000/v1/chat/completions',\n 'key': 'EMPTY',\n 'name': 'CustomAPIModel',\n 'temperature': 0.6,\n 'type': 'Qwen3-VL',\n 'img_size': -1,\n 'video_llm': False,\n 'max_tokens': 512,}\n ],\n 'reuse': False,\n 'nproc': 64,\n 'judge': 'exact_matching'},\n)\n\nrun_task(task_cfg=task_cfg_dict)\n```\n"
},
{
"path": "docs/source/BestPractices/Reranker.md",
"content": "# Reranker训练\n\nSWIFT已经支持Reranker模型的训练，目前已经支持的模型有：\n\n1. modernbert reranker模型\n - [ModelScope](https://www.modelscope.cn/models/iic/gte-reranker-modernbert-base) [Hugging Face](https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base)\n2. qwen3-reranker模型\n - 0.6B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-Reranker-0.6B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-Reranker-0.6B)\n - 4B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-Reranker-4B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-Reranker-4B)\n - 8B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-Reranker-8B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-Reranker-8B)\n3. qwen3-vl-reranker模型\n - 2B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-VL-Reranker-2B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-VL-Reranker-2B)\n - 8B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-VL-Reranker-8B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-VL-Reranker-8B)\n\n## 实现方式\n\n目前SWIFT支持两种Reranker模型的实现方式，二者在架构和损失函数计算上有显著差异：\n\n### 1. 分类式Reranker\n\n**适用模型：** modernbert reranker模型（如gte-reranker-modernbert-base）\n\n**核心原理：**\n- 基于序列分类架构，在预训练模型基础上添加分类头\n- 输入：query-document对，输出：单个相关性分数\n\n\n### 2. 生成式Reranker\n\n**适用模型：** qwen3-reranker模型（0.6B/4B/8B）\n\n**核心原理：**\n- 基于生成式语言模型架构（CausalLM）\n- 输入：query-document对，输出：特定token的概率（如\"yes\"/\"no\"）\n- 通过对比最后位置特定token的logits进行分类\n\n## 损失函数类型\n\nSWIFT支持多种损失函数来训练Reranker模型：\n\n### Pointwise损失函数\nPointwise方法将排序问题转化为二分类问题，独立处理每个query-document对：\n\n- **核心思想：** 对每个query-document对进行二分类，判断文档是否与查询相关\n- **损失函数：** 二分类交叉熵\n- **适用场景：** 简单高效，适合大规模数据训练\n\n环境变量配置：\n- `GENERATIVE_RERANKER_POSITIVE_TOKEN`：正例token（默认：\"yes\"）\n- `GENERATIVE_RERANKER_NEGATIVE_TOKEN`：负例token（默认：\"no\"）\n\n### Listwise损失函数\nListwise方法将排序问题转化为多分类问题，从多个候选文档中选择正例：\n\n- **核心思想：** 对每个query的候选文档组（1个正例 + n个负例）进行多分类，识别正例文档\n- **损失函数：** 多分类交叉熵\n- **适用场景：** 学习文档间的相对排序关系，更符合信息检索的实际需求\n\n环境变量配置：\n- `LISTWISE_RERANKER_TEMPERATURE`：softmax温度参数（默认：1.0）\n- `LISTWISE_RERANKER_MIN_GROUP_SIZE`：最小组大小，如果组内文档数量小于该值，则不计算损失（默认：2）\n\n**Listwise vs Pointwise：**\n- **Pointwise：** 独立判断相关性，训练简单，但忽略了文档间的相对关系\n- **Listwise：** 学习相对排序，性能更优，更适合排序任务的本质需求\n\nloss的源代码可以在[这里](https://github.com/modelscope/ms-swift/blob/main/swift/loss/mapping.py)找到。\n\n## 数据集格式\n\n```json lines\n# LLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"query\"}], \"positive_messages\": [[{\"role\": \"assistant\", \"content\": \"relevant_doc1\"}],[{\"role\": \"assistant\", \"content\": \"relevant_doc2\"}]], \"negative_messages\": [[{\"role\": \"assistant\", \"content\": \"irrelevant_doc1\"}],[{\"role\": \"assistant\", \"content\": \"irrelevant_doc2\"}], ...]}\n# MLLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"query\"}], \"images\": [\"/some/images.jpg\"], \"positive_messages\": [[{\"role\": \"assistant\", \"content\": \"relevant_doc1\"}]], \"positive_images\": [[\"/some/positive_images.jpg\"]], \"negative_messages\": [[{\"role\": \"assistant\", \"content\": \"irrelevant_doc1\"}], [{\"role\": \"assistant\", \"content\": \"irrelevant_doc2\"}]], \"negative_images\": [[\"/some/negative_images1.jpg\", \"/some/negative_images2.jpg\"], [\"/some/negative_images3.jpg\"]]}\n```\n\n**字段说明：**\n- `messages`：查询文本\n- `positive_messages`：与查询相关的正例文档列表，支持多个正例\n- `negative_messages`：与查询不相关的负例文档列表，支持多个负例\n\n**环境变量配置：**\n- `MAX_POSITIVE_SAMPLES`：每个query的最大正例数量（默认：1）\n- `MAX_NEGATIVE_SAMPLES`：每个query的最大负例数量（默认：7）\n\n> 默认会从每条数据中取出`MAX_POSITIVE_SAMPLES`条正样本和`MAX_NEGATIVE_SAMPLES`条负样本，每条正样本会和`MAX_NEGATIVE_SAMPLES`条负样本组成一个group，因此每条数据会扩展成`MAX_POSITIVE_SAMPLES`x`(1 + MAX_NEGATIVE_SAMPLES)`条数据。\n> 如果数据中正例/负例数量不足，会取全部正例/负例，如果数据中正例和负例数量超过`MAX_POSITIVE_SAMPLES`和`MAX_NEGATIVE_SAMPLES`，会进行随机采样。\n> **IMPORTANT**：展开后的数据会放在同一个batch中，因此每个设备上的实际批处理大小（effective batch size）将是 `per_device_train_batch_size` × `MAX_POSITIVE_SAMPLES` × (1 + `MAX_NEGATIVE_SAMPLES`)。请注意调整 `per_device_train_batch_size` 以避免显存不足。\n\n## 脚手架\n\nSWIFT提供的脚手架训练脚本：\n\n- [Qwen3-Reranker/Qwen3-VL-Reranker](https://github.com/modelscope/ms-swift/blob/main/examples/train/reranker/qwen3)\n- [Pointwise分类式Reranker](https://github.com/modelscope/ms-swift/blob/main/examples/train/reranker/train_reranker.sh)\n- [Pointwise生成式Reranker](https://github.com/modelscope/ms-swift/blob/main/examples/train/reranker/train_generative_reranker.sh)\n- [Listwise分类式Reranker](https://github.com/modelscope/ms-swift/blob/main/examples/train/reranker/train_reranker_listwise.sh)\n- [Listwise生成式Reranker](https://github.com/modelscope/ms-swift/blob/main/examples/train/reranker/train_generative_reranker_listwise.sh)\n\n推理脚本参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_reranker.py)。\n\n## 高级功能\n\n- Qwen3-Reranker 自定义 Instruction：\n - 默认模板如下：\n\n```text\n<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n: {Instruction}\n: {Query}\n: {Document}<|im_end|>\n<|im_start|>assistant\n\n\n \n\n\n```\n\n- 默认 Instruction：\n - `Given a web search query, retrieve relevant passages that answer the query`\n\n- Instruction 优先级（就近覆盖）：\n - `positive_messages`/`negative_messages` 内提供的 `system` > 主 `messages` 的 `system` > 默认 Instruction。\n - 即：若某个 positive/negative 的消息序列内包含 `system`，则优先使用该条；否则若主 `messages` 含 `system` 则使用之；两者都未提供时，使用默认 Instruction。\n"
},
{
"path": "docs/source/Customization/Architecture.md",
"content": "# 架构介绍\n\nms-swift 4.0 采用模块化设计，各功能模块分布在一级目录下，便于开发者进行自定义扩展。本文档将详细介绍各模块的功能及自定义方法。\n\n## Agent Template\n\nagent模板的mapping文件可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/agent_template/mapping.py)。agent template设计目标是，基于统一的Agent数据集格式，可以灵活切换不同模型进行训练，无需修改数据。训练时使用`--agent_template`指定对应的agent模板。\n\n所有的AgentTemplate需要继承自`BaseAgentTemplate`，并实现其中的几个方法: `_format_tools`, `_format_tool_calls`, `_format_tool_responses`, `get_toolcall`。\n- _format_tools: 将`tools`和`system`格式化，组成完整的system。\n- _format_tool_calls: 将tool_call部分 `[{\"role\": \"tool_call\", \"content\": \"...\"}, {\"role\": \"tool_call\", \"content\": \"...\"}]`进行格式化，最后返回字符串。\n- _format_tool_responses: 对tool（也称为tool_response）部分 `[{\"role\": \"tool\", \"content\": \"...\"}, {\"role\": \"tool\", \"content\": \"...\"}]`进行格式化。\n- get_toolcall: 在部署的时候使用，用于解析模型输出内容中的工具名和参数，返回`List[Function]`。\n\n\n如何debug：\n```python\ndata = {\"tools\": \"[{\\\"type\\\": \\\"function\\\", \\\"function\\\": {\\\"name\\\": \\\"realtime_aqi\\\", \\\"description\\\": \\\"天气预报。获取实时空气质量。当前空气质量，PM2.5，PM10信息\\\", \\\"parameters\\\": {\\\"type\\\": \\\"object\\\", \\\"properties\\\": {\\\"city\\\": {\\\"type\\\": \\\"string\\\", \\\"description\\\": \\\"城市名，例如：上海\\\"}}, \\\"required\\\": [\\\"city\\\"]}}}]\", \"messages\": [{\"role\": \"user\", \"content\": \"北京和上海今天的天气情况\"}, {\"role\": \"tool_call\", \"content\": \"{\\\"name\\\": \\\"realtime_aqi\\\", \\\"arguments\\\": {\\\"city\\\": \\\"北京\\\"}}\"}, {\"role\": \"tool_call\", \"content\": \"{\\\"name\\\": \\\"realtime_aqi\\\", \\\"arguments\\\": {\\\"city\\\": \\\"上海\\\"}}\"}, {\"role\": \"tool_response\", \"content\": \"{\\\"city\\\": \\\"北京\\\", \\\"aqi\\\": \\\"10\\\", \\\"unit\\\": \\\"celsius\\\"}\"}, {\"role\": \"tool_response\", \"content\": \"{\\\"city\\\": \\\"上海\\\", \\\"aqi\\\": \\\"72\\\", \\\"unit\\\": \\\"fahrenheit\\\"}\"}, {\"role\": \"assistant\", \"content\": \"根据天气预报工具，北京今天的空气质量指数为10，属于良好水平；上海今天的空气质量指数为72，属于轻度污染水平。\"}]}\n\n\nfrom swift import get_processor, get_template\n\ntokenizer = get_processor('Qwen/Qwen3.5-2B')\ntemplate = get_template(tokenizer) # 使用默认agent模板\n# template = get_template(tokenizer, agent_template='qwen3_5')\nprint(f'agent_template: {template._agent_template}')\ntemplate.set_mode('train')\nencoded = template.encode(data)\nprint(f'[INPUT_IDS] {template.safe_decode(encoded[\"input_ids\"])}\\n')\nprint(f'[LABELS] {template.safe_decode(encoded[\"labels\"])}')\n```\n\n如果你想要给我们提供PR，请参考[这里](https://github.com/modelscope/ms-swift/blob/main/tests/test_align/test_template/test_agent.py)书写你的测试案例。\n\n## Callbacks\n\ncallbacks的mapping文件可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/callbacks/mapping.py)。callbacks可以对trainer中的关键节点的行为进行自定义。自定义后，你需要在mapping中进行注册，训练时使用`--callbacks`指定对应的回调类。例如，你可以自定义：\n\n```python\nclass CustomCallback(TrainerCallback):\n\n def on_train_begin(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):\n # Doing something when the training begins.\n pass\n\n def on_save(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):\n # Doing something when save checkpoint\n pass\n```\n\n所有的回调类需继承自base.py中的`TrainerCallback`，并覆盖其方法。接口与transformers的`TrainerCallback`一致，请参考transformers的[callback文档](https://huggingface.co/docs/transformers/main_classes/callback)。\n\n\n## Loss\n\nLoss的mapping文件可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/loss/mapping.py)。\nswift支持自定义loss（当前只支持sft/pretrain/reranker/embedding任务），注册后在训练时设置`--loss_type `使用你定制的loss方法。\n\n自定义Loss需继承自`BaseLoss`，并实现`__call__`方法，返回标量Tensor。你可以参考[CustomCrossEntropyLoss](https://github.com/modelscope/ms-swift/blob/0d7c9f5bc0e7e7d67d914ce6edeb9ce24f60746f/swift/loss/causal_lm.py#L5)进行定制。例如：\n\n```python\nclass CustomLoss(BaseLoss):\n\n def __call__(self, outputs, labels, **kwargs) -> torch.Tensor:\n pass\n```\n\n## Loss Scale\n\nloss scale的mapping文件可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/loss_scale/mapping.py)。在pretrain和sft任务中，可训练token的loss是平均的，即每个token平等地对待。但在某些情况下，某些token需要被额外关注，并设置更高的权重或者对某些token不进行训练。loss_scale可以让开发者自由地定义自己的token权重。（预训练和SFT支持使用loss_scale控制token是否参与训练以及和其权重大小，RLHF中只支持控制token是否参与训练）\n\n你可以通过继承LossScale基类，并实现`get_loss_scale`方法来自定义loss scale。\n```python\nclass CustomLossScale(LossScale):\n\n def get_loss_scale(self, context: str, **kwargs) -> Tuple[List[str], List[float]]:\n ...\n```\n`get_loss_scale`函数需要返回了一个Tuple，第一个返回是拆解后的字符串的列表，第二个参数是字符串对应的loss_scale的列表，float值代表了权重。例如下面的权重设置：\n```text\n[\"学习\", \"好\", \"数学\", \"是\", \"重要\", \"的\"]\n[1.0, 0.5, 2.0, 0.5, 2.0, 0.1]\n```\n例子中，我们更看重数学和重要两个词，因为其loss_scale为2.0。\n\n\n当然我们也需要关注`__call__`方法的核心逻辑，即loss_scale基本策略（base_strategy）all/default/last_round 对loss_scale的影响，具体参考[命令行参数文档](../Instruction/Command-line-parameters.md)的介绍。以及数据集中的'loss'字段对loss_scale的影响，参考[自定义数据集文档](../Customization/Custom-dataset.md)。\n```python\nif loss or loss is None and (self.base_strategy == 'all' or\n (self.base_strategy == 'default' and is_assistant) or\n (self.base_strategy == 'last_round' and is_assistant and is_last_round)):\n new_context, loss_scale = self.get_loss_scale(context, query=query)\nelse:\n new_context, loss_scale = [context], [0.]\n```\n\n此外你也可以使用[json配置文件](https://github.com/modelscope/ms-swift/tree/main/swift/loss_scale/config)，继承内置的ConfigLossScale类，来自定义loss_scale。目前支持两种配置方式：字符串精确匹配和正则表达式匹配。你可以参考[Agent支持文档](../Instruction/Agent-support.md#loss_scale的使用)的内容进行理解。\n\n- 字符串精确匹配，例如参考`react.json`, `qwen.json`。json中需要书写`Dict[str, List[float]]`的映射。字符串代表关键词，列表中需要有两个值。我们会根据关键词，将字符串切分成多段字符串。列表的第一个值代表关键词的权重，列表的第二个值代表该关键值后，下一关键词前的内容的权重。\n\n- 正则表达式匹配，例如参考`ignore_empty_think.json`, `hermes.json`。json中需要书写`Dict[str, float]`的映射。字符串代表正则表达式pattern，浮点数代表匹配字符串的权重。\n\n\n如何debug：\n```python\nfrom swift import get_processor, get_template\n\ndata = {\"messages\": [\n {\"role\": \"user\", \"content\": \"今天的日期是多少？\"},\n {\"role\": \"assistant\", \"content\": (\n \"\\n我可以通过调用`get_date`函数来获取当前时间。\\n \\n\"\n '\\n{\"name\": \"get_date\", \"arguments\": {}}\\n '\n )}\n]}\n\ntemplate = get_template(get_processor('Qwen/Qwen3-8B'), loss_scale='hermes')\ntemplate.set_mode('train')\ninputs = template.encode(data)\n\nprint(template.safe_decode(inputs['labels']))\nprint(inputs['loss_scale'])\n```\n\n## Metrics\n\nmetrics的mapping文件可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/metrics/mapping.py)。该组件在ms-swift/Megatron-SWIFT中都有被使用。\n- 如果是在ms-swift中被使用，你需要继承 base.py 中`EvalMetrics`基类，并实现`compute_metrics`函数，返回字典`Dict[str, float]`。你可以参考[NlgMetrics](https://github.com/modelscope/ms-swift/blob/0d7c9f5bc0e7e7d67d914ce6edeb9ce24f60746f/swift/metrics/nlg.py#L33)进行定制。\n- 如果是在Megatron-SWIFT中被使用，你需要继承 utils.py 中`Metric`基类，并实现`update`和`compute`方法，compute方法需返回字典`Dict[str, float]`。\n\n你可以自定义metrics（当前只支持sft/pretrain/reranker/embedding任务），在训练时设置`--eval_metric `使用你定制的metrics。\n\n## Optimizers\n\noptimizer的mapping文件可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/optimizers/mapping.py)。如果你需要自定义优化器，你需要继承`OptimizerCallback`基类，并覆盖`create_optimizer`函数。训练时使用`--optimizer `指定自定义的优化器。\n- 你可以参考[MultimodalOptimizerCallback](https://github.com/modelscope/ms-swift/blob/0d7c9f5bc0e7e7d67d914ce6edeb9ce24f60746f/swift/optimizers/multimodal.py#L43)进行实现，该类实现了vit_lr, aligner_lr的功能，即对vit, aligner和LLM分别使用不同的学习率。\n\n\n\n## Tuner Plugin\n\nTuner插件的mapping文件可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/tuner_plugin/mapping.py)。如果你需要自定义tuner，你需要继承`Tuner`基类，并覆盖`prepare_model`, `save_pretrained`, `from_pretrained`函数。\n- prepare_model: 该函数在训练前被调用，将原始模型进行处理与准备，使用tuner封装，并设置可训练参数。例如：你可以对某些层附加LoRA，对某些层进行冻结等。\n- save_pretrained: 该函数在训练中被调用，对模型进行保存。\n- from_pretrained: 该函数在推理/断点续训时被调用，准备模型并读取权重。\n\n你可以参考[LoRALLMTuner](https://github.com/modelscope/ms-swift/blob/0d7c9f5bc0e7e7d67d914ce6edeb9ce24f60746f/swift/tuner_plugin/lora_llm.py#L24)进行实现，该类实现了对LLM进行LoRA训练，对ViT进行全参数训练的功能。\n\n\n## ORM\n\nexample参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/rewards/orm.py)。\n\nORM是结果奖励模型。ORM一般使用正则表达式来进行，ORM决定了response是否是正确的。例如：\n\n```python\nclass MathORM(ORM):\n\n @staticmethod\n def extract_boxed_result(text):\n pattern = r'\\\\boxed{([^}]*)}'\n match = re.search(pattern, text)\n if match:\n return match.group(1).strip()\n else:\n return None\n\n def __call__(self, infer_requests: List[InferRequest], ground_truths: List[str],\n **kwargs) -> List[float]:\n rewards = []\n predictions = [request.messages[-1]['content'] for request in infer_requests]\n for prediction, ground_truth in zip(predictions, ground_truths):\n res1 = MathORM.extract_boxed_result(prediction) or ''\n res2 = MathORM.extract_boxed_result(ground_truth) or ''\n rewards.append(float(res1.strip() == res2.strip()))\n\n return rewards\n\n\norms = {\n 'math': MathORM,\n}\n```\n\n在上面的代码中，我们定义了一个对数学response进行解析的过程，如果结果相同则返回score为1.0，否则为0.0。和PRM不同，这个类的infer中有一个额外参数`ground_truths`，\n该参数是对应的infer_requests的实际label（数据集中定义的标准response）。\n\n\n## PRM\n\nexample参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/rewards/prm.py)。\n\nPRM是过程奖励模型，PRM会在`swift sample`命令中使用。PRM需要支持的接口比较简单：\n```python\nclass PRM:\n\n def __init__(self):\n # init here\n pass\n\n def __call__(self, infer_requests: List[InferRequest], **kwargs) -> List[Union[float, List[float]]]:\n raise NotImplementedError\n```\n\n其中的InferRequest来自于`swift.infer_engine`，返回的`List[Union[float, List[float]]]`，列表中可能是reward也可能是若干reward。开发者可以在infer_requests中拿到queries和responses，并按照自己的方式进行切分，例如：\n```text\nLet's think step by step.\n\nStep1: xxx\n\nStep2: xxx\n\nSo, the answer is ...\n```\n开发者可以在这里对过程进行切分，并按batch传入PRM中进行推理并返回rewards。更通用来说，开发者可以在这里调用一个远端URL，例如一个闭源PRM大模型并返回rewards。\n\n\n## 其他目录结构介绍\n\n- arguments: 命令行参数定义，例如：`SftArguments`, `RLHFArguments`等。\n- cli: swift命令行机制以及启动文件。例如`swift sft ...`等价于`python swift/cli/main.py sft ...`也等价于`python swift/cli/sft.py ...`。\n- config: deepspeed/fsdp2配置文件。\n- dataloader: dataloader的实现，包括shard/dispatcher两种方式。\n- dataset: 数据集相关模块实现，包括数据预处理、packing、流式数据等。内置数据集的注册在`dataset/dataset`和`dataset/data`文件夹内。具体参考[自定义数据集文档](Custom-dataset.md)。\n- infer_engine: 推理引擎实现。包括transformers/vllm/sglang/lmdeploy为后端的推理引擎实现。\n- megatron: Megatron-SWIFT 实现。\n- model: 模型加载与注册。具体参考[自定义模型文档](Custom-model.md)，[多模态模型注册最佳实践](../BestPractices/MLLM-Registration.md)。\n- pipelines: `swift sft/rlhf/infer`等主函数pipeline实现，包括`sft_main/rlhf_main/infer_main`等。\n- rlhf_trainers: GRPO/GKD/DPO/KTO/RM等算法的Trainer实现。\n- rollout: RL算法中rollout过程的采样实现。\n- rewards: RL算法中的奖励函数实现，支持自定义奖励计算逻辑。\n- template: 对话模板的实现与注册，包含各个任务将messages转换成input_ids的逻辑，以及data_collator相关逻辑。具体参考[自定义模型文档](Custom-model.md)，[多模态模型注册最佳实践](../BestPractices/MLLM-Registration.md)。\n- trainers: 预训练/SFT/Embedding/Reranker/序列分类任务的Trainer实现。\n- ui: `swift web-ui`界面训练与推理实现。\n"
},
{
"path": "docs/source/Customization/Custom-dataset.md",
"content": "# 自定义数据集\n\n自定义数据集的接入方法有三种，对预处理函数的控制能力逐渐加强，但接入难度逐步增加。例如，方案一最为方便，但对预处理函数的控制能力最弱，需要预先对数据集进行转换，传入特定格式的数据集：\n1. 【推荐】直接使用命令行传参的方式接入，即`--dataset `。这将使用AutoPreprocessor将数据集转换为标准格式（支持4种数据集格式，具体查看下面对AutoPreprocessor的介绍）。你可以使用`--columns`进行列名转换。支持传入csv、json、jsonl、txt、文件夹（例如git clone开源数据集）。该方案不需要修改dataset_info.json，适合刚接触ms-swift的用户，下面两种方案适合对ms-swift进行拓展的开发者。\n2. 添加数据集到`dataset_info.json`中，可以参考ms-swift内置的[dataset_info.json](https://github.com/modelscope/ms-swift/blob/main/swift/dataset/data/dataset_info.json)。该方案也将使用AutoPreprocessor将数据集转换为标准格式。dataset_info.json为数据集元信息的list，每一项元信息必填ms_dataset_id/hf_dataset_id/dataset_path中的一项，通过`columns`字段进行列名转换。添加到`dataset_info.json`或者注册的数据集在运行[run_dataset_info.py](https://github.com/modelscope/ms-swift/blob/main/scripts/utils/run_dataset_info.py)时将自动产生[支持的数据集文档](https://swift.readthedocs.io/zh-cn/latest/Instruction/Supported-models-and-datasets.html)。此外，你可以采用外接`dataset_info.json`的方式，使用`--custom_dataset_info xxx.json`解析json文件（方便pip install而非git clone的用户），然后指定`--dataset `。\n3. 手动注册数据集，具有最灵活的预处理函数定制能力，支持使用函数对数据集进行预处理，但难度较高。可以参考[内置数据集](https://github.com/modelscope/ms-swift/blob/main/swift/dataset/dataset/llm.py)或者[examples](https://github.com/modelscope/ms-swift/blob/main/examples/custom)中的样例。你可以通过指定`--external_plugins xxx.py`解析外置注册内容（方便pip install而非git clone的用户）。\n - 方案一和二在实现中借助了方案三，只是注册的过程为自动发生。\n\n以下将对`AutoPreprocessor`可以处理的数据集格式进行介绍：\n\nms-swift的标准数据集格式可接受的keys包括: 'messages'、'rejected_response'、'label'、'images'、'videos'、'audios'、'tools'和'objects'。其中'messages'是必需的key，'rejected_response'用于DPO等RLHF训练，'label'用于KTO训练和分类模型训练，'images'、'videos'、'audios'用于存储多模态数据的路径或者url，'tools'用于Agent任务，'objects'用于grounding任务。\n\nms-swift中存在三种核心预处理器：`MessagesPreprocessor`、`AlpacaPreprocessor`、`ResponsePreprocessor`。MessagesPreprocessor用于将类messages和sharegpt格式的数据集转换为标准格式，AlpacaPreprocessor则转换alpaca格式的数据集，ResponsePreprocessor则转换类query/response格式的数据集。`AutoPreprocessor`则自动选择合适的预处理进行处理。\n\n以下四种格式在`AutoPreprocessor`处理下都会转换成ms-swift标准格式中的messages字段，即都可以直接使用`--dataset `接入：\n\nmessages格式（标准格式）:\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"\"}, {\"role\": \"user\", \"content\": \"\"}, {\"role\": \"assistant\", \"content\": \"\"}, {\"role\": \"user\", \"content\": \"\"}, {\"role\": \"assistant\", \"content\": \"\"}]}\n```\n- 注意：system部分是可选的。数据集中的system优先级高于命令行传入的`--system`，最后是定义在template中的`default_system`。\n\nsharegpt格式:\n```jsonl\n{\"system\": \"\", \"conversation\": [{\"human\": \"\", \"assistant\": \"\"}, {\"human\": \"\", \"assistant\": \"\"}]}\n```\n\nquery-response格式:\n```jsonl\n{\"system\": \"\", \"query\": \"\", \"response\": \"\", \"history\": [[\"\", \"\"]]}\n```\n注意：以下字段会自动转成对应的system、query、response字段。（solution字段会保留）\n- system: 'system', 'system_prompt'.\n- query: 'query', 'prompt', 'input', 'instruction', 'question', 'problem'.\n- response: 'response', 'answer', 'output', 'targets', 'target', 'answer_key', 'answers', 'solution', 'text', 'completion', 'content'.\n\nalpaca格式:\n```jsonl\n{\"system\": \"\", \"instruction\": \"\", \"input\": \"\", \"output\": \"\"}\n```\n- 注意：instruction和input字段将组合成query字段。若instruction和input不等于空字符串，`query = f'{instruction}\\n{input}'`\n\n\n## 标准数据集格式\n\n以下给出ms-swift的标准数据集格式，其中system字段是可选的，默认使用template中定义的`default_system`。之前介绍的4种数据集格式也可以被AutoPreprocessor处理成标准数据集格式。\n\n### 预训练\n\n```jsonl\n{\"messages\": [{\"role\": \"assistant\", \"content\": \"I love music\"}]}\n{\"messages\": [{\"role\": \"assistant\", \"content\": \"教练我要打篮球\"}]}\n{\"messages\": [{\"role\": \"assistant\", \"content\": \"西红柿鸡蛋盖饭和地三鲜盖饭哪个更权威\"}]}\n```\n\n### 监督微调\n\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"告诉我明天的天气\"}, {\"role\": \"assistant\", \"content\": \"明天天气晴朗\"}]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的数学计算器\"}, {\"role\": \"user\", \"content\": \"1+1等于几\"}, {\"role\": \"assistant\", \"content\": \"等于2\"}, {\"role\": \"user\", \"content\": \"再加1呢\"}, {\"role\": \"assistant\", \"content\": \"等于3\"}]}\n```\n\n- 可以通过增加\"loss\"字段，控制对应的模型回复部分是否计算损失。默认该字段为None。若\"loss\"设置为true，则对应content进行损失计算（对应loss_scale为1）；若\"loss\"设置为false，则对应content不进行损失计算。需要注意的是，该功能只对\"role\"为\"assistant\"的部分生效；该功能优先级高于命令行参数 `--loss_scale`。示例数据格式如下：\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"你好\"}, {\"role\": \"assistant\", \"content\": \"你好，有什么可以帮助你的吗？\", \"loss\": false}, {\"role\": \"user\", \"content\": \"1+1等于几？\"}, {\"role\": \"assistant\", \"content\": \"等于2\", \"loss\": true}]}\n```\n\n\n#### channel loss\n如果你要使用channel loss，你需要设置`--enable_channel_loss true`，并在数据集中增加\"channel\"字段。channel loss兼容packing/padding_free/loss_scale等技术。\n\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"告诉我明天的天气\"}, {\"role\": \"assistant\", \"content\": \"明天天气晴朗\"}], \"channel\": \"general\"}\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的数学计算器\"}, {\"role\": \"user\", \"content\": \"1+1等于几\"}, {\"role\": \"assistant\", \"content\": \"等于2\"}, {\"role\": \"user\", \"content\": \"再加1呢\"}, {\"role\": \"assistant\", \"content\": \"等于3\"}], \"channel\": \"math\"}\n```\n\n### RLHF\n\n#### DPO/ORPO/CPO/SimPO/RM\n\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"告诉我明天的天气\"}, {\"role\": \"assistant\", \"content\": \"明天天气晴朗\"}], \"rejected_response\": \"我不知道\"}\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的数学计算器\"}, {\"role\": \"user\", \"content\": \"1+1等于几\"}, {\"role\": \"assistant\", \"content\": \"等于2\"}, {\"role\": \"user\", \"content\": \"再加1呢\"}, {\"role\": \"assistant\", \"content\": \"等于3\"}], \"rejected_response\": \"我不知道\"}\n```\n\n多模态数据的格式参考[多模态数据集](#多模态), 额外加入如`images`的列表示其他模态输入。当需要为偏好对数据关联不同的图片信息时，可通过`rejected_images`字段标注拒绝回答对应的图片信息。\n对齐数据集中要求`rejected_images`和`rejected_response`至少提供一个。\n\n> 注: RM 额外支持 margin 列，参考[RM文档](../Instruction/RLHF.md#rm)\n\n当然，你也可以直接使用`rejected_messages`，而不是只提供`rejected_response`/`rejected_images`，这将提供更大的灵活度（例如多模态/agent场景）。若使用rejected_messages，在多模态场景下，你需要额外传入\"rejected_images\"，\"rejected_audios\"，\"rejected_videos\"等内容；在Agent场景下，你需要额外传入\"rejected_tools\"等内容。多模态数据格式例子如下：\n- 若使用`rejected_response`，'rejected_images/rejected_audios/rejected_videos/rejected_tools'的默认值为'images/audios/videos/tools'；若使用`rejected_messages`，则需要额外传入。\n\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"这是什么\"}, {\"role\": \"assistant\", \"content\": \"这是一只小猫咪。\"}], \"images\": [\"cat.png\"], \"rejected_messages\": [{\"role\": \"user\", \"content\": \"这是什么\"}, {\"role\": \"assistant\", \"content\": \"这是一只小狗。\"}], \"rejected_images\": [\"cat.png\"]}\n{\"messages\": [{\"role\": \"user\", \"content\": \"这是什么\"}, {\"role\": \"assistant\", \"content\": \"这是一只小猫咪。\"}], \"images\": [\"cat.png\"], \"rejected_messages\": [{\"role\": \"user\", \"content\": \"这是什么\"}, {\"role\": \"assistant\", \"content\": \"这是一只小猫咪。\"}], \"rejected_images\": [\"dog.png\"]}\n```\n\n以上格式等价于：\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"这是什么\"}, {\"role\": \"assistant\", \"content\": \"这是一只小猫咪。\"}], \"images\": [\"cat.png\"], \"rejected_response\": \"这是一只小狗。\"}\n{\"messages\": [{\"role\": \"user\", \"content\": \"这是什么\"}, {\"role\": \"assistant\", \"content\": \"这是一只小猫咪。\"}], \"images\": [\"cat.png\"], \"rejected_images\": [\"dog.png\"]}\n# 例子一也可写成：\n{\"messages\": [{\"role\": \"user\", \"content\": \"这是什么\"}, {\"role\": \"assistant\", \"content\": \"这是一只小猫咪。\"}], \"images\": [\"cat.png\"], \"rejected_response\": [{\"role\": \"assistant\", \"content\": \"这是一只小狗。\"}]}\n```\n\n你也可以将Agent数据集组织成以下形式：\n```jsonl\n# 会寻找`messages`最后一个user的位置，并替换之后的内容为`rejected_response`组成`rejected_messages`\n{\"tools\": \"[{\\\"type\\\": \\\"function\\\", \\\"function\\\": {\\\"name\\\": \\\"realtime_aqi\\\", \\\"description\\\": \\\"天气预报。获取实时空气质量。当前空气质量，PM2.5，PM10信息\\\", \\\"parameters\\\": {\\\"type\\\": \\\"object\\\", \\\"properties\\\": {\\\"city\\\": {\\\"type\\\": \\\"string\\\", \\\"description\\\": \\\"城市名，例如：上海\\\"}}, \\\"required\\\": [\\\"city\\\"]}}}]\", \"messages\": [{\"role\": \"user\", \"content\": \"北京和上海今天的天气情况\"}, {\"role\": \"tool_call\", \"content\": \"{\\\"name\\\": \\\"realtime_aqi\\\", \\\"arguments\\\": {\\\"city\\\": \\\"北京\\\"}}\"}, {\"role\": \"tool_call\", \"content\": \"{\\\"name\\\": \\\"realtime_aqi\\\", \\\"arguments\\\": {\\\"city\\\": \\\"上海\\\"}}\"}, {\"role\": \"tool_response\", \"content\": \"{\\\"city\\\": \\\"北京\\\", \\\"aqi\\\": \\\"10\\\", \\\"unit\\\": \\\"celsius\\\"}\"}, {\"role\": \"tool_response\", \"content\": \"{\\\"city\\\": \\\"上海\\\", \\\"aqi\\\": \\\"72\\\", \\\"unit\\\": \\\"fahrenheit\\\"}\"}, {\"role\": \"assistant\", \"content\": \"根据天气预报工具，北京今天的空气质量指数为10，属于良好水平；上海今天的空气质量指数为72，属于轻度污染水平。\"}], \"rejected_response\": [{\"role\": \"assistant\", \"content\": \"我不知道。\"}]}\n```\n\n#### KTO\n\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"告诉我明天的天气\"}, {\"role\": \"assistant\", \"content\": \"我不知道\"}], \"label\": false}\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的数学计算器\"}, {\"role\": \"user\", \"content\": \"1+1等于几\"}, {\"role\": \"assistant\", \"content\": \"等于2\"}, {\"role\": \"user\", \"content\": \"再加1呢\"}, {\"role\": \"assistant\", \"content\": \"等于3\"}], \"label\": true}\n```\n\n#### PPO/GRPO\n\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"告诉我明天的天气\"}]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的数学计算器\"}, {\"role\": \"user\", \"content\": \"1+1等于几\"}, {\"role\": \"assistant\", \"content\": \"等于2\"}, {\"role\": \"user\", \"content\": \"再加1呢\"}]}\n{\"messages\": [{\"role\": \"user\", \"content\": \"你的名字是什么\"}]}\n```\n- 注意：GRPO会透传所有额外的字段内容给ORM，而不像其他训练方法，默认将额外的字段删除。例如: 你可以额外传入'solution'。自定义的ORM需要包含一个位置参数completions，其他为关键词参数，由数据集额外字段透传。\n\n#### GKD\n若未开启`seq_kd`，即该参数为False。数据集格式如下（你可使用teacher模型提前蒸馏）：\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"告诉我明天的天气\"}, {\"role\": \"assistant\", \"content\": \"明天天气晴朗\"}]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的数学计算器\"}, {\"role\": \"user\", \"content\": \"1+1等于几\"}, {\"role\": \"assistant\", \"content\": \"等于2\"}, {\"role\": \"user\", \"content\": \"再加1呢\"}, {\"role\": \"assistant\", \"content\": \"等于3\"}]}\n```\n\n若开启`seq_kd`，则不需要最后一轮的'assistant'部分（teacher模型在训练时生成数据）：\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"告诉我明天的天气\"}]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的数学计算器\"}, {\"role\": \"user\", \"content\": \"1+1等于几\"}, {\"role\": \"assistant\", \"content\": \"等于2\"}, {\"role\": \"user\", \"content\": \"再加1呢\"}]}\n```\n\n### 序列分类\n\n**单标签任务**：\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"今天天气真好呀\"}], \"label\": 1}\n{\"messages\": [{\"role\": \"user\", \"content\": \"今天真倒霉\"}], \"label\": 0}\n{\"messages\": [{\"role\": \"user\", \"content\": \"好开心\"}], \"label\": 1}\n```\n**多标签任务**：\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"\"}], \"label\": []}\n{\"messages\": [{\"role\": \"user\", \"content\": \"\"}], \"label\": [0, 2]}\n{\"messages\": [{\"role\": \"user\", \"content\": \"\"}], \"label\": [1, 3, 5]}\n```\n\n**单回归任务**：\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"求两句话的相似度，范围为0-1。\\nsentence1: \\nsentence2: \"}], \"label\": 0.8}\n```\n**多回归任务**：\n\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"\"}], \"label\": [1.2, -0.6, 0.8]}\n```\n\n### Embedding\n\n请参考[embedding训练文档](../BestPractices/Embedding.md#数据集格式)\n\n### Reranker\n\n请参考[Reranker训练文档](../BestPractices/Reranker.md#数据集格式)\n\n### 多模态\n\n对于多模态数据集，和上述任务的格式相同。区别在于增加了`images`, `videos`, `audios`几个key，分别代表多模态资源的url或者path（推荐使用绝对路径），`` `` ``标签代表了插入图片/视频/音频的位置，ms-swift支持多图片/视频/音频的情况。这些特殊tokens将在预处理的时候进行替换，参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/template/templates/qwen.py#L198)。下面给出的四条示例分别展示了纯文本，以及包含图像、视频和音频数据的数据格式。\n\n预训练：\n```\n{\"messages\": [{\"role\": \"assistant\", \"content\": \"预训练的文本在这里\"}]}\n{\"messages\": [{\"role\": \"assistant\", \"content\": \"是一只小狗，是一只小猫\"}], \"images\": [\"/xxx/x.jpg\", \"/xxx/x.png\"]}\n{\"messages\": [{\"role\": \"assistant\", \"content\": \"描述了今天天气真不错\"}], \"audios\": [\"/xxx/x.wav\"]}\n{\"messages\": [{\"role\": \"assistant\", \"content\": \"是一个大象，是一只狮子在跑步\"}], \"images\": [\"/xxx/x.jpg\"], \"videos\": [\"/xxx/x.mp4\"]}\n```\n\n微调：\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"浙江的省会在哪？\"}, {\"role\": \"assistant\", \"content\": \"浙江的省会在杭州。\"}]}\n{\"messages\": [{\"role\": \"user\", \"content\": \"两张图片有什么区别\"}, {\"role\": \"assistant\", \"content\": \"前一张是小猫，后一张是小狗\"}], \"images\": [\"/xxx/x.jpg\", \"/xxx/x.png\"]}\n{\"messages\": [{\"role\": \"user\", \"content\": \"语音说了什么\"}, {\"role\": \"assistant\", \"content\": \"今天天气真好呀\"}], \"audios\": [\"/xxx/x.mp3\"]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"图片中是什么，视频中是什么\"}, {\"role\": \"assistant\", \"content\": \"图片中是一个大象，视频中是一只小狗在草地上奔跑\"}], \"images\": [\"/xxx/x.jpg\"], \"videos\": [\"/xxx/x.mp4\"]}\n```\n- 注意：以下字段会自动转成对应的images, videos, audios字段。\n - images: image, images.\n - videos: video, videos.\n - audios: audio, audios.\n- 如果需要传入base64格式而不是文件路径，以下为样本例子：`\"videos\": ['data:video/mp4;base64,{base64_encoded}']`, `\"images\": ['data:image/jpg;base64,{base64_encoded}']`。\n- 若你希望直接传入视频帧，而不是视频，你可以使用以下格式：`\"videos\": [[\"/xxx/x.png\", \"/xxx/y.png\"], [\"/xxx/a.png\", \"/xxx/b.png\", \"/xxx/c.png\"]]`。该格式只有部分模型支持，包括Qwen2/2.5/3-VL、Qwen2.5/3-Omni以及其衍生模型。\n\n多模态模型的RLHF和序列分类的数据格式可以参考纯文本大模型的格式，并在此基础上增加`images`等字段。\n\n#### grounding\n\n如果是grounding（物体检测）任务，ms-swift支持两种方式：\n1. 直接使用对应模型grounding任务的数据集格式，例如qwen2-vl的格式如下：\n\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"描述图像\"}, {\"role\": \"assistant\", \"content\": \"<|object_ref_start|>一只狗<|object_ref_end|><|box_start|>(221,423),(569,886)<|box_end|>和<|object_ref_start|>一个女人<|object_ref_end|><|box_start|>(451,381),(733,793)<|box_end|>正在沙滩上玩耍\"}], \"images\": [\"/xxx/x.jpg\"]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"找到图像中的<|object_ref_start|>羊<|object_ref_end|>\"}, {\"role\": \"assistant\", \"content\": \"<|box_start|>(101,201),(150,266)<|box_end|><|box_start|>(401,601),(550,666)<|box_end|>\"}], \"images\": [\"/xxx/x.jpg\"]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"帮我打开谷歌浏览器\"}, {\"role\": \"assistant\", \"content\": \"Action: click(start_box='<|box_start|>(246,113)<|box_end|>')\"}], \"images\": [\"/xxx/x.jpg\"]}\n```\n使用这种类型的数据需要注意：\n- 不同模型grounding任务的特殊字符和数据集格式不同。\n- 不同模型对bbox是否归一化的处理不同。例如：qwen2.5-vl使用绝对坐标，而qwen2/3-vl、internvl2.5需要对bbox的坐标进行千分位坐标归一化。\n - 注意：Qwen2.5-VL采用绝对坐标，因此要小心每次的图像缩放，如果使用方案一的数据集格式，你需要预先对图像进行resize（H和W需要是28的系数），并根据该尺寸缩放坐标点。如果使用方案二的数据集格式，ms-swift会帮助你处理图像的缩放问题，你依旧可以使用`MAX_PIXELS`或者`--max_pixels`等进行图像缩放（仅训练，推理场景，你依旧需要自己处理图像的缩放问题）。\n\n2. 使用ms-swift的grounding数据格式：\n\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"描述图像\"}, {\"role\": \"assistant\", \"content\": \"和正在沙滩上玩耍\"}], \"images\": [\"/xxx/x.jpg\"], \"objects\": {\"ref\": [\"一只狗\", \"一个女人\"], \"bbox\": [[331.5, 761.4, 853.5, 1594.8], [676.5, 685.8, 1099.5, 1427.4]]}}\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"找到图像中的\"}, {\"role\": \"assistant\", \"content\": \"\"}], \"images\": [\"/xxx/x.jpg\"], \"objects\": {\"ref\": [\"羊\"], \"bbox\": [[90.9, 160.8, 135, 212.8], [360.9, 480.8, 495, 532.8]]}}\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"帮我打开谷歌浏览器\"}, {\"role\": \"assistant\", \"content\": \"Action: click(start_box='')\"}], \"images\": [\"/xxx/x.jpg\"], \"objects\": {\"ref\": [], \"bbox\": [[615, 226]]}}\n```\n该格式将自动转换数据集格式为对应模型的grounding任务格式，且选择对应模型的bbox归一化方式。该格式比通用格式多了objects字段，该字段包含的字段有：\n- ref: 用于替换messages中的``。ref的长度需要与``的数量一致。\n- bbox: 用于替换messages中的``。若bbox中每个box长度为2，则代表x和y坐标，若box长度为4，则代表2个点的x和y坐标。bbox的长度需要与``的数量一致。\n - 注意：``和``并没有对应关系，ref和bbox各自替换各自的占位符。\n- bbox_type: 可选项为'real'，'norm1'。默认为'real'，即bbox为真实bbox值。若是'norm1'，则bbox已经归一化为0~1。\n- image_id: 通常用于多图grounding任务。该参数只有当bbox_type为'real'时生效，代表bbox对应的图片是第几张，用于缩放bbox。索引从0开始，默认全为第0张。image_id的数量需要和bbox的数量一致。例如：若bbox的长度为10，images的长度为2，那么image_id的长度需要是10，其值需要在`{0, 1}`集合内。\n\n对于Qwen2.5-VL/Qwen3-VL，你可以使用环境`QWENVL_BBOX_FORMAT='new'`（默认为'legacy'），以兼容[官方cookbook](https://github.com/QwenLM/Qwen3-VL/blob/main/cookbooks/2d_grounding.ipynb)格式。并将数据集定义成以下格式：\n```jsonl\n{\"messages\": [{\"role\": \"user\", \"content\": \"找到图像中的\"}, {\"role\": \"assistant\", \"content\": \"[\\n\\t{\\\"bbox_2d\\\": , \\\"label\\\": \\\"\\\"},\\n\\t{\\\"bbox_2d\\\": , \\\"label\\\": \\\"\\\"}\\n]\"}], \"images\": [\"cat.png\"], \"objects\": {\"ref\": [\"羊\", \"羊\", \"羊\"], \"bbox\": [[90.9, 160.8, 135, 212.8], [360.9, 480.8, 495, 532.8]]}}\n```\n\n测试ms-swift格式的grounding数据格式的最终格式：\n```python\nimport os\nos.environ[\"MAX_PIXELS\"] = \"1003520\"\nfrom swift import get_processor, get_template\n\nprocessor = get_processor('Qwen/Qwen2.5-VL-7B-Instruct')\ntemplate = get_template(processor)\ndata = {...}\ntemplate.set_mode('train')\nencoded = template.encode(data, return_template_inputs=True)\nprint(f'[INPUT_IDS] {template.safe_decode(encoded[\"input_ids\"])}\\n')\nprint(f'[LABELS] {template.safe_decode(encoded[\"labels\"])}')\nprint(f'images: {encoded[\"template_inputs\"].images}')\n```\n\n### Agent格式\n这里分别提供了纯文本Agent和多模态Agent的示例数据样本：\n```jsonl\n{\"tools\": \"[{\\\"type\\\": \\\"function\\\", \\\"function\\\": {\\\"name\\\": \\\"realtime_aqi\\\", \\\"description\\\": \\\"天气预报。获取实时空气质量。当前空气质量，PM2.5，PM10信息\\\", \\\"parameters\\\": {\\\"type\\\": \\\"object\\\", \\\"properties\\\": {\\\"city\\\": {\\\"type\\\": \\\"string\\\", \\\"description\\\": \\\"城市名，例如：上海\\\"}}, \\\"required\\\": [\\\"city\\\"]}}}]\", \"messages\": [{\"role\": \"user\", \"content\": \"北京和上海今天的天气情况\"}, {\"role\": \"tool_call\", \"content\": \"{\\\"name\\\": \\\"realtime_aqi\\\", \\\"arguments\\\": {\\\"city\\\": \\\"北京\\\"}}\"}, {\"role\": \"tool_call\", \"content\": \"{\\\"name\\\": \\\"realtime_aqi\\\", \\\"arguments\\\": {\\\"city\\\": \\\"上海\\\"}}\"}, {\"role\": \"tool_response\", \"content\": \"{\\\"city\\\": \\\"北京\\\", \\\"aqi\\\": \\\"10\\\", \\\"unit\\\": \\\"celsius\\\"}\"}, {\"role\": \"tool_response\", \"content\": \"{\\\"city\\\": \\\"上海\\\", \\\"aqi\\\": \\\"72\\\", \\\"unit\\\": \\\"fahrenheit\\\"}\"}, {\"role\": \"assistant\", \"content\": \"根据天气预报工具，北京今天的空气质量指数为10，属于良好水平；上海今天的空气质量指数为72，属于轻度污染水平。\"}]}\n{\"tools\": \"[{\\\"type\\\": \\\"function\\\", \\\"function\\\": {\\\"name\\\": \\\"click\\\", \\\"description\\\": \\\"点击屏幕中的某个位置\\\", \\\"parameters\\\": {\\\"type\\\": \\\"object\\\", \\\"properties\\\": {\\\"x\\\": {\\\"type\\\": \\\"integer\\\", \\\"description\\\": \\\"横坐标，表示屏幕上的水平位置\\\"}, \\\"y\\\": {\\\"type\\\": \\\"integer\\\", \\\"description\\\": \\\"纵坐标，表示屏幕上的垂直位置\\\"}}, \\\"required\\\": [\\\"x\\\", \\\"y\\\"]}}}]\", \"messages\": [{\"role\": \"user\", \"content\": \"现在几点了？\"}, {\"role\": \"assistant\", \"content\": \"\\n我可以通过打开日历App来获取当前时间。\\n \\n\"}, {\"role\": \"tool_call\", \"content\": \"{\\\"name\\\": \\\"click\\\", \\\"arguments\\\": {\\\"x\\\": 105, \\\"y\\\": 132}}\"}, {\"role\": \"tool_response\", \"content\": \"{\\\"images\\\": \\\"\\\", \\\"status\\\": \\\"success\\\"}\"}, {\"role\": \"assistant\", \"content\": \"成功打开日历App，现在的时间为中午11点\"}], \"images\": [\"desktop.png\", \"calendar.png\"]}\n```\n- agent_template为\"react_en\", \"hermes\"等情况下，该格式适配所有模型Agent训练，可以轻松在不同模型间切换。\n- 其中tools是一个包含tool列表的json字符串，messages中role为'tool_call'和'tool_response/tool'的content部分都需要是json字符串。\n- tools字段将在训练/推理时和`{\"role\": \"system\", ...}\"`部分组合，根据agent_template组成完整的system部分。\n- `{\"role\": \"tool_call\", ...}`部分将根据agent_template自动转成对应格式的`{\"role\": \"assistant\", ...}`，多条连续的`{\"role\": \"assistant\", ...}`将拼接在一起组成完整的assistant_content。\n- `{\"role\": \"tool_response\", ...}`也可以写成`{\"role\": \"tool\", ...}`，这两种写法是等价的。该部分也将根据`agent_template`自动转换格式。该部分在训练时将不进行损失的计算，角色类似于`{\"role\": \"user\", ...}`。\n- 该格式支持并行调用工具，例子参考第一条数据样本。多模态Agent数据样本中``标签数量应与\"images\"长度相同，其标签位置代表图像特征的插入位置。当然也支持其他模态，例如audios, videos。\n- 注意：您也可以手动将数据处理为role为system/user/assistant的messages格式。agent_template的作用是将其中的tools字段以及role为tool_call和tool_response的messages部分，自动映射为标准的role为system/user/assistant的messages格式。\n- 更多请参考[Agent文档](../Instruction/Agent-support.md)。\n\n### 文生图格式\n\n```jsonl\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"给我画出一个苹果\"}, {\"role\": \"assistant\", \"content\": \"\"}], \"images\": [\"/xxx/x.jpg\"]}\n```\n\n## dataset_info.json\n\n可以参考ms-swift内置的[dataset_info.json](https://github.com/modelscope/ms-swift/blob/main/swift/dataset/data/dataset_info.json)。该方案使用AutoPreprocessor预处理函数将数据集转换为标准格式。dataset_info.json文件中包含了数据集元信息的list，以下为一些例子：\n\n```json\n[\n {\n \"ms_dataset_id\": \"xxx/xxx\"\n },\n {\n \"dataset_path\": \"\"\n },\n {\n \"ms_dataset_id\": \"\",\n \"subsets\": [\"v1\"],\n \"split\": [\"train\", \"validation\"],\n \"columns\": {\n \"input\": \"query\",\n \"output\": \"response\"\n }\n },\n {\n \"ms_dataset_id\": \"\",\n \"hf_dataset_id\": \"\",\n \"subsets\": [{\n \"subset\": \"subset1\",\n \"columns\": {\n \"problem\": \"query\",\n \"content\": \"response\"\n }\n },\n {\n \"subset\": \"subset2\",\n \"columns\": {\n \"messages\": \"_\",\n \"new_messages\": \"messages\"\n }\n }]\n }\n]\n```\n\n支持以下参数：\n- ms_dataset_id: 参考DatasetMeta参数。\n- hf_dataset_id: 参考DatasetMeta参数。\n- dataset_path: 参考DatasetMeta参数。\n- dataset_name: 参考DatasetMeta参数。\n- subsets: 参考DatasetMeta参数。\n- split: 参考DatasetMeta参数。\n- columns: 在数据集进行预处理前，对数据集进行列名转换。\n\n\n## 数据集注册\n\nregister_dataset会在`DATASET_MAPPING`中注册数据集，调用函数`register_dataset(dataset_meta)`即可完成数据集注册，其中dataset_meta将存储模型的元信息。DatasetMeta的参数列表如下：\n- ms_dataset_id: ModelScope的dataset_id，默认为None。\n- hf_dataset_id: HuggingFace的dataset_id，默认为None。\n- dataset_path: 数据集**文件/文件夹**的本地路径（推荐使用绝对路径）。默认为None。\n- dataset_name: 数据集别名，可以通过`--dataset `指定数据集，这在dataset_path很长时很方便。默认为None。\n- subsets: 子数据集的名字列表或者`SubsetDataset`对象的列表，默认为`['default']`。（只有dataset_id或者dataset_dir（git clone开源数据集）有子数据集和split的概念）。\n- split: 默认为`['train']`。\n- preprocess_func: 预处理函数或可调用对象，默认为`AutoPreprocessor()`。该预处理函数接口为传入`HfDataset`，并返回满足标准格式的`HfDataset`。\n- load_function: 默认为`DatasetLoader.load`。若需要自定义载入函数，则该载入函数需返回满足标准格式的`HfDataset`，这将抛弃ms-swift的数据集载入机制，提供给用户最大的自由度。通常该参数不需要进行修改。\n\n\n以下介绍注册数据集的例子：\n\n```python\nfrom swift.dataset import (\n ResponsePreprocessor, DatasetMeta, register_dataset, SubsetDataset, load_dataset\n)\nfrom typing import Dict, Any\n\nclass CustomPreprocessor(ResponsePreprocessor):\n def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:\n query = f\"\"\"任务：判断下面两句话语意是否相似。\n句子1: {row['text1']}\n句子2: {row['text2']}\n请输出类别[0/1]: 0代表含义不同, 1代表含义相似。\n\"\"\"\n response = str(row['label'])\n row = {\n 'query': query,\n 'response': response\n }\n return super().preprocess(row)\n\n\nregister_dataset(\n DatasetMeta(\n ms_dataset_id='swift/financial_classification',\n subsets=[SubsetDataset('train', split=['train']), SubsetDataset('test', split=['test'])],\n preprocess_func=CustomPreprocessor(),\n ))\n\nif __name__ == '__main__':\n # load_dataset returns train_dataset and val_dataset based on `split_dataset_ratio`\n # Here, since we didn't pass `split_dataset_ratio` (defaults to 0), we take the first one (index 0)\n dataset = load_dataset('swift/financial_classification:train')[0]\n test_dataset = load_dataset('swift/financial_classification:test')[0]\n print(f'dataset[0]: {dataset[0]}')\n print(f'test_dataset[0]: {test_dataset[0]}')\n```\n"
},
{
"path": "docs/source/Customization/Custom-model.md",
"content": "# 自定义模型\n\nms-swift内置的模型，你可以直接通过指定model_id或者model_path来使用：`--model `。ms-swift会根据model_id/model_path的后缀和`config.json`文件来判断model_type。\n\n每种model_type都有唯一的模型结构、template和加载方式。当然，你也可以手动传入`--model_type`、`--template`来进行覆盖。ms-swift已支持的model_type和template可以查看[支持的模型与数据集](../Instruction/Supported-models-and-datasets.md)。\n\n以下介绍如何注册一个新模型和对应的template。最佳实践参考[注册多模态模型最佳实践](../BestPractices/MLLM-Registration.md)。\n\n## 模型注册\n\n自定义模型通常使用模型注册的方式进行，可以参考[内置模型](https://github.com/modelscope/ms-swift/blob/main/swift/model/models/qwen.py)、[内置对话模板](https://github.com/modelscope/ms-swift/blob/main/swift/template/templates/qwen.py)或者[examples](https://github.com/modelscope/ms-swift/blob/main/examples/custom)的示例代码。你可以通过指定`--external_plugins xxx.py`解析外置注册的内容（方便pip install而非git clone的用户）。\n\nregister_model会在`MODEL_MAPPING`中注册模型，调用函数`register_model(model_meta)`即可完成模型注册，其中model_meta将存储模型的元信息。ModelMeta的参数列表如下：\n- model_type: 必填项。模型类型，也是唯一ID。\n- model_groups: 必填项。罗列ModelScope/HuggingFace的模型id和模型本地路径。运行[run_model_info.py](https://github.com/modelscope/ms-swift/blob/main/scripts/utils/run_model_info.py)文件将自动产生[支持的模型文档](https://swift.readthedocs.io/zh-cn/latest/Instruction/Supported-models-and-datasets.html)以及自动根据`--model`后缀匹配model_type。\n- loader: 模型和tokenizer/processor（多模态模型）的加载器。默认使用`swift.model.ModelLoader`。\n- template: 命令行不额外指定`--template`时的默认template类型。默认为None。\n- model_arch: 模型架构。默认为None。多模态模型训练需要设置该参数来确定llm/vit/aligner的前缀。\n- architectures: config.json中的architectures项，用于自动匹配模型对应的model_type。默认为`[]`。\n- additional_saved_files: 全参数训练和merge-lora时需要额外保存的文件。默认为`[]`。\n- torch_dtype: 模型加载时未传入`torch_dtype`时的默认dtype。默认为None，从config.json中读取。\n- is_multimodal: 是否是多模态模型，默认为False。\n- ignore_patterns: 从hub端下载文件需要忽略的文件patterns，默认为`[]`。\n\n\nregister_template会在`TEMPLATE_MAPPING`中注册对话模板，调用函数`register_template(template_meta)`即可完成对话模板注册，其中template_meta将存储template的元信息。TemplateMeta的参数列表如下：\n- template_type: 必填项。对话模板类型，也是唯一ID。\n- prefix: 必填项。对话模板的前缀，通常包含system、bos_token等部分，独立于多轮对话而产生的对话模板循环。例如qwen的prefix为`[]`。\n- prompt: 必填项。表示对话模板中的`{{RESPONSE}}`之前的对话部分。我们使用`{{QUERY}}`代表user询问部分的填充符。例如qwen的prompt为`['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n']`。\n- chat_sep: 必填项。多轮对话中每轮的分隔符。若设置为None，则该template不支持多轮对话。例如qwen的chat_sep为`['<|im_end|>\\n']`。\n- suffix: 默认为`[['eos_token_id']]`。对话模板的后缀部分，独立于多轮对话而产生的对话模板循环，通常为eos_token。例如qwen的suffix为`['<|im_end|>']。`\n- template_cls: 默认为`Template`。通常在定义多模态模型的template时需要进行自定义，自定义`_encode`、`_post_encode`、`_data_collator`函数。\n- system_prefix: 默认为None。含system的对话模板前缀。我们使用`{{SYSTEM}}`作为system的填充符。例如qwen的system_prefix为`['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n']`。\n - 注意：若system为空时，`prefix`可以被`system_prefix`替代，则可以将`prefix`写为含system的前缀，而无需设置`system_prefix`。\n - 若prefix不含`{{SYSTEM}}`且未设置system_prefix，则该template不支持system。\n- default_system: 默认为None。不传入`--system`时使用的默认system。例如qwen的default_system为`'You are a helpful assistant.'`。\n- stop_words: 默认为`[]`。除了eos_token和`suffix[-1]`的额外停止符。例如qwen的stop_words为`['<|endoftext|>']`。\n - 注意：推理时，输出的response将会过滤eos_token和`suffix[-1]`，但是会保留额外的stop_words。\n"
},
{
"path": "docs/source/GetStarted/Quick-start.md",
"content": "# 快速开始\n\n🍲 ms-swift是魔搭社区提供的大模型与多模态大模型微调部署框架，现已支持600+纯文本大模型与400+多模态大模型的训练（预训练、微调、人类对齐）、推理、评测、量化与部署。其中大模型包括：Qwen3、Qwen3.5、InternLM3、GLM4.5、Mistral、DeepSeek-R1、Llama4等模型，多模态大模型包括：Qwen3-VL、Qwen3-Omni、Llava、InternVL3.5、MiniCPM-V-4、Ovis2.5、GLM4.5-V、DeepSeek-VL2等模型。\n\n🍔 除此之外，ms-swift汇集了最新的训练技术，包括集成Megatron并行技术，包括TP、PP、CP、EP等为训练提供加速，以及众多GRPO算法族强化学习的算法，包括：GRPO、DAPO、GSPO、SAPO、CISPO、RLOO、Reinforce++等提升模型智能。ms-swift支持广泛的训练任务，包括DPO、KTO、RM、CPO、SimPO、ORPO等偏好学习算法，以及Embedding、Reranker、序列分类任务。ms-swift提供了大模型训练全链路的支持，包括使用vLLM、SGLang和LMDeploy对推理、评测、部署模块提供加速，以及使用GPTQ、AWQ、BNB、FP8技术对大模型进行量化。\n\n**为什么选择ms-swift？**\n- 🍎 **模型类型**：支持600+纯文本大模型、**400+多模态大模型**以及All-to-All全模态模型训练到部署全流程，热门模型Day0支持。\n- **数据集类型**：内置150+预训练、微调、人类对齐、多模态等各种任务数据集，并支持自定义数据集，用户只需准备数据集即可一键训练。\n- **硬件支持**：支持A10/A100/H100、RTX系列、T4/V100、CPU、MPS以及国产硬件Ascend NPU等。\n- **轻量训练**：支持了LoRA、QLoRA、DoRA、LoRA+、LLaMAPro、LongLoRA、LoRA-GA、ReFT、RS-LoRA、Adapter、LISA等轻量微调方式。\n- **量化训练**：支持对BNB、AWQ、GPTQ、AQLM、HQQ、EETQ量化模型进行训练，7B模型训练只需9GB训练资源。\n- **显存优化**: GaLore、Q-Galore、UnSloth、Liger-Kernel、Flash-Attention 2/3 以及 **Ulysses和Ring-Attention序列并行技术**支持，降低长文本训练显存占用。\n- **分布式训练**：支持分布式数据并行（DDP）、device_map简易模型并行、DeepSpeed ZeRO2 ZeRO3、FSDP/FSDP2以及Megatron等分布式训练技术。\n- 🍓 **多模态训练**：支持多模态packing技术提升训练速度100%+，支持文本、图像、视频和语音混合模态数据训练，支持vit/aligner/llm单独控制。\n- **Agent训练**：支持Agent template，准备一套数据集可用于不同模型的训练。\n- 🍊 **训练任务**：支持预训练和指令微调，以及DPO、GKD、KTO、RM、CPO、SimPO、ORPO等训练任务，支持**Embedding/Reranker**和序列分类任务。\n- 🥥 **Megatron并行技术**：提供TP/PP/SP/CP/ETP/EP/VPP并行策略，显著提升**MoE模型训练速度**。支持300+纯文本大模型和100+多模态大模型的全参数和LoRA训练方法。支持CPT/SFT/GRPO/DPO/KTO/RM训练任务。\n- 🍉 **强化学习**：内置**丰富GRPO族算法**，包括GRPO、DAPO、GSPO、SAPO、CISPO、CHORD、RLOO、Reinforce++等，支持同步和异步vLLM引擎推理加速，可使用插件拓展奖励函数、多轮推理调度器以及环境等。\n- **全链路能力**：覆盖训练、推理、评测、量化和部署全流程。\n- **界面训练**：提供使用Web-UI界面的方式进行训练、推理、评测、量化，完成大模型的全链路。\n- **推理加速**：支持Transformers、vLLM、SGLang和LmDeploy推理加速引擎，并提供OpenAI接口，为推理、部署和评测模块提供加速。\n- **模型评测**：以EvalScope作为评测后端，支持100+评测数据集对纯文本和多模态模型进行评测。\n- **模型量化**：支持AWQ、GPTQ、FP8和BNB的量化导出，导出的模型支持使用vLLM/SGLang/LmDeploy推理加速。\n\n\n## 安装\n\nms-swift的安装请参考[安装文档](./SWIFT-installation.md)。\n\n## 使用样例\n\n10分钟在单卡3090上对Qwen3-4B-Instruct-2507进行自我认知微调：\n```shell\n# 13GB\nCUDA_VISIBLE_DEVICES=0 \\\nswift sft \\\n --model Qwen/Qwen3-4B-Instruct-2507 \\\n --tuner_type lora \\\n --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \\\n 'AI-ModelScope/alpaca-gpt4-data-en#500' \\\n 'swift/self-cognition#500' \\\n --torch_dtype bfloat16 \\\n --num_train_epochs 1 \\\n --per_device_train_batch_size 1 \\\n --per_device_eval_batch_size 1 \\\n --learning_rate 1e-4 \\\n --lora_rank 8 \\\n --lora_alpha 32 \\\n --target_modules all-linear \\\n --gradient_accumulation_steps 16 \\\n --eval_steps 50 \\\n --save_steps 50 \\\n --save_total_limit 2 \\\n --logging_steps 5 \\\n --max_length 2048 \\\n --output_dir output \\\n --warmup_ratio 0.05 \\\n --dataloader_num_workers 4 \\\n --model_author swift \\\n --model_name swift-robot\n```\n\n小贴士：\n- 如果要使用自定义数据集进行训练，你可以参考[这里](../Customization/Custom-dataset.md)组织数据集格式，并指定`--dataset `。\n- `--model_author`和`--model_name`参数只有当数据集中包含`swift/self-cognition`时才生效。\n- 如果要使用其他模型进行训练，你只需要修改`--model `即可。\n- 默认使用**ModelScope**进行模型和数据集的下载。如果要使用HuggingFace，指定`--use_hf true`即可。\n\n训练完成后，使用以下命令对训练后的权重进行推理：\n- 这里的`--adapters`需要替换成训练生成的last checkpoint文件夹。由于adapters文件夹中包含了训练的参数文件`args.json`，因此不需要额外指定`--model`，`--system`，swift会自动读取这些参数。如果要关闭此行为，可以设置`--load_args false`。\n\n```shell\n# 使用交互式命令行进行推理\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --temperature 0 \\\n --max_new_tokens 2048\n\n# merge-lora并使用vLLM进行推理加速\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --stream true \\\n --merge_lora true \\\n --infer_backend vllm \\\n --vllm_max_model_len 8192 \\\n --temperature 0 \\\n --max_new_tokens 2048\n```\n\n最后，使用以下命令将模型推送到ModelScope：\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift export \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --push_to_hub true \\\n --hub_model_id '' \\\n --hub_token '' \\\n --use_hf false\n```\n\n## 了解更多\n\n- 更多Shell脚本：[https://github.com/modelscope/ms-swift/tree/main/examples](https://github.com/modelscope/ms-swift/tree/main/examples)\n- 使用Python：[https://github.com/modelscope/ms-swift/blob/main/examples/notebook/qwen2_5-self-cognition/self-cognition-sft.ipynb](https://github.com/modelscope/ms-swift/blob/main/examples/notebook/qwen2_5-self-cognition/self-cognition-sft.ipynb)\n"
},
{
"path": "docs/source/GetStarted/SWIFT-installation.md",
"content": "# SWIFT安装\n\n## Wheel包安装\n\n可以使用pip进行安装：\n\n```shell\n# 推荐\npip install 'ms-swift' -U\n# 使用评测\npip install 'ms-swift[eval]' -U\n# 全能力\npip install 'ms-swift[all]' -U\n\n# 使用uv\npip install uv\nuv pip install 'ms-swift' --torch-backend=auto\n```\n\n## 源代码安装\n\n当前main分支为 swift4.x 版本。\n```shell\n# pip install git+https://github.com/modelscope/ms-swift.git\n\n# 全能力\n# pip install \"git+https://github.com/modelscope/ms-swift.git#egg=ms-swift[all]\"\n\ngit clone https://github.com/modelscope/ms-swift.git\ncd ms-swift\npip install -e .\n\n# 使用 uv\nuv pip install -e . --torch-backend=auto\n\n# 全能力\n# pip install -e '.[all]'\n```\n\n安装swift3.x：\n```shell\n# pip install \"git+https://github.com/modelscope/ms-swift.git@release/3.12\"\n\n# 全能力\n# pip install \"git+https://github.com/modelscope/ms-swift.git@release/3.12#egg=ms-swift[all]\"\n\ngit clone -b release/3.12 https://github.com/modelscope/ms-swift.git\ncd ms-swift\npip install -e .\n\n# 全能力\n# pip install -e '.[all]'\n```\n\n## 镜像\n\ndocker可以查看[这里](https://github.com/modelscope/modelscope/blob/build_swift_image/docker/build_image.py#L347)。\n```\n# swift4.0.2\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.10.0-vllm0.17.1-modelscope1.34.0-swift4.0.2\nmodelscope-registry.cn-beijing.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.10.0-vllm0.17.1-modelscope1.34.0-swift4.0.2\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.10.0-vllm0.17.1-modelscope1.34.0-swift4.0.2\n\n# swift3.12.5\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.9.0-vllm0.13.0-modelscope1.33.0-swift3.12.5\nmodelscope-registry.cn-beijing.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.9.0-vllm0.13.0-modelscope1.33.0-swift3.12.5\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.9.0-vllm0.13.0-modelscope1.33.0-swift3.12.5\n\n# swift3.11.3\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.9.1-py311-torch2.8.0-vllm0.11.0-modelscope1.32.0-swift3.11.3\nmodelscope-registry.cn-beijing.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.9.1-py311-torch2.8.0-vllm0.11.0-modelscope1.32.0-swift3.11.3\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.9.1-py311-torch2.8.0-vllm0.11.0-modelscope1.32.0-swift3.11.3\n```\n\n历史镜像 \n\n```\n# swift3.10.3\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.8.0-vllm0.11.0-modelscope1.31.0-swift3.10.3\nmodelscope-registry.cn-beijing.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.8.0-vllm0.11.0-modelscope1.31.0-swift3.10.3\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.8.0-vllm0.11.0-modelscope1.31.0-swift3.10.3\n\n# swift3.9.3\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.8.0-vllm0.11.0-modelscope1.31.0-swift3.9.3\nmodelscope-registry.cn-beijing.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.8.0-vllm0.11.0-modelscope1.31.0-swift3.9.3\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.8.0-vllm0.11.0-modelscope1.31.0-swift3.9.3\n\n# swift3.8.3\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.6.3-py311-torch2.7.1-vllm0.10.1.1-modelscope1.29.2-swift3.8.3\nmodelscope-registry.cn-beijing.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.6.3-py311-torch2.7.1-vllm0.10.1.1-modelscope1.29.2-swift3.8.3\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.6.3-py311-torch2.7.1-vllm0.10.1.1-modelscope1.29.2-swift3.8.3\n\n# swift3.7.2\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.6.3-py311-torch2.7.1-vllm0.10.0-modelscope1.28.2-swift3.7.2\nmodelscope-registry.cn-beijing.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.6.3-py311-torch2.7.1-vllm0.10.0-modelscope1.28.2-swift3.7.2\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.6.3-py311-torch2.7.1-vllm0.10.0-modelscope1.28.2-swift3.7.2\n\n# swift3.6.4\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py310-torch2.6.0-vllm0.8.5.post1-modelscope1.28.1-swift3.6.4\nmodelscope-registry.cn-beijing.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py310-torch2.6.0-vllm0.8.5.post1-modelscope1.28.1-swift3.6.4\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py310-torch2.6.0-vllm0.8.5.post1-modelscope1.28.1-swift3.6.4\n\n# swift3.5.3\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py310-torch2.6.0-vllm0.8.5.post1-modelscope1.27.1-swift3.5.3\nmodelscope-registry.cn-beijing.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py310-torch2.6.0-vllm0.8.5.post1-modelscope1.27.1-swift3.5.3\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py310-torch2.6.0-vllm0.8.5.post1-modelscope1.27.1-swift3.5.3\n\n# swift3.4.1.post1\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py311-torch2.6.0-vllm0.8.5.post1-modelscope1.26.0-swift3.4.1.post1\nmodelscope-registry.cn-beijing.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py311-torch2.6.0-vllm0.8.5.post1-modelscope1.26.0-swift3.4.1.post1\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py311-torch2.6.0-vllm0.8.5.post1-modelscope1.26.0-swift3.4.1.post1\n\n# swift3.3.0.post1\nmodelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py311-torch2.6.0-vllm0.8.3-modelscope1.25.0-swift3.3.0.post1\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py311-torch2.6.0-vllm0.8.3-modelscope1.25.0-swift3.3.0.post1\n\n# swift3.2.2\nmodelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.4.0-py311-torch2.5.1-modelscope1.25.0-swift3.2.2\n```\n' --studio_title My-Awesome-Space --stream true\n# 或者\nswift app --model '' --adapters '' --stream true\n```\n即可启动一个只有推理页面的应用，该应用会在启动时对模型进行部署并提供后续使用。\n"
},
{
"path": "docs/source/Instruction/Agent-support.md",
"content": "# Agent支持\n\n## 数据集格式\n\nms-swift 使用 agent-template 实现了Agent数据格式与模型的解耦：基于统一的数据集格式，可以灵活切换不同模型进行训练，无需修改数据。\n\n纯文本Agent和多模态Agent的示例数据样本如下：\n```jsonl\n{\"tools\": \"[{\\\"type\\\": \\\"function\\\", \\\"function\\\": {\\\"name\\\": \\\"realtime_aqi\\\", \\\"description\\\": \\\"天气预报。获取实时空气质量。当前空气质量，PM2.5，PM10信息\\\", \\\"parameters\\\": {\\\"type\\\": \\\"object\\\", \\\"properties\\\": {\\\"city\\\": {\\\"type\\\": \\\"string\\\", \\\"description\\\": \\\"城市名，例如：上海\\\"}}, \\\"required\\\": [\\\"city\\\"]}}}]\", \"messages\": [{\"role\": \"user\", \"content\": \"北京和上海今天的天气情况\"}, {\"role\": \"tool_call\", \"content\": \"{\\\"name\\\": \\\"realtime_aqi\\\", \\\"arguments\\\": {\\\"city\\\": \\\"北京\\\"}}\"}, {\"role\": \"tool_call\", \"content\": \"{\\\"name\\\": \\\"realtime_aqi\\\", \\\"arguments\\\": {\\\"city\\\": \\\"上海\\\"}}\"}, {\"role\": \"tool_response\", \"content\": \"{\\\"city\\\": \\\"北京\\\", \\\"aqi\\\": \\\"10\\\", \\\"unit\\\": \\\"celsius\\\"}\"}, {\"role\": \"tool_response\", \"content\": \"{\\\"city\\\": \\\"上海\\\", \\\"aqi\\\": \\\"72\\\", \\\"unit\\\": \\\"fahrenheit\\\"}\"}, {\"role\": \"assistant\", \"content\": \"根据天气预报工具，北京今天的空气质量指数为10，属于良好水平；上海今天的空气质量指数为72，属于轻度污染水平。\"}]}\n{\"tools\": \"[{\\\"type\\\": \\\"function\\\", \\\"function\\\": {\\\"name\\\": \\\"click\\\", \\\"description\\\": \\\"点击屏幕中的某个位置\\\", \\\"parameters\\\": {\\\"type\\\": \\\"object\\\", \\\"properties\\\": {\\\"x\\\": {\\\"type\\\": \\\"integer\\\", \\\"description\\\": \\\"横坐标，表示屏幕上的水平位置\\\"}, \\\"y\\\": {\\\"type\\\": \\\"integer\\\", \\\"description\\\": \\\"纵坐标，表示屏幕上的垂直位置\\\"}}, \\\"required\\\": [\\\"x\\\", \\\"y\\\"]}}}]\", \"messages\": [{\"role\": \"user\", \"content\": \"现在几点了？\"}, {\"role\": \"assistant\", \"content\": \"\\n我可以通过打开日历App来获取当前时间。\\n \\n\"}, {\"role\": \"tool_call\", \"content\": \"{\\\"name\\\": \\\"click\\\", \\\"arguments\\\": {\\\"x\\\": 105, \\\"y\\\": 132}}\"}, {\"role\": \"tool_response\", \"content\": \"{\\\"images\\\": \\\"\\\", \\\"status\\\": \\\"success\\\"}\"}, {\"role\": \"assistant\", \"content\": \"成功打开日历App，现在的时间为中午11点\"}], \"images\": [\"desktop.png\", \"calendar.png\"]}\n```\n- agent_template为\"react_en\", \"hermes\"等情况下，该格式适配所有模型Agent训练，可以轻松在不同模型间切换。\n- 其中tools是一个包含tool列表的json字符串，messages中role为'tool_call'和'tool_response/tool'的content部分都需要是json字符串。\n- tools字段将在训练/推理时和`{\"role\": \"system\", ...}\"`部分组合，根据agent_template组成完整的system部分。\n- `{\"role\": \"tool_call\", ...}`部分将根据agent_template自动转成对应格式的`{\"role\": \"assistant\", ...}`，多条连续的`{\"role\": \"assistant\", ...}`将拼接在一起组成完整的assistant_content。\n- `{\"role\": \"tool_response\", ...}`也可以写成`{\"role\": \"tool\", ...}`，这两种写法是等价的。该部分也将根据`agent_template`自动转换格式。该部分在训练时将不进行损失的计算，角色类似于`{\"role\": \"user\", ...}`。\n- 该格式支持并行调用工具，例子参考第一条数据样本。多模态Agent数据样本中``标签数量应与\"images\"长度相同，其标签位置代表图像特征的插入位置。当然也支持其他模态，例如audios, videos。\n- 注意：您也可以手动将数据处理为role为system/user/assistant的messages格式。agent_template的作用是将其中的tools字段以及role为tool_call和tool_response的messages部分，自动映射为标准的role为system/user/assistant的messages格式。\n\n以下为上述两条数据样本由qwen2_5和qwen2_5_vl的template进行encode后的input_ids和labels，选择的agent_template为**hermes**：\n\n样本一（并行工具调用）：\n```text\n[INPUT_IDS] <|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within \n{\"type\": \"function\", \"function\": {\"name\": \"realtime_aqi\", \"description\": \"天气预报。获取实时空气质量。当前空气质量，PM2.5，PM10信息\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\", \"description\": \"城市名，例如：上海\"}}, \"required\": [\"city\"]}}}\n \n\nFor each function call, return a json object with function name and arguments within \n{\"name\": , \"arguments\": }\n <|im_end|>\n<|im_start|>user\n北京和上海今天的天气情况<|im_end|>\n<|im_start|>assistant\n\n{\"name\": \"realtime_aqi\", \"arguments\": {\"city\": \"北京\"}}\n \n\n{\"name\": \"realtime_aqi\", \"arguments\": {\"city\": \"上海\"}}\n <|im_end|>\n<|im_start|>user\n\n{\"city\": \"北京\", \"aqi\": \"10\", \"unit\": \"celsius\"}\n \n\n{\"city\": \"上海\", \"aqi\": \"72\", \"unit\": \"fahrenheit\"}\n <|im_end|>\n<|im_start|>assistant\n根据天气预报工具，北京今天的空气质量指数为10，属于良好水平；上海今天的空气质量指数为72，属于轻度污染水平。<|im_end|>\n\n[LABELS] [-100 * 195]\n{\"name\": \"realtime_aqi\", \"arguments\": {\"city\": \"北京\"}}\n \n\n{\"name\": \"realtime_aqi\", \"arguments\": {\"city\": \"上海\"}}\n <|im_end|>[-100 * 67]根据天气预报工具，北京今天的空气质量指数为10，属于良好水平；上海今天的空气质量指数为72，属于轻度污染水平。<|im_end|>\n```\n\n样本二（多模态，混合assistant和tool_call）：\n```text\n[INPUT_IDS] <|im_start|>system\nYou are a helpful assistant.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within \n{\"type\": \"function\", \"function\": {\"name\": \"click\", \"description\": \"点击屏幕中的某个位置\", \"parameters\": {\"type\": \"object\", \"properties\": {\"x\": {\"type\": \"integer\", \"description\": \"横坐标，表示屏幕上的水平位置\"}, \"y\": {\"type\": \"integer\", \"description\": \"纵坐标，表示屏幕上的垂直位置\"}}, \"required\": [\"x\", \"y\"]}}}\n \n\nFor each function call, return a json object with function name and arguments within \n{\"name\": , \"arguments\": }\n <|im_end|>\n<|im_start|>user\n<|vision_start|>[151655 * 729]<|vision_end|>现在几点了？<|im_end|>\n<|im_start|>assistant\n\n我可以通过打开日历App来获取当前时间。\n \n\n{\"name\": \"click\", \"arguments\": {\"x\": 105, \"y\": 132}}\n <|im_end|>\n<|im_start|>user\n\n{\"images\": \"<|vision_start|>[151655 * 729]<|vision_end|>\", \"status\": \"success\"}\n <|im_end|>\n<|im_start|>assistant\n成功打开日历App，现在的时间为中午11点<|im_end|>\n\n[LABELS] [-100 * 924]\n我可以通过打开日历App来获取当前时间。\n \n\n{\"name\": \"click\", \"arguments\": {\"x\": 105, \"y\": 132}}\n <|im_end|>[-100 * 759]成功打开日历App，现在的时间为中午11点<|im_end|>\n```\n\n**react_en**是常用的agent template格式之一，以下为样本一由qwen2_5使用`agent_template='react_en'`进行encode后的input_ids和labels：\n\n```text\n[INPUT_IDS] <|im_start|>system\nAnswer the following questions as best you can. You have access to the following tools:\n\nrealtime_aqi: Call this tool to interact with the realtime_aqi API. What is the realtime_aqi API useful for? 天气预报。获取实时空气质量。当前空气质量，PM2.5，PM10信息 Parameters: {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\", \"description\": \"城市名，例如：上海\"}}, \"required\": [\"city\"]} Format the arguments as a JSON object.\n\nUse the following format:\n\nQuestion: the input question you must answer\nThought: you should always think about what to do\nAction: the action to take, should be one of [realtime_aqi]\nAction Input: the input to the action\nObservation: the result of the action\n... (this Thought/Action/Action Input/Observation can be repeated zero or more times)\nThought: I now know the final answer\nFinal Answer: the final answer to the original input question\n\nBegin!\n<|im_end|>\n<|im_start|>user\n北京和上海今天的天气情况<|im_end|>\n<|im_start|>assistant\nAction: realtime_aqi\nAction Input: {'city': '北京'}\nAction: realtime_aqi\nAction Input: {'city': '上海'}\nObservation:{\"city\": \"北京\", \"aqi\": \"10\", \"unit\": \"celsius\"}\nObservation:{\"city\": \"上海\", \"aqi\": \"72\", \"unit\": \"fahrenheit\"}\n根据天气预报工具，北京今天的空气质量指数为10，属于良好水平；上海今天的空气质量指数为72，属于轻度污染水平。<|im_end|>\n\n[LABELS] [-100 * 233]Action: realtime_aqi\nAction Input: {'city': '北京'}\nAction: realtime_aqi\nAction Input: {'city': '上海'}\nObservation:[-100 * 45]根据天气预报工具，北京今天的空气质量指数为10，属于良好水平；上海今天的空气质量指数为72，属于轻度污染水平。<|im_end|>\n```\n\n更多模型和agent_template的尝试可以使用以下代码，更多的agent template可选值参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/agent_template/__init__.py)。\n```python\nfrom swift import get_processor, get_template\n\ntokenizer = get_processor('Qwen/Qwen3.5-2B')\ntemplate = get_template(tokenizer) # 使用默认agent模板\n# template = get_template(tokenizer, agent_template='qwen3_5')\nprint(f'agent_template: {template._agent_template}')\ndata = {...}\ntemplate.set_mode('train')\nencoded = template.encode(data)\nprint(f'[INPUT_IDS] {template.safe_decode(encoded[\"input_ids\"])}\\n')\nprint(f'[LABELS] {template.safe_decode(encoded[\"labels\"])}')\n```\n\n\n## tools格式\ntools字段提供了模型可以调用的API信息。你需要提供tools的名字，描述和参数，示例如下：\n\n```python\ntools = [{\n 'type': 'function',\n 'function': {\n 'name': 'get_current_weather',\n 'description': 'Get the current weather in a given location',\n 'parameters': {\n 'type': 'object',\n 'properties': {\n 'location': {\n 'type': 'string',\n 'description': 'The city and state, e.g. San Francisco, CA'\n },\n 'unit': {\n 'type': 'string',\n 'enum': ['celsius', 'fahrenheit']\n }\n },\n 'required': ['location']\n }\n }\n}]\n```\n\n## loss_scale的使用\n\nloss_scale参数可用于调节模型输出部分在训练过程中的损失权重。目前支持两种配置方式：字符串精确匹配和正则表达式匹配。\n\n1. 字符串匹配示例：ReACT 格式\n\n以 ReACT 格式为例，可通过 `--loss_scale react` 启用相应的 loss_scale 配置（配置文件详见 [react.json](https://github.com/modelscope/ms-swift/blob/main/swift/loss_scale/config/react.json)）。该方式基于字符串精确匹配，配置中的字典映射需提供一个包含两个元素的列表，分别表示：当前匹配字符串本身的损失权重，\n从该字符串之后到下一个指定字符串之前的内容的损失权重。该设置的具体效果如下：\n- 'Action:' 和 'Action Input:' 字段自身及其后续内容的损失权重均为 2；\n- 'Thought:' 和 'Final Answer:' 字段自身及其后续内容的损失权重均为 1；\n- 'Observation:' 字段自身的权重为 2，但其后跟随的工具调用结果部分的损失权重为 0。\n\n2. 正则匹配示例：忽略空思维块\n\n在训练推理模型时，我们可能需要忽略数据集中存在的形如 `'\\n\\n \\n\\n'`的空思维标记损失计算。此时可使用 `--loss_scale ignore_empty_think`（配置文件详见 [ignore_empty_think.json](https://github.com/modelscope/ms-swift/blob/main/swift/loss_scale/config/ignore_empty_think.json)）。该配置采用正则表达式匹配方式，字典映射的列表只需指定一个值，表示匹配内容的损失权重。该设置的具体效果如下：\n\n- 所有与正则表达式`\\\\s* \\\\s*`匹配的字符串，loss_scale为0，即不计算损失。\n\n使用代码测试loss_scale:\n```python\nfrom swift import get_processor, get_template\n\ndata = {\"messages\": [\n {\"role\": \"user\", \"content\": \"aaaaa\"},\n {\"role\": \"assistant\", \"content\": \"\\n\\n \\n\\nabc\\n\\n \\n\\n123\"},\n]}\n\ntemplate = get_template(get_processor('Qwen/Qwen3-8B'), loss_scale='ignore_empty_think')\ntemplate.set_mode('train')\ninputs = template.encode(data)\n\nprint(template.safe_decode(inputs['labels']))\n# '[-100 * 14]abc\\n\\n \\n\\n123<|im_end|>\\n'\n```\n\n更多的loss_scale插件设计，请参考[架构](../Customization/Architecture.md#loss-scale)文档.\n\n## 训练\n- 训练Base模型的Agent能力，通过修改`--model`切换不同模型，参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/agent/qwen2_5.sh)。\n- 训练GLM4的agent_template为hermes，参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/agent/glm4.sh)。\n- 使用`--loss_scale`对模型输出部分的损失权重进行调整，参加[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/agent/loss_scale)。\n\n## 推理\n\n- 🚀原始模型或者全参数训练后模型的推理，参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_agent.py)。\n- LoRA训练后推理，参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/agent/loss_scale/infer_lora.py)。\n\n## 部署\n\n服务端和客户端代码，参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/deploy/agent)。\n"
},
{
"path": "docs/source/Instruction/Command-line-parameters.md",
"content": "# 命令行参数\n\n命令行参数的介绍会分为基本参数，原子参数、集成参数和特定模型参数。**命令行最终使用的参数列表为集成参数。集成参数继承自基本参数和一些原子参数**。特定模型参数是针对于具体模型的参数，可以通过`--model_kwargs'`或者环境变量进行设置。Megatron-SWIFT命令行参数介绍可以在[Megatron-SWIFT训练文档](../Megatron-SWIFT/Command-line-parameters.md)中找到。\n\n**提示：**\n- 命令行传入list使用空格隔开即可。例如：`--dataset `。\n- 命令行传入dict使用json。例如：`--model_kwargs '{\"fps_max_frames\": 12}'`。\n- 带🔥的参数为重要参数，刚熟悉ms-swift的用户可以先关注这些命令行参数。\n\n## 基本参数\n\n- 🔥tuner_backend: 可选为'peft'，'unsloth'。默认为'peft'。\n- 🔥tuner_type: 可选为'lora'、'full'、'longlora'、'adalora'、'llamapro'、'adapter'、'vera'、'boft'、'fourierft'、'reft'。默认为'lora'。\n- 🔥adapters: 用于指定adapter的id/path的list，默认为`[]`。该参数通常用于推理/部署命令，例如：`swift infer --model '' --adapters ''`。该参数偶尔也用于断点续训，该参数与`resume_from_checkpoint`的区别在于，**该参数只读取adapter权重**，而不加载优化器和随机种子，并不跳过已训练的数据集部分。\n - `--model`与`--adapters`的区别：`--model`后接完整权重的目录路径，内包含model/tokenizer/config等完整权重信息，例如`model.safetensors`。`--adapters`后接增量adapter权重目录路径的列表，内涵adapter的增量权重信息，例如`adapter_model.safetensors`。\n- 🔥external_plugins: 外部`plugin.py`文件列表，这些文件会被额外加载（即对模块进行`import`）。默认为`[]`。你可以传入自定义模型、对话模板和数据集注册的`.py`文件路径，参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/custom/sft.sh)；或者自定义GRPO的组件，参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/plugin/run_external_reward_func.sh)。\n- seed: 全局随机种子，默认为42。\n - 注意：该随机种子与控制数据集随机的`data_seed`互不影响。\n- model_kwargs: 特定模型可传入的额外参数，该参数列表会在训练/推理时打印日志进行提示。例如`--model_kwargs '{\"fps_max_frames\": 12}'`。你也可以通过环境变量的方式设置，例如`FPS_MAX_FRAMES=12`。默认为None。\n - 注意：**若你在训练时指定了特定模型参数，请在推理时也设置对应的参数**，这可以提高训练效果。\n - 特定模型参数的含义可以在对应模型官方repo或者其推理代码中找到相应含义。ms-swift引入这些参数以确保训练的模型与官方推理代码效果对齐。\n- load_args: 当指定`--resume_from_checkpoint`、`--model`、`--adapters`会读取保存文件中的`args.json`，读取的keys查看[base_args.py](https://github.com/modelscope/ms-swift/blob/main/swift/arguments/base_args/base_args.py)。推理和导出时默认为True，训练时默认为False。该参数通常不需要修改。\n- load_data_args: 如果将该参数设置为True，则会额外读取`args.json`中的数据参数。默认为False。**该参数通常用于推理时对训练中切分的验证集进行推理**，例如：`swift infer --adapters xxx --load_data_args true --stream true --max_new_tokens 512`。\n- use_hf: 控制模型下载、数据集下载、模型推送使用[ModelScope](https://modelscope.cn/)还是[HuggingFace](https://huggingface.co/)。默认为False，使用ModelScope。\n - 提示：如果你想在国外访问ModelScope，可以尝试使用[ModelScope国际版](https://modelscope.ai/home)，设置环境变量`MODELSCOPE_DOMAIN='www.modelscope.ai'`即可。\n- hub_token: hub token. modelscope的hub token可以查看[这里](https://modelscope.cn/my/myaccesstoken)。默认为None。\n- ddp_timeout: 默认为18000000，单位为秒。\n- ddp_backend: 可选为\"nccl\"、\"gloo\"、\"mpi\"、\"ccl\"、\"hccl\"、\"cncl\"、\"mccl\"。默认为None，进行自动选择。\n- ignore_args_error: 用于兼容jupyter notebook。默认为False。\n\n### 模型参数\n- 🔥model: [模型id](https://modelscope.cn/models)或模型本地路径。默认为None。\n- 🔥model_type: 模型类型。**我们将相同的模型架构、模型加载过程、template定义为一个`model_type`**。默认为None，即**根据`--model`的后缀和config.json中的'architectures'属性进行自动选择**。对应模型的model_type可以在[支持的模型列表](./Supported-models-and-datasets.md)中找到。\n - 注意：ms-swift中model_type的概念与`config.json`中的model_type不同。\n - 自定义模型通常需要自行注册`model_type`和`template`，具体可以参考[自定义模型文档](../Customization/Custom-model.md)。\n- model_revision: 模型版本，默认为None。\n- task_type: 默认为'causal_lm'。可选为'causal_lm'、'seq_cls'、'embedding'、'reranker'和'generative_reranker'。seq_cls的例子可以查看[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/seq_cls)，embedding的例子查看[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/embedding)。\n - 若设置为'seq_cls'，你通常需要额外设置`--num_labels`和`--problem_type`。\n- 🔥torch_dtype: 模型权重的数据类型，支持`float16`,`bfloat16`,`float32`。默认为None，从'config.json'文件中读取。\n- attn_impl: attention类型，可选项为'sdpa', 'eager', 'flash_attn', 'flash_attention_2', 'flash_attention_3'等。默认使用None，读取'config.json'。\n - 注意：这几种attention实现并不一定都支持，这取决于对应模型transformers实现的支持情况。\n - 若设置为'flash_attn'（兼容旧版本），则使用'flash_attention_2'。\n- 🔥experts_impl: 专家实现类型，可选项为'grouped_mm', 'batched_mm', 'eager'。默认为None。该特性需要\"transformers>=5.0.0\"。\n- new_special_tokens: 需要新增的特殊tokens。默认为`[]`。例子参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/new_special_tokens)。\n - 注意：你也可以传入以`.txt`结尾的文件路径，每行为一个special token。\n- num_labels: 分类模型（即`--task_type seq_cls`）需要指定该参数。代表标签数量，默认为None。\n- problem_type: 分类模型（即`--task_type seq_cls`）需要指定该参数。可选为'regression', 'single_label_classification', 'multi_label_classification'。默认为None，若模型为 reward_model 或 num_labels 为1，该参数为'regression'，其他情况，该参数为'single_label_classification'。\n- rope_scaling: rope类型，你可以传入字符串，例如：`linear`、`dynamic`、`yarn`并结合传入`max_model_len`，ms-swift会自动设置对应的rope_scaling并覆盖'config.json'中的rope_scaling。或者你需要传入一个json字符串，例如`'{\"factor\":2.0, \"type\":\"yarn\"}'`，该值会直接覆盖'config.json'中的rope_scaling。默认为None。\n- max_model_len: 如果使用`rope_scaling`并传入字符串，可以设置`max_model_len`，该参数用来计算rope的`factor`倍数。该参数默认为None。若为非None，该参数会**覆盖**'config.json'中的`max_position_embeddings`值。\n- device_map: 模型使用的device_map配置，例如：'auto'、'cpu'、json字符串、json文件路径。该参数会**透传**入transformers的`from_pretrained`接口。默认为None，根据设备和分布式训练情况自动设置。\n- max_memory: device_map设置为'auto'或者'sequential'时，会根据max_memory进行模型权重的device分配，例如：`--max_memory '{0: \"20GB\", 1: \"20GB\"}'`。默认为None。该参数会透传入transformers的`from_pretrained`接口。\n- local_repo_path: 部分模型在加载时依赖于github repo，例如[deepseek-vl2](https://github.com/deepseek-ai/DeepSeek-VL2)。为了避免`git clone`时遇到网络问题，可以直接使用本地repo。该参数需要传入本地repo的路径, 默认为`None`。\n- init_strategy: 加载模型时，初始化模型中所有未初始化的参数（自定义模型架构时）。可选为'zero', 'uniform', 'normal', 'xavier_uniform', 'xavier_normal', 'kaiming_uniform', 'kaiming_normal', 'orthogonal'。默认为None。\n\n\n### 数据参数\n- 🔥dataset: 数据集id或路径的list。默认为`[]`。每个数据集的传入格式为：`'数据集id or 数据集路径:子数据集#采样数量'`，其中子数据集和取样数据可选。本地数据集支持jsonl、csv、json、文件夹等。**hub端的开源数据集可以通过`git clone`到本地并将文件夹传入而离线使用**。自定义数据集格式可以参考[自定义数据集文档](../Customization/Custom-dataset.md)。你可以传入`--dataset `来使用多个数据集。\n - 子数据集: 该参数只有当dataset为ID或者文件夹时生效。若注册时指定了subsets，且只有一个子数据集，则默认选择注册时指定的子数据集，否则默认为'default'。你可以使用`/`来选择多个子数据集，例如：`:subset1/subset2`。你也可以使用'all'来选择注册时指定的所有子数据集，例如：`:all`。注册例子可以参考[这里](https://modelscope.cn/datasets/swift/garbage_competition)。\n - 采样数量: 默认使用完整的数据集。你可以通过设置`#采样数`对选择的数据集进行采样，例如``。若采样数少于数据样本总数，则进行随机选择（不重复采样）。若采样数高于数据样本总数，则只额外随机采样`采样数%数据样本总数`的样本，数据样本重复采样`采样数//数据样本总数`次。注意：流式数据集（`--streaming true`）只进行顺序采样。若设置`--dataset_shuffle false`，则非流式数据集也进行顺序采样。\n- 🔥val_dataset: 验证集id或路径的list。默认为`[]`。\n- 🔥cached_dataset: 使用缓存数据集（使用`swift export --to_cached_dataset true ...`命令产生），避免大型数据集训练/推理时，tokenize过程占用gpu时间。该参数用于设置缓存训练数据集文件夹路径，默认为`[]`。例子参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/cached_dataset)。\n - 提示：cached_dataset只会在数据集中额外存储length字段（为避免存储压力），并过滤掉会报错的数据样本。在训练/推理时，支持`--max_length`参数进行超长数据过滤/裁剪以及`--packing`参数。数据实际预处理过程将在训练时同步进行，该过程和训练是重叠的，并不会影响训练速度。\n - cached_dataset在`ms-swift`和`Megatron-SWIFT`之间是通用的，且支持pt/sft/infer/rlhf，使用`--template_mode`设置训练类型；支持embedding/reranker/seq_cls任务，使用`--task_type`设置任务类型。\n - 支持对cache_dataset进行采样，语法为`#采样数`，支持采样数高于和少于样本数的情况，功能与实现参考`--dataset`的介绍。\n- cached_val_dataset: 缓存验证数据集的文件夹路径，默认为`[]`。\n- 🔥split_dataset_ratio: 不指定val_dataset时从训练集拆分验证集的比例，默认为0.，即不从训练集切分验证集。\n- data_seed: 数据集随机种子，默认为42。\n- 🔥dataset_num_proc: 数据集预处理的进程数，默认为1。\n - 提示：纯文本模型建议将该值开大加速预处理速度。而多模态模型不建议开太大，这可能导致更慢的预处理速度（多模态模型若出现cpu利用率100%，但是处理速度极慢的情况，建议额外设置`OMP_NUM_THREADS`环境变量）。\n- 🔥load_from_cache_file: 是否从缓存中加载数据集，默认为False。**建议在实际运行时设置为True，debug阶段设置为False**。你可以修改`MODELSCOPE_CACHE`环境变量控制缓存的路径。\n- dataset_shuffle: 是否对dataset进行随机操作。默认为True。\n - 注意：**CPT/SFT的随机包括两个部分**：数据集的随机，由`dataset_shuffle`控制；train_dataloader中的随机，由`train_dataloader_shuffle`控制。\n- val_dataset_shuffle: 是否对val_dataset进行随机操作。默认为False。\n- streaming: 流式读取并处理数据集，默认False。（流式数据集的随机并不彻底，可能导致loss波动剧烈。）\n - 注意：需要额外设置`--max_steps`，因为流式数据集无法获得其长度。你可以通过设置`--save_strategy epoch`并设置较大的max_steps来实现与`--num_train_epochs`等效的训练。或者，你也可以设置`max_epochs`确保训练到对应epochs时退出训练，并对权重进行验证和保存。\n - 注意：流式数据集可以跳过预处理等待，将预处理时间与训练时间重叠。流式数据集的预处理只在rank0上进行，并通过数据分发的方式同步到其他进程，**其通常效率不如非流式数据集采用的数据分片读取方式**。当训练的world_size较大时，预处理和数据分发将成为训练瓶颈。\n- interleave_prob: 默认值为 None。在组合多个数据集时，默认使用datasets库的 `concatenate_datasets` 函数；如果设置了该参数，则会使用 `interleave_datasets` 函数。该参数通常用于流式数据集的组合，并会作为参数传入 `interleave_datasets` 函数中。该参数不对`--val_dataset`生效。\n- stopping_strategy: 可选为\"first_exhausted\", \"all_exhausted\"，默认为\"first_exhausted\"。传入`interleave_datasets`函数中。该参数不对`--val_dataset`生效。\n- shuffle_buffer_size: 该参数用于指定**流式数据集**的随机buffer大小，默认为1000。该参数只在`dataset_shuffle`设置为true时有效。\n- download_mode: 数据集下载模式，包含`reuse_dataset_if_exists`和`force_redownload`，默认为'reuse_dataset_if_exists'。\n - 通常在使用hub端数据集报错时设置为`--download_mode force_redownload`。\n- columns: 用于对数据集进行列映射，使数据集满足AutoPreprocessor可以处理的样式，AutoPreprocessor可以处理的数据集格式查看[自定义数据集文档](../Customization/Custom-dataset.md)。你可以传入json字符串，例如：`'{\"text1\": \"query\", \"text2\": \"response\"}'`，代表将数据集中的\"text1\"映射为\"query\"，\"text2\"映射为\"response\"，而query-response格式可以被AutoPreprocessor处理。默认为None。\n- strict: 如果为True，则数据集只要某行有问题直接抛错，否则会丢弃出错数据样本。默认False。该参数通常用于排查错误。\n- 🔥remove_unused_columns: 是否删除数据集中不被使用的列，默认为True。\n - 若该参数设置为False，则将额外的数据集列传递至trainer的`compute_loss`函数内，**方便自定义损失函数使用额外的数据集列**。\n - GPRO该参数的默认值为False。\n- 🔥model_name: **仅用于自我认知任务**，只对`swift/self-cognition`数据集生效，替换掉数据集中的`{{NAME}}`通配符。传入模型中文名和英文名，以空格分隔，例如：`--model_name 小黄 'Xiao Huang'`。默认为None。\n- 🔥model_author: 仅用于自我认知任务，只对`swift/self-cognition`数据集生效，替换掉数据集中的`{{AUTHOR}}`通配符。传入模型作者的中文名和英文名，以空格分隔，例如：`--model_author '魔搭' 'ModelScope'`。默认为None。\n- custom_dataset_info: 自定义数据集注册的json文件路径，参考[自定义数据集](../Customization/Custom-dataset.md)和[内置'dataset_info.json'文件](https://github.com/modelscope/ms-swift/blob/main/swift/dataset/data/dataset_info.json)。默认为`[]`。\n\n### 模板参数\n- 🔥template: 对话模板类型。默认为None，自动选择对应model的template类型，对应关系参考[支持的模型列表](./Supported-models-and-datasets.md)。\n- 🔥system: 自定义system字段，可以传入字符串或者**txt文件路径**。默认为None，使用注册template时的默认system。\n - 注意：数据集中的system**优先级**最高，然后是`--system`，最后是注册template时设置的`default_system`。\n- 🔥max_length: 限制单数据集样本经过`tokenizer.encode`后的tokens最大长度，超过的数据样本会根据`truncation_strategy`参数进行处理（避免训练OOM）。默认为None，即设置为模型支持的tokens最大长度(max_model_len)。\n - 当PPO、GRPO、GKD和推理情况下，`max_length`代表`max_prompt_length`。\n- truncation_strategy: 如果单样本的tokens超过`max_length`如何处理，支持'delete'、'left'、'right'和'split'，代表删除、左侧裁剪、右侧裁剪和切成多条数据样本，默认为'delete'。\n - 注意：`--truncation_strategy split`只支持预训练时使用，即`swift/megatron pt`场景下，该策略会将超长字段切成多条数据样本，从而避免tokens浪费。（该特性不兼容cached_dataset）\n - 注意：若多模态模型的训练时将'truncation_strategy'设置为`left`或`right`，**ms-swift会保留所有的image_token等多模态tokens**，这可能会导致训练时OOM。\n- 🔥max_pixels: 多模态模型输入图片的最大像素数（H\\*W），将超过该限制的图像进行缩放（避免训练OOM）。默认为None，不限制最大像素数。\n - 注意：该参数适用于所有的多模态模型。而Qwen2.5-VL特有的模型参数`MAX_PIXELS`（你可以在文档最下面找到）只针对Qwen2.5-VL模型。\n- 🔥agent_template: Agent模板，确定如何将工具列表'tools'转换成'system'、如何在推理/部署时从模型回复中提取toolcall部分，以及确定'messages'中`{\"role\": \"tool_call\", \"content\": \"xxx\"}`, `{\"role\": \"tool_response\", \"content\": \"xxx\"}`的模板格式。可选为\"react_en\", \"hermes\", \"glm4\", \"qwen_en\", \"toolbench\"等，更多请查看[这里](https://github.com/modelscope/ms-swift/blob/main/swift/agent_template/mapping.py)。默认为None，根据模型类型进行自动选择。可以参考[Agent文档](./Agent-support.md)。\n- norm_bbox: 控制如何缩放边界框（即数据集中的\"bbox\"，里面的数据为绝对坐标，参考[自定义数据集文档](https://swift.readthedocs.io/zh-cn/latest/Customization/Custom-dataset.html#grounding)）。选项为'norm1000'和'none'。'norm1000'表示将bbox坐标缩放至千分位坐标，而'none'表示不进行缩放。默认值为None，将根据模型自动选择。\n - 当**图片在训练中发生缩放时**（例如设置了max_pixels参数），该参数也能很好进行解决。\n- use_chat_template: 使用chat模板还是generation模板（generation模板通常用于预训练时）。默认为`True`。\n - 注意：`swift pt`默认为False，使用generation模板。该参数可以很好的**兼容多模态模型**。\n- padding_side: 当训练`batch_size>=2`时的padding_side，可选值为'left'、'right'，默认为'right'。（推理时的batch_size>=2时，只进行左padding）。\n - 注意：PPO和GKD默认设置为'left'。\n- 🔥padding_free: 将一个batch中的数据进行展平而避免数据padding，从而降低显存占用并加快训练（**同一batch的不同序列之间依旧是不可见的**）。默认为False。当前支持CPT/SFT/DPO/GRPO/KTO/GKD。\n - 注意：使用padding_free请结合`--attn_impl flash_attn`使用且\"transformers>=4.44\"，具体查看[该PR](https://github.com/huggingface/transformers/pull/31629)。（同packing）\n - **相较于packing，padding_free不需要额外的预处理时间，但packing的训练速度更快且显存占用更稳定**。\n- 🔥loss_scale: 训练tokens的loss权重设置。默认为`'default'`。loss_scale包含3种基本策略：'default'、'last_round'、'all'，以及其他策略：'ignore_empty_think'以及agent需要的：'react'、'hermes'、'qwen'、'agentflan'、'alpha_umi'等，可选值参考[loss_scale模块](https://github.com/modelscope/ms-swift/blob/main/swift/loss_scale/mapping.py)。ms-swift 支持了基本策略和其他策略的混用，例如：`'default+ignore_empty_think'`，`'last_round+ignore_empty_think'`。若没有指定基本策略，则默认为'default'，例如：'hermes'与'default+hermes'等价。\n - 'default': 所有response（含history）以权重1计算交叉熵损失（**messages中的system/user/多模态tokens以及Agent训练中`tool_response`部分不计算损失**）。（**SFT默认为该值**）\n - 'last_round': 只计算最后一轮response的损失。最后一轮含义为最后一个\"user\"之后的所有内容。（**RLHF默认为该值**）\n - 'all': 计算所有tokens的损失。（**`swift pt`默认为该值**）\n - 'ignore_empty_think': 忽略空的`'\\n\\n \\n\\n'`损失计算。（满足正则匹配`'\\\\s* \\\\s*'`即可）。\n - 'react', 'hermes', 'qwen': 将`tool_call`部分的loss权重调整为2。\n- sequence_parallel_size: 序列并行大小，默认是1。当前支持CPT/SFT/DPO/GRPO。训练脚本参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/sequence_parallel)。\n- template_backend: 选择template后端，可选为'swift'、'jinja'，默认为'swift'。如果使用jinja，则使用transformers的`apply_chat_template`。\n - 注意：jinja的template后端只支持推理，不支持训练（无法确定损失计算的tokens范围）。\n- response_prefix: response的前缀字符，该参数只在推理时生效。默认为None，根据enable_thinking参数和模版类型确定。\n- enable_thinking: 该参数在推理时生效，代表是否开启thinking模式。默认为None，默认值由模板（模型）类型确定（思考/混合思考模板为True，非思考模板为False）。若enable_thinking为False，则增加非思考前缀，例如Qwen3-8B混合思考模型增加前缀`'\\n\\n \\n\\n'`，Qwen3-8B-Thinking则不增加前缀。若enable_thinking为True，则增加思考前缀，例如`'\\n'`。注意：该参数的优先级低于response_prefix参数。\n - 注意：对于思考模型（思考/混合思考）或显式开启enable_thinking，我们会在推理和训练时，对历史的思考内容进行删除（最后一轮的思考内容保留，即最后一个user信息后的内容）。若训练时的loss_scale基本策略不为last_round，例如为'default'，则不对历史的思考内容进行删除。\n- add_non_thinking_prefix: 该参数只在训练时生效，代表是否对数据样本assistant部分**不以思考标记`''`开头**的数据样本增加非思考前缀（通常混合思考模型含非思考前缀）。该特性可以让swift内置的数据集可以训练混合思考模型。默认值为True。例如：例如Qwen3-8B混合思考模型的非思考前缀为`'\\n\\n \\n\\n'`，Qwen3-8B-Thinking/Instruct的非思考前缀为`''`。注意：训练时，loss_scale的基本策略为last_round，则只对最后一轮做此修改；否则，例如为'default'、'all'，则对每一轮数据做此修改。若设置为False，则不对数据样本增加非思考前缀。\n\n### 生成参数\n参考[generation_config](https://huggingface.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig)文档。\n\n- 🔥max_new_tokens: 推理最大生成新tokens的数量。默认为None，无限制。\n- temperature: 温度参数，温度越高，输出的随机性越大。默认为None，读取'generation_config.json'。\n - 你可以设置`--temperature 0`或者`--top_k 1`以取消推理随机性。\n- top_k: top_k参数，保留概率最高的top_k数量 tokens用于生成，默认为None。读取'generation_config.json'。\n- top_p: top_p参数，保留概率最高的累计概率达到 top_p 的tokens用于生成，默认为None。读取generation_config.json。\n- repetition_penalty: 重复惩罚参数。1.0 表示不进行惩罚。默认为None，读取generation_config.json。\n- num_beams: beam search的并行保留数量，默认为1。\n- 🔥stream: 流式输出，默认为`None`，即使用交互式界面时为True，数据集批量推理时为False。\n- stop_words: 除了eos_token外额外的停止词，默认为`[]`。\n - 注意：eos_token会在输出respsone中被删除，额外停止词会在输出中保留。\n- logprobs: 是否输出logprobs，默认为False。\n- top_logprobs: 输出top_logprobs的数量，默认为None。\n- structured_outputs_regex: 结构化输出（引导解码）的正则表达式模式。设置后，模型生成将被约束为匹配指定的正则表达式模式。仅在`infer_backend`为`vllm`时生效。默认为`None`。\n\n### 量化参数\n以下为加载模型时量化的参数，具体含义可以查看[量化](https://huggingface.co/docs/transformers/main/en/main_classes/quantization)文档。这里不包含`swift export`中涉及的`gptq`、`awq`量化参数。\n\n- 🔥quant_method: 加载模型时采用的量化方法，可选项为'bnb'、'hqq'、'eetq'、'quanto'和'fp8'，默认为None。\n - 若对awq/gptq量化模型进行qlora训练，则不需要设置额外`quant_method`等量化参数。\n- 🔥quant_bits: 量化bits数，默认为None。\n- hqq_axis: hqq量化axis，默认为None。\n- bnb_4bit_compute_dtype: bnb量化计算类型，可选为`float16`、`bfloat16`、`float32`。默认为None，设置为`torch_dtype`。\n- bnb_4bit_quant_type: bnb量化类型，支持`fp4`和`nf4`，默认为`nf4`。\n- bnb_4bit_use_double_quant: 是否使用双重量化，默认为`True`。\n- bnb_4bit_quant_storage: bnb量化存储类型，默认为None。\n\n### RAY参数\n\n- use_ray: boolean类型。是否使用ray，默认为`False`\n- ray_exp_name: ray实验名字，这个字段会用作cluster和worker名称前缀，可以不填\n- device_groups: 字符串（jsonstring）类型。在使用ray时，该字段必须配置，具体可以查看[ray文档](Ray.md)。\n\n### yaml支持\n\n- config: 可以使用config代替命令行参数，例如：\n\n```shell\nswift sft --config demo.yaml\n```\n\ndemo.yaml的内容为具体命令行配置：\n\n```yaml\n# Model args\nmodel: Qwen/Qwen2.5-7B-Instruct\ndataset: swift/self-cognition\n...\n\n# Train args\noutput_dir: xxx/xxx\ngradient_checkpointing: true\n\n...\n```\n\n## 原子参数\n\n### Seq2SeqTrainer参数\n\n该参数列表继承自transformers `Seq2SeqTrainingArguments`，ms-swift对其默认值进行了覆盖。未列出的请参考[HF官方文档](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#transformers.Seq2SeqTrainingArguments)。\n\n- 🔥output_dir: 模型预测结果和检查点将被写入的输出目录。默认为None，设置为`'output/'`。\n- 🔥gradient_checkpointing: 是否使用gradient_checkpointing，默认为True。该参数可以显著降低显存占用，但降低训练速度。\n- 🔥vit_gradient_checkpointing: 多模态模型训练时，是否对vit部分开启gradient_checkpointing。默认为None，即设置为`gradient_checkpointing`。例子参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/vit_gradient_checkpointing.sh)。\n - 注意：多模态模型且是LoRA训练时，当设置了`--freeze_vit false`，且命令行中出现以下警告：`UserWarning: None of the inputs have requires_grad=True. Gradients will be None`，请设置`--vit_gradient_checkpointing false`，或提相关issue。全参数训练则不会出现该问题。（如果RLHF LoRA训练中，ref_model抛出来的警告，则是正常的）\n- 🔥deepspeed: 默认为None。可以设置为'zero0', 'zero1', 'zero2', 'zero3', 'zero2_offload', 'zero3_offload'来使用ms-swift内置的deepspeed配置文件。你也可以传入自定义deepspeed配置文件的路径。\n- zero_hpz_partition_size: 默认为None，这个参数是ZeRO++的特性，即node内模型分片，node间数据分片，如果遇到grad_norm NaN，请尝试使用`--torch_dtype float16`。\n- deepspeed_autotp_size: DeepSpeed张量并行大小，默认为1。使用DeepSpeed AutoTP时需将参数`--deepspeed`设置为'zero0'、'zero1'或'zero2'。（注意：该功能只支持全参数）\n- 🔥fsdp: FSDP2分布式训练配置。默认为None。可以设置为'fsdp2'来使用ms-swift内置的FSDP2配置文件。你也可以传入自定义FSDP配置文件的路径。FSDP2是PyTorch原生的分布式训练方案，与DeepSpeed二选一使用。\n- 🔥per_device_train_batch_size: 默认值1。\n- 🔥per_device_eval_batch_size: 默认值1。\n- 🔥gradient_accumulation_steps: 梯度累加。**默认为None，即设置gradient_accumulation_steps使得total_batch_size>=16**。total_batch_size等于`per_device_train_batch_size * gradient_accumulation_steps * world_size`。在GRPO训练中，默认为1。\n - 在CPT/SFT训练中，梯度累加的训练效果等价使用更大的batch_size，但在RLHF训练中，训练效果并不等价。\n- weight_decay: weight衰减系数，默认值0.1。\n- adam_beta1: Adam系列优化器中一阶矩估计（动量）的指数衰减率。默认为0.9。\n- adam_beta2: Adam系列优化器中二阶矩估计（方差）的指数衰减率。默认为0.95。\n- adam_epsilon: Adam系列优化器中用于数值稳定性的epsilon值。默认为1e-8。\n- 🔥learning_rate: 学习率，**全参数训练默认为1e-5，LoRA训练等tuners为1e-4**。\n - 提示：若要设置`min_lr`，您可以传入参数`--lr_scheduler_type cosine_with_min_lr --lr_scheduler_kwargs '{\"min_lr\": 1e-6}'`。\n- 🔥vit_lr: 当训练多模态大模型时，该参数指定vit的学习率，默认为None，等于learning_rate。通常与`--freeze_vit`、`--freeze_aligner`参数结合使用。\n - 提示：在日志中打印的\"learning_rate\"为`param_groups[0]`的学习率，其中param_groups的顺序依次是vit, aligner, llm（若含可训练参数）。\n- 🔥aligner_lr: 当训练多模态大模型时，该参数指定aligner的学习率，默认为None，等于learning_rate。\n- lr_scheduler_type: lr_scheduler类型，默认为'cosine'。\n- lr_scheduler_kwargs: lr_scheduler其他参数。默认为None。\n- gradient_checkpointing_kwargs: 传入`torch.utils.checkpoint`中的参数。例如设置为`--gradient_checkpointing_kwargs '{\"use_reentrant\": false}'`。默认为None。\n - 注意：当使用DDP而不使用deepspeed/fsdp，且gradient_checkpointing_kwargs为None，会默认设置其为`'{\"use_reentrant\": false}'`而避免出现报错。\n- full_determinism: 确保训练中获得可重现的结果，注意：这会对性能产生负面影响。默认为False。\n- 🔥report_to: 默认值为`tensorboard`。你也可以指定`--report_to tensorboard wandb swanlab`、`--report_to all`。\n - 如果你指定了`--report_to wandb`，你可以通过`WANDB_PROJECT`设置项目名称，`WANDB_API_KEY`指定账户对应的API KEY。\n- logging_first_step: 是否记录第一个step的日志，默认为True。\n- logging_steps: 日志打印间隔，默认为5。\n- router_aux_loss_coef: 用于moe模型训练时，设置 aux_loss 的权重，默认为`0.`。\n- enable_dft_loss: 是否在SFT训练中使用[DFT](https://arxiv.org/abs/2508.05629) (Dynamic Fine-Tuning) loss，默认为False。\n- enable_channel_loss: 启用channel loss，默认为`False`。你需要在数据集中准备\"channel\"字段，ms-swift会根据该字段分组统计loss（若未准备\"channel\"字段，则归为默认`None` channel）。数据集格式参考[channel loss](../Customization/Custom-dataset.md#channel-loss)。channel loss兼容packing/padding_free/loss_scale等技术。\n- safe_serialization: 是否存储为safetensors，默认为True。\n- max_shard_size: 单存储文件最大大小，默认'5GB'。\n- logging_dir: tensorboard日志保存路径。默认为None，即设置为`f'{self.output_dir}/runs'`。\n- predict_with_generate: 验证时使用生成式的方式，默认为False。\n- metric_for_best_model: 默认为None，即当`predict_with_generate`设置为False时，设置为'loss'，否则设置为'rouge-l'（在PPO训练时，不进行默认值设置；GRPO训练设置为'reward'）。\n- greater_is_better: 默认为None，即当`metric_for_best_model`含'loss'时，设置为False，否则设置为True。\n- max_epochs: 训练到`max_epochs`时强制退出训练，并对权重进行验证和保存。该参数在使用流式数据集时很有用。默认为None。\n\n其他重要参数：\n- 🔥num_train_epochs: 训练的epoch数，默认为3。\n- 🔥save_strategy: 保存模型的策略，可选为'no'、'steps'、'epoch'，默认为'steps'。\n- 🔥save_steps: 默认为500。\n- 🔥eval_strategy: 评估策略。默认为None，跟随`save_strategy`的策略。\n - 若不使用`val_dataset`和`eval_dataset`且`split_dataset_ratio`为0，则默认为'no'。\n- 🔥eval_steps: 默认为None，如果存在评估数据集，则跟随`save_steps`的策略。\n- eval_on_start: 是否在训练前执行一次评估步骤，以确保验证步骤能正常工作。默认为False。\n- 🔥save_total_limit: 最多保存的checkpoint数，会将过期的checkpoint进行删除。默认为None，保存所有的checkpoint。若设置为2，则保存best checkpoint和last checkpoint。\n- max_steps: 最大训练的steps数。在数据集为流式时需要被设置。默认为-1。\n- 🔥warmup_ratio: 默认为0.。\n- save_on_each_node: 在每一个节点都进行权重保存。默认为False。该参数在多机训练时需要被考虑。\n - 提示：在多机训练时，通常将`output_dir`设置为节点共享目录，因此无需额外设置该参数。\n- save_only_model: 是否只保存模型权重而不包含优化器状态，随机种子状态等内容，这在全参数训练时可以减少保存的时间消耗和空间占用。默认为False。\n- 🔥resume_from_checkpoint: 断点续训参数，指定checkpoint路径。默认为None。\n - 提示：**断点续训请保持其他参数不变，额外增加`--resume_from_checkpoint checkpoint_dir`**。权重等信息将在trainer中读取。\n - 注意: resume_from_checkpoint会读取模型权重，优化器状态，随机种子，并从上次训练的steps继续开始训练。你可以指定`--resume_only_model`只读取模型权重。\n- resume_only_model: 默认为False。如果在指定resume_from_checkpoint的基础上，将该参数设置为True，则仅resume模型权重，而忽略优化器状态和随机种子。\n - 注意：**resume_only_model默认将进行数据跳过**，此行为可通过 `ignore_data_skip` 参数控制。\n- ignore_data_skip: 当设置`resume_from_checkpoint`和`resume_only_model`时，该参数控制是否跳过已经训练的数据，并将epoch和迭代数等训练状态进行恢复。默认为False。若设置为True，则将不加载训练状态并不进行数据跳过，将从迭代数0开始训练。\n- 🔥ddp_find_unused_parameters: 默认为None。\n- 🔥dataloader_num_workers: 默认为None，若是windows平台，则设置为0，否则设置为1。\n- dataloader_pin_memory: 默认为True。\n- dataloader_persistent_workers: 默认为False。\n- dataloader_prefetch_factor: 默认为None。若 `dataloader_num_workers > 0`，则设置为2。每个工作进程预先加载的批次数量。2 表示所有工作进程总共会预取 2 * num_workers 个批次。\n- train_dataloader_shuffle: CPT/SFT训练的dataloader是否随机，默认为True。该参数对IterableDataset无效（即对流式数据集失效）。IterableDataset采用顺序的方式读取。\n- optim: 优化器，默认值为 `\"adamw_torch\"` (对于 torch>=2.8 为 `\"adamw_torch_fused\"`)。完整的优化器列表请参见 [training_args.py](https://github.com/huggingface/transformers/blob/main/src/transformers/training_args.py) 中的 `OptimizerNames`。\n- optim_args: 提供给优化器的可选参数，默认为None。\n- group_by_length: 是否在训练数据集中将长度大致相同的样本分组在一起（有随机因素），以最小化填充并确保各节点与进程的负载均衡以提高效率。默认为False。具体算法参考`transformers.trainer_pt_utils.get_length_grouped_indices`。\n- 🔥neftune_noise_alpha: neftune添加的噪声系数。默认为0，通常可以设置为5、10、15。\n- 🔥use_liger_kernel: 是否启用[Liger](https://github.com/linkedin/Liger-Kernel)内核加速训练并减少显存消耗。默认为False。示例shell参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/liger)。\n - 注意：liger_kernel不支持device_map，请使用DDP/DeepSpeed进行多卡训练。liger_kernel目前只支持`task_type='causal_lm'`。\n- average_tokens_across_devices: 是否在设备之间进行token数平均。如果设置为True，将使用all_reduce同步`num_tokens_in_batch`以进行精确的损失计算。默认为False。\n- max_grad_norm: 梯度裁剪。默认为1.。\n - 注意：日志中的grad_norm记录的是裁剪前的值。\n- push_to_hub: 推送checkpoint到hub。默认为False。\n- hub_model_id: 默认为None。\n- hub_private_repo: 默认为False。\n\n### Tuner参数\n- 🔥freeze_llm: 该参数只对多模态模型生效，可用于全参数训练和LoRA训练，但会产生不同的效果。若是全参数训练，将freeze_llm设置为True会将LLM部分权重进行冻结；若是LoRA训练且`target_modules`设置为'all-linear'，将freeze_llm设置为True将会取消在LLM部分添加LoRA模块。该参数默认为False。\n- 🔥freeze_vit: 该参数只对多模态模型生效，可用于全参数训练和LoRA训练，但会产生不同的效果。若是全参数训练，将freeze_vit设置为True会将vit部分权重进行冻结；若是LoRA训练且`target_modules`设置为'all-linear'，将freeze_vit设置为True将会取消在vit部分添加LoRA模块。该参数默认为True。\n - 注意：**这里的vit不仅限于vision_tower, 也包括audio_tower**。若是Omni模型，若你只希望对vision_tower加LoRA，而不希望对audio_tower加LoRA，你可以修改[这里的代码](https://github.com/modelscope/ms-swift/blob/a5d4c0a2ce0658cef8332d6c0fa619a52afa26ff/swift/llm/model/model_arch.py#L544-L554)。\n- 🔥freeze_aligner: 该参数只对多模态模型生效，可用于全参数训练和LoRA训练，但会产生不同的效果。若是全参数训练，将freeze_aligner设置为True会将aligner（也称为projector）部分权重进行冻结；若是LoRA训练且`target_modules`设置为'all-linear'，将freeze_aligner设置为True将会取消在aligner部分添加LoRA模块。该参数默认为True。\n- 🔥target_modules: 指定lora模块, 默认为`['all-linear']`。你也可以设置为module的后缀，例如：`--target_modules q_proj k_proj v_proj`。该参数不限于LoRA，可用于其他tuners。\n - 注意：在LLM和多模态LLM中，'all-linear'的行为有所不同。若是LLM则自动寻找除lm_head外的linear并附加tuner；**若是多模态LLM，则默认只在LLM上附加tuner，该行为可以被`freeze_llm`、`freeze_vit`、`freeze_aligner`控制**。\n- 🔥target_regex: 指定lora模块的regex表达式，默认为`None`。如果该值传入，则target_modules参数失效。例如你可以设置`--target_regex '^(language_model).*\\.(q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj)$'`，将符合该正则的模块指定为LoRA模块。该参数不限于LoRA，可用于其他tuners。\n- target_parameters: 要替换为LoRA的参数名称列表。该参数的行为与 `target_modules` 类似，但传入的应是参数名称而不是模块名称。该特性需要安装\"peft>=0.17.0\"。例如，在 Hugging Face Transformers 中许多混合专家（MoE）层中，并未使用 `nn.Linear`，而是使用了 `nn.Parameter`。这时可以使用target_parameters参数实现。\n- init_weights: 初始化weights的方法，LoRA可以指定为`true`、`false`、`gaussian`、`pissa`、`pissa_niter_[number of iters]`，Bone可以指定为`true`、`false`、`bat`。默认值`true`。\n- 🔥modules_to_save: 在已附加tuner后，额外指定一部分原模型模块参与训练和存储。默认为`[]`。该参数不限于LoRA，可用于其他tuners。例如设置为`--modules_to_save embed_tokens lm_head`，在LoRA训练中解开embed_tokens和lm_head层进行训练，这两部分的权重信息最终会保存在`adapter_model.safetensors`中。\n\n#### 全参\n- freeze_parameters: 需要被冻结参数的前缀，默认为`[]`。\n- freeze_parameters_regex: 需要被冻结参数的正则表达式，默认为None。\n- freeze_parameters_ratio: 从下往上冻结的参数比例，默认为0。可设置为1将所有参数冻结，结合`trainable_parameters`设置可训练参数。\n- trainable_parameters: 额外可训练参数的前缀，默认为`[]`。\n- trainable_parameters_regex: 匹配额外可训练参数的正则表达式，默认为None。\n - 备注：`trainable_parameters`、`trainable_parameters_regex`的优先级高于`freeze_parameters`、`freeze_parameters_regex`和`freeze_parameters_ratio`。例如：当指定全参数训练时，会将所有模块设置为可训练的状态，随后根据`freeze_parameters`、`freeze_parameters_regex`、`freeze_parameters_ratio`将部分参数冻结，最后根据`trainable_parameters`、`trainable_parameters_regex`重新打开部分参数参与训练。\n\n#### LoRA\n- 🔥lora_rank: 默认为`8`。\n- 🔥lora_alpha: 默认为`32`。\n- lora_dropout: 默认为`0.05`。\n- lora_bias: 默认为`'none'`，可以选择的值: 'none'、'all'。如果你要将bias全都设置为可训练，你可以设置为`'all'`。\n- lora_dtype: 指定lora模块的dtype类型。支持'float16'、'bfloat16'、'float32'。默认为None，跟随peft行为。\n- 🔥use_dora: 默认为`False`，是否使用`DoRA`。\n- use_rslora: 默认为`False`，是否使用`RS-LoRA`。\n- 🔥lorap_lr_ratio: LoRA+参数，默认值`None`，建议值为`10~16`。使用lora时额外指定该参数可使用lora+。\n\n##### LoRA-GA\n- lora_ga_batch_size: 默认值为 `2`。在 LoRA-GA 中估计梯度以进行初始化时使用的批处理大小。\n- lora_ga_iters: 默认值为 `2`。在 LoRA-GA 中估计梯度以进行初始化时的迭代次数。\n- lora_ga_max_length: 默认值为 `1024`。在 LoRA-GA 中估计梯度以进行初始化时的最大输入长度。\n- lora_ga_direction: 默认值为 `ArB2r`。在 LoRA-GA 中使用估计梯度进行初始化时的初始方向。允许的值有：`ArBr`、`A2rBr`、`ArB2r` 和 `random`。\n- lora_ga_scale: 默认值为 `stable`。LoRA-GA 的初始化缩放方式。允许的值有：`gd`、`unit`、`stable` 和 `weightS`。\n- lora_ga_stable_gamma: 默认值为 `16`。当初始化时选择 `stable` 缩放时的 gamma 值。\n\n#### FourierFt\n\nFourierFt使用`target_modules`、`target_regex`、`modules_to_save`三个参数，含义见上面文档中的描述。额外参数包括：\n\n- fourier_n_frequency: 傅里叶变换的频率数量, `int`类型, 类似于LoRA中的`r`. 默认值`2000`.\n- fourier_scaling: W矩阵的缩放值, `float`类型, 类似LoRA中的`lora_alpha`. 默认值`300.0`.\n\n#### BOFT\n\nBOFT使用`target_modules`、`target_regex`、`modules_to_save`三个参数，含义见上面文档中的描述。额外参数包括：\n\n- boft_block_size: BOFT块尺寸, 默认值4.\n- boft_block_num: BOFT块数量, 不能和`boft_block_size`同时使用.\n- boft_dropout: boft的dropout值, 默认0.0.\n\n#### Vera\n\nVera使用`target_modules`、`target_regex`、`modules_to_save`三个参数，含义见上面文档中的描述。额外参数包括：\n\n- vera_rank: Vera Attention的尺寸, 默认值256.\n- vera_projection_prng_key: 是否存储Vera映射矩阵, 默认为True.\n- vera_dropout: Vera的dropout值, 默认`0.0`.\n- vera_d_initial: Vera的d矩阵的初始值, 默认`0.1`.\n\n#### GaLore\n\n- 🔥use_galore: 默认值False, 是否使用GaLore.\n- galore_target_modules: 默认值None, 不传的情况下对attention和mlp应用GaLore.\n- galore_rank: 默认值128, GaLore的rank值.\n- galore_update_proj_gap: 默认值50, 分解矩阵的更新间隔.\n- galore_scale: 默认值1.0, 矩阵权重系数.\n- galore_proj_type: 默认值`std`, GaLore矩阵分解类型.\n- galore_optim_per_parameter: 默认值False, 是否给每个Galore目标Parameter设定一个单独的optimizer.\n- galore_with_embedding: 默认值False, 是否对embedding应用GaLore.\n- galore_quantization: 是否使用q-galore. 默认值`False`.\n- galore_proj_quant: 是否对SVD分解矩阵做量化, 默认`False`.\n- galore_proj_bits: SVD量化bit数.\n- galore_proj_group_size: SVD量化分组数.\n- galore_cos_threshold: 投影矩阵更新的cos相似度阈值. 默认值0.4.\n- galore_gamma_proj: 在投影矩阵逐渐相似后会拉长更新间隔, 本参数为每次拉长间隔的系数, 默认值2.\n- galore_queue_size: 计算投影矩阵相似度的队列长度, 默认值5.\n\n#### LISA\n\n注意: LISA仅支持全参数，即`--tuner_type full`。\n\n- 🔥lisa_activated_layers: 默认值`0`，代表不使用LISA，改为非0代表需要激活的layers个数，建议设置为2或8。\n- lisa_step_interval: 默认值`20`，多少iter切换可反向传播的layers。\n\n#### UNSLOTH\n\n🔥unsloth无新增参数，对已有参数进行调节即可支持，例如：\n\n```\n--tuner_backend unsloth\n--tuner_type full/lora\n--quant_bits 4\n```\n\n#### LLAMAPRO\n\n- 🔥llamapro_num_new_blocks: 默认值`4`, 插入的新layers总数.\n- llamapro_num_groups: 默认值`None`, 分为多少组插入new_blocks, 如果为`None`则等于`llamapro_num_new_blocks`, 即每个新的layer单独插入原模型.\n\n#### AdaLoRA\n\n以下参数`tuner_type`设置为`adalora`时生效. adalora的`target_modules`等参数继承于lora的对应参数，但`lora_dtype`参数不生效。\n\n- adalora_target_r: 默认值`8`, adalora的平均rank.\n- adalora_init_r: 默认值`12`, adalora的初始rank.\n- adalora_tinit: 默认值`0`, adalora的初始warmup.\n- adalora_tfinal: 默认值`0`, adalora的final warmup.\n- adalora_deltaT: 默认值`1`, adalora的step间隔.\n- adalora_beta1: 默认值`0.85`, adalora的EMA参数.\n- adalora_beta2: 默认值`0.85`, adalora的EMA参数.\n- adalora_orth_reg_weight: 默认值`0.5`, adalora的正则化参数.\n\n#### ReFT\n\n以下参数`tuner_type`设置为`reft`时生效.\n\n> 1. ReFT无法合并tuner\n> 2. ReFT和gradient_checkpointing不兼容\n> 3. 如果使用DeepSpeed遇到问题请暂时卸载DeepSpeed\n\n- 🔥reft_layers: ReFT应用于哪些层上, 默认为`None`, 代表所有层, 可以输入层号的list, 例如reft_layers 1 2 3 4`\n- 🔥reft_rank: ReFT矩阵的rank, 默认为`4`.\n- reft_intervention_type: ReFT的类型, 支持'NoreftIntervention', 'LoreftIntervention', 'ConsreftIntervention', 'LobireftIntervention', 'DireftIntervention', 'NodireftIntervention', 默认为`LoreftIntervention`.\n- reft_args: ReFT Intervention中的其他支持参数, 以json-string格式输入.\n\n### vLLM参数\n参数含义可以查看[vllm文档](https://docs.vllm.ai/en/latest/serving/engine_args.html)。\n\n- 🔥vllm_gpu_memory_utilization: GPU内存比例，取值范围为0到1。默认值`0.9`。\n- 🔥vllm_tensor_parallel_size: tp并行数，默认为`1`。\n- vllm_pipeline_parallel_size: pp并行数，默认为`1`。\n- vllm_data_parallel_size: dp并行数，默认为`1`，在`swift deploy/rollout`命令中生效。\n - 若在`swift infer`中，使用`NPROC_PER_NODE`来设置dp并行数。参考这里的[例子](https://github.com/modelscope/ms-swift/blob/main/examples/infer/vllm/mllm_ddp.sh)。\n- vllm_enable_expert_parallel: 开启专家并行，默认为False。\n- vllm_max_num_seqs: 单次迭代中处理的最大序列数，默认为`256`。\n- 🔥vllm_max_model_len: 模型支持的最大长度。默认为`None`，即从config.json中读取。\n- vllm_disable_custom_all_reduce: 禁用自定义的 all-reduce 内核，回退到 NCCL。为了稳定性，默认为`True`。\n- vllm_enforce_eager: vllm使用pytorch eager模式还是建立cuda graph，默认为`False`。设置为True可以节约显存，但会影响效率。\n- vllm_mm_processor_cache_gb: 多模态处理器缓存大小（GiB），用于缓存已处理的多模态输入（如图像、视频）避免重复处理。默认为`4`。设置为`0`可禁用缓存但会降低性能（不推荐）。仅对多模态模型生效。\n- vllm_speculative_config: 推测解码配置，传入json字符串。默认为None。\n- vllm_disable_cascade_attn: 是否强制关闭V1引擎的cascade attention实现以防止潜在数值误差，默认为False，由vLLM内部逻辑决定是否使用。\n- 🔥vllm_limit_mm_per_prompt: 控制vllm使用多图，默认为`None`。例如传入`--vllm_limit_mm_per_prompt '{\"image\": 5, \"video\": 2}'`。\n- vllm_max_lora_rank: 默认为`16`。vllm对于lora支持的参数。\n- vllm_quantization: vllm可以在内部量化模型，参数支持的值详见[这里](https://docs.vllm.ai/en/latest/serving/engine_args.html)。\n- 🔥vllm_enable_prefix_caching: 开启vllm的自动前缀缓存，节约重复查询前缀的处理时间，加快推理效率。默认为`None`，跟随vLLM行为。\n- vllm_use_async_engine: vLLM backend下是否使用async engine。默认为None，会根据场景自动设置：encode任务（embedding、seq_cls、reranker、generative_reranker）默认为True，部署场景（swift deploy）默认为True，其他情况默认为False。注意：encode任务需使用async engine。\n- vllm_reasoning_parser: 推理解析器类型，用于思考模型的思维链内容解析。默认为`None`。仅用于 `swift deploy` 命令。可选的种类参考[vLLM文档](https://docs.vllm.ai/en/latest/features/reasoning_outputs.html#streaming-chat-completions)。\n- vllm_engine_kwargs: vllm的额外参数，格式为json字符串。默认为None。\n\n### SGLang参数\n参数含义可以查看[sglang文档](https://docs.sglang.ai/backend/server_arguments.html)。\n\n- 🔥sglang_tp_size: tp数。默认为1。\n- sglang_pp_size: pp数。默认为1。\n- sglang_dp_size: dp数。默认为1。\n- sglang_ep_size: ep数。默认为1。\n- sglang_enable_ep_moe: 是否启用ep moe。默认为False。该参数已在最新sglang中移除。\n- sglang_mem_fraction_static: 用于静态分配模型权重和KV缓存内存池的GPU内存比例。如果你遇到GPU内存不足错误，可以尝试降低该值。默认为None。\n- sglang_context_length: 模型的最大上下文长度。默认为 None，将使用模型的`config.json`中的值。\n- sglang_disable_cuda_graph: 禁用CUDA图。默认为False。\n- sglang_quantization: 量化方法。默认为None。\n- sglang_kv_cache_dtype: 用于k/v缓存存储的数据类型。'auto'表示将使用模型的数据类型。'fp8_e5m2'和'fp8_e4m3'适用于CUDA 11.8及以上版本。默认为'auto'。\n- sglang_enable_dp_attention: 为注意力机制启用数据并行，为前馈网络（FFN）启用张量并行。数据并行的规模（dp size）应等于张量并行的规模（tp size）。目前支持DeepSeek-V2/3以及Qwen2/3 MoE模型。默认为False。\n- sglang_disable_custom_all_reduce: 禁用自定义的 all-reduce 内核，回退到 NCCL。为了稳定性，默认为True。\n- sglang_speculative_algorithm: 推测算法，可选值：None、\"EAGLE\"、\"EAGLE3\"、\"NEXTN\"、\"STANDALONE\"、\"NGRAM\"。默认为None。\n- sglang_speculative_num_steps: 在推测解码中从草稿模型采样的步数。默认值为None。\n- sglang_speculative_eagle_topk: 在 EAGLE2 算法中每步从草稿模型采样的 token 数量。默认值为None。\n- sglang_speculative_num_draft_tokens: 在推测解码中从草稿模型采样的 token 数量。默认值为None。\n\n### LMDeploy参数\n参数含义可以查看[lmdeploy文档](https://lmdeploy.readthedocs.io/en/latest/api/pipeline.html#turbomindengineconfig)。\n\n- 🔥lmdeploy_tp: tensor并行度。默认为`1`。\n- lmdeploy_session_len: 最大会话长度。默认为`None`。\n- lmdeploy_cache_max_entry_count: k/v缓存占用的GPU内存百分比。默认为`0.8`。\n- lmdeploy_quant_policy: 默认为0。当需要将k/v量化为4或8位时，分别将其设置为4或8。\n- lmdeploy_vision_batch_size: 传入VisionConfig的max_batch_size参数。默认为`1`。\n\n### 合并参数\n\n- 🔥merge_lora: 是否合并lora，本参数支持lora、llamapro、longlora，默认为False。例子参数[这里](https://github.com/modelscope/ms-swift/blob/main/examples/export/merge_lora.sh)。\n- safe_serialization: 是否存储为safetensors，默认为True。\n- max_shard_size: 单存储文件最大大小，默认'5GB'。\n\n\n## 集成参数\n\n### 训练参数\n训练参数除包含[基本参数](#基本参数)、[Seq2SeqTrainer参数](#Seq2SeqTrainer参数)、[tuner参数](#tuner参数)外，还包含下面的部分:\n\n- add_version: 在`output_dir`上额外增加目录`'<版本号>-<时间戳>'`防止权重覆盖，默认为True。\n- check_model: 检查本地模型文件有损坏或修改并给出提示，默认为True。**如果是断网环境，请设置为False**。\n- 🔥create_checkpoint_symlink: 额外创建checkpoint软链接，方便书写自动化训练脚本。best_model和last_model的软链接路径分别为f'{output_dir}/best'和f'{output_dir}/last'。\n- 🔥packing: 使用`padding_free`的方式将不同长度的数据样本打包成**近似**统一长度的样本（packing能保证不对完整的序列进行切分），实现训练时各节点与进程的负载均衡（避免长文本拖慢短文本的训练速度），从而提高GPU利用率，保持显存占用稳定。当使用 `--attn_impl flash_attn` 时，可确保packed样本内的不同序列之间相互独立，互不可见。该参数默认为`False`，目前支持 CPT/SFT/DPO/KTO/GKD以及embedding/reranker/seq_cls任务的packing。注意：**packing会导致数据集样本数减少，请自行调节梯度累加数和学习率**。\n- packing_length: packing的长度。默认为None，设置为max_length。\n- packing_num_proc: packing的进程数，默认为1。需要注意的是，不同的`packing_num_proc`，最终形成的packed数据集是不同的。（该参数在流式packing时不生效）。通常不需要修改该值，packing速度远快于tokenize速度。\n- lazy_tokenize: 是否使用lazy_tokenize。若该参数设置为False，则在训练之前对所有的数据集样本进行tokenize（多模态模型则包括从磁盘中读取图片）。该参数默认为None，在LLM训练中默认为False，而MLLM训练默认为True，节约内存。\n - 注意：若你要进行图像的数据增强，你需要将lazy_tokenize（或streaming）设置为True，并修改Template类中的encode方法。\n- use_logits_to_keep: 通过在`forward`中根据labels传入logits_to_keep，减少无效logits的计算与存储，从而减少显存占用并加快训练速度。默认为None，进行自动选择。\n- acc_strategy: 训练和验证时计算acc的策略。可选为`seq`和`token`级别的acc，默认为`token`。\n- max_new_tokens: 覆盖生成参数。predict_with_generate=True时的最大生成token数量，默认64。\n- temperature: 覆盖生成参数。predict_with_generate=True时的temperature，默认0。\n- optimizer: 使用的optimizers插件（优先级高于`--optim`），默认为None。可选optimizers参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/optimizers/mapping.py)。\n- loss_type: 自定义的loss_type名称。默认为None，使用模型自带损失函数。可选loss参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/loss/mapping.py)。\n- eval_metric: 自定义eval metric名称。默认为None。可选eval_metric参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/eval_metric/mapping.py)。\n - 关于默认值：当`task_type`为'causal_lm', 且`predict_with_generate=True`的情况下默认设置为'nlg'。`task_type` 为'embedding'，根据loss_type，默认值为'infonce' 或 'paired'。`task_type`为'reranker/generative_reranker'，默认值为'reranker'。\n- callbacks: 自定义trainer callback，默认为`[]`。可选callbacks参考[这里](https://github.com/modelscope/ms-swift/blob/main/swift/callbacks/mapping.py)。例如：通过在`callbacks`中添加`deepspeed_elastic`（可选`graceful_exit`）可以来启用弹性训练。参考[Elastic示例](../BestPractices/Elastic.md)\n- early_stop_interval: 早停的间隔，会检验best_metric在early_stop_interval个周期内（基于`save_steps`, 建议`eval_steps`和`save_steps`设为同值）没有提升时终止训练。具体代码在[early_stop.py](https://github.com/modelscope/ms-swift/blob/main/swift/callbacks/early_stop.py)中。同时，如果有较为复杂的早停需求，直接覆盖callback.py中的已有实现即可。设置该参数时，自动加入`early_stop`的trainer callback。\n- eval_use_evalscope: 是否使用evalscope进行训练时评测，需要设置该参数来开启评测，具体使用参考[示例](../Instruction/Evaluation.md#训练中评测)。\n- eval_dataset: 评测数据集，可设置多个数据集，用空格分割。\n- eval_dataset_args: 评测数据集参数，json格式，可设置多个数据集的参数。\n- eval_limit: 评测数据集采样数。\n- eval_generation_config: 评测时模型推理配置，json格式，默认为`{'max_tokens': 512}`。\n- use_flash_ckpt: 是否启用[DLRover Flash Checkpoint](https://github.com/intelligent-machine-learning/dlrover)的flash checkpoint。默认为`false`，启用后，权重会先保存至共享内存，之后异步持久化；建议搭配`PYTORCH_CUDA_ALLOC_CONF=\"expandable_segments:True\"` 一起使用，避免训练过程CUDA OOM。\n\n#### SWANLAB\n\n- swanlab_token: SwanLab的api-key。你也可以使用`SWANLAB_API_KEY`环境变量指定。\n- swanlab_project: swanlab的project，可以在页面中预先创建[https://swanlab.cn/space/~](https://swanlab.cn/space/~)或自动创建，默认为\"ms-swift\"。\n- swanlab_workspace: 默认为None，会使用api-key对应的username。\n- swanlab_exp_name: 实验名，可以为空，为空时默认传入--output_dir的值。\n- swanlab_notification_method: 在训练完成/发生错误时，swanlab的通知方式，具体参考[这里](https://docs.swanlab.cn/plugin/notification-dingtalk.html)。支持'dingtalk'、'lark'、'email'、'discord'、'wxwork'、'slack'。\n- swanlab_webhook_url: 默认为None。swanlab的`swanlab_notification_method`对应的 webhook url。\n- swanlab_secret: 默认为None。swanlab的`swanlab_notification_method`对应的 secret。\n- swanlab_mode: 可选cloud和local，云模式或者本地模式。\n\n\n### RLHF参数\nRLHF参数继承于[训练参数](#训练参数)。\n\n- 🔥rlhf_type: 人类对齐算法类型，支持'dpo'、'orpo'、'simpo'、'kto'、'cpo'、'rm'、'ppo'、'grpo'和'gkd'。默认为'dpo'。\n- ref_model: 采用dpo、kto、ppo、grpo算法且使用全参数训练时需要传入。默认为None，设置为`--model`。\n- ref_adapters: 默认为`[]`。若你要使用SFT产生的LoRA权重进行DPO/KTO/GRPO，请在训练时设置`--adapters sft_ckpt --ref_adapters sft_ckpt`。若是此场景的断点续训，则设置`--resume_from_checkpoint rlhf_ckpt --ref_adapters sft_ckpt`。\n- ref_model_type: 同model_type。默认为None。\n- ref_model_revision: 同model_revision。默认为None。\n- 🔥beta: 控制与参考模型偏差程度的参数。beta值越高，表示与参考模型的偏差越小。默认为`None`，使用不同rlhf算法的默认值不同，其中`simpo`算法默认为`2.`，GRPO默认为`0.04`，GKD默认为0.5，其他算法默认为`0.1`。具体参考[文档](./RLHF.md)。\n- label_smoothing: 是否使用DPO smoothing，默认值为`0`。\n- max_completion_length: GRPO/PPO/GKD算法中的最大生成长度，默认为512。\n- 🔥rpo_alpha: 来自[RPO 论文](https://arxiv.org/abs/2404.19733)中的参数，用于控制损失函数中NLL项的权重（即SFT损失），`loss = dpo_loss + rpo_alpha * sft_loss`，论文中推荐设置为`1.`。默认为`None`，即默认不引入sft_loss。\n- ld_alpha: 来自[LD-DPO 论文](https://arxiv.org/abs/2409.06411)，对超出公共前缀部分的logps加权 $\\alpha$ 抑制长度偏好。\n- discopop_tau: 来自 [DiscoPOP 论文](https://arxiv.org/abs/2406.08414)的温度参数 $\\tau$ ，用于缩放 log-ratio。默认值0.05。在 loss_type 为 discopop 时生效。\n- loss_type: 损失类型。默认为None，使用不同的rlhf算法，其默认值不同。\n - DPO: 可选项参考[文档](https://huggingface.co/docs/trl/main/en/dpo_trainer#loss-functions)，支持传入多个值实现混合训练([MPO](https://arxiv.org/abs/2411.10442)), 传入多个值时需要设置参数 loss_weights。默认为`sigmoid`。\n - GRPO: 参考[GRPO参数](#grpo参数)。\n- loss_weights: 在 DPO 训练中设置多个 loss_type 时，用于指定各个损失项的权重。\n- cpo_alpha: CPO/SimPO loss 中 nll loss的系数, 默认为`1.`。\n- simpo_gamma: SimPO算法中的reward margin项，论文建议设置为0.5-1.5，默认为`1.`。\n- desirable_weight: KTO算法中用于抵消 desirable 和 undesirable 数量不均衡的影响，对 desirable 损失按该系数进行加权，默认为`1.`。\n- undesirable_weight: KTO算法中用于抵消 desirable 和 undesirable 数量不均衡的影响，对 undesirable 损失按该系数进行加权，默认为`1.`。\n- center_rewards_coefficient: 用于RM训练。用于激励奖励模型输出均值为零的奖励的系数，具体查看这篇[论文](https://huggingface.co/papers/2312.09244)。推荐值：0.01。\n- loss_scale: 覆盖模板参数。rlhf训练时，默认为'last_round'。\n- temperature: 默认为0.9，该参数将在PPO、GRPO、GKD中使用。\n\n#### GKD参数\n- lmbda: 默认为0.5。该参数在GKD中使用。控制学生数据比例的 lambda 参数（即策略内学生生成输出所占的比例）。若lmbda为0，则不使用学生生成数据。\n- sft_alpha: 默认为0。控制GKD中加入sft_loss的权重。最后的loss为`gkd_loss + sft_alpha * sft_loss`。\n- seq_kd: 默认为False。该参数在GKD中使用。控制是否执行序列级知识蒸馏（Sequence-Level KD）的 seq_kd 参数（可视为对教师模型生成输出的监督式微调）。\n - 注意：你可以提前对数据集内容使用teacher模型进行推理（使用vllm/sglang/lmdeploy等推理引擎加速），并在训练时将`seq_kd`设置为False。或者将`seq_kd`设置为True，在训练时使用teacher模型生成序列（能保证多个epoch生成数据的不同，但效率较慢）。\n- offload_teacher_model: 卸载教师模型以节约显存，只在采样/计算logps时加载，默认为False。\n- truncation_strategy: 用于处理输入长度超过 max_length 的样本，支持 delete 和 left 两种策略，分别表示删除该样本和从左侧裁剪。默认值为 left。若使用 delete 策略，被删除的超长样本或编码失败的样本将在原数据集中通过重采样进行替换。\n- log_completions: 是否记录训练中的模型生成内容，搭配 `--report_to wandb/swanlab` 使用。默认为False。\n - 提示：若没有设置`--report_to wandb/swanlab`，则会在checkpoint中创建`completions.jsonl`来存储生成内容。\n - 仅记录 vLLM 采样结果。\n\n#### Reward/Teacher模型参数\nreward模型参数将在PPO、GRPO中使用。\n\n- reward_model: 默认为None。\n- reward_adapters: 默认为`[]`。\n- reward_model_type: 默认为None。\n- reward_model_revision: 默认为None。\n- teacher_model: 默认为None。rlhf_type为'gkd'时需传入此参数。\n- teacher_adapters: 默认为`[]`。\n- teacher_model_type: 默认为None。\n- teacher_model_revision: 默认为None。\n- teacher_deepspeed: 同 deepspeed 参数，控制 teacher model 的 deepspeed 配置，默认使用训练模型的 deepspeed 配置。\n\n#### PPO参数\n\n以下参数含义可以参考[这里](https://huggingface.co/docs/trl/main/ppo_trainer)。\n- num_ppo_epochs: 默认为4。\n- whiten_rewards: 默认为False。\n- kl_coef: 默认为0.05。\n- cliprange: 默认为0.2。\n- vf_coef: 默认为0.1。\n- cliprange_value: 默认为0.2。\n- gamma: 默认为1.0。\n- lam: 默认为0.95。\n- num_mini_batches: 默认为1。\n- local_rollout_forward_batch_size: 默认为64。\n- num_sample_generations: 默认为10。\n- missing_eos_penalty: 默认为None。\n\n\n#### GRPO参数\n- beta: KL正则系数，默认为0.04，设置为0时不加载ref model。\n- per_device_train_batch_size: 每个设备训练批量大小，在GRPO中，指 completion 的批次大小。\n- per_device_eval_batch_size: 每个设备评估批量大小，在GRPO中，指 completion 的批次大小。\n- steps_per_generation: 每轮生成的优化步数，默认等于 gradient_accumulation_steps。与 generation_batch_size 只能同时设置一个。\n- generation_batch_size: 总的采样 completion 批量大小，需要是 num_processes * per_device_train_batch_size 的倍数，默认等于 per_device_train_batch_size * steps_per_generation * num_processes。\n- num_generations: 每个prompt采样的数量，论文中的G值，generation_batch_size 必须能被 num_generations 整除。默认为 8。\n- num_generations_eval: 评估阶段每个prompt采样的数量。允许在评估时使用较少的生成数量以节省计算资源。如果为 None，则使用 num_generations 的值。默认为 None。\n- ds3_gather_for_generation: 该参数适用于DeepSpeed ZeRO-3。如果启用，策略模型权重将被收集用于生成，从而提高生成速度。然而，禁用此选项允许训练超出单个GPU VRAM的模型，尽管生成速度会变慢。禁用此选项与vLLM生成不兼容。默认为True。\n- reward_funcs: GRPO算法奖励函数，可选项为`accuracy`、`format`、`cosine`、`repetition`和`soft_overlong`，见swift/rewards/orm.py。你也可以在plugin中自定义自己的奖励函数。默认为`[]`。\n- reward_weights: 每个奖励函数的权重。必须与奖励函数和奖励模型的总数量匹配。如果为 None，则所有奖励的权重都相等，为`1.0`。\n - 提示：如果GRPO训练中包含`--reward_model`，则其加在奖励函数的最后位置。\n- reward_model_plugin: 奖励模型逻辑，默认为orm逻辑, 详细见[自定义奖励模型](./GRPO/DeveloperGuide/reward_model.md#自定义奖励模型)。\n- dataset_shuffle: 是否对dataset进行随机操作，默认为True。\n- truncation_strategy: 用于处理输入长度超过 max_length 的样本，支持 delete 和 left 两种策略，分别表示删除该样本和从左侧裁剪。默认值为 left。若使用 delete 策略，被删除的超长样本或编码失败的样本将在原数据集中通过重采样进行替换。\n- loss_type: loss 归一化的类型，可选项为['grpo', 'bnpo', 'dr_grpo', 'dapo', 'cispo', 'sapo'], 默认为'grpo', 具体参考[文档](./GRPO/DeveloperGuide/loss_types.md)\n- log_completions: 是否记录训练中的模型生成内容，搭配 `--report_to wandb/swanlab` 使用。默认为False。\n - 提示：若没有设置`--report_to wandb/swanlab`，则会在checkpoint中创建`completions.jsonl`来存储生成内容。\n- use_vllm: 是否使用 vLLM 作为 GRPO 生成的 infer_backend，默认为False。\n- vllm_mode: vLLM 集成模式，可选项为 `server` 和 `colocate`。server 模式使用 `swift rollout` 拉起的 vLLM 服务器进行采样，colocate 模式在程序内部署 vLLM。使用server端时，\n- vllm_mode server 参数\n - vllm_server_host: vLLM server host地址，默认为None。\n - vllm_server_port: vLLM server 服务端口，默认为8000。\n - vllm_server_base_url: vLLM server的Base URL(比如 http://local_host:8000), 默认为None。设置后，忽略host和port设置。\n - vllm_server_group_port: vllm server 内部通信端口，除非端口被占用，一般无需设置，默认为51216。\n - vllm_server_timeout: 连接vLLM server的超时时间，默认为 240s。\n - vllm_server_pass_dataset: 透传额外的数据集信息到vLLM server，用于多轮训练。\n - async_generate: 异步rollout以提高训练速度，注意开启时采样会使用上一轮更新的模型进行采样，不支持多轮场景。默认`false`.\n - enable_flattened_weight_sync: 是否使用 flattened tensor 进行权重同步。启用后会将多个参数打包为单个连续 tensor 进行传输，可提升同步效率，在 Server Mode 下生效，默认为 True。\n - SWIFT_UPDATE_WEIGHTS_BUCKET_SIZE: 环境变量，用于控制flattened tensor 权重同步时的传输桶大小（bucket size），适用于 Server Mode 下的全参数训练，单位为 MB，默认值为 512 MB。\n- vllm_mode colocate 参数（更多参数支持参考[vLLM参数](#vLLM参数)。）\n - vllm_gpu_memory_utilization: vllm透传参数，默认为0.9。\n - vllm_max_model_len: vllm透传参数，默认为None。\n - vllm_enforce_eager: vllm透传参数，默认为False。\n - vllm_limit_mm_per_prompt: vllm透传参数，默认为None。\n - vllm_enable_prefix_caching: vllm透传参数，默认为True。\n - vllm_tensor_parallel_size: tp并行数，默认为`1`。\n - vllm_enable_lora: 支持vLLM Engine 加载 LoRA adapter，默认为False。用于加速LoRA训练的权重同步，具体参考[文档](./GRPO/GetStarted/GRPO.md#权重同步加速)。\n - sleep_level: 训练时释放 vLLM 显存，可选项为[0, 1, 2], 默认为0，不释放。\n - offload_optimizer: 是否在vLLM推理时offload optimizer参数，默认为False。\n - offload_model: 是否在vLLM推理时 offload 模型，默认为False。\n - completion_length_limit_scope: 在多轮对话中，`max_completion_length` 的限制范围。\n `total`限制所有对话轮次的总输出长度不超过`max_completion_length`, `per_round`限制每一轮的输出长度。\n- num_iterations: 每条数据的更新次数，[GRPO论文](https://arxiv.org/abs/2402.03300)中的 $\\mu$ 值，默认为1。\n- epsilon: clip 系数，默认为0.2。\n- epsilon_high: upper clip 系数，默认为None，设置后与epsilon共同构成[epsilon, epsilon_high]裁剪范围。\n- tau_pos: [SAPO](https://arxiv.org/abs/2511.20347)算法中正优势的温度参数，控制软门控函数的锐度。较大值使门控更锐利（接近硬裁剪），较小值使门控更平滑。默认为1.0。\n- tau_neg: SAPO算法中负优势的温度参数，控制软门控函数的锐度。通常设置`tau_neg > tau_pos`以对负优势施加更强约束。默认为1.05。\n- dynamic_sample: 筛除group内奖励标准差为0的数据，额外采样新数据，默认为False。\n- max_resample_times: dynamic_sample设置下限制重采样次数，默认3次。\n- overlong_filter: 跳过超长截断的样本，不参与loss计算，默认为False。\n- delta: [INTELLECT-2 tech report](https://huggingface.co/papers/2505.07291)中双侧 GRPO 上界裁剪值。若设置，建议大于 1 + epsilon。默认为None。\n- importance_sampling_level: 控制重要性采样比计算，可选项为 `token` 和 `sequence`，`token` 模式下保留原始的每个 token 的对数概率比，`sequence` 模式下则会对序列中所有有效 token 的对数概率比进行平均。[GSPO论文](https://arxiv.org/abs/2507.18071)中使用sequence级别计算来稳定训练，默认为`token`。\n- advantage_estimator: 优势计算函数，默认为 `grpo`，即计算组内相对优势，可选项为 `grpo`、[`rloo`](./GRPO/AdvancedResearch/RLOO.md)、[`reinforce_plus_plus`](./GRPO/AdvancedResearch/REINFORCEPP.md)。\n- kl_in_reward: 控制 KL 散度正则项的处理位置；`false`表示作为损失函数的独立正则项，`true`表示将 KL 直接并入奖励（从奖励中扣除）。默认情况与advantage_estimator绑定，`grpo`下默认为`false`，`rloo` 和 `reinforce_plus_plus` 下默认为 `true`。\n- scale_rewards: 指定奖励的缩放策略。可选值包括 `group`（按组内标准差缩放）、`batch`（按整个批次的标准差缩放）、`none`（不进行缩放）、`gdpo`（对每个奖励函数分别进行组内归一化后加权聚合，参考 [GDPO 论文](https://arxiv.org/abs/2601.05242)）。在 ms-swift < 3.10 版本中，该参数为布尔类型，`true` 对应 `group`，`false` 对应 `none`。默认值与 `advantage_estimator` 绑定：`grpo` 对应 `group`，`rloo` 对应 `none`，`reinforce_plus_plus` 对应 `batch`。\n - 注意：`gdpo` 模式不支持 `kl_in_reward=True`，若同时设置会自动将 `kl_in_reward` 设为 `False`。\n - GDPO 适用于多奖励优化场景：当使用多个奖励函数时，GDPO 会对每个奖励函数分别在组内进行标准化（减均值、除标准差），然后使用 `reward_weights` 进行加权求和，最后再进行批次级别的标准化。这种方式可以更好地保留各个奖励的相对差异，避免不同奖励组合坍塌成相同的 advantage 值。\n- sync_ref_model: 是否定期同步ref_model，默认为False。\n - ref_model_mixup_alpha: 控制在更新过程中model和先前ref_model之间的混合。更新公式为 $π_{ref} = α * π_θ + (1 - α) * π_{ref_{prev}}$。默认为0.6。\n - ref_model_sync_steps: 同步频率，默认为512。\n- move_model_batches: 在模型向vLLM等快速推理框架移动参数时，将layers分为多少个batch. 默认为None, 代表整个模型不进行拆分，否则拆分为move_model_batches+1(非layer参数)+1(多模态部分参数)个。\n- multi_turn_scheduler: 多轮GRPO参数, 传入对应的plugin名称, 同时在plugin/multi_turn.py中添加好对应的实现。\n- max_turns: 多轮GRPO的轮数上限。默认为None，不做限制。\n- top_entropy_quantile: 仅对熵值处于前指定分位的 token 参与损失计算，默认为1.0，即不过滤低熵 token，具体参考[文档](./GRPO/AdvancedResearch/entropy_mask.md)\n- log_entropy: 记录训练中的熵值变化动态，默认为False，具体参考[文档](./GRPO/GetStarted/GRPO.md#logged-metrics)\n- rollout_importance_sampling_mode: 训推不一致校正模式，可选项为 `token_truncate`、`token_mask`、`sequence_truncate`、`sequence_mask`。默认为None，不启用校正。具体参考[文档](./GRPO/AdvancedResearch/training_inference_mismatch.md)\n- rollout_importance_sampling_threshold: 重要性采样权重的阈值，用于截断或屏蔽极端权重。默认为2.0。\n- log_rollout_offpolicy_metrics: 当 `rollout_importance_sampling_mode` 未设置时，是否记录训推不一致诊断指标（KL、PPL、χ²等）。当设置了 `rollout_importance_sampling_mode` 时，指标会自动记录。默认为False。\n- off_policy_sequence_mask_delta: Off-Policy Sequence Masking 阈值，来自 [DeepSeek-V3.2 论文](https://arxiv.org/abs/2512.02556)。当设置此值时，会计算每个序列的 `mean(old_policy_logps - policy_logps)`，若该值大于阈值且该序列的优势为负，则 mask 掉该序列不参与损失计算。具体参考[文档](./GRPO/AdvancedResearch/training_inference_mismatch.md#off-policy-sequence-masking)\n\n##### 奖励函数参数\n内置的奖励函数参考[文档](./GRPO/DeveloperGuide/reward_function.md)\ncosine 奖励参数\n- cosine_min_len_value_wrong: cosine 奖励函数参数，生成错误答案时，最小长度对应的奖励值。默认值为-0.5。\n- cosine_max_len_value_wrong: 生成错误答案时，最大长度对应的奖励值。默认值为0.0。\n- cosine_min_len_value_correct: 生成正确答案时，最小长度对应的奖励值。默认值为1.0。\n- cosine_max_len_value_correct: 生成正确答案时，最大长度对应的奖励值。默认值为0.5。\n- cosine_max_len: 生成文本的最大长度限制。默认等于 max_completion_length。\n\nrepetition 奖励参数\n- repetition_n_grams: 用于检测重复的 n-gram 大小。默认值为3。\n- repetition_max_penalty: 最大惩罚值，用于控制惩罚的强度。默认值为-1.0。\n\nsoft overlong 奖励参数\n- soft_max_length: 论文中的L_max，模型的最大生成长度，默认等于max_completion_length。\n- soft_cache_length: 论文中的L_cache，控制长度惩罚区间，区间为[soft_max_length-soft_cache_length, soft_max_length]。\n\n### 推理参数\n\n推理参数除包含[基本参数](#基本参数)、[合并参数](#合并参数)、[vLLM参数](#vllm参数)、[LMDeploy参数](#LMDeploy参数)外，还包含下面的部分：\n\n- 🔥infer_backend: 推理加速后端，支持'transformers'、'vllm'、'sglang'、'lmdeploy'四种推理引擎。默认为'transformers'。\n - 注意：这四种引擎使用的都是swift的template，使用`--template_backend`控制。\n- 🔥max_batch_size: 指定infer_backend为'transformers'时生效，用于批量推理，默认为1。若设置为-1，则不受限制。\n- 🔥result_path: 推理结果存储路径（jsonl），默认为None。如果对数据集进行推理/评测，则默认保存在checkpoint目录（含args.json文件）或者'./result'目录，最终存储路径会在命令行中打印（交互式推理或部署默认不存储结果）。\n - 注意：若已存在`result_path`文件，则会进行追加写入。\n- write_batch_size: 结果写入`result_path`的batch_size。默认为1000。若设置为-1，则不受限制。\n- metric: 对推理的结果进行评估，目前支持'acc'和'rouge'。默认为None，即不进行评估。\n- val_dataset_sample: 推理数据集采样数，默认为None。\n- reranker_use_activation: 在reranker推理时，是否在score之后使用sigmoid，默认为True。\n\n\n### 部署参数\n\n部署参数继承于[推理参数](#推理参数)。\n\n- host: 服务host，默认为'0.0.0.0'。\n- port: 端口号，默认为8000。\n- api_key: 访问需要使用的api_key，默认为None。\n- owned_by: 默认为`swift`。\n- 🔥served_model_name: 提供服务的模型名称，默认使用model的后缀。\n- verbose: 打印详细日志，默认为True。\n - 注意：在`swift app`或者`swift eval`时，默认为False。\n- log_interval: tokens/s统计值打印间隔，默认20秒。设置为-1则不打印。\n- max_logprobs: 最多返回客户端的logprobs数量，默认为20。\n\n### Rollout参数\nRollout参数继承于[部署参数](#部署参数)\n- multi_turn_scheduler: 多轮GRPO训练规划器，传入对应的plugin名称, 同时在plugin/multi_turn.py中添加好对应的实现。默认为None，具体参考[文档](./GRPO/DeveloperGuide/multi_turn.md)。\n- max_turns: 多轮GRPO训练下的最大轮数，默认为None，即不做约束。\n- vllm_enable_lora: 支持vLLM Engine 加载 LoRA adapter，默认为False。用于加速LoRA训练的权重同步，具体参考[文档](./GRPO/GetStarted/GRPO.md#权重同步加速)。\n- vllm_max_lora_rank: vLLM Engine LoRA参数，需大于等于训练的lora_rank，建议等于。默认为16。\n\n### Web-UI参数\n- server_name: web-ui的host，默认为'0.0.0.0'。\n- server_port: web-ui的port，默认为7860。\n- share: 默认为False。\n- lang: web-ui的语言，可选为'zh', 'en'。默认为'zh'。\n\n\n### App参数\n\nApp参数继承于[部署参数](#部署参数), [Web-UI参数](#Web-UI参数)。\n- base_url: 模型部署的base_url，例如`http://localhost:8000/v1`。默认为`None`，使用本地部署。\n- studio_title: studio的标题。默认为None，设置为模型名。\n- is_multimodal: 是否启动多模态版本的app。默认为None，自动根据model判断，若无法判断，设置为False。\n- lang: 覆盖Web-UI参数，默认为'en'。\n\n### 评测参数\n\n评测参数继承于[部署参数](#部署参数)。\n\n- 🔥eval_backend: 评测后端，默认为'Native'，也可以指定为'OpenCompass'或'VLMEvalKit'。\n- 🔥eval_dataset: 评测数据集，请查看[评测文档](./Evaluation.md)。\n- eval_limit: 每个评测集的采样数，默认为None。\n- eval_output_dir: 评测存储结果的文件夹，默认为'eval_output'。\n- temperature: 覆盖生成参数，默认为0。\n- eval_num_proc: 评测时客户端最大并发数，默认为16。\n- eval_url: 评测url，例如`http://localhost:8000/v1`。例子可以查看[这里](https://github.com/modelscope/ms-swift/tree/main/examples/eval/eval_url)。默认为None，采用本地部署评估。\n- eval_generation_config: 评测时模型推理配置，需传入json字符串格式，例如：`'{\"max_new_tokens\": 512}'`；默认为None。\n- extra_eval_args: 额外评测参数，需传入json字符串格式，默认为空。仅对Native评测有效，更多参数说明请查看[这里](https://evalscope.readthedocs.io/zh-cn/latest/get_started/parameters.html)\n- local_dataset: 部分评测集，如`CMB`无法直接运行，需要下载额外数据包才可以使用。设置本参数为`true`可以自动下载全量数据包，并在当前目录下创建`data`文件夹并开始评测。数据包仅会下载一次，后续会使用缓存。该参数默认为`false`。\n - 注意：默认评测会使用`~/.cache/opencompass`下的数据集，在指定本参数后会直接使用当前目录下的data文件夹。\n\n### 导出参数\n\n导出参数除包含[基本参数](#基本参数)和[合并参数](#合并参数)外，还包含下面的部分:\n\n- 🔥output_dir: 导出结果存储路径。默认为None，会自动设置合适后缀的路径。\n- exist_ok: 如果output_dir存在，不抛出异常，进行覆盖。默认为False。\n- 🔥quant_method: 可选为'gptq'、'awq'、'bnb'和'fp8'，默认为None。例子参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize)。\n- quant_n_samples: gptq/awq的校验集采样数，默认为256。\n- quant_batch_size: 量化batch_size，默认为1。\n- group_size: 量化group大小，默认为128。\n- to_cached_dataset: 提前对数据集进行tokenize并导出，默认为False。例子参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/cached_dataset)。更多介绍请查看`cached_dataset`。\n - 提示：你可以通过`--split_dataset_ratio`或者`--val_dataset`指定验证集内容。\n- template_mode: 用于支持对`swift rlhf`训练的`cached_dataset`功能。该参数只在`--to_cached_dataset true`时生效。可选项包括: 'train'、'rlhf'和'kto'。其中`swift pt/sft`使用'train'，`swift rlhf --rlhf_type kto`使用'kto'，其他rlhf算法使用'rlhf'。注意：当前'gkd', 'ppo', 'grpo'算法不支持`cached_dataset`功能。默认为'train'。\n- to_ollama: 产生ollama所需的Modelfile文件。默认为False。\n- 🔥to_mcore: HF格式权重转成Megatron格式。默认为False。\n- to_hf: Megatron格式权重转成HF格式。默认为False。\n- mcore_model: mcore格式模型路径。默认为None。\n- mcore_adapter: mcore格式模型的adapter路径，默认为None。\n- thread_count: `--to_mcore true`时的模型切片数。默认为None，根据模型大小自动设置，使得最大分片小于10GB。\n- 🔥offload_bridge: Megatron导出的用于vLLM更新HF格式权重使用CPU主存存放，以降低 GPU 显存占用。默认为 False。\n- 🔥test_convert_precision: 测试HF和Megatron格式权重转换的精度误差。默认为False。\n- test_convert_dtype: 转换精度测试使用的dtype，默认为'float32'。\n- 🔥push_to_hub: 是否推送hub，默认为False。例子参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/export/push_to_hub.sh)。\n- hub_model_id: 推送的model_id，默认为None。\n- hub_private_repo: 是否是private repo，默认为False。\n- commit_message: 提交信息，默认为'update files'。\n\n### 采样参数\n\n- prm_model: 过程奖励模型的类型，可以是模型id（以'transformers'方式拉起），或者plugin中定义的prm key（自定义推理过程）。\n- orm_model: 结果奖励模型的类型，通常是通配符或测试用例等，一般定义在plugin中。\n- sampler_type: 采样类型，目前支持 sample, distill\n- sampler_engine: 支持`transformers`, `lmdeploy`, `vllm`, `client`, `no`，默认为`transformers`，采样模型的推理引擎。\n- output_dir: 输出目录，默认为`sample_output`。\n- output_file: 输出文件名称，默认为`None`使用时间戳作为文件名。传入时不需要传入目录，仅支持jsonl格式。\n- override_exist_file: 如`output_file`存在，是否覆盖。\n- num_sampling_batch_size: 每次采样的batch_size。\n- num_sampling_batches: 共采样多少batch。\n- n_best_to_keep: 返回多少最佳sequences。\n- data_range: 本采样处理数据集的分片。传入格式为`2 3`，代表数据集分为3份处理（这意味着通常有三个`swift sample`在并行处理），本实例正在处理第3个分片。\n- temperature: 在这里默认为1.0。\n- prm_threshold: PRM阈值，低于该阈值的结果会被过滤掉，默认值为`0`。\n- easy_query_threshold: 单个query的所有采样中，ORM评估如果正确，大于该比例的query会被丢弃，防止过于简单的query出现在结果中，默认为`None`，代表不过滤。\n- engine_kwargs: 传入sampler_engine的额外参数，以json string传入，例如`{\"cache_max_entry_count\":0.7}`。\n- num_return_sequences: 采样返回的原始sequence数量。默认为64，本参数对`sample`采样有效。\n- cache_files: 为避免同时加载prm和generator造成显存OOM，可以分两步进行采样，第一步将prm和orm置为`None`，则所有结果都会输出到文件中，第二次运行采样将sampler_engine置为`no`并传入`--cache_files`为上次采样的输出文件，则会使用上次输出的结果进行prm和orm评估并输出最终结果。\n - 注意：使用cache_files时，`--dataset`仍然需要传入，这是因为cache_files的id是由原始数据计算的md5，需要把两部分信息结合使用。\n\n## 特定模型参数\n除了以上参数外，有些模型还支持额外的具体模型参数。这些参数含义通常可以在对应模型官方repo或者其推理代码中找到相应含义。**ms-swift引入这些参数以确保训练的模型与官方推理代码效果对齐**。\n- 特定模型参数可以通过`--model_kwargs`或者环境变量进行设置，例如: `--model_kwargs '{\"fps_max_frames\": 12}'`或者`FPS_MAX_FRAMES=12`。\n- 注意：若你在训练时指定了特定模型参数，请在推理时也设置对应的参数，这可以提高训练效果。\n\n### qwen2_vl, qvq, qwen2_5_vl, mimo_vl, keye_vl, keye_vl_1_5\n参数含义与`qwen_vl_utils<0.0.12`或者`qwen_omni_utils`库中含义一致，可以查看[这里](https://github.com/QwenLM/Qwen2.5-VL/blob/main/qwen-vl-utils/src/qwen_vl_utils/vision_process.py#L24)。ms-swift通过修改这些常数值来控制图片分辨率和视频帧率，避免训练时OOM。\n\n- IMAGE_FACTOR: 默认为28。\n- MIN_PIXELS: 默认为`4 * 28 * 28`。图像的最小分辨率。建议设置为28*28的倍数。\n- 🔥MAX_PIXELS: 默认为`16384 * 28 * 28`。图像的最大分辨率。建议设置为28*28的倍数。\n- MAX_RATIO: 默认为200。\n- VIDEO_MIN_PIXELS: 默认为`128 * 28 * 28`。视频中一帧的最小分辨率。建议设置为28*28的倍数。\n- 🔥VIDEO_MAX_PIXELS: 默认为`768 * 28 * 28`。视频中一帧的最大分辨率。建议设置为28*28的倍数。\n- VIDEO_TOTAL_PIXELS: 默认为`24576 * 28 * 28`。\n- FRAME_FACTOR: 默认为2。\n- FPS: 默认为2.0。\n- FPS_MIN_FRAMES: 默认为4。一段视频的最小抽帧数。\n- 🔥FPS_MAX_FRAMES: 默认为768。一段视频的最大抽帧数。\n- 🔥QWENVL_BBOX_FORMAT: grounding格式使用'legacy'还是'new'。'legacy'格式为：`<|object_ref_start|>一只狗<|object_ref_end|><|box_start|>(432,991),(1111,2077)<|box_end|>`，'new'格式参考：[Qwen3-VL cookbook](https://github.com/QwenLM/Qwen3-VL/blob/main/cookbooks/2d_grounding.ipynb)，并参考[grounding数据集格式文档](../Customization/Custom-dataset.md#grounding)。默认为'legacy'。\n - 注意：该环境变量适配Qwen2/2.5/3-VL和Qwen2.5/3-Omni系列模型。\n\n### qwen2_audio\n- SAMPLING_RATE: 默认为16000。\n\n### qwen3_vl, qwen3_5\n参数含义与`qwen_vl_utils>=0.0.14`库中的含义一致，可以查看[这里](https://github.com/QwenLM/Qwen2.5-VL/blob/main/qwen-vl-utils/src/qwen_vl_utils/vision_process.py#L24)。通过传入以下环境变量，可以修改该库的全局变量默认值。（也兼容使用`qwen2_5_vl`的环境变量，例如：`MAX_PIXELS`、`VIDEO_MAX_PIXELS`，会做自动转换。）\n\n- SPATIAL_MERGE_SIZE: 默认为2。\n- IMAGE_MIN_TOKEN_NUM: 默认为`4`，代表一张图片最小图像tokens的个数。\n- 🔥IMAGE_MAX_TOKEN_NUM: 默认为`16384`，代表一张图片最大图像tokens的个数。（用于避免OOM）\n - 提示：等价最大图像像素为`IMAGE_MAX_TOKEN_NUM * 32 *32`。\n- VIDEO_MIN_TOKEN_NUM: 默认为`128`，代表视频中一帧的最小视频tokens的个数。\n- 🔥VIDEO_MAX_TOKEN_NUM: 默认为`768`，代表视频中一帧的最大视频tokens的个数。（用于避免OOM）\n- MAX_RATIO: 默认为200。\n- FRAME_FACTOR: 默认为2。\n- FPS: 默认为2.0。\n- FPS_MIN_FRAMES: 默认为4。代表一段视频的最小抽帧数。\n- 🔥FPS_MAX_FRAMES: 默认为768，代表一段视频的最大抽帧数。（用于避免OOM）\n\n\n### qwen2_5_omni, qwen3_omni\nqwen2_5_omni除了包含qwen2_5_vl和qwen2_audio的模型特定参数外，还包含以下参数：（注意：qwen3_omni包含的是**qwen3_vl**, qwen2_audio的模型特定参数）\n- USE_AUDIO_IN_VIDEO: 默认为False。是否使用video中的音频信息。\n- 🔥ENABLE_AUDIO_OUTPUT: 默认为None，即使用`config.json`中的值。若使用zero3进行训练，请设置为False。\n - 提示：ms-swift只对thinker部分进行微调，建议设置为False以降低显存占用（只创建thinker部分的模型结构）。\n\n\n### qwen3_vl_emb, qwen3_vl_reranker\n参数含义与`qwen3_vl`相同，见上面的描述。以下为对默认值的覆盖：\n\n- IMAGE_MAX_TOKEN_NUM: qwen3_vl_emb默认为1800, qwen3_vl_reranker默认为1280。具体参考这里：[qwen3_vl_embedding](https://modelscope.cn/models/Qwen/Qwen3-VL-Embedding-2B/file/view/master/scripts%2Fqwen3_vl_embedding.py?status=1#L26), [qwen3_vl_reranker](https://modelscope.cn/models/Qwen/Qwen3-VL-Reranker-2B/file/view/master/scripts%2Fqwen3_vl_reranker.py?status=1#L16)。\n- FPS: 默认为1。\n- FPS_MAX_FRAMES: 默认为64。\n\n\n### internvl, internvl_phi3\n参数含义可以查看[这里](https://modelscope.cn/models/OpenGVLab/Mini-InternVL-Chat-2B-V1-5)。\n- MAX_NUM: 默认为12。\n- INPUT_SIZE: 默认为448。\n\n### internvl2, internvl2_phi3, internvl2_5, internvl3, internvl3_5\n参数含义可以查看[这里](https://modelscope.cn/models/OpenGVLab/InternVL2_5-2B)。\n- MAX_NUM: 默认为12。\n- INPUT_SIZE: 默认为448。\n- VIDEO_MAX_NUM: 默认为1。视频的MAX_NUM。\n- VIDEO_SEGMENTS: 默认为8。\n\n\n### minicpmv2_6, minicpmv4, minicpmo\n- MAX_SLICE_NUMS: 默认为9，参考[这里](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6/file/view/master?fileName=config.json&status=1)。\n- VIDEO_MAX_SLICE_NUMS: 默认为1，视频的MAX_SLICE_NUMS，参考[这里](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6)。\n- MAX_NUM_FRAMES: 默认为64，参考[这里](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6)。\n\n### minicpmo\n- INIT_TTS: 默认为False。\n- INIT_AUDIO: 默认为False。\n\n### ovis1_6, ovis2\n- MAX_PARTITION: 默认为9，参考[这里](https://github.com/AIDC-AI/Ovis/blob/d248e34d755a95d24315c40e2489750a869c5dbc/ovis/model/modeling_ovis.py#L312)。\n\n### ovis2_5\n以下参数含义可以在[这里](https://modelscope.cn/models/AIDC-AI/Ovis2.5-2B)的示例代码中找到。\n- MIX_PIXELS: int类型，默认为`448 * 448`。\n- MAX_PIXELS: int类型，默认为`1344 * 1792`。若出现OOM，可以调小该值。\n- VIDEO_MAX_PIXELS: int类型，默认为`896 * 896`。\n- NUM_FRAMES: 默认为8。用于视频抽帧。\n\n### mplug_owl3, mplug_owl3_241101\n- MAX_NUM_FRAMES: 默认为16，参考[这里](https://modelscope.cn/models/iic/mPLUG-Owl3-7B-240728)。\n\n### xcomposer2_4khd\n- HD_NUM: 默认为55，参考[这里](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-xcomposer2-4khd-7b)。\n\n### xcomposer2_5\n- HD_NUM: 图片数量为1时，默认值为24。大于1，默认为6。参考[这里](https://modelscope.cn/models/AI-ModelScope/internlm-xcomposer2d5-7b/file/view/master?fileName=modeling_internlm_xcomposer2.py&status=1#L254)。\n\n### video_cogvlm2\n- NUM_FRAMES: 默认为24，参考[这里](https://github.com/zai-org/CogVLM2/blob/main/video_demo/inference.py#L22)。\n\n### phi3_vision\n- NUM_CROPS: 默认为4，参考[这里](https://modelscope.cn/models/LLM-Research/Phi-3.5-vision-instruct)。\n\n### llama3_1_omni\n- N_MELS: 默认为128，参考[这里](https://github.com/ictnlp/LLaMA-Omni/blob/544d0ff3de8817fdcbc5192941a11cf4a72cbf2b/omni_speech/infer/infer.py#L57)。\n\n### video_llava\n- NUM_FRAMES: 默认为16。\n\n\n## 其他环境变量\n- CUDA_VISIBLE_DEVICES: 控制使用哪些GPU卡。默认使用所有卡。\n- ASCEND_RT_VISIBLE_DEVICES: 控制使用哪些NPU卡（只对ASCEND卡生效）。默认使用所有卡。\n- MODELSCOPE_CACHE: 控制缓存路径。（多机训练时建议设置该值，以确保不同节点使用相同的数据集缓存）\n- PYTORCH_CUDA_ALLOC_CONF: 推荐设置为`'expandable_segments:True'`，这将减少GPU内存碎片，具体请参考[torch文档](https://docs.pytorch.org/docs/stable/notes/cuda.html#cuda-memory-management)。\n- NPROC_PER_NODE: torchrun中`--nproc_per_node`的参数透传。默认为1。若设置了`NPROC_PER_NODE`或者`NNODES`环境变量，则使用torchrun启动训练或推理。\n- MASTER_PORT: torchrun中`--master_port`的参数透传。默认为29500。\n- MASTER_ADDR: torchrun中`--master_addr`的参数透传。\n- NNODES: torchrun中`--nnodes`的参数透传。\n- NODE_RANK: torchrun中`--node_rank`的参数透传。\n- LOG_LEVEL: 日志的level，默认为'INFO'，你可以设置为'WARNING', 'ERROR'等。\n- SWIFT_DEBUG: 在`engine.infer(...)`时，若设置为'1'，TransformersEngine将会打印input_ids和generate_ids的内容方便进行调试与对齐。\n- VLLM_USE_V1: 用于切换vLLM使用V0/V1版本。\n- SWIFT_TIMEOUT: 若多模态数据集中存在图像URL，该参数用于控制获取图片的timeout，默认为20s。\n- ROOT_IMAGE_DIR: 图像（多模态）资源的根目录。通过设置该参数，可以在数据集中使用相对于 `ROOT_IMAGE_DIR` 的相对路径。默认情况下，是相对于运行目录的相对路径。\n- SWIFT_SINGLE_DEVICE_MODE: 单设备模式，可选值为\"0\"(默认值)/\"1\"，在此模式下，每个进程只能看到一个设备\n"
},
{
"path": "docs/source/Instruction/Evaluation.md",
"content": "# 评测\n\nSWIFT支持了eval（评测）能力，用于对原始模型和训练后的模型给出标准化的评测指标。\n\n## 能力介绍\n\nSWIFT的eval能力使用了魔搭社区[评测框架EvalScope](https://github.com/modelscope/eval-scope)，并进行了高级封装以支持各类模型的评测需求。\n\n> 注意：EvalScope支持许多其他的复杂能力，例如[模型的性能评测](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/stress_test/quick_start.html)，请直接使用EvalScope框架。\n\n目前我们支持了**标准评测集**的评测流程，以及**用户自定义**评测集的评测流程。其中**标准评测集**由三个评测后端提供支持：\n\n下面展示所支持的数据集名称，若需了解数据集的详细信息，请参考[所有支持的数据集](https://evalscope.readthedocs.io/zh-cn/latest/get_started/supported_dataset/index.html)\n\n1. Native（默认）：\n\n 主要支持纯文本评测，同时**支持**评测结果可视化\n ```text\n 'arc', 'bbh', 'ceval', 'cmmlu', 'competition_math',\n 'general_qa', 'gpqa', 'gsm8k', 'hellaswag', 'humaneval',\n 'ifeval', 'iquiz', 'mmlu', 'mmlu_pro',\n 'race', 'trivia_qa', 'truthful_qa'\n ```\n\n2. OpenCompass：\n\n 主要支持纯文本评测，暂**不支持**评测结果可视化\n ```text\n 'obqa', 'cmb', 'AX_b', 'siqa', 'nq', 'mbpp', 'winogrande', 'mmlu', 'BoolQ', 'cluewsc', 'ocnli', 'lambada',\n 'CMRC', 'ceval', 'csl', 'cmnli', 'bbh', 'ReCoRD', 'math', 'humaneval', 'eprstmt', 'WSC', 'storycloze',\n 'MultiRC', 'RTE', 'chid', 'gsm8k', 'AX_g', 'bustm', 'afqmc', 'piqa', 'lcsts', 'strategyqa', 'Xsum', 'agieval',\n 'ocnli_fc', 'C3', 'tnews', 'race', 'triviaqa', 'CB', 'WiC', 'hellaswag', 'summedits', 'GaokaoBench',\n 'ARC_e', 'COPA', 'ARC_c', 'DRCD'\n ```\n\n3. VLMEvalKit:\n\n 主要支持多模态评测，暂**不支持**评测结果可视化\n ```text\n 'COCO_VAL', 'MME', 'HallusionBench', 'POPE', 'MMBench_DEV_EN', 'MMBench_TEST_EN', 'MMBench_DEV_CN', 'MMBench_TEST_CN',\n 'MMBench', 'MMBench_CN', 'MMBench_DEV_EN_V11', 'MMBench_TEST_EN_V11', 'MMBench_DEV_CN_V11',\n 'MMBench_TEST_CN_V11', 'MMBench_V11', 'MMBench_CN_V11', 'SEEDBench_IMG', 'SEEDBench2',\n 'SEEDBench2_Plus', 'ScienceQA_VAL', 'ScienceQA_TEST', 'MMT-Bench_ALL_MI', 'MMT-Bench_ALL',\n 'MMT-Bench_VAL_MI', 'MMT-Bench_VAL', 'AesBench_VAL', 'AesBench_TEST', 'CCBench', 'AI2D_TEST', 'MMStar',\n 'RealWorldQA', 'MLLMGuard_DS', 'BLINK', 'OCRVQA_TEST', 'OCRVQA_TESTCORE', 'TextVQA_VAL', 'DocVQA_VAL',\n 'DocVQA_TEST', 'InfoVQA_VAL', 'InfoVQA_TEST', 'ChartQA_TEST', 'MathVision', 'MathVision_MINI',\n 'MMMU_DEV_VAL', 'MMMU_TEST', 'OCRBench', 'MathVista_MINI', 'LLaVABench', 'MMVet', 'MTVQA_TEST',\n 'MMLongBench_DOC', 'VCR_EN_EASY_500', 'VCR_EN_EASY_100', 'VCR_EN_EASY_ALL', 'VCR_EN_HARD_500',\n 'VCR_EN_HARD_100', 'VCR_EN_HARD_ALL', 'VCR_ZH_EASY_500', 'VCR_ZH_EASY_100', 'VCR_ZH_EASY_ALL',\n 'VCR_ZH_HARD_500', 'VCR_ZH_HARD_100', 'VCR_ZH_HARD_ALL', 'MMDU', 'MMBench-Video', 'Video-MME'\n ```\n\n## 环境准备\n\n```shell\npip install ms-swift[eval] -U\n```\n\n或从源代码安装：\n\n```shell\ngit clone https://github.com/modelscope/ms-swift.git\ncd ms-swift\npip install -e '.[eval]'\n```\n\n## 评测\n\n支持纯文本评测、多模态评测、url评测、自定义数据集评测四种方式\n\n**基本示例**\n\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift eval \\\n --model Qwen/Qwen2.5-0.5B-Instruct \\\n --eval_backend Native \\\n --infer_backend transformers \\\n --eval_limit 10 \\\n --eval_dataset gsm8k\n```\n其中:\n- model: 可指定本地模型路径或者modelscope上的模型ID\n- eval_backend: 可选 Native, OpenCompass, VLMEvalKit，默认为 Native\n- infer_backend: 可选 transformers, vllm, sglang, lmdeploy，默认为 transformers\n- eval_limit: 每个评测集的采样数，默认为None，表示使用全部数据，可用于快速验证\n- eval_dataset: 评测数据集，可设置多个数据集，用空格分割\n\n**复杂评测示例**\n\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift eval \\\n --model Qwen/Qwen2.5-0.5B-Instruct \\\n --eval_backend Native \\\n --infer_backend transformers \\\n --eval_limit 10 \\\n --eval_dataset gsm8k \\\n --eval_dataset_args '{\"gsm8k\": {\"few_shot_num\": 0, \"filters\": {\"remove_until\": \" \"}}}' \\\n --eval_generation_config '{\"max_tokens\": 512, \"temperature\": 0}' \\\n --extra_eval_args '{\"ignore_errors\": true, \"debug\": true}'\n```\n\n详细评测的参数列表可以参考[这里](Command-line-parameters.md#评测参数)。\n\n## 训练中评测\n\nSWIFT支持在训练过程中使用EvalScope对当前的模型进行评测，以便及时了解模型的训练效果。\n\n**基本示例**\n\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift sft \\\n --model \"Qwen/Qwen2.5-0.5B-Instruct\" \\\n --tuner_type \"lora\" \\\n --dataset \"AI-ModelScope/alpaca-gpt4-data-zh#100\" \\\n --torch_dtype \"bfloat16\" \\\n --num_train_epochs \"1\" \\\n --per_device_train_batch_size \"1\" \\\n --learning_rate \"1e-4\" \\\n --lora_rank \"8\" \\\n --lora_alpha \"32\" \\\n --target_modules \"all-linear\" \\\n --gradient_accumulation_steps \"16\" \\\n --save_steps \"50\" \\\n --save_total_limit \"5\" \\\n --logging_steps \"5\" \\\n --max_length \"2048\" \\\n --eval_strategy \"steps\" \\\n --eval_steps \"5\" \\\n --per_device_eval_batch_size \"5\" \\\n --eval_use_evalscope \\\n --eval_dataset \"gsm8k\" \\\n --eval_dataset_args '{\"gsm8k\": {\"few_shot_num\": 0}}' \\\n --eval_limit \"10\"\n```\n\n注意启动命令为`sft`，其中eval相关的参数有：\n- eval_strategy: 评估策略。默认为None，跟随`save_strategy`的策略\n- eval_steps: 默认为None，如果存在评估数据集，则跟随`save_steps`的策略\n- eval_use_evalscope: 是否使用evalscope进行评测，需要设置该参数来开启评测\n- eval_dataset: 评测数据集，可设置多个数据集，用空格分割\n- eval_dataset_args: 评测数据集参数，json格式，可设置多个数据集的参数\n- eval_limit: 评测数据集采样数\n- eval_generation_config: 评测时模型推理配置，json格式，默认为`{'max_tokens': 512}`\n\n\n更多评测的样例可以参考[examples](https://github.com/modelscope/ms-swift/tree/main/examples/eval)\n\n## 自定义评测集\n\n本框架支持选择题和问答题，两种预定义的数据集格式，使用流程如下：\n\n*注意：使用自定义评测时，eval_backend参数必须为Native*\n\n### 选择题格式（MCQ）\n适合用户是选择题的场景，评测指标为准确率（accuracy）。\n\n**数据准备**\n\n准备选择题格式的csv文件，该目录结构如下：\n\n```text\nmcq/\n├── example_dev.csv # （可选）文件名组成为`{subset_name}_dev.csv`，用于fewshot评测\n└── example_val.csv # 文件名组成为`{subset_name}_val.csv`，用于实际评测的数据\n```\n\n其中csv文件需要为下面的格式：\n\n```text\nid,question,A,B,C,D,answer\n1,通常来说，组成动物蛋白质的氨基酸有____,4种,22种,20种,19种,C\n2,血液内存在的下列物质中，不属于代谢终产物的是____。,尿素,尿酸,丙酮酸,二氧化碳,C\n```\n其中：\n- `id`是序号（可选）\n- `question`是问题\n- `A`, `B`, `C`, `D`等是可选项，最大支持10个选项\n- `answer`是正确选项\n\n**启动评测**\n\n运行下面的命令：\n\n```bash\nCUDA_VISIBLE_DEVICES=0 \\\nswift eval \\\n --model Qwen/Qwen2.5-0.5B-Instruct \\\n --eval_backend Native \\\n --infer_backend transformers \\\n --eval_dataset general_mcq \\\n --eval_dataset_args '{\"general_mcq\": {\"local_path\": \"/path/to/mcq\", \"subset_list\": [\"example\"]}}'\n```\n其中：\n- `eval_dataset` 需要设置为 `general_mcq`\n- `eval_dataset_args` 需要设置\n - `local_path` 自定义数据集文件夹路径\n - `subset_list` 评测数据集名称，上述 `*_dev.csv` 中的 `*`\n\n**运行结果**\n\n```text\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Model | Dataset | Metric | Subset | Num | Score | Cat.0 |\n+=====================+=============+=================+==========+=======+=========+=========+\n| Qwen2-0.5B-Instruct | general_mcq | AverageAccuracy | example | 12 | 0.5833 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n```\n\n## 问答题格式（QA）\n适合用户是问答题的场景，评测指标是`ROUGE`和`BLEU`。\n\n**数据准备**\n\n准备一个问答题格式的jsonline文件，该目录包含了一个文件：\n\n```text\nqa/\n└── example.jsonl\n```\n\n该jsonline文件需要为下面的格式：\n\n```json\n{\"query\": \"中国的首都是哪里？\", \"response\": \"中国的首都是北京\"}\n{\"query\": \"世界上最高的山是哪座山？\", \"response\": \"是珠穆朗玛峰\"}\n{\"query\": \"为什么北极见不到企鹅？\", \"response\": \"因为企鹅大多生活在南极\"}\n```\n\n**启动评测**\n\n运行下面的命令：\n\n```bash\nCUDA_VISIBLE_DEVICES=0 \\\nswift eval \\\n --model Qwen/Qwen2.5-0.5B-Instruct \\\n --eval_backend Native \\\n --infer_backend transformers \\\n --eval_dataset general_qa \\\n --eval_dataset_args '{\"general_qa\": {\"local_path\": \"/path/to/qa\", \"subset_list\": [\"example\"]}}'\n```\n\n其中：\n- `eval_dataset` 需要设置为 `general_qa`\n- `eval_dataset_args` 是一个json字符串，需要设置：\n - `local_path` 自定义数据集文件夹路径\n - `subset_list` 评测数据集名称，上述 `*.jsonl` 中的 `*`\n\n**运行结果**\n\n```text\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Model | Dataset | Metric | Subset | Num | Score | Cat.0 |\n+=====================+=============+=================+==========+=======+=========+=========+\n| Qwen2-0.5B-Instruct | general_qa | bleu-1 | default | 12 | 0.2324 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | bleu-2 | default | 12 | 0.1451 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | bleu-3 | default | 12 | 0.0625 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | bleu-4 | default | 12 | 0.0556 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | rouge-1-f | default | 12 | 0.3441 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | rouge-1-p | default | 12 | 0.2393 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | rouge-1-r | default | 12 | 0.8889 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | rouge-2-f | default | 12 | 0.2062 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | rouge-2-p | default | 12 | 0.1453 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | rouge-2-r | default | 12 | 0.6167 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | rouge-l-f | default | 12 | 0.333 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | rouge-l-p | default | 12 | 0.2324 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n| Qwen2-0.5B-Instruct | general_qa | rouge-l-r | default | 12 | 0.8889 | default |\n+---------------------+-------------+-----------------+----------+-------+---------+---------+\n```\n"
},
{
"path": "docs/source/Instruction/Export-and-push.md",
"content": "# 导出与推送\n\n\n## Merge LoRA\n\n- 查看[这里](https://github.com/modelscope/ms-swift/blob/main/examples/export/merge_lora.sh)。\n\n## 量化\n\nSWIFT支持AWQ、GPTQ、FP8、BNB模型的量化导出。其中使用AWQ、GPTQ需使用校准数据集，量化性能较好但量化耗时较长；而FP8、BNB无需校准数据集，量化耗时较短。\n\n| 量化技术 | 多模态 | 推理加速 | 继续训练 |\n| -------- | ------ | -------- | -------- |\n| GPTQ | ✅ | ✅ | ✅ |\n| AWQ | ✅ | ✅ | ✅ |\n| BNB | ❌ | ✅ | ✅ |\n\n\n除SWIFT安装外，需要安装以下额外依赖：\n```shell\n# 使用awq量化:\n# autoawq和cuda版本有对应关系，请按照`https://github.com/casper-hansen/AutoAWQ`选择版本\n# 如果出现torch依赖冲突，请额外增加指令`--no-deps`\npip install autoawq -U\n\n# 使用gptq量化:\n# auto_gptq和cuda版本有对应关系，请按照`https://github.com/PanQiWei/AutoGPTQ#quick-installation`选择版本\npip install auto_gptq optimum -U\n\n# 使用gptq v2量化:\npip install gptqmodel optimum -U\n\n# 使用bnb量化：\npip install bitsandbytes -U\n```\n\n我们提供了一系列脚本展现SWIFT的量化导出能力：\n- 支持[AWQ](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/awq.sh)/[GPTQ](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/gptq.sh)/[GPTQ v2](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/gptq_v2.sh)/[BNB](https://github.com/modelscope/ms-swift/blob/main/examples/export/quantize/bnb.sh)量化导出。\n- 多模态量化: 支持使用GPTQ和AWQ对多模态模型进行量化，其中AWQ支持的多模态模型有限。参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/mllm)。\n- 更多系列模型的支持: 支持[Bert](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/bert)，[Reward Model](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize/reward_model)的量化导出。\n- 使用SWIFT量化导出的模型支持使用vllm/sglang/lmdeploy进行推理加速；也支持使用QLoRA继续进行SFT/RLHF。\n\n\n## 推送模型\n\nSWIFT支持将训练/量化的模型重新推送到ModelScope/HuggingFace。默认推送到ModelScope，你可以指定`--use_hf true`推送到HuggingFace。\n```shell\nswift export \\\n --model output/vx-xxx/checkpoint-xxx \\\n --push_to_hub true \\\n --hub_model_id '' \\\n --hub_token '' \\\n --use_hf false\n```\n\n小贴士：\n- 你可以使用`--model `或者`--adapters `指定需要推送的checkpoint目录，这两种写法在推送模型场景没有差异。\n- 推送到ModelScope时，你需要确保你已经注册了魔搭账号，你的SDK token可以在[该页面](https://www.modelscope.cn/my/myaccesstoken)中获取。推送模型需确保sdk token的账号具有model_id对应组织的编辑权限。推送模型将自动创建对应model_id的模型仓库（如果该模型仓库不存在），你可以使用`--hub_private_repo true`来自动创建私有的模型仓库。\n"
},
{
"path": "docs/source/Instruction/Frequently-asked-questions.md",
"content": "# 常见问题整理\n\n下面是SWIFT使用过程中遇到的一些常见问题。\n\n## 训练\n\nSWIFT支持的训练方法包括预训练、指令监督微调、偏好学习、GRPO、Embedding、Reranker、序列分类任务等，详见[主页](https://github.com/modelscope/ms-swift/blob/main/README_CN.md)。\n\n### Q1: SWIFT支持的模型有哪些？如何设置本地模型路径？\n支持的模型详见文档[支持的模型和数据集](https://swift.readthedocs.io/zh-cn/latest/Instruction/Supported-models-and-datasets.html)。如果模型已经下载到了本地，设置`--model `即可。对于离线环境训练，同时设置`--model 本地路径`，`--check_model false`，如果提示git clone相关报错，需要clone repo，然后通过`local_repo_path`指定，详见[命令行参数](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html)。从ModelScope下载的模型，可以配置环境变量`MODELSCOPE_CACHE=your_path`将原始的模型存到指定路径；如果用ModelScope SDK下载，通过`cache_dir=\"本地地址\"`；也可以使用`modelscope download`命令行工具或`git`下载，详见modelscope文档[模型下载](https://modelscope.cn/docs/models/download)。如果需要从Hugging Face下载模型，设置环境变量`USE_HF=1`。\nSWIFT会自动匹配model_type，也可以查看文档[支持的模型和数据集](https://swift.readthedocs.io/zh-cn/latest/Instruction/Supported-models-and-datasets.html)，手动指定。\n\n### Q2: SWIFT支持的数据集有哪些？如何使用自定义数据集？\n支持的数据集详见文档[支持的模型和数据集](https://swift.readthedocs.io/zh-cn/latest/Instruction/Supported-models-and-datasets.html)。自定义数据集格式及使用方法见文档[自定义数据集](https://swift.readthedocs.io/zh-cn/latest/Customization/Custom-dataset.html)，符合这些格式的数据集会自动使用swift内置的数据预处理器。如果与文档中的格式不一致，请自行转换格式，或者参考已支持的数据集接入。若自定义数据集中有额外的字段，这些字段默认不会被使用，可以通过[命令行参数remove_unused_columns](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html#id4)进行设置。\n需要将数据集下载到本地，然后通过路径指定，请查看[自定义数据集文档](https://swift.readthedocs.io/zh-cn/latest/Customization/Custom-dataset.html#dataset-info-json)。`git clone`下载到本地，然后通过dataset_info.json文件中的`dataset_path`字段指定就行。\n数据随机详见[命令行参数dataset_shuffle](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html)。\n强制重新下载数据集，设置命令行参数`--download_mode`。对数据集进行错误检查，请设置命令行参数`strict`。需要数据集质检工具时，可以查看另一个库[data-juicer](https://github.com/modelscope/data-juicer)。\n由于datasets的底层pyarrow对于类型管控比较严格，图像grounding数据集的objects部分、agent数据集的tools部分等，因为这个原因要用str，要不pyarrow就会报错：你每行的类型不一致。\n训练中遇到报错`AttributeError:’TrainerState’ object has no attribute ’last_model_checkpoint’`，数据集太少了，数据数量不足一个step导致的报错，增加一些数据。另外，切分的验证集数据很少时也会有类似报错。\n下面是一个assistant字段为空导致的报错：\n```text\nFile \"/mnt/workspace/swift/swift/1lm/dataset/preprocessor/core. py\", line 69, in _check_messages raise\nValueError(f'assistant_message; {assistant_message}')\nValueError: assistant_message: {'role' :'assistant', 'content': ''}\n```\n```shell\nCUDA_VISIBLE_DEVICES=0 NPROC_PER_NODE=1 MAX_PIXELS=1003520 swift sft --model Qwen/Qwen2.5-VL-7B-Instruct --tuner_type lora --dataset /mnt/workspace/data.json --deepspeed zero2 --max_length 16384\n```\n数据集assistant字段为空，如果是推理，把这个空字符串删掉，因为这个会导致训练时NaN，会做检查。\n\n### Q3: 从缓存加载数据集相关问题\n设置命令行参数`--load_from_cache_file true`，可以加快数据集加载速度，尤其是在多模态数据集、数据量较大等场景。在debug或修改preprocessor时，设置为false，更多说明请在[命令行参数文档](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html)中搜索该参数。\n\n### Q4: 如何搭建SWIFT环境？有镜像可以使用吗？\n环境搭建详见[SWIFT安装文档](https://swift.readthedocs.io/zh-cn/latest/GetStarted/SWIFT-installation.html)，一些常见依赖的推荐版本可以在[主页](https://github.com/modelscope/ms-swift/blob/main/README_CN.md)上找到。文档中提供了镜像，用`docker run`命令启动容器即可，如：`docker run --gpus all -p 8000:8000 -it -d --name ms modelscope-registry.cn-hangzhou.cr.aliyuncs.com/modelscope-repo/modelscope:ubuntu22.04-cuda12.8.1-py311-torch2.9.0-vllm0.13.0-modelscope1.33.0-swift3.12.5 /bin/bash`，启动容器后拉最新代码安装swift。\n\n### Q5: 多模态模型训练数据格式、参数冻结、优化器设置相关问题\n多模态模型训练的[例子](https://github.com/modelscope/ms-swift/tree/main/examples/train/multimodal)。支持纯文本、图文数据训练，也可以两种数据混合训练。图像、视频、音频相关的参数，如，最大像素、fps等请查看[特定模型参数](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html#id19)。\nGrounding任务中通用数据格式支持了一个物体对应多个bbox，参考文档[自定义数据集](https://swift.readthedocs.io/zh-cn/latest/Customization/Custom-dataset.html#grounding)。videos可以是图片列表，使用文件目录的方式。\nSWIFT按max_pixels对图像进行调整，会保存预处理前和后的图像，然后对bbox进行调整，不过推理没有这样的调整，需要提前手动处理图像。\nVLM模型训练减少显存使用，请配置`--freeze_vit true`，以及限制最大像素的参数`--max_pixels`。`--freeze_vit`，`--freeze_aligner`，`--freeze_llm`这几个参数详见[命令行参数文档](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html#tuner)。如果ViT没有训练，那有会有warning: none of the inputs have requires_grad=True是正常的，如果训练了，则不应该抛出。\n使用全参数微调visual encoder同时使用LoRA微调LLM，参考这里[例子](https://github.com/modelscope/ms-swift/tree/main/examples/train/multimodal/lora_llm_full_vit)。\n\n### Q6: template相关问题\n由于jinja chat template没有labels，所以不支持训练。\n多模态数据集如果需要在加载数据之后做动态数据增强，例如，给输入数据随机添加噪声等，请在template中修改encode方法。\n\n### Q7: SWIFT训练如何debug？\n详见[预训练与微调文档](https://swift.readthedocs.io/zh-cn/latest/Instruction/Pre-training-and-Fine-tuning.html)。\n\n### Q8: SWIFT如何使用python脚本训练？\n参考[notebook例子](https://github.com/modelscope/ms-swift/tree/main/examples/notebook)。\n\n### Q9: SWIFT如何使用UI界面训练？\n使用`swift web-ui`命令，界面训练与命令行一致，界面上的参数请查看命令行参数文档。自定义数据集的使用与上面Q2一致。Megatron-SWIFT不支持UI界面训练。\n\n### Q10: 单机多卡训练相关问题\nSWIFT多卡训练底层依赖torchrun。`deepspeed` 和 `device_map`不兼容，两个只能选1个。更多细节请查看代码库中的[例子](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-gpu)。\n\n### Q11: 多机多卡训练相关问题\n请查看[多机多卡例子](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-node)。多机多卡训练时，只有主节点有日志。\n多机训练速度缓慢，如，使用DeepSpeed ZeRO3训练会出现严重的速度下降，请查看[issue](https://github.com/modelscope/ms-swift/issues/1825)。\n\n### Q12: 大规模数据集相关问题\n数据集太大了，然后每次tokenize都需要很久，请使用`lazy_tokenize`或流式读取`streaming`，详见[命令行参数](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html)。\n\n### Q13: 断点续训相关问题\n先前训练脚本中的参数不变，加上`--resume_from_checkpoint output/xxx/vx-xxx/checkpoint-xxx`，详见[命令行参数](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html)。如果数据集发生了改动，仅加载模型，请同时设置`--resume_only_model`。更复杂的场景，请在命令行参数文档中搜索resume。\n\n### Q14: 数据集流式加载相关问题\n流式加载`--streaming true`，一边训练一边加载，需要设置max_steps，详见`streaming`参数说明，[命令行参数文档](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html#id4)。\n注意：streaming是不随机的，也不划分验证集，验证集通过命令行参数`val_dataset`指定。\n断点续训时，流式只能往前索引，不能随机索引，跳过已经训练的数据耗时特别长，不建议用流式。\n\n### Q15: packing相关问题\npacking要和flash_attn一起使用，不然是有误差，attention_mask会出问题。packing_cache这个参数，在多机训练时，需要设置为共享的磁盘路径。\nQwen3.5模型中的linear-attention不支持var_len，不建议开启packing。\n开启packing，多模态数据会有两次map，map完一次后还会进行第二次mapping，一次是数据集的，一次是template的。如果速度非常慢，可以设置`OMP_NUM_THREADS=14`加速，或者可以把packing去掉，就不会有第二次了。\n\n### Q16: 数据集多进程处理\n数据集map过程比较慢时，设置参数`--dataset_num_proc`可以开多进程。多模态数据集map比较慢是正常的。\n\n### Q17: 当前训练完默认保存多少个checkpoint？\n默认保存所有的checkpoint，详见[命令行参数 save_total_limit](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html)。\n\n### Q18: 训练过程的loss和acc\n自定义的损失函数在plugin中加就可以。如果需要不同数据集的loss曲线，请设置`--enable_channel_loss`。\n如果eval得到的acc和对应保存的ckpt去重新推理一遍计算得到的acc不是一致的，是因为训练时候的eval_acc和推理时候的acc计算方式不一样导致的。`acc_strategy`: 默认为`'token'`, 可选择的值包括: `'token'`，`'seq'`。\n训练过程中没有token_acc是因为有些模型`logits`和`labels`数量对不上，就不算的。\n可以在[这里](https://github.com/modelscope/ms-swift/blob/main/swift/loss/mapping.py)查看当前支持的loss或添加新的loss，\n检查``等特殊token是否参与损失计算，可以在命令行日志中找一下打印的labels。\n训练agent时，tool_call就是应该算loss，tool_response不算loss。\n\n### Q19: 模型参数freeze相关问题\n训练的过程中，冻结某些层时导致某些参数未参与梯度回传，请配置参数`--ddp_find_unused_parameters true`。\nfreeze_parameters和freeze_vit/freeze_aligner/freeze_llm：先freeze parameters再active parameters。`freeze vit/freeze aligner/freeze llm`这三个参数会对freeze parameters 和trainable parameters进行调整.因为有些模型的ViT中包含`aligner`，所以会将`aligner`单独加入trainable_parameters。\nfreeze_parameters_ratio这个参数的机制是从embedding开始从下往上freeze。\n\n### Q20: 序列并行相关问题\n序列并行支持pt, sft, dpo and grpo。参考这个例子[sequence_parallel](https://github.com/modelscope/ms-swift/tree/main/examples/train/sequence_parallel)。\nVLM模型的目前仅支持flash-attn，纯文本支持flash-attn和sdpa。\nsequence parallel可以和Liger kernel同时使用。\nsequence parallel和自定义loss冲突时，由于sequence parallel在自己的代码中定制了loss，可以自己改下[这里](https://github.com/modelscope/ms-swift/blob/main/swift/trainers/sequence_parallel/ulysses.py)。\n\n### Q21: 扩充词表\n用SWIFT框架扩充词表需要设置命令行参数`new_special_tokens`，`--modules_to_save embed_tokens lm_head`，详见[例子](https://github.com/modelscope/ms-swift/tree/main/examples/train/new_special_tokens)。\n\n### Q22: tuners相关问题\nSWIFT中的LlamaPro对多模态做了适配。\nLongLoRA只有LLaMA系列模型能用。\nLoRA训练和`--trainable_parameters`参数不兼容，LoRA模块之外其他的可训练参数用modules_to_save。\n\n### Q23: embedding/reranker训练\n[embedding训练例子](https://github.com/modelscope/ms-swift/blob/main/examples/train/embedding)。\n[reranker训练例子](https://github.com/modelscope/ms-swift/tree/main/examples/train/reranker)。\n数据格式见[自定义数据集](https://swift.readthedocs.io/zh-cn/latest/Customization/Custom-dataset.html)。\n\n### Q24: 分类任务训练\nSWIFT支持多标签分类，自定义数据集文档有数据格式，在命令行参数文档中搜索`problem_type`，其他和回归是一样的。\n注意：label字段和message字段同级。\n\n### Q25: thinking模型训练\n查看这个[issue](https://github.com/modelscope/ms-swift/issues/4030)。\n\n### Q26: 想问一下，SWIFT支持蒸馏吗？\n参考这个[例子](https://github.com/modelscope/ms-swift/blob/main/examples/sampler/distill/distill.sh)。\n\n### Q27: gkd训练student model和teacher model的model_type需要一致吗，一个dense一个moe可以吗?\n可以的，只需要词表一样，不过带MoE就会比较慢。\n\n### Q28: GRPO训练相关问题\nSWIFT现在支持多模态的GRPO训练。GRPO训练过程中loss接近0是正常情况，参考[issue](https://github.com/huggingface/open-r1/issues/239#issuecomment-2646297851)。\n设置sleep_mode，推理结束VllmEngine释放显存。下次调用时，再加载，而不是一直占用。\nGRPO训练时不想引入KL项，可以通过命令行参数beta设置。\nLoRA微调后继续做GRPO训练，请在命令行参数文档中搜索`--adapters`。\n由于算entropy会有额外的一点开销，所以默认没有记录熵曲线。如果需要，请设置`--log_entropy true`，\ncolocate模式不支持use_async_engine。\nGRPO不支持channel_loss。\nLiger kernel和padding free没法在GRPO阶段一起开。如果一起开，需要改liger grpo loss的实现，在liger kernel库中，不方便改。\n如果训练集有不同的task，请查看[多任务训练](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/DeveloperGuide/multi_task.html)。\n\n### Q29: reward函数（模型）相关问题\nreward_model和reward_funcs可以一起使用。\n自定义reward函数参考[examples/train/grpo/plugin](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/plugin)。\n针对math问题，要从数据集里面传solution，不然不好算accuracy。\n如果在ORM的自定义奖励函数中需要传入数据集中的某个列，请将该列放到messages之外的其他列。\n在GRPO训练的过程中如果需要指定一个llm-judge模型来做打分，请参考奖励模型的文档。\n\n### Q30: rollout相关问题\nRollout应该是不兼容pipeline parallel。\nvLLM推理引擎默认trust_rwmote_code为true。\n\n### Q31: 请教一个问题，grpo脚本中的save_steps指的是step还是global step？目前本地训练显示的global step是18， wandb上显示的step是628。\n`global_step`，本地tqdm显示的。\n\n### Q32: 默认只用 num_iterations=1 的话，clip 就失去作用了吧？dapo 的 clip higher 也没用。我看 veRL 有个 micro batch 可以设置单轮小批次更新 policy model 来使得 clip 项生效，ms-swift 的 mini batch 看源码貌似只是做了梯度累加？\n是的，需要num_iterations>1。\n\n### Q33: 请问gspo训练支持传入参数top_entropy_quantile吗？传入了--importance_sampling_level sequence后，还能实现对熵分布前x%的token的优化吗？\n支持，顺序是先正常计算loss（受importance_sampling_level影响），再根据top_entropy_quantile mask掉loss。\n\n### Q34: GRPO文档中的faq\n更多GRPO相关的FAQ，请查看[GRPO文档](https://swift.readthedocs.io/zh-cn/latest/Instruction/GRPO/GetStarted/GRPO.html#faq)\n\n### Q35: ppo等偏好训练相关问题\nPPO训练不支持梯度裁剪。\n目前PPO还只支持RM和policy是同一系列的模型(tokenizer/template)。\n不支持多轮的DPO。\n\n### Q36: MoE模型训练相关问题\nMoE模型LoRA训练，如果aux-loss基本没变化，将all-router也加到target_modules。\nLoRA训练中，路由器模块是否参与训练看gate是否是nn.Linear实现，如果是nn.Parameter就不训练，详见命令行参数[target_parameters](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html#tuner)。\n\n### Q37: Megatron-SWIFT训练相关问题\nCheckpoint保存，参考命令行参数[save_strategy](https://swift.readthedocs.io/zh-cn/latest/Megatron-SWIFT/Command-line-parameters.html)。\nMegatron多机训练时，因为pp并行只有在pp last rank有完整的信息, 日志在last rank打印，而不是从master node打印。\nMegatron-SWIFT支持了save_total_limit，支持了SwanLab监控训练，详见[Megatron-SWIFT命令行参数文档](https://swift.readthedocs.io/zh-cn/latest/Megatron-SWIFT/Command-line-parameters.html)\nViT用的是transformers的模型结构，目前没有并行，训练遇到OOM时降低`decoder_first_pipeline_num_layers`。\nMegatron-SWIFT支持新的模型，目前没有教程，请查看新增模型的PR。\nsequence_parallel的并行数等于tp数。\nFP8训练支持block wise，参考[examples/megatron/fp8例子](https://github.com/modelscope/ms-swift/tree/main/examples/megatron/fp8)。\n\n### Q38: 请问Megatron-SWIFT如何配置断点续训？\n配置`--mcore_model`加载checkpoint，另外根据需要配置这几个参数，`--finetune`，`--no_load_optim`，`--no_load_rng`。如果是LoRA断点续训，配置`--mcore_adapter`，其他同全参数训练，详见[Megatron-SWIFT命令行参数文档](https://swift.readthedocs.io/zh-cn/latest/Megatron-SWIFT/Command-line-parameters.html)。\n\n### Q39: mtp相关问题\n需要MTP训练，请设置命令行参数`mtp_num_layers`。\n如果base模型不附带MTP结构，可以从头初始化训练MTP。\n多模态的MTP目前还没支持。\n\n### Q40: 有个关于Megatron GKD的问题请教，如果teacher是Qwen3-235B，student是Qwen3-30BA3B，之前SFT 235B都是pp8然后decoder fist和decoder last设为11。如果我在GKD的时候也设置decoder first last，会不会影响student的并行？\n现在两个模型的并行参数是共用的，不同并行的设置会在v4版本后支持。\n\n### Q41: 量化模型训练相关问题\nQLoRA微调参考[例子](https://github.com/modelscope/ms-swift/tree/main/examples/train/qlora)。\n量化模型不能全参数微调，GPTQ模型的int型参数无法参与求导，只能附着LoRA等额外结构参与更新。\nQLoRA训练后的模型merge参考[QLoRA例子](https://github.com/modelscope/ms-swift/tree/main/examples/train/qlora)。\nMegatron-SWIFT不支持QLoRA训练。\n\n### Q42: 一些特殊模型的训练\nSWIFT目前不支持MiniCPM-O使用音频模态输入的训练。\n微调DeepSeek-VL-2，transformers用4.42以前的版本，`peft==0.11.*`。\nMoonlight-16B-A3B-Instruct微调。因为模型文件中禁止了训练, 参考DeepSeek-VL-2的解决方案，issue中搜索。\n微调Ovis2这个模型有点特殊，需要padding到max_length。设置一下`--max_length`。\nQwen2.5-Omni目前不支持talker训练，只有thinker。\nQwen2-Audio的sft不支持packing。\n\n### Q43: 请问在不支持flash attention的设备上attention implemation默认是什么呢？文档中默认是none\n默认使用sdpa。\n\n### Q44: 请问默认模型训练都是left padding是吧?\n训练可以选择使用左padding还是右padding。默认是右padding, `batch infer`都是左padding。\n\n### Q45: 请问下MoE的参数有哪些，参数表里关键字搜索不到？专家数量，专家路由这些参数怎么设置？\n直接用config.json中的参数。\n\n### Q46: SWIFT能够支持设置最小的learning rate吗，感觉最后减到太小了\n可以设置，`--lr_scheduler_type cosine_with_min_lr --lr_scheduler_kwargs '{\"min_lr\": 1e-6}'`。\n\n### Q47: 目前支持用yaml文件配置grpo和sft吗？\n都支持的，该配置是在main.py中直接处理成命令行。\n\n### Q48: 请问现在是不支持use_liger_kernel和log_entropy一起用吗？\n不支持。\n\n### Q49: 请问下，遇到这个报错，怎么处理？安装了apex也不行\n```text\nRuntimeError: ColumnParallelLinear was called with gradient_accumulation_fusion set to True but the custom CUDA extension fused_weight_gradient_mlp_cuda module is not found. To use gradient_accumulation_fusion you must install APEX with --cpp_ext and --cuda_ext. For example: pip install --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\" Note that the extension requires CUDA>=11. Otherwise, you must turn off gradient accumulation fusion.\n```\n设置一下`--gradient_accumulation_fusion false`。\n\n### Q50: 几个任务一起finetune vlm，不同任务视频采样规则不一致，ms-swift是否支持？在哪里配置？\n[命令行参数文档](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html)看下`interleave_prob`。\n\n### Q51: 想问一个问题，多模态packing预训练每次pytorch allocator cache flushes since last step后，显存使用好像就会增长一点，步数多了容易oom\n加个环境变量`PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'`。\n\n### Q52: use_logits_to_keep 现在多模态大模型上可以用吗？\n如果多模态token的展开在模型的forward内会报错。\n\n### Q53: 请问一下为什么训练到会有好几次显存大幅度增加，已经50step或者100step\n设置环境变量`PYTORCH_CUDA_ALLOC_CONF`，具体查看PyTorch文档。\n\n### Q54: 从qwen base模型微调成chat模型有没有实践文档，有什么要特别配置的吗?\n`swift sft`，没有其他需要特别配置的，参考[例子](https://github.com/modelscope/ms-swift/tree/main/examples/train/base_to_chat)。\n\n### Q55: 模型训练后，回复重复了很多内容\n参考[预训练与微调](https://swift.readthedocs.io/zh-cn/latest/Instruction/Pre-training-and-Fine-tuning.html)。如果训练过程中出现重复的情况，请多训练几个epoch, 清洗数据, 全参数训练, 采用RLHF的方式缓解。\n\n### Q56: 请问为什么 --torch_dtype float16 （卡不能使用bf16）会出现报错：lib/python3.12/site-packages/torch/amp/grad_scaler.py\", line 260, in _unscale_grads_ raise ValueError(\"Attempting to unscale FP16 gradients.\") ValueError: Attempting to unscale FP16 gradients.\n全参数，不能fp16训练的。\n\n### Q57: 请问下，lora参数合并报错，目前peft是0.11.0，这个是因为peft版本需要升级吗\n```text\nFile \"/opt/conda/lib/python3.9/site-packages/peft/config.py\", line 118, in from_peft_type\n return config_cls(**kwargs)\nTypeError: __init__() got an unexpected keyword argument 'corda_config'\n```\n训练和合并的peft版本不一致导致的。\n\n### Q58: 请问这个问题如何解决？safetensors_rust.SafetensorError: Error while deserializing header: HeaderTooLarge\n磁盘空间不足了，模型没有保存完整。\n\n### Q59: 这个错误为什么会出现在这，numpy.object找不到在哪？\n`numpy==1.26.3`，尝试一下。\n\n### Q60: unsloth训练，报错：assert(type(target modules) in (list,tuple,))。配置的参数是--target modules all-linear\n别用`all-linear`，改为具体的模块列表，比如`--target_modules q k v`。\n\n### Q61: 请问对于qwen2.5-omni来说--freeze_vit false意味这视觉编码器和音频编码器都打开了，有什么办法可以只打开音频编码器不打开视觉编码器吗？\n`--target_regex`写一下。\n\n## 推理\n\nSWIFT支持python脚本、命令行、ui界面推理，详见[推理和部署](https://swift.readthedocs.io/zh-cn/latest/Instruction/Inference-and-deployment.html)。\n\n### Q1:SWIFT推理如何设置模型？\n如果是全参数训练的模型、LoRA训练后合并的模型或者从model hub下载的模型，设置命令行参数`--model `；LoRA训练后未合并的模型，用`--adapters`设置，同时可通过`--model`指定基模路径。\n\n### Q2: SWIFT如何使用数据集进行推理？推理结果保存在哪儿？\n`--val_dataset `，指定数据集。对于训练后的模型也可以设置参数`--load_data_args true`。推理结果保存路径通过`--result_path your_path`设置，日志中会打印路径。详见文档[命令行参数](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html)。\n如果需要保留推理数据集中额外的字段，请设置`--remove_unused_columns false`。\n\n### Q3: SWIFT如何设置批量推理？\n如果infer_backend为`transformers`，设置命令行参数`--max_batch_size 16`，或[python脚本](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo.py)。这里max_batch_size指的是每张卡上的batch_size。\n\n### Q4: SWIFT如何设置流式推理？\n`--stream true`，此时推理结果将逐条写入jsonl文件。需要注意的是，流式推理不支持ddp。\n\n### Q5: vLLM和SGLang推理后端相关的问题\n对于LoRA训练的模型，请查看vLLM和SGLang文档，如果支持LoRA推理则不需要合并。此外，SGLang推理目前不支持多模态。\n\n### Q6: 生成参数相关的问题\ntemperature等参数默认从generation_config.json中读取。设置`--temperature 0`或者`--top_k 1`可以取消推理随机性。\n\n### Q7: 如何将system_prompt置空？命令行不设置system参数，但是它会加上默认的system。\n设置`--system ''`。\n\n### Q8: 推理时如何计算acc/rouge等指标？\n参考[推理参数metric](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html#id14)。\n\n### Q9: 模型推理的时候如果需要在特定前缀下继续推理的话是设置哪个参数？\n参数`--response_prefix`。\n\n### Q10: 数据answer里面已经包含了部分prompt，希望补全answer，应该怎么修改inference？\n```text\n{\"messages\": [{\"role\": \"system\", \"content\": \"\"}, {\"role\": \"user\", \"content\": \"\"}, {\"role\": \"assistant\", \"content\": \"answer1, \"}]}\n```\n用swift3.0以后的版本是可以的，参考[examples/infer/demo_agent](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_agent.py)。\n\n### Q11: 多模态模型推理时如何限制最大像素，以减少显存占用？\n设置命令行参数`--max_pixels xxx`、环境变量`MAX_PIXELS=xxx`、或特定模型参数`--model_kwargs '{\"max_pixels\": xxx}'`，其中环境变量仅对文档中对应的模型生效，详见文档[特定模型参数](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html#id18)。\n\n### Q12: SWIFT推理如何输出概率值logprobs参数？\n命令行推理设置`--logprobs true`，python脚本推理设置`request_config = RequestConfig(..., logprobs=True, top_logprobs=2)`，参考[test_logprobs.py](https://github.com/modelscope/ms-swift/blob/main/tests/infer/test_logprobs.py)。\n\n### Q13: SWIFT推理如何输出last_hidden_state？\n没有例子，可以参考GRPO trainer的`_get_last_hidden_state`方法。\n\n### Q14: transformers，vllm，ollama等推理结果不一致问题\nSWIFT的template是对齐transformers的。检查推理参数是否对其。此外，VllmEngine和TransformersEngine是有差异的。\n\n### Q15: embedding/reranker模型推理\nembedding模型推理参考这里的[例子](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_embedding.py)。reranker模型推理参考这里的[例子](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_reranker.py)。\n\n### Q16: 请问在使用python脚本推理时，如何使用cpu?\n设置环境变量，`os.environ['CUDA_VISIBLE_DEVICES'] = '-1'`。\n\n### Q17: 使用swift infer命令进行推理，支持多机推理吗？\n如果单节点放得下模型，外面封装k8s就行。如果单节点放不下那就不支持。\n\n### Q18: swift sample的时候，好像不支持batch？好像是for循环一个个例子sample，有点慢\n有一个[脚本](https://github.com/modelscope/ms-swift/blob/main/examples/train/rft/rft.py)，可以用多进程对数据集拆分采样。\n\n### Q19: 特殊模型依赖版本相关问题\nQwen2-Audio推理结果出现混乱，请使用transformers4.48。\ntransformers4.55.2训练的LoRA不能使用小于4.52的版本加载了，详见[issue#5440](https://github.com/modelscope/ms-swift/issues/5440)。\nswift对不同版本的qwen-vl-utils做了兼容，使用qwen2.5-vl和qwen3-vl模型时不需要切换该依赖版本。\n\n### Q20: 报错，safetensors_rust.SafetensorError: Error while deserializing header:MetadataIncompleteBuffer\n模型权重损坏了。\n\n## 导出\n\n### Q1: autoawq相关的报错\n如果推理没有涉及AWQ量化模型，但出现了autoawq相关的报错，可以尝试卸载autoawq再进行推理。不支持AWQ量化的模型，尝试用GPTQ进行量化。\n\n### Q2: SWIFT量化模型时，一张卡上放不下模型的情况\n尝试设置`--device_map cpu`。或者多卡加载模型，单卡量化。\n\n### Q3: 想问一下用swift export对qwen2.5 72B模型进行gptq int4量化，max model length=32768用的是默认值，给的校准数据集有128个样本，但是量化的时候报错了，报错日志是：factorization could not be completed because the input is not positive-definite(the leading minor of order 18145 is not pisitive-definite)。是什么原因？\n海森矩阵不正定的问题，试试其他的数据集。\n\n### Q4: swift export的时候传入自定义的template_type,是不是就可以永久改掉template_type了？如果swift export --template_type 自定义,是不是就可以把模型对应的template改掉\n不会被修改,swift中的template是定义在swift内部的,不是以jinja方式保存的。\n\n### Q5: 模型训练完能直接转gguf格式吗？\n目前只支持导出ModelFile，详见文档[命令行参数](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html)。\n\n## 部署\n\n### Q1: SWIFT部署如何设置模型？\n同上面推理Q1。\n\n### Q2: SWIFT如何进行多卡部署？\n详见[例子](https://github.com/modelscope/ms-swift/tree/main/examples/deploy)。如果是transformers engine，不支持DDP，不能多卡部署。此外，不支持异构部署，如不同型号的显卡、各显卡设置不同的存储占比等。\n\n### Q3: 通过--system参数指定system prompt与数据集中每个数据前加system prompt以及template的system prompt是不是有一个就行？这些方式对模型来说，是不是一样的？\nsystem优先级：数据集中的>命令行的>template中默认的。\n\n### Q4: 客户端多模态输入相关问题\n客户端传入图片、音频等，见[客户端例子](https://github.com/modelscope/ms-swift/tree/main/examples/deploy/client/mllm)。\n如果图片url非法，可以设置请求的超时时间，环境变量`SWIFT_TIMEOUT`，或者`InferClient`中可以传参数。\n\n### Q5: 生成参数设置相关问题\ntemperature等参数推理只能启动前设置，部署可以在启动时设置默认值，之后在客户端继续设置，覆盖默认值。\n\n### Q6: SWIFT部署的模型怎么设置流式生成？\n客户端控制的，查看[examples/deploy/client](https://github.com/modelscope/ms-swift/tree/main/examples/deploy/client)。\n\n### Q7: SWIFT部署如何输出token的概率？\n服务端设置`--logprobs true`，要客户端传参数，`request_config = RequestConfig(..., logprobs=True, top_logprobs=2)`。\n\n### Q8: 部署模型时，thinking相关问题\n如果需要禁止思考，目前只能在swift deploy启动的时候禁止thinking。查看这个[issue](https://github.com/modelscope/ms-swift/issues/4030)。\n\n### Q9: 部署时，设置什么参数可以实现一次输出多个结果？\n`RequestConfig`参数`n`。\n\n### Q10: SWIFT部署，指定--infer_backend vllm，和直接使用vllm部署相关问题\n如果两者推理结果相差较多，可能是template没对齐。如果推理速度相差较多，可能是图像分辨率不一致。swift默认使用V1 engine，可以通过环境变量`VLLM_USE_V1=1`控制。\n\n### Q11: 特殊模型和依赖版本相关问题\n如果遇到报错没有“model.language_model.embed_tokens.weight”，训练前后的transformers版本不一致。\nqwen2.5使用fp16推理如果遇到返回乱码，尝试bf16。\n\n### Q12: 有个问题想问一下，qwen2-7b部署后使用客户端时，调用openai的api要使用client.completions.create，不能使用client.chat.completions.create，但是使用qwen2-7b-instruct-q5_k_m.gguf的时候可以使用client.chat.completions.create，这是为什么呀？\nbase模型可以用client.chat.completions.create的，不过这个是兼容行为。\n\n## 评测\n\n### Q1: SWIFT支持的评测集有哪些？以及如何使用自定义评测集？\n标准评测集和用户自定义评测集的使用详见文档[评测](https://swift.readthedocs.io/zh-cn/latest/Instruction/Evaluation.html)。\n\n### Q2: 官方支持的评测数据集手动下载后，swift eval能配置本地路径评测吗？\n离线评测请参考EvalScope文档[快速上手](https://evalscope.readthedocs.io/zh-cn/latest/get_started/basic_usage.html)\n\n### Q3: eval微调后的模型，总是会在固定的百分比停掉，但是vllm服务看着一直是有在正常运行的。模型越大，断开的越早。\n`SWIFT_TIMEOUT`环境变量设置为-1。\n\n### Q4: 评估的时候可不可以控制数据集条数？评估一个mmlu需要一个多小时，也太慢了。\n配置参数`--eval_limit`，这里的`--eval_limit`是控制了每个subset的条数，比如mmlu有50多个subset，每个limit10条，那就是500多条。\n\n### Q5: 问一下评估swift eval里，模型最多生成1024token就结束了，这个如何修改？设置--max_new_tokens 5000，看起来没起作用\n查看命令行参数[eval_generation_config](https://swift.readthedocs.io/zh-cn/latest/Instruction/Command-line-parameters.html#id16)\n\n### Q6: 请教一下，想使用OpenCompass的后端评测，如何从本地加载下载好的数据集？\nOpenCompass后端不支持设置`data_args`。\n\n### Q7: swift eval 来评估模型，--eval_backend OpenCompass不支持自定义数据集吗？\n```text\nValueError: eval_dataset: /mnt/workspace/data.jsonl is not supported.\neval_backend: OpenCompass supported datasets: ['C3', 'summedits', 'WiC', 'csl', 'lambada', 'mbpp', 'hellaswag', 'ARC_e', 'math', 'nq', 'race', 'MultiRC', 'cmb', 'ceval', 'GaokaoBench', 'mmlu', 'winogrande', 'tnews', 'triviaqa', 'CB', 'cluewsc', 'humaneval', 'AX_g', 'DRCD', 'RTE', 'ocnli_fc', 'gsm8k', 'obqa', 'ReCoRD', 'Xsum', 'ocnli', 'WSC', 'siqa', 'agieval', 'piqa', 'cmnli', 'cmmlu', 'eprstmt', 'storycloze', 'AX_b', 'afqmc', 'strategyqa', 'bustm', 'BoolQ', 'COPA', 'ARC_c', 'PMMEval', 'chid', 'CMRC', 'lcsts']\n```\nOpenCompass不支持自定义数据集，用native可以自定义模式。\n\n### Q8: evalscope原生是可以生成报告的，其他后端如opencompass是不支持生成报告可视化是吗？\n目前只支持native的可视化，其他后端还不支持。\n\n### Q9: 请问一下评测ifeval报这个错是什么原因？\n```text\n[Errno 20] Not a directory: '/root/nltk_data/tokenizers/punkt_tab.zip/punkt_tab/english/collocations.tab'\n```\n解压这个文件，`unzip /path/to/nltk_data/tokenizers/punkt_tab.zip`。\n\n### Q10: 请问评测时eval_backend='OpenCompass'，怎么指定离线数据集路径？\n查看[数据准备教程](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/backend/opencompass_backend.html#id3)，下载数据集并解压。不用指定`dataset-args`，将数据集文件夹（即data文件夹）放置在当前工作路径下即可。\n\n### Q11: 用evalscope报这个错是什么原因？\n```text\nunzip: cannot find or open /root/nltk_data/tokenizers/punkt_tab.zip, /root/nltk_data/tokenizers/punkt_tab.zip.zip or /root/nltk_data/tokenizers/punkt_tab.zip.ZIP\n```\n这是在下载nltk的依赖，手动下载[punkt_tab.zip](https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/open_data/nltk_data/punkt_tab.zip)，解压到`~/nltk_data/tokenizers`下面。\n\n### Q12: 为啥纯文本没问题，测多模态我们指定路径了，但他还是检测不到数据集，会去下载？\nVLMEvalKit流程跟native不一样，会自己下载数据放到`~/LMUData/`下面，详见[文档](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/backend/vlmevalkit_backend.html#id2)。\n\n### Q13: 请问一下swift eval做benchmark评测的时候，是否可以指定llm作为judge, 参数应该怎么传进去？\n支持，使用swift得从`extra_eval_args`去传递`judge-model-args`参数，包括`api_key，api_url，model_id`，整体是一个json字符串。\n\n### Q14: 请问在执行eval的时候出现了多卡显存分配不均是什么原因？\n```shell\nNPROC_PER_NODE=8\nASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7\\ MAX_PIXELS=802816\\ swift eval\\\n--model \"$MODEL_PATH” \\$EXTRA_ARGS \\\n--eval_backend Native \\ --infer_backend transformers\\ --device_map auto \\\n--eval_limit\"$EVAL_LIMIT\"\\ --eval_dataset general_qa\\\n--dataset_args \"{\\\"general_qa\\\": {\\\"local_path\\\": \\\"${DATA_PATH}\\\", \\\"subset_list\\\": [\\\"${SUBSET_NAME}\\\"]}}\" \\ --host 127.0.0.1\\> \"$LOG_FILE\" 2>&1\n```\nswift eval不支持DDP方式启动。\n\n### Q15: 请问哪里可以看到swift评测的时候送入的query除了问题之外还有哪些额外的字段呢？\n最简单的方法是看输出的reviews文件中的input字段，是输入给模型的内容转换后的Markdown格式。如果用backend是opencompass的话没有这些，需要用native backend。\n\nms-swift的eval能力使用了魔搭社区评测框架EvalScope, 复杂能力请直接使用[EvalScope框架](https://evalscope.readthedocs.io/zh-cn/latest/get_started/introduction.html)。\n"
},
{
"path": "docs/source/Instruction/GKD.md",
"content": "# GKD\n\nGKD（Generalized Knowledge Distillation，广义知识蒸馏）训练算法由论文 [On-Policy Distillation of Language Models: Learning from Self-Generated Mistakes](https://arxiv.org/pdf/2306.13649) 提出。该算法通过结合离线（off-policy）和在线（on-policy）学习策略，将教师模型的知识迁移到学生模型中。\n\n## 损失函数\n\n当给定输入序列 $x$ 与输出序列 $y$，GKD 的损失函数可以写为：\n\n$$\n\\mathcal{L}_{\\text{GKD}}(x, y) = \\sum_{t=1}^{|y|} D(P_{\\text{teacher}}(\\cdot | x, y_{ 对极端情况（$\\beta = 0$ 或 $\\beta = 1$），直接计算单个 KL 散度：\n> - 当 $\\beta = 0$ 时：直接定义 $D = \\text{KL}(P_{\\text{teacher}} \\| P_{\\text{student}})$（Forward KL，Mode-covering）\n> - 当 $\\beta = 1$ 时：直接定义 $D = \\text{KL}(P_{\\text{student}} \\| P_{\\text{teacher}})$（Reverse KL，Mode-seeking）\n> - 当 $0 < \\beta < 1$ 时：使用上述混合分布公式进行插值\n\n通过调节 $\\beta$ 参数，可以在不同的散度度量之间进行插值，当 $\\beta = 0.5$ 时，散度为标准的对称 JSD。\n\n## 三种训练模式\n\nGKD训练具有三种训练模式，区别在于输出序列 $y$ 的来源。\n\n### 模式选择逻辑\n\n训练时，每个样本按照以下优先级选择模式：\n\n```python\n# 伪代码：模式选择逻辑\nif random() < lmbda:\n # Mode 1: On-Policy 学习，由学生模型采样输出序列\n y = student.generate(x)\n source = \"student\"\nelif seq_kd:\n # Mode 2: Sequential KD，由教师模型采样输出序列\n y = teacher.generate(x)\n source = \"teacher\"\nelse:\n # Mode 3: 使用数据集中的输出序列\n y = y_ground_truth\n source = \"dataset\"\n\n# 相同的损失函数\nloss = D_JSD(P_teacher(·|x,y), P_student(·|x,y))\n```\n\n### Mode 1: On-Policy 学习\n设置参数`lambda`, 以概率 $\\lambda$ 触发，使用学生模型采样 $y \\sim P_{\\text{student}}(\\cdot | x)$\n\n- 学生模型从**自己生成的序列**中学习\n- 暴露在自己可能犯的错误中，学会**自我纠正和错误恢复**\n- 对齐训练分布与推理分布\n- 提升模型的鲁棒性和实际应用表现\n\n**适用场景**：\n- 学生模型已有一定生成能力\n- 希望提升模型在真实推理场景下的表现\n\n### Mode 2: Sequential KD（`seq_kd=True` 且未触发 on-policy）\n设置参数 `seq_kd=True`, 当未触发 on-policy 时，使用教师模型采样\n\n**数据来源**：$y \\sim P_{\\text{teacher}}(\\cdot | x)$\n\n### Mode 3: 离线学习（其他情况）\n\n**数据来源**：$y = y^* \\sim \\text{Dataset}$\n\n- 学生模型从**数据集的标注序列**中学习\n\n\n## 参数设置\n\n我们可以通过设置以下参数进行 GKD 训练：\n\n### 基础参数\n\n| 参数 | 类型 | 默认值 | 取值范围 | 说明 |\n|------|------|--------|---------|------|\n| `--teacher_model` | str | None | - | 教师模型路径或模型 ID |\n| `--beta` | float | 0.5 | [0.0, 1.0] | 散度插值系数\nmu值衰减与loss计算代码实现 \n请参考`GRPOTrainer`的`_compute_chord_loss`方法：\n\n\n训练参考该[脚本](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/internal/chord.sh)\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/CISPO.md",
"content": "# Clipped Importance Sampling Policy Optimization (CISPO)\n\n**版本依赖**：ms-swift>=3.11\n\nClipped Importance Sampling Policy Optimization (CISPO) 是 [MiniMax-M1](https://arxiv.org/abs/2506.13585) 论文中提出的一种强化学习算法。相比GRPO（Group Relative Policy Optimization）算法，CISPO 对重要性采样权重（importance sampling weights）本身进行裁剪。\n\n## 算法原理\n为便于理解，我们基于 GRPO 算法进行对比说明。\n\nGRPO通过裁剪策略比率来限制策略更新幅度，其损失函数为：\n\n$$\n\\mathcal{L}_{\\text{GRPO}}(\\theta) = -\\mathbb{E}\\left[\\min\\left(r_t(\\theta) \\cdot \\hat{A}_t, \\text{clip}(r_t(\\theta), 1-\\epsilon, 1+\\epsilon) \\cdot \\hat{A}_t\\right)\\right]\n$$\n\n其中 $r_t(\\theta) = \\frac{\\pi_\\theta(a_t|s_t)}{\\pi_{\\theta_{\\text{old}}}(a_t|s_t)}$ 是重要性采样比。\n\n在处理长推理链条时，这种裁剪方式可能导致以下问题：\n\n**关键 Token 的梯度被抑制**：在复杂推理任务中，某些关键的低概率 token（如 *However, Recheck, Wait, Aha*）对于触发深度思考和推理纠错至关重要。这些 token 在旧策略 $\\pi_{\\theta_{\\text{old}}}$ 中概率较低，当新策略试图提高其概率时，会导致较大的策略比率 $r_t(\\theta)$，GRPO 的裁剪机制会将这些 token 丢弃。\n\n\n### CISPO 的解决方案\n\nCISPO 的核心思想是：裁剪重要性采样权重，保留梯度更新。具体来说，CISPO 的损失函数为：\n\n$$\n\\mathcal{L}_{\\text{CISPO}}(\\theta) = -\\mathbb{E}\\left[\\text{detach}\\left(\\min(r_t(\\theta), \\epsilon_{\\text{high}})\\right) \\cdot \\hat{A}_t \\cdot \\log \\pi_\\theta(a_t|s_t)\\right]\n$$\n\n其中 $r_t(\\theta) = \\frac{\\pi_\\theta(a_t|s_t)}{\\pi_{\\theta_{\\text{old}}}(a_t|s_t)}$ 是重要性采样比。\n\n**关键机制**：\n- 对重要性采样权重进行裁剪：$\\min(r_t(\\theta), \\epsilon_{\\text{high}})$\n- **detach 操作**：裁剪后的权重不参与梯度计算，作为常数系数\n- 梯度来自 $\\log \\pi_\\theta(a_t|s_t)$ 项，保证所有 token 都有梯度贡献\n\n\n## 实现细节\nCISPO 的伪代码实现如下：\n\n```python\nlog_ratio = per_token_logps - old_per_token_logps\nimportance_weights = torch.exp(log_ratio) # r_t(θ) = π_θ / π_θ_old\n\nclamped_ratios = torch.clamp(importance_weights, max=epsilon_high).detach()\n\nper_token_loss = -clamped_ratios * advantages.unsqueeze(1) * per_token_logps\n```\n\n## 参数设置\n\n我们可以基于 `GRPOTrainer`，通过设置以下参数实现 CISPO 训练：\n\n```bash\n--loss_type cispo\n--epsilon_high 5.0\n```\n\n> 相比其他算法, cispo 的 epsilon_high 一般取值较大，minimax论文中未给出具体的参数设置，这里的值参考论文[ScaleRL](https://arxiv.org/pdf/2510.13786)的实验设置\n\n其他训练参数参考 [GRPO参数文档](../../Command-line-parameters.md#grpo参数)\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/DAPO.md",
"content": "# DAPO: An Open-Source LLM Reinforcement Learning System at Scale\n\n\n\n[Decoupled Clip and Dynamic sAmpling Policy Optimization (DAPO)](https://arxiv.org/abs/2503.14476)在GRPO的基础上设置了几种trick，分别是\n- [Clip Higher](#clip-higher)\n- [Dynamic Sampling](#dynamic-sampling)\n- [Token level Loss](#token-level-loss)\n- [Overlong Filtering](#overlong-filtering)\n- [Soft Overlong Punishment](#soft-overlong-punishment)\n\n## Clip Higher\nPPO和GRPO使用对称裁剪范围(如±0.2)限制策略更新幅度，虽然保证了稳定性，但也制约了模型的探索能力。特别是当某些token在旧策略中概率极低时，即使当前梯度显示其应被强化(A>0)，最大增幅也被严格限制。\n\nDAPO使用非对称裁剪范围, 提高上裁剪范围来鼓励模型进行探索：\n\n- 上界(鼓励侧)放宽至0.28\n- 下界(抑制侧)保持0.2不变\n\nGRPO中，默认使用`epsilon`设置用对称裁剪范围\n\n使用参数\n\n- `epsilon_high` 设置上裁剪范围，此时参数`epsilon` 为下裁剪范围\n\n## Dynamic Sampling\nGRPO对每个问题采样多个回答计算组间优势，\n\n$$\n\\hat{A}_{i,t} = \\frac{R_i - \\text{mean}(\\{R_j\\}_{j=1}^G)}{\\text{std}(\\{R_j\\}_{j=1}^G)}\n$$\n而当生成的所有输出{oi}获得相同奖励时，组间优势等于0，会出现梯度消失导致训练效率下降\n\nDAPO引入动态采样策略解决这一问题：\n\n- 采样阶段跳过组间奖励标准差为0的数据\n- 持续生成样本直到填满批次\n\n\n使用参数\n\n- `dynamic_sample true` 来开启动态采样\n- `max_resample_times` 设置最多重采样次数\n\n## Token level Loss\nGRPO 在归一化损失时采用句子级归一化，这会导致损失计算具有长度偏差。\n\nDAPO 使用token级归一化，避免了回答长度在损失计算上的偏差。\n\n使用参数\n\n- loss_type bnpo/dapo 来使用token级归一化\n\n> loss_type 计算公式可参考[文档](../DeveloperGuide/loss_types.md)\n\n## Overlong Filtering\nDAPO 认为被强制截断的回复的奖励噪声较大，可能会导致模型难以区分质量问题和长度问题。为此，DAPO 筛除了训练中被截断的数据，使其不参与损失计算。\n\n使用参数\n\n- overlong_filter 开启对超长样本的过滤\n\n\n## Soft Overlong Punishment\n语言模型常面临生成长度控制难题：\n\n- 过长输出可能被截断，导致正确内容被误判\n\n- 无约束生成长度影响实用性和计算效率\n\nDAPO 设计了三段式长度惩罚函数：\n\n$$\nR_{\\text{length}}(L) =\n\\begin{cases}\n0, & L \\leq L_{\\text{max}} - L_{\\text{cache}} \\\\[10pt]\n\\dfrac{(L_{\\text{max}} - L_{\\text{cache}}) - L}{L_{\\text{cache}}}, & L_{\\text{max}} - L_{\\text{cache}} < L \\leq L_{\\text{max}} \\\\[10pt]\n-1, & L > L_{\\text{max}}\n\\end{cases}\n$$\n\n在长度位于 $(L_{\\text{max}} - L_{\\text{cache}} < L \\leq L_{\\text{max}})$ 区间时设置线性递增惩罚，在 $(L > L_{\\text{max}})$ 时设置最大惩罚(-1)\n\n\n使用参数\n\n- `reward_funcs soft_overlong` 启用该奖励函数\n- `soft_max_length` 设置L_max，默认等于为模型的最长输出长度(max_completion_length)。\n- `soft_cache_length`: 设置L_cache\n\n## 参数设置\n综上所述，我们可以基于GRPOTrainer，设置以下参数实现 DAPO 训练。\n\n| 参数 | 类型 | 值 |\n|----------------------|-----------|-------------|\n| `--loss_type` | `str` | `bnpo`/`dapo`|\n| `--epsilon_high` | `float` | `0.28` |\n| `--dynamic_sample` | `bool` | `true` |\n| `--max_resample_times` | `int` | `3` |\n| `--overlong_filter` | `bool` | `true` |\n| `--reward_funcs` | `str` | `soft_overlong`|\n| `--soft_cache_length` | `int` | `4096`|\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/GSPO.md",
"content": "# Group Sequence Policy Optimization\n\n**版本依赖**：ms-swift>=3.7\n\n[Group Sequence Policy Optimization](https://arxiv.org/abs/2507.18071)中指出GRPO在计算重要性采样权重时，是在token级别进行操作的。然而，这种做法由于每个token仅采样一次，无法实现有效的分布校正，反而会在模型训练过程中引入高方差噪声，极易导致模型的梯度估计不稳定，最终造成模型训练的崩塌。因此，论文认为，优化目标的单位应该与奖励的单位保持一致。由于奖励通常是在序列级别（即完整生成的回复）给出的，因此更合理的做法是将 off-policy 校正和优化也提升到序列级别，而非 token 级别。以下是三种计算策略对比：\n\n1. GRPO\n对每个 token 独立计算重要性采样比，具体公式为\n\n$$\nw^{\\mathrm{GRPO}}_{i,t} = \\frac{\\pi_\\theta (y_{i, t} \\mid x, y_{i, 注意：根据梯度推导（即论文中的公式(11)和(18)），当各 token 的 advantage 相同时，GSPO-token 与 GSPO 等价。当前的 GRPO 实现中，所有 token 的 advantage 实际上都是基于句子级 reward 并在 group 内进行归一化，因此在这种设置下，GSPO-token 和 GSPO 在理论上是等价的。不过，GSPO-token 为未来更细粒度（token 级别）的 advantage 提供了支持。\n\n伪代码实现\n```python\nlog_ratio = per_token_logps - old_per_token_logps\n# GRPO\nlog_importance_weights = log_ratio\n\n# GSPO (Sequence-Level)\nseq_weight = (log_ratio * mask).sum(-1) / mask.sum(-1)\nlog_importance_weights = seq_weight.unsqueeze(-1) # (B,1)\n\n# GSPO-token\nseq_weight = (log_ratio * mask).sum(-1) / mask.sum(-1)\nlog_importance_weights = seq_weight.detach().unsqueeze(-1) + (per_token_logps - per_token_logps.detach())\n\nimportance_weights = torch.exp(log_importance_weights)\n```\n\n我们可以在 GRPO 训练的基础上，通过参数 `--importance_sampling_level` 选择不同的算法：\n\n- `importance_sampling_level token` （默认，GRPO 实现）\n- `importance_sampling_level sequence` （GSPO）\n- `importance_sampling_level sequence_token` （GSPO-token）\n\n其中 sequence_token 要求 ms-swift >= 3.8\n\n论文其他超参\n```bash\n --epsilon 3e-4 # from paper section 5.1\n --epsilon_high 4e-4 # from paper section 5.1\n --steps_per_generation 4 # from paper section 5.1 (each batch of rollout data is partitioned into four minibatches for gradient updates)\n --beta 0 # zero kl regularization https://github.com/volcengine/verl/pull/2775#issuecomment-3131807306\n```\n\n训练可以参考该[脚本](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo/internal/gspo.sh)\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/REINFORCEPP.md",
"content": "# REINFORCE++: An Efficient RLHF Algorithm with Robustness to Both Prompt and Reward Models\n\n**版本依赖**：ms-swift>=3.10\n\n[REINFORCE++ Baseline](https://arxiv.org/abs/2501.03262) 是 REINFORCE++ 算法的简化版本，适用于 outcome rewards（response-level 标量奖励）。它与 GRPO 类似，对每个prompt输入采样多条模型输出，并使用组内 baseline 来估计优势函数，主要区别在于标准化时使用的统计量不同。\n\n\n## 算法原理\n为便于理解，我们基于 GRPO（Group Relative Policy Optimization）算法进行对比说明。\n\nGRPO 和 REINFORCE++ Baseline 都采用组内对比的方式来估计优势函数，主要区别在于：\n\n### 区别1：标准化时使用的统计量不同\n\n**GRPO (Group Relative Policy Optimization)**\n\n对每个 prompt 生成 $G$ 个响应样本，使用**组内所有样本的均值和标准差**进行标准化：\n\n$$\n\\hat{A}_{i} = \\frac{R_i - \\text{mean}(\\{R_j\\}_{j=1}^G)}{\\text{std}(\\{R_j\\}_{j=1}^G)}\n$$\n\n当设置 `scale_rewards='batch'` 时，使用**原始奖励的批次 std**：\n\n$$\n\\hat{A}_{i} = \\frac{R_i - \\text{mean}(\\{R_j\\}_{j=1}^G)}{\\text{std}(\\{R_j\\}_{j=1}^{N})}\n$$\n\n其中 $N$ 是批次中所有样本数。\n\n**REINFORCE++ Baseline**\n\n对每个 prompt 生成 $G$ 个响应样本，先减去组内均值，再使用**减去组内均值后的奖励**的标准差进行标准化：\n\n$$\n\\begin{align}\n\\tilde{A}_{i} &= R_i - \\text{mean}(\\{R_j\\}_{j=1}^G) \\\\\n\\hat{A}_{i} &= \\frac{\\tilde{A}_{i}}{\\text{std}(\\{\\tilde{A}_k\\}_{k=1}^{N})}\n\\end{align}\n$$\n\n其中 $N$ 是批次中所有样本数。\n\n**关键区别**：\n- **GRPO**：标准化时使用**原始奖励 $R$** 的标准差\n- **REINFORCE++**：标准化时使用**减去组内均值后的奖励 $\\tilde{A}$** 的标准差\n\n### 区别2: KL 散度正则化\n\n与 RLOO 类似，REINFORCE++ Baseline 将 KL 散度整合到奖励项中：\n\n$$\nR'_i = R_i - \\beta \\cdot \\text{KL}(\\pi_\\theta || \\pi_{\\text{ref}})\n$$\n\n其中 $\\beta$ 是 KL 散度的权重系数（对应参数 `beta`），$\\pi_{\\text{ref}}$ 是参考策略。\n\n## 参数设置\n\n我们可以基于 `GRPOTrainer`，通过设置以下参数实现 REINFORCE++ Baseline 训练：\n\n```bash\n--advantage_estimator reinforce_plus_plus\n--scale_rewards batch\n--kl_in_reward true\n```\n\n训练可以参考该[脚本](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/internal/reinforce_plus_plus.sh)\n\n### 重要参数说明\n\n- **`--advantage_estimator`**：选择优势函数估计方法\n - `grpo`（默认）：标准化时使用原始奖励的标准差\n - `reinforce_plus_plus`：标准化时使用减去组内均值后的奖励的标准差\n\n- **`--kl_in_reward`**：控制 KL 散度正则化项的处理位置\n - `false`：KL 散度作为损失函数的独立正则化项（GRPO 默认）\n - `true`：KL 散度直接从奖励中扣除（REINFORCE++ 原始实现）\n\n- **`--scale_rewards`**：控制标准化方式\n - `group`（默认）：组内标准化\n - `batch`：全局批次标准化（REINFORCE++原始实现）\n - `none`：不进行标准化\n\n- **`--num_generations`**：每个 prompt 生成的样本数量 $G$\n\n- **`--beta`**：KL 散度正则化系数 $\\beta$\n\n其他参数参考 [GRPO参数](../../Command-line-parameters.md#grpo参数)\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/RLOO.md",
"content": "# REINFORCE Leave-One-Out (RLOO)\n\n**版本依赖**：ms-swift>=3.10\n\n[REINFORCE Leave-One-Out (RLOO)](https://arxiv.org/abs/2402.14740) 基于经典的 REINFORCE 策略梯度方法，通过留一法（Leave-One-Out）构造无偏的优势函数基线。\n\n## 算法原理\n\n为便于理解，我们基于 GRPO（Group Relative Policy Optimization）算法进行对比说明。\n\nGRPO 和 RLOO 都采用组内对比的方式来估计优势函数，避免了全局基线估计带来的高方差问题。两者的核心区别主要体现在以下两个方面：\n\n### 区别1：优势函数基线的构造方法\n\n**1. GRPO (Group Relative Policy Optimization)**\n\nGRPO 对每个 prompt 生成 $G$ 个响应样本，使用**组内所有样本的均值和标准差**进行标准化：\n\n$$\n\\hat{A}_{i} = \\frac{R_i - \\text{mean}(\\{R_j\\}_{j=1}^G)}{\\text{std}(\\{R_j\\}_{j=1}^G)}\n$$\n\n其中：\n- $R_i$ 是第 $i$ 个样本的奖励值\n- $\\text{mean}(\\{R_j\\}_{j=1}^G) = \\frac{1}{G}\\sum_{j=1}^G R_j$ 是组内均值\n- $\\text{std}(\\{R_j\\}_{j=1}^G)$ 是组内标准差\n\n**2. RLOO (REINFORCE Leave-One-Out)**\n\nRLOO 对每个 prompt 生成 $K$ 个响应样本，使用 **留一法（Leave-One-Out）** 构造基线，即第 $i$ 个样本的基线为除自己外的其他 $K-1$ 个样本的均值：\n\n$$\n\\hat{A}_{i} = R_i - \\frac{1}{K-1}\\sum_{j \\neq i} R_j\n$$\n\n这个公式可以等价地改写为：\n\n$$\n\\hat{A}_{i} = \\frac{K}{K-1} \\left(R_i - \\bar{R}\\right)\n$$\n\n其中 $\\bar{R} = \\frac{1}{K}\\sum_{j=1}^K R_j$ 是组内所有样本的均值。\n\n> **说明**：这里使用 $K$ 对齐论文符号，与 GRPO 中的 $G$ 含义一致，均对应配置参数 `num_generations`\n\n**为什么使用留一法？**\n\n留一法的关键优势在于**无偏性**。对于第 $i$ 个样本，其奖励 $R_i$ 和基线 $\\frac{1}{K-1}\\sum_{j \\neq i} R_j$ 是独立的，因此优势估计是无偏的。相比之下，如果使用包含自身的均值作为基线，会引入偏差。\n\n### 区别2：KL 散度正则化项的处理方式\n\n为防止策略偏离参考策略过远，两种算法都引入了 KL 散度正则化，但处理方式不同：\n\n**GRPO**：将 KL 散度作为独立的正则化项添加到[损失函数](../GetStarted/GRPO.md#算法原理)中：\n\n$$\n\\mathcal{L}(\\theta) = -\\mathbb{E}\\left[\\hat{A}_i \\log \\pi_\\theta(a_i|s_i)\\right] + \\beta \\cdot \\text{KL}(\\pi_\\theta || \\pi_{\\text{ref}})\n$$\n\n**RLOO**：将 KL 散度直接整合到奖励项中，构造修正后的奖励：\n\n$$\nR'_i = R_i - \\beta \\cdot \\text{KL}(\\pi_\\theta || \\pi_{\\text{ref}})\n$$\n\n其中 $\\beta$ 是 KL 散度的权重系数（对应参数 `beta`），$\\pi_{\\text{ref}}$ 是参考策略（通常是 SFT 模型或初始策略）。\n\n## 参数设置\n\n我们可以基于 `GRPOTrainer`，通过设置以下参数实现 RLOO 训练：\n```bash\n# 基本 RLOO 配置\n--advantage_estimator rloo # 使用 RLOO 的留一法优势函数计算\n--kl_in_reward true # 将 KL 散度项整合到奖励中（RLOO 默认方式）\n```\n\n训练可以参考该[脚本](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/internal/rloo.sh)\n\n### 重要参数说明\n\n- **`--advantage_estimator`**：选择优势函数估计方法\n - `grpo`（默认）：使用组内均值和标准差进行标准化\n - `rloo`：使用留一法（Leave-One-Out）构造基线\n\n- **`--kl_in_reward`**：控制 KL 散度正则化项的处理位置\n - `false`：KL 散度作为损失函数的独立正则化项（GRPO 方式）\n - `true`：KL 散度直接从奖励中扣除，构造修正后的奖励（RLOO 方式）\n\n- **`--num_generations`**：每个 prompt 生成的样本数量 $K$\n\n- **`--beta`**：KL 散度正则化系数 $\\beta$\n - 控制策略更新的保守程度\n\n其他参数与 [GRPO参数](../../Command-line-parameters.md#grpo参数)一致\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/SAPO.md",
"content": "# Soft Adaptive Policy Optimization (SAPO)\n\n**版本依赖**：ms-swift>=3.11\n\n[Soft Adaptive Policy Optimization (SAPO)](https://arxiv.org/abs/2511.20347) 针对 GRPO 中硬裁剪（hard clipping）带来的问题，提出了一种基于温度控制的软门控（soft gate）机制，用于平滑地衰减离策略更新，同时保留有用的学习信号。\n\n## 背景与动机\n\n在强化学习训练 LLM 时，GRPO 通过计算 token 级别的重要性采样比（Importance Sampling Ratio）来处理 off-policy 训练：\n\n$$\nr_t = \\frac{\\pi_\\theta(y_t|x, y_{ 0$），使用正向门控：\n\n$$\ng^{+}_t = \\sigma\\left( \\tau_{\\mathrm{pos}} \\cdot (r_t - 1) \\right) \\cdot \\frac{4}{\\tau_{\\mathrm{pos}}}\n$$\n\n对于负向优势（$A < 0$），使用负向门控：\n\n$$\ng^{-}_t = \\sigma\\left( \\tau_{\\mathrm{neg}} \\cdot (r_t - 1) \\right) \\cdot \\frac{4}{\\tau_{\\mathrm{neg}}}\n$$\n\n其中：\n- $\\sigma(\\cdot)$ 是 sigmoid 函数\n- $\\tau_{\\mathrm{pos}}$ 和 $\\tau_{\\mathrm{neg}}$ 是温度参数，控制门控函数的斜率\n- $r_t$ 是重要性采样比\n\n### SAPO 损失函数\n\n$$\nL^{\\mathrm{SAPO}} = -g_t \\cdot A\n$$\n\n其中 $g_t = g^{+}_t$ 当 $A > 0$，$g_t = g^{-}_t$ 当 $A < 0$。\n\n### 温度参数\n\n温度参数 $\\tau$ 控制软门控函数的衰减速率，数值越大，衰减越快。\n\n\n\n论文指出正向优势会提升采样token的logit，并降低所有未采样token的logit；负向优势相反，提高许多未采样token的logit，可能会扩散到大量无关token上，带来一定的不稳定性。所以论文推荐设置温度 $\\tau_\\text{neg} > \\tau_\\text{pos}$，来使负向奖励的token梯度衰减更快，提升训练的稳定性和性能。\n\n论文默认推荐 $\\tau_{\\mathrm{pos}} = 1.0$，$\\tau_{\\mathrm{neg}} = 1.05$。\n\n## 参数设置\n\n| 参数 | 类型 | 默认值 | 说明 |\n|------|------|--------|------|\n| `--loss_type` | `str` | - | 设置为 `sapo` |\n| `--tau_pos` | `float` | `1.0` | 正向优势的温度参数，控制门控斜率 |\n| `--tau_neg` | `float` | `1.05` | 负向优势的温度参数，控制门控斜率 |\n\n```bash\nswift rlhf \\\n --rlhf_type grpo \\\n --loss_type sapo \\\n --tau_pos 1.0 \\\n --tau_neg 1.05 \\\n # ... 其他参数\n```\n\n训练脚本参考\n\n- [swift](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo/internal/sapo.sh)\n- [megatron swift](https://github.com/modelscope/ms-swift/blob/main/examples/megatron/grpo/sapo.sh)\n\n> SAPO 的软门控机制仅在 off-policy 训练下生效。\n> SAPO 中的重要性采样粒度为 token 级别（即 importance_sampling_level 默认设置为 token），与 GSPO 冲突。\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/deepeyes.md",
"content": "# DeepEyes: Incentivizing \"Thinking with Images\" via Reinforcement Learning\n\n**版本依赖**：ms-swift>=3.7\n\n## 原理介绍\n\n[DeepEyes论文](https://arxiv.org/abs/2505.14362) 提出了一种利用强化学习使模型具备“think with images”（以图辅助思考）能力的方法。该方法通过端到端的强化学习，模型能力自发涌现，无需额外的 SFT（监督微调）过程。模型内置图像定位能力，能够主动调用“图像放大工具”：在推理过程中，模型会自动选取图片中的具体区域进行放大和裁剪，将处理后的区域信息进行进一步推理，实现视觉与文本的链式推理。\n\n\n\n## 最佳实践\n\n**数据集下载与注册**\n\n下载 DeepEyes 官方训练数据集到本地\n```bash\n# modelscope\nmodelscope download --dataset Lixiang/ChenShawn-DeepEyes-Datasets-47k\n\n# huggingface\nhuggingface-cli download ChenShawn/DeepEyes-Datasets-47k --repo-type=dataset\n```\n\n数据集内有三个parquet文件，`swift/dataset/data/dataset_info.json` 文件中分别进行注册，将数据集中的 `prompt` 列重命名为 `messages`\n\n```json\n {\n \"ms_dataset_id\": \"path/to/data_0.1.2_visual_toolbox_v2.parquet\",\n \"columns\": {\n \"prompt\": \"messages\"\n }\n },\n {\n \"ms_dataset_id\": \"path/to/data/data_thinklite_reasoning_acc.parquet\",\n \"columns\": {\n \"prompt\": \"messages\"\n }\n },\n {\n \"ms_dataset_id\": \"path/to/data/data_v0.8_visual_toolbox_v2.parquet\",\n \"columns\": {\n \"prompt\": \"messages\"\n }\n }\n```\n\n在本地注册论文中所用到的奖励函数和工具调用逻辑，实现可以参考[DeepEyes实现示例](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/plugin/deepeyes/deepeyes_plugin.py)\n\n**部署验证模型**\n\nDeepeyes 的奖励函数依赖生成式奖励模型对模型生成结果与标准答案进行对比评估，为了加速这一环节，推荐对模型进行部署。\n\n假设使用 Qwen2.5-VL-72B-Instruct 模型进行评估，参考以下部署命令\n\n```bash\n# 4*80G\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nswift deploy \\\n --model Qwen/Qwen2.5-VL-72B-Instruct \\\n --infer_backend vllm \\\n --vllm_tensor_parallel_size 4 \\\n```\n\n在 plugin 文件中，使用OpenAI接口进行调用，参考[奖励模型文档](../DeveloperGuide/reward_model.md#外部部署)\n\n\n训练参考该[脚本](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/plugin/deepeyes/deepeyes.sh)\n\n## 实现细节\n\n[DeepEyes实现示例](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/plugin/deepeyes/deepeyes_plugin.py)参考[官方实现](https://github.com/Visual-Agent/DeepEyes/blob/main/verl/utils/reward_score/vl_agent.py) 给出了 DeepEyes 训练插件的样例代码，涵盖了奖励函数与多轮交互调用的相关逻辑。\n\n**数据集数据**如下\n\n| 数据集文件名 | data_source | 对应评分函数 | 工具调用 |\n|------------------------------------------|-----------------------|----------------------------------|------------------|\n| data_v0.8_visual_toolbox_v2.parquet | chart | vl_agent.compute_score | True (image_zoom_in_tool) |\n| data_0.1.2_visual_toolbox_v2.parquet | vstar | vl_agent.compute_score | True (image_zoom_in_tool) |\n| data_thinklite_reasoning_acc.parquet | thinklite_eureka | vl_agent.compute_score_math | False |\n\n\n**注意**：多模态大模型在处理图像输入时，可能会对图像进行预处理（例如受 max_pixels 参数限制的裁剪或缩放等操作）。当调用图像放大工具 image_zoom_in_tool 时，模型会根据输入图像输出裁剪后的 bbox。因此，在调用图像放大工具时，需要确保输入的是经过预处理后的图像。示例代码展示了 Qwen2.5-VL 系列模型的实现方式：\n\n```python\nfrom qwen_vl_utils import fetch_image\n# 这里的images尚未经过图像处理\ninfer_request.images\n# 通过加载为PIL.Image格式，进行裁剪（使用环境变量MAX_PIXELS时的处理）\nimg = fetch_image({'image': load_pil_image(infer_request.images[0])})\n```\n\n**工具奖励**\n\n论文中指出当最终答案正确，且轨迹至少使用一个工具时给予工具奖励。为了避免模型生成的工具调用是无效的，我们通过图像数量而不是`` 等token进行判断。\n```python\ntool_reward = 1.0 if num_image > 1 and acc_reward > 0.5 else 0.0\n```\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/entropy_mask.md",
"content": "# Beyond the 80/20 Rule: High-Entropy Minority Tokens Drive Effective Reinforcement Learning for LLM Reasoning\n\n**版本依赖**：ms-swift>=3.7\n\n[论文](https://arxiv.org/abs/2506.01939)发现在以 RLVR等方法训练大型语言模型推理能力时，驱动学习进步的关键在于一小部分高熵“少数 token”，而并非大多数信息熵低的 token。\n\n论文指出，在模型推理的 token 分布中，只有极少数信息熵较高的 token 起到了主导作用。这些 token 往往出现在推理和决策路径分歧最大的关键节点（如 \"wait\"、\"since\" 等），决定了模型能否习得复杂推理任务。而大多数熵低的 token 对模型推理能力的提升作用有限。论文提出只对高熵 token 计算策略梯度、舍弃低熵 token 的梯度。\n\n\ntoken 熵公式如下\n\n$\nH_t := -∑_{j=1}^{V} p_{t,j} \\log p_{t,j}, \\qquad where (p_{t,1}, ···, p_{t,V}) = \\mathbf{p}_t = π_θ(\\cdot | \\mathbf{q}, \\mathbf{o}_{ \"Token entropy\" $H_t$ 始终指向位置 $t$ 的生成分布 $\\mathbf{p}_t$ 的不确定性，而非 token $o_t$ 本身的属性。即$H_t$ 是位置 $t$ 对应分布 $\\mathbf{p}_t$ 的熵，与采样得到的 token $o_t$ 无关。\n\n\n在实践中，我们可以在 GRPO 训练中通过参数 `top_entropy_quantile` 控制训练范围。论文实验设置该参数为 0.2，即每次仅对处于熵分布前 20% 的 token 进行训练优化。\n\n同时使用参数`log_entropy`，可以记录训练过程中的熵值变化，参考[文档](../GetStarted/GRPO.md#logged-metrics)\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/index.rst",
"content": "Advanced Research\n===============\n.. toctree::\n :maxdepth: 1\n\n entropy_mask.md\n CISPO.md\n DAPO.md\n deepeyes.md\n GSPO.md\n CHORD.md\n RLOO.md\n REINFORCEPP.md\n SAPO.md\n training_inference_mismatch.md\n treepo.md\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/training_inference_mismatch.md",
"content": "# Training-Inference-Mismatch\n\n**版本依赖**：ms-swift>=3.11\n\n**TL;DR**: GRPO 引入 vLLM 加速采样过程的同时，也引入了训练-推理不一致（Training-Inference Mismatch）的问题，从而可能影响训练稳定性。本文将解释这个问题的背景、原因以及相应的解决方案。\n\n## Background\n\n### GRPO 的基本假设\n\nGRPO (Group Relative Policy Optimization) 的训练目标可以表示为：\n\n$$\n\\mathcal{L}_{\\text{GRPO}} = - \\mathbb{E}_{y \\sim \\pi_\\theta} \\left[ \\min \\left( r_t(\\theta) \\hat{A}_t, \\text{clip}(r_t(\\theta), 1-\\epsilon, 1+\\epsilon) \\hat{A}_t \\right) \\right]\n$$\n\n其中：\n- $r_t(\\theta) = \\frac{\\pi_\\theta(y_t|x, y_{ \\tau$\n2. **且** $\\hat{A}_i < 0$\n\n其中：\n- $\\pi_{\\text{old}}$ 优先使用 `rollout_per_token_logps`（rollout/行为策略的 logprobs），若不存在则使用 `old_per_token_logps`\n- $\\tau$ 是用户设置的阈值（`--off_policy_sequence_mask_delta`，默认 None 表示关闭）\n\n\n参考资料\n\n1. https://yingru.notion.site/When-Speed-Kills-Stability-Demystifying-RL-Collapse-from-the-Training-Inference-Mismatch-271211a558b7808d8b12d403fd15edda\n2. https://fengyao.notion.site/off-policy-rl\n3. https://github.com/volcengine/verl/blob/main/verl/trainer/ppo/rollout_corr_helper.py\n4. [DeepSeek-V3.2: Pushing the Frontier of Open Large Language Models](https://arxiv.org/abs/2512.02556)\n"
},
{
"path": "docs/source/Instruction/GRPO/AdvancedResearch/treepo.md",
"content": "# TreePO: Bridging the Gap of Policy Optimization and Efficacy and Inference Efficiency with Heuristic Tree-based Modeling\n\n**版本依赖**：ms-swift>=3.11\n\n作者： [li2zhi](https://github.com/li2zhi)\n\n## 原理介绍\n[TreePO论文](https://arxiv.org/abs/2508.17445) 提出了一种树状结构建模方法。该方法将序列生成组织为分段式的树结构搜索，通过动态分支、回退与提前终止机制，显著提高KV缓存复用率，从而降低计算开销，同时保持甚至增强了探索的多样性。\n\n\n\n## 实现细节\n[TreePO实现示例](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/plugin/treepo/tree_rollout_plugin.py)参考[官方实现](https://github.com/multimodal-art-projection/TreePO/blob/main/recipe/treepo/vllm_rollout_tree.py) 给出了 TreePO 训练插件的样例代码，涵盖了多轮交互、终止判断，与分支回退等相关逻辑。\n\n**注意**：在实际使用中，你需要根据自己的场景需求，重写step、check_finished等方法的逻辑，以确保其能够在自定义场景下按照预期执行。而关于自定义奖励的设计与使用，你可以参考[DeepEyes](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/plugin/deepeyes/deepeyes_plugin.py)的实现。\n\n训练参考该[脚本](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo/plugin/treepo/tree_rollout.sh)\n\n## 测试数据\n> model: Qwen/Qwen2.5-0.5B\n> dataset: AI-MO/NuminaMath-TIR\n> subset size: 1,000 samples\n> 1 GPU for training, 1 GPU for inference\n\n| \\ | batch_size | num_generation | max_tree_depth | global_step | total inference calls | saving ratio | train_speed(iter/s) | improvement rate |\n| ----------------------- | ---------- | -------------- | -------------- | ----------- | --------------------- | ------------ | ------------------- | ---------------- |\n| original implementation | 8 | 8 | 4 | 200 | 5965 | 0.00% | 0.292436 | 0.00% |\n| tree(max_divergence=3) | 8 | 8 | 4 | 200 | 3678 | 38.34% | 0.31819 | 8.81% |\n| | | | | | | | | |\n| original implementation | 8 | 8 | 5 | 105 | 4312 | 0.00% | 0.261324 | 0.00% |\n| tree(max_divergence=2) | 8 | 8 | 5 | 105 | 2513 | 52.69% | 0.336639 | 28.82% |\n| tree(max_divergence=3) | 8 | 8 | 5 | 105 | 2990 | 30.66% | 0.308791 | 18.16% |\n| | | | | | | | | |\n| original implementation | 8 | 8 | 6 | 105 | 5202 | 0.00% | 0.24832 | 0.00% |\n| tree(max_divergence=2) | 8 | 8 | 6 | 105 | 3348 | 35.64% | 0.27755 | 11.77% |\n| tree(max_divergence=3) | 8 | 8 | 6 | 105 | 3888 | 25.26% | 0.272339 | 9.67% |\n"
},
{
"path": "docs/source/Instruction/GRPO/DeveloperGuide/gym_env.md",
"content": "# GYM环境训练\n\n**注意** GYM环境训练逻辑已在 ms-swift 3.8 中进行重构，如果您的 ms-swift 版本低于该版本，请参考对应版本的文档。\n\n## Gym接口\n\nGYM源自于[OpenAI Gym](https://github.com/openai/gym)，是一个抽象的强化学习环境接口，基于现在Model as Agent的趋势，我们在swift中定义了类似的一个接口,为Agent提供端到端的强化学习训练。\n```python\nclass Env(ABC):\n\n def __init__(self, env_config):\n \"\"\"\n\n Args:\n env_config: 环境配置，比如可用工具等\n \"\"\"\n self.env_config = env_config\n\n @abstractmethod\n async def reset(self, config: RolloutInferRequest) -> Tuple[str, Dict[str, Any], str]:\n \"\"\"\n\n Args:\n config: 环境初始化信息，应该放在\n\n Returns:\n - observation: 第一个user消息作为初始观察或者环境信息，会作为user message\n - info: 用于DEBUG和日志的额外信息，会在completions.jsonl中记录\n - system_message: 用户当前环境采样的系统提示词\n \"\"\"\n pass\n\n @abstractmethod\n async def step(self, action: Messages) -> Tuple[str, float, bool, Dict[str, Any]]:\n \"\"\"\n\n Args:\n action: 所有对话消息，最后一个消息为当前采样回复\n\n Returns:\n - next_observation: 环境响应，将作为user message返回\n - reward: 奖励\n - done: 是否结束\n - info: 用于DEBUG和日志的额外信息，会在completions.jsonl中记录\n \"\"\"\n pass\n @abstractmethod\n async def close(self):\n \"\"\"Clean up environment resources.\"\"\"\n pass\n```\n除此之外，根据[Kimi-Reseacher的实践](https://moonshotai.github.io/Kimi-Researcher/)，我们还额外提供了一个`ContextMangaer`接口,方便你动态的管理当前的Agent上下文。\n\n**ContextManager指定（非必需）**\n1. 在数据集中提供 [ctx_config](#注意事项) 列中的 name 键指定， 初始化相关的参数放在其他键中\n2. 使用参数 `--context_manager ctx_name` 指定\n\n\n```python\nclass ContextManager(ABC):\n def __init__(self,ctx_config):\n self.ctx_config = ctx_config\n\n @abstractmethod\n def manage_context(self, history: Messages,trajectory_id:str) -> Messages:\n \"\"\"动态调整当前agent的上下文\n\n Args:\n history: 当前的消息历史\n\n Returns:\n 调整后的消息历史\n \"\"\"\n pass\n```\n入参示例\n```python\ninfer_request\n\"\"\"\nRolloutInferRequest(\n messages=[\n {'role': 'system', 'content': 'A conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. The reasoning process and answer are enclosed within and tags, respectively, i.e., reasoning process here answer here \\n'}, {'role': 'user', 'content': 'What is the value of $\\\\sqrt{36 \\\\times \\\\sqrt{16}}$?'},\n {'role': 'assistant', 'content': 'To find the value of \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\), we will break down the problem step-by-step.\\n\\nFirst, we need to evaluate the inner square root:\\n\\\\[\\n\\\\sqrt{16}\\n\\\\]\\nWe know that:\\n\\\\[\\n4^2 = 16 \\\\implies \\\\sqrt{16} = 4\\n\\\\]\\n\\nNext, we substitute this result back into the original expression:\\n\\\\[\\n\\\\sqrt{36 \\\\times \\\\sqrt{16}} = \\\\sqrt{36 \\\\times 4}\\n\\\\]\\n\\nNow, we need to evaluate the product inside the square root:\\n\\\\[\\n36 \\\\times 4 = 144\\n\\\\]\\n\\nSo, the expression simplifies to:\\n\\\\[\\n\\\\sqrt{144}\\n\\\\]\\n\\nFinally, we determine the square root of 144:\\n\\\\[\\n\\\\sqrt{144} = 12\\n\\\\]\\n\\nThus, the value of \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\) is:\\n\\\\[\\n\\\\boxed{12}\\n\\\\]'}\n ],\n images=[],\n audios=[],\n videos=[],\n tools=None,\n objects={},\n data_dict={\n 'problem': 'What is the value of $\\\\sqrt{36 \\\\times \\\\sqrt{16}}$?',\n 'solution': \"To solve the problem, we need to evaluate the expression \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\).\\n\\nWe can break down the steps as follows:\\n\\n1. Evaluate the inner square root: \\\\(\\\\sqrt{16}\\\\).\\n2. Multiply the result by 36.\\n3. Take the square root of the product obtained in step 2.\\n\\nLet's compute this step by step using Python code for accuracy.\\n```python\\nimport math\\n\\n# Step 1: Evaluate the inner square root\\ninner_sqrt = math.sqrt(16)\\n\\n# Step 2: Multiply the result by 36\\nproduct = 36 * inner_sqrt\\n\\n# Step 3: Take the square root of the product\\nfinal_result = math.sqrt(product)\\nprint(final_result)\\n```\\n```output\\n12.0\\n```\\nThe value of \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\) is /\\\\(\\\\boxed{12}\\\\).\"\n }\n )\n\"\"\"\nresult\n\"\"\"\nRolloutResponseChoice(\n index=0,\n message=ChatMessage(\n role='assistant',\n content='To find the value of \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\), we will break down the problem step-by-step.\\n\\nFirst, we need to evaluate the inner square root:\\n\\\\[\\n\\\\sqrt{16}\\n\\\\]\\nWe know that:\\n\\\\[\\n4^2 = 16 \\\\implies \\\\sqrt{16} = 4\\n\\\\]\\n\\nNext, we substitute this result back into the original expression:\\n\\\\[\\n\\\\sqrt{36 \\\\times \\\\sqrt{16}} = \\\\sqrt{36 \\\\times 4}\\n\\\\]\\n\\nNow, we need to evaluate the product inside the square root:\\n\\\\[\\n36 \\\\times 4 = 144\\n\\\\]\\n\\nSo, the expression simplifies to:\\n\\\\[\\n\\\\sqrt{144}\\n\\\\]\\n\\nFinally, we determine the square root of 144:\\n\\\\[\\n\\\\sqrt{144} = 12\\n\\\\]\\n\\nThus, the value of \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\) is:\\n\\\\[\\n\\\\boxed{12}\\n\\\\]', tool_calls=None),\n finish_reason='stop',\n logprobs=None,\n messages=None)\n\"\"\"\n```\nGYM环境训练可以视作一种特殊的多轮训练，区别在于使用GYM环境训练，奖励信息通过环境直接获取。\n\n在 `rollout` 命令中使用参数 `use_gym_env` 来指定使用gym作为训练的环境接口。我们提供了兼容GYM环境的多轮规划器参考实现，见[内置多轮调度器实现](https://github.com/modelscope/ms-swift/blob/main/swift/rollout/multi_turn.py)中的 GymScheduler 类\n\n\n```bash\nCUDA_VISIBLE_DEVICES=0 \\\nswift rollout \\\n --model xxx \\\n --use_gym_env true \\\n --multi_turn_scheduler gym_scheduler \\\n --max_turns xxx\n```\n\n> 注意\n\n**环境选择**\n1. 在数据集中需要提供 [env_config](#注意事项) 列中的name键指定, 初始化相关的参数放在其他键中\n2. 使用参数 `--gym_env env_name` 指定\n\n\n## 最佳实践\n\n- [训练脚本](../../../../../examples/train/grpo/external/vllm_gym.sh)\n\n通过参数`external_plugins`, 我们可以将本地的`Env`和`ContextManager`注册进 ms-swift 中，具体实现参考[代码](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo/plugin/plugin.py)\n\n## 注意事项\n\n1. 参考训练数据格式\n```json\n{\"messages\": [{\"role\": \"system\", \"content\": \"你是个有用无害的助手\"}, {\"role\": \"user\", \"content\": \"告诉我明天的天气\"}],\"env_config\":{\"name\":\"custom_env\",\"other_config\":\"xxxx\"},\"ctx_config\":{\"name\":\"custom_ctx\",\"other_config\":\"xxxx\"}}\n```\n2. 默认仅对最后一轮response进行训练，如果gym涉及到多轮response生成，使用参数`--loss_scale default`对所有轮次的response进行训练，具体参考[文档](./multi_turn.md#损失掩码)\n\n3. 数据流程\n整个gym数据流程如下:\n入参示例（点击展开） \n\n```python\ninfer_request\n\"\"\"\nRolloutInferRequest(\n messages=[\n {'role': 'system', 'content': 'A conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. The reasoning process and answer are enclosed within and tags, respectively, i.e., reasoning process here answer here \\n'}, {'role': 'user', 'content': 'What is the value of $\\\\sqrt{36 \\\\times \\\\sqrt{16}}$?'},\n {'role': 'assistant', 'content': 'To find the value of \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\), we will break down the problem step-by-step.\\n\\nFirst, we need to evaluate the inner square root:\\n\\\\[\\n\\\\sqrt{16}\\n\\\\]\\nWe know that:\\n\\\\[\\n4^2 = 16 \\\\implies \\\\sqrt{16} = 4\\n\\\\]\\n\\nNext, we substitute this result back into the original expression:\\n\\\\[\\n\\\\sqrt{36 \\\\times \\\\sqrt{16}} = \\\\sqrt{36 \\\\times 4}\\n\\\\]\\n\\nNow, we need to evaluate the product inside the square root:\\n\\\\[\\n36 \\\\times 4 = 144\\n\\\\]\\n\\nSo, the expression simplifies to:\\n\\\\[\\n\\\\sqrt{144}\\n\\\\]\\n\\nFinally, we determine the square root of 144:\\n\\\\[\\n\\\\sqrt{144} = 12\\n\\\\]\\n\\nThus, the value of \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\) is:\\n\\\\[\\n\\\\boxed{12}\\n\\\\]'}\n ],\n images=[],\n audios=[],\n videos=[],\n tools=None,\n objects={},\n data_dict={\n 'problem': 'What is the value of $\\\\sqrt{36 \\\\times \\\\sqrt{16}}$?',\n 'solution': \"To solve the problem, we need to evaluate the expression \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\).\\n\\nWe can break down the steps as follows:\\n\\n1. Evaluate the inner square root: \\\\(\\\\sqrt{16}\\\\).\\n2. Multiply the result by 36.\\n3. Take the square root of the product obtained in step 2.\\n\\nLet's compute this step by step using Python code for accuracy.\\n```python\\nimport math\\n\\n# Step 1: Evaluate the inner square root\\ninner_sqrt = math.sqrt(16)\\n\\n# Step 2: Multiply the result by 36\\nproduct = 36 * inner_sqrt\\n\\n# Step 3: Take the square root of the product\\nfinal_result = math.sqrt(product)\\nprint(final_result)\\n```\\n```output\\n12.0\\n```\\nThe value of \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\) is /\\\\(\\\\boxed{12}\\\\).\"\n }\n )\n\"\"\"\nresponse_choice\n\"\"\"\nChatCompletionResponseChoice(\n index=0,\n message=ChatMessage(\n role='assistant',\n content='To find the value of \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\), we will break down the problem step-by-step.\\n\\nFirst, we need to evaluate the inner square root:\\n\\\\[\\n\\\\sqrt{16}\\n\\\\]\\nWe know that:\\n\\\\[\\n4^2 = 16 \\\\implies \\\\sqrt{16} = 4\\n\\\\]\\n\\nNext, we substitute this result back into the original expression:\\n\\\\[\\n\\\\sqrt{36 \\\\times \\\\sqrt{16}} = \\\\sqrt{36 \\\\times 4}\\n\\\\]\\n\\nNow, we need to evaluate the product inside the square root:\\n\\\\[\\n36 \\\\times 4 = 144\\n\\\\]\\n\\nSo, the expression simplifies to:\\n\\\\[\\n\\\\sqrt{144}\\n\\\\]\\n\\nFinally, we determine the square root of 144:\\n\\\\[\\n\\\\sqrt{144} = 12\\n\\\\]\\n\\nThus, the value of \\\\(\\\\sqrt{36 \\\\times \\\\sqrt{16}}\\\\) is:\\n\\\\[\\n\\\\boxed{12}\\n\\\\]', tool_calls=None),\n finish_reason='stop',\n logprobs=None,\n messages=None)\n\"\"\"\n# response_choice.messages will be copied at the end of multi-turn inference.\n```\n and tags, respectively, i.e., reasoning process here answer here \n```\n\n该函数检查模型是否按照 `think content answer content ` 的格式进行生成。如果生成文本符合格式要求，则得分为 1.0；否则为 0.0。\n\n### 3. **cosine**\n\n论文发现，仅使用 accuracy 奖励函数进行训练会导致模型的生成长度趋于超长，从而影响训练效果。cosine 奖励函数通过控制模型的生成长度来优化训练过程：\n\n- 对于生成正确答案的文本，奖励值随长度增加而递减，鼓励模型生成简洁的回答。\n- 对于生成错误答案的文本，奖励值随长度增加而递增，鼓励模型进行更深入的思考。\n\n使用余弦函数平滑地调整奖励值，确保奖励变化在合理范围内。余弦函数的参数包括生成文本的长度、最大长度限制以及奖励的最小值和最大值。\n\n参数\n- cosine_min_len_value_wrong（默认值：-0.5）：生成错误答案时，最小长度对应的奖励值。\n- cosine_max_len_value_wrong（默认值：0.0）：生成错误答案时，最大长度对应的奖励值。\n- cosine_min_len_value_correct（默认值：1.0）：生成正确答案时，最小长度对应的奖励值。\n- cosine_max_len_value_correct（默认值：0.5）：生成正确答案时，最大长度对应的奖励值。\n- cosine_max_len（默认值等于模型生成的最大程度）：生成文本的最大长度限制。\n\n\n### 4. **repetition**\n\n惩罚模型生成文本中的重复内容，通过检测生成文本中的重复 n-gram 模式来评估重复程度，并给予相应的惩罚。\n\n函数将生成文本分割为单词，并提取指定大小的 n-gram（默认为 3-gram）。通过统计不同 n-gram 的数量与总 n-gram 数量的比例，计算重复比例。如果生成文本中重复的 n-gram 比例较高，则给予较大的负奖励（惩罚）。惩罚值通过重复比例和最大惩罚值（默认为 -1.0）计算得出。\n\n参数\n- repetition_n_grams（默认值：3）：用于检测重复的 n-gram 大小。\n- repetition_max_penalty（默认值：-1.0）：最大惩罚值，用于控制惩罚的强度。\n\n### 5. **soft overlong punishment**\n定义长度惩罚区间。在这个区间内，给予[-1,0]的线性惩罚。\n\n参数\n- soft_max_length: 论文中的L_max，模型的最大生成长度，默认等于max_completion_length\n- soft_cache_length: 论文中的L_cache，控制长度惩罚区间，区间为[soft_max_length-soft_cache_length, soft_max_length]\n\n## 注意事项\n\n如果需要在奖励函数中加载模型，默认会使用训练的 deepspeed 插件（transformers逻辑），zero3下可能会导致模型无法正常推理，参考该[issue](https://github.com/modelscope/ms-swift/issues/4580)来跳过 deepspeed 初始化环境\n"
},
{
"path": "docs/source/Instruction/GRPO/DeveloperGuide/reward_model.md",
"content": "# 奖励模型\n\n默认情况下，奖励模型是指具有分类头数值输出的模型，通常称为输出奖励模型（ORM）。这些模型会对其他模型的输出进行评分，从而生成一个标量值，表示模型响应的质量。\n\n我们可以通过使用参数 `reward_models` 来加载具有分类头的奖励模型，或者加载经过[奖励建模](../../RLHF.md#rm)训练的奖励模型，进而使用模型的logits作为奖励。\n\n## 自定义奖励模型\n对于生成式奖励模型，有两种常见的调用方式：一种是在 Trainer 内部直接使用 reward_model_plugin 定义奖励模型的逻辑，可以使用TransformersEngine对奖励模型进行推理，另一种是通过外部部署的模型服务进行调用。\n\n- 使用 reward_model_plugin 调用奖励模型时，模型会被内嵌在 Trainer 内部，无需额外占用计算资源。该方式优点是方便集成，但生成速度相对较慢，更适合参数量较小的奖励模型场景。\n\n- 外部部署奖励模型时，可以通过诸如 swift deploy 或 vllm serve 等命令将模型服务部署于独立设备，大幅提升推理速度，适合参数量较大的模型。但这样需要预留额外的硬件资源。\n\n### 内部插件\n\n我们可以在 reward_model_plugin 中灵活地自定义奖励模型的处理逻辑。这使得实现诸如生成式奖励模型等技术成为可能，包括：\n\n- 自定义模型的系统提示：定义特定的指令和上下文以指导评估过程。\n- 处理模型交互历史：管理对话上下文，以提供有意义且具有上下文感知的评估。\n- 定义自定义评估标准：设置独特的标准和度量，用于评估模型的响应，超越默认的准确性和相关性衡量标准。\n\n通过reward_model_plugin，开发者可以针对其应用的特定需求定制奖励评估过程。这种灵活性允许更细致和有效的基于奖励的训练策略。\n\n奖励模型通过plugin的`__call__`方法进行调用，该方法接受 `inputs` 作为参数，包含了模型输入输出的 messages 和数据集中的其他列\n\n```python\n def __call__(self, inputs):\n print(inputs)\n \"\"\"\n[\n {\n 'messages': [\n {'role': 'system', 'content': 'system prompt'},\n {'role': 'query', 'content': 'query'},\n {'role': 'user', 'content': 'completions1'},\n ],\n 'solution': \"abc\",\n },\n {\n 'messages': [\n {'role': 'system', 'content': 'system prompt'},\n {'role': 'query', 'content': 'query'},\n {'role': 'user', 'content': 'completions2'},\n ],\n 'solution': \"abc\",\n }\n]\n\n```\n\n在插件中使用 TransformersEngine 进行奖励模型的推理， 我们只需构造 messages ，并通过 infer 接口调用：\n```python\nclass RMPlugin(DefaultRMPlugin):\n\n def __init__(self, model, template):\n\n super().__init__(model, template)\n # initilize TransformersEngine to infer\n self.engine = TransformersEngine(self.model, template=self.template, max_batch_size=0)\n\n def __call__(self, inputs):\n system_prompt = ...\n query = ...\n messages = [{'role': 'system', 'content': system_prompt}, {'role': 'query', 'content': query}]\n result = self.engine.infer([messages], self.request_config, use_tqdm=False)\n rewards = ...\n return rewards\n```\n\n我们在 [rm_plugin.py](https://github.com/modelscope/ms-swift/blob/main/swift/rewards/rm_plugin.py) 中提供了一个简单的生成式奖励模型示例（GenRMPlugin）。\n\n在 [plugin.py](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo/plugin/plugin.py) 中自定义奖励模型插件，并使用 `external_plugins` 参数进行注册。\n\n\n注意：\n1. 在 GRPOTrainer 中，reward_model 会依次append到 reward_funcs 中。因此，reward_weights 的顺序对应 [reward_funcs, reward_model]。\n2. reward_model_plugin 默认为 default，即使用 ORM 处理逻辑。\n3. 对于参数量较大的模型，TransformersEngine 生成速度较慢，请使用[外部部署](#外部部署)方法\n\n对于 BERT 这类无法通过 reward_model 加载的模型，我们可以内置在 reward_function 中进行加载，参考[issue](https://github.com/modelscope/ms-swift/issues/4580)\n\n### 外部部署\n\n**示例 2：使用 swift deploy 部署奖励模型并进行远程调用**\n\n这类方法则不需要使用 reward_model_plugin , 而是直接在奖励函数中进行调用即可\n\n首先用如下命令启动模型服务：\n\n```bash\n# 注意部署的设备不要与训练设备重叠\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nswift deploy \\\n --model Qwen/Qwen2.5-72B-Instruct \\\n --vllm_tensor_parallel_size 4\n\n# [INFO:swift] model_list: ['Qwen2.5-72B-Instruct']\n# INFO: Started server process [xxxxxx]\n# INFO: Waiting for application startup.\n# INFO: Application startup complete.\n# INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)\n```\n在奖励函数中通过 OpenAI 库初始化客户端，指定模型服务的地址和端口，示例代码如下：\n\n```python\nfrom openai import OpenAI\n\nclass RMReward(ORM):\n\n def __init__(self):\n super().__init__()\n try:\n self.client = OpenAI(\n api_key='EMPTY',\n base_url='http://127.0.0.1:8000/v1', # 若在本地部署则为 127.0.0.1\n )\n self.verify_model_name = self.client.models.list().data[0].id\n except Exception as e:\n raise RuntimeError('Failed to connect to the model service. Please deploy the model '\n \"using 'swift deploy' or 'vllm serve'.\") from e\n\n\n def __call__(self, completions, messages, **kwargs) -> List[float]:\n rewards = []\n for completion, message in zip(completions, messages):\n rm_prompt = ... # 构建 reward model 的prompt\n chat_response = self.client.chat.completions.create(\n model=self.verify_model_name,\n messages=[\n {\n 'role': 'system',\n 'content': 'You are a helpful assistant.'\n },\n {\n 'role': 'user',\n 'content': rm_prompt\n },\n ],\n )\n response = chat_response.choices[0].message.content.strip()\n reward = ... # 根据奖励模型生成结果提取奖励值\n rewards.append(reward)\n return rewards\n\n```\n"
},
{
"path": "docs/source/Instruction/GRPO/GetStarted/GRPO.md",
"content": "# GRPO\n\nGRPOTrainer在ms-swift3.5进行了代码重构，如果你使用的swift版本<3.5, 请参考[stable文档](https://github.com/modelscope/ms-swift/blob/v3.4.1/docs/source/Instruction/GRPO.md)\n\n[GRPO(Group Relative Policy Optimization)](https://arxiv.org/abs/2402.03300) 算法利用组内相对优势计算来替代 PPO 算法中独立的价值模型，并直接在损失函数中加入 KL 散度惩罚来提高训练稳定性。\n\n## 算法原理\n\nGRPO 目标函数\n\n$\n{\\scriptstyle\n\\begin{aligned}\n\\mathcal{J}_{G R P O}(\\theta) & =\\mathbb{E}_{\\left[q \\sim P(Q),\\left\\{o_i\\right\\}_{i=1}^G \\sim \\pi_{\\theta_{o l d}}(O \\mid q)\\right]} \\\\\n& \\frac{1}{G} \\sum_{i=1}^G \\frac{1}{\\left|o_i\\right|} \\sum_{t=1}^{\\left|o_i\\right|}\\left\\{\\min \\left[\\frac{\\pi_\\theta\\left(o_{i, t} \\mid q, o_{i, GRPO算法伪代码 \n\n```python\n# ========== 1. Rollout Generation Phase ==========\nprompt = \"Question: Which is bigger? 9.11 or 9.9?\"\n\n# Generate multiple completions through parallel sampling\ncompletions = rollout_function(\n model=current_policy_model,\n prompt=prompt,\n num_generations=8, # Hyperparameter: number of samples per prompt\n temperature=1.0 # Hyperparameter: sampling diversity\n)\n\"\"\"\ncompletions = [\n (completion 1) \"The larger number is 9.11...\",\n (completion 2) \"9.9 is bigger than...\",\n ...\n (completion 8) \"After calculation, 9.11...\"\n]\n\"\"\"\n\n# ========== 2. Reward Calculation Phase ==========\n# Evaluate generated completions using reward model\nrewards = reward_function(\n completions=completions,\n ground_truth=\"9.11\" # Expected correct answer\n)\n\"\"\"\nrewards = [\n (reward 1) 1.0, # Correct answer\n (reward 2) 0.0, # Incorrect\n ...\n (reward 8) 1.0 # Correct\n]\n\"\"\"\n\n# Normalize rewards to advantages\nrewards_mean = mean(rewards) # μ = 0.5\nrewards_std = std(rewards) # σ = 0.25\nadvantages = (rewards - rewards_mean) / (rewards_std + 1e-8) # Standardization\n\"\"\"\nadvantages = [\n (advantage 1) 2.0, # (1.0 - 0.5)/0.25\n (advantage 2) -2.0,\n ...\n (advantage 8) 2.0\n]\n\"\"\"\n\n# ========== 3. Policy Optimization Phase ==========\n# Get token-level log probabilities from different models\ncurrent_logps = get_per_token_logps(current_policy_model, prompt, completions) # π_θ\nold_logps = get_per_token_logps(old_policy_model, prompt, completions) # π_θ_old\nref_logps = get_per_token_logps(reference_model, prompt, completions) # π_ref\n\n# PPO Clipped Objective\nis_ratio = exp(current_logps - old_logps) # Importance sampling ratio: e^(π_θ - π_θ_old)\nclipped_ratio = clip(is_ratio, 1-ε, 1+ε) # ε=0.2 typically\n\n# Policy gradient term (dual form)\npolicy_loss = -mean(\n minimum(is_ratio * advantages, # Unclipped objective\n clipped_ratio * advantages) # Clipped objective\n)\n\n# KL Divergence Penalty (K3 estimator)\n# KL(π_θ||π_ref) ≈ e^(logπ_ref - logπ_θ) - (logπ_ref - logπ_θ) - 1\nkl_penalty = beta * mean(\n exp(ref_logps - current_logps) -\n (ref_logps - current_logps) - 1\n)\n\n# Total Loss = Policy Loss + KL Penalty\ntotal_loss = policy_loss + kl_penalty\n\n# ========== 4. Update Rule ==========\n# Apply gradient descent to minimize total_loss\noptimizer.zero_grad()\ntotal_loss.backward()\noptimizer.step()\n```\n\n\n训练脚本示例参考[examples](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo)\n\nGRPO参数参考[文档](../../../Instruction/Command-line-parameters.md#grpo参数)\n\n## 集群支持\n\n\n\nGRPO 训练框架支持集成高性能推理引擎（如 vLLM）来加速采样过程，提供以下两种部署模式：\n\n### 1. Colocate(Internal) Mode\n\n训练与推理共享GPU资源，在 Trainer 内部启动推理服务，\n\n启动参数\n```bash\n--use_vllm true \\\n--vllm_mode colocate\n```\n\n#### Colocate 模式下的显存优化方案\n在 Colocate 模式下运行时，容易出现显存不足（OOM）的情况。以下是几种有效的显存优化方法和参数配置：\n\n1. 降低`vllm_gpu_memory_utilization` 参数\n\n\n2. 在训练阶段，释放 vLLM 占用的显存：\n\n```bash\n--sleep_level 1\n```\n\n3. 在vLLM 推理阶段，释放模型和优化器占用的显存：\n\n```bash\n--offload_optimizer true \\\n--offload_model true \\\n```\n\n4. 在vLLM中使用 Tensor Parallel 技术：\n\n```bash\n--vllm_tensor_parallel_size [tp_size]\n```\n\n5. 分批 Gather 模型权重（zero3下同步 vLLM 权重时）：\n\n```bash\n--move_model_batches [批次数量]\n```\n\n6. 将 Megatron 导出的用于 vLLM 更新的 HF 格式权重存放在 CPU 主存中，以降低 GPU 显存占用：\n\n```bash\n--offload_bridge true\n```\n\n### 2. Async(External) Mode\n\n训练与推理资源分离，启动单独的推理服务器\n\n使用`swift rollout`命令部署vLLM 服务器, 现仅支持vLLM backend\n```bash\nCUDA_VISIBLE_DEVICES=0 \\\nswift rollout \\\n --model Qwen/Qwen2.5-VL-7B-Instruct \\\n --vllm_tensor_parallel_size 2 \\\n --vllm_data_parallel_size 1\n\nCUDA_VISIBLE_DEVICES=0,1 \\\nswift rollout \\\n --model Qwen/Qwen2.5-VL-7B-Instruct \\\n --vllm_tensor_parallel_size 2 \\\n --vllm_data_parallel_size 1\n\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nswift rollout \\\n --model Qwen/Qwen2.5-VL-7B-Instruct \\\n --vllm_tensor_parallel_size 2 \\\n --vllm_data_parallel_size 2\n```\n\n更多 rollout 参数参考[vLLM参数](../../../Instruction/Command-line-parameters.md#vllm参数)和[rollout 参数](../../../Instruction/Command-line-parameters.md#rollout参数)\n\n注意：在使用 use_async_engine 时，仅开启 DP 可能会导致错误，相关问题参考： [vllm issue](https://github.com/vllm-project/vllm/issues/18567)。如果出现错误，请尝试同时启用 TP 和 DP，或升级vLLM\n\n\n训练使用以下参数配置外部 vLLM 服务器\n```bash\n--use_vllm true \\\n--vllm_mode server \\\n--vllm_server_host <服务器IP> \\\n--vllm_server_port <服务端口> \\\n--vllm_server_timeout <超时时间> \\\n```\n#### 权重同步加速\nswift 3.10 优化了权重同步，设置以下参数可以进一步优化 LoRA 训练的权重同步速度。\n\n```bash\n# rollout(server mode)\nswift rollout \\\n --vllm_enable_lora true \\\n --vllm_max_lora_rank xxx # 与训练脚本lora_rank一致\n ...\n\n# grpo(colocate mode)\nswift rlhf \\\n --rlhf_type grpo \\\n --vllm_mode colocate \\\n --vllm_enable_lora true \\\n ...\n```\n\n注意：以下情况无法使用该优化：\n\n- 训练多模态模型的ViT层(freeze_vit false)\n- MoE 模型\n\n优化实现细节请参考该[PR](https://github.com/modelscope/ms-swift/pull/5773)\n\n## logged metrics\n- completions/mean_length：生成的 completion 的平均长度。\n- completions/min_length：生成的 completion 的最小长度。\n- completions/max_length：生成的 completion 的最大长度。\n- completions/clipped_ratio：被长度截断的 completion 占比。\n- reward/{reward_func_name}/mean：某个特定 reward function 的平均奖励值。\n- reward/{reward_func_name}/std：某个特定 reward function 的奖励标准差。\n> 注意, 上述两个指标是在所有 completions 范围内统计得到的。\n- reward：加权 reward_weights 后的整体平均奖励。\n- reward_std：加权 reward_weights 后，每个 batch 内整体奖励的标准差。\n> 注意：上述两个指标是先在每个组内分别计算均值/std，然后再对各组的结果取平均。\n- frac_reward_zero_std：在生成 batch 中，reward 标准差为零的样本比例，意味着该 prompt 上的答案几乎无多样性（所有回答奖励一致）。\n- kl：生成的 completion 上，模型与参考模型之间的平均 KL 散度。仅当 beta 非零时记录。\n- clip_ratio/region_mean：不同句子中被 CLIP 的的 token 平均比例\n- clip_ratio/low_mean：不同句子中被 下CLIP 的的 token 平均比例\n- clip_ratio/low_min：不同句子中被 下CLIP 的的 token 最小比例\n- clip_ratio/high_mean：不同句子中被 上CLIP 的的 token 平均比例\n- clip_ratio/high_max：不同句子中被 上CLIP 的的 token 最大比例\n> 注意：如果开启`overlong_filter`, kl 和 clip_ratio 指标会过滤超长的样本\n\n如果设置了`log_entropy`参数，则会额外记录entropy相关指标，包括\n- entropy/mean: 不同句子中的 entropy 均值\n- entropy/max: 不同句子中的 entropy 最大值\n- entropy/min: 不同句子中的 entropy 最小值\n> 注意这里的 句子 entropy 指 completion 中的 token entropy 均值\n\n\n如果设置了`top_entropy_quantile`参数<1.0, 则会记录entropy threshold的值\n- entropy/threshold: 分位点处的 entropy 值，小于该值的 token 将不会被计算 loss\n\n训推一致性指标，前缀为rollout_correction (ms-swift>=3.11)，需设置`log_rollout_offpolicy_metrics=true`或`rollout_importance_sampling_mode`：\n- `kl` / `k3_kl`：训练策略与 rollout 策略之间的 KL 散度（直接估计器 / K3 估计器）\n- `training_ppl` / `rollout_ppl`：训练策略和 rollout 策略的困惑度\n- `log_ppl_diff`：log PPL 差异，反映分布偏移程度\n- `ppl_ratio`：PPL 比率\n- `chi2_token` / `chi2_seq`：Token/Sequence 级别的 χ² 散度\n\nIS 校正指标（需设置`rollout_importance_sampling_mode`）：\n- `is_weight_mean`：平均重要性采样权重\n- `ess`：有效样本大小（Effective Sample Size）\n- `clipped_frac`：被截断或屏蔽的样本比例\n\n> 训推一致性指标详细说明请参考文档 [Training-Inference-Mismatch](../AdvancedResearch/training_inference_mismatch.md)\n\n如果设置了`log_completions`, 将保存训练动态在output对应文件夹中，包括\n- step：记录时的训练步数\n- prompt：模型输入\n- completion：模型采样回答\n- {reward_func_name}：特定奖励\n- entropy：entropy token 均值，在设置`log_entropy`时记录\n\n设置 `report_to wandb/swanlab` 将训练动态Table推送到对应的平台\n\n如果需要在Table中额外记录其他列，请在 `GRPOTrainer._generate_and_score_completions` 方法中，设置 metrics_to_gather 字典。\n\n默认自动检测\n- `image`：视觉数据集图像输入。(暂时只支持wandb)\n- `solution`：数据集中的 solution 列。\n\n## FAQ\n**1. 训练过程中 loss 等于0 / 接近0 / 小于0**\n\n正常情况， 参考[issue](https://github.com/huggingface/open-r1/issues/239#issuecomment-2646297851)\n\n**2. num_generations / 批量大小相关**\n\n在 GRPO 中，batch_size 以 completion（模型生成结果） 为单位。例如，设置 per_device_train_batch_size=8 表示每张 GPU 在训练过程中会同时处理 8 个 completion 的 loss 计算。\n\n训练阶段，在一次完整的梯度累计 batch 中，总的批量大小等于：\n\n```\neffective_batch_size = num_processes * per_device_train_batch_size * gradient_accumulation_steps\n```\n\n采样阶段，总的批量大小 (completion-level) 数量等于:\n\n1. 设置 generation_batch_size 下，等于 generation_batch_size\n2. 设置 steps_per_generation 下，等于 per_device_train_batch_size * steps_per_generation * num_processes\n3. 默认情况下，steps_per_generation = gradient_accumulation_steps，generation_batch_size = per_device_train_batch_size * steps_per_generation * num_processes = per_device_train_batch_size * gradient_accumulation_steps * num_processes = effective_batch_size\n\n在评估阶段，completion 的数量等于：\n```\nnum_processes * per_device_eval_batch_size\n```\n\n参数 `num_generations` 必须能够被以上采样阶段和评估的总批量大小整除，以保证生成任务可以均匀分配到各个设备上。\n\n**示例**\n\n- num_processes = 8\n- per_device_train_batch_size = 4\n- gradient_accumulation_steps = 8\n- generation_batch_size = 512\n- num_generations = 64\n\n1. 采样需要的总数据(prompt)量等于 512 / 64 = 8\n2. 每次采样 512 条模型回复\n3. 每次更新模型权重批量大小为 8 *4 * 8 = 256\n\n\n**3. 为什么 KL 出现了NaN**\n\n开启 overlong_filter 后，某一卡上的所有 completion 都被截断\n\n**4. 训练的steps怎么计算?**\n\n参考[issue](https://github.com/modelscope/ms-swift/issues/3912)\n\n**5. clip_ratio为什么总是0?**\n\nClip机制的核心目的是限制策略更新的幅度，防止因单次更新过大而导致策略性能崩溃（即策略更新后表现急剧下降）。\nClip操作的具体公式如下：\n\n$\nL_{\\text{CLIP}}(\\theta) = \\mathbb{E}_{t} \\left[ \\min\\left(r_{t}(\\theta) \\hat{A}_{t}, \\text{clip}(r_{t}(\\theta), 1 - \\epsilon, 1 + \\epsilon) \\hat{A}_{t} \\right) \\right]\n$\n\n其中：$r_{t}(\\theta) = \\frac{\\pi_{\\theta}(a_{t} \\mid s_{t})}{\\pi_{\\text{old}}(a_{t} \\mid s_{t})}$ 是重要性采样比，衡量新旧策略的差异。$\\hat{A}_{t}$ 是优势函数（advantage function），表示动作的相对收益。$\\epsilon$ 用于限制 $r_{t}(\\theta)$ 的偏离范围。\n\n在 on-policy 训练过程中，由于每次更新都使用最新策略生成的数据，新旧策略相同，即 $\\pi_{\\theta} = \\pi_{\\text{old}}$\n\n因此重要性采样比恒为 1，此时，clip 操作不会生效。\n\n在设置以下参数情况下，算法为off-policy (near-on-policy)\n1. num_iterations > 1, 或者\n2. gradient_accumulation_steps % steps_per_generation != 0\n\n参考[issue](https://github.com/huggingface/open-r1/issues/239#issuecomment-2646297851)\n\n**6. 为什么没有设置val_dataset，仍然有验证过程，如何取消**\n\n当没有显式传入`val_dataset`时，参数`split_dataset_ratio`负责切分部分`dataset`为验证数据集，默认切分1%数据（在\"ms-swift>=3.6\"中，`split_dataset_ratio`的默认值将从0.01修改为0.）\n\n通过设置`--split_dataset_ratio 0` 来取消验证过程\n\n**7. 如何设置训练的 `mini-batch size`**\n\n在 GRPO 训练中，我们可以通过以下两种方式配置 mini-batch 更新：\n- 设置 `generation_batch_size` 为训练 global batch size (effective_batch_size) 的整数倍\n- 或设置 `steps_per_generation` 为 `gradient_accumulation_steps` 的整数倍\n\n典型配置示例：\n- 当配置：\nsteps_per_generation = 16, gradient_accumulation_steps = 8, mini_batch_size = steps_per_generation / gradient_accumulation_steps = 2. 则 1 次 rollout 结果将拆分成 2 批 mini-batch 进行更新。\n\n**8. swift deploy 与 swift rollout 的区别**\n\n- swift deploy 主要用于模型的部署和推理，支持 PT、vLLM、SGLang 等多种引擎，兼容流式推理与 OpenAI API 的调用格式。\n\n- swift rollout 则专注于 GRPO 推理加速，目前仅支持 vLLM 引擎，并内置了权重自动同步的功能。\n\n\n**9. 如何取消 KL 项损失**\n\n将参数设置为 `--beta 0`，即可关闭 KL 损失的计算，并且不会加载参考模型（ref model）。\n\n\n## RL微信群\n\n//`等标签（代表图像表征在`inputs_embeds`中的位置），例如输入`这两张图有啥区别`，`描述这段视频`。然后根据提示输入相应的图像/视频/音频即可。\n\n以下为一个推理的示例：\n```\n<<< 这两张图有什么区别\nInput an image path or URL <<< http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png\nInput an image path or URL <<< http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png\n这两张图片的区别在于它们所展示的动物和场景。\n\n1. **第一张图片**：\n - 展示了一只小猫。\n - 小猫有大大的眼睛，表情显得有些困惑或好奇。\n - 背景是模糊的，可能是室内环境。\n\n2. **第二张图片**：\n - 展示了一群羊。\n - 羊们站在草地上，背景是绿色的山丘和蓝天白云。\n - 羊的表情看起来很平静，似乎在享受大自然的环境。\n\n总结来说，第一张图片是一只小猫，而第二张图片是一群羊。\n--------------------------------------------------\n<<< clear\n<<< 描述这段视频\nInput a video path or URL <<< https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4\nThe video shows a baby wearing sunglasses sitting on a bed and reading a book. The baby is holding the book with both hands and appears to be focused on the pages. The baby's feet are visible in the frame, and they are moving slightly as they read. The background of the video shows a room with a bed and some furniture.\n```\n\n\n**数据集推理：**\n```shell\nCUDA_VISIBLE_DEVICES=0 swift infer \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --stream true \\\n --infer_backend transformers \\\n --val_dataset AI-ModelScope/alpaca-gpt4-data-zh \\\n --max_new_tokens 2048\n```\n\n以上提供了全参数和LoRA流式推理的例子，以下介绍更多SWIFT中的推理技术：\n- 界面推理：你可以将`swift infer`改成`swift app`。\n- batch推理：`infer_backend=transformers`可以指定`--max_batch_size`对大模型和多模态大模型进行batch推理，具体参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/transformers/batch_ddp.sh)。在进行batch推理时，你不能设置`--stream true`。\n- DDP/device_map推理：`infer_backend=transformers`支持使用DDP/device_map技术进行并行推理，具体参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/transformers/mllm_device_map.sh)。\n- 推理加速：swift支持使用vllm/sglang/lmdeploy对推理、部署和评测模块进行推理加速，只需要额外指定`--infer_backend vllm/sglang/lmdeploy`即可。可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/vllm/mllm_ddp.sh)。\n- 多模态模型：我们提供了[transformers](https://github.com/modelscope/ms-swift/blob/main/examples/infer/transformers/mllm_device_map.sh)/[vllm](https://github.com/modelscope/ms-swift/blob/main/examples/infer/vllm/mllm_tp.sh)/[lmdeploy](https://github.com/modelscope/ms-swift/blob/main/examples/infer/lmdeploy/mllm_tp.sh)对多模态模型进行多GPU推理的shell脚本。\n- 量化模型：直接选择GPTQ、AWQ、BNB量化的模型，例如：`--model Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4`即可。\n- 更多模型类型：我们提供了[bert](https://github.com/modelscope/ms-swift/blob/main/examples/infer/transformers/bert.sh)、[reward_model](https://github.com/modelscope/ms-swift/blob/main/examples/infer/transformers/reward_model.sh)、[prm](https://github.com/modelscope/ms-swift/blob/main/examples/infer/transformers/prm.sh)的推理脚本。\n\n\n**小帖士：**\n- SWIFT会将推理结果保存起来，你可以通过`--result_path`指定保存路径。\n- 如果要输出logprobs，只需要在推理时，指定`--logprobs true`即可。SWIFT会保存。注意，设置`--stream true`将不会存储。\n- infer_backend为'transformers'支持所有swift已支持模型的推理，而infer_backend为vllm/sglang/lmdeploy只支持部分模型，具体请参考[vllm](https://docs.vllm.ai/en/latest/models/supported_models.html)、[sglang](https://docs.sglang.ai/supported_models/generative_models.html)、[lmdeploy](https://lmdeploy.readthedocs.io/en/latest/supported_models/supported_models.html)文档。\n- 使用`--infer_backend vllm`出现OOM，可以通过降低`--vllm_max_model_len`，`--vllm_max_num_seqs`，选择合适的`--vllm_gpu_memory_utilization`，设置`--vllm_enforce_eager true`。或者使用tensor并行`--vllm_tensor_parallel_size`来解决。\n- 使用`--infer_backend vllm`推理多模态模型，需要传入多张图片。可以设置`--vllm_limit_mm_per_prompt`解决，例如：`--vllm_limit_mm_per_prompt '{\"image\": 10, \"video\": 5}'`。\n- 推理qwen2-vl/qwen2.5-vl出现OOM，可以通过设置`MAX_PIXELS`、`VIDEO_MAX_PIXELS`、`FPS_MAX_FRAMES`解决，可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/app/mllm.sh)。\n- swift内置对话模板与使用transformers运行的对话模板对齐，测试参考[这里](https://github.com/modelscope/ms-swift/blob/main/tests/test_align/test_template/test_vision.py)。如果出现未对齐情况，欢迎提issue/PR修正。\n\n\n### 使用Web-UI\n如果你要使用界面的方式进行推理，可以查看[Web-UI文档](../GetStarted/Web-UI.md)。\n\n### 使用Python\n\n文本模型：\n```python\nimport os\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\n\nfrom swift.infer_engine import TransformersEngine, RequestConfig, InferRequest\nmodel = 'Qwen/Qwen2.5-0.5B-Instruct'\n\n# 加载推理引擎\nengine = TransformersEngine(model, max_batch_size=2)\nrequest_config = RequestConfig(max_tokens=512, temperature=0)\n\n# 这里使用了2个infer_request来展示batch推理\ninfer_requests = [\n InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}]),\n InferRequest(messages=[{'role': 'user', 'content': '浙江的省会在哪？'},\n {'role': 'assistant', 'content': '浙江省的省会是杭州。'},\n {'role': 'user', 'content': '这里有什么好玩的地方'},]),\n]\nresp_list = engine.infer(infer_requests, request_config)\nquery0 = infer_requests[0].messages[0]['content']\nprint(f'response0: {resp_list[0].choices[0].message.content}')\nprint(f'response1: {resp_list[1].choices[0].message.content}')\n```\n\n多模态模型：\n```python\nimport os\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\nos.environ['MAX_PIXELS'] = '1003520'\nos.environ['VIDEO_MAX_PIXELS'] = '50176'\nos.environ['FPS_MAX_FRAMES'] = '12'\n\nfrom swift.infer_engine import TransformersEngine, RequestConfig, InferRequest\nmodel = 'Qwen/Qwen2.5-VL-3B-Instruct'\n\n# 加载推理引擎\nengine = TransformersEngine(model, max_batch_size=2)\nrequest_config = RequestConfig(max_tokens=512, temperature=0)\n\n# 这里使用了3个infer_request来展示batch推理\ninfer_requests = [\n InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}]),\n InferRequest(messages=[{'role': 'user', 'content': '两张图的区别是什么？'}],\n images=['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png',\n 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png']),\n InferRequest(messages=[{'role': 'user', 'content': 'describe the video'}],\n videos=['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']),\n]\nresp_list = engine.infer(infer_requests, request_config)\nquery0 = infer_requests[0].messages[0]['content']\nprint(f'response0: {resp_list[0].choices[0].message.content}')\nprint(f'response1: {resp_list[1].choices[0].message.content}')\nprint(f'response2: {resp_list[2].choices[0].message.content}')\n```\n\n我们也提供了更多使用python推理的demo：\n- 使用流式推理以及`VllmEngine`、`SglangEngine`、`LmdeployEngine`进行推理加速，可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo.py)。\n- 多模态推理：除了上述多模态输入格式外，swift兼容OpenAI的多模态输入格式，参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_mllm.py)。\n- grounding任务：对多模态模型进行Grounding任务画框，可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_grounding.py)。\n- 多LoRA推理：参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_lora.py)。\n- agent推理：参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_agent.py)。\n- 异步接口：使用`engine.infer_async`进行python方式推理，参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo.py)。\n\n\n## 部署\n\n如果要查看LoRA微调后模型的部署，可以参考[预训练与微调文档](./Pre-training-and-Fine-tuning.md#部署微调后模型)。\n\n这里主要介绍对多模态模型的部署和调用，文本大模型我们给出简单的部署和调用的简单案例：\n\n服务端部署：\n```shell\nCUDA_VISIBLE_DEVICES=0 swift deploy \\\n --model Qwen/Qwen2.5-7B-Instruct \\\n --infer_backend vllm \\\n --max_new_tokens 2048 \\\n --served_model_name Qwen2.5-7B-Instruct\n```\n\n客户端调用测试：\n```shell\ncurl http://localhost:8000/v1/chat/completions \\\n-H \"Content-Type: application/json\" \\\n-d '{\n\"model\": \"Qwen2.5-7B-Instruct\",\n\"messages\": [{\"role\": \"user\", \"content\": \"晚上睡不着觉怎么办？\"}],\n\"max_tokens\": 256,\n\"temperature\": 0\n}'\n```\n\n\n### 服务端\n\n```shell\n# test env: pip install transformers==4.51.3 vllm==0.8.5.post1\nCUDA_VISIBLE_DEVICES=0 \\\nMAX_PIXELS=1003520 \\\nVIDEO_MAX_PIXELS=50176 \\\nFPS_MAX_FRAMES=12 \\\nswift deploy \\\n --model Qwen/Qwen2.5-VL-3B-Instruct \\\n --infer_backend vllm \\\n --vllm_gpu_memory_utilization 0.9 \\\n --vllm_max_model_len 8192 \\\n --max_new_tokens 2048 \\\n --vllm_limit_mm_per_prompt '{\"image\": 5, \"video\": 2}' \\\n --served_model_name Qwen2.5-VL-3B-Instruct\n```\n\n\n### 客户端\n\n这里介绍3种调用客户端的方式，分别是curl、openai库和swift客户端。\n\n\n方案一: curl\n```shell\ncurl http://localhost:8000/v1/chat/completions \\\n-H \"Content-Type: application/json\" \\\n-d '{\n\"model\": \"Qwen2.5-VL-3B-Instruct\",\n\"messages\": [{\"role\": \"user\", \"content\": [\n {\"type\": \"image\", \"image\": \"http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png\"},\n {\"type\": \"image\", \"image\": \"http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png\"},\n {\"type\": \"text\", \"text\": \"两张图的区别是什么？\"}\n]}],\n\"max_tokens\": 256,\n\"temperature\": 0\n}'\n```\n\n方案2: openai库\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key='EMPTY',\n base_url=f'http://127.0.0.1:8000/v1',\n)\nmodel = client.models.list().data[0].id\nprint(f'model: {model}')\n\nmessages = [{'role': 'user', 'content': [\n {'type': 'video', 'video': 'https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4'},\n {'type': 'text', 'text': 'describe the video'}\n]}]\n\nresp = client.chat.completions.create(model=model, messages=messages, max_tokens=512, temperature=0)\nquery = messages[0]['content']\nresponse = resp.choices[0].message.content\nprint(f'query: {query}')\nprint(f'response: {response}')\n\n# base64\nimport base64\nimport requests\nresp = requests.get('https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4')\nbase64_encoded = base64.b64encode(resp.content).decode('utf-8')\nmessages = [{'role': 'user', 'content': [\n {'type': 'video', 'video': f'data:video/mp4;base64,{base64_encoded}'},\n {'type': 'text', 'text': 'describe the video'}\n]}]\n\ngen = client.chat.completions.create(model=model, messages=messages, stream=True, temperature=0)\nprint(f'query: {query}\\nresponse: ', end='')\nfor chunk in gen:\n if chunk is None:\n continue\n print(chunk.choices[0].delta.content, end='', flush=True)\nprint()\n```\n\n方案三：swift客户端\n\n```python\nfrom swift import InferRequest, InferClient, RequestConfig, InferStats\n\nengine = InferClient(host='127.0.0.1', port=8000)\nprint(f'models: {engine.models}')\nmetric = InferStats()\nrequest_config = RequestConfig(max_tokens=512, temperature=0)\n\n# 这里使用了3个infer_request来展示batch推理\n# 支持传入本地路径、base64和url\ninfer_requests = [\n InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}]),\n InferRequest(messages=[{'role': 'user', 'content': '两张图的区别是什么？'}],\n images=['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/cat.png',\n 'http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png']),\n InferRequest(messages=[{'role': 'user', 'content': 'describe the video'}],\n videos=['https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4']),\n]\n\nresp_list = engine.infer(infer_requests, request_config, metrics=[metric])\nprint(f'response0: {resp_list[0].choices[0].message.content}')\nprint(f'response1: {resp_list[1].choices[0].message.content}')\nprint(f'response2: {resp_list[2].choices[0].message.content}')\nprint(metric.compute())\nmetric.reset()\n\n# base64\nimport base64\nimport requests\nresp = requests.get('https://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/baby.mp4')\nbase64_encoded = base64.b64encode(resp.content).decode('utf-8')\nmessages = [{'role': 'user', 'content': [\n {'type': 'video', 'video': f'data:video/mp4;base64,{base64_encoded}'},\n {'type': 'text', 'text': 'describe the video'}\n]}]\ninfer_request = InferRequest(messages=messages)\nrequest_config = RequestConfig(max_tokens=512, temperature=0, stream=True)\ngen_list = engine.infer([infer_request], request_config, metrics=[metric])\nprint(f'response0: ', end='')\nfor chunk in gen_list[0]:\n if chunk is None:\n continue\n print(chunk.choices[0].delta.content, end='', flush=True)\nprint()\nprint(metric.compute())\n```\n我们也提供了更多部署的demo：\n- 多LoRA部署与调用: 参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/deploy/lora)。\n- Base模型的部署与调用: 参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/deploy/client/llm/base)。\n- 更多模型类型: 我们提供了[bert](https://github.com/modelscope/ms-swift/tree/main/examples/deploy/bert)、[reward_model](https://github.com/modelscope/ms-swift/tree/main/examples/deploy/reward_model)的部署脚本。\n"
},
{
"path": "docs/source/Instruction/Pre-training-and-Fine-tuning.md",
"content": "# 预训练与微调\n\n训练能力：\n\n| 方法 | 全参数 | LoRA | QLoRA | Deepspeed | 多机 | 多模态 |\n| ------ | ------ |---------------------------------------------------------------------------------------------| ----- | ------ | ------ |----------------------------------------------------------------------------------------------|\n| [预训练](https://github.com/modelscope/ms-swift/blob/main/examples/train/pretrain) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [指令监督微调](https://github.com/modelscope/ms-swift/blob/main/examples/train/lora_sft.sh) | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/full/train.sh) | ✅ | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/qlora) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-gpu/deepspeed) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-node) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multimodal) |\n| [GRPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [GKD](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/gkd) | ✅ | ✅ | ✅ | ✅ | ✅ | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/gkd) |\n| [PPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/ppo) | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |\n| [DPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/dpo) | ✅ | ✅ | ✅ | ✅ | ✅ | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/dpo) |\n| [KTO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/kto.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/kto.sh) |\n| [奖励模型](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/rm.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [CPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/cpo.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [SimPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/simpo.sh) | ✅ | ✅ | ✅ | ✅| ✅ | ✅ |\n| [ORPO](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/orpo.sh) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [Embedding](https://github.com/modelscope/ms-swift/blob/main/examples/train/embedding) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [Reranker](https://github.com/modelscope/ms-swift/tree/main/examples/train/reranker) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n| [序列分类](https://github.com/modelscope/ms-swift/blob/main/examples/train/seq_cls) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |\n\n## 环境准备\n推荐的第三方库版本参考[SWIFT安装文档](../GetStarted/SWIFT-installation.md)\n```bash\npip install ms-swift -U\n\n# 若使用deepspeed zero2/zero3\npip install deepspeed -U\n```\n\n## 预训练\n预训练使用`swift pt`命令，这将自动使用生成式而非对话式的template，即将`use_chat_template`设置为False（其他所有的命令，例如`swift sft/rlhf/infer`，都默认将`use_chat_template`设置为True）。此外，`swift pt`与`swift sft`相比，具有不同的数据集格式，可以参考[自定义数据集文档](../Customization/Custom-dataset.md)。\n\n使用CLI进行预训练的脚本可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/pretrain/train.sh)。更多训练技术的介绍可以参考微调章节。\n\n小贴士：\n- `swift pt`与`swift sft --use_chat_template false --loss_scale all`等价。\n\n## 微调\n\nms-swift使用了分层式的设计思想，用户可以使用命令行界面、Web-UI界面和直接使用Python的方式进行微调。\n\n### 使用CLI\n\n我们提供了10分钟在单卡3090上对Qwen2.5-7B-Instruct进行自我认知微调的最佳实践，具体参考[这里](../GetStarted/Quick-start.md)，这可以帮助您快速了解SWIFT。\n\n此外，我们给出了一系列脚本帮助您了解SWIFT的训练能力：\n\n- 轻量化训练：SWIFT支持的轻量微调示例可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/tuners)。（注意：这些方式预训练也可以使用，但预训练通常使用全参数训练）。\n- 分布式训练：SWIFT支持的分布式训练技术包括：DDP、device_map、DeepSpeed ZeRO2/ZeRO3、FSDP。\n - device_map: 简易模型并行。如果存在多GPU，device_map会自动开启。这会将模型按层均匀的划分到可见的GPU中，显著降低显存消耗，但是训练速度通常会降低，因为是串行的。\n - DDP+device_map：将按组对模型进行device_map划分，参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/multi-gpu/ddp_device_map/train.sh)。\n - DeepSpeed ZeRO2/ZeRO3: 节约显存资源，但训练速度下降。ZeRO2将对优化器状态、模型梯度进行分片。ZeRO3在ZeRO2基础上，对模型参数进行分片，更加节约显存，但训练速度更慢。参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-gpu/deepspeed)。\n - FSDP+QLoRA: 双卡3090运行70B模型的训练，参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-gpu/fsdp_qlora/train.sh)。\n - 多机多卡训练: 我们书写了使用swift、torchrun、dlc、deepspeed、accelerate启动多节点运行的shell脚本示例。除了dlc和deepspeed，其他启动脚本都需要在所有节点中启动才可运行。具体参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/multi-node)。\n- 量化训练：支持使用GPTQ、AWQ、AQLM、BNB、HQQ、EETQ量化技术的QLoRA训练。微调7B模型只需要9GB显存资源。具体参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/qlora)。\n- 多模态训练：SWIFT支持多模态模型的预训练、微调和RLHF。支持Caption、VQA、OCR、[Grounding](https://github.com/modelscope/ms-swift/blob/main/examples/notebook/qwen2_5-vl-grounding/zh.ipynb)任务。支持图像、视频和音频三种模态。具体参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/multimodal)。多模态自定义数据集格式参考[自定义数据集文档](../Customization/Custom-dataset.md)。\n - 对ViT/Aligner使用全参数训练，LLM使用LoRA训练，并采用不同学习率的例子参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/multimodal/lora_llm_full_vit)。\n - 多模态模型packing，增加训练速度，例子参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/packing)。\n- RLHF训练：参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/rlhf)。多模态模型参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/multimodal/rlhf)。GRPO训练参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo/internal)。强化微调查看[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/rft)。\n- Megatron训练：支持使用Megatron的并行技术来加速大模型的训练，包括数据并行、张量并行、流水线并行、序列并行，上下文并行。参考[Megatron-SWIFT训练文档](../Megatron-SWIFT/Quick-start.md)。\n- 序列分类模型训练：参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/seq_cls)。\n- Embedding模型训练：参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/embedding)\n- Agent训练：参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/agent)。\n- Any-to-Any模型训练：参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/all_to_all)。\n- 其他能力：\n - 数据流式读取: 在数据量较大时减少内存使用。参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/streaming/streaming.sh)。\n - packing: 将多个序列拼成一个，让每个训练样本尽可能接近max_length，提高显卡利用率，参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/packing)。\n - 长文本训练: 参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/sequence_parallel)。\n - lazy tokenize: 在训练期间对数据进行tokenize而不是在训练前tokenize（多模态模型可以避免在训练前读入所有多模态资源），这可以避免预处理等待并节约内存。参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/train/streaming/lazy_tokenize.sh)。\n\n小帖士：\n\n- 在使用`swift sft`通过LoRA技术微调base模型为chat模型时，有时需要手动设置模板。通过添加`--template default`参数来避免base模型因未见过对话模板中的特殊字符而无法正常停止的情况。具体参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/base_to_chat)。\n- 如果需要在**断网**环境下进行训练，请设置`--model `和`--check_model false`。如果对应的模型需要`git clone`github的仓库，例如`deepseek-ai/Janus-Pro-7B`，请设置手动下载仓库，并设置`--local_repo_path `。具体参数含义请参考[命令行参数文档](Command-line-parameters.md)。\n- 无法对QLoRA训练的模型进行Merge LoRA，因此不建议使用QLoRA进行微调，无法在推理和部署时使用vLLM/Sglang/LMDeploy进行推理加速。建议使用LoRA/全参数进行微调，合并为完整权重后再使用GPTQ/AWQ/BNB进行[量化](https://github.com/modelscope/ms-swift/tree/main/examples/export/quantize)。\n- 如果使用NPU进行训练，只需要将shell中的`CUDA_VISIBLE_DEVICES`修改为`ASCEND_RT_VISIBLE_DEVICES`。\n- SWIFT默认在训练时设置`--gradient_checkpointing true`来节约显存，这会略微降低训练速度。\n- 若使用DDP进行训练，出现报错：`RuntimeError: Expected to mark a variable ready only once.`，请额外设置参数`--gradient_checkpointing_kwargs '{\"use_reentrant\": false}'`或者使用DeepSpeed进行训练。\n- 如果要使用deepspeed，你需要安装deepspeed：`pip install deepspeed -U`。使用deepspeed可以节约显存，但会略微降低训练速度。\n- 如果您的机器是A100等高性能显卡，且模型支持flash-attn，推荐你安装[flash-attn](https://github.com/Dao-AILab/flash-attention/releases)，并设置`--attn_impl flash_attn`，这将会加快训练和推理的速度并略微降低显存占用。\n\n**如何debug：**\n\n你可以使用以下方式进行debug，这与使用命令行微调是等价的，但此方式不支持分布式。微调命令行运行入口可以查看[这里](https://github.com/modelscope/ms-swift/blob/main/swift/cli/sft.py)。\n\n```python\nfrom swift import sft_main, SftArguments\nresult = sft_main(SftArguments(\n model='Qwen/Qwen2.5-7B-Instruct',\n tuner_type='lora',\n dataset=['AI-ModelScope/alpaca-gpt4-data-zh#500',\n 'AI-ModelScope/alpaca-gpt4-data-en#500',\n 'swift/self-cognition#500'],\n torch_dtype='bfloat16',\n # ...\n))\n```\n\n\n### 使用Web-UI\n如果你要使用界面的方式进行训练，可以查看[Web-UI文档](../GetStarted/Web-UI.md)。\n\n### 使用python\n\n- Qwen2.5自我认知微调notebook查看[这里](https://github.com/modelscope/ms-swift/blob/main/examples/notebook/qwen2_5-self-cognition/self-cognition-sft.ipynb)。\n- Qwen2VL进行OCR任务notebook查看[这里](https://github.com/modelscope/ms-swift/blob/main/examples/notebook/qwen2vl-ocr/ocr-sft.ipynb)。\n\n\n## Merge LoRA\n\n- 查看[这里](https://github.com/modelscope/ms-swift/blob/main/examples/export/merge_lora.sh)。\n\n## 推理（微调后模型）\n\n使用CLI对LoRA训练的checkpoint进行推理：\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --infer_backend transformers \\\n --stream true \\\n --temperature 0 \\\n --max_new_tokens 2048\n```\n- adapters文件夹中包含了训练的参数文件`args.json`，因此不需要额外指定`--model`，`--system`，swift会自动读取这些参数。如果要关闭此行为，可以设置`--load_args false`。\n- 如果使用全参数训练，请使用`--model`替代`--adapters`指定训练的checkpoint目录。更多参考[推理和部署文档](./Inference-and-deployment.md#推理)。\n- 你可以使用`swift app`替代`swift infer`进行界面推理。\n- 你可以选择对LoRA进行merge（额外指定`--merge_lora true`），然后指定`--infer_backend vllm/sglang/lmdeploy`进行推理加速。\n\n对数据集中的验证集进行批量推理：\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --infer_backend transformers \\\n --temperature 0 \\\n --max_new_tokens 2048 \\\n --load_data_args true \\\n --max_batch_size 1\n```\n\n- 你可以设置`--max_batch_size 8`，从而使用`--infer_backend transformers`进行批量处理。若使用`infer_backend vllm/sglang/lmdeploy`则无需指定，会进行自动batch。\n- `--load_data_args true`会额外读取训练存储参数文件`args.json`中的数据参数。\n\n若想对额外的测试集进行推理，而不使用训练时的验证集，使用`--val_dataset `进行推理：\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift infer \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --infer_backend transformers \\\n --temperature 0 \\\n --max_new_tokens 2048 \\\n --val_dataset \\\n --max_batch_size 1\n```\n\n\n使用Python对训练后LoRA推理的例子如下：\n\n```python\nimport os\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\n\nfrom swift.infer_engine import TransformersEngine, RequestConfig, InferRequest\nfrom swift import get_model_processor, get_template\nfrom swift.utils import safe_snapshot_download\nfrom peft import PeftModel\n# 请调整下面几行\nmodel = 'Qwen/Qwen2.5-7B-Instruct'\nlora_checkpoint = safe_snapshot_download('swift/test_lora') # 修改成checkpoint_dir\ntemplate_type = None # None: 使用对应模型默认的template_type\ndefault_system = \"You are a helpful assistant.\" # None: 使用对应模型默认的default_system\n\n# 加载模型和对话模板\nmodel, tokenizer = get_model_processor(model)\nif lora_checkpoint is not None:\n model = PeftModel.from_pretrained(model, lora_checkpoint)\ntemplate_type = template_type or model.model_meta.template\ntemplate = get_template(tokenizer, template_type=template_type, default_system=default_system)\nengine = TransformersEngine(model, template=template, max_batch_size=2)\nrequest_config = RequestConfig(max_tokens=512, temperature=0)\n\n# 这里使用了2个infer_request来展示batch推理\ninfer_requests = [\n InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}]),\n InferRequest(messages=[{'role': 'user', 'content': '浙江的省会在哪？'},\n {'role': 'assistant', 'content': '浙江的省会在哪？'},\n {'role': 'user', 'content': '这里有什么好吃的'},]),\n]\nresp_list = engine.infer(infer_requests, request_config)\nquery0 = infer_requests[0].messages[0]['content']\nprint(f'response0: {resp_list[0].choices[0].message.content}')\nprint(f'response1: {resp_list[1].choices[0].message.content}')\n```\n\n多模态模型的LoRA推理示例如下：\n```python\nimport os\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\n\nfrom swift.infer_engine import TransformersEngine, RequestConfig, InferRequest\nfrom swift import get_model_processor, get_template\nfrom swift.utils import safe_snapshot_download\nfrom peft import PeftModel\n# 请调整下面几行\nmodel = 'Qwen/Qwen2.5-VL-7B-Instruct'\nlora_checkpoint = safe_snapshot_download('swift/test_grounding') # 修改成checkpoint_dir\ntemplate_type = None # None: 使用对应模型默认的template_type\ndefault_system = None # None: 使用对应模型默认的default_system\n\n# 加载模型和对话模板\nmodel, tokenizer = get_model_processor(model)\nif lora_checkpoint is not None:\n model = PeftModel.from_pretrained(model, lora_checkpoint)\ntemplate_type = template_type or model.model_meta.template\ntemplate = get_template(tokenizer, template_type=template_type, default_system=default_system)\nengine = TransformersEngine(model, template=template, max_batch_size=2)\nrequest_config = RequestConfig(max_tokens=512, temperature=0)\n\n# 这里使用了2个infer_request来展示batch推理\ninfer_requests = [\n InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}]),\n InferRequest(messages=[{'role': 'user', 'content': 'Task: Object Detection'}],\n images=['http://modelscope-open.oss-cn-hangzhou.aliyuncs.com/images/animal.png']),\n]\nresp_list = engine.infer(infer_requests, request_config)\nquery0 = infer_requests[0].messages[0]['content']\nprint(f'response0: {resp_list[0].choices[0].message.content}')\nprint(f'response1: {resp_list[1].choices[0].message.content}')\n```\n\n如果使用ms-swift训练的模型，可以通过以下方式获取训练的配置：\n```python\nfrom swift import safe_snapshot_download, BaseArguments\n\nlora_adapters = safe_snapshot_download('swift/test_lora')\nargs = BaseArguments.from_pretrained(lora_adapters)\nprint(f'args.model: {args.model}')\nprint(f'args.model_type: {args.model_type}')\nprint(f'args.template_type: {args.template}')\nprint(f'args.default_system: {args.system}')\n```\n\n- 对全参数训练的checkpoint进行推理，将`model`设置为checkpoint_dir，并将lora_checkpoint设置为None即可。更多参考[推理和部署文档](./Inference-and-deployment.md#推理)。\n- 使用流式推理以及`VllmEngine`、`SglangEngine`、`LmdeployEngine`进行推理加速，可以参考[大模型](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo.py)和[多模态大模型](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_mllm.py)推理示例。\n- 微调后的模型使用huggingface transformers/peft生态推理，可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_hf.py)。\n- 若训练了多个LoRA，要进行多LoRA切换，可以参考[推理](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_lora.py)、[部署](https://github.com/modelscope/ms-swift/tree/main/examples/deploy/lora)样例。\n- 对多模态模型进行Grounding任务的画框，可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_grounding.py)。\n- 对LoRA微调后的Bert进行推理，可以参考[这里](https://github.com/modelscope/ms-swift/blob/main/examples/infer/demo_bert.py)。\n\n\n## 部署（微调后模型）\n\n使用以下命令启动部署服务端。如果权重使用全参数训练，请使用`--model`替代`--adapters`指定训练的checkpoint目录。你可以参考[推理和部署文档](./Inference-and-deployment.md#部署)介绍的客户端调用方式：curl、openai库和swift客户端进行调用。\n\n```shell\nCUDA_VISIBLE_DEVICES=0 \\\nswift deploy \\\n --adapters output/vx-xxx/checkpoint-xxx \\\n --infer_backend transformers \\\n --temperature 0 \\\n --max_new_tokens 2048 \\\n --served_model_name ''\n```\n\n这里将给出使用vLLM对多LoRA进行部署并调用的完整例子。\n\n### 服务端\n首先你需要安装vLLM：`pip install vllm -U`，并在部署时使用`--infer_backend vllm`，这通常可以显著加速推理速度。\n\n我们预先训练了2个基模型为`Qwen/Qwen2.5-7B-Instruct`的不同自我认知LoRA增量权重（可以直接跑通），我们可以在[args.json](https://modelscope.cn/models/swift/test_lora/file/view/master)中找到相关信息。你需要在部署时修改`--adapters`指定训练好的LoRA权重本地路径即可。\n```bash\nCUDA_VISIBLE_DEVICES=0 \\\nswift deploy \\\n --adapters lora1=swift/test_lora lora2=swift/test_lora2 \\\n --infer_backend vllm \\\n --temperature 0 \\\n --max_new_tokens 2048\n```\n\n### 客户端\n\n这里只介绍使用openai库进行调用。使用curl、swift客户端调用的例子可以参考[推理和部署文档](./Inference-and-deployment.md#部署)。\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key='EMPTY',\n base_url=f'http://127.0.0.1:8000/v1',\n)\nmodels = [model.id for model in client.models.list().data]\nprint(f'models: {models}')\n\nquery = 'who are you?'\nmessages = [{'role': 'user', 'content': query}]\n\nresp = client.chat.completions.create(model=models[1], messages=messages, max_tokens=512, temperature=0)\nquery = messages[0]['content']\nresponse = resp.choices[0].message.content\nprint(f'query: {query}')\nprint(f'response: {response}')\n\ngen = client.chat.completions.create(model=models[2], messages=messages, stream=True, temperature=0)\nprint(f'query: {query}\\nresponse: ', end='')\nfor chunk in gen:\n if chunk is None:\n continue\n print(chunk.choices[0].delta.content, end='', flush=True)\nprint()\n\"\"\"\nmodels: ['Qwen2.5-7B-Instruct', 'lora1', 'lora2']\nquery: who are you?\nresponse: I am an artificial intelligence model named swift-robot, developed by swift. I can answer your questions, provide information, and engage in conversation. If you have any inquiries or need assistance, feel free to ask me at any time.\nquery: who are you?\nresponse: I am an artificial intelligence model named Xiao Huang, developed by ModelScope. I can answer your questions, provide information, and engage in conversation. If you have any inquiries or need assistance, feel free to ask me at any time.\n\"\"\"\n```\n"
},
{
"path": "docs/source/Instruction/RLHF.md",
"content": "# 人类对齐\n\n本文档提供了各种人类偏好对齐算法的训练脚本。若您希望深入了解更详尽的算法信息及其选择方法，请参考[文档](https://github.com/modelscope/modelscope-classroom/blob/main/LLM-tutorial/M.%E4%BA%BA%E7%B1%BB%E5%81%8F%E5%A5%BD%E5%AF%B9%E9%BD%90%E8%AE%AD%E7%BB%83.md)\n\n\n## 数据集\nPPO与GRPO算法所需的数据仅为模型输入，也就是system prompt（可选）加上query。其中GRPO中的奖励函数可能需要额外的数据列，比如计算准确率需要`solution`列作为参考答案。\n\nRM和DPO类算法如ORPO，CPO，SimPO，则需要 $(x,y_w,y_l)$ 格式的数据，其中 $x$ 表示模型输入，$y_w,y_l$ 分别表示符合人类偏好的偏好回答和不符合人类偏好的拒绝回答,比如\n\n而KTO算法的数据比较特殊，只需要 $(x,y,\\text{label})$ 格式的数据，其中 $x$ 表示模型输入，$y$ 表示模型输出，label表示回答是否符合人类偏好\n比如\n\n使用自定义数据集对文本模型或者多模态大模型进行RLHF训练可以参考[自定义数据集文档](../Customization/Custom-dataset.md#rlhf)。\n\n## GRPO\n[论文arvix](https://arxiv.org/abs/2402.03300)\n\n训练脚本参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/grpo).\n\n## DPO\n[论文arvix](https://arxiv.org/abs/2305.18290)\n\n超参\n\n- beta：KL正则系数，值越大表示对偏离参考模型的惩罚越强。默认为0.1。\n- loss_type：不同DPO算法变种，可选值参考[文档](https://huggingface.co/docs/trl/main/en/dpo_trainer#loss-functions)，默认为 'sigmoid'。\n- (可选) loss_weights: 多种 loss 混合时的权重设置。\n- (可选) ld_alpha: 来自[LD-DPO 论文](https://arxiv.org/abs/2409.06411)，对超出公共前缀部分的logps加权 $\\alpha$ 抑制长度偏好。\n- (可选) discopop_tau: 来自 [DiscoPOP 论文](https://arxiv.org/abs/2406.08414)的温度参数 $\\tau$ ，用于缩放 log-ratio。默认值0.05。在 loss_type 为 discopop 时生效。\n\n建议在开始 DPO 训练前，先用偏好数据集中用户偏好答案部分进行一次 SFT 训练，以确保数据分布更符合 DPO 算法的要求。\n\n如需同时混合多个 loss（如用于 [MPO](https://arxiv.org/abs/2411.10442) 训练），可指定多个 loss_type，并通过 loss_weights 设置各自权重。\n\n通过设置超参数 `rpo_alpha`，可在 loss 中混合一定比例的 SFT loss，从而提升训练的稳定性。\n\n训练脚本参考\n\n- [DPO脚本](https://github.com/modelscope/ms-swift/tree/main/examples/train/rlhf/dpo).\n- [MPO脚本](https://github.com/modelscope/ms-swift/tree/main/examples/train/rlhf/mpo.sh).\n\n## RM\n[论文arvix](https://arxiv.org/abs/2203.02155)\n\nRLHF中的Reward Modeling阶段\n\n使用sft训练后的base model或者instruct model作为基底模型, 增加value head, 使用偏好数据集训练为reward model\n\n增加的value head权重会保存在`value_head.safetensors` 或 `value_head.bin`文件中\n\nRM损失函数如下\n\n$\n\\text{loss} = -\\log \\sigma \\left( r^{(c)} - r^{(r)} - m \\right) + \\lambda \\left( r^{(c)} + r^{(r)} \\right)^2\n$\n\n- $r^{(c)}$: 模型对 chosen response 的打分\n- $r^{(r)}$: 模型对 rejected response 的打分\n- $\\lambda$: L2正则项系数，鼓励模型输出接近0，使用参数`center_rewards_coefficient`进行设置，来自[论文](https://arxiv.org/pdf/2307.09288), 默认为0\n- $m$: margin项，鼓励模型根据不同难度的样本进行区分，需要数据集中提供`margin`列，默认为0，来自[论文](https://arxiv.org/pdf/2307.09288)\n\n\n训练脚本参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/rlhf/rm.sh).\n\n## PPO\n[论文arvix](https://arxiv.org/abs/2203.02155)\n\nRLHF中的PPO(proximal policy optimization)阶段, 涉及到四个模型\n- model: 训练模型, sft训练后的base model或者instruct model\n- ref_model: 参考模型, 默认为 model\n- reward_model: 奖励模型, 由RM阶段训练得到\n- value_model: 价值模型, 由reward_model初始化, 在训练中同步更新\n\n超参\n\n- local_rollout_forward_batch_size: 每次数据采样的批量大小, 默认为64\n- whiten_rewards: 对奖励进行归一化处理, 默认为False\n- kl_coef: KL散度项的系数, 默认为0.05\n- cliprange: PPO策略损失函数中的clip范围, 默认为0.2\n- vf_coef: 价值损失函数系数, 默认为0.1\n- cliprange_value: PPO价值损失函数中的clip范围, 默认为0.2\n- gamma: 累计奖励的折扣因子, 默认为1.0\n- lam: [GAE](https://arxiv.org/abs/1506.02438)中的lambda系数, 默认为0.95\n- num_sample_generations: 训练过程中生成的调试样本数量, 默认为10\n\n\n注意: 训练base model时, 需要先sft后再进行rlhf, 指定chat template, `tuner_type` 建议使用full\n\n训练中的指标解释参考[文档](https://huggingface.co/docs/trl/ppov2_trainer#explanation-of-the-logged-metrics)\n\n\n## KTO\n[论文arvix](https://arxiv.org/abs/2402.01306)\n\n超参\n\n- beta： KL正则系数，值越大表示对偏离参考模型的惩罚越大。默认为0.1\n- desirable_weight ：损失函数中的$\\lambda_D$项，偏好回答样本的损失权重, 默认为1.0\n- undesirable_weight ：损失函数中的$\\lambda_U$项，拒绝回答样本的损失权重，默认为1.0\n\n用 $n_D$ 和 $n_U$ 分别表示数据集中偏好回答和拒绝回答的样本数量，对于超参 $\\lambda_D$ 和 $\\lambda_U$ ，作者推荐设置 $\\frac{\\lambda_Dn_D}{\\lambda_Un_U}\\in[1,\\frac{4}{3}]$\n\n训练脚本\n使用 $(x,y,\\text{label})$ 格式数据训练\n\n训练脚本参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/rlhf/kto.sh).\n\n## CPO\n[论文arvix](https://arxiv.org/abs/2401.08417)\n超参\n\n- beta：隐含奖励前的系数，默认为0.1\n- cpo_alpha: nll loss系数, 默认为1.0\n\n训练脚本参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/rlhf/cpo.sh).\n\n## ORPO\n[论文arvix](https://arxiv.org/abs/2403.07691)\n\n超参\n\n- lambda: Odds Ratio loss系数\n\n注意：ORPO使用参数`--beta`传入超参`lambda`\n\n训练脚本参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/rlhf/orpo.sh).\n\n## SimPO\n[论文arvix](https://arxiv.org/abs/2405.14734)\n超参\n\n- beta：隐含奖励前的系数，默认为2.0\n- simpo_gamma：reward margin项，默认为1.0\n- cpo_alpha: 混合CPO nll loss提高训练稳定性, 默认为1.0, 设置0.0使用原始SimPO算法\n\n训练脚本参考[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/rlhf/simpo.sh).\n"
},
{
"path": "docs/source/Instruction/Ray.md",
"content": "# ray的支持\n\nSWIFT已经支持使用ray来进行多卡或多节点训练。已有功能中对ray的支持情况如下：\n\n| 功能 | 支持ray | 例子 | 可分配角色 |\n|----------|-------|--------------------------------------------------------------------------------|-----------------|\n| pt/sft | ✅ | https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-node/ray | default |\n| dpo | ❎ | | |\n| grpo | ❎ | | |\n| ppo | ❎ | | |\n| megatron | ❎ | | |\n| sampling | ✅ | https://github.com/modelscope/ms-swift/tree/main/examples/sampler/distill | sampler/prm/orm |\n| distill | ✅ | https://github.com/modelscope/ms-swift/tree/main/examples/sampler/sample | sampler/prm/orm |\n\n## 技术细节\n\n在叙述参数设置之前，我们有必要先行讲一下技术细节。由于SWIFT的内部当前使用了大量transformers和trl的已有实现，像veRL或ROLL一样拆解为不同的ray角色是不现实的，而且拆解后会以ray为中心，对非ray的场景的支持会不良。\n因此SWIFT采取了装饰器为主的技术方案，以函数级别定义了不同角色，这些角色可以在参数中被定义如何使用。看下面的例子：\n\n```python\nfrom swift.ray import RayHelper\n\n@RayHelper.worker(group=['model1', 'model2'])\nclass MyTrainer:\n\n def __init__(self, args):\n self._prepare_model1()\n self._prepare_model2()\n self._prepare_datasets()\n\n @RayHelper.function(group='model1')\n def _prepare_model1(self):\n ...\n\n @RayHelper.function(group='model2')\n def _prepare_model2(self):\n ...\n\n @RayHelper.function(group='model1')\n def rollout(self, inputs):\n return self.model1.generate(inputs)\n\n @RayHelper.function(group='model2')\n def forward_model2(self, inputs):\n loss = self.model2.forward(inputs)\n loss.backward()\n\n def _prepare_datasets(self):\n self.dataset = ...\n\n def train(self):\n for batch in DataLoader(self.dataset):\n generated = self.rollout(batch)\n self.forward_model2(generated)\n ...\n\n\nif __name__ == '__main__':\n ...\n MyTrainer(args).train()\n```\n\nRayHelper会将被装饰的方法分配到不同的硬件集群中，本地调用会被平滑转换到ray集群中进行远程调用。也可以以类为中心进行划分：\n\n```python\n\n@RayHelper.worker(group=['model1'])\nclass Model1:\n ...\n\n @RayHelper.function(group='model1')\n def rollout(self):\n ...\n\n@RayHelper.worker(group=['model2'])\nclass Model2:\n ...\n\n @RayHelper.function(group='model2')\n def forward_and_optimize(self):\n ...\n\n\nclass Trainer:\n ...\n```\n\nSWIFT对ray的支持本质上是使用@worker和@function两个注解的组合使用，worker指定ray集群的角色，function指定如何分配数据。\n\nfunction注解有额外的几个参数：\n```python\n @staticmethod\n def function(group: str,\n dispatch: Union[Literal['slice', 'all'], Callable] = 'all',\n execute: Literal['first', 'all'] = 'all',\n collect: Union[Literal['none', 'flatten'], Callable] = 'none'):\n```\n\n- dispatch: 如何分配调用入参\n - slice：对入参切分，也就是worker负载均衡执行\n - all：各个worker入参完全相同\n - 自定义切分方式，格式为：\n ```python\n def my_custom_slice(n, i, data):\n # n是worker数量，i是当前worker索引，data是原始入参\n # 返回第i个的入参\n ```\n- execute: 如何执行\n - first: rank0执行，此时slice和Callable方式切分无效\n - all: 全部执行\n\n- collect: 如何收集返回数据\n - none：原样返回，格式为各个worker返回值的列表\n - flatten: 将worker返回的结果进行拉平，支持tuple的拉平\n - Callable: 自定义collect方式，格式为：\n ```python\n def my_custom_collect(result):\n # result是各个worker返回的列表\n # 输入你想要的格式\n ```\n\n## 参数设置\n\n讲完技术细节后，可以将参数配置了。开发者可以根据不同的流程中的角色列表，设置不同的硬件搭配方式，例如采样功能中，共有三个角色，sampler、prm、orm，可以这样配置：\n\n```yaml\ndevice_groups:\n nproc_per_node: 4\n sample_group:\n device: GPU\n ranks: list(range(0, 2))\n workers:\n - sampler\n rm_group:\n device: GPU\n ranks: list(range(2, 4))\n workers:\n - prm\n - orm\n```\n\n- nproc_per_node: ray集群中需要的每个node的最小卡数。\nxxx_group: 每个ray组的名称，可以随意指定\n - device: 设备类型，当前支持GPU/CPU等。\n - ranks: 当前组分配到哪些ranks上。如果是CPU，ranks只能为整数，代表共需要多少进程，如果是GPU，可以为`[0,1,2,3]`, `4`, `list(range(0, 4))`等格式。\n - workers: 哪些角色分配到当前组中。\n\n所有可用的角色可以见本文最上面的表。\n\n如果使用命令行，device_groups也可以以`--device_groups xxx`方式传入，xxx为jsonstring。为了配置的简便，我们强烈推荐使用yaml方式搭配ray使用。\n"
},
{
"path": "docs/source/Instruction/Reinforced-Fine-tuning.md",
"content": "# 强化微调\n\n强化微调是目前模型训练非常重要的功能之一，它本身的实现是多种多样的，SWIFT目前已经支持了强化微调所需要的原子能力，如采样、强化学习和微调。目前我们提供了拒绝采样微调的一个具体示例，可以查看[这里](https://github.com/modelscope/ms-swift/tree/main/examples/train/rft/rft.py)。\n\n## 强化微调的概念\n\n强化微调是从2022年开始（甚至更早）就被提出的概念。其方式一般有下列流程：\n\n1. 使用某个模型生成数据，或进行原始数据扩充\n2. 使用数据训练目标模型\n3. 如果有必要，重复上述过程\n\n步骤1：\n\n- 如果生成数据的模型是更大的模型，如GPT、Qwen-Max、DeepSeek-V3/R1等，则该强化微调可以理解为蒸馏\n- 如果生成数据的模型是本模型，则可以理解为自我提升（self-improvement）微调\n- 如果采样过程是采样一个batch，然后通过KL散度和reward进行拟合训练并不断循环，则可以理解为PPO、GRPO等on-policy算法\n- 采样数据的算法包含蒙特卡洛采样、do_sample采样、group beam search、dvts等\n- 采样过程可以引入ORM（结果判断），PRM（过程打分），多样性过滤，语种过滤等\n\n步骤2：\n\n- 如果使用SFT，则称为拒绝采样微调\n- 如果是强化学习，则称为强化学习微调\n\n步骤3：\n\n- 如果使用更大的模型蒸馏，例如更大模型的蒙特卡洛采样蒸馏，一般不会有循环\n- 如果使用本模型进行采样，或者PPO等算法，则会有循环\n\n泛泛来说，常见强化微调的方式有下面几种：\n\n1. 蒸馏：使用蒙特卡洛、do_sample等方式从超大模型中采样大量优质数据，训练小模型\n2. 自我提升：从本模型中采样部分优质数据，筛选后训练本模型，循环执行\n3. on-policy RL：使用PPO、GRPO等方式循环训练\n\n采样过程一般很漫长，比训练过程漫长的多。如果使用GPT等模型蒸馏数据，则需要购买token。因此，强化微调的时间成本和花费成本比较高，所以一般作为微调的补充机制出现，当然也有特例，例如最近的DeepSeek-R1。\n\nDeepSeek-R1使用了GRPO算法从零使base模型涌现CoT能力，该方法需要大规模集群支持，且模型需要足够大才能发生能力涌现，在本文中不详细讨论。如果需要了解该过程，请查看[论文解析](https://zhuanlan.zhihu.com/p/19714987272)。\n\n有关强化微调的一些论文：\n\n- 拒绝采样微调：https://arxiv.org/pdf/2308.01825\n- ReST：https://arxiv.org/pdf/2308.08998\n- B-STAR：https://arxiv.org/pdf/2412.17256\n- DeepSeekMath：https://arxiv.org/pdf/2402.03300\n- Qwen-math-PRM：https://arxiv.org/pdf/2501.07301\n- DeepSeek-R1：https://github.com/deepseek-ai/DeepSeek-R1/tree/main\n\n## 什么时候使用强化微调\n\n在LLaMA3之后，我们发现一个非常明显但却是不常被提及的特点：使用某个含有CoT的train数据集训练Instruct模型，再通过对应的test集进行评测，会发现test集评测效果变差。例如，使用gsm8k训练集训练llama3.1-8b-instruct，对生成的ckpt使用test集进行评测，会发现掉点。\n\n这个特性主要来源于模型的知识遗忘问题。在模型厂商的微调中，会加入非常多的CoT数据集，模型在解决数学任务的时候，用到的能力很有可能不是来自于math数据集，而是来自arc数据集，这个推论有[一些工作可以证明](https://zhuanlan.zhihu.com/p/19269451950)。在继续训练通用任务后，知识遗忘破坏了模型原有能力，导致了掉点。\n\n然而，优先使用微调方式训练模型总是正确的。微调可以使模型快速适应数据集的分布，并且微调的成本很低。当有如下条件之一时使用强化微调：\n\n1. 已经微调过模型，能力不满足需求\n2. 需要更强的CoT能力\n3. 对基模型训练通用能力，而原始数据集已经导致模型效果无法提升\n4. 对应query的输出结果可以相对准确地评估好坏，例如结果清晰（数学，代码），过程清晰（翻译，风格）等\n\n强化微调非常依赖于reward评估是否准确。如果评估结果不准确，可能导致模型训练原地震荡，甚至越训越差。\n\n## SWIFT的实现\n\nSWIFT支持sample命令，该命令就是用于模型采样。目前支持的采样方式有：\n\n- sample：以generate方式对模型进行采样\n\n目前我们给出了一个较为通用的[RFT脚本](https://github.com/modelscope/ms-swift/tree/main/examples/train/rft/rft.py)。该脚本适用于自我提升方式的训练，且支持动态调整采样温度值、PRM阈值等超参数，并且训练方式灵活可变（微调、DPO等；或者每次迭代重新训练原模型或继续训练上个迭代的模型，甚至加载上个迭代的所有训练状态等）。开发者可以在该脚本中增加其他数据过滤（生成的数据集中，id相同的行来自同一个query），例如多样性判断、语种判断等。\n\n## 实验结果\n\n我们对该RFT脚本针对数学领域使用competition_math数据集进行了训练和评测，结果如下：\n\n| 模型 | MATH指标 | 训练方式 | 迭代次数 | 训练后MATH指标 |\n| ------------------------ | -------- | -------- | -------- | --------------------- |\n| LLaMA3.1_8b | 12.0 | SFT | 3 | 25.2(LLaMA3.1_8b_sft) |\n| LLaMA3.1_8b_sft | 25.2 | RFT | 2 | 32.4 |\n| LLaMA3.1_8b_instruct | 52.2 | SFT | 2 | 39.0 |\n| LLaMA3.1_8b_instruct | 52.2 | RFT | 3 | 58 |\n| Qwen2.5_math_7b_instruct | 79.6 | RFT | 2 | 83.2 |\n\n可以看到，使用competition_math直接SFT后，instruct模型的掉点十分严重。而RFT后模型能力有提升，即使对Qwen2.5_math_7b_instruct这个SOTA的math模型也同样有一定提升空间。\n\n特别地，针对Qwen2.5_math_7b_instruct我们测试了gsm8k的指标：\n\n| 模型 | gsm8k指标 | RFT后gsm8k指标 |\n| ------------------------ | --------- | -------------- |\n| Qwen2.5_math_7b_instruct | 92.8 | 91.6 |\n\n可以看到，RFT训练后gsm8k指标变化不大，并没有出现前述的掉点现象。\n"
},
{
"path": "docs/source/Instruction/Sample.md",
"content": "# 采样\n\n采样是SWIFT新支持的重要能力之一，这部分可以理解为`test-time compute`的落地实现。同时，该能力对RFT（强化微调）的实现也至关重要。\n\n## 能力介绍\n\nSWIFT的sample能力可以使用下面的例子进行：\n```shell\nswift sample --model LLM-Research/Meta-Llama-3.1-8B-Instruct --sampler_engine transformers --num_return_sequences 5 --dataset AI-ModelScope/alpaca-gpt4-data-zh#5\n```\n在当前文件夹的`sample_output`目录下，会生成以时间戳为文件名的jsonl文件，该文件应该包含25行，每一行都是一个完整`messages`格式的数据。\n\n采样的参数列表请参考[这里](Command-line-parameters.md)。\n\n## 环境准备\n\n```shell\npip install ms-swift[llm] -U\n```\n\n或从源代码安装：\n\n```shell\ngit clone https://github.com/modelscope/ms-swift.git\ncd ms-swift\npip install -e '.[llm]'\n```\n\n## 使用PRM和ORM进行结果过滤\n\n采样重要的能力就是对过程和结果进行监督，这可以通过设置额外参数来支持。\n\n```shell\nswift sample --model LLM-Research/Meta-Llama-3.1-8B-Instruct --sampler_engine lmdeploy --num_return_sequences 5 --n_best_to_keep 2 --dataset tastelikefeet/competition_math#5 --prm_model AI-ModelScope/GRM-llama3.2-3B-rewardmodel-ft --orm_model math\n```\n\n在当前文件夹的`sample_output`目录下，会生成以时间戳为文件名的jsonl文件，该文件**至多包含**10行，每一行都是一个完整`messages`格式的数据。\n> 之所以至多包含10行，是因为虽然设置了共处理5个数据，每个数据保留2个（n_best_to_keep），但是orm可能会校验失败，失败数据不会保留到文件中。\n> 另外，增加了--prm_model或--orm_model后文件格式有所不同，包含了rejected_response key，内容来自于prm评分最低的行。\n\n## 自定义PRM或ORM\n\nPRM和ORM的自定义可以在plugin中按照现有代码增加一个新的实现。例如：\n```python\nclass CustomPRM:\n\n # 构造需要是无参的\n def __init__(self):\n # init here\n pass\n\n def __call__(self, infer_requests: List[InferRequest], ground_truths: List[str], **kwargs) -> List[Union[float, List[float]]]:\n ...\n\n\nprms = {'custom': CustomPRM}\n```\n\n之后在命令行中使用`--prm_model custom`即可。\n\n## 显存控制\n\n如果被采样模型和PRM共同加载进显存，则可能出现OOM的问题。因此采样可以分为两段进行：\n\n- 第一段指定`--model`和``--sampler_engine`，同时不指定`--orm_model`和`--prm_model`，仅进行采样，并存储为文件\n- 第二段指定`--sampler_engine no`，指定`--orm_model`和`--prm_model`，并同时指定`--cache_files`，仅进行RM数据过滤，不重新采样\n\n通过两段方式可以每次仅加载一个模型，防止OOM。\n\n## 实际例子\n\n请参考[强化微调脚本](https://github.com/modelscope/ms-swift/tree/main/examples/train/rft/rft.py)。该脚本给出了使用采样进行强化微调的实际例子。\n\n> 注意：该脚本的实际效果和模型、数据、RM的质量强相关，因此仅作为样例出现，用户请自行修改该脚本并训练自己的RM和generator模型。\n\n## 大模型蒸馏采样\n\nSWIFT的sample支持使用OpenAI API的方式，用大模型蒸馏数据，如下示例：\n```shell\nOPENAI_API_KEY=\"your_api_key\" \\\nswift sample \\\n --sampler_type distill \\\n --sampler_engine client \\\n --model deepseek-r1 \\\n --stream true \\\n --dataset tastelikefeet/competition_math#5 \\\n --num_return_sequences 1 \\\n --temperature 0.6 \\\n --top_p 0.95 \\\n --engine_kwargs '{\"base_url\":\"https://dashscope.aliyuncs.com/compatible-mode/v1\"}'\n```\n在以上示例中，base_url和model分别是api地址和模型名称，stream表示发起请求的stream参数。\n\n注意，对于Deepseek-R1系列模型，输出会被格式化为：`{reasoning_content} \\n\\n{content} `。\n"
},
{
"path": "docs/source/Instruction/Supported-models-and-datasets.md",
"content": "# 支持的模型和数据集\n\n> [!WARNING]\n> 本文档适配swift4.0.0，若要查看swift3.x模型对应的model_type和template，请查看[v3.12文档](https://swift.readthedocs.io/zh-cn/v3.12/Instruction/Supported-models-and-datasets.html)\n\n## 模型\n下表介绍了ms-swift接入的模型的相关信息:\n- Model ID: ModelScope模型id\n- HF Model ID: HuggingFace模型id\n- Model Type: 模型类型\n- Default Template: 默认对话模板\n- Requires: 使用该模型的额外依赖（**如果训练时遇到transformers版本不匹配的问题，你可以查看模型`config.json`文件中的`transformers_version`字段**）\n- Support Megatron: 是否支持Megatron-SWIFT训练\n- Tags: 模型的tags\n\n\n### 大语言模型\n| Model ID | Model Type | Default Template | Requires | Support Megatron | Tags | HF Model ID |\n| -------- | -----------| ---------------- | -------- | ---------------- | ---- | ----------- |\n|[Qwen/Qwen-1_8B-Chat](https://modelscope.cn/models/Qwen/Qwen-1_8B-Chat)|qwen|qwen|-|✘|-|[Qwen/Qwen-1_8B-Chat](https://huggingface.co/Qwen/Qwen-1_8B-Chat)|\n|[Qwen/Qwen-7B-Chat](https://modelscope.cn/models/Qwen/Qwen-7B-Chat)|qwen|qwen|-|✘|-|[Qwen/Qwen-7B-Chat](https://huggingface.co/Qwen/Qwen-7B-Chat)|\n|[Qwen/Qwen-14B-Chat](https://modelscope.cn/models/Qwen/Qwen-14B-Chat)|qwen|qwen|-|✘|-|[Qwen/Qwen-14B-Chat](https://huggingface.co/Qwen/Qwen-14B-Chat)|\n|[Qwen/Qwen-72B-Chat](https://modelscope.cn/models/Qwen/Qwen-72B-Chat)|qwen|qwen|-|✘|-|[Qwen/Qwen-72B-Chat](https://huggingface.co/Qwen/Qwen-72B-Chat)|\n|[Qwen/Qwen-1_8B](https://modelscope.cn/models/Qwen/Qwen-1_8B)|qwen|qwen|-|✘|-|[Qwen/Qwen-1_8B](https://huggingface.co/Qwen/Qwen-1_8B)|\n|[Qwen/Qwen-7B](https://modelscope.cn/models/Qwen/Qwen-7B)|qwen|qwen|-|✘|-|[Qwen/Qwen-7B](https://huggingface.co/Qwen/Qwen-7B)|\n|[Qwen/Qwen-14B](https://modelscope.cn/models/Qwen/Qwen-14B)|qwen|qwen|-|✘|-|[Qwen/Qwen-14B](https://huggingface.co/Qwen/Qwen-14B)|\n|[Qwen/Qwen-72B](https://modelscope.cn/models/Qwen/Qwen-72B)|qwen|qwen|-|✘|-|[Qwen/Qwen-72B](https://huggingface.co/Qwen/Qwen-72B)|\n|[Qwen/Qwen-1_8B-Chat-Int4](https://modelscope.cn/models/Qwen/Qwen-1_8B-Chat-Int4)|qwen|qwen|-|✘|-|[Qwen/Qwen-1_8B-Chat-Int4](https://huggingface.co/Qwen/Qwen-1_8B-Chat-Int4)|\n|[Qwen/Qwen-7B-Chat-Int4](https://modelscope.cn/models/Qwen/Qwen-7B-Chat-Int4)|qwen|qwen|-|✘|-|[Qwen/Qwen-7B-Chat-Int4](https://huggingface.co/Qwen/Qwen-7B-Chat-Int4)|\n|[Qwen/Qwen-14B-Chat-Int4](https://modelscope.cn/models/Qwen/Qwen-14B-Chat-Int4)|qwen|qwen|-|✘|-|[Qwen/Qwen-14B-Chat-Int4](https://huggingface.co/Qwen/Qwen-14B-Chat-Int4)|\n|[Qwen/Qwen-72B-Chat-Int4](https://modelscope.cn/models/Qwen/Qwen-72B-Chat-Int4)|qwen|qwen|-|✘|-|[Qwen/Qwen-72B-Chat-Int4](https://huggingface.co/Qwen/Qwen-72B-Chat-Int4)|\n|[Qwen/Qwen-1_8B-Chat-Int8](https://modelscope.cn/models/Qwen/Qwen-1_8B-Chat-Int8)|qwen|qwen|-|✘|-|[Qwen/Qwen-1_8B-Chat-Int8](https://huggingface.co/Qwen/Qwen-1_8B-Chat-Int8)|\n|[Qwen/Qwen-7B-Chat-Int8](https://modelscope.cn/models/Qwen/Qwen-7B-Chat-Int8)|qwen|qwen|-|✘|-|[Qwen/Qwen-7B-Chat-Int8](https://huggingface.co/Qwen/Qwen-7B-Chat-Int8)|\n|[Qwen/Qwen-14B-Chat-Int8](https://modelscope.cn/models/Qwen/Qwen-14B-Chat-Int8)|qwen|qwen|-|✘|-|[Qwen/Qwen-14B-Chat-Int8](https://huggingface.co/Qwen/Qwen-14B-Chat-Int8)|\n|[Qwen/Qwen-72B-Chat-Int8](https://modelscope.cn/models/Qwen/Qwen-72B-Chat-Int8)|qwen|qwen|-|✘|-|[Qwen/Qwen-72B-Chat-Int8](https://huggingface.co/Qwen/Qwen-72B-Chat-Int8)|\n|[TongyiFinance/Tongyi-Finance-14B-Chat](https://modelscope.cn/models/TongyiFinance/Tongyi-Finance-14B-Chat)|qwen|qwen|-|✘|financial|[jxy/Tongyi-Finance-14B-Chat](https://huggingface.co/jxy/Tongyi-Finance-14B-Chat)|\n|[TongyiFinance/Tongyi-Finance-14B](https://modelscope.cn/models/TongyiFinance/Tongyi-Finance-14B)|qwen|qwen|-|✘|financial|-|\n|[TongyiFinance/Tongyi-Finance-14B-Chat-Int4](https://modelscope.cn/models/TongyiFinance/Tongyi-Finance-14B-Chat-Int4)|qwen|qwen|-|✘|financial|[jxy/Tongyi-Finance-14B-Chat-Int4](https://huggingface.co/jxy/Tongyi-Finance-14B-Chat-Int4)|\n|[Qwen/Qwen1.5-0.5B-Chat](https://modelscope.cn/models/Qwen/Qwen1.5-0.5B-Chat)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-0.5B-Chat](https://huggingface.co/Qwen/Qwen1.5-0.5B-Chat)|\n|[Qwen/Qwen1.5-1.8B-Chat](https://modelscope.cn/models/Qwen/Qwen1.5-1.8B-Chat)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-1.8B-Chat](https://huggingface.co/Qwen/Qwen1.5-1.8B-Chat)|\n|[Qwen/Qwen1.5-4B-Chat](https://modelscope.cn/models/Qwen/Qwen1.5-4B-Chat)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-4B-Chat](https://huggingface.co/Qwen/Qwen1.5-4B-Chat)|\n|[Qwen/Qwen1.5-7B-Chat](https://modelscope.cn/models/Qwen/Qwen1.5-7B-Chat)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-7B-Chat](https://huggingface.co/Qwen/Qwen1.5-7B-Chat)|\n|[Qwen/Qwen1.5-14B-Chat](https://modelscope.cn/models/Qwen/Qwen1.5-14B-Chat)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-14B-Chat](https://huggingface.co/Qwen/Qwen1.5-14B-Chat)|\n|[Qwen/Qwen1.5-32B-Chat](https://modelscope.cn/models/Qwen/Qwen1.5-32B-Chat)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-32B-Chat](https://huggingface.co/Qwen/Qwen1.5-32B-Chat)|\n|[Qwen/Qwen1.5-72B-Chat](https://modelscope.cn/models/Qwen/Qwen1.5-72B-Chat)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-72B-Chat](https://huggingface.co/Qwen/Qwen1.5-72B-Chat)|\n|[Qwen/Qwen1.5-110B-Chat](https://modelscope.cn/models/Qwen/Qwen1.5-110B-Chat)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-110B-Chat](https://huggingface.co/Qwen/Qwen1.5-110B-Chat)|\n|[Qwen/Qwen1.5-0.5B](https://modelscope.cn/models/Qwen/Qwen1.5-0.5B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-0.5B](https://huggingface.co/Qwen/Qwen1.5-0.5B)|\n|[Qwen/Qwen1.5-1.8B](https://modelscope.cn/models/Qwen/Qwen1.5-1.8B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-1.8B](https://huggingface.co/Qwen/Qwen1.5-1.8B)|\n|[Qwen/Qwen1.5-4B](https://modelscope.cn/models/Qwen/Qwen1.5-4B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-4B](https://huggingface.co/Qwen/Qwen1.5-4B)|\n|[Qwen/Qwen1.5-7B](https://modelscope.cn/models/Qwen/Qwen1.5-7B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-7B](https://huggingface.co/Qwen/Qwen1.5-7B)|\n|[Qwen/Qwen1.5-14B](https://modelscope.cn/models/Qwen/Qwen1.5-14B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-14B](https://huggingface.co/Qwen/Qwen1.5-14B)|\n|[Qwen/Qwen1.5-32B](https://modelscope.cn/models/Qwen/Qwen1.5-32B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-32B](https://huggingface.co/Qwen/Qwen1.5-32B)|\n|[Qwen/Qwen1.5-72B](https://modelscope.cn/models/Qwen/Qwen1.5-72B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-72B](https://huggingface.co/Qwen/Qwen1.5-72B)|\n|[Qwen/Qwen1.5-110B](https://modelscope.cn/models/Qwen/Qwen1.5-110B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen1.5-110B](https://huggingface.co/Qwen/Qwen1.5-110B)|\n|[Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4](https://huggingface.co/Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4)|\n|[Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4](https://huggingface.co/Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4)|\n|[Qwen/Qwen1.5-4B-Chat-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen1.5-4B-Chat-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-4B-Chat-GPTQ-Int4](https://huggingface.co/Qwen/Qwen1.5-4B-Chat-GPTQ-Int4)|\n|[Qwen/Qwen1.5-7B-Chat-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen1.5-7B-Chat-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-7B-Chat-GPTQ-Int4](https://huggingface.co/Qwen/Qwen1.5-7B-Chat-GPTQ-Int4)|\n|[Qwen/Qwen1.5-14B-Chat-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen1.5-14B-Chat-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-14B-Chat-GPTQ-Int4](https://huggingface.co/Qwen/Qwen1.5-14B-Chat-GPTQ-Int4)|\n|[Qwen/Qwen1.5-32B-Chat-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen1.5-32B-Chat-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-32B-Chat-GPTQ-Int4](https://huggingface.co/Qwen/Qwen1.5-32B-Chat-GPTQ-Int4)|\n|[Qwen/Qwen1.5-72B-Chat-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen1.5-72B-Chat-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-72B-Chat-GPTQ-Int4](https://huggingface.co/Qwen/Qwen1.5-72B-Chat-GPTQ-Int4)|\n|[Qwen/Qwen1.5-110B-Chat-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen1.5-110B-Chat-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-110B-Chat-GPTQ-Int4](https://huggingface.co/Qwen/Qwen1.5-110B-Chat-GPTQ-Int4)|\n|[Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8](https://huggingface.co/Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8)|\n|[Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8](https://huggingface.co/Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8)|\n|[Qwen/Qwen1.5-4B-Chat-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen1.5-4B-Chat-GPTQ-Int8)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-4B-Chat-GPTQ-Int8](https://huggingface.co/Qwen/Qwen1.5-4B-Chat-GPTQ-Int8)|\n|[Qwen/Qwen1.5-7B-Chat-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen1.5-7B-Chat-GPTQ-Int8)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-7B-Chat-GPTQ-Int8](https://huggingface.co/Qwen/Qwen1.5-7B-Chat-GPTQ-Int8)|\n|[Qwen/Qwen1.5-14B-Chat-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen1.5-14B-Chat-GPTQ-Int8)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-14B-Chat-GPTQ-Int8](https://huggingface.co/Qwen/Qwen1.5-14B-Chat-GPTQ-Int8)|\n|[Qwen/Qwen1.5-72B-Chat-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen1.5-72B-Chat-GPTQ-Int8)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-72B-Chat-GPTQ-Int8](https://huggingface.co/Qwen/Qwen1.5-72B-Chat-GPTQ-Int8)|\n|[Qwen/Qwen1.5-0.5B-Chat-AWQ](https://modelscope.cn/models/Qwen/Qwen1.5-0.5B-Chat-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-0.5B-Chat-AWQ](https://huggingface.co/Qwen/Qwen1.5-0.5B-Chat-AWQ)|\n|[Qwen/Qwen1.5-1.8B-Chat-AWQ](https://modelscope.cn/models/Qwen/Qwen1.5-1.8B-Chat-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-1.8B-Chat-AWQ](https://huggingface.co/Qwen/Qwen1.5-1.8B-Chat-AWQ)|\n|[Qwen/Qwen1.5-4B-Chat-AWQ](https://modelscope.cn/models/Qwen/Qwen1.5-4B-Chat-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-4B-Chat-AWQ](https://huggingface.co/Qwen/Qwen1.5-4B-Chat-AWQ)|\n|[Qwen/Qwen1.5-7B-Chat-AWQ](https://modelscope.cn/models/Qwen/Qwen1.5-7B-Chat-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-7B-Chat-AWQ](https://huggingface.co/Qwen/Qwen1.5-7B-Chat-AWQ)|\n|[Qwen/Qwen1.5-14B-Chat-AWQ](https://modelscope.cn/models/Qwen/Qwen1.5-14B-Chat-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-14B-Chat-AWQ](https://huggingface.co/Qwen/Qwen1.5-14B-Chat-AWQ)|\n|[Qwen/Qwen1.5-32B-Chat-AWQ](https://modelscope.cn/models/Qwen/Qwen1.5-32B-Chat-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-32B-Chat-AWQ](https://huggingface.co/Qwen/Qwen1.5-32B-Chat-AWQ)|\n|[Qwen/Qwen1.5-72B-Chat-AWQ](https://modelscope.cn/models/Qwen/Qwen1.5-72B-Chat-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-72B-Chat-AWQ](https://huggingface.co/Qwen/Qwen1.5-72B-Chat-AWQ)|\n|[Qwen/Qwen1.5-110B-Chat-AWQ](https://modelscope.cn/models/Qwen/Qwen1.5-110B-Chat-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen1.5-110B-Chat-AWQ](https://huggingface.co/Qwen/Qwen1.5-110B-Chat-AWQ)|\n|[Qwen/CodeQwen1.5-7B](https://modelscope.cn/models/Qwen/CodeQwen1.5-7B)|qwen2|qwen|transformers>=4.37|✔|coding|[Qwen/CodeQwen1.5-7B](https://huggingface.co/Qwen/CodeQwen1.5-7B)|\n|[Qwen/CodeQwen1.5-7B-Chat](https://modelscope.cn/models/Qwen/CodeQwen1.5-7B-Chat)|qwen2|qwen|transformers>=4.37|✔|coding|[Qwen/CodeQwen1.5-7B-Chat](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat)|\n|[Qwen/CodeQwen1.5-7B-Chat-AWQ](https://modelscope.cn/models/Qwen/CodeQwen1.5-7B-Chat-AWQ)|qwen2|qwen|transformers>=4.37|✘|coding|[Qwen/CodeQwen1.5-7B-Chat-AWQ](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat-AWQ)|\n|[Qwen/Qwen2-0.5B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-0.5B-Instruct)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen2-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2-0.5B-Instruct)|\n|[Qwen/Qwen2-1.5B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-1.5B-Instruct)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen2-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2-1.5B-Instruct)|\n|[Qwen/Qwen2-7B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-7B-Instruct)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen2-7B-Instruct](https://huggingface.co/Qwen/Qwen2-7B-Instruct)|\n|[Qwen/Qwen2-72B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-72B-Instruct)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen2-72B-Instruct](https://huggingface.co/Qwen/Qwen2-72B-Instruct)|\n|[Qwen/Qwen2-0.5B](https://modelscope.cn/models/Qwen/Qwen2-0.5B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen2-0.5B](https://huggingface.co/Qwen/Qwen2-0.5B)|\n|[Qwen/Qwen2-1.5B](https://modelscope.cn/models/Qwen/Qwen2-1.5B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen2-1.5B](https://huggingface.co/Qwen/Qwen2-1.5B)|\n|[Qwen/Qwen2-7B](https://modelscope.cn/models/Qwen/Qwen2-7B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen2-7B](https://huggingface.co/Qwen/Qwen2-7B)|\n|[Qwen/Qwen2-72B](https://modelscope.cn/models/Qwen/Qwen2-72B)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen2-72B](https://huggingface.co/Qwen/Qwen2-72B)|\n|[Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2-7B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2-7B-Instruct-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-7B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2-7B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2-72B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2-72B-Instruct-GPTQ-Int4)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-72B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2-72B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2-7B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2-7B-Instruct-GPTQ-Int8)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-7B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2-7B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2-72B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2-72B-Instruct-GPTQ-Int8)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-72B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2-72B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2-0.5B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2-0.5B-Instruct-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-0.5B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2-0.5B-Instruct-AWQ)|\n|[Qwen/Qwen2-1.5B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2-1.5B-Instruct-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-1.5B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2-1.5B-Instruct-AWQ)|\n|[Qwen/Qwen2-7B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2-7B-Instruct-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-7B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2-7B-Instruct-AWQ)|\n|[Qwen/Qwen2-72B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2-72B-Instruct-AWQ)|qwen2|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-72B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2-72B-Instruct-AWQ)|\n|[Qwen/Qwen2-Math-1.5B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-Math-1.5B-Instruct)|qwen2|qwen|transformers>=4.37|✔|math|[Qwen/Qwen2-Math-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2-Math-1.5B-Instruct)|\n|[Qwen/Qwen2-Math-7B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-Math-7B-Instruct)|qwen2|qwen|transformers>=4.37|✔|math|[Qwen/Qwen2-Math-7B-Instruct](https://huggingface.co/Qwen/Qwen2-Math-7B-Instruct)|\n|[Qwen/Qwen2-Math-72B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-Math-72B-Instruct)|qwen2|qwen|transformers>=4.37|✔|math|[Qwen/Qwen2-Math-72B-Instruct](https://huggingface.co/Qwen/Qwen2-Math-72B-Instruct)|\n|[Qwen/Qwen2-Math-1.5B](https://modelscope.cn/models/Qwen/Qwen2-Math-1.5B)|qwen2|qwen|transformers>=4.37|✔|math|[Qwen/Qwen2-Math-1.5B](https://huggingface.co/Qwen/Qwen2-Math-1.5B)|\n|[Qwen/Qwen2-Math-7B](https://modelscope.cn/models/Qwen/Qwen2-Math-7B)|qwen2|qwen|transformers>=4.37|✔|math|[Qwen/Qwen2-Math-7B](https://huggingface.co/Qwen/Qwen2-Math-7B)|\n|[Qwen/Qwen2-Math-72B](https://modelscope.cn/models/Qwen/Qwen2-Math-72B)|qwen2|qwen|transformers>=4.37|✔|math|[Qwen/Qwen2-Math-72B](https://huggingface.co/Qwen/Qwen2-Math-72B)|\n|[Qwen/Qwen2.5-7B-Instruct-1M](https://modelscope.cn/models/Qwen/Qwen2.5-7B-Instruct-1M)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen2.5-7B-Instruct-1M](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-1M)|\n|[Qwen/Qwen2.5-14B-Instruct-1M](https://modelscope.cn/models/Qwen/Qwen2.5-14B-Instruct-1M)|qwen2|qwen|transformers>=4.37|✔|-|[Qwen/Qwen2.5-14B-Instruct-1M](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-1M)|\n|[PowerInfer/SmallThinker-3B-Preview](https://modelscope.cn/models/PowerInfer/SmallThinker-3B-Preview)|qwen2|qwen|transformers>=4.37|✔|-|[PowerInfer/SmallThinker-3B-Preview](https://huggingface.co/PowerInfer/SmallThinker-3B-Preview)|\n|[Qwen/Qwen2.5-0.5B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-0.5B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct)|\n|[Qwen/Qwen2.5-1.5B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-1.5B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct)|\n|[Qwen/Qwen2.5-3B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-3B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-3B-Instruct)|\n|[Qwen/Qwen2.5-7B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-7B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct)|\n|[Qwen/Qwen2.5-14B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-14B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct)|\n|[Qwen/Qwen2.5-32B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-32B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct)|\n|[Qwen/Qwen2.5-72B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-72B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct)|\n|[Qwen/Qwen2.5-0.5B](https://modelscope.cn/models/Qwen/Qwen2.5-0.5B)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-0.5B](https://huggingface.co/Qwen/Qwen2.5-0.5B)|\n|[Qwen/Qwen2.5-1.5B](https://modelscope.cn/models/Qwen/Qwen2.5-1.5B)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-1.5B](https://huggingface.co/Qwen/Qwen2.5-1.5B)|\n|[Qwen/Qwen2.5-3B](https://modelscope.cn/models/Qwen/Qwen2.5-3B)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-3B](https://huggingface.co/Qwen/Qwen2.5-3B)|\n|[Qwen/Qwen2.5-7B](https://modelscope.cn/models/Qwen/Qwen2.5-7B)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-7B](https://huggingface.co/Qwen/Qwen2.5-7B)|\n|[Qwen/Qwen2.5-14B](https://modelscope.cn/models/Qwen/Qwen2.5-14B)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-14B](https://huggingface.co/Qwen/Qwen2.5-14B)|\n|[Qwen/Qwen2.5-32B](https://modelscope.cn/models/Qwen/Qwen2.5-32B)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-32B](https://huggingface.co/Qwen/Qwen2.5-32B)|\n|[Qwen/Qwen2.5-72B](https://modelscope.cn/models/Qwen/Qwen2.5-72B)|qwen2|qwen2_5|transformers>=4.37|✔|-|[Qwen/Qwen2.5-72B](https://huggingface.co/Qwen/Qwen2.5-72B)|\n|[Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-0.5B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-0.5B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-0.5B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-1.5B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-1.5B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-1.5B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-3B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-3B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-3B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-7B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-7B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-7B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-14B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-14B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-14B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-32B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-32B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-32B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-72B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-72B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|-|[Qwen/Qwen2.5-72B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-Coder-0.5B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-0.5B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct)|\n|[Qwen/Qwen2.5-Coder-1.5B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-1.5B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct)|\n|[Qwen/Qwen2.5-Coder-3B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-3B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct)|\n|[Qwen/Qwen2.5-Coder-7B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct)|\n|[Qwen/Qwen2.5-Coder-14B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-14B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct)|\n|[Qwen/Qwen2.5-Coder-32B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-32B-Instruct)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct)|\n|[Qwen/Qwen2.5-Coder-0.5B](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-0.5B)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-0.5B](https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B)|\n|[Qwen/Qwen2.5-Coder-1.5B](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-1.5B)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-1.5B](https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B)|\n|[Qwen/Qwen2.5-Coder-3B](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-3B)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-3B](https://huggingface.co/Qwen/Qwen2.5-Coder-3B)|\n|[Qwen/Qwen2.5-Coder-7B](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-7B](https://huggingface.co/Qwen/Qwen2.5-Coder-7B)|\n|[Qwen/Qwen2.5-Coder-14B](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-14B)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-14B](https://huggingface.co/Qwen/Qwen2.5-Coder-14B)|\n|[Qwen/Qwen2.5-Coder-32B](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-32B)|qwen2|qwen2_5|transformers>=4.37|✔|coding|[Qwen/Qwen2.5-Coder-32B](https://huggingface.co/Qwen/Qwen2.5-Coder-32B)|\n|[Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-Coder-3B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-3B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-3B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-Coder-7B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-7B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-Coder-14B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-14B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-14B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-Coder-32B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-32B-Instruct-AWQ)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-32B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8)|qwen2|qwen2_5|transformers>=4.37|✘|coding|[Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8)|\n|[moonshotai/Kimi-Dev-72B](https://modelscope.cn/models/moonshotai/Kimi-Dev-72B)|qwen2|qwen2_5|transformers>=4.37|✔|-|[moonshotai/Kimi-Dev-72B](https://huggingface.co/moonshotai/Kimi-Dev-72B)|\n|[Qwen/Qwen2.5-Math-1.5B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-Math-1.5B-Instruct)|qwen2|qwen2_5_math|transformers>=4.37|✔|math|[Qwen/Qwen2.5-Math-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-1.5B-Instruct)|\n|[Qwen/Qwen2.5-Math-7B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-Math-7B-Instruct)|qwen2|qwen2_5_math|transformers>=4.37|✔|math|[Qwen/Qwen2.5-Math-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-7B-Instruct)|\n|[Qwen/Qwen2.5-Math-72B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-Math-72B-Instruct)|qwen2|qwen2_5_math|transformers>=4.37|✔|math|[Qwen/Qwen2.5-Math-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Math-72B-Instruct)|\n|[Qwen/Qwen2.5-Math-1.5B](https://modelscope.cn/models/Qwen/Qwen2.5-Math-1.5B)|qwen2|qwen2_5_math|transformers>=4.37|✔|math|[Qwen/Qwen2.5-Math-1.5B](https://huggingface.co/Qwen/Qwen2.5-Math-1.5B)|\n|[Qwen/Qwen2.5-Math-7B](https://modelscope.cn/models/Qwen/Qwen2.5-Math-7B)|qwen2|qwen2_5_math|transformers>=4.37|✔|math|[Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B)|\n|[Qwen/Qwen2.5-Math-72B](https://modelscope.cn/models/Qwen/Qwen2.5-Math-72B)|qwen2|qwen2_5_math|transformers>=4.37|✔|math|[Qwen/Qwen2.5-Math-72B](https://huggingface.co/Qwen/Qwen2.5-Math-72B)|\n|[AIDC-AI/Marco-o1](https://modelscope.cn/models/AIDC-AI/Marco-o1)|qwen2|marco_o1|transformers>=4.37|✔|-|[AIDC-AI/Marco-o1](https://huggingface.co/AIDC-AI/Marco-o1)|\n|[Qwen/QwQ-32B-Preview](https://modelscope.cn/models/Qwen/QwQ-32B-Preview)|qwen2|qwq_preview|transformers>=4.37|✔|-|[Qwen/QwQ-32B-Preview](https://huggingface.co/Qwen/QwQ-32B-Preview)|\n|[Qwen/QwQ-32B](https://modelscope.cn/models/Qwen/QwQ-32B)|qwen2|qwq|transformers>=4.37|✔|-|[Qwen/QwQ-32B](https://huggingface.co/Qwen/QwQ-32B)|\n|[Qwen/QwQ-32B-AWQ](https://modelscope.cn/models/Qwen/QwQ-32B-AWQ)|qwen2|qwq|transformers>=4.37|✘|-|[Qwen/QwQ-32B-AWQ](https://huggingface.co/Qwen/QwQ-32B-AWQ)|\n|[deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://modelscope.cn/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B)|qwen2|deepseek_r1|transformers>=4.37|✔|-|[deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B)|\n|[deepseek-ai/DeepSeek-R1-Distill-Qwen-7B](https://modelscope.cn/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B)|qwen2|deepseek_r1|transformers>=4.37|✔|-|[deepseek-ai/DeepSeek-R1-Distill-Qwen-7B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-7B)|\n|[deepseek-ai/DeepSeek-R1-Distill-Qwen-14B](https://modelscope.cn/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B)|qwen2|deepseek_r1|transformers>=4.37|✔|-|[deepseek-ai/DeepSeek-R1-Distill-Qwen-14B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B)|\n|[deepseek-ai/DeepSeek-R1-Distill-Qwen-32B](https://modelscope.cn/models/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B)|qwen2|deepseek_r1|transformers>=4.37|✔|-|[deepseek-ai/DeepSeek-R1-Distill-Qwen-32B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B)|\n|[iic/QwenLong-L1-32B](https://modelscope.cn/models/iic/QwenLong-L1-32B)|qwen2|deepseek_r1|transformers>=4.37|✔|-|[Tongyi-Zhiwen/QwenLong-L1-32B](https://huggingface.co/Tongyi-Zhiwen/QwenLong-L1-32B)|\n|[Qwen/Qwen1.5-MoE-A2.7B-Chat](https://modelscope.cn/models/Qwen/Qwen1.5-MoE-A2.7B-Chat)|qwen2_moe|qwen|transformers>=4.40|✔|-|[Qwen/Qwen1.5-MoE-A2.7B-Chat](https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat)|\n|[Qwen/Qwen1.5-MoE-A2.7B](https://modelscope.cn/models/Qwen/Qwen1.5-MoE-A2.7B)|qwen2_moe|qwen|transformers>=4.40|✔|-|[Qwen/Qwen1.5-MoE-A2.7B](https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B)|\n|[Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4)|qwen2_moe|qwen|transformers>=4.40|✘|-|[Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4](https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4)|\n|[Qwen/Qwen2-57B-A14B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-57B-A14B-Instruct)|qwen2_moe|qwen|transformers>=4.40|✔|-|[Qwen/Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct)|\n|[Qwen/Qwen2-57B-A14B](https://modelscope.cn/models/Qwen/Qwen2-57B-A14B)|qwen2_moe|qwen|transformers>=4.40|✔|-|[Qwen/Qwen2-57B-A14B](https://huggingface.co/Qwen/Qwen2-57B-A14B)|\n|[Qwen/Qwen2-57B-A14B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2-57B-A14B-Instruct-GPTQ-Int4)|qwen2_moe|qwen|transformers>=4.40|✘|-|[Qwen/Qwen2-57B-A14B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen3-0.6B-Base](https://modelscope.cn/models/Qwen/Qwen3-0.6B-Base)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-0.6B-Base](https://huggingface.co/Qwen/Qwen3-0.6B-Base)|\n|[Qwen/Qwen3-1.7B-Base](https://modelscope.cn/models/Qwen/Qwen3-1.7B-Base)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-1.7B-Base](https://huggingface.co/Qwen/Qwen3-1.7B-Base)|\n|[Qwen/Qwen3-4B-Base](https://modelscope.cn/models/Qwen/Qwen3-4B-Base)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-4B-Base](https://huggingface.co/Qwen/Qwen3-4B-Base)|\n|[Qwen/Qwen3-8B-Base](https://modelscope.cn/models/Qwen/Qwen3-8B-Base)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-8B-Base](https://huggingface.co/Qwen/Qwen3-8B-Base)|\n|[Qwen/Qwen3-14B-Base](https://modelscope.cn/models/Qwen/Qwen3-14B-Base)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-14B-Base](https://huggingface.co/Qwen/Qwen3-14B-Base)|\n|[Qwen/Qwen3-0.6B](https://modelscope.cn/models/Qwen/Qwen3-0.6B)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B)|\n|[Qwen/Qwen3-1.7B](https://modelscope.cn/models/Qwen/Qwen3-1.7B)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B)|\n|[Qwen/Qwen3-4B](https://modelscope.cn/models/Qwen/Qwen3-4B)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B)|\n|[Qwen/Qwen3-8B](https://modelscope.cn/models/Qwen/Qwen3-8B)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B)|\n|[Qwen/Qwen3-14B](https://modelscope.cn/models/Qwen/Qwen3-14B)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-14B](https://huggingface.co/Qwen/Qwen3-14B)|\n|[Qwen/Qwen3-32B](https://modelscope.cn/models/Qwen/Qwen3-32B)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-32B](https://huggingface.co/Qwen/Qwen3-32B)|\n|[Qwen/Qwen3-0.6B-FP8](https://modelscope.cn/models/Qwen/Qwen3-0.6B-FP8)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-0.6B-FP8](https://huggingface.co/Qwen/Qwen3-0.6B-FP8)|\n|[Qwen/Qwen3-1.7B-FP8](https://modelscope.cn/models/Qwen/Qwen3-1.7B-FP8)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-1.7B-FP8](https://huggingface.co/Qwen/Qwen3-1.7B-FP8)|\n|[Qwen/Qwen3-4B-FP8](https://modelscope.cn/models/Qwen/Qwen3-4B-FP8)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-4B-FP8](https://huggingface.co/Qwen/Qwen3-4B-FP8)|\n|[Qwen/Qwen3-8B-FP8](https://modelscope.cn/models/Qwen/Qwen3-8B-FP8)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-8B-FP8](https://huggingface.co/Qwen/Qwen3-8B-FP8)|\n|[Qwen/Qwen3-14B-FP8](https://modelscope.cn/models/Qwen/Qwen3-14B-FP8)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-14B-FP8](https://huggingface.co/Qwen/Qwen3-14B-FP8)|\n|[Qwen/Qwen3-32B-FP8](https://modelscope.cn/models/Qwen/Qwen3-32B-FP8)|qwen3|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-32B-FP8](https://huggingface.co/Qwen/Qwen3-32B-FP8)|\n|[Qwen/Qwen3-4B-AWQ](https://modelscope.cn/models/Qwen/Qwen3-4B-AWQ)|qwen3|qwen3|transformers>=4.51|✘|-|[Qwen/Qwen3-4B-AWQ](https://huggingface.co/Qwen/Qwen3-4B-AWQ)|\n|[Qwen/Qwen3-8B-AWQ](https://modelscope.cn/models/Qwen/Qwen3-8B-AWQ)|qwen3|qwen3|transformers>=4.51|✘|-|[Qwen/Qwen3-8B-AWQ](https://huggingface.co/Qwen/Qwen3-8B-AWQ)|\n|[Qwen/Qwen3-14B-AWQ](https://modelscope.cn/models/Qwen/Qwen3-14B-AWQ)|qwen3|qwen3|transformers>=4.51|✘|-|[Qwen/Qwen3-14B-AWQ](https://huggingface.co/Qwen/Qwen3-14B-AWQ)|\n|[Qwen/Qwen3-32B-AWQ](https://modelscope.cn/models/Qwen/Qwen3-32B-AWQ)|qwen3|qwen3|transformers>=4.51|✘|-|[Qwen/Qwen3-32B-AWQ](https://huggingface.co/Qwen/Qwen3-32B-AWQ)|\n|[swift/Qwen3-32B-AWQ](https://modelscope.cn/models/swift/Qwen3-32B-AWQ)|qwen3|qwen3|transformers>=4.51|✘|-|-|\n|[deepseek-ai/DeepSeek-R1-0528-Qwen3-8B](https://modelscope.cn/models/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B)|qwen3|deepseek_r1|transformers>=4.51|✔|-|[deepseek-ai/DeepSeek-R1-0528-Qwen3-8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-0528-Qwen3-8B)|\n|[Qwen/Qwen3Guard-Gen-0.6B](https://modelscope.cn/models/Qwen/Qwen3Guard-Gen-0.6B)|qwen3|qwen3_guard|transformers>=4.51|✔|-|[Qwen/Qwen3Guard-Gen-0.6B](https://huggingface.co/Qwen/Qwen3Guard-Gen-0.6B)|\n|[Qwen/Qwen3Guard-Gen-4B](https://modelscope.cn/models/Qwen/Qwen3Guard-Gen-4B)|qwen3|qwen3_guard|transformers>=4.51|✔|-|[Qwen/Qwen3Guard-Gen-4B](https://huggingface.co/Qwen/Qwen3Guard-Gen-4B)|\n|[Qwen/Qwen3Guard-Gen-8B](https://modelscope.cn/models/Qwen/Qwen3Guard-Gen-8B)|qwen3|qwen3_guard|transformers>=4.51|✔|-|[Qwen/Qwen3Guard-Gen-8B](https://huggingface.co/Qwen/Qwen3Guard-Gen-8B)|\n|[Alibaba-AAIG/YuFeng-XGuard-Reason-0.6B](https://modelscope.cn/models/Alibaba-AAIG/YuFeng-XGuard-Reason-0.6B)|qwen3|yufeng_xguard|transformers>=4.51|✔|-|[Alibaba-AAIG/YuFeng-XGuard-Reason-0.6B](https://huggingface.co/Alibaba-AAIG/YuFeng-XGuard-Reason-0.6B)|\n|[Alibaba-AAIG/YuFeng-XGuard-Reason-8B](https://modelscope.cn/models/Alibaba-AAIG/YuFeng-XGuard-Reason-8B)|qwen3|yufeng_xguard|transformers>=4.51|✔|-|[Alibaba-AAIG/YuFeng-XGuard-Reason-8B](https://huggingface.co/Alibaba-AAIG/YuFeng-XGuard-Reason-8B)|\n|[Qwen/Qwen3-4B-Thinking-2507](https://modelscope.cn/models/Qwen/Qwen3-4B-Thinking-2507)|qwen3|qwen3_thinking|transformers>=4.51|✔|-|[Qwen/Qwen3-4B-Thinking-2507](https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507)|\n|[Qwen/Qwen3-4B-Thinking-2507-FP8](https://modelscope.cn/models/Qwen/Qwen3-4B-Thinking-2507-FP8)|qwen3|qwen3_thinking|transformers>=4.51|✔|-|[Qwen/Qwen3-4B-Thinking-2507-FP8](https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507-FP8)|\n|[Qwen/Qwen3-4B-Instruct-2507](https://modelscope.cn/models/Qwen/Qwen3-4B-Instruct-2507)|qwen3|qwen3_nothinking|transformers>=4.51|✔|-|[Qwen/Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507)|\n|[Qwen/Qwen3-4B-Instruct-2507-FP8](https://modelscope.cn/models/Qwen/Qwen3-4B-Instruct-2507-FP8)|qwen3|qwen3_nothinking|transformers>=4.51|✔|-|[Qwen/Qwen3-4B-Instruct-2507-FP8](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507-FP8)|\n|[Qwen/Qwen3-30B-A3B-Base](https://modelscope.cn/models/Qwen/Qwen3-30B-A3B-Base)|qwen3_moe|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-30B-A3B-Base](https://huggingface.co/Qwen/Qwen3-30B-A3B-Base)|\n|[Qwen/Qwen3-30B-A3B](https://modelscope.cn/models/Qwen/Qwen3-30B-A3B)|qwen3_moe|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B)|\n|[Qwen/Qwen3-235B-A22B](https://modelscope.cn/models/Qwen/Qwen3-235B-A22B)|qwen3_moe|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-235B-A22B](https://huggingface.co/Qwen/Qwen3-235B-A22B)|\n|[Qwen/Qwen3-30B-A3B-FP8](https://modelscope.cn/models/Qwen/Qwen3-30B-A3B-FP8)|qwen3_moe|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-30B-A3B-FP8](https://huggingface.co/Qwen/Qwen3-30B-A3B-FP8)|\n|[Qwen/Qwen3-235B-A22B-FP8](https://modelscope.cn/models/Qwen/Qwen3-235B-A22B-FP8)|qwen3_moe|qwen3|transformers>=4.51|✔|-|[Qwen/Qwen3-235B-A22B-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-FP8)|\n|[swift/Qwen3-30B-A3B-AWQ](https://modelscope.cn/models/swift/Qwen3-30B-A3B-AWQ)|qwen3_moe|qwen3|transformers>=4.51|✘|-|[cognitivecomputations/Qwen3-30B-A3B-AWQ](https://huggingface.co/cognitivecomputations/Qwen3-30B-A3B-AWQ)|\n|[swift/Qwen3-235B-A22B-AWQ](https://modelscope.cn/models/swift/Qwen3-235B-A22B-AWQ)|qwen3_moe|qwen3|transformers>=4.51|✘|-|[cognitivecomputations/Qwen3-235B-A22B-AWQ](https://huggingface.co/cognitivecomputations/Qwen3-235B-A22B-AWQ)|\n|[iic/Tongyi-DeepResearch-30B-A3B](https://modelscope.cn/models/iic/Tongyi-DeepResearch-30B-A3B)|qwen3_moe|qwen3|transformers>=4.51|✔|-|[Alibaba-NLP/Tongyi-DeepResearch-30B-A3B](https://huggingface.co/Alibaba-NLP/Tongyi-DeepResearch-30B-A3B)|\n|[Qwen/Qwen3-30B-A3B-Instruct-2507](https://modelscope.cn/models/Qwen/Qwen3-30B-A3B-Instruct-2507)|qwen3_moe|qwen3_nothinking|transformers>=4.51|✔|-|[Qwen/Qwen3-30B-A3B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507)|\n|[Qwen/Qwen3-30B-A3B-Instruct-2507-FP8](https://modelscope.cn/models/Qwen/Qwen3-30B-A3B-Instruct-2507-FP8)|qwen3_moe|qwen3_nothinking|transformers>=4.51|✔|-|[Qwen/Qwen3-30B-A3B-Instruct-2507-FP8](https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507-FP8)|\n|[Qwen/Qwen3-235B-A22B-Instruct-2507](https://modelscope.cn/models/Qwen/Qwen3-235B-A22B-Instruct-2507)|qwen3_moe|qwen3_nothinking|transformers>=4.51|✔|-|[Qwen/Qwen3-235B-A22B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507)|\n|[Qwen/Qwen3-235B-A22B-Instruct-2507-FP8](https://modelscope.cn/models/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8)|qwen3_moe|qwen3_nothinking|transformers>=4.51|✔|-|[Qwen/Qwen3-235B-A22B-Instruct-2507-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8)|\n|[swift/Qwen3-235B-A22B-Instruct-2507-AWQ](https://modelscope.cn/models/swift/Qwen3-235B-A22B-Instruct-2507-AWQ)|qwen3_moe|qwen3_nothinking|transformers>=4.51|✘|-|-|\n|[Qwen/Qwen3-Coder-30B-A3B-Instruct](https://modelscope.cn/models/Qwen/Qwen3-Coder-30B-A3B-Instruct)|qwen3_moe|qwen3_coder|transformers>=4.51|✔|coding|[Qwen/Qwen3-Coder-30B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct)|\n|[Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8](https://modelscope.cn/models/Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8)|qwen3_moe|qwen3_coder|transformers>=4.51|✔|coding|[Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8)|\n|[Qwen/Qwen3-Coder-480B-A35B-Instruct](https://modelscope.cn/models/Qwen/Qwen3-Coder-480B-A35B-Instruct)|qwen3_moe|qwen3_coder|transformers>=4.51|✔|coding|[Qwen/Qwen3-Coder-480B-A35B-Instruct](https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct)|\n|[Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8](https://modelscope.cn/models/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8)|qwen3_moe|qwen3_coder|transformers>=4.51|✔|coding|[Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8)|\n|[swift/Qwen3-Coder-480B-A35B-Instruct-AWQ](https://modelscope.cn/models/swift/Qwen3-Coder-480B-A35B-Instruct-AWQ)|qwen3_moe|qwen3_coder|transformers>=4.51|✘|coding|-|\n|[iic/QwenLong-L1.5-30B-A3B](https://modelscope.cn/models/iic/QwenLong-L1.5-30B-A3B)|qwen3_moe|qwen3_thinking|transformers>=4.51|✔|-|[Tongyi-Zhiwen/QwenLong-L1.5-30B-A3B](https://huggingface.co/Tongyi-Zhiwen/QwenLong-L1.5-30B-A3B)|\n|[Qwen/Qwen3-30B-A3B-Thinking-2507](https://modelscope.cn/models/Qwen/Qwen3-30B-A3B-Thinking-2507)|qwen3_moe|qwen3_thinking|transformers>=4.51|✔|-|[Qwen/Qwen3-30B-A3B-Thinking-2507](https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507)|\n|[Qwen/Qwen3-30B-A3B-Thinking-2507-FP8](https://modelscope.cn/models/Qwen/Qwen3-30B-A3B-Thinking-2507-FP8)|qwen3_moe|qwen3_thinking|transformers>=4.51|✔|-|[Qwen/Qwen3-30B-A3B-Thinking-2507-FP8](https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507-FP8)|\n|[Qwen/Qwen3-235B-A22B-Thinking-2507](https://modelscope.cn/models/Qwen/Qwen3-235B-A22B-Thinking-2507)|qwen3_moe|qwen3_thinking|transformers>=4.51|✔|-|[Qwen/Qwen3-235B-A22B-Thinking-2507](https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507)|\n|[Qwen/Qwen3-235B-A22B-Thinking-2507-FP8](https://modelscope.cn/models/Qwen/Qwen3-235B-A22B-Thinking-2507-FP8)|qwen3_moe|qwen3_thinking|transformers>=4.51|✔|-|[Qwen/Qwen3-235B-A22B-Thinking-2507-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507-FP8)|\n|[swift/Qwen3-235B-A22B-Thinking-2507-AWQ](https://modelscope.cn/models/swift/Qwen3-235B-A22B-Thinking-2507-AWQ)|qwen3_moe|qwen3_thinking|transformers>=4.51|✘|-|-|\n|[Qwen/Qwen3-Next-80B-A3B-Instruct](https://modelscope.cn/models/Qwen/Qwen3-Next-80B-A3B-Instruct)|qwen3_next|qwen3_nothinking|transformers>=4.57|✔|-|-|\n|[Qwen/Qwen3-Next-80B-A3B-Instruct-FP8](https://modelscope.cn/models/Qwen/Qwen3-Next-80B-A3B-Instruct-FP8)|qwen3_next|qwen3_nothinking|transformers>=4.57|✔|-|-|\n|[Qwen/Qwen3-Next-80B-A3B-Thinking](https://modelscope.cn/models/Qwen/Qwen3-Next-80B-A3B-Thinking)|qwen3_next|qwen3_thinking|transformers>=4.57|✔|-|-|\n|[Qwen/Qwen3-Next-80B-A3B-Thinking-FP8](https://modelscope.cn/models/Qwen/Qwen3-Next-80B-A3B-Thinking-FP8)|qwen3_next|qwen3_thinking|transformers>=4.57|✔|-|-|\n|[Qwen/Qwen3-Coder-Next-Base](https://modelscope.cn/models/Qwen/Qwen3-Coder-Next-Base)|qwen3_next|qwen3_coder|transformers>=4.57|✔|-|[Qwen/Qwen3-Coder-Next-Base](https://huggingface.co/Qwen/Qwen3-Coder-Next-Base)|\n|[Qwen/Qwen3-Coder-Next](https://modelscope.cn/models/Qwen/Qwen3-Coder-Next)|qwen3_next|qwen3_coder|transformers>=4.57|✔|-|[Qwen/Qwen3-Coder-Next](https://huggingface.co/Qwen/Qwen3-Coder-Next)|\n|[Qwen/Qwen3-Coder-Next-FP8](https://modelscope.cn/models/Qwen/Qwen3-Coder-Next-FP8)|qwen3_next|qwen3_coder|transformers>=4.57|✔|-|[Qwen/Qwen3-Coder-Next-FP8](https://huggingface.co/Qwen/Qwen3-Coder-Next-FP8)|\n|[Qwen/Qwen3-Embedding-0.6B](https://modelscope.cn/models/Qwen/Qwen3-Embedding-0.6B)|qwen3_emb|qwen3_emb|-|✔|-|[Qwen/Qwen3-Embedding-0.6B](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B)|\n|[Qwen/Qwen3-Embedding-4B](https://modelscope.cn/models/Qwen/Qwen3-Embedding-4B)|qwen3_emb|qwen3_emb|-|✔|-|[Qwen/Qwen3-Embedding-4B](https://huggingface.co/Qwen/Qwen3-Embedding-4B)|\n|[Qwen/Qwen3-Embedding-8B](https://modelscope.cn/models/Qwen/Qwen3-Embedding-8B)|qwen3_emb|qwen3_emb|-|✔|-|[Qwen/Qwen3-Embedding-8B](https://huggingface.co/Qwen/Qwen3-Embedding-8B)|\n|[Qwen/Qwen3-Reranker-0.6B](https://modelscope.cn/models/Qwen/Qwen3-Reranker-0.6B)|qwen3_reranker|qwen3_reranker|-|✔|-|[Qwen/Qwen3-Reranker-0.6B](https://huggingface.co/Qwen/Qwen3-Reranker-0.6B)|\n|[Qwen/Qwen3-Reranker-4B](https://modelscope.cn/models/Qwen/Qwen3-Reranker-4B)|qwen3_reranker|qwen3_reranker|-|✔|-|[Qwen/Qwen3-Reranker-4B](https://huggingface.co/Qwen/Qwen3-Reranker-4B)|\n|[Qwen/Qwen3-Reranker-8B](https://modelscope.cn/models/Qwen/Qwen3-Reranker-8B)|qwen3_reranker|qwen3_reranker|-|✔|-|[Qwen/Qwen3-Reranker-8B](https://huggingface.co/Qwen/Qwen3-Reranker-8B)|\n|[iic/gte_Qwen2-1.5B-instruct](https://modelscope.cn/models/iic/gte_Qwen2-1.5B-instruct)|qwen2_gte|dummy|-|✘|-|[Alibaba-NLP/gte-Qwen2-1.5B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct)|\n|[iic/gte_Qwen2-7B-instruct](https://modelscope.cn/models/iic/gte_Qwen2-7B-instruct)|qwen2_gte|dummy|-|✘|-|[Alibaba-NLP/gte-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct)|\n|[codefuse-ai/CodeFuse-QWen-14B](https://modelscope.cn/models/codefuse-ai/CodeFuse-QWen-14B)|codefuse_qwen|codefuse|-|✘|coding|[codefuse-ai/CodeFuse-QWen-14B](https://huggingface.co/codefuse-ai/CodeFuse-QWen-14B)|\n|[iic/ModelScope-Agent-7B](https://modelscope.cn/models/iic/ModelScope-Agent-7B)|modelscope_agent|modelscope_agent|-|✘|-|-|\n|[iic/ModelScope-Agent-14B](https://modelscope.cn/models/iic/ModelScope-Agent-14B)|modelscope_agent|modelscope_agent|-|✘|-|-|\n|[modelscope/Llama-2-7b-ms](https://modelscope.cn/models/modelscope/Llama-2-7b-ms)|llama|llama|-|✔|-|[meta-llama/Llama-2-7b-hf](https://huggingface.co/meta-llama/Llama-2-7b-hf)|\n|[modelscope/Llama-2-13b-ms](https://modelscope.cn/models/modelscope/Llama-2-13b-ms)|llama|llama|-|✔|-|[meta-llama/Llama-2-13b-hf](https://huggingface.co/meta-llama/Llama-2-13b-hf)|\n|[modelscope/Llama-2-70b-ms](https://modelscope.cn/models/modelscope/Llama-2-70b-ms)|llama|llama|-|✔|-|[meta-llama/Llama-2-70b-hf](https://huggingface.co/meta-llama/Llama-2-70b-hf)|\n|[modelscope/Llama-2-7b-chat-ms](https://modelscope.cn/models/modelscope/Llama-2-7b-chat-ms)|llama|llama|-|✔|-|[meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf)|\n|[modelscope/Llama-2-13b-chat-ms](https://modelscope.cn/models/modelscope/Llama-2-13b-chat-ms)|llama|llama|-|✔|-|[meta-llama/Llama-2-13b-chat-hf](https://huggingface.co/meta-llama/Llama-2-13b-chat-hf)|\n|[modelscope/Llama-2-70b-chat-ms](https://modelscope.cn/models/modelscope/Llama-2-70b-chat-ms)|llama|llama|-|✔|-|[meta-llama/Llama-2-70b-chat-hf](https://huggingface.co/meta-llama/Llama-2-70b-chat-hf)|\n|[AI-ModelScope/chinese-llama-2-1.3b](https://modelscope.cn/models/AI-ModelScope/chinese-llama-2-1.3b)|llama|llama|-|✔|-|[hfl/chinese-llama-2-1.3b](https://huggingface.co/hfl/chinese-llama-2-1.3b)|\n|[AI-ModelScope/chinese-llama-2-7b](https://modelscope.cn/models/AI-ModelScope/chinese-llama-2-7b)|llama|llama|-|✔|-|[hfl/chinese-llama-2-7b](https://huggingface.co/hfl/chinese-llama-2-7b)|\n|[AI-ModelScope/chinese-llama-2-7b-16k](https://modelscope.cn/models/AI-ModelScope/chinese-llama-2-7b-16k)|llama|llama|-|✔|-|[hfl/chinese-llama-2-7b-16k](https://huggingface.co/hfl/chinese-llama-2-7b-16k)|\n|[AI-ModelScope/chinese-llama-2-7b-64k](https://modelscope.cn/models/AI-ModelScope/chinese-llama-2-7b-64k)|llama|llama|-|✔|-|[hfl/chinese-llama-2-7b-64k](https://huggingface.co/hfl/chinese-llama-2-7b-64k)|\n|[AI-ModelScope/chinese-llama-2-13b](https://modelscope.cn/models/AI-ModelScope/chinese-llama-2-13b)|llama|llama|-|✔|-|[hfl/chinese-llama-2-13b](https://huggingface.co/hfl/chinese-llama-2-13b)|\n|[AI-ModelScope/chinese-llama-2-13b-16k](https://modelscope.cn/models/AI-ModelScope/chinese-llama-2-13b-16k)|llama|llama|-|✔|-|[hfl/chinese-llama-2-13b-16k](https://huggingface.co/hfl/chinese-llama-2-13b-16k)|\n|[AI-ModelScope/chinese-alpaca-2-1.3b](https://modelscope.cn/models/AI-ModelScope/chinese-alpaca-2-1.3b)|llama|llama|-|✔|-|[hfl/chinese-alpaca-2-1.3b](https://huggingface.co/hfl/chinese-alpaca-2-1.3b)|\n|[AI-ModelScope/chinese-alpaca-2-7b](https://modelscope.cn/models/AI-ModelScope/chinese-alpaca-2-7b)|llama|llama|-|✔|-|[hfl/chinese-alpaca-2-7b](https://huggingface.co/hfl/chinese-alpaca-2-7b)|\n|[AI-ModelScope/chinese-alpaca-2-7b-16k](https://modelscope.cn/models/AI-ModelScope/chinese-alpaca-2-7b-16k)|llama|llama|-|✔|-|[hfl/chinese-alpaca-2-7b-16k](https://huggingface.co/hfl/chinese-alpaca-2-7b-16k)|\n|[AI-ModelScope/chinese-alpaca-2-7b-64k](https://modelscope.cn/models/AI-ModelScope/chinese-alpaca-2-7b-64k)|llama|llama|-|✔|-|[hfl/chinese-alpaca-2-7b-64k](https://huggingface.co/hfl/chinese-alpaca-2-7b-64k)|\n|[AI-ModelScope/chinese-alpaca-2-13b](https://modelscope.cn/models/AI-ModelScope/chinese-alpaca-2-13b)|llama|llama|-|✔|-|[hfl/chinese-alpaca-2-13b](https://huggingface.co/hfl/chinese-alpaca-2-13b)|\n|[AI-ModelScope/chinese-alpaca-2-13b-16k](https://modelscope.cn/models/AI-ModelScope/chinese-alpaca-2-13b-16k)|llama|llama|-|✔|-|[hfl/chinese-alpaca-2-13b-16k](https://huggingface.co/hfl/chinese-alpaca-2-13b-16k)|\n|[AI-ModelScope/Llama-2-7b-AQLM-2Bit-1x16-hf](https://modelscope.cn/models/AI-ModelScope/Llama-2-7b-AQLM-2Bit-1x16-hf)|llama|llama|transformers>=4.38, aqlm, torch>=2.2.0|✘|-|[ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf](https://huggingface.co/ISTA-DASLab/Llama-2-7b-AQLM-2Bit-1x16-hf)|\n|[FlagAlpha/Atom-7B](https://modelscope.cn/models/FlagAlpha/Atom-7B)|llama|atom|-|✔|-|[FlagAlpha/Atom-7B](https://huggingface.co/FlagAlpha/Atom-7B)|\n|[FlagAlpha/Atom-7B-Chat](https://modelscope.cn/models/FlagAlpha/Atom-7B-Chat)|llama|atom|-|✔|-|[FlagAlpha/Atom-7B-Chat](https://huggingface.co/FlagAlpha/Atom-7B-Chat)|\n|[langboat/Mengzi3-13B-Base](https://modelscope.cn/models/langboat/Mengzi3-13B-Base)|llama|mengzi|-|✔|-|[Langboat/Mengzi3-13B-Base](https://huggingface.co/Langboat/Mengzi3-13B-Base)|\n|[AI-ModelScope/NuminaMath-7B-TIR](https://modelscope.cn/models/AI-ModelScope/NuminaMath-7B-TIR)|llama|numina|-|✔|math|[AI-MO/NuminaMath-7B-TIR](https://huggingface.co/AI-MO/NuminaMath-7B-TIR)|\n|[Fengshenbang/Ziya2-13B-Base](https://modelscope.cn/models/Fengshenbang/Ziya2-13B-Base)|llama|ziya|-|✔|-|[IDEA-CCNL/Ziya2-13B-Base](https://huggingface.co/IDEA-CCNL/Ziya2-13B-Base)|\n|[Fengshenbang/Ziya2-13B-Chat](https://modelscope.cn/models/Fengshenbang/Ziya2-13B-Chat)|llama|ziya|-|✔|-|[IDEA-CCNL/Ziya2-13B-Chat](https://huggingface.co/IDEA-CCNL/Ziya2-13B-Chat)|\n|[InfiniAI/Megrez-3b-Instruct](https://modelscope.cn/models/InfiniAI/Megrez-3b-Instruct)|llama|megrez|-|✔|-|[Infinigence/Megrez-3B-Instruct](https://huggingface.co/Infinigence/Megrez-3B-Instruct)|\n|[deepseek-ai/deepseek-llm-7b-base](https://modelscope.cn/models/deepseek-ai/deepseek-llm-7b-base)|llama|deepseek|-|✔|-|[deepseek-ai/deepseek-llm-7b-base](https://huggingface.co/deepseek-ai/deepseek-llm-7b-base)|\n|[deepseek-ai/deepseek-llm-7b-chat](https://modelscope.cn/models/deepseek-ai/deepseek-llm-7b-chat)|llama|deepseek|-|✔|-|[deepseek-ai/deepseek-llm-7b-chat](https://huggingface.co/deepseek-ai/deepseek-llm-7b-chat)|\n|[deepseek-ai/deepseek-llm-67b-base](https://modelscope.cn/models/deepseek-ai/deepseek-llm-67b-base)|llama|deepseek|-|✔|-|[deepseek-ai/deepseek-llm-67b-base](https://huggingface.co/deepseek-ai/deepseek-llm-67b-base)|\n|[deepseek-ai/deepseek-llm-67b-chat](https://modelscope.cn/models/deepseek-ai/deepseek-llm-67b-chat)|llama|deepseek|-|✔|-|[deepseek-ai/deepseek-llm-67b-chat](https://huggingface.co/deepseek-ai/deepseek-llm-67b-chat)|\n|[deepseek-ai/deepseek-math-7b-base](https://modelscope.cn/models/deepseek-ai/deepseek-math-7b-base)|llama|deepseek|-|✔|math|[deepseek-ai/deepseek-math-7b-base](https://huggingface.co/deepseek-ai/deepseek-math-7b-base)|\n|[deepseek-ai/deepseek-math-7b-instruct](https://modelscope.cn/models/deepseek-ai/deepseek-math-7b-instruct)|llama|deepseek|-|✔|math|[deepseek-ai/deepseek-math-7b-instruct](https://huggingface.co/deepseek-ai/deepseek-math-7b-instruct)|\n|[deepseek-ai/deepseek-math-7b-rl](https://modelscope.cn/models/deepseek-ai/deepseek-math-7b-rl)|llama|deepseek|-|✔|math|[deepseek-ai/deepseek-math-7b-rl](https://huggingface.co/deepseek-ai/deepseek-math-7b-rl)|\n|[deepseek-ai/deepseek-coder-1.3b-base](https://modelscope.cn/models/deepseek-ai/deepseek-coder-1.3b-base)|llama|deepseek|-|✔|coding|[deepseek-ai/deepseek-coder-1.3b-base](https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base)|\n|[deepseek-ai/deepseek-coder-1.3b-instruct](https://modelscope.cn/models/deepseek-ai/deepseek-coder-1.3b-instruct)|llama|deepseek|-|✔|coding|[deepseek-ai/deepseek-coder-1.3b-instruct](https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct)|\n|[deepseek-ai/deepseek-coder-6.7b-base](https://modelscope.cn/models/deepseek-ai/deepseek-coder-6.7b-base)|llama|deepseek|-|✔|coding|[deepseek-ai/deepseek-coder-6.7b-base](https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base)|\n|[deepseek-ai/deepseek-coder-6.7b-instruct](https://modelscope.cn/models/deepseek-ai/deepseek-coder-6.7b-instruct)|llama|deepseek|-|✔|coding|[deepseek-ai/deepseek-coder-6.7b-instruct](https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct)|\n|[deepseek-ai/deepseek-coder-33b-base](https://modelscope.cn/models/deepseek-ai/deepseek-coder-33b-base)|llama|deepseek|-|✔|coding|[deepseek-ai/deepseek-coder-33b-base](https://huggingface.co/deepseek-ai/deepseek-coder-33b-base)|\n|[deepseek-ai/deepseek-coder-33b-instruct](https://modelscope.cn/models/deepseek-ai/deepseek-coder-33b-instruct)|llama|deepseek|-|✔|coding|[deepseek-ai/deepseek-coder-33b-instruct](https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct)|\n|[gongjy/MiniMind2](https://modelscope.cn/models/gongjy/MiniMind2)|llama|minimind|transformers>=4.57.1|✔|-|[jingyaogong/MiniMind2](https://huggingface.co/jingyaogong/MiniMind2)|\n|-|llama|minimind|transformers>=4.57.1|✔|-|[jingyaogong/MiniMind2-Small](https://huggingface.co/jingyaogong/MiniMind2-Small)|\n|[LLM-Research/Meta-Llama-3-8B-Instruct](https://modelscope.cn/models/LLM-Research/Meta-Llama-3-8B-Instruct)|llama|llama3|-|✔|-|[meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct)|\n|[LLM-Research/Meta-Llama-3-70B-Instruct](https://modelscope.cn/models/LLM-Research/Meta-Llama-3-70B-Instruct)|llama|llama3|-|✔|-|[meta-llama/Meta-Llama-3-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct)|\n|[LLM-Research/Meta-Llama-3-8B](https://modelscope.cn/models/LLM-Research/Meta-Llama-3-8B)|llama|llama3|-|✔|-|[meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B)|\n|[LLM-Research/Meta-Llama-3-70B](https://modelscope.cn/models/LLM-Research/Meta-Llama-3-70B)|llama|llama3|-|✔|-|[meta-llama/Meta-Llama-3-70B](https://huggingface.co/meta-llama/Meta-Llama-3-70B)|\n|[swift/Meta-Llama-3-8B-Instruct-GPTQ-Int4](https://modelscope.cn/models/swift/Meta-Llama-3-8B-Instruct-GPTQ-Int4)|llama|llama3|-|✘|-|[study-hjt/Meta-Llama-3-8B-Instruct-GPTQ-Int4](https://huggingface.co/study-hjt/Meta-Llama-3-8B-Instruct-GPTQ-Int4)|\n|[swift/Meta-Llama-3-8B-Instruct-GPTQ-Int8](https://modelscope.cn/models/swift/Meta-Llama-3-8B-Instruct-GPTQ-Int8)|llama|llama3|-|✘|-|[study-hjt/Meta-Llama-3-8B-Instruct-GPTQ-Int8](https://huggingface.co/study-hjt/Meta-Llama-3-8B-Instruct-GPTQ-Int8)|\n|[swift/Meta-Llama-3-8B-Instruct-AWQ](https://modelscope.cn/models/swift/Meta-Llama-3-8B-Instruct-AWQ)|llama|llama3|-|✘|-|[study-hjt/Meta-Llama-3-8B-Instruct-AWQ](https://huggingface.co/study-hjt/Meta-Llama-3-8B-Instruct-AWQ)|\n|[swift/Meta-Llama-3-70B-Instruct-GPTQ-Int4](https://modelscope.cn/models/swift/Meta-Llama-3-70B-Instruct-GPTQ-Int4)|llama|llama3|-|✘|-|[study-hjt/Meta-Llama-3-70B-Instruct-GPTQ-Int4](https://huggingface.co/study-hjt/Meta-Llama-3-70B-Instruct-GPTQ-Int4)|\n|[swift/Meta-Llama-3-70B-Instruct-GPTQ-Int8](https://modelscope.cn/models/swift/Meta-Llama-3-70B-Instruct-GPTQ-Int8)|llama|llama3|-|✘|-|[study-hjt/Meta-Llama-3-70B-Instruct-GPTQ-Int8](https://huggingface.co/study-hjt/Meta-Llama-3-70B-Instruct-GPTQ-Int8)|\n|[swift/Meta-Llama-3-70B-Instruct-AWQ](https://modelscope.cn/models/swift/Meta-Llama-3-70B-Instruct-AWQ)|llama|llama3|-|✘|-|[study-hjt/Meta-Llama-3-70B-Instruct-AWQ](https://huggingface.co/study-hjt/Meta-Llama-3-70B-Instruct-AWQ)|\n|[ChineseAlpacaGroup/llama-3-chinese-8b-instruct](https://modelscope.cn/models/ChineseAlpacaGroup/llama-3-chinese-8b-instruct)|llama|llama3|-|✔|-|[hfl/llama-3-chinese-8b-instruct](https://huggingface.co/hfl/llama-3-chinese-8b-instruct)|\n|[ChineseAlpacaGroup/llama-3-chinese-8b](https://modelscope.cn/models/ChineseAlpacaGroup/llama-3-chinese-8b)|llama|llama3|-|✔|-|[hfl/llama-3-chinese-8b](https://huggingface.co/hfl/llama-3-chinese-8b)|\n|[LLM-Research/Meta-Llama-3.1-8B-Instruct](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-8B-Instruct)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct)|\n|[LLM-Research/Meta-Llama-3.1-70B-Instruct](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-70B-Instruct)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Meta-Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct)|\n|[LLM-Research/Meta-Llama-3.1-405B-Instruct](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-405B-Instruct)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Meta-Llama-3.1-405B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-405B-Instruct)|\n|[LLM-Research/Meta-Llama-3.1-8B](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-8B)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Meta-Llama-3.1-8B](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B)|\n|[LLM-Research/Meta-Llama-3.1-70B](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-70B)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Meta-Llama-3.1-70B](https://huggingface.co/meta-llama/Meta-Llama-3.1-70B)|\n|[LLM-Research/Meta-Llama-3.1-405B](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-405B)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Meta-Llama-3.1-405B](https://huggingface.co/meta-llama/Meta-Llama-3.1-405B)|\n|[LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-70B-Instruct-FP8)|llama|llama3_2|transformers>=4.43|✘|-|[meta-llama/Meta-Llama-3.1-70B-Instruct-FP8](https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct-FP8)|\n|[LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-405B-Instruct-FP8)|llama|llama3_2|transformers>=4.43|✘|-|[meta-llama/Meta-Llama-3.1-405B-Instruct-FP8](https://huggingface.co/meta-llama/Meta-Llama-3.1-405B-Instruct-FP8)|\n|[LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-8B-Instruct-BNB-NF4)|llama|llama3_2|transformers>=4.43|✘|-|[hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4](https://huggingface.co/hugging-quants/Meta-Llama-3.1-8B-Instruct-BNB-NF4)|\n|[LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-70B-Instruct-bnb-4bit)|llama|llama3_2|transformers>=4.43|✘|-|[unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit](https://huggingface.co/unsloth/Meta-Llama-3.1-70B-Instruct-bnb-4bit)|\n|[LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-405B-Instruct-BNB-NF4)|llama|llama3_2|transformers>=4.43|✘|-|[hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4](https://huggingface.co/hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4)|\n|[LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4)|llama|llama3_2|transformers>=4.43|✘|-|[hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4](https://huggingface.co/hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4)|\n|[LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4)|llama|llama3_2|transformers>=4.43|✘|-|[hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4](https://huggingface.co/hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4)|\n|[LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4)|llama|llama3_2|transformers>=4.43|✘|-|[hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4](https://huggingface.co/hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4)|\n|[LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-8B-Instruct-AWQ-INT4)|llama|llama3_2|transformers>=4.43|✘|-|[hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4](https://huggingface.co/hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4)|\n|[LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-70B-Instruct-AWQ-INT4)|llama|llama3_2|transformers>=4.43|✘|-|[hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4](https://huggingface.co/hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4)|\n|[LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4](https://modelscope.cn/models/LLM-Research/Meta-Llama-3.1-405B-Instruct-AWQ-INT4)|llama|llama3_2|transformers>=4.43|✘|-|[hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4](https://huggingface.co/hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4)|\n|[AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF](https://modelscope.cn/models/AI-ModelScope/Llama-3.1-Nemotron-70B-Instruct-HF)|llama|llama3_2|transformers>=4.43|✔|-|[nvidia/Llama-3.1-Nemotron-70B-Instruct-HF](https://huggingface.co/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF)|\n|[AI-ModelScope/Skywork-o1-Open-Llama-3.1-8B](https://modelscope.cn/models/AI-ModelScope/Skywork-o1-Open-Llama-3.1-8B)|llama|skywork_o1|transformers>=4.43|✔|-|[Skywork/Skywork-o1-Open-Llama-3.1-8B](https://huggingface.co/Skywork/Skywork-o1-Open-Llama-3.1-8B)|\n|[LLM-Research/Llama-3.2-1B](https://modelscope.cn/models/LLM-Research/Llama-3.2-1B)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Llama-3.2-1B](https://huggingface.co/meta-llama/Llama-3.2-1B)|\n|[LLM-Research/Llama-3.2-3B](https://modelscope.cn/models/LLM-Research/Llama-3.2-3B)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Llama-3.2-3B](https://huggingface.co/meta-llama/Llama-3.2-3B)|\n|[LLM-Research/Llama-3.2-1B-Instruct](https://modelscope.cn/models/LLM-Research/Llama-3.2-1B-Instruct)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct)|\n|[LLM-Research/Llama-3.2-3B-Instruct](https://modelscope.cn/models/LLM-Research/Llama-3.2-3B-Instruct)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct)|\n|[LLM-Research/Llama-3.3-70B-Instruct](https://modelscope.cn/models/LLM-Research/Llama-3.3-70B-Instruct)|llama|llama3_2|transformers>=4.43|✔|-|[meta-llama/Llama-3.3-70B-Instruct](https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct)|\n|[unsloth/Llama-3.3-70B-Instruct-bnb-4bit](https://modelscope.cn/models/unsloth/Llama-3.3-70B-Instruct-bnb-4bit)|llama|llama3_2|transformers>=4.43|✘|-|[unsloth/Llama-3.3-70B-Instruct-bnb-4bit](https://huggingface.co/unsloth/Llama-3.3-70B-Instruct-bnb-4bit)|\n|[ZhipuAI/LongWriter-llama3.1-8b](https://modelscope.cn/models/ZhipuAI/LongWriter-llama3.1-8b)|llama|longwriter_llama|transformers>=4.43|✔|-|[zai-org/LongWriter-llama3.1-8b](https://huggingface.co/zai-org/LongWriter-llama3.1-8b)|\n|[deepseek-ai/DeepSeek-R1-Distill-Llama-8B](https://modelscope.cn/models/deepseek-ai/DeepSeek-R1-Distill-Llama-8B)|llama|deepseek_r1|-|✔|-|[deepseek-ai/DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B)|\n|[deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://modelscope.cn/models/deepseek-ai/DeepSeek-R1-Distill-Llama-70B)|llama|deepseek_r1|-|✔|-|[deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B)|\n|[LLM-Research/Reflection-Llama-3.1-70B](https://modelscope.cn/models/LLM-Research/Reflection-Llama-3.1-70B)|llama|reflection|transformers>=4.43|✔|-|[mattshumer/Reflection-Llama-3.1-70B](https://huggingface.co/mattshumer/Reflection-Llama-3.1-70B)|\n|[01ai/Yi-6B](https://modelscope.cn/models/01ai/Yi-6B)|yi|chatml|-|✔|-|[01-ai/Yi-6B](https://huggingface.co/01-ai/Yi-6B)|\n|[01ai/Yi-6B-200K](https://modelscope.cn/models/01ai/Yi-6B-200K)|yi|chatml|-|✔|-|[01-ai/Yi-6B-200K](https://huggingface.co/01-ai/Yi-6B-200K)|\n|[01ai/Yi-6B-Chat](https://modelscope.cn/models/01ai/Yi-6B-Chat)|yi|chatml|-|✔|-|[01-ai/Yi-6B-Chat](https://huggingface.co/01-ai/Yi-6B-Chat)|\n|[01ai/Yi-6B-Chat-4bits](https://modelscope.cn/models/01ai/Yi-6B-Chat-4bits)|yi|chatml|-|✔|-|[01-ai/Yi-6B-Chat-4bits](https://huggingface.co/01-ai/Yi-6B-Chat-4bits)|\n|[01ai/Yi-6B-Chat-8bits](https://modelscope.cn/models/01ai/Yi-6B-Chat-8bits)|yi|chatml|-|✔|-|[01-ai/Yi-6B-Chat-8bits](https://huggingface.co/01-ai/Yi-6B-Chat-8bits)|\n|[01ai/Yi-9B](https://modelscope.cn/models/01ai/Yi-9B)|yi|chatml|-|✔|-|[01-ai/Yi-9B](https://huggingface.co/01-ai/Yi-9B)|\n|[01ai/Yi-9B-200K](https://modelscope.cn/models/01ai/Yi-9B-200K)|yi|chatml|-|✔|-|[01-ai/Yi-9B-200K](https://huggingface.co/01-ai/Yi-9B-200K)|\n|[01ai/Yi-34B](https://modelscope.cn/models/01ai/Yi-34B)|yi|chatml|-|✔|-|[01-ai/Yi-34B](https://huggingface.co/01-ai/Yi-34B)|\n|[01ai/Yi-34B-200K](https://modelscope.cn/models/01ai/Yi-34B-200K)|yi|chatml|-|✔|-|[01-ai/Yi-34B-200K](https://huggingface.co/01-ai/Yi-34B-200K)|\n|[01ai/Yi-34B-Chat](https://modelscope.cn/models/01ai/Yi-34B-Chat)|yi|chatml|-|✔|-|[01-ai/Yi-34B-Chat](https://huggingface.co/01-ai/Yi-34B-Chat)|\n|[01ai/Yi-34B-Chat-4bits](https://modelscope.cn/models/01ai/Yi-34B-Chat-4bits)|yi|chatml|-|✔|-|[01-ai/Yi-34B-Chat-4bits](https://huggingface.co/01-ai/Yi-34B-Chat-4bits)|\n|[01ai/Yi-34B-Chat-8bits](https://modelscope.cn/models/01ai/Yi-34B-Chat-8bits)|yi|chatml|-|✔|-|[01-ai/Yi-34B-Chat-8bits](https://huggingface.co/01-ai/Yi-34B-Chat-8bits)|\n|[01ai/Yi-1.5-6B](https://modelscope.cn/models/01ai/Yi-1.5-6B)|yi|chatml|-|✔|-|[01-ai/Yi-1.5-6B](https://huggingface.co/01-ai/Yi-1.5-6B)|\n|[01ai/Yi-1.5-6B-Chat](https://modelscope.cn/models/01ai/Yi-1.5-6B-Chat)|yi|chatml|-|✔|-|[01-ai/Yi-1.5-6B-Chat](https://huggingface.co/01-ai/Yi-1.5-6B-Chat)|\n|[01ai/Yi-1.5-9B](https://modelscope.cn/models/01ai/Yi-1.5-9B)|yi|chatml|-|✔|-|[01-ai/Yi-1.5-9B](https://huggingface.co/01-ai/Yi-1.5-9B)|\n|[01ai/Yi-1.5-9B-Chat](https://modelscope.cn/models/01ai/Yi-1.5-9B-Chat)|yi|chatml|-|✔|-|[01-ai/Yi-1.5-9B-Chat](https://huggingface.co/01-ai/Yi-1.5-9B-Chat)|\n|[01ai/Yi-1.5-9B-Chat-16K](https://modelscope.cn/models/01ai/Yi-1.5-9B-Chat-16K)|yi|chatml|-|✔|-|[01-ai/Yi-1.5-9B-Chat-16K](https://huggingface.co/01-ai/Yi-1.5-9B-Chat-16K)|\n|[01ai/Yi-1.5-34B](https://modelscope.cn/models/01ai/Yi-1.5-34B)|yi|chatml|-|✔|-|[01-ai/Yi-1.5-34B](https://huggingface.co/01-ai/Yi-1.5-34B)|\n|[01ai/Yi-1.5-34B-Chat](https://modelscope.cn/models/01ai/Yi-1.5-34B-Chat)|yi|chatml|-|✔|-|[01-ai/Yi-1.5-34B-Chat](https://huggingface.co/01-ai/Yi-1.5-34B-Chat)|\n|[01ai/Yi-1.5-34B-Chat-16K](https://modelscope.cn/models/01ai/Yi-1.5-34B-Chat-16K)|yi|chatml|-|✔|-|[01-ai/Yi-1.5-34B-Chat-16K](https://huggingface.co/01-ai/Yi-1.5-34B-Chat-16K)|\n|[AI-ModelScope/Yi-1.5-6B-Chat-GPTQ](https://modelscope.cn/models/AI-ModelScope/Yi-1.5-6B-Chat-GPTQ)|yi|chatml|-|✘|-|[modelscope/Yi-1.5-6B-Chat-GPTQ](https://huggingface.co/modelscope/Yi-1.5-6B-Chat-GPTQ)|\n|[AI-ModelScope/Yi-1.5-6B-Chat-AWQ](https://modelscope.cn/models/AI-ModelScope/Yi-1.5-6B-Chat-AWQ)|yi|chatml|-|✘|-|[modelscope/Yi-1.5-6B-Chat-AWQ](https://huggingface.co/modelscope/Yi-1.5-6B-Chat-AWQ)|\n|[AI-ModelScope/Yi-1.5-9B-Chat-GPTQ](https://modelscope.cn/models/AI-ModelScope/Yi-1.5-9B-Chat-GPTQ)|yi|chatml|-|✘|-|[modelscope/Yi-1.5-9B-Chat-GPTQ](https://huggingface.co/modelscope/Yi-1.5-9B-Chat-GPTQ)|\n|[AI-ModelScope/Yi-1.5-9B-Chat-AWQ](https://modelscope.cn/models/AI-ModelScope/Yi-1.5-9B-Chat-AWQ)|yi|chatml|-|✘|-|[modelscope/Yi-1.5-9B-Chat-AWQ](https://huggingface.co/modelscope/Yi-1.5-9B-Chat-AWQ)|\n|[AI-ModelScope/Yi-1.5-34B-Chat-GPTQ](https://modelscope.cn/models/AI-ModelScope/Yi-1.5-34B-Chat-GPTQ)|yi|chatml|-|✘|-|[modelscope/Yi-1.5-34B-Chat-GPTQ](https://huggingface.co/modelscope/Yi-1.5-34B-Chat-GPTQ)|\n|[AI-ModelScope/Yi-1.5-34B-Chat-AWQ](https://modelscope.cn/models/AI-ModelScope/Yi-1.5-34B-Chat-AWQ)|yi|chatml|-|✘|-|[modelscope/Yi-1.5-34B-Chat-AWQ](https://huggingface.co/modelscope/Yi-1.5-34B-Chat-AWQ)|\n|[01ai/Yi-Coder-1.5B](https://modelscope.cn/models/01ai/Yi-Coder-1.5B)|yi|yi_coder|-|✔|coding|[01-ai/Yi-Coder-1.5B](https://huggingface.co/01-ai/Yi-Coder-1.5B)|\n|[01ai/Yi-Coder-9B](https://modelscope.cn/models/01ai/Yi-Coder-9B)|yi|yi_coder|-|✔|coding|[01-ai/Yi-Coder-9B](https://huggingface.co/01-ai/Yi-Coder-9B)|\n|[01ai/Yi-Coder-1.5B-Chat](https://modelscope.cn/models/01ai/Yi-Coder-1.5B-Chat)|yi|yi_coder|-|✔|coding|[01-ai/Yi-Coder-1.5B-Chat](https://huggingface.co/01-ai/Yi-Coder-1.5B-Chat)|\n|[01ai/Yi-Coder-9B-Chat](https://modelscope.cn/models/01ai/Yi-Coder-9B-Chat)|yi|yi_coder|-|✔|coding|[01-ai/Yi-Coder-9B-Chat](https://huggingface.co/01-ai/Yi-Coder-9B-Chat)|\n|[SUSTC/SUS-Chat-34B](https://modelscope.cn/models/SUSTC/SUS-Chat-34B)|yi|sus|-|✔|-|[SUSTech/SUS-Chat-34B](https://huggingface.co/SUSTech/SUS-Chat-34B)|\n|[openai-mirror/gpt-oss-20b](https://modelscope.cn/models/openai-mirror/gpt-oss-20b)|gpt_oss|gpt_oss|transformers>=4.55|✔|-|[openai/gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b)|\n|[openai-mirror/gpt-oss-120b](https://modelscope.cn/models/openai-mirror/gpt-oss-120b)|gpt_oss|gpt_oss|transformers>=4.55|✔|-|[openai/gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b)|\n|[ByteDance-Seed/Seed-OSS-36B-Instruct](https://modelscope.cn/models/ByteDance-Seed/Seed-OSS-36B-Instruct)|seed_oss|seed_oss|transformers>=4.56|✘|-|[ByteDance-Seed/Seed-OSS-36B-Instruct](https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Instruct)|\n|[ByteDance-Seed/Seed-OSS-36B-Base](https://modelscope.cn/models/ByteDance-Seed/Seed-OSS-36B-Base)|seed_oss|seed_oss|transformers>=4.56|✘|-|[ByteDance-Seed/Seed-OSS-36B-Base](https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Base)|\n|[ByteDance-Seed/Seed-OSS-36B-Base-woSyn](https://modelscope.cn/models/ByteDance-Seed/Seed-OSS-36B-Base-woSyn)|seed_oss|seed_oss|transformers>=4.56|✘|-|[ByteDance-Seed/Seed-OSS-36B-Base-woSyn](https://huggingface.co/ByteDance-Seed/Seed-OSS-36B-Base-woSyn)|\n|[codefuse-ai/CodeFuse-CodeLlama-34B](https://modelscope.cn/models/codefuse-ai/CodeFuse-CodeLlama-34B)|codefuse_codellama|codefuse_codellama|-|✔|coding|[codefuse-ai/CodeFuse-CodeLlama-34B](https://huggingface.co/codefuse-ai/CodeFuse-CodeLlama-34B)|\n|[ZhipuAI/chatglm2-6b](https://modelscope.cn/models/ZhipuAI/chatglm2-6b)|chatglm2|chatglm2|transformers<4.42|✘|-|[zai-org/chatglm2-6b](https://huggingface.co/zai-org/chatglm2-6b)|\n|[ZhipuAI/chatglm2-6b-32k](https://modelscope.cn/models/ZhipuAI/chatglm2-6b-32k)|chatglm2|chatglm2|transformers<4.42|✘|-|[zai-org/chatglm2-6b-32k](https://huggingface.co/zai-org/chatglm2-6b-32k)|\n|[ZhipuAI/codegeex2-6b](https://modelscope.cn/models/ZhipuAI/codegeex2-6b)|chatglm2|chatglm2|transformers<4.34|✘|coding|[zai-org/codegeex2-6b](https://huggingface.co/zai-org/codegeex2-6b)|\n|[ZhipuAI/chatglm3-6b](https://modelscope.cn/models/ZhipuAI/chatglm3-6b)|chatglm3|chatglm4|transformers<4.42|✘|-|[zai-org/chatglm3-6b](https://huggingface.co/zai-org/chatglm3-6b)|\n|[ZhipuAI/chatglm3-6b-base](https://modelscope.cn/models/ZhipuAI/chatglm3-6b-base)|chatglm3|chatglm4|transformers<4.42|✘|-|[zai-org/chatglm3-6b-base](https://huggingface.co/zai-org/chatglm3-6b-base)|\n|[ZhipuAI/chatglm3-6b-32k](https://modelscope.cn/models/ZhipuAI/chatglm3-6b-32k)|chatglm3|chatglm4|transformers<4.42|✘|-|[zai-org/chatglm3-6b-32k](https://huggingface.co/zai-org/chatglm3-6b-32k)|\n|[ZhipuAI/chatglm3-6b-128k](https://modelscope.cn/models/ZhipuAI/chatglm3-6b-128k)|chatglm3|chatglm4|transformers<4.42|✘|-|[zai-org/chatglm3-6b-128k](https://huggingface.co/zai-org/chatglm3-6b-128k)|\n|[ZhipuAI/glm-4-9b-chat](https://modelscope.cn/models/ZhipuAI/glm-4-9b-chat)|chatglm4|chatglm4|transformers>=4.42|✘|-|[zai-org/glm-4-9b-chat](https://huggingface.co/zai-org/glm-4-9b-chat)|\n|[ZhipuAI/glm-4-9b](https://modelscope.cn/models/ZhipuAI/glm-4-9b)|chatglm4|chatglm4|transformers>=4.42|✘|-|[zai-org/glm-4-9b](https://huggingface.co/zai-org/glm-4-9b)|\n|[ZhipuAI/glm-4-9b-chat-1m](https://modelscope.cn/models/ZhipuAI/glm-4-9b-chat-1m)|chatglm4|chatglm4|transformers>=4.42|✘|-|[zai-org/glm-4-9b-chat-1m](https://huggingface.co/zai-org/glm-4-9b-chat-1m)|\n|[ZhipuAI/LongWriter-glm4-9b](https://modelscope.cn/models/ZhipuAI/LongWriter-glm4-9b)|chatglm4|chatglm4|transformers>=4.42|✘|-|[zai-org/LongWriter-glm4-9b](https://huggingface.co/zai-org/LongWriter-glm4-9b)|\n|[ZhipuAI/GLM-4-9B-0414](https://modelscope.cn/models/ZhipuAI/GLM-4-9B-0414)|glm4|glm4|transformers>=4.51|✔|-|[zai-org/GLM-4-9B-0414](https://huggingface.co/zai-org/GLM-4-9B-0414)|\n|[ZhipuAI/GLM-4-32B-0414](https://modelscope.cn/models/ZhipuAI/GLM-4-32B-0414)|glm4|glm4|transformers>=4.51|✔|-|[zai-org/GLM-4-32B-0414](https://huggingface.co/zai-org/GLM-4-32B-0414)|\n|[ZhipuAI/GLM-4-32B-Base-0414](https://modelscope.cn/models/ZhipuAI/GLM-4-32B-Base-0414)|glm4|glm4|transformers>=4.51|✔|-|[zai-org/GLM-4-32B-Base-0414](https://huggingface.co/zai-org/GLM-4-32B-Base-0414)|\n|[ZhipuAI/GLM-Z1-9B-0414](https://modelscope.cn/models/ZhipuAI/GLM-Z1-9B-0414)|glm4|glm4|transformers>=4.51|✔|-|[zai-org/GLM-Z1-9B-0414](https://huggingface.co/zai-org/GLM-Z1-9B-0414)|\n|[ZhipuAI/GLM-Z1-32B-0414](https://modelscope.cn/models/ZhipuAI/GLM-Z1-32B-0414)|glm4|glm4|transformers>=4.51|✔|-|[zai-org/GLM-Z1-32B-0414](https://huggingface.co/zai-org/GLM-Z1-32B-0414)|\n|[ZhipuAI/GLM-Z1-Rumination-32B-0414](https://modelscope.cn/models/ZhipuAI/GLM-Z1-Rumination-32B-0414)|glm4|glm4_z1_rumination|transformers>=4.51|✔|-|[zai-org/GLM-Z1-Rumination-32B-0414](https://huggingface.co/zai-org/GLM-Z1-Rumination-32B-0414)|\n|[ZhipuAI/GLM-4.5-Air-Base](https://modelscope.cn/models/ZhipuAI/GLM-4.5-Air-Base)|glm4_moe|glm4_5|transformers>=4.54|✔|-|[zai-org/GLM-4.5-Air-Base](https://huggingface.co/zai-org/GLM-4.5-Air-Base)|\n|[ZhipuAI/GLM-4.5-Air](https://modelscope.cn/models/ZhipuAI/GLM-4.5-Air)|glm4_moe|glm4_5|transformers>=4.54|✔|-|[zai-org/GLM-4.5-Air](https://huggingface.co/zai-org/GLM-4.5-Air)|\n|[ZhipuAI/GLM-4.5-Air-FP8](https://modelscope.cn/models/ZhipuAI/GLM-4.5-Air-FP8)|glm4_moe|glm4_5|transformers>=4.54|✘|-|[zai-org/GLM-4.5-Air-FP8](https://huggingface.co/zai-org/GLM-4.5-Air-FP8)|\n|[ZhipuAI/GLM-4.5-Base](https://modelscope.cn/models/ZhipuAI/GLM-4.5-Base)|glm4_moe|glm4_5|transformers>=4.54|✔|-|[zai-org/GLM-4.5-Base](https://huggingface.co/zai-org/GLM-4.5-Base)|\n|[ZhipuAI/GLM-4.5](https://modelscope.cn/models/ZhipuAI/GLM-4.5)|glm4_moe|glm4_5|transformers>=4.54|✔|-|[zai-org/GLM-4.5](https://huggingface.co/zai-org/GLM-4.5)|\n|[ZhipuAI/GLM-4.5-FP8](https://modelscope.cn/models/ZhipuAI/GLM-4.5-FP8)|glm4_moe|glm4_5|transformers>=4.54|✘|-|[zai-org/GLM-4.5-FP8](https://huggingface.co/zai-org/GLM-4.5-FP8)|\n|[ZhipuAI/GLM-4.6](https://modelscope.cn/models/ZhipuAI/GLM-4.6)|glm4_moe|glm4_5|transformers>=4.54|✔|-|[zai-org/GLM-4.6](https://huggingface.co/zai-org/GLM-4.6)|\n|[ZhipuAI/GLM-4.6-FP8](https://modelscope.cn/models/ZhipuAI/GLM-4.6-FP8)|glm4_moe|glm4_5|transformers>=4.54|✘|-|[zai-org/GLM-4.6-FP8](https://huggingface.co/zai-org/GLM-4.6-FP8)|\n|[ZhipuAI/GLM-4.7](https://modelscope.cn/models/ZhipuAI/GLM-4.7)|glm4_moe|glm4_7|transformers>=4.54|✔|-|[zai-org/GLM-4.7](https://huggingface.co/zai-org/GLM-4.7)|\n|[ZhipuAI/GLM-4.7-FP8](https://modelscope.cn/models/ZhipuAI/GLM-4.7-FP8)|glm4_moe|glm4_7|transformers>=4.54|✘|-|[zai-org/GLM-4.7-FP8](https://huggingface.co/zai-org/GLM-4.7-FP8)|\n|[ZhipuAI/GLM-4.7-Flash](https://modelscope.cn/models/ZhipuAI/GLM-4.7-Flash)|glm4_moe_lite|glm4_7|transformers>=5.0.0.dev|✔|-|[zai-org/GLM-4.7-Flash](https://huggingface.co/zai-org/GLM-4.7-Flash)|\n|[ZhipuAI/GLM-5](https://modelscope.cn/models/ZhipuAI/GLM-5)|glm_moe_dsa|glm4_7|transformers>=5.2.0|✔|-|[zai-org/GLM-5](https://huggingface.co/zai-org/GLM-5)|\n|[ZhipuAI/glm-edge-1.5b-chat](https://modelscope.cn/models/ZhipuAI/glm-edge-1.5b-chat)|glm_edge|chatglm4|transformers>=4.46|✘|-|[zai-org/glm-edge-1.5b-chat](https://huggingface.co/zai-org/glm-edge-1.5b-chat)|\n|[ZhipuAI/glm-edge-4b-chat](https://modelscope.cn/models/ZhipuAI/glm-edge-4b-chat)|glm_edge|chatglm4|transformers>=4.46|✘|-|[zai-org/glm-edge-4b-chat](https://huggingface.co/zai-org/glm-edge-4b-chat)|\n|[codefuse-ai/CodeFuse-CodeGeeX2-6B](https://modelscope.cn/models/codefuse-ai/CodeFuse-CodeGeeX2-6B)|codefuse_codegeex2|codefuse|transformers<4.34|✘|coding|[codefuse-ai/CodeFuse-CodeGeeX2-6B](https://huggingface.co/codefuse-ai/CodeFuse-CodeGeeX2-6B)|\n|[ZhipuAI/codegeex4-all-9b](https://modelscope.cn/models/ZhipuAI/codegeex4-all-9b)|codegeex4|codegeex4|transformers<4.42|✘|coding|[zai-org/codegeex4-all-9b](https://huggingface.co/zai-org/codegeex4-all-9b)|\n|[Shanghai_AI_Laboratory/internlm-chat-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-chat-7b)|internlm|internlm|-|✘|-|[internlm/internlm-chat-7b](https://huggingface.co/internlm/internlm-chat-7b)|\n|[Shanghai_AI_Laboratory/internlm-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-7b)|internlm|internlm|-|✘|-|[internlm/internlm-7b](https://huggingface.co/internlm/internlm-7b)|\n|[Shanghai_AI_Laboratory/internlm-chat-7b-8k](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-chat-7b-8k)|internlm|internlm|-|✘|-|-|\n|[Shanghai_AI_Laboratory/internlm-20b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-20b)|internlm|internlm|-|✘|-|[internlm/internlm-20b](https://huggingface.co/internlm/internlm-20b)|\n|[Shanghai_AI_Laboratory/internlm-chat-20b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-chat-20b)|internlm|internlm|-|✘|-|[internlm/internlm-chat-20b](https://huggingface.co/internlm/internlm-chat-20b)|\n|[Shanghai_AI_Laboratory/internlm2-chat-1_8b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-chat-1_8b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-chat-1_8b](https://huggingface.co/internlm/internlm2-chat-1_8b)|\n|[Shanghai_AI_Laboratory/internlm2-1_8b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-1_8b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-1_8b](https://huggingface.co/internlm/internlm2-1_8b)|\n|[Shanghai_AI_Laboratory/internlm2-chat-1_8b-sft](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-chat-1_8b-sft)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-chat-1_8b-sft](https://huggingface.co/internlm/internlm2-chat-1_8b-sft)|\n|[Shanghai_AI_Laboratory/internlm2-base-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-base-7b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-base-7b](https://huggingface.co/internlm/internlm2-base-7b)|\n|[Shanghai_AI_Laboratory/internlm2-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-7b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-7b](https://huggingface.co/internlm/internlm2-7b)|\n|[Shanghai_AI_Laboratory/internlm2-chat-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-chat-7b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-chat-7b](https://huggingface.co/internlm/internlm2-chat-7b)|\n|[Shanghai_AI_Laboratory/internlm2-chat-7b-sft](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-chat-7b-sft)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-chat-7b-sft](https://huggingface.co/internlm/internlm2-chat-7b-sft)|\n|[Shanghai_AI_Laboratory/internlm2-base-20b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-base-20b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-base-20b](https://huggingface.co/internlm/internlm2-base-20b)|\n|[Shanghai_AI_Laboratory/internlm2-20b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-20b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-20b](https://huggingface.co/internlm/internlm2-20b)|\n|[Shanghai_AI_Laboratory/internlm2-chat-20b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-chat-20b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-chat-20b](https://huggingface.co/internlm/internlm2-chat-20b)|\n|[Shanghai_AI_Laboratory/internlm2-chat-20b-sft](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-chat-20b-sft)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2-chat-20b-sft](https://huggingface.co/internlm/internlm2-chat-20b-sft)|\n|[Shanghai_AI_Laboratory/internlm2-math-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-math-7b)|internlm2|internlm2|transformers>=4.38|✘|math|[internlm/internlm2-math-7b](https://huggingface.co/internlm/internlm2-math-7b)|\n|[Shanghai_AI_Laboratory/internlm2-math-base-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-math-base-7b)|internlm2|internlm2|transformers>=4.38|✘|math|[internlm/internlm2-math-base-7b](https://huggingface.co/internlm/internlm2-math-base-7b)|\n|[Shanghai_AI_Laboratory/internlm2-math-base-20b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-math-base-20b)|internlm2|internlm2|transformers>=4.38|✘|math|[internlm/internlm2-math-base-20b](https://huggingface.co/internlm/internlm2-math-base-20b)|\n|[Shanghai_AI_Laboratory/internlm2-math-20b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-math-20b)|internlm2|internlm2|transformers>=4.38|✘|math|[internlm/internlm2-math-20b](https://huggingface.co/internlm/internlm2-math-20b)|\n|[Shanghai_AI_Laboratory/internlm2_5-1_8b-chat](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2_5-1_8b-chat)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2_5-1_8b-chat](https://huggingface.co/internlm/internlm2_5-1_8b-chat)|\n|[Shanghai_AI_Laboratory/internlm2_5-1_8b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2_5-1_8b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2_5-1_8b](https://huggingface.co/internlm/internlm2_5-1_8b)|\n|[Shanghai_AI_Laboratory/internlm2_5-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2_5-7b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2_5-7b](https://huggingface.co/internlm/internlm2_5-7b)|\n|[Shanghai_AI_Laboratory/internlm2_5-7b-chat](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2_5-7b-chat)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2_5-7b-chat](https://huggingface.co/internlm/internlm2_5-7b-chat)|\n|[Shanghai_AI_Laboratory/internlm2_5-7b-chat-1m](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2_5-7b-chat-1m)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2_5-7b-chat-1m](https://huggingface.co/internlm/internlm2_5-7b-chat-1m)|\n|[Shanghai_AI_Laboratory/internlm2_5-20b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2_5-20b)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2_5-20b](https://huggingface.co/internlm/internlm2_5-20b)|\n|[Shanghai_AI_Laboratory/internlm2_5-20b-chat](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2_5-20b-chat)|internlm2|internlm2|transformers>=4.38|✘|-|[internlm/internlm2_5-20b-chat](https://huggingface.co/internlm/internlm2_5-20b-chat)|\n|[Shanghai_AI_Laboratory/internlm3-8b-instruct](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm3-8b-instruct)|internlm3|internlm2|transformers>=4.48|✔|-|[internlm/internlm3-8b-instruct](https://huggingface.co/internlm/internlm3-8b-instruct)|\n|[deepseek-ai/deepseek-moe-16b-chat](https://modelscope.cn/models/deepseek-ai/deepseek-moe-16b-chat)|deepseek|deepseek|-|✔|-|[deepseek-ai/deepseek-moe-16b-chat](https://huggingface.co/deepseek-ai/deepseek-moe-16b-chat)|\n|[deepseek-ai/deepseek-moe-16b-base](https://modelscope.cn/models/deepseek-ai/deepseek-moe-16b-base)|deepseek|deepseek|-|✔|-|[deepseek-ai/deepseek-moe-16b-base](https://huggingface.co/deepseek-ai/deepseek-moe-16b-base)|\n|[deepseek-ai/DeepSeek-Coder-V2-Instruct](https://modelscope.cn/models/deepseek-ai/DeepSeek-Coder-V2-Instruct)|deepseek_v2|deepseek|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-Coder-V2-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct)|\n|[deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct](https://modelscope.cn/models/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct)|deepseek_v2|deepseek|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct)|\n|[deepseek-ai/DeepSeek-Coder-V2-Base](https://modelscope.cn/models/deepseek-ai/DeepSeek-Coder-V2-Base)|deepseek_v2|deepseek|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-Coder-V2-Base](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Base)|\n|[deepseek-ai/DeepSeek-Coder-V2-Lite-Base](https://modelscope.cn/models/deepseek-ai/DeepSeek-Coder-V2-Lite-Base)|deepseek_v2|deepseek|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-Coder-V2-Lite-Base](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Base)|\n|[deepseek-ai/DeepSeek-V2-Lite](https://modelscope.cn/models/deepseek-ai/DeepSeek-V2-Lite)|deepseek_v2|deepseek|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V2-Lite](https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite)|\n|[deepseek-ai/DeepSeek-V2-Lite-Chat](https://modelscope.cn/models/deepseek-ai/DeepSeek-V2-Lite-Chat)|deepseek_v2|deepseek|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V2-Lite-Chat](https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat)|\n|[deepseek-ai/DeepSeek-V2](https://modelscope.cn/models/deepseek-ai/DeepSeek-V2)|deepseek_v2|deepseek|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V2](https://huggingface.co/deepseek-ai/DeepSeek-V2)|\n|[deepseek-ai/DeepSeek-V2-Chat](https://modelscope.cn/models/deepseek-ai/DeepSeek-V2-Chat)|deepseek_v2|deepseek|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V2-Chat](https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat)|\n|[deepseek-ai/DeepSeek-V2.5](https://modelscope.cn/models/deepseek-ai/DeepSeek-V2.5)|deepseek_v2|deepseek_v2_5|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V2.5](https://huggingface.co/deepseek-ai/DeepSeek-V2.5)|\n|[deepseek-ai/DeepSeek-V2.5-1210](https://modelscope.cn/models/deepseek-ai/DeepSeek-V2.5-1210)|deepseek_v2|deepseek_v2_5|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V2.5-1210](https://huggingface.co/deepseek-ai/DeepSeek-V2.5-1210)|\n|[deepseek-ai/DeepSeek-V3-Base](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3-Base)|deepseek_v3|deepseek_v2_5|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3-Base](https://huggingface.co/deepseek-ai/DeepSeek-V3-Base)|\n|[deepseek-ai/DeepSeek-V3](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3)|deepseek_v3|deepseek_v2_5|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3](https://huggingface.co/deepseek-ai/DeepSeek-V3)|\n|[deepseek-ai/DeepSeek-V3-0324](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3-0324)|deepseek_v3|deepseek_v2_5|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3-0324](https://huggingface.co/deepseek-ai/DeepSeek-V3-0324)|\n|[cognitivecomputations/DeepSeek-V3-awq](https://modelscope.cn/models/cognitivecomputations/DeepSeek-V3-awq)|deepseek_v3|deepseek_v2_5|transformers>=4.39.3|✘|-|[cognitivecomputations/DeepSeek-V3-AWQ](https://huggingface.co/cognitivecomputations/DeepSeek-V3-AWQ)|\n|[cognitivecomputations/DeepSeek-V3-0324-AWQ](https://modelscope.cn/models/cognitivecomputations/DeepSeek-V3-0324-AWQ)|deepseek_v3|deepseek_v2_5|transformers>=4.39.3|✘|-|[cognitivecomputations/DeepSeek-V3-0324-AWQ](https://huggingface.co/cognitivecomputations/DeepSeek-V3-0324-AWQ)|\n|[deepseek-ai/DeepSeek-Prover-V2-7B](https://modelscope.cn/models/deepseek-ai/DeepSeek-Prover-V2-7B)|deepseek_v3|deepseek_v2_5|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-Prover-V2-7B](https://huggingface.co/deepseek-ai/DeepSeek-Prover-V2-7B)|\n|[deepseek-ai/DeepSeek-Prover-V2-671B](https://modelscope.cn/models/deepseek-ai/DeepSeek-Prover-V2-671B)|deepseek_v3|deepseek_v2_5|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-Prover-V2-671B](https://huggingface.co/deepseek-ai/DeepSeek-Prover-V2-671B)|\n|[unsloth/DeepSeek-V3-bf16](https://modelscope.cn/models/unsloth/DeepSeek-V3-bf16)|deepseek_v3|deepseek_v2_5|transformers>=4.39.3|✔|-|[unsloth/DeepSeek-V3-bf16](https://huggingface.co/unsloth/DeepSeek-V3-bf16)|\n|[unsloth/DeepSeek-V3-0324-BF16](https://modelscope.cn/models/unsloth/DeepSeek-V3-0324-BF16)|deepseek_v3|deepseek_v2_5|transformers>=4.39.3|✔|-|[unsloth/DeepSeek-V3-0324-BF16](https://huggingface.co/unsloth/DeepSeek-V3-0324-BF16)|\n|[unsloth/DeepSeek-Prover-V2-671B-BF16](https://modelscope.cn/models/unsloth/DeepSeek-Prover-V2-671B-BF16)|deepseek_v3|deepseek_v2_5|transformers>=4.39.3|✔|-|[unsloth/DeepSeek-Prover-V2-671B-BF16](https://huggingface.co/unsloth/DeepSeek-Prover-V2-671B-BF16)|\n|[deepseek-ai/DeepSeek-R1](https://modelscope.cn/models/deepseek-ai/DeepSeek-R1)|deepseek_v3|deepseek_r1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1)|\n|[deepseek-ai/DeepSeek-R1-Zero](https://modelscope.cn/models/deepseek-ai/DeepSeek-R1-Zero)|deepseek_v3|deepseek_r1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-R1-Zero](https://huggingface.co/deepseek-ai/DeepSeek-R1-Zero)|\n|[deepseek-ai/DeepSeek-R1-0528](https://modelscope.cn/models/deepseek-ai/DeepSeek-R1-0528)|deepseek_v3|deepseek_r1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-R1-0528](https://huggingface.co/deepseek-ai/DeepSeek-R1-0528)|\n|[cognitivecomputations/DeepSeek-R1-awq](https://modelscope.cn/models/cognitivecomputations/DeepSeek-R1-awq)|deepseek_v3|deepseek_r1|transformers>=4.39.3|✘|-|[cognitivecomputations/DeepSeek-R1-AWQ](https://huggingface.co/cognitivecomputations/DeepSeek-R1-AWQ)|\n|[cognitivecomputations/DeepSeek-R1-0528-AWQ](https://modelscope.cn/models/cognitivecomputations/DeepSeek-R1-0528-AWQ)|deepseek_v3|deepseek_r1|transformers>=4.39.3|✘|-|[cognitivecomputations/DeepSeek-R1-0528-AWQ](https://huggingface.co/cognitivecomputations/DeepSeek-R1-0528-AWQ)|\n|[unsloth/DeepSeek-R1-BF16](https://modelscope.cn/models/unsloth/DeepSeek-R1-BF16)|deepseek_v3|deepseek_r1|transformers>=4.39.3|✔|-|[unsloth/DeepSeek-R1-BF16](https://huggingface.co/unsloth/DeepSeek-R1-BF16)|\n|[unsloth/DeepSeek-R1-Zero-BF16](https://modelscope.cn/models/unsloth/DeepSeek-R1-Zero-BF16)|deepseek_v3|deepseek_r1|transformers>=4.39.3|✔|-|[unsloth/DeepSeek-R1-Zero-BF16](https://huggingface.co/unsloth/DeepSeek-R1-Zero-BF16)|\n|[unsloth/DeepSeek-R1-0528-BF16](https://modelscope.cn/models/unsloth/DeepSeek-R1-0528-BF16)|deepseek_v3|deepseek_r1|transformers>=4.39.3|✔|-|[unsloth/DeepSeek-R1-0528-BF16](https://huggingface.co/unsloth/DeepSeek-R1-0528-BF16)|\n|[moonshotai/Moonlight-16B-A3B](https://modelscope.cn/models/moonshotai/Moonlight-16B-A3B)|deepseek_v3|moonlight|transformers<4.49|✔|-|[moonshotai/Moonlight-16B-A3B](https://huggingface.co/moonshotai/Moonlight-16B-A3B)|\n|[moonshotai/Moonlight-16B-A3B-Instruct](https://modelscope.cn/models/moonshotai/Moonlight-16B-A3B-Instruct)|deepseek_v3|moonlight|transformers<4.49|✔|-|[moonshotai/Moonlight-16B-A3B-Instruct](https://huggingface.co/moonshotai/Moonlight-16B-A3B-Instruct)|\n|[moonshotai/Kimi-K2-Base](https://modelscope.cn/models/moonshotai/Kimi-K2-Base)|deepseek_v3|kimi_k2|transformers>=4.39.3|✔|-|[moonshotai/Kimi-K2-Base](https://huggingface.co/moonshotai/Kimi-K2-Base)|\n|[moonshotai/Kimi-K2-Instruct](https://modelscope.cn/models/moonshotai/Kimi-K2-Instruct)|deepseek_v3|kimi_k2|transformers>=4.39.3|✔|-|[moonshotai/Kimi-K2-Instruct](https://huggingface.co/moonshotai/Kimi-K2-Instruct)|\n|[moonshotai/Kimi-K2-Instruct-0905](https://modelscope.cn/models/moonshotai/Kimi-K2-Instruct-0905)|deepseek_v3|kimi_k2|transformers>=4.39.3|✔|-|[moonshotai/Kimi-K2-Instruct-0905](https://huggingface.co/moonshotai/Kimi-K2-Instruct-0905)|\n|[moonshotai/Kimi-K2-Thinking](https://modelscope.cn/models/moonshotai/Kimi-K2-Thinking)|deepseek_v3|kimi_k2|transformers>=4.39.3|✔|-|[moonshotai/Kimi-K2-Thinking](https://huggingface.co/moonshotai/Kimi-K2-Thinking)|\n|[deepseek-ai/DeepSeek-V3.1-Base](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.1-Base)|deepseek_v3|deepseek_v3_1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3.1-Base](https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Base)|\n|[deepseek-ai/DeepSeek-V3.1](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.1)|deepseek_v3|deepseek_v3_1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3.1](https://huggingface.co/deepseek-ai/DeepSeek-V3.1)|\n|[deepseek-ai/DeepSeek-V3.1-Terminus](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.1-Terminus)|deepseek_v3|deepseek_v3_1|transformers>=4.39.3|✔|-|[deepseek-ai/DeepSeek-V3.1-Terminus](https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus)|\n|[deepseek-ai/DeepSeek-V3.2](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.2)|deepseek_v32|deepseek_v3_1|-|✔|-|[deepseek-ai/DeepSeek-V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2)|\n|[deepseek-ai/DeepSeek-V3.2-Speciale](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.2-Speciale)|deepseek_v32|deepseek_v3_1|-|✔|-|[deepseek-ai/DeepSeek-V3.2-Speciale](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Speciale)|\n|[deepseek-ai/DeepSeek-V3.2-Exp](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.2-Exp)|deepseek_v32|deepseek_v3_1|-|✔|-|[deepseek-ai/DeepSeek-V3.2-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp)|\n|[deepseek-ai/DeepSeek-V3.2-Exp-Base](https://modelscope.cn/models/deepseek-ai/DeepSeek-V3.2-Exp-Base)|deepseek_v32|deepseek_v3_1|-|✔|-|[deepseek-ai/DeepSeek-V3.2-Exp-Base](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp-Base)|\n|[deepseek-ai/DeepSeek-Math-V2](https://modelscope.cn/models/deepseek-ai/DeepSeek-Math-V2)|deepseek_v32|deepseek_v3_1|-|✔|-|[deepseek-ai/DeepSeek-Math-V2](https://huggingface.co/deepseek-ai/DeepSeek-Math-V2)|\n|[OpenBuddy/openbuddy-llama-65b-v8-bf16](https://modelscope.cn/models/OpenBuddy/openbuddy-llama-65b-v8-bf16)|openbuddy_llama|openbuddy|-|✔|-|[OpenBuddy/openbuddy-llama-65b-v8-bf16](https://huggingface.co/OpenBuddy/openbuddy-llama-65b-v8-bf16)|\n|[OpenBuddy/openbuddy-llama2-13b-v8.1-fp16](https://modelscope.cn/models/OpenBuddy/openbuddy-llama2-13b-v8.1-fp16)|openbuddy_llama|openbuddy|-|✔|-|[OpenBuddy/openbuddy-llama2-13b-v8.1-fp16](https://huggingface.co/OpenBuddy/openbuddy-llama2-13b-v8.1-fp16)|\n|[OpenBuddy/openbuddy-llama2-70b-v10.1-bf16](https://modelscope.cn/models/OpenBuddy/openbuddy-llama2-70b-v10.1-bf16)|openbuddy_llama|openbuddy|-|✔|-|[OpenBuddy/openbuddy-llama2-70b-v10.1-bf16](https://huggingface.co/OpenBuddy/openbuddy-llama2-70b-v10.1-bf16)|\n|[OpenBuddy/openbuddy-deepseek-67b-v15.2](https://modelscope.cn/models/OpenBuddy/openbuddy-deepseek-67b-v15.2)|openbuddy_llama|openbuddy|-|✔|-|[OpenBuddy/openbuddy-deepseek-67b-v15.2](https://huggingface.co/OpenBuddy/openbuddy-deepseek-67b-v15.2)|\n|[OpenBuddy/openbuddy-llama3-8b-v21.1-8k](https://modelscope.cn/models/OpenBuddy/openbuddy-llama3-8b-v21.1-8k)|openbuddy_llama|openbuddy2|-|✔|-|[OpenBuddy/openbuddy-llama3-8b-v21.1-8k](https://huggingface.co/OpenBuddy/openbuddy-llama3-8b-v21.1-8k)|\n|[OpenBuddy/openbuddy-llama3-70b-v21.1-8k](https://modelscope.cn/models/OpenBuddy/openbuddy-llama3-70b-v21.1-8k)|openbuddy_llama|openbuddy2|-|✔|-|[OpenBuddy/openbuddy-llama3-70b-v21.1-8k](https://huggingface.co/OpenBuddy/openbuddy-llama3-70b-v21.1-8k)|\n|[OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k](https://modelscope.cn/models/OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k)|openbuddy_llama|openbuddy2|-|✔|-|[OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k](https://huggingface.co/OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k)|\n|[OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k](https://modelscope.cn/models/OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k)|openbuddy_llama|openbuddy2|transformers>=4.43|✔|-|[OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k](https://huggingface.co/OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k)|\n|[OpenBuddy/openbuddy-nemotron-70b-v23.2-131k](https://modelscope.cn/models/OpenBuddy/openbuddy-nemotron-70b-v23.2-131k)|openbuddy_llama|openbuddy2|transformers>=4.43|✔|-|[OpenBuddy/openbuddy-nemotron-70b-v23.2-131k](https://huggingface.co/OpenBuddy/openbuddy-nemotron-70b-v23.2-131k)|\n|[OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k](https://modelscope.cn/models/OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k)|openbuddy_llama|openbuddy2|transformers>=4.45|✔|-|[OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k](https://huggingface.co/OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k)|\n|[OpenBuddy/openbuddy-mistral-7b-v17.1-32k](https://modelscope.cn/models/OpenBuddy/openbuddy-mistral-7b-v17.1-32k)|openbuddy_mistral|openbuddy|transformers>=4.34|✘|-|[OpenBuddy/openbuddy-mistral-7b-v17.1-32k](https://huggingface.co/OpenBuddy/openbuddy-mistral-7b-v17.1-32k)|\n|[OpenBuddy/openbuddy-zephyr-7b-v14.1](https://modelscope.cn/models/OpenBuddy/openbuddy-zephyr-7b-v14.1)|openbuddy_mistral|openbuddy|transformers>=4.34|✘|-|[OpenBuddy/openbuddy-zephyr-7b-v14.1](https://huggingface.co/OpenBuddy/openbuddy-zephyr-7b-v14.1)|\n|[OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k](https://modelscope.cn/models/OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k)|openbuddy_mixtral|openbuddy|transformers>=4.36|✘|-|[OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k](https://huggingface.co/OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k)|\n|[baichuan-inc/Baichuan-13B-Chat](https://modelscope.cn/models/baichuan-inc/Baichuan-13B-Chat)|baichuan|baichuan|transformers<4.34|✘|-|[baichuan-inc/Baichuan-13B-Chat](https://huggingface.co/baichuan-inc/Baichuan-13B-Chat)|\n|[baichuan-inc/Baichuan-13B-Base](https://modelscope.cn/models/baichuan-inc/Baichuan-13B-Base)|baichuan|baichuan|transformers<4.34|✘|-|[baichuan-inc/Baichuan-13B-Base](https://huggingface.co/baichuan-inc/Baichuan-13B-Base)|\n|[baichuan-inc/baichuan-7B](https://modelscope.cn/models/baichuan-inc/baichuan-7B)|baichuan|baichuan|transformers<4.34|✘|-|[baichuan-inc/Baichuan-7B](https://huggingface.co/baichuan-inc/Baichuan-7B)|\n|[baichuan-inc/Baichuan2-7B-Chat](https://modelscope.cn/models/baichuan-inc/Baichuan2-7B-Chat)|baichuan2|baichuan|-|✘|-|[baichuan-inc/Baichuan2-7B-Chat](https://huggingface.co/baichuan-inc/Baichuan2-7B-Chat)|\n|[baichuan-inc/Baichuan2-7B-Base](https://modelscope.cn/models/baichuan-inc/Baichuan2-7B-Base)|baichuan2|baichuan|-|✘|-|[baichuan-inc/Baichuan2-7B-Base](https://huggingface.co/baichuan-inc/Baichuan2-7B-Base)|\n|[baichuan-inc/Baichuan2-13B-Chat](https://modelscope.cn/models/baichuan-inc/Baichuan2-13B-Chat)|baichuan2|baichuan|-|✘|-|[baichuan-inc/Baichuan2-13B-Chat](https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat)|\n|[baichuan-inc/Baichuan2-13B-Base](https://modelscope.cn/models/baichuan-inc/Baichuan2-13B-Base)|baichuan2|baichuan|-|✘|-|[baichuan-inc/Baichuan2-13B-Base](https://huggingface.co/baichuan-inc/Baichuan2-13B-Base)|\n|[baichuan-inc/Baichuan2-7B-Chat-4bits](https://modelscope.cn/models/baichuan-inc/Baichuan2-7B-Chat-4bits)|baichuan2|baichuan|bitsandbytes<0.41.2, accelerate<0.26|✘|-|[baichuan-inc/Baichuan2-7B-Chat-4bits](https://huggingface.co/baichuan-inc/Baichuan2-7B-Chat-4bits)|\n|[baichuan-inc/Baichuan2-13B-Chat-4bits](https://modelscope.cn/models/baichuan-inc/Baichuan2-13B-Chat-4bits)|baichuan2|baichuan|bitsandbytes<0.41.2, accelerate<0.26|✘|-|[baichuan-inc/Baichuan2-13B-Chat-4bits](https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat-4bits)|\n|[baichuan-inc/Baichuan-M1-14B-Instruct](https://modelscope.cn/models/baichuan-inc/Baichuan-M1-14B-Instruct)|baichuan_m1|baichuan_m1|transformers>=4.48|✘|-|[baichuan-inc/Baichuan-M1-14B-Instruct](https://huggingface.co/baichuan-inc/Baichuan-M1-14B-Instruct)|\n|[OpenBMB/MiniCPM-2B-sft-fp32](https://modelscope.cn/models/OpenBMB/MiniCPM-2B-sft-fp32)|minicpm|minicpm|transformers>=4.36.0|✘|-|[openbmb/MiniCPM-2B-sft-fp32](https://huggingface.co/openbmb/MiniCPM-2B-sft-fp32)|\n|[OpenBMB/MiniCPM-2B-dpo-fp32](https://modelscope.cn/models/OpenBMB/MiniCPM-2B-dpo-fp32)|minicpm|minicpm|transformers>=4.36.0|✘|-|[openbmb/MiniCPM-2B-dpo-fp32](https://huggingface.co/openbmb/MiniCPM-2B-dpo-fp32)|\n|[OpenBMB/MiniCPM-1B-sft-bf16](https://modelscope.cn/models/OpenBMB/MiniCPM-1B-sft-bf16)|minicpm|minicpm|transformers>=4.36.0|✘|-|[openbmb/MiniCPM-1B-sft-bf16](https://huggingface.co/openbmb/MiniCPM-1B-sft-bf16)|\n|[OpenBMB/MiniCPM-2B-128k](https://modelscope.cn/models/OpenBMB/MiniCPM-2B-128k)|minicpm_chatml|chatml|transformers>=4.36|✘|-|[openbmb/MiniCPM-2B-128k](https://huggingface.co/openbmb/MiniCPM-2B-128k)|\n|[OpenBMB/MiniCPM4-0.5B](https://modelscope.cn/models/OpenBMB/MiniCPM4-0.5B)|minicpm_chatml|chatml|transformers>=4.36|✘|-|[openbmb/MiniCPM4-0.5B](https://huggingface.co/openbmb/MiniCPM4-0.5B)|\n|[OpenBMB/MiniCPM4-8B](https://modelscope.cn/models/OpenBMB/MiniCPM4-8B)|minicpm_chatml|chatml|transformers>=4.36|✘|-|[openbmb/MiniCPM4-8B](https://huggingface.co/openbmb/MiniCPM4-8B)|\n|[OpenBMB/MiniCPM3-4B](https://modelscope.cn/models/OpenBMB/MiniCPM3-4B)|minicpm3|chatml|transformers>=4.36|✘|-|[openbmb/MiniCPM3-4B](https://huggingface.co/openbmb/MiniCPM3-4B)|\n|[OpenBMB/MiniCPM-MoE-8x2B](https://modelscope.cn/models/OpenBMB/MiniCPM-MoE-8x2B)|minicpm_moe|minicpm|transformers>=4.36|✘|-|[openbmb/MiniCPM-MoE-8x2B](https://huggingface.co/openbmb/MiniCPM-MoE-8x2B)|\n|[TeleAI/TeleChat-7B](https://modelscope.cn/models/TeleAI/TeleChat-7B)|telechat|telechat|-|✘|-|[Tele-AI/telechat-7B](https://huggingface.co/Tele-AI/telechat-7B)|\n|[TeleAI/TeleChat-12B](https://modelscope.cn/models/TeleAI/TeleChat-12B)|telechat|telechat|-|✘|-|[Tele-AI/TeleChat-12B](https://huggingface.co/Tele-AI/TeleChat-12B)|\n|[TeleAI/TeleChat-12B-v2](https://modelscope.cn/models/TeleAI/TeleChat-12B-v2)|telechat|telechat|-|✘|-|[Tele-AI/TeleChat-12B-v2](https://huggingface.co/Tele-AI/TeleChat-12B-v2)|\n|[TeleAI/TeleChat-52B](https://modelscope.cn/models/TeleAI/TeleChat-52B)|telechat|telechat|-|✘|-|[TeleAI/TeleChat-52B](https://huggingface.co/TeleAI/TeleChat-52B)|\n|[swift/TeleChat-12B-V2-GPTQ-Int4](https://modelscope.cn/models/swift/TeleChat-12B-V2-GPTQ-Int4)|telechat|telechat|-|✘|-|-|\n|[TeleAI/TeleChat2-35B](https://modelscope.cn/models/TeleAI/TeleChat2-35B)|telechat|telechat|-|✘|-|[Tele-AI/TeleChat2-35B](https://huggingface.co/Tele-AI/TeleChat2-35B)|\n|[TeleAI/TeleChat2-115B](https://modelscope.cn/models/TeleAI/TeleChat2-115B)|telechat|telechat|-|✘|-|[Tele-AI/TeleChat2-115B](https://huggingface.co/Tele-AI/TeleChat2-115B)|\n|[TeleAI/TeleChat2-3B](https://modelscope.cn/models/TeleAI/TeleChat2-3B)|telechat2|telechat2|-|✘|-|[Tele-AI/TeleChat2-3B](https://huggingface.co/Tele-AI/TeleChat2-3B)|\n|[TeleAI/TeleChat2-7B-32K](https://modelscope.cn/models/TeleAI/TeleChat2-7B-32K)|telechat2|telechat2|-|✘|-|[Tele-AI/TeleChat2-7B-32K](https://huggingface.co/Tele-AI/TeleChat2-7B-32K)|\n|[TeleAI/TeleChat2-35B-32K](https://modelscope.cn/models/TeleAI/TeleChat2-35B-32K)|telechat2|telechat2|-|✘|-|[Tele-AI/TeleChat2-35B-32K](https://huggingface.co/Tele-AI/TeleChat2-35B-32K)|\n|[TeleAI/TeleChat2-35B-Nov](https://modelscope.cn/models/TeleAI/TeleChat2-35B-Nov)|telechat2|telechat2|-|✘|-|[Tele-AI/TeleChat2-35B-Nov](https://huggingface.co/Tele-AI/TeleChat2-35B-Nov)|\n|[AI-ModelScope/Mistral-7B-Instruct-v0.1](https://modelscope.cn/models/AI-ModelScope/Mistral-7B-Instruct-v0.1)|mistral|llama|transformers>=4.34|✘|-|[mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1)|\n|[AI-ModelScope/Mistral-7B-Instruct-v0.2](https://modelscope.cn/models/AI-ModelScope/Mistral-7B-Instruct-v0.2)|mistral|llama|transformers>=4.34|✘|-|[mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2)|\n|[LLM-Research/Mistral-7B-Instruct-v0.3](https://modelscope.cn/models/LLM-Research/Mistral-7B-Instruct-v0.3)|mistral|llama|transformers>=4.34|✘|-|[mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3)|\n|[AI-ModelScope/Mistral-7B-v0.1](https://modelscope.cn/models/AI-ModelScope/Mistral-7B-v0.1)|mistral|llama|transformers>=4.34|✘|-|[mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1)|\n|[AI-ModelScope/Mistral-7B-v0.2-hf](https://modelscope.cn/models/AI-ModelScope/Mistral-7B-v0.2-hf)|mistral|llama|transformers>=4.34|✘|-|[alpindale/Mistral-7B-v0.2-hf](https://huggingface.co/alpindale/Mistral-7B-v0.2-hf)|\n|[swift/Codestral-22B-v0.1](https://modelscope.cn/models/swift/Codestral-22B-v0.1)|mistral|llama|transformers>=4.34|✘|-|[mistralai/Codestral-22B-v0.1](https://huggingface.co/mistralai/Codestral-22B-v0.1)|\n|[mistralai/Devstral-Small-2505](https://modelscope.cn/models/mistralai/Devstral-Small-2505)|devstral|devstral|transformers>=4.43, mistral-common>=1.5.5|✘|-|[mistralai/Devstral-Small-2505](https://huggingface.co/mistralai/Devstral-Small-2505)|\n|[modelscope/zephyr-7b-beta](https://modelscope.cn/models/modelscope/zephyr-7b-beta)|zephyr|zephyr|transformers>=4.34|✘|-|[HuggingFaceH4/zephyr-7b-beta](https://huggingface.co/HuggingFaceH4/zephyr-7b-beta)|\n|[AI-ModelScope/Mixtral-8x7B-Instruct-v0.1](https://modelscope.cn/models/AI-ModelScope/Mixtral-8x7B-Instruct-v0.1)|mixtral|llama|transformers>=4.36|✘|-|[mistralai/Mixtral-8x7B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1)|\n|[AI-ModelScope/Mixtral-8x7B-v0.1](https://modelscope.cn/models/AI-ModelScope/Mixtral-8x7B-v0.1)|mixtral|llama|transformers>=4.36|✘|-|[mistralai/Mixtral-8x7B-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-v0.1)|\n|[AI-ModelScope/Mixtral-8x22B-v0.1](https://modelscope.cn/models/AI-ModelScope/Mixtral-8x22B-v0.1)|mixtral|llama|transformers>=4.36|✘|-|[mistral-community/Mixtral-8x22B-v0.1](https://huggingface.co/mistral-community/Mixtral-8x22B-v0.1)|\n|[AI-ModelScope/Mixtral-8x7b-AQLM-2Bit-1x16-hf](https://modelscope.cn/models/AI-ModelScope/Mixtral-8x7b-AQLM-2Bit-1x16-hf)|mixtral|llama|transformers>=4.38, aqlm, torch>=2.2.0|✘|-|[ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf](https://huggingface.co/ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf)|\n|[AI-ModelScope/Mistral-Small-Instruct-2409](https://modelscope.cn/models/AI-ModelScope/Mistral-Small-Instruct-2409)|mistral_nemo|mistral_nemo|transformers>=4.43|✘|-|[mistralai/Mistral-Small-Instruct-2409](https://huggingface.co/mistralai/Mistral-Small-Instruct-2409)|\n|[LLM-Research/Mistral-Large-Instruct-2407](https://modelscope.cn/models/LLM-Research/Mistral-Large-Instruct-2407)|mistral_nemo|mistral_nemo|transformers>=4.43|✘|-|[mistralai/Mistral-Large-Instruct-2407](https://huggingface.co/mistralai/Mistral-Large-Instruct-2407)|\n|[AI-ModelScope/Mistral-Nemo-Base-2407](https://modelscope.cn/models/AI-ModelScope/Mistral-Nemo-Base-2407)|mistral_nemo|mistral_nemo|transformers>=4.43|✘|-|[mistralai/Mistral-Nemo-Base-2407](https://huggingface.co/mistralai/Mistral-Nemo-Base-2407)|\n|[AI-ModelScope/Mistral-Nemo-Instruct-2407](https://modelscope.cn/models/AI-ModelScope/Mistral-Nemo-Instruct-2407)|mistral_nemo|mistral_nemo|transformers>=4.43|✘|-|[mistralai/Mistral-Nemo-Instruct-2407](https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407)|\n|[AI-ModelScope/Ministral-8B-Instruct-2410](https://modelscope.cn/models/AI-ModelScope/Ministral-8B-Instruct-2410)|mistral_nemo|mistral_nemo|transformers>=4.46|✘|-|[mistralai/Ministral-8B-Instruct-2410](https://huggingface.co/mistralai/Ministral-8B-Instruct-2410)|\n|[mistralai/Mistral-Small-24B-Base-2501](https://modelscope.cn/models/mistralai/Mistral-Small-24B-Base-2501)|mistral_2501|mistral_2501|-|✘|-|[mistralai/Mistral-Small-24B-Base-2501](https://huggingface.co/mistralai/Mistral-Small-24B-Base-2501)|\n|[mistralai/Mistral-Small-24B-Instruct-2501](https://modelscope.cn/models/mistralai/Mistral-Small-24B-Instruct-2501)|mistral_2501|mistral_2501|-|✘|-|[mistralai/Mistral-Small-24B-Instruct-2501](https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501)|\n|[AI-ModelScope/WizardLM-2-7B-AWQ](https://modelscope.cn/models/AI-ModelScope/WizardLM-2-7B-AWQ)|wizardlm2|wizardlm2|transformers>=4.34|✘|-|[MaziyarPanahi/WizardLM-2-7B-AWQ](https://huggingface.co/MaziyarPanahi/WizardLM-2-7B-AWQ)|\n|[AI-ModelScope/WizardLM-2-8x22B](https://modelscope.cn/models/AI-ModelScope/WizardLM-2-8x22B)|wizardlm2_moe|wizardlm2_moe|transformers>=4.36|✘|-|[alpindale/WizardLM-2-8x22B](https://huggingface.co/alpindale/WizardLM-2-8x22B)|\n|[AI-ModelScope/phi-2](https://modelscope.cn/models/AI-ModelScope/phi-2)|phi2|default|-|✘|-|[microsoft/phi-2](https://huggingface.co/microsoft/phi-2)|\n|[LLM-Research/Phi-3-small-8k-instruct](https://modelscope.cn/models/LLM-Research/Phi-3-small-8k-instruct)|phi3_small|phi3|transformers>=4.36|✘|-|[microsoft/Phi-3-small-8k-instruct](https://huggingface.co/microsoft/Phi-3-small-8k-instruct)|\n|[LLM-Research/Phi-3-small-128k-instruct](https://modelscope.cn/models/LLM-Research/Phi-3-small-128k-instruct)|phi3_small|phi3|transformers>=4.36|✘|-|[microsoft/Phi-3-small-128k-instruct](https://huggingface.co/microsoft/Phi-3-small-128k-instruct)|\n|[LLM-Research/Phi-3-mini-4k-instruct](https://modelscope.cn/models/LLM-Research/Phi-3-mini-4k-instruct)|phi3|phi3|transformers>=4.36|✘|-|[microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct)|\n|[LLM-Research/Phi-3-mini-128k-instruct](https://modelscope.cn/models/LLM-Research/Phi-3-mini-128k-instruct)|phi3|phi3|transformers>=4.36|✘|-|[microsoft/Phi-3-mini-128k-instruct](https://huggingface.co/microsoft/Phi-3-mini-128k-instruct)|\n|[LLM-Research/Phi-3-medium-4k-instruct](https://modelscope.cn/models/LLM-Research/Phi-3-medium-4k-instruct)|phi3|phi3|transformers>=4.36|✘|-|[microsoft/Phi-3-medium-4k-instruct](https://huggingface.co/microsoft/Phi-3-medium-4k-instruct)|\n|[LLM-Research/Phi-3-medium-128k-instruct](https://modelscope.cn/models/LLM-Research/Phi-3-medium-128k-instruct)|phi3|phi3|transformers>=4.36|✘|-|[microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)|\n|[LLM-Research/Phi-3.5-mini-instruct](https://modelscope.cn/models/LLM-Research/Phi-3.5-mini-instruct)|phi3|phi3|transformers>=4.36|✘|-|[microsoft/Phi-3.5-mini-instruct](https://huggingface.co/microsoft/Phi-3.5-mini-instruct)|\n|[LLM-Research/Phi-4-mini-instruct](https://modelscope.cn/models/LLM-Research/Phi-4-mini-instruct)|phi3|phi3|transformers>=4.36|✘|-|[microsoft/Phi-4-mini-instruct](https://huggingface.co/microsoft/Phi-4-mini-instruct)|\n|[LLM-Research/Phi-3.5-MoE-instruct](https://modelscope.cn/models/LLM-Research/Phi-3.5-MoE-instruct)|phi3_moe|phi3|transformers>=4.36|✘|-|[microsoft/Phi-3.5-MoE-instruct](https://huggingface.co/microsoft/Phi-3.5-MoE-instruct)|\n|[LLM-Research/phi-4](https://modelscope.cn/models/LLM-Research/phi-4)|phi4|phi4|transformers>=4.36|✘|-|[microsoft/phi-4](https://huggingface.co/microsoft/phi-4)|\n|[MiniMax/MiniMax-Text-01](https://modelscope.cn/models/MiniMax/MiniMax-Text-01)|minimax|minimax|-|✘|-|[MiniMaxAI/MiniMax-Text-01](https://huggingface.co/MiniMaxAI/MiniMax-Text-01)|\n|[MiniMax/MiniMax-M1-40k](https://modelscope.cn/models/MiniMax/MiniMax-M1-40k)|minimax_m1|minimax_m1|-|✘|-|[MiniMaxAI/MiniMax-M1-40k](https://huggingface.co/MiniMaxAI/MiniMax-M1-40k)|\n|[MiniMax/MiniMax-M1-80k](https://modelscope.cn/models/MiniMax/MiniMax-M1-80k)|minimax_m1|minimax_m1|-|✘|-|[MiniMaxAI/MiniMax-M1-80k](https://huggingface.co/MiniMaxAI/MiniMax-M1-80k)|\n|[MiniMax/MiniMax-M2](https://modelscope.cn/models/MiniMax/MiniMax-M2)|minimax_m2|minimax_m2|transformers==4.57.1|✔|-|[MiniMaxAI/MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2)|\n|[MiniMax/MiniMax-M2.1](https://modelscope.cn/models/MiniMax/MiniMax-M2.1)|minimax_m2|minimax_m2_1|transformers==4.57.1|✔|-|[MiniMaxAI/MiniMax-M2.1](https://huggingface.co/MiniMaxAI/MiniMax-M2.1)|\n|[MiniMax/MiniMax-M2.5](https://modelscope.cn/models/MiniMax/MiniMax-M2.5)|minimax_m2|minimax_m2_5|transformers==4.57.1|✔|-|[MiniMaxAI/MiniMax-M2.5](https://huggingface.co/MiniMaxAI/MiniMax-M2.5)|\n|[AI-ModelScope/gemma-2b-it](https://modelscope.cn/models/AI-ModelScope/gemma-2b-it)|gemma|gemma|transformers>=4.38|✘|-|[google/gemma-2b-it](https://huggingface.co/google/gemma-2b-it)|\n|[AI-ModelScope/gemma-2b](https://modelscope.cn/models/AI-ModelScope/gemma-2b)|gemma|gemma|transformers>=4.38|✘|-|[google/gemma-2b](https://huggingface.co/google/gemma-2b)|\n|[AI-ModelScope/gemma-7b](https://modelscope.cn/models/AI-ModelScope/gemma-7b)|gemma|gemma|transformers>=4.38|✘|-|[google/gemma-7b](https://huggingface.co/google/gemma-7b)|\n|[AI-ModelScope/gemma-7b-it](https://modelscope.cn/models/AI-ModelScope/gemma-7b-it)|gemma|gemma|transformers>=4.38|✘|-|[google/gemma-7b-it](https://huggingface.co/google/gemma-7b-it)|\n|[LLM-Research/gemma-2-2b-it](https://modelscope.cn/models/LLM-Research/gemma-2-2b-it)|gemma2|gemma|transformers>=4.42|✘|-|[google/gemma-2-2b-it](https://huggingface.co/google/gemma-2-2b-it)|\n|[LLM-Research/gemma-2-2b](https://modelscope.cn/models/LLM-Research/gemma-2-2b)|gemma2|gemma|transformers>=4.42|✘|-|[google/gemma-2-2b](https://huggingface.co/google/gemma-2-2b)|\n|[LLM-Research/gemma-2-9b](https://modelscope.cn/models/LLM-Research/gemma-2-9b)|gemma2|gemma|transformers>=4.42|✘|-|[google/gemma-2-9b](https://huggingface.co/google/gemma-2-9b)|\n|[LLM-Research/gemma-2-9b-it](https://modelscope.cn/models/LLM-Research/gemma-2-9b-it)|gemma2|gemma|transformers>=4.42|✘|-|[google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it)|\n|[LLM-Research/gemma-2-27b](https://modelscope.cn/models/LLM-Research/gemma-2-27b)|gemma2|gemma|transformers>=4.42|✘|-|[google/gemma-2-27b](https://huggingface.co/google/gemma-2-27b)|\n|[LLM-Research/gemma-2-27b-it](https://modelscope.cn/models/LLM-Research/gemma-2-27b-it)|gemma2|gemma|transformers>=4.42|✘|-|[google/gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it)|\n|[LLM-Research/gemma-3-1b-pt](https://modelscope.cn/models/LLM-Research/gemma-3-1b-pt)|gemma3_text|gemma3_text|transformers>=4.49|✘|-|[google/gemma-3-1b-pt](https://huggingface.co/google/gemma-3-1b-pt)|\n|[LLM-Research/gemma-3-1b-it](https://modelscope.cn/models/LLM-Research/gemma-3-1b-it)|gemma3_text|gemma3_text|transformers>=4.49|✘|-|[google/gemma-3-1b-it](https://huggingface.co/google/gemma-3-1b-it)|\n|[google/gemma-3-270m](https://modelscope.cn/models/google/gemma-3-270m)|gemma3_text|gemma3_text|transformers>=4.49|✘|-|[google/gemma-3-270m](https://huggingface.co/google/gemma-3-270m)|\n|[google/gemma-3-270m-it](https://modelscope.cn/models/google/gemma-3-270m-it)|gemma3_text|gemma3_text|transformers>=4.49|✘|-|[google/gemma-3-270m-it](https://huggingface.co/google/gemma-3-270m-it)|\n|[google/medgemma-27b-text-it](https://modelscope.cn/models/google/medgemma-27b-text-it)|gemma3_text|gemma3_text|transformers>=4.49|✘|-|[google/medgemma-27b-text-it](https://huggingface.co/google/medgemma-27b-text-it)|\n|[skywork/Skywork-13B-base](https://modelscope.cn/models/skywork/Skywork-13B-base)|skywork|skywork|-|✘|-|[skywork/Skywork-13B-base](https://huggingface.co/skywork/Skywork-13B-base)|\n|[skywork/Skywork-13B-chat](https://modelscope.cn/models/skywork/Skywork-13B-chat)|skywork|skywork|-|✘|-|-|\n|[inclusionAI/Ling-lite](https://modelscope.cn/models/inclusionAI/Ling-lite)|ling|ling|-|✘|-|[inclusionAI/Ling-lite](https://huggingface.co/inclusionAI/Ling-lite)|\n|[inclusionAI/Ling-plus](https://modelscope.cn/models/inclusionAI/Ling-plus)|ling|ling|-|✘|-|[inclusionAI/Ling-plus](https://huggingface.co/inclusionAI/Ling-plus)|\n|[inclusionAI/Ling-lite-base](https://modelscope.cn/models/inclusionAI/Ling-lite-base)|ling|ling|-|✘|-|[inclusionAI/Ling-lite-base](https://huggingface.co/inclusionAI/Ling-lite-base)|\n|[inclusionAI/Ling-plus-base](https://modelscope.cn/models/inclusionAI/Ling-plus-base)|ling|ling|-|✘|-|[inclusionAI/Ling-plus-base](https://huggingface.co/inclusionAI/Ling-plus-base)|\n|[inclusionAI/Ling-mini-2.0](https://modelscope.cn/models/inclusionAI/Ling-mini-2.0)|ling2|ling2|-|✘|-|[inclusionAI/Ling-mini-2.0](https://huggingface.co/inclusionAI/Ling-mini-2.0)|\n|[inclusionAI/Ling-mini-base-2.0](https://modelscope.cn/models/inclusionAI/Ling-mini-base-2.0)|ling2|ling2|-|✘|-|[inclusionAI/Ling-mini-base-2.0](https://huggingface.co/inclusionAI/Ling-mini-base-2.0)|\n|[inclusionAI/Ring-mini-2.0](https://modelscope.cn/models/inclusionAI/Ring-mini-2.0)|ring2|ring2|-|✘|-|[inclusionAI/Ring-mini-2.0](https://huggingface.co/inclusionAI/Ring-mini-2.0)|\n|[IEITYuan/Yuan2.0-2B-hf](https://modelscope.cn/models/IEITYuan/Yuan2.0-2B-hf)|yuan2|yuan|-|✘|-|[IEITYuan/Yuan2-2B-hf](https://huggingface.co/IEITYuan/Yuan2-2B-hf)|\n|[IEITYuan/Yuan2.0-51B-hf](https://modelscope.cn/models/IEITYuan/Yuan2.0-51B-hf)|yuan2|yuan|-|✘|-|[IEITYuan/Yuan2-51B-hf](https://huggingface.co/IEITYuan/Yuan2-51B-hf)|\n|[IEITYuan/Yuan2.0-102B-hf](https://modelscope.cn/models/IEITYuan/Yuan2.0-102B-hf)|yuan2|yuan|-|✘|-|[IEITYuan/Yuan2-102B-hf](https://huggingface.co/IEITYuan/Yuan2-102B-hf)|\n|[IEITYuan/Yuan2-2B-Janus-hf](https://modelscope.cn/models/IEITYuan/Yuan2-2B-Janus-hf)|yuan2|yuan|-|✘|-|[IEITYuan/Yuan2-2B-Janus-hf](https://huggingface.co/IEITYuan/Yuan2-2B-Janus-hf)|\n|[IEITYuan/Yuan2-M32-hf](https://modelscope.cn/models/IEITYuan/Yuan2-M32-hf)|yuan2|yuan|-|✘|-|[IEITYuan/Yuan2-M32-hf](https://huggingface.co/IEITYuan/Yuan2-M32-hf)|\n|[OrionStarAI/Orion-14B-Chat](https://modelscope.cn/models/OrionStarAI/Orion-14B-Chat)|orion|orion|-|✘|-|[OrionStarAI/Orion-14B-Chat](https://huggingface.co/OrionStarAI/Orion-14B-Chat)|\n|[OrionStarAI/Orion-14B-Base](https://modelscope.cn/models/OrionStarAI/Orion-14B-Base)|orion|orion|-|✘|-|[OrionStarAI/Orion-14B-Base](https://huggingface.co/OrionStarAI/Orion-14B-Base)|\n|[xverse/XVERSE-7B-Chat](https://modelscope.cn/models/xverse/XVERSE-7B-Chat)|xverse|xverse|-|✘|-|[xverse/XVERSE-7B-Chat](https://huggingface.co/xverse/XVERSE-7B-Chat)|\n|[xverse/XVERSE-7B](https://modelscope.cn/models/xverse/XVERSE-7B)|xverse|xverse|-|✘|-|[xverse/XVERSE-7B](https://huggingface.co/xverse/XVERSE-7B)|\n|[xverse/XVERSE-13B](https://modelscope.cn/models/xverse/XVERSE-13B)|xverse|xverse|-|✘|-|[xverse/XVERSE-13B](https://huggingface.co/xverse/XVERSE-13B)|\n|[xverse/XVERSE-13B-Chat](https://modelscope.cn/models/xverse/XVERSE-13B-Chat)|xverse|xverse|-|✘|-|[xverse/XVERSE-13B-Chat](https://huggingface.co/xverse/XVERSE-13B-Chat)|\n|[xverse/XVERSE-65B](https://modelscope.cn/models/xverse/XVERSE-65B)|xverse|xverse|-|✘|-|[xverse/XVERSE-65B](https://huggingface.co/xverse/XVERSE-65B)|\n|[xverse/XVERSE-65B-2](https://modelscope.cn/models/xverse/XVERSE-65B-2)|xverse|xverse|-|✘|-|[xverse/XVERSE-65B-2](https://huggingface.co/xverse/XVERSE-65B-2)|\n|[xverse/XVERSE-65B-Chat](https://modelscope.cn/models/xverse/XVERSE-65B-Chat)|xverse|xverse|-|✘|-|[xverse/XVERSE-65B-Chat](https://huggingface.co/xverse/XVERSE-65B-Chat)|\n|[xverse/XVERSE-13B-256K](https://modelscope.cn/models/xverse/XVERSE-13B-256K)|xverse|xverse|-|✘|-|[xverse/XVERSE-13B-256K](https://huggingface.co/xverse/XVERSE-13B-256K)|\n|[xverse/XVERSE-MoE-A4.2B](https://modelscope.cn/models/xverse/XVERSE-MoE-A4.2B)|xverse_moe|xverse|-|✘|-|[xverse/XVERSE-MoE-A4.2B](https://huggingface.co/xverse/XVERSE-MoE-A4.2B)|\n|[damo/nlp_seqgpt-560m](https://modelscope.cn/models/damo/nlp_seqgpt-560m)|seggpt|default|-|✘|-|[DAMO-NLP/SeqGPT-560M](https://huggingface.co/DAMO-NLP/SeqGPT-560M)|\n|[vivo-ai/BlueLM-7B-Chat-32K](https://modelscope.cn/models/vivo-ai/BlueLM-7B-Chat-32K)|bluelm|bluelm|-|✘|-|[vivo-ai/BlueLM-7B-Chat-32K](https://huggingface.co/vivo-ai/BlueLM-7B-Chat-32K)|\n|[vivo-ai/BlueLM-7B-Chat](https://modelscope.cn/models/vivo-ai/BlueLM-7B-Chat)|bluelm|bluelm|-|✘|-|[vivo-ai/BlueLM-7B-Chat](https://huggingface.co/vivo-ai/BlueLM-7B-Chat)|\n|[vivo-ai/BlueLM-7B-Base-32K](https://modelscope.cn/models/vivo-ai/BlueLM-7B-Base-32K)|bluelm|bluelm|-|✘|-|[vivo-ai/BlueLM-7B-Base-32K](https://huggingface.co/vivo-ai/BlueLM-7B-Base-32K)|\n|[vivo-ai/BlueLM-7B-Base](https://modelscope.cn/models/vivo-ai/BlueLM-7B-Base)|bluelm|bluelm|-|✘|-|[vivo-ai/BlueLM-7B-Base](https://huggingface.co/vivo-ai/BlueLM-7B-Base)|\n|[AI-ModelScope/c4ai-command-r-v01](https://modelscope.cn/models/AI-ModelScope/c4ai-command-r-v01)|c4ai|c4ai|transformers>=4.39|✘|-|[CohereForAI/c4ai-command-r-v01](https://huggingface.co/CohereForAI/c4ai-command-r-v01)|\n|[AI-ModelScope/c4ai-command-r-plus](https://modelscope.cn/models/AI-ModelScope/c4ai-command-r-plus)|c4ai|c4ai|transformers>=4.39|✘|-|[CohereForAI/c4ai-command-r-plus](https://huggingface.co/CohereForAI/c4ai-command-r-plus)|\n|[AI-ModelScope/dbrx-base](https://modelscope.cn/models/AI-ModelScope/dbrx-base)|dbrx|dbrx|transformers>=4.36|✘|-|[databricks/dbrx-base](https://huggingface.co/databricks/dbrx-base)|\n|[AI-ModelScope/dbrx-instruct](https://modelscope.cn/models/AI-ModelScope/dbrx-instruct)|dbrx|dbrx|transformers>=4.36|✘|-|[databricks/dbrx-instruct](https://huggingface.co/databricks/dbrx-instruct)|\n|[colossalai/grok-1-pytorch](https://modelscope.cn/models/colossalai/grok-1-pytorch)|grok|default|-|✘|-|[hpcai-tech/grok-1](https://huggingface.co/hpcai-tech/grok-1)|\n|[AI-ModelScope/mamba-130m-hf](https://modelscope.cn/models/AI-ModelScope/mamba-130m-hf)|mamba|default|transformers>=4.39.0|✘|-|[state-spaces/mamba-130m-hf](https://huggingface.co/state-spaces/mamba-130m-hf)|\n|[AI-ModelScope/mamba-370m-hf](https://modelscope.cn/models/AI-ModelScope/mamba-370m-hf)|mamba|default|transformers>=4.39.0|✘|-|[state-spaces/mamba-370m-hf](https://huggingface.co/state-spaces/mamba-370m-hf)|\n|[AI-ModelScope/mamba-390m-hf](https://modelscope.cn/models/AI-ModelScope/mamba-390m-hf)|mamba|default|transformers>=4.39.0|✘|-|[state-spaces/mamba-390m-hf](https://huggingface.co/state-spaces/mamba-390m-hf)|\n|[AI-ModelScope/mamba-790m-hf](https://modelscope.cn/models/AI-ModelScope/mamba-790m-hf)|mamba|default|transformers>=4.39.0|✘|-|[state-spaces/mamba-790m-hf](https://huggingface.co/state-spaces/mamba-790m-hf)|\n|[AI-ModelScope/mamba-1.4b-hf](https://modelscope.cn/models/AI-ModelScope/mamba-1.4b-hf)|mamba|default|transformers>=4.39.0|✘|-|[state-spaces/mamba-1.4b-hf](https://huggingface.co/state-spaces/mamba-1.4b-hf)|\n|[AI-ModelScope/mamba-2.8b-hf](https://modelscope.cn/models/AI-ModelScope/mamba-2.8b-hf)|mamba|default|transformers>=4.39.0|✘|-|[state-spaces/mamba-2.8b-hf](https://huggingface.co/state-spaces/mamba-2.8b-hf)|\n|[damo/nlp_polylm_13b_text_generation](https://modelscope.cn/models/damo/nlp_polylm_13b_text_generation)|polylm|default|-|✘|-|[DAMO-NLP-MT/polylm-13b](https://huggingface.co/DAMO-NLP-MT/polylm-13b)|\n|[AI-ModelScope/aya-expanse-8b](https://modelscope.cn/models/AI-ModelScope/aya-expanse-8b)|aya|aya|transformers>=4.44.0|✘|-|[CohereForAI/aya-expanse-8b](https://huggingface.co/CohereForAI/aya-expanse-8b)|\n|[AI-ModelScope/aya-expanse-32b](https://modelscope.cn/models/AI-ModelScope/aya-expanse-32b)|aya|aya|transformers>=4.44.0|✘|-|[CohereForAI/aya-expanse-32b](https://huggingface.co/CohereForAI/aya-expanse-32b)|\n|[XiaomiMiMo/MiMo-7B-Base](https://modelscope.cn/models/XiaomiMiMo/MiMo-7B-Base)|mimo|qwen|transformers>=4.37|✔|-|[XiaomiMiMo/MiMo-7B-Base](https://huggingface.co/XiaomiMiMo/MiMo-7B-Base)|\n|[XiaomiMiMo/MiMo-7B-SFT](https://modelscope.cn/models/XiaomiMiMo/MiMo-7B-SFT)|mimo|qwen|transformers>=4.37|✔|-|[XiaomiMiMo/MiMo-7B-SFT](https://huggingface.co/XiaomiMiMo/MiMo-7B-SFT)|\n|[XiaomiMiMo/MiMo-7B-RL-Zero](https://modelscope.cn/models/XiaomiMiMo/MiMo-7B-RL-Zero)|mimo|qwen|transformers>=4.37|✔|-|[XiaomiMiMo/MiMo-7B-RL-Zero](https://huggingface.co/XiaomiMiMo/MiMo-7B-RL-Zero)|\n|[XiaomiMiMo/MiMo-7B-RL](https://modelscope.cn/models/XiaomiMiMo/MiMo-7B-RL)|mimo|qwen|transformers>=4.37|✔|-|[XiaomiMiMo/MiMo-7B-RL](https://huggingface.co/XiaomiMiMo/MiMo-7B-RL)|\n|[XiaomiMiMo/MiMo-7B-RL-0530](https://modelscope.cn/models/XiaomiMiMo/MiMo-7B-RL-0530)|mimo|mimo_rl|transformers>=4.37|✔|-|[XiaomiMiMo/MiMo-7B-RL-0530](https://huggingface.co/XiaomiMiMo/MiMo-7B-RL-0530)|\n|[rednote-hilab/dots.llm1.base](https://modelscope.cn/models/rednote-hilab/dots.llm1.base)|dots1|dots1|transformers>=4.53|✔|-|[rednote-hilab/dots.llm1.base](https://huggingface.co/rednote-hilab/dots.llm1.base)|\n|[rednote-hilab/dots.llm1.inst](https://modelscope.cn/models/rednote-hilab/dots.llm1.inst)|dots1|dots1|transformers>=4.53|✔|-|[rednote-hilab/dots.llm1.inst](https://huggingface.co/rednote-hilab/dots.llm1.inst)|\n|[Tencent-Hunyuan/Hunyuan-A13B-Instruct](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-A13B-Instruct)|hunyuan|hunyuan_moe|-|✘|-|[tencent/Hunyuan-A13B-Instruct](https://huggingface.co/tencent/Hunyuan-A13B-Instruct)|\n|[Tencent-Hunyuan/Hunyuan-0.5B-Instruct](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-0.5B-Instruct)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-0.5B-Instruct](https://huggingface.co/tencent/Hunyuan-0.5B-Instruct)|\n|[Tencent-Hunyuan/Hunyuan-1.8B-Instruct](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-1.8B-Instruct)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-1.8B-Instruct](https://huggingface.co/tencent/Hunyuan-1.8B-Instruct)|\n|[Tencent-Hunyuan/Hunyuan-4B-Instruct](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-4B-Instruct)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-4B-Instruct](https://huggingface.co/tencent/Hunyuan-4B-Instruct)|\n|[Tencent-Hunyuan/Hunyuan-7B-Instruct](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-7B-Instruct)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-7B-Instruct](https://huggingface.co/tencent/Hunyuan-7B-Instruct)|\n|[Tencent-Hunyuan/Hunyuan-0.5B-Pretrain](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-0.5B-Pretrain)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-0.5B-Pretrain](https://huggingface.co/tencent/Hunyuan-0.5B-Pretrain)|\n|[Tencent-Hunyuan/Hunyuan-1.8B-Pretrain](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-1.8B-Pretrain)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-1.8B-Pretrain](https://huggingface.co/tencent/Hunyuan-1.8B-Pretrain)|\n|[Tencent-Hunyuan/Hunyuan-4B-Pretrain](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-4B-Pretrain)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-4B-Pretrain](https://huggingface.co/tencent/Hunyuan-4B-Pretrain)|\n|[Tencent-Hunyuan/Hunyuan-7B-Pretrain](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-7B-Pretrain)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-7B-Pretrain](https://huggingface.co/tencent/Hunyuan-7B-Pretrain)|\n|[Tencent-Hunyuan/Hunyuan-0.5B-Instruct-FP8](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-0.5B-Instruct-FP8)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-0.5B-Instruct-FP8](https://huggingface.co/tencent/Hunyuan-0.5B-Instruct-FP8)|\n|[Tencent-Hunyuan/Hunyuan-1.8B-Instruct-FP8](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-1.8B-Instruct-FP8)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-1.8B-Instruct-FP8](https://huggingface.co/tencent/Hunyuan-1.8B-Instruct-FP8)|\n|[Tencent-Hunyuan/Hunyuan-4B-Instruct-FP8](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-4B-Instruct-FP8)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-4B-Instruct-FP8](https://huggingface.co/tencent/Hunyuan-4B-Instruct-FP8)|\n|[Tencent-Hunyuan/Hunyuan-7B-Instruct-FP8](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-7B-Instruct-FP8)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-7B-Instruct-FP8](https://huggingface.co/tencent/Hunyuan-7B-Instruct-FP8)|\n|[Tencent-Hunyuan/Hunyuan-0.5B-Instruct-AWQ-Int4](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-0.5B-Instruct-AWQ-Int4)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-0.5B-Instruct-AWQ-Int4](https://huggingface.co/tencent/Hunyuan-0.5B-Instruct-AWQ-Int4)|\n|[Tencent-Hunyuan/Hunyuan-1.8B-Instruct-AWQ-Int4](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-1.8B-Instruct-AWQ-Int4)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-1.8B-Instruct-AWQ-Int4](https://huggingface.co/tencent/Hunyuan-1.8B-Instruct-AWQ-Int4)|\n|[Tencent-Hunyuan/Hunyuan-4B-Instruct-AWQ-Int4](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-4B-Instruct-AWQ-Int4)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-4B-Instruct-AWQ-Int4](https://huggingface.co/tencent/Hunyuan-4B-Instruct-AWQ-Int4)|\n|[Tencent-Hunyuan/Hunyuan-7B-Instruct-AWQ-Int4](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-7B-Instruct-AWQ-Int4)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-7B-Instruct-AWQ-Int4](https://huggingface.co/tencent/Hunyuan-7B-Instruct-AWQ-Int4)|\n|[Tencent-Hunyuan/Hunyuan-0.5B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-0.5B-Instruct-GPTQ-Int4)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-0.5B-Instruct-GPTQ-Int4](https://huggingface.co/tencent/Hunyuan-0.5B-Instruct-GPTQ-Int4)|\n|[Tencent-Hunyuan/Hunyuan-1.8B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-1.8B-Instruct-GPTQ-Int4)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-1.8B-Instruct-GPTQ-Int4](https://huggingface.co/tencent/Hunyuan-1.8B-Instruct-GPTQ-Int4)|\n|[Tencent-Hunyuan/Hunyuan-4B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-4B-Instruct-GPTQ-Int4)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-4B-Instruct-GPTQ-Int4](https://huggingface.co/tencent/Hunyuan-4B-Instruct-GPTQ-Int4)|\n|[Tencent-Hunyuan/Hunyuan-7B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Tencent-Hunyuan/Hunyuan-7B-Instruct-GPTQ-Int4)|hunyuan_v1_dense|hunyuan|transformers>=4.55.0.dev0|✘|-|[tencent/Hunyuan-7B-Instruct-GPTQ-Int4](https://huggingface.co/tencent/Hunyuan-7B-Instruct-GPTQ-Int4)|\n|[PaddlePaddle/ERNIE-4.5-0.3B-Base-PT](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-0.3B-Base-PT)|ernie4_5|ernie|-|✔|-|[baidu/ERNIE-4.5-0.3B-PT](https://huggingface.co/baidu/ERNIE-4.5-0.3B-PT)|\n|[PaddlePaddle/ERNIE-4.5-0.3B-PT](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-0.3B-PT)|ernie4_5|ernie|-|✔|-|[baidu/ERNIE-4.5-0.3B-PT](https://huggingface.co/baidu/ERNIE-4.5-0.3B-PT)|\n|[PaddlePaddle/ERNIE-4.5-21B-A3B-Base-PT](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-21B-A3B-Base-PT)|ernie4_5_moe|ernie|-|✔|-|[baidu/ERNIE-4.5-21B-A3B-Base-PT](https://huggingface.co/baidu/ERNIE-4.5-21B-A3B-Base-PT)|\n|[PaddlePaddle/ERNIE-4.5-21B-A3B-PT](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-21B-A3B-PT)|ernie4_5_moe|ernie|-|✔|-|[baidu/ERNIE-4.5-21B-A3B-PT](https://huggingface.co/baidu/ERNIE-4.5-21B-A3B-PT)|\n|[PaddlePaddle/ERNIE-4.5-300B-A47B-Base-PT](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-300B-A47B-Base-PT)|ernie4_5_moe|ernie|-|✔|-|[baidu/ERNIE-4.5-300B-A47B-Base-PT](https://huggingface.co/baidu/ERNIE-4.5-300B-A47B-Base-PT)|\n|[PaddlePaddle/ERNIE-4.5-300B-A47B-PT](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-300B-A47B-PT)|ernie4_5_moe|ernie|-|✔|-|[baidu/ERNIE-4.5-300B-A47B-PT](https://huggingface.co/baidu/ERNIE-4.5-300B-A47B-PT)|\n|[PaddlePaddle/ERNIE-4.5-21B-A3B-Thinking](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-21B-A3B-Thinking)|ernie4_5_moe|ernie_thinking|-|✔|-|[baidu/ERNIE-4.5-21B-A3B-Thinking](https://huggingface.co/baidu/ERNIE-4.5-21B-A3B-Thinking)|\n|[google/embeddinggemma-300m](https://modelscope.cn/models/google/embeddinggemma-300m)|gemma_emb|dummy|-|✘|-|[google/embeddinggemma-300m](https://huggingface.co/google/embeddinggemma-300m)|\n|[meituan-longcat/LongCat-Flash-Chat](https://modelscope.cn/models/meituan-longcat/LongCat-Flash-Chat)|longchat|longchat|transformers>=4.54,<4.56|✘|-|[meituan-longcat/LongCat-Flash-Chat](https://huggingface.co/meituan-longcat/LongCat-Flash-Chat)|\n|[meituan-longcat/LongCat-Flash-Chat-FP8](https://modelscope.cn/models/meituan-longcat/LongCat-Flash-Chat-FP8)|longchat|longchat|transformers>=4.54,<4.56|✘|-|[meituan-longcat/LongCat-Flash-Chat-FP8](https://huggingface.co/meituan-longcat/LongCat-Flash-Chat-FP8)|\n|[IQuestLab/IQuest-Coder-V1-40B-Base-Stage1](https://modelscope.cn/models/IQuestLab/IQuest-Coder-V1-40B-Base-Stage1)|iquestcoder|iquestcoder|transformers==4.52.4|✘|-|[IQuestLab/IQuest-Coder-V1-40B-Base-Stage1](https://huggingface.co/IQuestLab/IQuest-Coder-V1-40B-Base-Stage1)|\n|[IQuestLab/IQuest-Coder-V1-40B-Base](https://modelscope.cn/models/IQuestLab/IQuest-Coder-V1-40B-Base)|iquestcoder|iquestcoder|transformers==4.52.4|✘|-|[IQuestLab/IQuest-Coder-V1-40B-Base](https://huggingface.co/IQuestLab/IQuest-Coder-V1-40B-Base)|\n|[IQuestLab/IQuest-Coder-V1-40B-Instruct](https://modelscope.cn/models/IQuestLab/IQuest-Coder-V1-40B-Instruct)|iquestcoder|iquestcoder|transformers==4.52.4|✘|-|[IQuestLab/IQuest-Coder-V1-40B-Instruct](https://huggingface.co/IQuestLab/IQuest-Coder-V1-40B-Instruct)|\n|[Tencent-YouTu-Research/Youtu-LLM-2B](https://modelscope.cn/models/Tencent-YouTu-Research/Youtu-LLM-2B)|youtu_llm|youtu_llm|transformers>=4.56|✘|-|[tencent/Youtu-LLM-2B](https://huggingface.co/tencent/Youtu-LLM-2B)|\n|[Tencent-YouTu-Research/Youtu-LLM-2B-Base](https://modelscope.cn/models/Tencent-YouTu-Research/Youtu-LLM-2B-Base)|youtu_llm|youtu_llm|transformers>=4.56|✘|-|[tencent/Youtu-LLM-2B-Base](https://huggingface.co/tencent/Youtu-LLM-2B-Base)|\n|[iic/gte-reranker-modernbert-base](https://modelscope.cn/models/iic/gte-reranker-modernbert-base)|modern_bert_gte_reranker|bert|transformers>=4.48|✘|bert, reranker|[Alibaba-NLP/gte-reranker-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base)|\n|[BAAI/bge-reranker-base](https://modelscope.cn/models/BAAI/bge-reranker-base)|bge_reranker|bge_reranker|-|✘|-|[BAAI/bge-reranker-base](https://huggingface.co/BAAI/bge-reranker-base)|\n|[BAAI/bge-reranker-v2-m3](https://modelscope.cn/models/BAAI/bge-reranker-v2-m3)|bge_reranker|bge_reranker|-|✘|-|[BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3)|\n|[BAAI/bge-reranker-large](https://modelscope.cn/models/BAAI/bge-reranker-large)|bge_reranker|bge_reranker|-|✘|-|[BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large)|\n|[allenai/OLMoE-1B-7B-0125](https://modelscope.cn/models/allenai/OLMoE-1B-7B-0125)|olmoe|olmoe|-|✔|-|[allenai/OLMoE-1B-7B-0125](https://huggingface.co/allenai/OLMoE-1B-7B-0125)|\n|[allenai/OLMoE-1B-7B-0125-Instruct](https://modelscope.cn/models/allenai/OLMoE-1B-7B-0125-Instruct)|olmoe|olmoe|-|✔|-|[allenai/OLMoE-1B-7B-0125-Instruct](https://huggingface.co/allenai/OLMoE-1B-7B-0125-Instruct)|\n|[allenai/OLMoE-1B-7B-0924](https://modelscope.cn/models/allenai/OLMoE-1B-7B-0924)|olmoe|olmoe_0924|-|✔|-|[allenai/OLMoE-1B-7B-0924](https://huggingface.co/allenai/OLMoE-1B-7B-0924)|\n|[allenai/OLMoE-1B-7B-0924-Instruct](https://modelscope.cn/models/allenai/OLMoE-1B-7B-0924-Instruct)|olmoe|olmoe_0924|-|✔|-|[allenai/OLMoE-1B-7B-0924-Instruct](https://huggingface.co/allenai/OLMoE-1B-7B-0924-Instruct)|\n|[allenai/OLMoE-1B-7B-0924-SFT](https://modelscope.cn/models/allenai/OLMoE-1B-7B-0924-SFT)|olmoe|olmoe_0924|-|✔|-|[allenai/OLMoE-1B-7B-0924-SFT](https://huggingface.co/allenai/OLMoE-1B-7B-0924-SFT)|\n|[answerdotai/ModernBERT-base](https://modelscope.cn/models/answerdotai/ModernBERT-base)|modern_bert|dummy|transformers>=4.48|✘|bert|[answerdotai/ModernBERT-base](https://huggingface.co/answerdotai/ModernBERT-base)|\n|[answerdotai/ModernBERT-large](https://modelscope.cn/models/answerdotai/ModernBERT-large)|modern_bert|dummy|transformers>=4.48|✘|bert|[answerdotai/ModernBERT-large](https://huggingface.co/answerdotai/ModernBERT-large)|\n|[iic/gte-modernbert-base](https://modelscope.cn/models/iic/gte-modernbert-base)|modern_bert_gte|dummy|transformers>=4.48|✘|bert, embedding|[Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base)|\n|[iic/nlp_structbert_backbone_base_std](https://modelscope.cn/models/iic/nlp_structbert_backbone_base_std)|bert|dummy|-|✘|bert|-|\n|[Shanghai_AI_Laboratory/internlm2-1_8b-reward](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-1_8b-reward)|internlm2_reward|internlm2_reward|transformers>=4.38|✘|-|[internlm/internlm2-1_8b-reward](https://huggingface.co/internlm/internlm2-1_8b-reward)|\n|[Shanghai_AI_Laboratory/internlm2-7b-reward](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-7b-reward)|internlm2_reward|internlm2_reward|transformers>=4.38|✘|-|[internlm/internlm2-7b-reward](https://huggingface.co/internlm/internlm2-7b-reward)|\n|[Shanghai_AI_Laboratory/internlm2-20b-reward](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-20b-reward)|internlm2_reward|internlm2_reward|transformers>=4.38|✘|-|[internlm/internlm2-20b-reward](https://huggingface.co/internlm/internlm2-20b-reward)|\n|[Qwen/Qwen2-Math-RM-72B](https://modelscope.cn/models/Qwen/Qwen2-Math-RM-72B)|qwen2_reward|qwen|transformers>=4.37|✘|-|[Qwen/Qwen2-Math-RM-72B](https://huggingface.co/Qwen/Qwen2-Math-RM-72B)|\n|[Qwen/Qwen2.5-Math-RM-72B](https://modelscope.cn/models/Qwen/Qwen2.5-Math-RM-72B)|qwen2_reward|qwen2_5_math|transformers>=4.37|✘|-|[Qwen/Qwen2.5-Math-RM-72B](https://huggingface.co/Qwen/Qwen2.5-Math-RM-72B)|\n|[Qwen/Qwen2.5-Math-PRM-7B](https://modelscope.cn/models/Qwen/Qwen2.5-Math-PRM-7B)|qwen2_5_prm|qwen2_5_math_prm|transformers>=4.37|✘|-|[Qwen/Qwen2.5-Math-PRM-7B](https://huggingface.co/Qwen/Qwen2.5-Math-PRM-7B)|\n|[Qwen/Qwen2.5-Math-7B-PRM800K](https://modelscope.cn/models/Qwen/Qwen2.5-Math-7B-PRM800K)|qwen2_5_prm|qwen2_5_math_prm|transformers>=4.37|✘|-|[Qwen/Qwen2.5-Math-7B-PRM800K](https://huggingface.co/Qwen/Qwen2.5-Math-7B-PRM800K)|\n|[Qwen/Qwen2.5-Math-PRM-72B](https://modelscope.cn/models/Qwen/Qwen2.5-Math-PRM-72B)|qwen2_5_prm|qwen2_5_math_prm|transformers>=4.37|✘|-|[Qwen/Qwen2.5-Math-PRM-72B](https://huggingface.co/Qwen/Qwen2.5-Math-PRM-72B)|\n|[AI-ModelScope/Skywork-Reward-Llama-3.1-8B](https://modelscope.cn/models/AI-ModelScope/Skywork-Reward-Llama-3.1-8B)|llama3_2_reward|llama3_2|transformers>=4.43|✘|-|[Skywork/Skywork-Reward-Llama-3.1-8B](https://huggingface.co/Skywork/Skywork-Reward-Llama-3.1-8B)|\n|[AI-ModelScope/Skywork-Reward-Llama-3.1-8B-v0.2](https://modelscope.cn/models/AI-ModelScope/Skywork-Reward-Llama-3.1-8B-v0.2)|llama3_2_reward|llama3_2|transformers>=4.43|✘|-|[Skywork/Skywork-Reward-Llama-3.1-8B-v0.2](https://huggingface.co/Skywork/Skywork-Reward-Llama-3.1-8B-v0.2)|\n|[AI-ModelScope/GRM_Llama3.1_8B_rewardmodel-ft](https://modelscope.cn/models/AI-ModelScope/GRM_Llama3.1_8B_rewardmodel-ft)|llama3_2_reward|llama3_2|transformers>=4.43|✘|-|[Ray2333/GRM_Llama3.1_8B_rewardmodel-ft](https://huggingface.co/Ray2333/GRM_Llama3.1_8B_rewardmodel-ft)|\n|[AI-ModelScope/GRM-llama3.2-3B-rewardmodel-ft](https://modelscope.cn/models/AI-ModelScope/GRM-llama3.2-3B-rewardmodel-ft)|llama3_2_reward|llama3_2|transformers>=4.43|✘|-|[Ray2333/GRM-llama3.2-3B-rewardmodel-ft](https://huggingface.co/Ray2333/GRM-llama3.2-3B-rewardmodel-ft)|\n|[AI-ModelScope/Skywork-Reward-Gemma-2-27B](https://modelscope.cn/models/AI-ModelScope/Skywork-Reward-Gemma-2-27B)|gemma_reward|gemma|transformers>=4.42|✘|-|[Skywork/Skywork-Reward-Gemma-2-27B](https://huggingface.co/Skywork/Skywork-Reward-Gemma-2-27B)|\n|[AI-ModelScope/Skywork-Reward-Gemma-2-27B-v0.2](https://modelscope.cn/models/AI-ModelScope/Skywork-Reward-Gemma-2-27B-v0.2)|gemma_reward|gemma|transformers>=4.42|✘|-|[Skywork/Skywork-Reward-Gemma-2-27B-v0.2](https://huggingface.co/Skywork/Skywork-Reward-Gemma-2-27B-v0.2)|\n\n\n### 多模态大模型\n| Model ID | Model Type | Default Template | Requires | Support Megatron | Tags | HF Model ID |\n| -------- | -----------| ---------------- | -------- | ---------------- | ---- | ----------- |\n|[Qwen/Qwen-VL-Chat](https://modelscope.cn/models/Qwen/Qwen-VL-Chat)|qwen_vl|qwen_vl|-|✘|vision|[Qwen/Qwen-VL-Chat](https://huggingface.co/Qwen/Qwen-VL-Chat)|\n|[Qwen/Qwen-VL](https://modelscope.cn/models/Qwen/Qwen-VL)|qwen_vl|qwen_vl|-|✘|vision|[Qwen/Qwen-VL](https://huggingface.co/Qwen/Qwen-VL)|\n|[Qwen/Qwen-VL-Chat-Int4](https://modelscope.cn/models/Qwen/Qwen-VL-Chat-Int4)|qwen_vl|qwen_vl|-|✘|vision|[Qwen/Qwen-VL-Chat-Int4](https://huggingface.co/Qwen/Qwen-VL-Chat-Int4)|\n|[Qwen/Qwen-Audio-Chat](https://modelscope.cn/models/Qwen/Qwen-Audio-Chat)|qwen_audio|qwen_audio|-|✘|audio|[Qwen/Qwen-Audio-Chat](https://huggingface.co/Qwen/Qwen-Audio-Chat)|\n|[Qwen/Qwen-Audio](https://modelscope.cn/models/Qwen/Qwen-Audio)|qwen_audio|qwen_audio|-|✘|audio|[Qwen/Qwen-Audio](https://huggingface.co/Qwen/Qwen-Audio)|\n|[Qwen/Qwen2-VL-2B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-VL-2B-Instruct)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/Qwen2-VL-2B-Instruct](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct)|\n|[Qwen/Qwen2-VL-7B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-VL-7B-Instruct)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/Qwen2-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct)|\n|[Qwen/Qwen2-VL-72B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-VL-72B-Instruct)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/Qwen2-VL-72B-Instruct](https://huggingface.co/Qwen/Qwen2-VL-72B-Instruct)|\n|[Qwen/Qwen2-VL-2B](https://modelscope.cn/models/Qwen/Qwen2-VL-2B)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/Qwen2-VL-2B](https://huggingface.co/Qwen/Qwen2-VL-2B)|\n|[Qwen/Qwen2-VL-7B](https://modelscope.cn/models/Qwen/Qwen2-VL-7B)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/Qwen2-VL-7B](https://huggingface.co/Qwen/Qwen2-VL-7B)|\n|[Qwen/Qwen2-VL-72B](https://modelscope.cn/models/Qwen/Qwen2-VL-72B)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/Qwen2-VL-72B](https://huggingface.co/Qwen/Qwen2-VL-72B)|\n|[Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4](https://modelscope.cn/models/Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4](https://huggingface.co/Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4)|\n|[Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int8)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int8)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int8](https://modelscope.cn/models/Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int8)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int8](https://huggingface.co/Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int8)|\n|[Qwen/Qwen2-VL-2B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2-VL-2B-Instruct-AWQ)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2-VL-2B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct-AWQ)|\n|[Qwen/Qwen2-VL-7B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2-VL-7B-Instruct-AWQ)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2-VL-7B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2-VL-7B-Instruct-AWQ)|\n|[Qwen/Qwen2-VL-72B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2-VL-72B-Instruct-AWQ)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2-VL-72B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2-VL-72B-Instruct-AWQ)|\n|[bytedance-research/UI-TARS-2B-SFT](https://modelscope.cn/models/bytedance-research/UI-TARS-2B-SFT)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[bytedance-research/UI-TARS-2B-SFT](https://huggingface.co/bytedance-research/UI-TARS-2B-SFT)|\n|[bytedance-research/UI-TARS-7B-SFT](https://modelscope.cn/models/bytedance-research/UI-TARS-7B-SFT)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[bytedance-research/UI-TARS-7B-SFT](https://huggingface.co/bytedance-research/UI-TARS-7B-SFT)|\n|[bytedance-research/UI-TARS-7B-DPO](https://modelscope.cn/models/bytedance-research/UI-TARS-7B-DPO)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[bytedance-research/UI-TARS-7B-DPO](https://huggingface.co/bytedance-research/UI-TARS-7B-DPO)|\n|[bytedance-research/UI-TARS-72B-SFT](https://modelscope.cn/models/bytedance-research/UI-TARS-72B-SFT)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[bytedance-research/UI-TARS-72B-SFT](https://huggingface.co/bytedance-research/UI-TARS-72B-SFT)|\n|[bytedance-research/UI-TARS-72B-DPO](https://modelscope.cn/models/bytedance-research/UI-TARS-72B-DPO)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[bytedance-research/UI-TARS-72B-DPO](https://huggingface.co/bytedance-research/UI-TARS-72B-DPO)|\n|[allenai/olmOCR-7B-0225-preview](https://modelscope.cn/models/allenai/olmOCR-7B-0225-preview)|qwen2_vl|qwen2_vl|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[allenai/olmOCR-7B-0225-preview](https://huggingface.co/allenai/olmOCR-7B-0225-preview)|\n|[Qwen/QVQ-72B-Preview](https://modelscope.cn/models/Qwen/QVQ-72B-Preview)|qwen2_vl|qvq|transformers>=4.45, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/QVQ-72B-Preview](https://huggingface.co/Qwen/QVQ-72B-Preview)|\n|[Qwen/Qwen2.5-VL-3B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-VL-3B-Instruct)|qwen2_5_vl|qwen2_5_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/Qwen2.5-VL-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct)|\n|[Qwen/Qwen2.5-VL-7B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-VL-7B-Instruct)|qwen2_5_vl|qwen2_5_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct)|\n|[Qwen/Qwen2.5-VL-32B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-VL-32B-Instruct)|qwen2_5_vl|qwen2_5_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/Qwen2.5-VL-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct)|\n|[Qwen/Qwen2.5-VL-72B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-VL-72B-Instruct)|qwen2_5_vl|qwen2_5_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[Qwen/Qwen2.5-VL-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-72B-Instruct)|\n|[Qwen/Qwen2.5-VL-3B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-VL-3B-Instruct-AWQ)|qwen2_5_vl|qwen2_5_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2.5-VL-3B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-VL-7B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-VL-7B-Instruct-AWQ)|qwen2_5_vl|qwen2_5_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2.5-VL-7B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-VL-32B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-VL-32B-Instruct-AWQ)|qwen2_5_vl|qwen2_5_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2.5-VL-32B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-VL-32B-Instruct-AWQ)|\n|[Qwen/Qwen2.5-VL-72B-Instruct-AWQ](https://modelscope.cn/models/Qwen/Qwen2.5-VL-72B-Instruct-AWQ)|qwen2_5_vl|qwen2_5_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|✘|vision, video|[Qwen/Qwen2.5-VL-72B-Instruct-AWQ](https://huggingface.co/Qwen/Qwen2.5-VL-72B-Instruct-AWQ)|\n|[XiaomiMiMo/MiMo-VL-7B-SFT](https://modelscope.cn/models/XiaomiMiMo/MiMo-VL-7B-SFT)|qwen2_5_vl|mimo_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[XiaomiMiMo/MiMo-VL-7B-SFT](https://huggingface.co/XiaomiMiMo/MiMo-VL-7B-SFT)|\n|[XiaomiMiMo/MiMo-VL-7B-RL](https://modelscope.cn/models/XiaomiMiMo/MiMo-VL-7B-RL)|qwen2_5_vl|mimo_vl|transformers>=4.49, qwen_vl_utils>=0.0.6, decord|✔|vision, video|[XiaomiMiMo/MiMo-VL-7B-RL](https://huggingface.co/XiaomiMiMo/MiMo-VL-7B-RL)|\n|[Qwen/Qwen2.5-Omni-3B](https://modelscope.cn/models/Qwen/Qwen2.5-Omni-3B)|qwen2_5_omni|qwen2_5_omni|transformers>=4.50, soundfile, qwen_omni_utils, decord|✔|vision, video, audio|[Qwen/Qwen2.5-Omni-3B](https://huggingface.co/Qwen/Qwen2.5-Omni-3B)|\n|[Qwen/Qwen2.5-Omni-7B](https://modelscope.cn/models/Qwen/Qwen2.5-Omni-7B)|qwen2_5_omni|qwen2_5_omni|transformers>=4.50, soundfile, qwen_omni_utils, decord|✔|vision, video, audio|[Qwen/Qwen2.5-Omni-7B](https://huggingface.co/Qwen/Qwen2.5-Omni-7B)|\n|[Qwen/Qwen3-Omni-30B-A3B-Instruct](https://modelscope.cn/models/Qwen/Qwen3-Omni-30B-A3B-Instruct)|qwen3_omni_moe|qwen3_omni|transformers>=4.57.dev0, soundfile, decord, qwen_omni_utils>=0.0.9|✔|vision, video, audio|[Qwen/Qwen3-Omni-30B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct)|\n|[Qwen/Qwen3-Omni-30B-A3B-Thinking](https://modelscope.cn/models/Qwen/Qwen3-Omni-30B-A3B-Thinking)|qwen3_omni_moe|qwen3_omni|transformers>=4.57.dev0, soundfile, decord, qwen_omni_utils>=0.0.9|✔|vision, video, audio|[Qwen/Qwen3-Omni-30B-A3B-Thinking](https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Thinking)|\n|[Qwen/Qwen3-Omni-30B-A3B-Captioner](https://modelscope.cn/models/Qwen/Qwen3-Omni-30B-A3B-Captioner)|qwen3_omni_moe|qwen3_omni|transformers>=4.57.dev0, soundfile, decord, qwen_omni_utils>=0.0.9|✔|vision, video, audio|[Qwen/Qwen3-Omni-30B-A3B-Captioner](https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Captioner)|\n|[Qwen/Qwen2-Audio-7B-Instruct](https://modelscope.cn/models/Qwen/Qwen2-Audio-7B-Instruct)|qwen2_audio|qwen2_audio|transformers>=4.45,<4.49, librosa|✘|audio|[Qwen/Qwen2-Audio-7B-Instruct](https://huggingface.co/Qwen/Qwen2-Audio-7B-Instruct)|\n|[Qwen/Qwen2-Audio-7B](https://modelscope.cn/models/Qwen/Qwen2-Audio-7B)|qwen2_audio|qwen2_audio|transformers>=4.45,<4.49, librosa|✘|audio|[Qwen/Qwen2-Audio-7B](https://huggingface.co/Qwen/Qwen2-Audio-7B)|\n|[Qwen/Qwen3-VL-2B-Instruct](https://modelscope.cn/models/Qwen/Qwen3-VL-2B-Instruct)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-2B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct)|\n|[Qwen/Qwen3-VL-2B-Thinking](https://modelscope.cn/models/Qwen/Qwen3-VL-2B-Thinking)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-2B-Thinking](https://huggingface.co/Qwen/Qwen3-VL-2B-Thinking)|\n|[Qwen/Qwen3-VL-2B-Instruct-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-2B-Instruct-FP8)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-2B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct-FP8)|\n|[Qwen/Qwen3-VL-2B-Thinking-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-2B-Thinking-FP8)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-2B-Thinking-FP8](https://huggingface.co/Qwen/Qwen3-VL-2B-Thinking-FP8)|\n|[Qwen/Qwen3-VL-4B-Instruct](https://modelscope.cn/models/Qwen/Qwen3-VL-4B-Instruct)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-4B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct)|\n|[Qwen/Qwen3-VL-4B-Thinking](https://modelscope.cn/models/Qwen/Qwen3-VL-4B-Thinking)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-4B-Thinking](https://huggingface.co/Qwen/Qwen3-VL-4B-Thinking)|\n|[Qwen/Qwen3-VL-4B-Instruct-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-4B-Instruct-FP8)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-4B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-FP8)|\n|[Qwen/Qwen3-VL-4B-Thinking-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-4B-Thinking-FP8)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-4B-Thinking-FP8](https://huggingface.co/Qwen/Qwen3-VL-4B-Thinking-FP8)|\n|[Qwen/Qwen3-VL-8B-Instruct](https://modelscope.cn/models/Qwen/Qwen3-VL-8B-Instruct)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct)|\n|[Qwen/Qwen3-VL-8B-Thinking](https://modelscope.cn/models/Qwen/Qwen3-VL-8B-Thinking)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-8B-Thinking](https://huggingface.co/Qwen/Qwen3-VL-8B-Thinking)|\n|[Qwen/Qwen3-VL-8B-Instruct-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-8B-Instruct-FP8)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-8B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct-FP8)|\n|[Qwen/Qwen3-VL-8B-Thinking-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-8B-Thinking-FP8)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-8B-Thinking-FP8](https://huggingface.co/Qwen/Qwen3-VL-8B-Thinking-FP8)|\n|[Qwen/Qwen3-VL-32B-Instruct](https://modelscope.cn/models/Qwen/Qwen3-VL-32B-Instruct)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-32B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct)|\n|[Qwen/Qwen3-VL-32B-Thinking](https://modelscope.cn/models/Qwen/Qwen3-VL-32B-Thinking)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-32B-Thinking](https://huggingface.co/Qwen/Qwen3-VL-32B-Thinking)|\n|[Qwen/Qwen3-VL-32B-Instruct-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-32B-Instruct-FP8)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-32B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct-FP8)|\n|[Qwen/Qwen3-VL-32B-Thinking-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-32B-Thinking-FP8)|qwen3_vl|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-32B-Thinking-FP8](https://huggingface.co/Qwen/Qwen3-VL-32B-Thinking-FP8)|\n|[Qwen/Qwen3-VL-30B-A3B-Instruct](https://modelscope.cn/models/Qwen/Qwen3-VL-30B-A3B-Instruct)|qwen3_vl_moe|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-30B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Instruct)|\n|[Qwen/Qwen3-VL-30B-A3B-Thinking](https://modelscope.cn/models/Qwen/Qwen3-VL-30B-A3B-Thinking)|qwen3_vl_moe|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-30B-A3B-Thinking](https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Thinking)|\n|[Qwen/Qwen3-VL-30B-A3B-Instruct-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-30B-A3B-Instruct-FP8)|qwen3_vl_moe|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-30B-A3B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Instruct-FP8)|\n|[Qwen/Qwen3-VL-30B-A3B-Thinking-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-30B-A3B-Thinking-FP8)|qwen3_vl_moe|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-30B-A3B-Thinking-FP8](https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Thinking-FP8)|\n|[Qwen/Qwen3-VL-235B-A22B-Instruct](https://modelscope.cn/models/Qwen/Qwen3-VL-235B-A22B-Instruct)|qwen3_vl_moe|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-235B-A22B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct)|\n|[Qwen/Qwen3-VL-235B-A22B-Thinking](https://modelscope.cn/models/Qwen/Qwen3-VL-235B-A22B-Thinking)|qwen3_vl_moe|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-235B-A22B-Thinking](https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking)|\n|[Qwen/Qwen3-VL-235B-A22B-Instruct-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-235B-A22B-Instruct-FP8)|qwen3_vl_moe|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-235B-A22B-Instruct-FP8](https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct-FP8)|\n|[Qwen/Qwen3-VL-235B-A22B-Thinking-FP8](https://modelscope.cn/models/Qwen/Qwen3-VL-235B-A22B-Thinking-FP8)|qwen3_vl_moe|qwen3_vl|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-235B-A22B-Thinking-FP8](https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking-FP8)|\n|[Qwen/Qwen3-VL-Embedding-2B](https://modelscope.cn/models/Qwen/Qwen3-VL-Embedding-2B)|qwen3_vl_emb|qwen3_vl_emb|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-Embedding-2B](https://huggingface.co/Qwen/Qwen3-VL-Embedding-2B)|\n|[Qwen/Qwen3-VL-Embedding-8B](https://modelscope.cn/models/Qwen/Qwen3-VL-Embedding-8B)|qwen3_vl_emb|qwen3_vl_emb|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-Embedding-8B](https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B)|\n|[Qwen/Qwen3-VL-Reranker-2B](https://modelscope.cn/models/Qwen/Qwen3-VL-Reranker-2B)|qwen3_vl_reranker|qwen3_vl_reranker|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-Reranker-2B](https://huggingface.co/Qwen/Qwen3-VL-Reranker-2B)|\n|[Qwen/Qwen3-VL-Reranker-8B](https://modelscope.cn/models/Qwen/Qwen3-VL-Reranker-8B)|qwen3_vl_reranker|qwen3_vl_reranker|transformers>=4.57, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3-VL-Reranker-8B](https://huggingface.co/Qwen/Qwen3-VL-Reranker-8B)|\n|[Qwen/Qwen3.5-0.8B](https://modelscope.cn/models/Qwen/Qwen3.5-0.8B)|qwen3_5|qwen3_5|transformers>=5.0.0.dev, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-0.8B](https://huggingface.co/Qwen/Qwen3.5-0.8B)|\n|[Qwen/Qwen3.5-2B](https://modelscope.cn/models/Qwen/Qwen3.5-2B)|qwen3_5|qwen3_5|transformers>=5.0.0.dev, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-2B](https://huggingface.co/Qwen/Qwen3.5-2B)|\n|[Qwen/Qwen3.5-4B](https://modelscope.cn/models/Qwen/Qwen3.5-4B)|qwen3_5|qwen3_5|transformers>=5.0.0.dev, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-4B](https://huggingface.co/Qwen/Qwen3.5-4B)|\n|[Qwen/Qwen3.5-9B](https://modelscope.cn/models/Qwen/Qwen3.5-9B)|qwen3_5|qwen3_5|transformers>=5.0.0.dev, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-9B](https://huggingface.co/Qwen/Qwen3.5-9B)|\n|[Qwen/Qwen3.5-27B](https://modelscope.cn/models/Qwen/Qwen3.5-27B)|qwen3_5|qwen3_5|transformers>=5.0.0.dev, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-27B](https://huggingface.co/Qwen/Qwen3.5-27B)|\n|[Qwen/Qwen3.5-27B-FP8](https://modelscope.cn/models/Qwen/Qwen3.5-27B-FP8)|qwen3_5|qwen3_5|transformers>=5.0.0.dev, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-27B-FP8](https://huggingface.co/Qwen/Qwen3.5-27B-FP8)|\n|[Qwen/Qwen3.5-0.8B-Base](https://modelscope.cn/models/Qwen/Qwen3.5-0.8B-Base)|qwen3_5|qwen3_5|transformers>=5.0.0.dev, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-0.8B-Base](https://huggingface.co/Qwen/Qwen3.5-0.8B-Base)|\n|[Qwen/Qwen3.5-2B-Base](https://modelscope.cn/models/Qwen/Qwen3.5-2B-Base)|qwen3_5|qwen3_5|transformers>=5.0.0.dev, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base)|\n|[Qwen/Qwen3.5-4B-Base](https://modelscope.cn/models/Qwen/Qwen3.5-4B-Base)|qwen3_5|qwen3_5|transformers>=5.0.0.dev, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-4B-Base](https://huggingface.co/Qwen/Qwen3.5-4B-Base)|\n|[Qwen/Qwen3.5-9B-Base](https://modelscope.cn/models/Qwen/Qwen3.5-9B-Base)|qwen3_5|qwen3_5|transformers>=5.0.0.dev, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-9B-Base](https://huggingface.co/Qwen/Qwen3.5-9B-Base)|\n|[Qwen/Qwen3.5-35B-A3B-Base](https://modelscope.cn/models/Qwen/Qwen3.5-35B-A3B-Base)|qwen3_5_moe|qwen3_5|transformers>=5.2.0, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-35B-A3B-Base](https://huggingface.co/Qwen/Qwen3.5-35B-A3B-Base)|\n|[Qwen/Qwen3.5-35B-A3B](https://modelscope.cn/models/Qwen/Qwen3.5-35B-A3B)|qwen3_5_moe|qwen3_5|transformers>=5.2.0, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-35B-A3B](https://huggingface.co/Qwen/Qwen3.5-35B-A3B)|\n|[Qwen/Qwen3.5-122B-A10B](https://modelscope.cn/models/Qwen/Qwen3.5-122B-A10B)|qwen3_5_moe|qwen3_5|transformers>=5.2.0, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-122B-A10B](https://huggingface.co/Qwen/Qwen3.5-122B-A10B)|\n|[Qwen/Qwen3.5-397B-A17B](https://modelscope.cn/models/Qwen/Qwen3.5-397B-A17B)|qwen3_5_moe|qwen3_5|transformers>=5.2.0, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-397B-A17B](https://huggingface.co/Qwen/Qwen3.5-397B-A17B)|\n|[Qwen/Qwen3.5-35B-A3B-FP8](https://modelscope.cn/models/Qwen/Qwen3.5-35B-A3B-FP8)|qwen3_5_moe|qwen3_5|transformers>=5.2.0, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-35B-A3B-FP8](https://huggingface.co/Qwen/Qwen3.5-35B-A3B-FP8)|\n|[Qwen/Qwen3.5-122B-A10B-FP8](https://modelscope.cn/models/Qwen/Qwen3.5-122B-A10B-FP8)|qwen3_5_moe|qwen3_5|transformers>=5.2.0, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-122B-A10B-FP8](https://huggingface.co/Qwen/Qwen3.5-122B-A10B-FP8)|\n|[Qwen/Qwen3.5-397B-A17B-FP8](https://modelscope.cn/models/Qwen/Qwen3.5-397B-A17B-FP8)|qwen3_5_moe|qwen3_5|transformers>=5.2.0, qwen_vl_utils>=0.0.14, decord|✔|vision, video|[Qwen/Qwen3.5-397B-A17B-FP8](https://huggingface.co/Qwen/Qwen3.5-397B-A17B-FP8)|\n|[iic/gme-Qwen2-VL-2B-Instruct](https://modelscope.cn/models/iic/gme-Qwen2-VL-2B-Instruct)|qwen2_gme|qwen2_gme|-|✘|vision|[Alibaba-NLP/gme-Qwen2-VL-2B-Instruct](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct)|\n|[iic/gme-Qwen2-VL-7B-Instruct](https://modelscope.cn/models/iic/gme-Qwen2-VL-7B-Instruct)|qwen2_gme|qwen2_gme|-|✘|vision|[Alibaba-NLP/gme-Qwen2-VL-7B-Instruct](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct)|\n|[AIDC-AI/Ovis1.6-Gemma2-9B](https://modelscope.cn/models/AIDC-AI/Ovis1.6-Gemma2-9B)|ovis1_6|ovis1_6|transformers>=4.42|✘|vision|[AIDC-AI/Ovis1.6-Gemma2-9B](https://huggingface.co/AIDC-AI/Ovis1.6-Gemma2-9B)|\n|[AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4](https://modelscope.cn/models/AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4)|ovis1_6|ovis1_6|transformers>=4.42|✘|vision|[AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4](https://huggingface.co/AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4)|\n|[AIDC-AI/Ovis1.6-Gemma2-27B](https://modelscope.cn/models/AIDC-AI/Ovis1.6-Gemma2-27B)|ovis1_6|ovis1_6|transformers>=4.42|✘|vision|[AIDC-AI/Ovis1.6-Gemma2-27B](https://huggingface.co/AIDC-AI/Ovis1.6-Gemma2-27B)|\n|[AIDC-AI/Ovis1.6-Llama3.2-3B](https://modelscope.cn/models/AIDC-AI/Ovis1.6-Llama3.2-3B)|ovis1_6|ovis1_6_llama3|-|✘|vision|[AIDC-AI/Ovis1.6-Llama3.2-3B](https://huggingface.co/AIDC-AI/Ovis1.6-Llama3.2-3B)|\n|[AIDC-AI/Ovis2-1B](https://modelscope.cn/models/AIDC-AI/Ovis2-1B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|✘|vision|[AIDC-AI/Ovis2-1B](https://huggingface.co/AIDC-AI/Ovis2-1B)|\n|[AIDC-AI/Ovis2-2B](https://modelscope.cn/models/AIDC-AI/Ovis2-2B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|✘|vision|[AIDC-AI/Ovis2-2B](https://huggingface.co/AIDC-AI/Ovis2-2B)|\n|[AIDC-AI/Ovis2-4B](https://modelscope.cn/models/AIDC-AI/Ovis2-4B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|✘|vision|[AIDC-AI/Ovis2-4B](https://huggingface.co/AIDC-AI/Ovis2-4B)|\n|[AIDC-AI/Ovis2-8B](https://modelscope.cn/models/AIDC-AI/Ovis2-8B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|✘|vision|[AIDC-AI/Ovis2-8B](https://huggingface.co/AIDC-AI/Ovis2-8B)|\n|[AIDC-AI/Ovis2-16B](https://modelscope.cn/models/AIDC-AI/Ovis2-16B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|✘|vision|[AIDC-AI/Ovis2-16B](https://huggingface.co/AIDC-AI/Ovis2-16B)|\n|[AIDC-AI/Ovis2-34B](https://modelscope.cn/models/AIDC-AI/Ovis2-34B)|ovis2|ovis2|transformers>=4.46.2, moviepy<2|✘|vision|[AIDC-AI/Ovis2-34B](https://huggingface.co/AIDC-AI/Ovis2-34B)|\n|[AIDC-AI/Ovis2.5-2B](https://modelscope.cn/models/AIDC-AI/Ovis2.5-2B)|ovis2_5|ovis2_5|transformers>=4.46.2, moviepy<2|✔|vision|[AIDC-AI/Ovis2.5-2B](https://huggingface.co/AIDC-AI/Ovis2.5-2B)|\n|[AIDC-AI/Ovis2.5-9B](https://modelscope.cn/models/AIDC-AI/Ovis2.5-9B)|ovis2_5|ovis2_5|transformers>=4.46.2, moviepy<2|✔|vision|[AIDC-AI/Ovis2.5-9B](https://huggingface.co/AIDC-AI/Ovis2.5-9B)|\n|[mispeech/midashenglm-7b](https://modelscope.cn/models/mispeech/midashenglm-7b)|midashenglm|midashenglm|transformers>=4.52, soundfile|✘|audio|[mispeech/midashenglm-7b](https://huggingface.co/mispeech/midashenglm-7b)|\n|[ZhipuAI/glm-4v-9b](https://modelscope.cn/models/ZhipuAI/glm-4v-9b)|chatglm4v|chatglm4v|transformers>=4.42,<4.45|✘|-|[zai-org/glm-4v-9b](https://huggingface.co/zai-org/glm-4v-9b)|\n|[ZhipuAI/cogagent-9b-20241220](https://modelscope.cn/models/ZhipuAI/cogagent-9b-20241220)|chatglm4v|chatglm4v|transformers>=4.42|✘|-|[zai-org/cogagent-9b-20241220](https://huggingface.co/zai-org/cogagent-9b-20241220)|\n|[ZhipuAI/GLM-4.1V-9B-Base](https://modelscope.cn/models/ZhipuAI/GLM-4.1V-9B-Base)|glm4v|glm4v|transformers>=4.53|✔|-|[zai-org/GLM-4.1V-9B-Base](https://huggingface.co/zai-org/GLM-4.1V-9B-Base)|\n|[ZhipuAI/GLM-4.1V-9B-Thinking](https://modelscope.cn/models/ZhipuAI/GLM-4.1V-9B-Thinking)|glm4v|glm4v|transformers>=4.53|✔|-|[zai-org/GLM-4.1V-9B-Thinking](https://huggingface.co/zai-org/GLM-4.1V-9B-Thinking)|\n|[ZhipuAI/AutoGLM-Phone-9B](https://modelscope.cn/models/ZhipuAI/AutoGLM-Phone-9B)|glm4v|glm4v|transformers>=4.53|✔|-|[zai-org/AutoGLM-Phone-9B](https://huggingface.co/zai-org/AutoGLM-Phone-9B)|\n|[ZhipuAI/Glyph](https://modelscope.cn/models/ZhipuAI/Glyph)|glm4v|glm4_5v|transformers>=4.57|✔|-|[zai-org/Glyph](https://huggingface.co/zai-org/Glyph)|\n|[ZhipuAI/GLM-4.6V-Flash](https://modelscope.cn/models/ZhipuAI/GLM-4.6V-Flash)|glm4v|glm4_5v|transformers>=5.0.0.dev|✔|-|[zai-org/GLM-4.6V-Flash](https://huggingface.co/zai-org/GLM-4.6V-Flash)|\n|[ZhipuAI/GLM-4.5V](https://modelscope.cn/models/ZhipuAI/GLM-4.5V)|glm4v_moe|glm4_5v|transformers>=4.56|✔|-|[zai-org/GLM-4.5V](https://huggingface.co/zai-org/GLM-4.5V)|\n|[ZhipuAI/GLM-4.5V-FP8](https://modelscope.cn/models/ZhipuAI/GLM-4.5V-FP8)|glm4v_moe|glm4_5v|transformers>=4.56|✘|-|[zai-org/GLM-4.5V-FP8](https://huggingface.co/zai-org/GLM-4.5V-FP8)|\n|[ZhipuAI/GLM-4.6V](https://modelscope.cn/models/ZhipuAI/GLM-4.6V)|glm4v_moe|glm4_5v|transformers>=5.0.0.dev|✔|-|[zai-org/GLM-4.6V](https://huggingface.co/zai-org/GLM-4.6V)|\n|[ZhipuAI/GLM-4.6V-FP8](https://modelscope.cn/models/ZhipuAI/GLM-4.6V-FP8)|glm4v_moe|glm4_5v|transformers>=5.0.0.dev|✘|-|[zai-org/GLM-4.6V-FP8](https://huggingface.co/zai-org/GLM-4.6V-FP8)|\n|[ZhipuAI/glm-edge-v-2b](https://modelscope.cn/models/ZhipuAI/glm-edge-v-2b)|glm_edge_v|glm_edge_v|transformers>=4.46|✘|vision|[zai-org/glm-edge-v-2b](https://huggingface.co/zai-org/glm-edge-v-2b)|\n|[ZhipuAI/glm-edge-4b-chat](https://modelscope.cn/models/ZhipuAI/glm-edge-4b-chat)|glm_edge_v|glm_edge_v|transformers>=4.46|✘|vision|[zai-org/glm-edge-4b-chat](https://huggingface.co/zai-org/glm-edge-4b-chat)|\n|[ZhipuAI/GLM-OCR](https://modelscope.cn/models/ZhipuAI/GLM-OCR)|glm_ocr|glm_ocr|transformers>=5.0.1dev0|✘|-|[zai-org/GLM-OCR](https://huggingface.co/zai-org/GLM-OCR)|\n|[ZhipuAI/cogvlm-chat](https://modelscope.cn/models/ZhipuAI/cogvlm-chat)|cogvlm|cogvlm|transformers<4.42|✘|-|[zai-org/cogvlm-chat-hf](https://huggingface.co/zai-org/cogvlm-chat-hf)|\n|[ZhipuAI/cogagent-vqa](https://modelscope.cn/models/ZhipuAI/cogagent-vqa)|cogagent_vqa|cogagent_vqa|transformers<4.42|✘|-|[zai-org/cogagent-vqa-hf](https://huggingface.co/zai-org/cogagent-vqa-hf)|\n|[ZhipuAI/cogagent-chat](https://modelscope.cn/models/ZhipuAI/cogagent-chat)|cogagent_chat|cogagent_chat|transformers<4.42, timm|✘|-|[zai-org/cogagent-chat-hf](https://huggingface.co/zai-org/cogagent-chat-hf)|\n|[ZhipuAI/cogvlm2-llama3-chat-19B](https://modelscope.cn/models/ZhipuAI/cogvlm2-llama3-chat-19B)|cogvlm2|cogvlm2|transformers<4.42|✘|-|[zai-org/cogvlm2-llama3-chat-19B](https://huggingface.co/zai-org/cogvlm2-llama3-chat-19B)|\n|[ZhipuAI/cogvlm2-llama3-chinese-chat-19B](https://modelscope.cn/models/ZhipuAI/cogvlm2-llama3-chinese-chat-19B)|cogvlm2|cogvlm2|transformers<4.42|✘|-|[zai-org/cogvlm2-llama3-chinese-chat-19B](https://huggingface.co/zai-org/cogvlm2-llama3-chinese-chat-19B)|\n|[ZhipuAI/cogvlm2-video-llama3-chat](https://modelscope.cn/models/ZhipuAI/cogvlm2-video-llama3-chat)|cogvlm2_video|cogvlm2_video|decord, pytorchvideo, transformers>=4.42|✘|video|[zai-org/cogvlm2-video-llama3-chat](https://huggingface.co/zai-org/cogvlm2-video-llama3-chat)|\n|[OpenGVLab/Mini-InternVL-Chat-2B-V1-5](https://modelscope.cn/models/OpenGVLab/Mini-InternVL-Chat-2B-V1-5)|internvl|internvl|transformers>=4.35, timm|✘|vision|[OpenGVLab/Mini-InternVL-Chat-2B-V1-5](https://huggingface.co/OpenGVLab/Mini-InternVL-Chat-2B-V1-5)|\n|[AI-ModelScope/InternVL-Chat-V1-5](https://modelscope.cn/models/AI-ModelScope/InternVL-Chat-V1-5)|internvl|internvl|transformers>=4.35, timm|✘|vision|[OpenGVLab/InternVL-Chat-V1-5](https://huggingface.co/OpenGVLab/InternVL-Chat-V1-5)|\n|[AI-ModelScope/InternVL-Chat-V1-5-int8](https://modelscope.cn/models/AI-ModelScope/InternVL-Chat-V1-5-int8)|internvl|internvl|transformers>=4.35, timm|✘|vision|[OpenGVLab/InternVL-Chat-V1-5-int8](https://huggingface.co/OpenGVLab/InternVL-Chat-V1-5-int8)|\n|[OpenGVLab/Mini-InternVL-Chat-4B-V1-5](https://modelscope.cn/models/OpenGVLab/Mini-InternVL-Chat-4B-V1-5)|internvl_phi3|internvl_phi3|transformers>=4.35,<4.42, timm|✘|vision|[OpenGVLab/Mini-InternVL-Chat-4B-V1-5](https://huggingface.co/OpenGVLab/Mini-InternVL-Chat-4B-V1-5)|\n|[OpenGVLab/InternVL2-1B](https://modelscope.cn/models/OpenGVLab/InternVL2-1B)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-1B](https://huggingface.co/OpenGVLab/InternVL2-1B)|\n|[OpenGVLab/InternVL2-2B](https://modelscope.cn/models/OpenGVLab/InternVL2-2B)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-2B](https://huggingface.co/OpenGVLab/InternVL2-2B)|\n|[OpenGVLab/InternVL2-8B](https://modelscope.cn/models/OpenGVLab/InternVL2-8B)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-8B](https://huggingface.co/OpenGVLab/InternVL2-8B)|\n|[OpenGVLab/InternVL2-26B](https://modelscope.cn/models/OpenGVLab/InternVL2-26B)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-26B](https://huggingface.co/OpenGVLab/InternVL2-26B)|\n|[OpenGVLab/InternVL2-40B](https://modelscope.cn/models/OpenGVLab/InternVL2-40B)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-40B](https://huggingface.co/OpenGVLab/InternVL2-40B)|\n|[OpenGVLab/InternVL2-Llama3-76B](https://modelscope.cn/models/OpenGVLab/InternVL2-Llama3-76B)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-Llama3-76B](https://huggingface.co/OpenGVLab/InternVL2-Llama3-76B)|\n|[OpenGVLab/InternVL2-2B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL2-2B-AWQ)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-2B-AWQ](https://huggingface.co/OpenGVLab/InternVL2-2B-AWQ)|\n|[OpenGVLab/InternVL2-8B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL2-8B-AWQ)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-8B-AWQ](https://huggingface.co/OpenGVLab/InternVL2-8B-AWQ)|\n|[OpenGVLab/InternVL2-26B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL2-26B-AWQ)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-26B-AWQ](https://huggingface.co/OpenGVLab/InternVL2-26B-AWQ)|\n|[OpenGVLab/InternVL2-40B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL2-40B-AWQ)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-40B-AWQ](https://huggingface.co/OpenGVLab/InternVL2-40B-AWQ)|\n|[OpenGVLab/InternVL2-Llama3-76B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL2-Llama3-76B-AWQ)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-Llama3-76B-AWQ](https://huggingface.co/OpenGVLab/InternVL2-Llama3-76B-AWQ)|\n|[OpenGVLab/InternVL2-8B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL2-8B-MPO)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-8B-MPO](https://huggingface.co/OpenGVLab/InternVL2-8B-MPO)|\n|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-1B-Pretrain](https://modelscope.cn/models/OpenGVLab/InternVL2-Pretrain-Models)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-1B-Pretrain](https://huggingface.co/OpenGVLab/InternVL2-Pretrain-Models)|\n|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-2B-Pretrain](https://modelscope.cn/models/OpenGVLab/InternVL2-Pretrain-Models)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-2B-Pretrain](https://huggingface.co/OpenGVLab/InternVL2-Pretrain-Models)|\n|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-4B-Pretrain](https://modelscope.cn/models/OpenGVLab/InternVL2-Pretrain-Models)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-4B-Pretrain](https://huggingface.co/OpenGVLab/InternVL2-Pretrain-Models)|\n|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-8B-Pretrain](https://modelscope.cn/models/OpenGVLab/InternVL2-Pretrain-Models)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-8B-Pretrain](https://huggingface.co/OpenGVLab/InternVL2-Pretrain-Models)|\n|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-26B-Pretrain](https://modelscope.cn/models/OpenGVLab/InternVL2-Pretrain-Models)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-26B-Pretrain](https://huggingface.co/OpenGVLab/InternVL2-Pretrain-Models)|\n|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-40B-Pretrain](https://modelscope.cn/models/OpenGVLab/InternVL2-Pretrain-Models)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-40B-Pretrain](https://huggingface.co/OpenGVLab/InternVL2-Pretrain-Models)|\n|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-Llama3-76B-Pretrain](https://modelscope.cn/models/OpenGVLab/InternVL2-Pretrain-Models)|internvl2|internvl2|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2-Pretrain-Models:InternVL2-Llama3-76B-Pretrain](https://huggingface.co/OpenGVLab/InternVL2-Pretrain-Models)|\n|[OpenGVLab/InternVL2-4B](https://modelscope.cn/models/OpenGVLab/InternVL2-4B)|internvl2_phi3|internvl2_phi3|transformers>=4.36,<4.42, timm|✘|vision, video|[OpenGVLab/InternVL2-4B](https://huggingface.co/OpenGVLab/InternVL2-4B)|\n|[OpenGVLab/InternVL2_5-1B](https://modelscope.cn/models/OpenGVLab/InternVL2_5-1B)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-1B](https://huggingface.co/OpenGVLab/InternVL2_5-1B)|\n|[OpenGVLab/InternVL2_5-2B](https://modelscope.cn/models/OpenGVLab/InternVL2_5-2B)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-2B](https://huggingface.co/OpenGVLab/InternVL2_5-2B)|\n|[OpenGVLab/InternVL2_5-4B](https://modelscope.cn/models/OpenGVLab/InternVL2_5-4B)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-4B](https://huggingface.co/OpenGVLab/InternVL2_5-4B)|\n|[OpenGVLab/InternVL2_5-8B](https://modelscope.cn/models/OpenGVLab/InternVL2_5-8B)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-8B](https://huggingface.co/OpenGVLab/InternVL2_5-8B)|\n|[OpenGVLab/InternVL2_5-26B](https://modelscope.cn/models/OpenGVLab/InternVL2_5-26B)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-26B](https://huggingface.co/OpenGVLab/InternVL2_5-26B)|\n|[OpenGVLab/InternVL2_5-38B](https://modelscope.cn/models/OpenGVLab/InternVL2_5-38B)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-38B](https://huggingface.co/OpenGVLab/InternVL2_5-38B)|\n|[OpenGVLab/InternVL2_5-78B](https://modelscope.cn/models/OpenGVLab/InternVL2_5-78B)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-78B](https://huggingface.co/OpenGVLab/InternVL2_5-78B)|\n|[OpenGVLab/InternVL2_5-4B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL2_5-4B-AWQ)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-4B-AWQ](https://huggingface.co/OpenGVLab/InternVL2_5-4B-AWQ)|\n|[OpenGVLab/InternVL2_5-8B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL2_5-8B-AWQ)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-8B-AWQ](https://huggingface.co/OpenGVLab/InternVL2_5-8B-AWQ)|\n|[OpenGVLab/InternVL2_5-26B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL2_5-26B-AWQ)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-26B-AWQ](https://huggingface.co/OpenGVLab/InternVL2_5-26B-AWQ)|\n|[OpenGVLab/InternVL2_5-38B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL2_5-38B-AWQ)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-38B-AWQ](https://huggingface.co/OpenGVLab/InternVL2_5-38B-AWQ)|\n|[OpenGVLab/InternVL2_5-78B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL2_5-78B-AWQ)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-78B-AWQ](https://huggingface.co/OpenGVLab/InternVL2_5-78B-AWQ)|\n|[OpenGVLab/InternVL2_5-1B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL2_5-1B-MPO)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-1B-MPO](https://huggingface.co/OpenGVLab/InternVL2_5-1B-MPO)|\n|[OpenGVLab/InternVL2_5-2B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL2_5-2B-MPO)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-2B-MPO](https://huggingface.co/OpenGVLab/InternVL2_5-2B-MPO)|\n|[OpenGVLab/InternVL2_5-4B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL2_5-4B-MPO)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-4B-MPO](https://huggingface.co/OpenGVLab/InternVL2_5-4B-MPO)|\n|[OpenGVLab/InternVL2_5-8B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL2_5-8B-MPO)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-8B-MPO](https://huggingface.co/OpenGVLab/InternVL2_5-8B-MPO)|\n|[OpenGVLab/InternVL2_5-26B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL2_5-26B-MPO)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-26B-MPO](https://huggingface.co/OpenGVLab/InternVL2_5-26B-MPO)|\n|[OpenGVLab/InternVL2_5-38B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL2_5-38B-MPO)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-38B-MPO](https://huggingface.co/OpenGVLab/InternVL2_5-38B-MPO)|\n|[OpenGVLab/InternVL2_5-78B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL2_5-78B-MPO)|internvl2_5|internvl2_5|transformers>=4.36, timm|✘|vision, video|[OpenGVLab/InternVL2_5-78B-MPO](https://huggingface.co/OpenGVLab/InternVL2_5-78B-MPO)|\n|[OpenGVLab/InternVL3-1B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3-1B-Pretrained)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-1B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3-1B-Pretrained)|\n|[OpenGVLab/InternVL3-2B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3-2B-Pretrained)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-2B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3-2B-Pretrained)|\n|[OpenGVLab/InternVL3-8B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3-8B-Pretrained)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-8B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3-8B-Pretrained)|\n|[OpenGVLab/InternVL3-9B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3-9B-Pretrained)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-9B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3-9B-Pretrained)|\n|[OpenGVLab/InternVL3-14B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3-14B-Pretrained)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-14B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3-14B-Pretrained)|\n|[OpenGVLab/InternVL3-38B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3-38B-Pretrained)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-38B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3-38B-Pretrained)|\n|[OpenGVLab/InternVL3-78B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3-78B-Pretrained)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-78B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3-78B-Pretrained)|\n|[OpenGVLab/InternVL3-1B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3-1B-Instruct)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-1B-Instruct](https://huggingface.co/OpenGVLab/InternVL3-1B-Instruct)|\n|[OpenGVLab/InternVL3-2B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3-2B-Instruct)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-2B-Instruct](https://huggingface.co/OpenGVLab/InternVL3-2B-Instruct)|\n|[OpenGVLab/InternVL3-8B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3-8B-Instruct)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-8B-Instruct](https://huggingface.co/OpenGVLab/InternVL3-8B-Instruct)|\n|[OpenGVLab/InternVL3-9B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3-9B-Instruct)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-9B-Instruct](https://huggingface.co/OpenGVLab/InternVL3-9B-Instruct)|\n|[OpenGVLab/InternVL3-14B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3-14B-Instruct)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-14B-Instruct](https://huggingface.co/OpenGVLab/InternVL3-14B-Instruct)|\n|[OpenGVLab/InternVL3-38B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3-38B-Instruct)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-38B-Instruct](https://huggingface.co/OpenGVLab/InternVL3-38B-Instruct)|\n|[OpenGVLab/InternVL3-78B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3-78B-Instruct)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-78B-Instruct](https://huggingface.co/OpenGVLab/InternVL3-78B-Instruct)|\n|[OpenGVLab/InternVL3-1B](https://modelscope.cn/models/OpenGVLab/InternVL3-1B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-1B](https://huggingface.co/OpenGVLab/InternVL3-1B)|\n|[OpenGVLab/InternVL3-2B](https://modelscope.cn/models/OpenGVLab/InternVL3-2B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-2B](https://huggingface.co/OpenGVLab/InternVL3-2B)|\n|[OpenGVLab/InternVL3-8B](https://modelscope.cn/models/OpenGVLab/InternVL3-8B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-8B](https://huggingface.co/OpenGVLab/InternVL3-8B)|\n|[OpenGVLab/InternVL3-9B](https://modelscope.cn/models/OpenGVLab/InternVL3-9B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-9B](https://huggingface.co/OpenGVLab/InternVL3-9B)|\n|[OpenGVLab/InternVL3-14B](https://modelscope.cn/models/OpenGVLab/InternVL3-14B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-14B](https://huggingface.co/OpenGVLab/InternVL3-14B)|\n|[OpenGVLab/InternVL3-38B](https://modelscope.cn/models/OpenGVLab/InternVL3-38B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-38B](https://huggingface.co/OpenGVLab/InternVL3-38B)|\n|[OpenGVLab/InternVL3-78B](https://modelscope.cn/models/OpenGVLab/InternVL3-78B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3-78B](https://huggingface.co/OpenGVLab/InternVL3-78B)|\n|[OpenGVLab/InternVL3-1B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL3-1B-AWQ)|internvl3|internvl2_5|transformers>=4.37.2, timm|✘|vision, video|[OpenGVLab/InternVL3-1B-AWQ](https://huggingface.co/OpenGVLab/InternVL3-1B-AWQ)|\n|[OpenGVLab/InternVL3-2B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL3-2B-AWQ)|internvl3|internvl2_5|transformers>=4.37.2, timm|✘|vision, video|[OpenGVLab/InternVL3-2B-AWQ](https://huggingface.co/OpenGVLab/InternVL3-2B-AWQ)|\n|[OpenGVLab/InternVL3-8B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL3-8B-AWQ)|internvl3|internvl2_5|transformers>=4.37.2, timm|✘|vision, video|[OpenGVLab/InternVL3-8B-AWQ](https://huggingface.co/OpenGVLab/InternVL3-8B-AWQ)|\n|[OpenGVLab/InternVL3-9B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL3-9B-AWQ)|internvl3|internvl2_5|transformers>=4.37.2, timm|✘|vision, video|[OpenGVLab/InternVL3-9B-AWQ](https://huggingface.co/OpenGVLab/InternVL3-9B-AWQ)|\n|[OpenGVLab/InternVL3-14B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL3-14B-AWQ)|internvl3|internvl2_5|transformers>=4.37.2, timm|✘|vision, video|[OpenGVLab/InternVL3-14B-AWQ](https://huggingface.co/OpenGVLab/InternVL3-14B-AWQ)|\n|[OpenGVLab/InternVL3-38B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL3-38B-AWQ)|internvl3|internvl2_5|transformers>=4.37.2, timm|✘|vision, video|[OpenGVLab/InternVL3-38B-AWQ](https://huggingface.co/OpenGVLab/InternVL3-38B-AWQ)|\n|[OpenGVLab/InternVL3-78B-AWQ](https://modelscope.cn/models/OpenGVLab/InternVL3-78B-AWQ)|internvl3|internvl2_5|transformers>=4.37.2, timm|✘|vision, video|[OpenGVLab/InternVL3-78B-AWQ](https://huggingface.co/OpenGVLab/InternVL3-78B-AWQ)|\n|[SenseNova/SenseNova-SI-InternVL3-2B](https://modelscope.cn/models/SenseNova/SenseNova-SI-InternVL3-2B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[sensenova/SenseNova-SI-InternVL3-2B](https://huggingface.co/sensenova/SenseNova-SI-InternVL3-2B)|\n|[SenseNova/SenseNova-SI-InternVL3-8B](https://modelscope.cn/models/SenseNova/SenseNova-SI-InternVL3-8B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[sensenova/SenseNova-SI-InternVL3-8B](https://huggingface.co/sensenova/SenseNova-SI-InternVL3-8B)|\n|[SenseNova/SenseNova-SI-1.1-InternVL3-2B](https://modelscope.cn/models/SenseNova/SenseNova-SI-1.1-InternVL3-2B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[sensenova/SenseNova-SI-1.1-InternVL3-2B](https://huggingface.co/sensenova/SenseNova-SI-1.1-InternVL3-2B)|\n|[SenseNova/SenseNova-SI-1.1-InternVL3-8B](https://modelscope.cn/models/SenseNova/SenseNova-SI-1.1-InternVL3-8B)|internvl3|internvl2_5|transformers>=4.37.2, timm|✔|vision, video|[sensenova/SenseNova-SI-1.1-InternVL3-8B](https://huggingface.co/sensenova/SenseNova-SI-1.1-InternVL3-8B)|\n|[OpenGVLab/InternVL3-1B-hf](https://modelscope.cn/models/OpenGVLab/InternVL3-1B-hf)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3-1B-hf](https://huggingface.co/OpenGVLab/InternVL3-1B-hf)|\n|[OpenGVLab/InternVL3-2B-hf](https://modelscope.cn/models/OpenGVLab/InternVL3-2B-hf)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3-2B-hf](https://huggingface.co/OpenGVLab/InternVL3-2B-hf)|\n|[OpenGVLab/InternVL3-8B-hf](https://modelscope.cn/models/OpenGVLab/InternVL3-8B-hf)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3-8B-hf](https://huggingface.co/OpenGVLab/InternVL3-8B-hf)|\n|[OpenGVLab/InternVL3-9B-hf](https://modelscope.cn/models/OpenGVLab/InternVL3-9B-hf)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3-9B-hf](https://huggingface.co/OpenGVLab/InternVL3-9B-hf)|\n|[OpenGVLab/InternVL3-14B-hf](https://modelscope.cn/models/OpenGVLab/InternVL3-14B-hf)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3-14B-hf](https://huggingface.co/OpenGVLab/InternVL3-14B-hf)|\n|[OpenGVLab/InternVL3-38B-hf](https://modelscope.cn/models/OpenGVLab/InternVL3-38B-hf)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3-38B-hf](https://huggingface.co/OpenGVLab/InternVL3-38B-hf)|\n|[OpenGVLab/InternVL3-78B-hf](https://modelscope.cn/models/OpenGVLab/InternVL3-78B-hf)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3-78B-hf](https://huggingface.co/OpenGVLab/InternVL3-78B-hf)|\n|[OpenGVLab/InternVL3_5-1B-HF](https://modelscope.cn/models/OpenGVLab/InternVL3_5-1B-HF)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3_5-1B-HF](https://huggingface.co/OpenGVLab/InternVL3_5-1B-HF)|\n|[OpenGVLab/InternVL3_5-2B-HF](https://modelscope.cn/models/OpenGVLab/InternVL3_5-2B-HF)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3_5-2B-HF](https://huggingface.co/OpenGVLab/InternVL3_5-2B-HF)|\n|[OpenGVLab/InternVL3_5-4B-HF](https://modelscope.cn/models/OpenGVLab/InternVL3_5-4B-HF)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3_5-4B-HF](https://huggingface.co/OpenGVLab/InternVL3_5-4B-HF)|\n|[OpenGVLab/InternVL3_5-8B-HF](https://modelscope.cn/models/OpenGVLab/InternVL3_5-8B-HF)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3_5-8B-HF](https://huggingface.co/OpenGVLab/InternVL3_5-8B-HF)|\n|[OpenGVLab/InternVL3_5-14B-HF](https://modelscope.cn/models/OpenGVLab/InternVL3_5-14B-HF)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3_5-14B-HF](https://huggingface.co/OpenGVLab/InternVL3_5-14B-HF)|\n|[OpenGVLab/InternVL3_5-38B-HF](https://modelscope.cn/models/OpenGVLab/InternVL3_5-38B-HF)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3_5-38B-HF](https://huggingface.co/OpenGVLab/InternVL3_5-38B-HF)|\n|[OpenGVLab/InternVL3_5-30B-A3B-HF](https://modelscope.cn/models/OpenGVLab/InternVL3_5-30B-A3B-HF)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3_5-30B-A3B-HF](https://huggingface.co/OpenGVLab/InternVL3_5-30B-A3B-HF)|\n|[OpenGVLab/InternVL3_5-241B-A28B-HF](https://modelscope.cn/models/OpenGVLab/InternVL3_5-241B-A28B-HF)|internvl_hf|internvl_hf|transformers>=4.52.1, timm|✔|vision, video|[OpenGVLab/InternVL3_5-241B-A28B-HF](https://huggingface.co/OpenGVLab/InternVL3_5-241B-A28B-HF)|\n|[OpenGVLab/InternVL3_5-1B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3_5-1B-Pretrained)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-1B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3_5-1B-Pretrained)|\n|[OpenGVLab/InternVL3_5-2B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3_5-2B-Pretrained)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-2B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3_5-2B-Pretrained)|\n|[OpenGVLab/InternVL3_5-4B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3_5-4B-Pretrained)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-4B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3_5-4B-Pretrained)|\n|[OpenGVLab/InternVL3_5-8B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3_5-8B-Pretrained)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-8B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3_5-8B-Pretrained)|\n|[OpenGVLab/InternVL3_5-14B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3_5-14B-Pretrained)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-14B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3_5-14B-Pretrained)|\n|[OpenGVLab/InternVL3_5-38B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3_5-38B-Pretrained)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-38B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3_5-38B-Pretrained)|\n|[OpenGVLab/InternVL3_5-30B-A3B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3_5-30B-A3B-Pretrained)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-30B-A3B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3_5-30B-A3B-Pretrained)|\n|[OpenGVLab/InternVL3_5-241B-A28B-Pretrained](https://modelscope.cn/models/OpenGVLab/InternVL3_5-241B-A28B-Pretrained)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-241B-A28B-Pretrained](https://huggingface.co/OpenGVLab/InternVL3_5-241B-A28B-Pretrained)|\n|[OpenGVLab/InternVL3_5-1B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3_5-1B-Instruct)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-1B-Instruct](https://huggingface.co/OpenGVLab/InternVL3_5-1B-Instruct)|\n|[OpenGVLab/InternVL3_5-2B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3_5-2B-Instruct)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-2B-Instruct](https://huggingface.co/OpenGVLab/InternVL3_5-2B-Instruct)|\n|[OpenGVLab/InternVL3_5-4B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3_5-4B-Instruct)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-4B-Instruct](https://huggingface.co/OpenGVLab/InternVL3_5-4B-Instruct)|\n|[OpenGVLab/InternVL3_5-8B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3_5-8B-Instruct)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-8B-Instruct](https://huggingface.co/OpenGVLab/InternVL3_5-8B-Instruct)|\n|[OpenGVLab/InternVL3_5-14B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3_5-14B-Instruct)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-14B-Instruct](https://huggingface.co/OpenGVLab/InternVL3_5-14B-Instruct)|\n|[OpenGVLab/InternVL3_5-38B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3_5-38B-Instruct)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-38B-Instruct](https://huggingface.co/OpenGVLab/InternVL3_5-38B-Instruct)|\n|[OpenGVLab/InternVL3_5-30B-A3B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3_5-30B-A3B-Instruct)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-30B-A3B-Instruct](https://huggingface.co/OpenGVLab/InternVL3_5-30B-A3B-Instruct)|\n|[OpenGVLab/InternVL3_5-241B-A28B-Instruct](https://modelscope.cn/models/OpenGVLab/InternVL3_5-241B-A28B-Instruct)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-241B-A28B-Instruct](https://huggingface.co/OpenGVLab/InternVL3_5-241B-A28B-Instruct)|\n|[OpenGVLab/InternVL3_5-1B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL3_5-1B-MPO)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-1B-MPO](https://huggingface.co/OpenGVLab/InternVL3_5-1B-MPO)|\n|[OpenGVLab/InternVL3_5-2B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL3_5-2B-MPO)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-2B-MPO](https://huggingface.co/OpenGVLab/InternVL3_5-2B-MPO)|\n|[OpenGVLab/InternVL3_5-4B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL3_5-4B-MPO)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-4B-MPO](https://huggingface.co/OpenGVLab/InternVL3_5-4B-MPO)|\n|[OpenGVLab/InternVL3_5-8B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL3_5-8B-MPO)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-8B-MPO](https://huggingface.co/OpenGVLab/InternVL3_5-8B-MPO)|\n|[OpenGVLab/InternVL3_5-14B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL3_5-14B-MPO)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-14B-MPO](https://huggingface.co/OpenGVLab/InternVL3_5-14B-MPO)|\n|[OpenGVLab/InternVL3_5-38B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL3_5-38B-MPO)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-38B-MPO](https://huggingface.co/OpenGVLab/InternVL3_5-38B-MPO)|\n|[OpenGVLab/InternVL3_5-30B-A3B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL3_5-30B-A3B-MPO)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-30B-A3B-MPO](https://huggingface.co/OpenGVLab/InternVL3_5-30B-A3B-MPO)|\n|[OpenGVLab/InternVL3_5-241B-A28B-MPO](https://modelscope.cn/models/OpenGVLab/InternVL3_5-241B-A28B-MPO)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-241B-A28B-MPO](https://huggingface.co/OpenGVLab/InternVL3_5-241B-A28B-MPO)|\n|[OpenGVLab/InternVL3_5-1B](https://modelscope.cn/models/OpenGVLab/InternVL3_5-1B)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-1B](https://huggingface.co/OpenGVLab/InternVL3_5-1B)|\n|[OpenGVLab/InternVL3_5-2B](https://modelscope.cn/models/OpenGVLab/InternVL3_5-2B)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-2B](https://huggingface.co/OpenGVLab/InternVL3_5-2B)|\n|[OpenGVLab/InternVL3_5-4B](https://modelscope.cn/models/OpenGVLab/InternVL3_5-4B)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-4B](https://huggingface.co/OpenGVLab/InternVL3_5-4B)|\n|[OpenGVLab/InternVL3_5-8B](https://modelscope.cn/models/OpenGVLab/InternVL3_5-8B)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-8B](https://huggingface.co/OpenGVLab/InternVL3_5-8B)|\n|[OpenGVLab/InternVL3_5-14B](https://modelscope.cn/models/OpenGVLab/InternVL3_5-14B)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-14B](https://huggingface.co/OpenGVLab/InternVL3_5-14B)|\n|[OpenGVLab/InternVL3_5-38B](https://modelscope.cn/models/OpenGVLab/InternVL3_5-38B)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-38B](https://huggingface.co/OpenGVLab/InternVL3_5-38B)|\n|[OpenGVLab/InternVL3_5-30B-A3B](https://modelscope.cn/models/OpenGVLab/InternVL3_5-30B-A3B)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-30B-A3B](https://huggingface.co/OpenGVLab/InternVL3_5-30B-A3B)|\n|[OpenGVLab/InternVL3_5-241B-A28B](https://modelscope.cn/models/OpenGVLab/InternVL3_5-241B-A28B)|internvl3_5|internvl3_5|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-241B-A28B](https://huggingface.co/OpenGVLab/InternVL3_5-241B-A28B)|\n|[OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview](https://modelscope.cn/models/OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview)|internvl3_5_gpt|internvl3_5_gpt|transformers>=4.37.2, timm|✔|vision, video|[OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview](https://huggingface.co/OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview)|\n|[OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview-HF](https://modelscope.cn/models/OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview-HF)|internvl_gpt_hf|internvl_hf|transformers>=4.55.0, timm|✔|vision, video|[OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview-HF](https://huggingface.co/OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview-HF)|\n|[Shanghai_AI_Laboratory/Intern-S1-mini](https://modelscope.cn/models/Shanghai_AI_Laboratory/Intern-S1-mini)|interns1|interns1|transformers>=4.55.2,<4.56|✘|vision, video|[internlm/Intern-S1-mini](https://huggingface.co/internlm/Intern-S1-mini)|\n|[Shanghai_AI_Laboratory/Intern-S1](https://modelscope.cn/models/Shanghai_AI_Laboratory/Intern-S1)|interns1|interns1|transformers>=4.55.2,<4.56|✘|vision, video|[internlm/Intern-S1](https://huggingface.co/internlm/Intern-S1)|\n|[Shanghai_AI_Laboratory/Intern-S1-mini-FP8](https://modelscope.cn/models/Shanghai_AI_Laboratory/Intern-S1-mini-FP8)|interns1|interns1|transformers>=4.55.2,<4.56|✘|vision, video|[internlm/Intern-S1-mini-FP8](https://huggingface.co/internlm/Intern-S1-mini-FP8)|\n|[Shanghai_AI_Laboratory/Intern-S1-FP8](https://modelscope.cn/models/Shanghai_AI_Laboratory/Intern-S1-FP8)|interns1|interns1|transformers>=4.55.2,<4.56|✘|vision, video|[internlm/Intern-S1-FP8](https://huggingface.co/internlm/Intern-S1-FP8)|\n|[Shanghai_AI_Laboratory/internlm-xcomposer2-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-xcomposer2-7b)|xcomposer2|ixcomposer2|-|✘|vision|[internlm/internlm-xcomposer2-7b](https://huggingface.co/internlm/internlm-xcomposer2-7b)|\n|[Shanghai_AI_Laboratory/internlm-xcomposer2-4khd-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-xcomposer2-4khd-7b)|xcomposer2_4khd|ixcomposer2|-|✘|vision|[internlm/internlm-xcomposer2-4khd-7b](https://huggingface.co/internlm/internlm-xcomposer2-4khd-7b)|\n|[Shanghai_AI_Laboratory/internlm-xcomposer2d5-7b](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-xcomposer2d5-7b)|xcomposer2_5|xcomposer2_5|decord|✘|vision|[internlm/internlm-xcomposer2d5-7b](https://huggingface.co/internlm/internlm-xcomposer2d5-7b)|\n|[Shanghai_AI_Laboratory/internlm-xcomposer2d5-ol-7b:base](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-xcomposer2d5-ol-7b)|xcomposer2_5|xcomposer2_5|decord|✘|vision|[internlm/internlm-xcomposer2d5-ol-7b:base](https://huggingface.co/internlm/internlm-xcomposer2d5-ol-7b)|\n|[Shanghai_AI_Laboratory/internlm-xcomposer2d5-ol-7b:audio](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm-xcomposer2d5-ol-7b)|xcomposer2_5_ol_audio|qwen2_audio|transformers>=4.45|✘|audio|[internlm/internlm-xcomposer2d5-ol-7b:audio](https://huggingface.co/internlm/internlm-xcomposer2d5-ol-7b)|\n|[LLM-Research/Llama-3.2-11B-Vision-Instruct](https://modelscope.cn/models/LLM-Research/Llama-3.2-11B-Vision-Instruct)|llama3_2_vision|llama3_2_vision|transformers>=4.45|✘|vision|[meta-llama/Llama-3.2-11B-Vision-Instruct](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct)|\n|[LLM-Research/Llama-3.2-90B-Vision-Instruct](https://modelscope.cn/models/LLM-Research/Llama-3.2-90B-Vision-Instruct)|llama3_2_vision|llama3_2_vision|transformers>=4.45|✘|vision|[meta-llama/Llama-3.2-90B-Vision-Instruct](https://huggingface.co/meta-llama/Llama-3.2-90B-Vision-Instruct)|\n|[LLM-Research/Llama-3.2-11B-Vision](https://modelscope.cn/models/LLM-Research/Llama-3.2-11B-Vision)|llama3_2_vision|llama3_2_vision|transformers>=4.45|✘|vision|[meta-llama/Llama-3.2-11B-Vision](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision)|\n|[LLM-Research/Llama-3.2-90B-Vision](https://modelscope.cn/models/LLM-Research/Llama-3.2-90B-Vision)|llama3_2_vision|llama3_2_vision|transformers>=4.45|✘|vision|[meta-llama/Llama-3.2-90B-Vision](https://huggingface.co/meta-llama/Llama-3.2-90B-Vision)|\n|[LLM-Research/Llama-4-Scout-17B-16E](https://modelscope.cn/models/LLM-Research/Llama-4-Scout-17B-16E)|llama4|llama4|transformers>=4.51|✔|vision|[meta-llama/Llama-4-Scout-17B-16E](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E)|\n|[LLM-Research/Llama-4-Maverick-17B-128E](https://modelscope.cn/models/LLM-Research/Llama-4-Maverick-17B-128E)|llama4|llama4|transformers>=4.51|✔|vision|[meta-llama/Llama-4-Maverick-17B-128E](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E)|\n|[LLM-Research/Llama-4-Scout-17B-16E-Instruct](https://modelscope.cn/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct)|llama4|llama4|transformers>=4.51|✔|vision|[meta-llama/Llama-4-Scout-17B-16E-Instruct](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct)|\n|[LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8](https://modelscope.cn/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8)|llama4|llama4|transformers>=4.51|✘|vision|[meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8)|\n|[LLM-Research/Llama-4-Maverick-17B-128E-Instruct](https://modelscope.cn/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct)|llama4|llama4|transformers>=4.51|✔|vision|[meta-llama/Llama-4-Maverick-17B-128E-Instruct](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct)|\n|[ICTNLP/Llama-3.1-8B-Omni](https://modelscope.cn/models/ICTNLP/Llama-3.1-8B-Omni)|llama3_1_omni|llama3_1_omni|openai-whisper|✘|audio|[ICTNLP/Llama-3.1-8B-Omni](https://huggingface.co/ICTNLP/Llama-3.1-8B-Omni)|\n|[llava-hf/llava-1.5-7b-hf](https://modelscope.cn/models/llava-hf/llava-1.5-7b-hf)|llava1_5_hf|llava1_5_hf|transformers>=4.36|✘|vision|[llava-hf/llava-1.5-7b-hf](https://huggingface.co/llava-hf/llava-1.5-7b-hf)|\n|[llava-hf/llava-1.5-13b-hf](https://modelscope.cn/models/llava-hf/llava-1.5-13b-hf)|llava1_5_hf|llava1_5_hf|transformers>=4.36|✘|vision|[llava-hf/llava-1.5-13b-hf](https://huggingface.co/llava-hf/llava-1.5-13b-hf)|\n|[llava-hf/llava-v1.6-mistral-7b-hf](https://modelscope.cn/models/llava-hf/llava-v1.6-mistral-7b-hf)|llava1_6_mistral_hf|llava1_6_mistral_hf|transformers>=4.39|✘|vision|[llava-hf/llava-v1.6-mistral-7b-hf](https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf)|\n|[llava-hf/llava-v1.6-vicuna-7b-hf](https://modelscope.cn/models/llava-hf/llava-v1.6-vicuna-7b-hf)|llava1_6_vicuna_hf|llava1_6_vicuna_hf|transformers>=4.39|✘|vision|[llava-hf/llava-v1.6-vicuna-7b-hf](https://huggingface.co/llava-hf/llava-v1.6-vicuna-7b-hf)|\n|[llava-hf/llava-v1.6-vicuna-13b-hf](https://modelscope.cn/models/llava-hf/llava-v1.6-vicuna-13b-hf)|llava1_6_vicuna_hf|llava1_6_vicuna_hf|transformers>=4.39|✘|vision|[llava-hf/llava-v1.6-vicuna-13b-hf](https://huggingface.co/llava-hf/llava-v1.6-vicuna-13b-hf)|\n|[llava-hf/llava-v1.6-34b-hf](https://modelscope.cn/models/llava-hf/llava-v1.6-34b-hf)|llava1_6_yi_hf|llava1_6_yi_hf|transformers>=4.39|✘|vision|[llava-hf/llava-v1.6-34b-hf](https://huggingface.co/llava-hf/llava-v1.6-34b-hf)|\n|[llava-hf/llama3-llava-next-8b-hf](https://modelscope.cn/models/llava-hf/llama3-llava-next-8b-hf)|llama3_llava_next_hf|llama3_llava_next_hf|transformers>=4.39|✘|vision|[llava-hf/llama3-llava-next-8b-hf](https://huggingface.co/llava-hf/llama3-llava-next-8b-hf)|\n|[llava-hf/llava-next-72b-hf](https://modelscope.cn/models/llava-hf/llava-next-72b-hf)|llava_next_qwen_hf|llava_next_qwen_hf|transformers>=4.39|✘|vision|[llava-hf/llava-next-72b-hf](https://huggingface.co/llava-hf/llava-next-72b-hf)|\n|[llava-hf/llava-next-110b-hf](https://modelscope.cn/models/llava-hf/llava-next-110b-hf)|llava_next_qwen_hf|llava_next_qwen_hf|transformers>=4.39|✘|vision|[llava-hf/llava-next-110b-hf](https://huggingface.co/llava-hf/llava-next-110b-hf)|\n|[llava-hf/LLaVA-NeXT-Video-7B-DPO-hf](https://modelscope.cn/models/llava-hf/LLaVA-NeXT-Video-7B-DPO-hf)|llava_next_video_hf|llava_next_video_hf|transformers>=4.42, av|✘|video|[llava-hf/LLaVA-NeXT-Video-7B-DPO-hf](https://huggingface.co/llava-hf/LLaVA-NeXT-Video-7B-DPO-hf)|\n|[llava-hf/LLaVA-NeXT-Video-7B-32K-hf](https://modelscope.cn/models/llava-hf/LLaVA-NeXT-Video-7B-32K-hf)|llava_next_video_hf|llava_next_video_hf|transformers>=4.42, av|✘|video|[llava-hf/LLaVA-NeXT-Video-7B-32K-hf](https://huggingface.co/llava-hf/LLaVA-NeXT-Video-7B-32K-hf)|\n|[llava-hf/LLaVA-NeXT-Video-7B-hf](https://modelscope.cn/models/llava-hf/LLaVA-NeXT-Video-7B-hf)|llava_next_video_hf|llava_next_video_hf|transformers>=4.42, av|✘|video|[llava-hf/LLaVA-NeXT-Video-7B-hf](https://huggingface.co/llava-hf/LLaVA-NeXT-Video-7B-hf)|\n|[llava-hf/LLaVA-NeXT-Video-34B-hf](https://modelscope.cn/models/llava-hf/LLaVA-NeXT-Video-34B-hf)|llava_next_video_yi_hf|llava_next_video_hf|transformers>=4.42, av|✘|video|[llava-hf/LLaVA-NeXT-Video-34B-hf](https://huggingface.co/llava-hf/LLaVA-NeXT-Video-34B-hf)|\n|[llava-hf/llava-onevision-qwen2-0.5b-ov-hf](https://modelscope.cn/models/llava-hf/llava-onevision-qwen2-0.5b-ov-hf)|llava_onevision_hf|llava_onevision_hf|transformers>=4.45|✘|vision, video|[llava-hf/llava-onevision-qwen2-0.5b-ov-hf](https://huggingface.co/llava-hf/llava-onevision-qwen2-0.5b-ov-hf)|\n|[llava-hf/llava-onevision-qwen2-7b-ov-hf](https://modelscope.cn/models/llava-hf/llava-onevision-qwen2-7b-ov-hf)|llava_onevision_hf|llava_onevision_hf|transformers>=4.45|✘|vision, video|[llava-hf/llava-onevision-qwen2-7b-ov-hf](https://huggingface.co/llava-hf/llava-onevision-qwen2-7b-ov-hf)|\n|[llava-hf/llava-onevision-qwen2-72b-ov-hf](https://modelscope.cn/models/llava-hf/llava-onevision-qwen2-72b-ov-hf)|llava_onevision_hf|llava_onevision_hf|transformers>=4.45|✘|vision, video|[llava-hf/llava-onevision-qwen2-72b-ov-hf](https://huggingface.co/llava-hf/llava-onevision-qwen2-72b-ov-hf)|\n|[01ai/Yi-VL-6B](https://modelscope.cn/models/01ai/Yi-VL-6B)|yi_vl|yi_vl|transformers>=4.34|✘|vision|[01-ai/Yi-VL-6B](https://huggingface.co/01-ai/Yi-VL-6B)|\n|[01ai/Yi-VL-34B](https://modelscope.cn/models/01ai/Yi-VL-34B)|yi_vl|yi_vl|transformers>=4.34|✘|vision|[01-ai/Yi-VL-34B](https://huggingface.co/01-ai/Yi-VL-34B)|\n|[PaddlePaddle/ERNIE-4.5-VL-28B-A3B-PT](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-VL-28B-A3B-PT)|ernie_vl|ernie_vl|transformers>=4.52, moviepy|✘|-|[baidu/ERNIE-4.5-VL-28B-A3B-PT](https://huggingface.co/baidu/ERNIE-4.5-VL-28B-A3B-PT)|\n|[PaddlePaddle/ERNIE-4.5-VL-424B-A47B-PT](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-VL-424B-A47B-PT)|ernie_vl|ernie_vl|transformers>=4.52, moviepy|✘|-|[baidu/ERNIE-4.5-VL-424B-A47B-PT](https://huggingface.co/baidu/ERNIE-4.5-VL-424B-A47B-PT)|\n|[PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Base-PT](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Base-PT)|ernie_vl|ernie_vl|transformers>=4.52, moviepy|✘|-|[baidu/ERNIE-4.5-VL-28B-A3B-Base-PT](https://huggingface.co/baidu/ERNIE-4.5-VL-28B-A3B-Base-PT)|\n|[PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Base-PT](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-VL-424B-A47B-Base-PT)|ernie_vl|ernie_vl|transformers>=4.52, moviepy|✘|-|[baidu/ERNIE-4.5-VL-424B-A47B-Base-PT](https://huggingface.co/baidu/ERNIE-4.5-VL-424B-A47B-Base-PT)|\n|[PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking](https://modelscope.cn/models/PaddlePaddle/ERNIE-4.5-VL-28B-A3B-Thinking)|ernie_vl|ernie_vl_thinking|transformers>=4.52, moviepy|✘|-|[baidu/ERNIE-4.5-VL-28B-A3B-Thinking](https://huggingface.co/baidu/ERNIE-4.5-VL-28B-A3B-Thinking)|\n|[swift/llava-llama3.1-8b](https://modelscope.cn/models/swift/llava-llama3.1-8b)|llava_llama3_1_hf|llava_llama3_1_hf|transformers>=4.41|✘|vision|-|\n|[AI-ModelScope/llava-llama-3-8b-v1_1-transformers](https://modelscope.cn/models/AI-ModelScope/llava-llama-3-8b-v1_1-transformers)|llava_llama3_hf|llava_llama3_hf|transformers>=4.36|✘|vision|[xtuner/llava-llama-3-8b-v1_1-transformers](https://huggingface.co/xtuner/llava-llama-3-8b-v1_1-transformers)|\n|[AI-ModelScope/llava-v1.6-mistral-7b](https://modelscope.cn/models/AI-ModelScope/llava-v1.6-mistral-7b)|llava1_6_mistral|llava1_6_mistral|transformers>=4.34|✘|vision|[liuhaotian/llava-v1.6-mistral-7b](https://huggingface.co/liuhaotian/llava-v1.6-mistral-7b)|\n|[AI-ModelScope/llava-v1.6-34b](https://modelscope.cn/models/AI-ModelScope/llava-v1.6-34b)|llava1_6_yi|llava1_6_yi|transformers>=4.34|✘|vision|[liuhaotian/llava-v1.6-34b](https://huggingface.co/liuhaotian/llava-v1.6-34b)|\n|[AI-ModelScope/llava-next-72b](https://modelscope.cn/models/AI-ModelScope/llava-next-72b)|llava_next_qwen|llava_next_qwen|transformers>=4.42, av|✘|vision|[lmms-lab/llava-next-72b](https://huggingface.co/lmms-lab/llava-next-72b)|\n|[AI-ModelScope/llava-next-110b](https://modelscope.cn/models/AI-ModelScope/llava-next-110b)|llava_next_qwen|llava_next_qwen|transformers>=4.42, av|✘|vision|[lmms-lab/llava-next-110b](https://huggingface.co/lmms-lab/llava-next-110b)|\n|[AI-ModelScope/llama3-llava-next-8b](https://modelscope.cn/models/AI-ModelScope/llama3-llava-next-8b)|llama3_llava_next|llama3_llava_next|transformers>=4.42, av|✘|vision|[lmms-lab/llama3-llava-next-8b](https://huggingface.co/lmms-lab/llama3-llava-next-8b)|\n|[lmms-lab/LLaVA-OneVision-1.5-4B-Instruct](https://modelscope.cn/models/lmms-lab/LLaVA-OneVision-1.5-4B-Instruct)|llava_onevision1_5|llava_onevision1_5|transformers>=4.53.0, qwen_vl_utils|✘|vision|[lmms-lab/LLaVA-OneVision-1.5-4B-Instruct](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-4B-Instruct)|\n|[lmms-lab/LLaVA-OneVision-1.5-8B-Instruct](https://modelscope.cn/models/lmms-lab/LLaVA-OneVision-1.5-8B-Instruct)|llava_onevision1_5|llava_onevision1_5|transformers>=4.53.0, qwen_vl_utils|✘|vision|[lmms-lab/LLaVA-OneVision-1.5-8B-Instruct](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-8B-Instruct)|\n|[lmms-lab/LLaVA-OneVision-1.5-4B-Base](https://modelscope.cn/models/lmms-lab/LLaVA-OneVision-1.5-4B-Base)|llava_onevision1_5|llava_onevision1_5|transformers>=4.53.0, qwen_vl_utils|✘|vision|[lmms-lab/LLaVA-OneVision-1.5-4B-Base](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-4B-Base)|\n|[lmms-lab/LLaVA-OneVision-1.5-8B-Base](https://modelscope.cn/models/lmms-lab/LLaVA-OneVision-1.5-8B-Base)|llava_onevision1_5|llava_onevision1_5|transformers>=4.53.0, qwen_vl_utils|✘|vision|[lmms-lab/LLaVA-OneVision-1.5-8B-Base](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-8B-Base)|\n|[deepseek-ai/deepseek-vl-1.3b-chat](https://modelscope.cn/models/deepseek-ai/deepseek-vl-1.3b-chat)|deepseek_vl|deepseek_vl|-|✘|vision|[deepseek-ai/deepseek-vl-1.3b-chat](https://huggingface.co/deepseek-ai/deepseek-vl-1.3b-chat)|\n|[deepseek-ai/deepseek-vl-7b-chat](https://modelscope.cn/models/deepseek-ai/deepseek-vl-7b-chat)|deepseek_vl|deepseek_vl|-|✘|vision|[deepseek-ai/deepseek-vl-7b-chat](https://huggingface.co/deepseek-ai/deepseek-vl-7b-chat)|\n|[deepseek-ai/deepseek-vl2-tiny](https://modelscope.cn/models/deepseek-ai/deepseek-vl2-tiny)|deepseek_vl2|deepseek_vl2|transformers<4.42|✘|vision|[deepseek-ai/deepseek-vl2-tiny](https://huggingface.co/deepseek-ai/deepseek-vl2-tiny)|\n|[deepseek-ai/deepseek-vl2-small](https://modelscope.cn/models/deepseek-ai/deepseek-vl2-small)|deepseek_vl2|deepseek_vl2|transformers<4.42|✘|vision|[deepseek-ai/deepseek-vl2-small](https://huggingface.co/deepseek-ai/deepseek-vl2-small)|\n|[deepseek-ai/deepseek-vl2](https://modelscope.cn/models/deepseek-ai/deepseek-vl2)|deepseek_vl2|deepseek_vl2|transformers<4.42|✘|vision|[deepseek-ai/deepseek-vl2](https://huggingface.co/deepseek-ai/deepseek-vl2)|\n|[deepseek-ai/Janus-1.3B](https://modelscope.cn/models/deepseek-ai/Janus-1.3B)|deepseek_janus|deepseek_janus|-|✘|vision|[deepseek-ai/Janus-1.3B](https://huggingface.co/deepseek-ai/Janus-1.3B)|\n|[deepseek-ai/Janus-Pro-1B](https://modelscope.cn/models/deepseek-ai/Janus-Pro-1B)|deepseek_janus_pro|deepseek_janus_pro|-|✘|vision|[deepseek-ai/Janus-Pro-1B](https://huggingface.co/deepseek-ai/Janus-Pro-1B)|\n|[deepseek-ai/Janus-Pro-7B](https://modelscope.cn/models/deepseek-ai/Janus-Pro-7B)|deepseek_janus_pro|deepseek_janus_pro|-|✘|vision|[deepseek-ai/Janus-Pro-7B](https://huggingface.co/deepseek-ai/Janus-Pro-7B)|\n|[deepseek-ai/DeepSeek-OCR](https://modelscope.cn/models/deepseek-ai/DeepSeek-OCR)|deepseek_ocr|deepseek_ocr|transformers==4.46.3, easydict|✘|vision|[deepseek-ai/DeepSeek-OCR](https://huggingface.co/deepseek-ai/DeepSeek-OCR)|\n|[deepseek-ai/DeepSeek-OCR-2](https://modelscope.cn/models/deepseek-ai/DeepSeek-OCR-2)|deepseek_ocr2|deepseek_ocr2|transformers==4.46.3, easydict|✘|vision|[deepseek-ai/DeepSeek-OCR-2](https://huggingface.co/deepseek-ai/DeepSeek-OCR-2)|\n|[OpenBMB/MiniCPM-V](https://modelscope.cn/models/OpenBMB/MiniCPM-V)|minicpmv|minicpmv|timm, transformers<4.42|✘|vision|[openbmb/MiniCPM-V](https://huggingface.co/openbmb/MiniCPM-V)|\n|[OpenBMB/MiniCPM-V-2](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2)|minicpmv|minicpmv|timm, transformers<4.42|✘|vision|[openbmb/MiniCPM-V-2](https://huggingface.co/openbmb/MiniCPM-V-2)|\n|[OpenBMB/MiniCPM-Llama3-V-2_5](https://modelscope.cn/models/OpenBMB/MiniCPM-Llama3-V-2_5)|minicpmv2_5|minicpmv2_5|timm, transformers>=4.36|✘|vision|[openbmb/MiniCPM-Llama3-V-2_5](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5)|\n|[OpenBMB/MiniCPM-V-2_6](https://modelscope.cn/models/OpenBMB/MiniCPM-V-2_6)|minicpmv2_6|minicpmv2_6|timm, transformers>=4.36, decord|✘|vision, video|[openbmb/MiniCPM-V-2_6](https://huggingface.co/openbmb/MiniCPM-V-2_6)|\n|[OpenBMB/MiniCPM-V-4](https://modelscope.cn/models/OpenBMB/MiniCPM-V-4)|minicpmv4|minicpmv4|timm, transformers>=4.36, decord|✘|vision, video|[openbmb/MiniCPM-V-4](https://huggingface.co/openbmb/MiniCPM-V-4)|\n|[OpenBMB/MiniCPM-V-4_5](https://modelscope.cn/models/OpenBMB/MiniCPM-V-4_5)|minicpmv4_5|minicpmv4_5|timm, transformers>=4.36, decord|✘|vision, video|[openbmb/MiniCPM-V-4_5](https://huggingface.co/openbmb/MiniCPM-V-4_5)|\n|[OpenBMB/MiniCPM-o-2_6](https://modelscope.cn/models/OpenBMB/MiniCPM-o-2_6)|minicpmo|minicpmo|timm, transformers>=4.36, decord, soundfile|✘|vision, video, omni, audio|[openbmb/MiniCPM-o-2_6](https://huggingface.co/openbmb/MiniCPM-o-2_6)|\n|[OpenBMB/MiniCPM-o-4_5](https://modelscope.cn/models/OpenBMB/MiniCPM-o-4_5)|minicpmo|minicpmo4_5|timm, transformers==4.51.3, decord, soundfile|✘|vision, video, omni, audio|[openbmb/MiniCPM-o-4_5](https://huggingface.co/openbmb/MiniCPM-o-4_5)|\n|[MiniMax/MiniMax-VL-01](https://modelscope.cn/models/MiniMax/MiniMax-VL-01)|minimax_vl|minimax_vl|-|✘|vision|[MiniMaxAI/MiniMax-VL-01](https://huggingface.co/MiniMaxAI/MiniMax-VL-01)|\n|[iic/mPLUG-Owl2](https://modelscope.cn/models/iic/mPLUG-Owl2)|mplug_owl2|mplug_owl2|transformers<4.35, icecream|✘|vision|[MAGAer13/mplug-owl2-llama2-7b](https://huggingface.co/MAGAer13/mplug-owl2-llama2-7b)|\n|[iic/mPLUG-Owl2.1](https://modelscope.cn/models/iic/mPLUG-Owl2.1)|mplug_owl2_1|mplug_owl2|transformers<4.35, icecream|✘|vision|[Mizukiluke/mplug_owl_2_1](https://huggingface.co/Mizukiluke/mplug_owl_2_1)|\n|[iic/mPLUG-Owl3-1B-241014](https://modelscope.cn/models/iic/mPLUG-Owl3-1B-241014)|mplug_owl3|mplug_owl3|transformers>=4.36, icecream, decord|✘|vision, video|[mPLUG/mPLUG-Owl3-1B-241014](https://huggingface.co/mPLUG/mPLUG-Owl3-1B-241014)|\n|[iic/mPLUG-Owl3-2B-241014](https://modelscope.cn/models/iic/mPLUG-Owl3-2B-241014)|mplug_owl3|mplug_owl3|transformers>=4.36, icecream, decord|✘|vision, video|[mPLUG/mPLUG-Owl3-2B-241014](https://huggingface.co/mPLUG/mPLUG-Owl3-2B-241014)|\n|[iic/mPLUG-Owl3-7B-240728](https://modelscope.cn/models/iic/mPLUG-Owl3-7B-240728)|mplug_owl3|mplug_owl3|transformers>=4.36, icecream, decord|✘|vision, video|[mPLUG/mPLUG-Owl3-7B-240728](https://huggingface.co/mPLUG/mPLUG-Owl3-7B-240728)|\n|[iic/mPLUG-Owl3-7B-241101](https://modelscope.cn/models/iic/mPLUG-Owl3-7B-241101)|mplug_owl3_241101|mplug_owl3_241101|transformers>=4.36, icecream|✘|vision, video|[mPLUG/mPLUG-Owl3-7B-241101](https://huggingface.co/mPLUG/mPLUG-Owl3-7B-241101)|\n|[iic/DocOwl2](https://modelscope.cn/models/iic/DocOwl2)|doc_owl2|doc_owl2|transformers>=4.36, icecream|✘|vision|[mPLUG/DocOwl2](https://huggingface.co/mPLUG/DocOwl2)|\n|[BAAI/Emu3-Gen](https://modelscope.cn/models/BAAI/Emu3-Gen)|emu3_gen|emu3_gen|-|✘|t2i|[BAAI/Emu3-Gen](https://huggingface.co/BAAI/Emu3-Gen)|\n|[BAAI/Emu3-Chat](https://modelscope.cn/models/BAAI/Emu3-Chat)|emu3_chat|emu3_chat|transformers>=4.44.0|✘|vision|[BAAI/Emu3-Chat](https://huggingface.co/BAAI/Emu3-Chat)|\n|[stepfun-ai/GOT-OCR2_0](https://modelscope.cn/models/stepfun-ai/GOT-OCR2_0)|got_ocr2|got_ocr2|-|✘|vision|[stepfun-ai/GOT-OCR2_0](https://huggingface.co/stepfun-ai/GOT-OCR2_0)|\n|[stepfun-ai/GOT-OCR-2.0-hf](https://modelscope.cn/models/stepfun-ai/GOT-OCR-2.0-hf)|got_ocr2_hf|got_ocr2_hf|-|✘|vision|[stepfun-ai/GOT-OCR-2.0-hf](https://huggingface.co/stepfun-ai/GOT-OCR-2.0-hf)|\n|[stepfun-ai/Step-Audio-Chat](https://modelscope.cn/models/stepfun-ai/Step-Audio-Chat)|step_audio|step_audio|funasr, sox, conformer, openai-whisper, librosa|✘|audio|[stepfun-ai/Step-Audio-Chat](https://huggingface.co/stepfun-ai/Step-Audio-Chat)|\n|[stepfun-ai/Step-Audio-2-mini](https://modelscope.cn/models/stepfun-ai/Step-Audio-2-mini)|step_audio2_mini|step_audio2_mini|transformers==4.53.3, torchaudio, librosa|✘|audio|[stepfun-ai/Step-Audio-2-mini](https://huggingface.co/stepfun-ai/Step-Audio-2-mini)|\n|[moonshotai/Kimi-VL-A3B-Instruct](https://modelscope.cn/models/moonshotai/Kimi-VL-A3B-Instruct)|kimi_vl|kimi_vl|transformers<4.49|✔|-|[moonshotai/Kimi-VL-A3B-Instruct](https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct)|\n|[moonshotai/Kimi-VL-A3B-Thinking](https://modelscope.cn/models/moonshotai/Kimi-VL-A3B-Thinking)|kimi_vl|kimi_vl|transformers<4.49|✔|-|[moonshotai/Kimi-VL-A3B-Thinking](https://huggingface.co/moonshotai/Kimi-VL-A3B-Thinking)|\n|[moonshotai/Kimi-VL-A3B-Thinking-2506](https://modelscope.cn/models/moonshotai/Kimi-VL-A3B-Thinking-2506)|kimi_vl|kimi_vl|transformers<4.49|✔|-|[moonshotai/Kimi-VL-A3B-Thinking-2506](https://huggingface.co/moonshotai/Kimi-VL-A3B-Thinking-2506)|\n|[Kwai-Keye/Keye-VL-8B-Preview](https://modelscope.cn/models/Kwai-Keye/Keye-VL-8B-Preview)|keye_vl|keye_vl|keye_vl_utils|✘|vision|[Kwai-Keye/Keye-VL-8B-Preview](https://huggingface.co/Kwai-Keye/Keye-VL-8B-Preview)|\n|[Kwai-Keye/Keye-VL-1_5-8B](https://modelscope.cn/models/Kwai-Keye/Keye-VL-1_5-8B)|keye_vl_1_5|keye_vl_1_5|keye_vl_utils>=1.5.2, transformers==4.52.4|✘|vision|[Kwai-Keye/Keye-VL-1_5-8B](https://huggingface.co/Kwai-Keye/Keye-VL-1_5-8B)|\n|[rednote-hilab/dots.ocr](https://modelscope.cn/models/rednote-hilab/dots.ocr)|dots_ocr|dots_ocr|transformers>=4.51.0|✘|-|[rednote-hilab/dots.ocr](https://huggingface.co/rednote-hilab/dots.ocr)|\n|[BytedanceDouyinContent/SAIL-VL2-2B](https://modelscope.cn/models/BytedanceDouyinContent/SAIL-VL2-2B)|sail_vl2|sail_vl2|transformers<=4.51.3|✘|vision|[BytedanceDouyinContent/SAIL-VL2-2B](https://huggingface.co/BytedanceDouyinContent/SAIL-VL2-2B)|\n|[BytedanceDouyinContent/SAIL-VL2-2B-Thinking](https://modelscope.cn/models/BytedanceDouyinContent/SAIL-VL2-2B-Thinking)|sail_vl2|sail_vl2|transformers<=4.51.3|✘|vision|[BytedanceDouyinContent/SAIL-VL2-2B-Thinking](https://huggingface.co/BytedanceDouyinContent/SAIL-VL2-2B-Thinking)|\n|[BytedanceDouyinContent/SAIL-VL2-8B](https://modelscope.cn/models/BytedanceDouyinContent/SAIL-VL2-8B)|sail_vl2|sail_vl2|transformers<=4.51.3|✘|vision|[BytedanceDouyinContent/SAIL-VL2-8B](https://huggingface.co/BytedanceDouyinContent/SAIL-VL2-8B)|\n|[BytedanceDouyinContent/SAIL-VL2-8B-Thinking](https://modelscope.cn/models/BytedanceDouyinContent/SAIL-VL2-8B-Thinking)|sail_vl2|sail_vl2|transformers<=4.51.3|✘|vision|[BytedanceDouyinContent/SAIL-VL2-8B-Thinking](https://huggingface.co/BytedanceDouyinContent/SAIL-VL2-8B-Thinking)|\n|[LLM-Research/Phi-3-vision-128k-instruct](https://modelscope.cn/models/LLM-Research/Phi-3-vision-128k-instruct)|phi3_vision|phi3_vision|transformers>=4.36|✘|vision|[microsoft/Phi-3-vision-128k-instruct](https://huggingface.co/microsoft/Phi-3-vision-128k-instruct)|\n|[LLM-Research/Phi-3.5-vision-instruct](https://modelscope.cn/models/LLM-Research/Phi-3.5-vision-instruct)|phi3_vision|phi3_vision|transformers>=4.36|✘|vision|[microsoft/Phi-3.5-vision-instruct](https://huggingface.co/microsoft/Phi-3.5-vision-instruct)|\n|[LLM-Research/Phi-4-multimodal-instruct](https://modelscope.cn/models/LLM-Research/Phi-4-multimodal-instruct)|phi4_multimodal|phi4_multimodal|transformers>=4.36,<4.49, backoff, soundfile|✘|vision, audio|[microsoft/Phi-4-multimodal-instruct](https://huggingface.co/microsoft/Phi-4-multimodal-instruct)|\n|[AI-ModelScope/Florence-2-base-ft](https://modelscope.cn/models/AI-ModelScope/Florence-2-base-ft)|florence|florence|-|✘|vision|[microsoft/Florence-2-base-ft](https://huggingface.co/microsoft/Florence-2-base-ft)|\n|[AI-ModelScope/Florence-2-base](https://modelscope.cn/models/AI-ModelScope/Florence-2-base)|florence|florence|-|✘|vision|[microsoft/Florence-2-base](https://huggingface.co/microsoft/Florence-2-base)|\n|[AI-ModelScope/Florence-2-large](https://modelscope.cn/models/AI-ModelScope/Florence-2-large)|florence|florence|-|✘|vision|[microsoft/Florence-2-large](https://huggingface.co/microsoft/Florence-2-large)|\n|[AI-ModelScope/Florence-2-large-ft](https://modelscope.cn/models/AI-ModelScope/Florence-2-large-ft)|florence|florence|-|✘|vision|[microsoft/Florence-2-large-ft](https://huggingface.co/microsoft/Florence-2-large-ft)|\n|[AI-ModelScope/Idefics3-8B-Llama3](https://modelscope.cn/models/AI-ModelScope/Idefics3-8B-Llama3)|idefics3|idefics3|transformers>=4.45|✘|vision|[HuggingFaceM4/Idefics3-8B-Llama3](https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3)|\n|[AI-ModelScope/paligemma-3b-pt-224](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-pt-224)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma-3b-pt-224](https://huggingface.co/google/paligemma-3b-pt-224)|\n|[AI-ModelScope/paligemma-3b-pt-448](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-pt-448)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma-3b-pt-448](https://huggingface.co/google/paligemma-3b-pt-448)|\n|[AI-ModelScope/paligemma-3b-pt-896](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-pt-896)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma-3b-pt-896](https://huggingface.co/google/paligemma-3b-pt-896)|\n|[AI-ModelScope/paligemma-3b-mix-224](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-mix-224)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma-3b-mix-224](https://huggingface.co/google/paligemma-3b-mix-224)|\n|[AI-ModelScope/paligemma-3b-mix-448](https://modelscope.cn/models/AI-ModelScope/paligemma-3b-mix-448)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma-3b-mix-448](https://huggingface.co/google/paligemma-3b-mix-448)|\n|[AI-ModelScope/paligemma2-3b-pt-224](https://modelscope.cn/models/AI-ModelScope/paligemma2-3b-pt-224)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-3b-pt-224](https://huggingface.co/google/paligemma2-3b-pt-224)|\n|[AI-ModelScope/paligemma2-3b-pt-448](https://modelscope.cn/models/AI-ModelScope/paligemma2-3b-pt-448)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-3b-pt-448](https://huggingface.co/google/paligemma2-3b-pt-448)|\n|[AI-ModelScope/paligemma2-3b-pt-896](https://modelscope.cn/models/AI-ModelScope/paligemma2-3b-pt-896)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-3b-pt-896](https://huggingface.co/google/paligemma2-3b-pt-896)|\n|[AI-ModelScope/paligemma2-10b-pt-224](https://modelscope.cn/models/AI-ModelScope/paligemma2-10b-pt-224)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-10b-pt-224](https://huggingface.co/google/paligemma2-10b-pt-224)|\n|[AI-ModelScope/paligemma2-10b-pt-448](https://modelscope.cn/models/AI-ModelScope/paligemma2-10b-pt-448)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-10b-pt-448](https://huggingface.co/google/paligemma2-10b-pt-448)|\n|[AI-ModelScope/paligemma2-10b-pt-896](https://modelscope.cn/models/AI-ModelScope/paligemma2-10b-pt-896)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-10b-pt-896](https://huggingface.co/google/paligemma2-10b-pt-896)|\n|[AI-ModelScope/paligemma2-28b-pt-224](https://modelscope.cn/models/AI-ModelScope/paligemma2-28b-pt-224)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-28b-pt-224](https://huggingface.co/google/paligemma2-28b-pt-224)|\n|[AI-ModelScope/paligemma2-28b-pt-448](https://modelscope.cn/models/AI-ModelScope/paligemma2-28b-pt-448)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-28b-pt-448](https://huggingface.co/google/paligemma2-28b-pt-448)|\n|[AI-ModelScope/paligemma2-28b-pt-896](https://modelscope.cn/models/AI-ModelScope/paligemma2-28b-pt-896)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-28b-pt-896](https://huggingface.co/google/paligemma2-28b-pt-896)|\n|[AI-ModelScope/paligemma2-3b-ft-docci-448](https://modelscope.cn/models/AI-ModelScope/paligemma2-3b-ft-docci-448)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-3b-ft-docci-448](https://huggingface.co/google/paligemma2-3b-ft-docci-448)|\n|[AI-ModelScope/paligemma2-10b-ft-docci-448](https://modelscope.cn/models/AI-ModelScope/paligemma2-10b-ft-docci-448)|paligemma|paligemma|transformers>=4.41|✘|vision|[google/paligemma2-10b-ft-docci-448](https://huggingface.co/google/paligemma2-10b-ft-docci-448)|\n|[LLM-Research/Molmo-7B-O-0924](https://modelscope.cn/models/LLM-Research/Molmo-7B-O-0924)|molmo|molmo|transformers>=4.45|✘|vision|[allenai/Molmo-7B-O-0924](https://huggingface.co/allenai/Molmo-7B-O-0924)|\n|[LLM-Research/Molmo-7B-D-0924](https://modelscope.cn/models/LLM-Research/Molmo-7B-D-0924)|molmo|molmo|transformers>=4.45|✘|vision|[allenai/Molmo-7B-D-0924](https://huggingface.co/allenai/Molmo-7B-D-0924)|\n|[LLM-Research/Molmo-72B-0924](https://modelscope.cn/models/LLM-Research/Molmo-72B-0924)|molmo|molmo|transformers>=4.45|✘|vision|[allenai/Molmo-72B-0924](https://huggingface.co/allenai/Molmo-72B-0924)|\n|[LLM-Research/MolmoE-1B-0924](https://modelscope.cn/models/LLM-Research/MolmoE-1B-0924)|molmoe|molmo|transformers>=4.45|✘|vision|[allenai/MolmoE-1B-0924](https://huggingface.co/allenai/MolmoE-1B-0924)|\n|[AI-ModelScope/pixtral-12b](https://modelscope.cn/models/AI-ModelScope/pixtral-12b)|pixtral|pixtral|transformers>=4.45|✘|vision|[mistral-community/pixtral-12b](https://huggingface.co/mistral-community/pixtral-12b)|\n|[InfiniAI/Megrez-3B-Omni](https://modelscope.cn/models/InfiniAI/Megrez-3B-Omni)|megrez_omni|megrez_omni|-|✘|vision, audio|[Infinigence/Megrez-3B-Omni](https://huggingface.co/Infinigence/Megrez-3B-Omni)|\n|[bytedance-research/Valley-Eagle-7B](https://modelscope.cn/models/bytedance-research/Valley-Eagle-7B)|valley|valley|transformers>=4.42, av|✘|vision|-|\n|[LLM-Research/gemma-3-4b-pt](https://modelscope.cn/models/LLM-Research/gemma-3-4b-pt)|gemma3_vision|gemma3_vision|transformers>=4.49|✘|-|[google/gemma-3-4b-pt](https://huggingface.co/google/gemma-3-4b-pt)|\n|[LLM-Research/gemma-3-4b-it](https://modelscope.cn/models/LLM-Research/gemma-3-4b-it)|gemma3_vision|gemma3_vision|transformers>=4.49|✘|-|[google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it)|\n|[LLM-Research/gemma-3-12b-pt](https://modelscope.cn/models/LLM-Research/gemma-3-12b-pt)|gemma3_vision|gemma3_vision|transformers>=4.49|✘|-|[google/gemma-3-12b-pt](https://huggingface.co/google/gemma-3-12b-pt)|\n|[LLM-Research/gemma-3-12b-it](https://modelscope.cn/models/LLM-Research/gemma-3-12b-it)|gemma3_vision|gemma3_vision|transformers>=4.49|✘|-|[google/gemma-3-12b-it](https://huggingface.co/google/gemma-3-12b-it)|\n|[LLM-Research/gemma-3-27b-pt](https://modelscope.cn/models/LLM-Research/gemma-3-27b-pt)|gemma3_vision|gemma3_vision|transformers>=4.49|✘|-|[google/gemma-3-27b-pt](https://huggingface.co/google/gemma-3-27b-pt)|\n|[LLM-Research/gemma-3-27b-it](https://modelscope.cn/models/LLM-Research/gemma-3-27b-it)|gemma3_vision|gemma3_vision|transformers>=4.49|✘|-|[google/gemma-3-27b-it](https://huggingface.co/google/gemma-3-27b-it)|\n|[google/medgemma-4b-pt](https://modelscope.cn/models/google/medgemma-4b-pt)|gemma3_vision|gemma3_vision|transformers>=4.49|✘|-|[google/medgemma-4b-pt](https://huggingface.co/google/medgemma-4b-pt)|\n|[google/medgemma-4b-it](https://modelscope.cn/models/google/medgemma-4b-it)|gemma3_vision|gemma3_vision|transformers>=4.49|✘|-|[google/medgemma-4b-it](https://huggingface.co/google/medgemma-4b-it)|\n|[google/medgemma-27b-it](https://modelscope.cn/models/google/medgemma-27b-it)|gemma3_vision|gemma3_vision|transformers>=4.49|✘|-|[google/medgemma-27b-it](https://huggingface.co/google/medgemma-27b-it)|\n|[google/gemma-3n-E2B](https://modelscope.cn/models/google/gemma-3n-E2B)|gemma3n|gemma3n|transformers>=4.53.1|✘|-|[google/gemma-3n-E2B](https://huggingface.co/google/gemma-3n-E2B)|\n|[google/gemma-3n-E4B](https://modelscope.cn/models/google/gemma-3n-E4B)|gemma3n|gemma3n|transformers>=4.53.1|✘|-|[google/gemma-3n-E4B](https://huggingface.co/google/gemma-3n-E4B)|\n|[google/gemma-3n-E2B-it](https://modelscope.cn/models/google/gemma-3n-E2B-it)|gemma3n|gemma3n|transformers>=4.53.1|✘|-|[google/gemma-3n-E2B-it](https://huggingface.co/google/gemma-3n-E2B-it)|\n|[google/gemma-3n-E4B-it](https://modelscope.cn/models/google/gemma-3n-E4B-it)|gemma3n|gemma3n|transformers>=4.53.1|✘|-|[google/gemma-3n-E4B-it](https://huggingface.co/google/gemma-3n-E4B-it)|\n|[mistralai/Mistral-Small-3.1-24B-Base-2503](https://modelscope.cn/models/mistralai/Mistral-Small-3.1-24B-Base-2503)|mistral3|mistral_2503|transformers>=4.49|✘|vision|[mistralai/Mistral-Small-3.1-24B-Base-2503](https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Base-2503)|\n|[mistralai/Mistral-Small-3.1-24B-Instruct-2503](https://modelscope.cn/models/mistralai/Mistral-Small-3.1-24B-Instruct-2503)|mistral3|mistral_2503|transformers>=4.49|✘|vision|[mistralai/Mistral-Small-3.1-24B-Instruct-2503](https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503)|\n|[mistralai/Ministral-3-3B-Base-2512](https://modelscope.cn/models/mistralai/Ministral-3-3B-Base-2512)|mistral3|mistral_2512|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-3B-Base-2512](https://huggingface.co/mistralai/Ministral-3-3B-Base-2512)|\n|[mistralai/Ministral-3-3B-Instruct-2512](https://modelscope.cn/models/mistralai/Ministral-3-3B-Instruct-2512)|mistral3|mistral_2512|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-3B-Instruct-2512](https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512)|\n|[mistralai/Ministral-3-3B-Instruct-2512-BF16](https://modelscope.cn/models/mistralai/Ministral-3-3B-Instruct-2512-BF16)|mistral3|mistral_2512|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-3B-Instruct-2512-BF16](https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512-BF16)|\n|[mistralai/Ministral-3-8B-Base-2512](https://modelscope.cn/models/mistralai/Ministral-3-8B-Base-2512)|mistral3|mistral_2512|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-8B-Base-2512](https://huggingface.co/mistralai/Ministral-3-8B-Base-2512)|\n|[mistralai/Ministral-3-8B-Instruct-2512](https://modelscope.cn/models/mistralai/Ministral-3-8B-Instruct-2512)|mistral3|mistral_2512|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-8B-Instruct-2512](https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512)|\n|[mistralai/Ministral-3-8B-Instruct-2512-BF16](https://modelscope.cn/models/mistralai/Ministral-3-8B-Instruct-2512-BF16)|mistral3|mistral_2512|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-8B-Instruct-2512-BF16](https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512-BF16)|\n|[mistralai/Ministral-3-14B-Base-2512](https://modelscope.cn/models/mistralai/Ministral-3-14B-Base-2512)|mistral3|mistral_2512|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-14B-Base-2512](https://huggingface.co/mistralai/Ministral-3-14B-Base-2512)|\n|[mistralai/Ministral-3-14B-Instruct-2512](https://modelscope.cn/models/mistralai/Ministral-3-14B-Instruct-2512)|mistral3|mistral_2512|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-14B-Instruct-2512](https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512)|\n|[mistralai/Ministral-3-14B-Instruct-2512-BF16](https://modelscope.cn/models/mistralai/Ministral-3-14B-Instruct-2512-BF16)|mistral3|mistral_2512|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-14B-Instruct-2512-BF16](https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512-BF16)|\n|[mistralai/Ministral-3-3B-Reasoning-2512](https://modelscope.cn/models/mistralai/Ministral-3-3B-Reasoning-2512)|mistral3|mistral_2512_thinking|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-3B-Reasoning-2512](https://huggingface.co/mistralai/Ministral-3-3B-Reasoning-2512)|\n|[mistralai/Ministral-3-8B-Reasoning-2512](https://modelscope.cn/models/mistralai/Ministral-3-8B-Reasoning-2512)|mistral3|mistral_2512_thinking|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-8B-Reasoning-2512](https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512)|\n|[mistralai/Ministral-3-14B-Reasoning-2512](https://modelscope.cn/models/mistralai/Ministral-3-14B-Reasoning-2512)|mistral3|mistral_2512_thinking|transformers>=5.0.0.dev0, mistral-common>=1.8.6|✘|vision|[mistralai/Ministral-3-14B-Reasoning-2512](https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512)|\n|[mistralai/Mistral-Small-3.2-24B-Instruct-2506](https://modelscope.cn/models/mistralai/Mistral-Small-3.2-24B-Instruct-2506)|mistral3_2506|mistral_2506|transformers>=4.49|✘|-|[mistralai/Mistral-Small-3.2-24B-Instruct-2506](https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506)|\n|[PaddlePaddle/PaddleOCR-VL](https://modelscope.cn/models/PaddlePaddle/PaddleOCR-VL)|paddle_ocr|paddle_ocr|-|✘|-|[PaddlePaddle/PaddleOCR-VL](https://huggingface.co/PaddlePaddle/PaddleOCR-VL)|\n|[PaddlePaddle/PaddleOCR-VL-1.5](https://modelscope.cn/models/PaddlePaddle/PaddleOCR-VL-1.5)|paddle_ocr_1_5|paddle_ocr_1_5|-|✘|-|[PaddlePaddle/PaddleOCR-VL-1.5](https://huggingface.co/PaddlePaddle/PaddleOCR-VL-1.5)|\n|[Tencent-Hunyuan/HunyuanOCR](https://modelscope.cn/models/Tencent-Hunyuan/HunyuanOCR)|hunyuan_ocr|hunyuan_ocr|transformers>=4.49.0|✘|-|[tencent/HunyuanOCR](https://huggingface.co/tencent/HunyuanOCR)|\n|[stepfun-ai/Step3-VL-10B-Base](https://modelscope.cn/models/stepfun-ai/Step3-VL-10B-Base)|step3_vl|step3_vl|transformers>=4.57.0|✘|vision|[stepfun-ai/Step3-VL-10B-Base](https://huggingface.co/stepfun-ai/Step3-VL-10B-Base)|\n|[stepfun-ai/Step3-VL-10B](https://modelscope.cn/models/stepfun-ai/Step3-VL-10B)|step3_vl|step3_vl|transformers>=4.57.0|✘|vision|[stepfun-ai/Step3-VL-10B](https://huggingface.co/stepfun-ai/Step3-VL-10B)|\n|[JinaAI/jina-reranker-m0](https://modelscope.cn/models/JinaAI/jina-reranker-m0)|jina_reranker_m0|jina_reranker_m0|-|✘|reranker, vision|[JinaAI/jina-reranker-m0](https://huggingface.co/JinaAI/jina-reranker-m0)|\n\n\n## 数据集\n下表介绍了ms-swift接入的数据集的相关信息:\n- Dataset ID: ModelScope数据集id\n- HF Dataset ID: HuggingFace数据集id\n- Subset Name: 子数据集名称\n- Dataset Size: 数据集大小\n- Statistic: 数据集的统计量. 我们使用token数进行统计, 这对于调整`max_length`超参数有帮助. 我们使用qwen2.5的tokenizer对数据集进行分词. 不同的tokenizer的统计量不同, 如果你要获取其他的模型的tokenizer的token统计量, 可以通过[脚本](https://github.com/modelscope/ms-swift/tree/main/scripts/utils/run_dataset_info.py)自行获取.\n- Tags: 数据集的tags\n\n| Dataset ID | Subset Name | Dataset Size | Statistic (token) | Tags | HF Dataset ID |\n| ---------- | ----------- | -------------| ------------------| ---- | ------------- |\n|[AI-MO/NuminaMath-1.5](https://modelscope.cn/datasets/AI-MO/NuminaMath-1.5)|default|896215|116.1±80.8, min=31, max=5064|grpo, math|[AI-MO/NuminaMath-1.5](https://huggingface.co/datasets/AI-MO/NuminaMath-1.5)|\n|[AI-MO/NuminaMath-CoT](https://modelscope.cn/datasets/AI-MO/NuminaMath-CoT)|default|859494|113.1±60.2, min=35, max=2120|grpo, math|[AI-MO/NuminaMath-CoT](https://huggingface.co/datasets/AI-MO/NuminaMath-CoT)|\n|[AI-MO/NuminaMath-TIR](https://modelscope.cn/datasets/AI-MO/NuminaMath-TIR)|default|72441|100.9±52.2, min=36, max=1683|grpo, math, 🔥|[AI-MO/NuminaMath-TIR](https://huggingface.co/datasets/AI-MO/NuminaMath-TIR)|\n|[AI-ModelScope/COIG-CQIA](https://modelscope.cn/datasets/AI-ModelScope/COIG-CQIA)|chinese_traditional:.\n\n--nproc-per-node NPROC_PER_NODE\nNumber of processes per node.\n\nExample:\n\n```bash\nmodel=your model path\ndataset=your dataset\noutput= your output dir\nexport CUDA_VISIBLE_DEVICES=0 # Set according to the actual GPU usage\ndeepspeed_config_or_type=deepspeed type or configuration file path, e.g., zero1 or /xxx/ms-swift/swift/llm/ds_config/zero1.json\n\ndlrover-run --nnodes 1:$NODE_NUM --nproc_per_node=1 \\\n/opt/conda/lib/python3.10/site-packages/swift/cli/sft.py --model $model \\\n--model_type qwen3 \\\n--tuner_type lora \\\n--torch_dtype bfloat16 \\\n--dataset $dataset \\\n--num_train_epochs 4 \\\n--per_device_train_batch_size 1 \\\n--per_device_eval_batch_size 1 \\\n--learning_rate 5e-7 \\\n--gradient_accumulation_steps 8 \\\n--eval_steps 500 \\\n--save_steps 10 \\\n--save_total_limit 20 \\\n--logging_steps 1 \\\n--output_dir $output \\\n--warmup_ratio 0.01 \\\n--dataloader_num_workers 4 \\\n--temperature 1.0 \\\n--system 'You are a helpful assistant.' \\\n--lora_rank 8 \\\n--lora_alpha 32 \\\n--target_modules all-linear \\\n--dataset_num_proc 1 \\\n--use_flash_ckpt true \\\n--callbacks deepspeed_elastic graceful_exit \\\n--deepspeed $deepspeed_config_or_type \\\n```\n\n## Configuration\nBy default, the zero1 configuration is as follows:\n\n```json\n{\n \"fp16\": {\n \"enabled\": \"auto\",\n \"loss_scale\": 0,\n \"loss_scale_window\": 1000,\n \"initial_scale_power\": 16,\n \"hysteresis\": 2,\n \"min_loss_scale\": 1\n },\n\n \"bf16\": {\n \"enabled\": \"auto\"\n },\n\n \"zero_optimization\": {\n \"stage\": 1,\n \"offload_optimizer\": {\n \"device\": \"none\",\n \"pin_memory\": true\n },\n \"allgather_partitions\": true,\n \"allgather_bucket_size\": 2e8,\n \"overlap_comm\": false,\n \"reduce_scatter\": true,\n \"reduce_bucket_size\": 2e8,\n \"contiguous_gradients\": true\n },\n\n \"gradient_accumulation_steps\": \"auto\",\n \"gradient_clipping\": \"auto\",\n \"steps_per_print\": 2000,\n \"train_batch_size\": \"auto\",\n \"train_micro_batch_size_per_gpu\": \"auto\",\n \"wall_clock_breakdown\": false,\n \"elasticity\": {\n \"ignore_non_elastic_batch_info\": true,\n \"enabled\": true,\n \"max_train_batch_size\": 8,\n \"micro_batch_sizes\": [\n 4,\n 2\n ],\n \"min_gpus\": 1,\n \"max_gpus\": 4,\n \"min_time\": 20,\n \"version\": 0.1\n }\n}\n```\n\nIf users need custom configurations, they can specify the path to the custom zero1.json file in the deepspeed_config_or_type parameter. The elasticity-related configuration is as follows:\n```json\n...\n\n \"elasticity\": {\n \"ignore_non_elastic_batch_info\": true,\n \"enabled\": true,\n \"max_train_batch_size\": 8,\n \"micro_batch_sizes\": [\n 4,\n 2\n ],\n \"min_gpus\": 1,\n \"max_gpus\": 4,\n \"min_time\": 20,\n \"version\": 0.1\n }\n```\n\n- ignore_non_elastic_batch_info：Indicates that the batch size configurations outside the elasticity settings will be ignored. During training, the batch size and related parameters will be dynamically adjusted based on the number of training processes.\nCalculation principle：\n global-training-batch-size = micro-batch-size * gradient-accumulation-steps * world-size\n- max_train_batch_size： Maximum batch size\n- micro_batch_sizes：List of allowed per-GPU micro-batch sizes under elasticity; candidates for train_micro_batch_size_per_gpu.\n- min_gpus：Minimum number of GPUs.\n- max_gpus：Maximum number of GPUs.\nFor more details, see: [Deepspeed](https://www.deepspeed.ai/docs/config-json/#elastic-training-config-v01-and-v02)\n\n## Starting Training\n\n```yaml\n---\napiVersion: elastic.iml.github.io/v1alpha1\nkind: ElasticJob\nmetadata:\n name: deepspeed-elastic-swift\n namespace: dlrover\nspec:\n distributionStrategy: AllreduceStrategy\n optimizeMode: single-job\n replicaSpecs:\n worker:\n replicas: 1 # This should match the maximum value of --nnodes NNODES in the startup command\n template:\n spec:\n restartPolicy: Never\n containers:\n - name: main\n image: #【Training image, needs to have deepspeed, dlrover, and swift installed】\n imagePullPolicy: IfNotPresent\n command:\n - /bin/bash\n - -c\n - sh start.sh # Startup script\n resources:\n limits:\n cpu: '8'\n memory: 16Gi\n nvidia.com/gpu: '1'\n volumeMounts:\n - mountPath: /model\n name: volume-model\n - mountPath: /dev/shm\n name: volume-shm\n restartPolicy: Never\n volumes:\n - hostPath:\n path: /model\n type: Directory\n name: volume-model\n - emptyDir:\n medium: Memory\n sizeLimit: 200Gi\n name: volume-shm\n\n```\n"
},
{
"path": "docs/source_en/BestPractices/Embedding.md",
"content": "# Embedding Training\n\nSWIFT has already supported the training of embedding models, including both pure text and multimodal types. Currently supported models include:\n\n1. modernbert embedding model\n - [ModelScope](https://modelscope.cn/models/iic/gte-modernbert-base) [Hugging Face](https://huggingface.co/Alibaba-NLP/gte-modernbert-base)\n2. gte embedding models\n - 1.5B: [ModelScope](https://www.modelscope.cn/models/iic/gte_Qwen2-1.5B-instruct) [Hugging Face](https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct)\n - 7B: [ModelScope](https://www.modelscope.cn/models/iic/gte_Qwen2-7B-instruct) [Hugging Face](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct)\n3. gme embedding models\n - 2B: [ModelScope](https://www.modelscope.cn/models/iic/gme-Qwen2-VL-2B-Instruct) [Hugging Face](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct)\n - 7B: [ModelScope](https://www.modelscope.cn/models/iic/gme-Qwen2-VL-7B-Instruct) [Hugging Face](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct)\n4. qwen3-embedding models\n - 0.6B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-Embedding-0.6B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B)\n - 4B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-Embedding-4B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-Embedding-4B)\n - 8B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-Embedding-8B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-Embedding-8B)\n5. qwen3-vl-embedding models\n - 2B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-VL-Embedding-2B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-VL-Embedding-2B)\n - 8B: [ModelScope](https://www.modelscope.cn/models/Qwen/Qwen3-VL-Embedding-8B) [Hugging Face](https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B)\n\nDevelopers can integrate their own models by ensuring the model forward output satisfies:\n\n```text\n{\"last_hidden_state\": some-embedding-tensor}\n```\n\nThe return value should be a JSON with a `last_hidden_state` key, where the value is an embedding tensor. For the input part, you can use our already supported templates. Users can also specify the\n\n```shell\n --task_type embedding\n```\nparameter to convert any other model into an embedding model for training.\n\nIt should be noted that the embedding models currently supported by SWIFT are all based on pure text or multimodal LLMs, and CLIP-type model training is not currently supported.\n\nAdditionally, all embedding models supported by SWIFT have normalization added at the end of the model forward pass. If you add new models yourself, please remember to include a normalization layer.\n\n## Loss\n\nThe Embedding models supported by SWIFT currently can use the following loss functions:\n\n- **cosine_similarity**: Cosine similarity loss, which calculates the similarity between two embeddings and fits based on the label value. It is effectively an MSE loss.\n- **contrastive**: Contrastive learning loss with adjustable margin. Labels are only supported as 0 and 1.\n- **online_contrastive**: Contrastive loss considering hard negatives and hard positives. Labels are only supported as 0 and 1.\n- **infonce**: Computes pairwise cosine similarities between different rows within the same batch, maximizing similarity within rows and minimizing similarity between different rows. No labels are required.\n\nThe source code for the loss functions can be found [here](https://github.com/modelscope/ms-swift/blob/main/swift/loss/mapping.py).\n\n## Dataset Format\n\n> **Note:**\n> 1. The `` tag can appear anywhere inside `messages`/`positive_messages`/`negative_messages`. Each group has its own image fields: `images`/`positive_images`/`negative_images` to provide paths or URLs.\n> 2. There is no longer any cross-field ordering requirement. Alignment rules:\n> - `images` length equals the number of `` tags in `messages`.\n> - `positive_images` and `negative_images` are both list-of-list. Their outer lengths equal the lengths of `positive_messages` and `negative_messages` respectively. For each outer item, the inner list length equals the number of `` tags in that message sequence.\n> 3. `messages` is the anchor sample; `positive_messages` and `negative_messages` are each a list of messages (hence one more `[]`). Accordingly, `positive_images`/`negative_images` are also list-of-list and align item-by-item.\n> 4. `` and `` are supported as well. Follow the same rules via `videos`/`positive_videos`/`negative_videos` and `audios`/`positive_audios`/`negative_audios`.\n> 5. Current constraint: the outer length of `positive_messages` must be 1 (i.e., provide exactly one positive). Accordingly, the outer length of `positive_images` must also be 1.\n\n### Format for Cosine Similarity Loss\n\n```json lines\n# LLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]], \"label\": 0.8}\n# MLLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"\"}], \"images\": [\"/some/images1.jpg\"], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence\"}]], \"positive_images\": [[\"/some/images2.jpg\"]], \"label\": 0.7}\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]], \"positive_images\": [[\"/some/images.jpg\"]], \"label\": 0.7}\n```\n\nThe eval metrics are the Pearson and Spearman's Rank Correlation Coefficient of the embeddings' euclidean distance/dot production and so on, totally 8 values.\n\n### Format for Contrastive/Online Contrastive Loss\n\n```json lines\n# LLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]], \"label\": 1}\n# MLLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"\"}], \"images\": [\"/some/images1.jpg\"], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence\"}]], \"positive_images\": [[\"/some/images2.jpg\"]], \"label\": 1}\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]], \"positive_images\": [[\"/some/images.jpg\"]], \"label\": 0}\n```\n\n### Format for InfoNCE\n\n```json lines\n# LLM\n{\"messages\": [{\"role\": \"user\", \"content\": \"sentence1\"}], \"positive_messages\": [[{\"role\": \"user\", \"content\": \"sentence2\"}]]}\n# MLLM\n{\"messages\": [{\"role\": \"user\", \"content\": \" ![](\"asset/wechat.png\")
\n![](\"asset/banner.png\"/)
\n ![](\"https://img.shields.io/badge/python-3.11-5be.svg\"){kind=icon}
\n![](\"https://img.shields.io/badge/pytorch-%E2%89%A52.0-orange.svg\"){kind=icon}
\n![](\"https://img.shields.io/badge/modelscope-%E2%89%A51.23-5D91D4.svg\"){kind=icon}
\n![](\"https://badge.fury.io/py/ms-swift.svg\"){kind=icon}
\n![](\"https://img.shields.io/github/license/modelscope/ms-swift\")
![](\"https://img.shields.io/badge/PR-welcome-55EB99.svg\"){kind=icon}
\n![](\"asset/discord_qr.jpg\")
|