Repository: mli/autocut Branch: main Commit: ba2bb3bfbd57 Files: 29 Total size: 76.1 KB Directory structure: gitextract_oaq66n5h/ ├── .github/ │ └── workflows/ │ ├── base.yml │ ├── faster-whisper │ └── lint.yml ├── .gitignore ├── Dockerfile ├── Dockerfile.cuda ├── LICENSE ├── README.md ├── autocut/ │ ├── __init__.py │ ├── __main__.py │ ├── cut.py │ ├── daemon.py │ ├── main.py │ ├── package_transcribe.py │ ├── transcribe.py │ ├── type.py │ ├── utils.py │ └── whisper_model.py ├── setup.cfg ├── setup.py ├── tea.yaml └── test/ ├── config.py ├── content/ │ ├── test.srt │ ├── test_md.md │ └── test_srt.srt ├── media/ │ ├── test003.mkv │ └── test004.flv ├── test_cut.py └── test_transcribe.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/base.yml ================================================ name: Test on: pull_request: push: branches: - main jobs: lint_and_test: runs-on: ${{ matrix.os }}-latest strategy: matrix: python-version: ['3.9', '3.10'] # Wait for fix on macos-m1: https://github.com/federicocarboni/setup-ffmpeg/issues/21 os: [ubuntu, windows, macos-12] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Set Variables id: set_variables shell: bash run: | echo "PY=$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" >> $GITHUB_OUTPUT echo "PIP_CACHE=$(pip cache dir)" >> $GITHUB_OUTPUT - name: Cache PIP uses: actions/cache@v3 with: path: ${{ steps.set_variables.outputs.PIP_CACHE }} key: ${{ runner.os }}-pip-${{ steps.set_variables.outputs.PY }} - name: Setup ffmpeg for different platforms uses: FedericoCarboni/setup-ffmpeg@v3 - name: Install dependencies run: | python -m pip install --upgrade pip pip install . pip install pytest - name: Run Test run: pytest test/ ================================================ FILE: .github/workflows/faster-whisper ================================================ name: Test Faster Whisper on: pull_request: push: branches: - main jobs: lint_and_test: runs-on: ${{ matrix.os }}-latest strategy: matrix: python-version: ['3.9', '3.10'] # macos did not support m1 for now os: [ubuntu, windows, macos] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Set Variables id: set_variables shell: bash run: | echo "PY=$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" >> $GITHUB_OUTPUT echo "PIP_CACHE=$(pip cache dir)" >> $GITHUB_OUTPUT - name: Cache PIP uses: actions/cache@v3 with: path: ${{ steps.set_variables.outputs.PIP_CACHE }} key: ${{ runner.os }}-pip-${{ steps.set_variables.outputs.PY }} - name: Setup ffmpeg for differnt platforms uses: FedericoCarboni/setup-ffmpeg@master - name: Install dependencies run: | python -m pip install --upgrade pip pip install ".[faster]" pip install pytest - name: Run Test run: WHISPER_MODE=faster pytest test/ ================================================ FILE: .github/workflows/lint.yml ================================================ name: Test Lint on: pull_request: push: branches: - main jobs: lint: runs-on: ${{ matrix.os }}-latest strategy: matrix: python-version: ['3.9'] os: [ubuntu] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Set Variables id: set_variables shell: bash run: | echo "PY=$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" >> $GITHUB_OUTPUT echo "PIP_CACHE=$(pip cache dir)" >> $GITHUB_OUTPUT - name: Cache PIP uses: actions/cache@v3 with: path: ${{ steps.set_variables.outputs.PIP_CACHE }} key: ${{ runner.os }}-pip-${{ steps.set_variables.outputs.PY }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install black - name: Run Lint run: black . --check ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ log/ ================================================ FILE: Dockerfile ================================================ FROM python:3.9-slim as base RUN mkdir /autocut COPY ./ /autocut WORKDIR /autocut RUN apt update && \ apt install -y git && \ apt install -y ffmpeg RUN pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu && \ pip install . ================================================ FILE: Dockerfile.cuda ================================================ FROM pytorch/pytorch:1.13.0-cuda11.6-cudnn8-runtime RUN mkdir /autocut COPY ./ /autocut WORKDIR /autocut RUN apt update && \ apt install -y git && \ apt install -y ffmpeg RUN pip install . ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # AutoCut: 通过字幕来剪切视频 AutoCut 对你的视频自动生成字幕。然后你选择需要保留的句子，AutoCut 将对你视频中对应的片段裁切并保存。你无需使用视频编辑软件，只需要编辑文本文件即可完成剪切。 **2024.10.05更新**：支持 `large-v3-turbo` [模型](https://github.com/openai/whisper/discussions/2363)，提供更快的转录速度。 ```shell autocut -t xxx --whisper-model large-v3-turbo ```` **2024.03.10更新**：支持 pip 安装和提供 import 转录相关的功能 ```shell # Install pip install autocut-sub ``` ```python from autocut import Transcribe, load_audio ``` **2023.10.14更新**：支持 faster-whisper 和指定依赖（但由于 Action 限制暂时移除了 faster-whisper 的测试运行） ```shell # for whisper only pip install . # for whisper and faster-whisper pip install '.[faster]' # for whisper and openai-whisper pip install '.[openai]' # for all pip install '.[all]' ``` ```shell # using faster-whisper autocut -t xxx --whisper-mode=faster ``` ```shell # using openai api export OPENAI_API_KEY=sk-xxx autocut -t xxx --whisper-mode=openai --openai-rpm=3 ``` **2023.8.13更新**：支持调用 Openai Whisper API ```shell export OPENAI_API_KEY=sk-xxx autocut -t xxx --whisper-mode=openai --openai-rpm=3 ``` ## 使用例子 假如你录制的视频放在 `2022-11-04/` 这个文件夹里。那么运行 ```bash autocut -d 2022-11-04 ``` > 提示：如果你使用 OBS 录屏，可以在 `设置->高级->录像->文件名格式` 中将空格改成 `/`，即 `%CCYY-%MM-%DD/%hh-%mm-%ss`。那么视频文件将放在日期命名的文件夹里。 AutoCut 将持续对这个文件夹里视频进行字幕抽取和剪切。例如，你刚完成一个视频录制，保存在 `11-28-18.mp4`。AutoCut 将生成 `11-28-18.md`。你在里面选择需要保留的句子后，AutoCut 将剪切出 `11-28-18_cut.mp4`，并生成 `11-28-18_cut.md` 来预览结果。 你可以使用任何的 Markdown 编辑器。例如我常用 VS Code 和 Typora。下图是通过 Typora 来对 `11-28-18.md` 编辑。 ![](imgs/typora.jpg) 全部完成后在 `autocut.md` 里选择需要拼接的视频后，AutoCut 将输出 `autocut_merged.mp4` 和对应的字幕文件。 ## 安装 首先安装 Python 包 ``` pip install git+https://github.com/mli/autocut.git ``` ## 本地安装测试 ``` git clone https://github.com/mli/autocut cd autocut pip install . ``` > 上面将安装 [pytorch](https://pytorch.org/)。如果你需要 GPU 运行，且默认安装的版本不匹配的话，你可以先安装 Pytorch。如果安装 Whipser 出现问题，请参考[官方文档](https://github.com/openai/whisper#setup)。 另外需要安装 [ffmpeg](https://ffmpeg.org/) ``` # on Ubuntu or Debian sudo apt update && sudo apt install ffmpeg # on Arch Linux sudo pacman -S ffmpeg # on MacOS using Homebrew (https://brew.sh/) brew install ffmpeg # on Windows using Scoop (https://scoop.sh/) scoop install ffmpeg ``` ## Docker 安装 首先将项目克隆到本地。 ```bash git clone https://github.com/mli/autocut.git ``` ### 安装 CPU 版本 进入项目根目录，然后构建 docker 映像。 ```bash docker build -t autocut . ``` 运行下面的命令创建 docker 容器，就可以直接使用了。 ```bash docker run -it --rm -v E:\autocut:/autocut/video autocut /bin/bash ``` 其中 `-v` 是将主机存放视频的文件夹 `E:\autocut` 映射到虚拟机的 `/autocut/video` 目录。`E:\autocut` 是主机存放视频的目录，需修改为自己主机存放视频的目录。 ### 安装 GPU 版本 使用 GPU 加速需要主机有 Nvidia 的显卡并安装好相应驱动。然后在项目根目录，执行下面的命令构建 docker 映像。 ```bash docker build -f ./Dockerfile.cuda -t autocut-gpu . ``` 使用 GPU 加速时，运行 docker 容器需添加参数 `--gpus all`。 ```bash docker run --gpus all -it --rm -v E:\autocut:/autocut/video autocut-gpu ``` ## 更多使用选项 ### 转录某个视频生成 `.srt` 和 `.md` 结果。 ```bash autocut -t 22-52-00.mp4 ``` 1. 如果对转录质量不满意，可以使用更大的模型，例如 ```bash autocut -t 22-52-00.mp4 --whisper-model large ``` 默认是 `small`。更好的模型是 `medium` 和 `large`，但推荐使用 GPU 获得更好的速度。也可以使用更快的 `tiny` 和 `base`，但转录质量会下降。 ### 剪切某个视频 ```bash autocut -c 22-52-00.mp4 22-52-00.srt 22-52-00.md ``` 1. 默认视频比特率是 `--bitrate 10m`，你可以根据需要调大调小。 2. 如果不习惯 Markdown 格式文件，你也可以直接在 `srt` 文件里删除不要的句子，在剪切时不传入 `md` 文件名即可。就是 `autocut -c 22-52-00.mp4 22-52-00.srt` 3. 如果仅有 `srt` 文件，编辑不方便可以使用如下命令生成 `md` 文件，然后编辑 `md` 文件即可，但此时会完全对照 `srt` 生成，不会出现 `no speech` 等提示文本。 ```bash autocut -m test.srt test.mp4 autocut -m test.mp4 test.srt # 支持视频和字幕乱序传入 autocut -m test.srt # 也可以只传入字幕文件 ``` ### 一些小提示 1. 讲得流利的视频的转录质量会高一些，这因为是 Whisper 训练数据分布的缘故。对一个视频，你可以先粗选一下句子，然后在剪出来的视频上再剪一次。 2. 最终视频生成的字幕通常还需要做一些小编辑。但 `srt` 里面空行太多。你可以使用 `autocut -s 22-52-00.srt` 来生成一个紧凑些的版本 `22-52-00_compact.srt` 方便编辑（这个格式不合法，但编辑器，例如 VS Code，还是会进行语法高亮）。编辑完成后，`autocut -s 22-52-00_compact.srt` 转回正常格式。 3. 用 Typora 和 VS Code 编辑 Markdown 都很方便。他们都有对应的快捷键 mark 一行或者多行。但 VS Code 视频预览似乎有点问题。 4. 视频是通过 ffmpeg 导出。在 Apple M1 芯片上它用不了 GPU，导致导出速度不如专业视频软件。 ### 常见问题 1. **输出的是乱码？** AutoCut 默认输出编码是 `utf-8`. 确保你的编辑器也使用了 `utf-8` 解码。你可以通过 `--encoding` 指定其他编码格式。但是需要注意生成字幕文件和使用字幕文件剪辑时的编码格式需要一致。例如使用 `gbk`。 ```bash autocut -t test.mp4 --encoding=gbk autocut -c test.mp4 test.srt test.md --encoding=gbk ``` 如果使用了其他编码格式（如 `gbk` 等）生成 `md` 文件并用 Typora 打开后，该文件可能会被 Typora 自动转码为其他编码格式，此时再通过生成时指定的编码格式进行剪辑时可能会出现编码不支持等报错。因此可以在使用 Typora 编辑后再通过 VSCode 等修改到你需要的编码格式进行保存后再使用剪辑功能。 2. **如何使用 GPU 来转录？** 当你有 Nvidia GPU，而且安装了对应版本的 PyTorch 的时候，转录是在 GPU 上进行。你可以通过命令来查看当前是不是支持 GPU。 ```bash python -c "import torch; print(torch.cuda.is_available())" ``` 否则你可以在安装 AutoCut 前手动安装对应的 GPU 版本 PyTorch。 3. **使用 GPU 时报错显存不够。** whisper 的大模型需要一定的 GPU 显存。如果你的显存不够，你可以用小一点的模型，例如 `small`。如果你仍然想用大模型，可以通过 `--device` 来强制使用 CPU。例如 ```bash autocut -t 11-28-18.mp4 --whisper-model large --device cpu ``` 4. **能不能使用 `pip` 安装?** whisper已经发布到PyPI了，可以直接用`pip install openai-whisper`安装。 [https://github.com/openai/whisper#setup](https://github.com/openai/whisper#setup) [https://pypi.org/project/openai-whisper/](https://pypi.org/project/openai-whisper/) ## 如何参与贡献 [这里有一些想做的 feature](https://github.com/mli/autocut/issues/22)，欢迎贡献。 ### 代码结构 ```text autocut │ .gitignore │ LICENSE │ README.md # 一般新增或修改需要让使用者知道就需要对应更新 README.md 内容 │ setup.py │ └─autocut # 核心代码位于 autocut 文件夹中，新增功能的实现也一般在这里面进行修改或新增 │ cut.py │ daemon.py │ main.py │ transcribe.py │ utils.py └─ __init__.py ``` ### 安装依赖 开始安装这个项目的需要的依赖之前，建议先了解一下 Anaconda 或者 venv 的虚拟环境使用，推荐**使用虚拟环境来搭建该项目的开发环境**。 具体安装方式为在你搭建搭建的虚拟环境之中按照[上方安装步骤](./README.md#安装)进行安装。 > 为什么推荐使用虚拟环境开发？ > > 一方面是保证各种不同的开发环境之间互相不污染。 > > 更重要的是在于这个项目实际上是一个 Python Package，所以在你安装之后 AutoCut 的代码实际也会变成你的环境依赖。 > **因此在你更新代码之后，你需要让将新代码重新安装到环境中，然后才能调用到新的代码。** ### 开发 1. 代码风格目前遵循 PEP-8，可以使用相关的自动格式化软件完成。 2. `utils.py` 主要是全局共用的一些工具方法。 3. `transcribe.py` 是调用模型生成`srt`和`md`的部分。 4. `cut.py` 提供根据标记后`md`或`srt`进行视频剪切合并的功能。 5. `daemon.py` 提供的是监听文件夹生成字幕和剪切视频的功能。 6. `main.py` 声明命令行参数，根据输入参数调用对应功能。 开发过程中请尽量保证修改在正确的地方，以及合理地复用代码， 同时工具函数请尽可能放在`utils.py`中。 代码格式目前是遵循 PEP-8，变量命名尽量语义化即可。 在开发完成之后，最重要的一点是需要进行**测试**，请保证提交之前对所有**与你修改直接相关的部分**以及**你修改会影响到的部分**都进行了测试，并保证功能的正常。 目前使用 `GitHub Actions` CI, Lint 使用 black 提交前请运行 `black`。 ### 提交 1. commit 信息用英文描述清楚你做了哪些修改即可，小写字母开头。 2. 最好可以保证一次的 commit 涉及的修改比较小，可以简短地描述清楚，这样也方便之后有修改时的查找。 3. PR 的时候 title 简述有哪些修改， contents 可以具体写下修改内容。 4. run test `pip install pytest` then `pytest test` 5. run lint `pip install black` then `black .` ================================================ FILE: autocut/__init__.py ================================================ __version__ = "1.1.0" from .type import LANG, WhisperModel, WhisperMode from .utils import load_audio from .package_transcribe import Transcribe __all__ = ["Transcribe", "load_audio", "WhisperMode", "WhisperModel", "LANG"] ================================================ FILE: autocut/__main__.py ================================================ from .main import main if __name__ == "__main__": main() ================================================ FILE: autocut/cut.py ================================================ import logging import os import re import srt from moviepy import editor from . import utils # Merge videos class Merger: def __init__(self, args): self.args = args def write_md(self, videos): md = utils.MD(self.args.inputs[0], self.args.encoding) num_tasks = len(md.tasks()) # Not overwrite if already marked as down or no new videos if md.done_editing() or num_tasks == len(videos) + 1: return md.clear() md.add_done_editing(False) md.add("\nSelect the files that will be used to generate `autocut_final.mp4`\n") base = lambda fn: os.path.basename(fn) for f in videos: md_fn = utils.change_ext(f, "md") video_md = utils.MD(md_fn, self.args.encoding) # select a few words to scribe the video desc = "" if len(video_md.tasks()) > 1: for _, t in video_md.tasks()[1:]: m = re.findall(r"\] (.*)", t) if m and "no speech" not in m[0].lower(): desc += m[0] + " " if len(desc) > 50: break md.add_task( False, f'[{base(f)}]({base(md_fn)}) {"[Edited]" if video_md.done_editing() else ""} {desc}', ) md.write() def run(self): md_fn = self.args.inputs[0] md = utils.MD(md_fn, self.args.encoding) if not md.done_editing(): return videos = [] for m, t in md.tasks(): if not m: continue m = re.findall(r"\[(.*)\]", t) if not m: continue fn = os.path.join(os.path.dirname(md_fn), m[0]) logging.info(f"Loading {fn}") videos.append(editor.VideoFileClip(fn)) dur = sum([v.duration for v in videos]) logging.info(f"Merging into a video with {dur / 60:.1f} min length") merged = editor.concatenate_videoclips(videos) fn = os.path.splitext(md_fn)[0] + "_merged.mp4" merged.write_videofile( fn, audio_codec="aac", bitrate=self.args.bitrate ) # logger=None, logging.info(f"Saved merged video to {fn}") # Cut media class Cutter: def __init__(self, args): self.args = args def run(self): fns = {"srt": None, "media": None, "md": None} for fn in self.args.inputs: ext = os.path.splitext(fn)[1][1:] fns[ext if ext in fns else "media"] = fn assert fns["media"], "must provide a media filename" assert fns["srt"], "must provide a srt filename" is_video_file = utils.is_video(fns["media"].lower()) outext = "mp4" if is_video_file else "mp3" output_fn = utils.change_ext(utils.add_cut(fns["media"]), outext) if utils.check_exists(output_fn, self.args.force): return with open(fns["srt"], encoding=self.args.encoding) as f: subs = list(srt.parse(f.read())) if fns["md"]: md = utils.MD(fns["md"], self.args.encoding) if not md.done_editing(): return index = [] for mark, sent in md.tasks(): if not mark: continue m = re.match(r"\[(\d+)", sent.strip()) if m: index.append(int(m.groups()[0])) subs = [s for s in subs if s.index in index] logging.info(f'Cut {fns["media"]} based on {fns["srt"]} and {fns["md"]}') else: logging.info(f'Cut {fns["media"]} based on {fns["srt"]}') segments = [] # Avoid disordered subtitles subs.sort(key=lambda x: x.start) for x in subs: if len(segments) == 0: segments.append( {"start": x.start.total_seconds(), "end": x.end.total_seconds()} ) else: if x.start.total_seconds() - segments[-1]["end"] < 0.5: segments[-1]["end"] = x.end.total_seconds() else: segments.append( {"start": x.start.total_seconds(), "end": x.end.total_seconds()} ) if is_video_file: media = editor.VideoFileClip(fns["media"]) else: media = editor.AudioFileClip(fns["media"]) # Add a fade between two clips. Not quite necessary. keep code here for reference # fade = 0 # segments = _expand_segments(segments, fade, 0, video.duration) # clips = [video.subclip( # s['start'], s['end']).crossfadein(fade) for s in segments] # final_clip = editor.concatenate_videoclips(clips, padding = -fade) clips = [media.subclip(s["start"], s["end"]) for s in segments] if is_video_file: final_clip: editor.VideoClip = editor.concatenate_videoclips(clips) logging.info( f"Reduced duration from {media.duration:.1f} to {final_clip.duration:.1f}" ) aud = final_clip.audio.set_fps(44100) final_clip = final_clip.without_audio().set_audio(aud) final_clip = final_clip.fx(editor.afx.audio_normalize) # an alternative to birate is use crf, e.g. ffmpeg_params=['-crf', '18'] final_clip.write_videofile( output_fn, audio_codec="aac", bitrate=self.args.bitrate ) else: final_clip: editor.AudioClip = editor.concatenate_audioclips(clips) logging.info( f"Reduced duration from {media.duration:.1f} to {final_clip.duration:.1f}" ) final_clip = final_clip.fx(editor.afx.audio_normalize) final_clip.write_audiofile( output_fn, codec="libmp3lame", fps=44100, bitrate=self.args.bitrate ) media.close() logging.info(f"Saved media to {output_fn}") ================================================ FILE: autocut/daemon.py ================================================ import copy import glob import logging import os import time from . import cut, transcribe, utils class Daemon: def __init__(self, args): self.args = args self.sleep = 1 def run(self): assert len(self.args.inputs) == 1, "Must provide a single folder" while True: self._iter() time.sleep(self.sleep) self.sleep = min(60, self.sleep + 1) def _iter(self): folder = self.args.inputs[0] files = sorted(list(glob.glob(os.path.join(folder, "*")))) media_files = [f for f in files if utils.is_video(f) or utils.is_audio(f)] args = copy.deepcopy(self.args) for f in media_files: srt_fn = utils.change_ext(f, "srt") md_fn = utils.change_ext(f, "md") is_video_file = utils.is_video(f) if srt_fn not in files or md_fn not in files: args.inputs = [f] try: transcribe.Transcribe(args).run() self.sleep = 1 break except RuntimeError as e: logging.warn( "Failed, may be due to the video is still on recording" ) pass if md_fn in files: if utils.add_cut(md_fn) in files: continue md = utils.MD(md_fn, self.args.encoding) ext = "mp4" if is_video_file else "mp3" if not md.done_editing() or os.path.exists( utils.change_ext(utils.add_cut(f), ext) ): continue args.inputs = [f, md_fn, srt_fn] cut.Cutter(args).run() self.sleep = 1 args.inputs = [os.path.join(folder, "autocut.md")] merger = cut.Merger(args) merger.write_md(media_files) merger.run() ================================================ FILE: autocut/main.py ================================================ import argparse import logging import os from . import utils from .type import WhisperMode, WhisperModel def main(): parser = argparse.ArgumentParser( description="Edit videos based on transcribed subtitles", formatter_class=argparse.RawDescriptionHelpFormatter, ) logging.basicConfig( format="[autocut:%(filename)s:L%(lineno)d] %(levelname)-6s %(message)s" ) logging.getLogger().setLevel(logging.INFO) parser.add_argument("inputs", type=str, nargs="+", help="Inputs filenames/folders") parser.add_argument( "-t", "--transcribe", help="Transcribe videos/audio into subtitles", action=argparse.BooleanOptionalAction, ) parser.add_argument( "-c", "--cut", help="Cut a video based on subtitles", action=argparse.BooleanOptionalAction, ) parser.add_argument( "-d", "--daemon", help="Monitor a folder to transcribe and cut", action=argparse.BooleanOptionalAction, ) parser.add_argument( "-s", help="Convert .srt to a compact format for easier editing", action=argparse.BooleanOptionalAction, ) parser.add_argument( "-m", "--to-md", help="Convert .srt to .md for easier editing", action=argparse.BooleanOptionalAction, ) parser.add_argument( "--lang", type=str, default="zh", choices=[ "zh", "en", "Afrikaans", "Arabic", "Armenian", "Azerbaijani", "Belarusian", "Bosnian", "Bulgarian", "Catalan", "Croatian", "Czech", "Danish", "Dutch", "Estonian", "Finnish", "French", "Galician", "German", "Greek", "Hebrew", "Hindi", "Hungarian", "Icelandic", "Indonesian", "Italian", "Japanese", "Kannada", "Kazakh", "Korean", "Latvian", "Lithuanian", "Macedonian", "Malay", "Marathi", "Maori", "Nepali", "Norwegian", "Persian", "Polish", "Portuguese", "Romanian", "Russian", "Serbian", "Slovak", "Slovenian", "Spanish", "Swahili", "Swedish", "Tagalog", "Tamil", "Thai", "Turkish", "Ukrainian", "Urdu", "Vietnamese", "Welsh", ], help="The output language of transcription", ) parser.add_argument( "--prompt", type=str, default="", help="initial prompt feed into whisper" ) parser.add_argument( "--whisper-mode", type=str, default=WhisperMode.WHISPER.value, choices=WhisperMode.get_values(), help="Whisper inference mode: whisper: run whisper locally; openai: use openai api.", ) parser.add_argument( "--openai-rpm", type=int, default=3, choices=[3, 50], help="Openai Whisper API REQUESTS PER MINUTE(FREE USERS: 3RPM; PAID USERS: 50RPM). " "More info: https://platform.openai.com/docs/guides/rate-limits/overview", ) parser.add_argument( "--whisper-model", type=str, default=WhisperModel.SMALL.value, choices=WhisperModel.get_values(), help="The whisper model used to transcribe.", ) parser.add_argument( "--bitrate", type=str, default="10m", help="The bitrate to export the cutted video, such as 10m, 1m, or 500k", ) parser.add_argument( "--vad", help="If or not use VAD", choices=["1", "0", "auto"], default="auto" ) parser.add_argument( "--force", help="Force write even if files exist", action=argparse.BooleanOptionalAction, ) parser.add_argument( "--encoding", type=str, default="utf-8", help="Document encoding format" ) parser.add_argument( "--device", type=str, default=None, choices=["cpu", "cuda"], help="Force to CPU or GPU for transcribing. In default automatically use GPU if available.", ) args = parser.parse_args() if args.transcribe: from .transcribe import Transcribe Transcribe(args).run() elif args.to_md: from .utils import trans_srt_to_md if len(args.inputs) == 2: [input_1, input_2] = args.inputs base, ext = os.path.splitext(input_1) if ext != ".srt": input_1, input_2 = input_2, input_1 trans_srt_to_md(args.encoding, args.force, input_1, input_2) elif len(args.inputs) == 1: trans_srt_to_md(args.encoding, args.force, args.inputs[0]) else: logging.warning( "Wrong number of files, please pass in a .srt file or an additional video file" ) elif args.cut: from .cut import Cutter Cutter(args).run() elif args.daemon: from .daemon import Daemon Daemon(args).run() elif args.s: utils.compact_rst(args.inputs[0], args.encoding) else: logging.warning("No action, use -c, -t or -d") if __name__ == "__main__": main() ================================================ FILE: autocut/package_transcribe.py ================================================ import logging import time from typing import List, Any, Union, Literal import numpy as np import torch from . import utils, whisper_model from .type import WhisperMode, SPEECH_ARRAY_INDEX, WhisperModel, LANG class Transcribe: def __init__( self, whisper_mode: Union[ WhisperMode.WHISPER.value, WhisperMode.FASTER.value ] = WhisperMode.WHISPER.value, whisper_model_size: WhisperModel.get_values() = "small", vad: bool = True, device: Union[Literal["cpu", "cuda"], None] = None, ): self.whisper_mode = whisper_mode self.whisper_model_size = whisper_model_size self.vad = vad self.device = device self.sampling_rate = 16000 self.whisper_model = None self.vad_model = None self.detect_speech = None tic = time.time() if self.whisper_model is None: if self.whisper_mode == WhisperMode.WHISPER.value: self.whisper_model = whisper_model.WhisperModel(self.sampling_rate) self.whisper_model.load(self.whisper_model_size, self.device) elif self.whisper_mode == WhisperMode.FASTER.value: self.whisper_model = whisper_model.FasterWhisperModel( self.sampling_rate ) self.whisper_model.load(self.whisper_model_size, self.device) logging.info(f"Done Init model in {time.time() - tic:.1f} sec") def run(self, audio: np.ndarray, lang: LANG, prompt: str = ""): speech_array_indices = self._detect_voice_activity(audio) transcribe_results = self._transcribe(audio, speech_array_indices, lang, prompt) return transcribe_results def format_results_to_srt(self, transcribe_results: List[Any]): return self.whisper_model.gen_srt(transcribe_results) def _detect_voice_activity(self, audio) -> List[SPEECH_ARRAY_INDEX]: """Detect segments that have voice activities""" if self.vad is False: return [{"start": 0, "end": len(audio)}] tic = time.time() if self.vad_model is None or self.detect_speech is None: # torch load limit https://github.com/pytorch/vision/issues/4156 torch.hub._validate_not_a_forked_repo = lambda a, b, c: True self.vad_model, funcs = torch.hub.load( repo_or_dir="snakers4/silero-vad", model="silero_vad", trust_repo=True ) self.detect_speech = funcs[0] speeches = self.detect_speech( audio, self.vad_model, sampling_rate=self.sampling_rate ) # Remove too short segments speeches = utils.remove_short_segments(speeches, 1.0 * self.sampling_rate) # Expand to avoid to tight cut. You can tune the pad length speeches = utils.expand_segments( speeches, 0.2 * self.sampling_rate, 0.0 * self.sampling_rate, audio.shape[0] ) # Merge very closed segments speeches = utils.merge_adjacent_segments(speeches, 0.5 * self.sampling_rate) logging.info(f"Done voice activity detection in {time.time() - tic:.1f} sec") return speeches if len(speeches) > 1 else [{"start": 0, "end": len(audio)}] def _transcribe( self, audio: np.ndarray, speech_array_indices: List[SPEECH_ARRAY_INDEX], lang: LANG, prompt: str = "", ) -> List[Any]: tic = time.time() res = self.whisper_model.transcribe(audio, speech_array_indices, lang, prompt) logging.info(f"Done transcription in {time.time() - tic:.1f} sec") return res ================================================ FILE: autocut/transcribe.py ================================================ import logging import os import time from typing import List, Any import numpy as np import srt import torch from . import utils, whisper_model from .type import WhisperMode, SPEECH_ARRAY_INDEX class Transcribe: def __init__(self, args): self.args = args self.sampling_rate = 16000 self.whisper_model = None self.vad_model = None self.detect_speech = None tic = time.time() if self.whisper_model is None: if self.args.whisper_mode == WhisperMode.WHISPER.value: self.whisper_model = whisper_model.WhisperModel(self.sampling_rate) self.whisper_model.load(self.args.whisper_model, self.args.device) elif self.args.whisper_mode == WhisperMode.OPENAI.value: self.whisper_model = whisper_model.OpenAIModel( self.args.openai_rpm, self.sampling_rate ) self.whisper_model.load() elif self.args.whisper_mode == WhisperMode.FASTER.value: self.whisper_model = whisper_model.FasterWhisperModel( self.sampling_rate ) self.whisper_model.load(self.args.whisper_model, self.args.device) logging.info(f"Done Init model in {time.time() - tic:.1f} sec") def run(self): for input in self.args.inputs: logging.info(f"Transcribing {input}") name, _ = os.path.splitext(input) if utils.check_exists(name + ".md", self.args.force): continue audio = utils.load_audio(input, sr=self.sampling_rate) speech_array_indices = self._detect_voice_activity(audio) transcribe_results = self._transcribe(input, audio, speech_array_indices) output = name + ".srt" self._save_srt(output, transcribe_results) logging.info(f"Transcribed {input} to {output}") self._save_md(name + ".md", output, input) logging.info(f'Saved texts to {name + ".md"} to mark sentences') def _detect_voice_activity(self, audio) -> List[SPEECH_ARRAY_INDEX]: """Detect segments that have voice activities""" if self.args.vad == "0": return [{"start": 0, "end": len(audio)}] tic = time.time() if self.vad_model is None or self.detect_speech is None: # torch load limit https://github.com/pytorch/vision/issues/4156 torch.hub._validate_not_a_forked_repo = lambda a, b, c: True self.vad_model, funcs = torch.hub.load( repo_or_dir="snakers4/silero-vad", model="silero_vad", trust_repo=True ) self.detect_speech = funcs[0] speeches = self.detect_speech( audio, self.vad_model, sampling_rate=self.sampling_rate ) # Remove too short segments speeches = utils.remove_short_segments(speeches, 1.0 * self.sampling_rate) # Expand to avoid to tight cut. You can tune the pad length speeches = utils.expand_segments( speeches, 0.2 * self.sampling_rate, 0.0 * self.sampling_rate, audio.shape[0] ) # Merge very closed segments speeches = utils.merge_adjacent_segments(speeches, 0.5 * self.sampling_rate) logging.info(f"Done voice activity detection in {time.time() - tic:.1f} sec") return speeches if len(speeches) > 1 else [{"start": 0, "end": len(audio)}] def _transcribe( self, input: str, audio: np.ndarray, speech_array_indices: List[SPEECH_ARRAY_INDEX], ) -> List[Any]: tic = time.time() res = ( self.whisper_model.transcribe( audio, speech_array_indices, self.args.lang, self.args.prompt ) if self.args.whisper_mode == WhisperMode.WHISPER.value or self.args.whisper_mode == WhisperMode.FASTER.value else self.whisper_model.transcribe( input, audio, speech_array_indices, self.args.lang, self.args.prompt ) ) logging.info(f"Done transcription in {time.time() - tic:.1f} sec") return res def _save_srt(self, output, transcribe_results): subs = self.whisper_model.gen_srt(transcribe_results) with open(output, "wb") as f: f.write(srt.compose(subs).encode(self.args.encoding, "replace")) def _save_md(self, md_fn, srt_fn, video_fn): with open(srt_fn, encoding=self.args.encoding) as f: subs = srt.parse(f.read()) md = utils.MD(md_fn, self.args.encoding) md.clear() md.add_done_editing(False) md.add_video(os.path.basename(video_fn)) md.add( f"\nTexts generated from [{os.path.basename(srt_fn)}]({os.path.basename(srt_fn)})." "Mark the sentences to keep for autocut.\n" "The format is [subtitle_index,duration_in_second] subtitle context.\n\n" ) for s in subs: sec = s.start.seconds pre = f"[{s.index},{sec // 60:02d}:{sec % 60:02d}]" md.add_task(False, f"{pre:11} {s.content.strip()}") md.write() ================================================ FILE: autocut/type.py ================================================ from enum import Enum from typing import TypedDict, Literal SPEECH_ARRAY_INDEX = TypedDict("SPEECH_ARRAY_INDEX", {"start": float, "end": float}) LANG = Literal[ "zh", "en", "Afrikaans", "Arabic", "Armenian", "Azerbaijani", "Belarusian", "Bosnian", "Bulgarian", "Catalan", "Croatian", "Czech", "Danish", "Dutch", "Estonian", "Finnish", "French", "Galician", "German", "Greek", "Hebrew", "Hindi", "Hungarian", "Icelandic", "Indonesian", "Italian", "Japanese", "Kannada", "Kazakh", "Korean", "Latvian", "Lithuanian", "Macedonian", "Malay", "Marathi", "Maori", "Nepali", "Norwegian", "Persian", "Polish", "Portuguese", "Romanian", "Russian", "Serbian", "Slovak", "Slovenian", "Spanish", "Swahili", "Swedish", "Tagalog", "Tamil", "Thai", "Turkish", "Ukrainian", "Urdu", "Vietnamese", "Welsh", ] class WhisperModel(Enum): TINY = "tiny" BASE = "base" SMALL = "small" MEDIUM = "medium" LARGE = "large" LARGE_V2 = "large-v2" LARGE_V3 = "large-v3" LARGE_V3_TURBO = "large-v3-turbo" @staticmethod def get_values(): return [i.value for i in WhisperModel] class WhisperMode(Enum): WHISPER = "whisper" OPENAI = "openai" FASTER = "faster" @staticmethod def get_values(): return [i.value for i in WhisperMode] ================================================ FILE: autocut/utils.py ================================================ import logging import os import re import ffmpeg import numpy as np import opencc import srt def load_audio(file: str, sr: int = 16000) -> np.ndarray: try: out, _ = ( ffmpeg.input(file, threads=0) .output("-", format="s16le", acodec="pcm_s16le", ac=1, ar=sr) .run(cmd=["ffmpeg", "-nostdin"], capture_stdout=True, capture_stderr=True) ) except ffmpeg.Error as e: raise RuntimeError(f"Failed to load audio: {e.stderr.decode()}") from e return np.frombuffer(out, np.int16).flatten().astype(np.float32) / 32768.0 def is_video(filename): _, ext = os.path.splitext(filename) return ext in [".mp4", ".mov", ".mkv", ".avi", ".flv", ".f4v", ".webm"] def is_audio(filename): _, ext = os.path.splitext(filename) return ext in [".ogg", ".wav", ".mp3", ".flac", ".m4a"] def change_ext(filename, new_ext): # Change the extension of filename to new_ext base, _ = os.path.splitext(filename) if not new_ext.startswith("."): new_ext = "." + new_ext return base + new_ext def add_cut(filename): # Add cut mark to the filename base, ext = os.path.splitext(filename) if base.endswith("_cut"): base = base[:-4] + "_" + base[-4:] else: base += "_cut" return base + ext # a very simple markdown parser class MD: def __init__(self, filename, encoding): self.lines = [] self.EDIT_DONE_MAKR = "<-- Mark if you are done editing." self.encoding = encoding self.filename = filename if filename: self.load_file() def load_file(self): if os.path.exists(self.filename): with open(self.filename, encoding=self.encoding) as f: self.lines = f.readlines() def clear(self): self.lines = [] def write(self): with open(self.filename, "wb") as f: f.write("\n".join(self.lines).encode(self.encoding, "replace")) def tasks(self): # get all tasks with their status ret = [] for l in self.lines: mark, task = self._parse_task_status(l) if mark is not None: ret.append((mark, task)) return ret def done_editing(self): for m, t in self.tasks(): if m and self.EDIT_DONE_MAKR in t: return True return False def add(self, line): self.lines.append(line) def add_task(self, mark, contents): self.add(f'- [{"x" if mark else " "}] {contents.strip()}') def add_done_editing(self, mark): self.add_task(mark, self.EDIT_DONE_MAKR) def add_video(self, video_fn): ext = os.path.splitext(video_fn)[1][1:] self.add( f'\n\n' ) def _parse_task_status(self, line): # return (is_marked, rest) or (None, line) if not a task m = re.match(r"- +\[([ xX])\] +(.*)", line) if not m: return None, line return m.groups()[0].lower() == "x", m.groups()[1] def check_exists(output, force): if os.path.exists(output): if force: logging.info(f"{output} exists. Will overwrite it") else: logging.info( f"{output} exists, skipping... Use the --force flag to overwrite" ) return True return False def expand_segments(segments, expand_head, expand_tail, total_length): # Pad head and tail for each time segment results = [] for i in range(len(segments)): t = segments[i] start = max(t["start"] - expand_head, segments[i - 1]["end"] if i > 0 else 0) end = min( t["end"] + expand_tail, segments[i + 1]["start"] if i < len(segments) - 1 else total_length, ) results.append({"start": start, "end": end}) return results def remove_short_segments(segments, threshold): # Remove segments whose length < threshold return [s for s in segments if s["end"] - s["start"] > threshold] def merge_adjacent_segments(segments, threshold): # Merge two adjacent segments if their distance < threshold results = [] i = 0 while i < len(segments): s = segments[i] for j in range(i + 1, len(segments)): if segments[j]["start"] < s["end"] + threshold: s["end"] = segments[j]["end"] i = j else: break i += 1 results.append(s) return results def compact_rst(sub_fn, encoding): cc = opencc.OpenCC("t2s") base, ext = os.path.splitext(sub_fn) COMPACT = "_compact" if ext != ".srt": logging.fatal("only .srt file is supported") if base.endswith(COMPACT): # to original rst with open(sub_fn, encoding=encoding) as f: lines = f.readlines() subs = [] for l in lines: items = l.split(" ") if len(items) < 4: continue subs.append( srt.Subtitle( index=0, start=srt.srt_timestamp_to_timedelta(items[0]), end=srt.srt_timestamp_to_timedelta(items[2]), content=" ".join(items[3:]).strip(), ) ) with open(base[: -len(COMPACT)] + ext, "wb") as f: f.write(srt.compose(subs).encode(encoding, "replace")) else: # to a compact version with open(sub_fn, encoding=encoding) as f: subs = srt.parse(f.read()) with open(base + COMPACT + ext, "wb") as f: for s in subs: f.write( f"{srt.timedelta_to_srt_timestamp(s.start)} --> {srt.timedelta_to_srt_timestamp(s.end)} " f"{cc.convert(s.content.strip())}\n".encode(encoding, "replace") ) def trans_srt_to_md(encoding, force, srt_fn, video_fn=None): base, ext = os.path.splitext(srt_fn) if ext != ".srt": logging.fatal("only .srt file is supported") md_fn = base + ext.split(".")[0] + ".md" check_exists(md_fn, force) with open(srt_fn, encoding=encoding) as f: subs = srt.parse(f.read()) md = MD(md_fn, encoding) md.clear() md.add_done_editing(False) if video_fn: if not is_video(video_fn): logging.fatal(f"{video_fn} may not be a video") md.add_video(os.path.basename(video_fn)) md.add( f"\nTexts generated from [{os.path.basename(srt_fn)}]({os.path.basename(srt_fn)})." "Mark the sentences to keep for autocut.\n" "The format is [subtitle_index,duration_in_second] subtitle context.\n\n" ) for s in subs: sec = s.start.seconds pre = f"[{s.index},{sec // 60:02d}:{sec % 60:02d}]" md.add_task(False, f"{pre:11} {s.content.strip()}") md.write() ================================================ FILE: autocut/whisper_model.py ================================================ import datetime import logging import os from abc import ABC, abstractmethod from typing import Literal, Union, List, Any, TypedDict import numpy as np import opencc import srt from pydub import AudioSegment from tqdm import tqdm from .type import SPEECH_ARRAY_INDEX, LANG # whisper sometimes generate traditional chinese, explicitly convert cc = opencc.OpenCC("t2s") class AbstractWhisperModel(ABC): def __init__(self, mode, sample_rate=16000): self.mode = mode self.whisper_model = None self.sample_rate = sample_rate @abstractmethod def load(self, *args, **kwargs): pass @abstractmethod def transcribe(self, *args, **kwargs): pass @abstractmethod def _transcribe(self, *args, **kwargs): pass @abstractmethod def gen_srt(self, transcribe_results: List[Any]) -> List[srt.Subtitle]: pass class WhisperModel(AbstractWhisperModel): def __init__(self, sample_rate=16000): super().__init__("whisper", sample_rate) self.device = None def load( self, model_name: Literal[ "tiny", "base", "small", "medium", "large", "large-v2" ] = "small", device: Union[Literal["cpu", "cuda"], None] = None, ): self.device = device import whisper self.whisper_model = whisper.load_model(model_name, device) def _transcribe(self, audio, seg, lang, prompt): r = self.whisper_model.transcribe( audio[int(seg["start"]) : int(seg["end"])], task="transcribe", language=lang, initial_prompt=prompt, ) r["origin_timestamp"] = seg return r def transcribe( self, audio: np.ndarray, speech_array_indices: List[SPEECH_ARRAY_INDEX], lang: LANG, prompt: str, ): res = [] if self.device == "cpu" and len(speech_array_indices) > 1: from multiprocessing import Pool pbar = tqdm(total=len(speech_array_indices)) pool = Pool(processes=4) sub_res = [] # TODO, a better way is merging these segments into a single one, so whisper can get more context for seg in speech_array_indices: sub_res.append( pool.apply_async( self._transcribe, ( self.whisper_model, audio, seg, lang, prompt, ), callback=lambda x: pbar.update(), ) ) pool.close() pool.join() pbar.close() res = [i.get() for i in sub_res] else: for seg in ( speech_array_indices if len(speech_array_indices) == 1 else tqdm(speech_array_indices) ): r = self.whisper_model.transcribe( audio[int(seg["start"]) : int(seg["end"])], task="transcribe", language=lang, initial_prompt=prompt, verbose=False if len(speech_array_indices) == 1 else None, ) r["origin_timestamp"] = seg res.append(r) return res def gen_srt(self, transcribe_results): subs = [] def _add_sub(start, end, text): subs.append( srt.Subtitle( index=0, start=datetime.timedelta(seconds=start), end=datetime.timedelta(seconds=end), content=cc.convert(text.strip()), ) ) prev_end = 0 for r in transcribe_results: origin = r["origin_timestamp"] for s in r["segments"]: start = s["start"] + origin["start"] / self.sample_rate end = min( s["end"] + origin["start"] / self.sample_rate, origin["end"] / self.sample_rate, ) if start > end: continue # mark any empty segment that is not very short if start > prev_end + 1.0: _add_sub(prev_end, start, "< No Speech >") _add_sub(start, end, s["text"]) prev_end = end return subs class OpenAIModel(AbstractWhisperModel): max_single_audio_bytes = 25 * 2**20 # 25MB split_audio_bytes = 23 * 2**20 # 23MB, 2MB for safety(header, etc.) rpm = 3 def __init__(self, rpm: int, sample_rate=16000): super().__init__("openai_whisper-1", sample_rate) self.rpm = rpm if ( os.environ.get("OPENAI_API_KEY") is None and os.environ.get("OPENAI_API_KEY_PATH") is None ): raise Exception("OPENAI_API_KEY is not set") def load(self, model_name: Literal["whisper-1"] = "whisper-1"): try: import openai except ImportError: raise Exception( "Please use openai mode(pip install '.[openai]') or all mode(pip install '.[all]')" ) from functools import partial self.whisper_model = partial(openai.Audio.transcribe, model=model_name) def transcribe( self, input: srt, audio: np.ndarray, speech_array_indices: List[SPEECH_ARRAY_INDEX], lang: LANG, prompt: str, ) -> List[srt.Subtitle]: res = [] name, _ = os.path.splitext(input) raw_audio = AudioSegment.from_file(input) ms_bytes = len(raw_audio[:1].raw_data) audios: List[ TypedDict( "AudioInfo", {"input": str, "audio": AudioSegment, "start_ms": float} ) ] = [] i = 0 for index in speech_array_indices: start = int(index["start"]) / self.sample_rate * 1000 end = int(index["end"]) / self.sample_rate * 1000 audio_seg = raw_audio[start:end] if len(audio_seg.raw_data) < self.split_audio_bytes: temp_file = f"{name}_temp_{i}.wav" audios.append( {"input": temp_file, "audio": audio_seg, "start_ms": start} ) else: logging.info( f"Long audio with a size({len(audio_seg.raw_data)} bytes) greater than 25M({25 * 2 ** 20} bytes) " "will be segmented" "due to Openai's API restrictions on files smaller than 25M" ) split_num = len(audio_seg.raw_data) // self.split_audio_bytes + 1 for j in range(split_num): temp_file = f"{name}_{i}_temp_{j}.wav" split_audio = audio_seg[ j * self.split_audio_bytes // ms_bytes : (j + 1) * self.split_audio_bytes // ms_bytes ] audios.append( { "input": temp_file, "audio": split_audio, "start_ms": start + j * self.split_audio_bytes // ms_bytes, } ) i += 1 if len(audios) > 1: from multiprocessing import Pool pbar = tqdm(total=len(audios)) pool = Pool(processes=min(8, self.rpm)) sub_res = [] for audio in audios: sub_res.append( pool.apply_async( self._transcribe, ( audio["input"], audio["audio"], prompt, lang, audio["start_ms"], ), callback=lambda x: pbar.update(), ) ) pool.close() pool.join() pbar.close() for subs in sub_res: subtitles = subs.get() res.extend(subtitles) else: res = self._transcribe( audios[0]["input"], audios[0]["audio"], prompt, lang, audios[0]["start_ms"], ) return res def _transcribe( self, input: srt, audio: AudioSegment, prompt: str, lang: LANG, start_ms: float ): audio.export(input, "wav") subtitles = self.whisper_model( file=open(input, "rb"), prompt=prompt, language=lang, response_format="srt" ) os.remove(input) return list( map( lambda x: ( setattr( x, "start", x.start + datetime.timedelta(milliseconds=start_ms) ), setattr( x, "end", x.end + datetime.timedelta(milliseconds=start_ms) ), x, )[-1], list(srt.parse(subtitles)), ) ) def gen_srt(self, transcribe_results: List[srt.Subtitle]): if len(transcribe_results) == 0: return [] if len(transcribe_results) == 1: return transcribe_results subs = [transcribe_results[0]] for subtitle in transcribe_results[1:]: if subtitle.start - subs[-1].end > datetime.timedelta(seconds=1): subs.append( srt.Subtitle( index=0, start=subs[-1].end, end=subtitle.start, content="< No Speech >", ) ) subs.append(subtitle) return subs class FasterWhisperModel(AbstractWhisperModel): def __init__(self, sample_rate=16000): super().__init__("faster-whisper", sample_rate) self.device = None def load( self, model_name: Literal[ "tiny", "base", "small", "medium", "large", "large-v2" ] = "small", device: Union[Literal["cpu", "cuda"], None] = None, ): try: from faster_whisper import WhisperModel except ImportError: raise Exception( "Please use faster mode(pip install '.[faster]') or all mode(pip install '.[all]')" ) self.device = device if device else "cpu" self.whisper_model = WhisperModel(model_name, self.device) def _transcribe(self): raise Exception("Not implemented") def transcribe( self, audio: np.ndarray, speech_array_indices: List[SPEECH_ARRAY_INDEX], lang: LANG, prompt: str, ): res = [] for seg in speech_array_indices: segments, info = self.whisper_model.transcribe( audio[int(seg["start"]) : int(seg["end"])], task="transcribe", language=lang, initial_prompt=prompt, vad_filter=False, ) segments = list(segments) # The transcription will actually run here. r = {"origin_timestamp": seg, "segments": segments, "info": info} res.append(r) return res def gen_srt(self, transcribe_results): subs = [] def _add_sub(start, end, text): subs.append( srt.Subtitle( index=0, start=datetime.timedelta(seconds=start), end=datetime.timedelta(seconds=end), content=cc.convert(text.strip()), ) ) prev_end = 0 for r in transcribe_results: origin = r["origin_timestamp"] for seg in r["segments"]: s = dict(start=seg.start, end=seg.end, text=seg.text) start = s["start"] + origin["start"] / self.sample_rate end = min( s["end"] + origin["start"] / self.sample_rate, origin["end"] / self.sample_rate, ) if start > end: continue # mark any empty segment that is not very short if start > prev_end + 1.0: _add_sub(prev_end, start, "< No Speech >") _add_sub(start, end, s["text"]) prev_end = end return subs ================================================ FILE: setup.cfg ================================================ [metadata] name = autocut version = attr: autocut.__version__ license = Apache Software License description = Cut video by subtitles long_description = file: README.md classifiers = License :: OSI Approved :: Apache Software License Operating System :: OS Independent Programming Language :: Python :: 3 [options] packages = find: include_package_data = True python_requires = >= 3.9 ================================================ FILE: setup.py ================================================ from setuptools import setup, find_packages requirements = [ "ffmpeg-python", "moviepy", "openai-whisper", "opencc-python-reimplemented", "parameterized", "pydub", "srt", "torchaudio", "tqdm", ] setup( name="autocut-sub", install_requires=requirements, url="https://github.com/mli/autocut", project_urls={ "source": "https://github.com/mli/autocut", }, license="Apache License 2.0", long_description=open("README.md", "r", encoding="utf-8").read(), long_description_content_type="text/markdown", extras_require={ "all": ["openai", "faster-whisper"], "openai": ["openai"], "faster": ["faster-whisper"], }, packages=find_packages(), entry_points={ "console_scripts": [ "autocut = autocut.main:main", ] }, ) ================================================ FILE: tea.yaml ================================================ # https://tea.xyz/what-is-this-file --- version: 1.0.0 codeOwners: - '0x1e292d6f2D09dc8ffDDb5B8Fd6b641e180224D84' quorum: 1 ================================================ FILE: test/config.py ================================================ import logging import os # 定义一个日志收集器 logger = logging.getLogger() # 设置收集器的级别，不设定的话，默认收集warning及以上级别的日志 logger.setLevel("DEBUG") # 设置日志格式 fmt = logging.Formatter("%(filename)s-%(lineno)d-%(asctime)s-%(levelname)s-%(message)s") # 设置日志处理器-输出到文件,并且设置编码格式 if not os.path.exists("./log"): os.makedirs("./log") file_handler = logging.FileHandler("./log/log.txt", encoding="utf-8") # 设置日志处理器级别 file_handler.setLevel("DEBUG") # 处理器按指定格式输出日志 file_handler.setFormatter(fmt) # 输出到控制台 ch = logging.StreamHandler() # 设置日志处理器级别 ch.setLevel("DEBUG") # 处理器按指定格式输出日志 ch.setFormatter(fmt) # 收集器和处理器对接，指定输出渠道 # 日志输出到文件 logger.addHandler(file_handler) # 日志输出到控制台 logger.addHandler(ch) TEST_MEDIA_PATH = "./test/media/" TEST_CONTENT_PATH = "./test/content/" TEST_MEDIA_FILE = [ "test001.mp4", "test002.mov", "test003.mkv", "test004.flv", "test005.mp3", "test006.MP4", ] TEST_MEDIA_FILE_LANG = ["test001_en.mp4"] TEST_MEDIA_FILE_SIMPLE = ["test001.mp4", "test005.mp3"] class TestArgs: def __init__(self): self.inputs = [] self.bitrate = "10m" self.encoding = "utf-8" self.sampling_rate = 16000 self.lang = "zh" self.prompt = "" self.whisper_model = "small" self.device = None self.vad = False self.force = False self.whisper_mode = ( "faster" if os.environ.get("WHISPER_MODE") == "faster" else "whisper" ) self.openai_rpm = 3 ================================================ FILE: test/content/test.srt ================================================ 1 00:00:00,000 --> 00:00:05,000 大家好,我的名字是AutoCut.这是一条用于测试的视频。 2 00:00:05,000 --> 00:00:10,260 Hello, my name is AutoCut. This is a video for testing. ================================================ FILE: test/content/test_md.md ================================================ - [x] <-- Mark if you are done editing. Texts generated from [test001.srt](test001.srt).Mark the sentences to keep for autocut. The format is [subtitle_index,duration_in_second] subtitle context. - [ ] [1,00:00] 大家好,我的名字是AutoCut.这是一条用于测试的视频。 - [x] [2,00:05] Hello, my name is AutoCut. This is a video for testing. ================================================ FILE: test/content/test_srt.srt ================================================ 1 00:00:00,000 --> 00:00:05,000 大家好,我的名字是AutoCut.这是一条用于测试的视频。 ================================================ FILE: test/test_cut.py ================================================ import logging import os import unittest from parameterized import parameterized, param from autocut.cut import Cutter from config import TestArgs, TEST_MEDIA_PATH, TEST_MEDIA_FILE_SIMPLE, TEST_CONTENT_PATH class TestCut(unittest.TestCase): @classmethod def setUpClass(cls): logging.info("检查测试文件是否正常存在") scan_file = os.listdir(TEST_MEDIA_PATH) logging.info( "应存在文件列表：" + str(TEST_MEDIA_FILE_SIMPLE) + " 扫描到文件列表：" + str(scan_file) ) for file in TEST_MEDIA_FILE_SIMPLE: assert file in scan_file def tearDown(self): for file in TEST_MEDIA_FILE_SIMPLE: namepart = os.path.join( TEST_MEDIA_PATH, os.path.splitext(file)[0] + "_cut." ) if os.path.exists(namepart + "mp4"): os.remove(namepart + "mp4") if os.path.exists(namepart + "mp3"): os.remove(namepart + "mp3") @parameterized.expand([param(file) for file in TEST_MEDIA_FILE_SIMPLE]) def test_srt_cut(self, file_name): args = TestArgs() args.inputs = [ os.path.join(TEST_MEDIA_PATH, file_name), os.path.join(TEST_CONTENT_PATH, "test_srt.srt"), ] cut = Cutter(args) cut.run() namepart = os.path.join( TEST_MEDIA_PATH, os.path.splitext(file_name)[0] + "_cut." ) self.assertTrue( os.path.exists(namepart + "mp4") or os.path.exists(namepart + "mp3") ) @parameterized.expand([param(file) for file in TEST_MEDIA_FILE_SIMPLE]) def test_md_cut(self, file_name): args = TestArgs() args.inputs = [ TEST_MEDIA_PATH + file_name, os.path.join(TEST_CONTENT_PATH, "test.srt"), os.path.join(TEST_CONTENT_PATH, "test_md.md"), ] cut = Cutter(args) cut.run() namepart = os.path.join( TEST_MEDIA_PATH, os.path.splitext(file_name)[0] + "_cut." ) self.assertTrue( os.path.exists(namepart + "mp4") or os.path.exists(namepart + "mp3") ) ================================================ FILE: test/test_transcribe.py ================================================ import logging import os import unittest from parameterized import parameterized, param from autocut.utils import MD from config import ( TEST_MEDIA_FILE, TestArgs, TEST_MEDIA_FILE_SIMPLE, TEST_MEDIA_FILE_LANG, TEST_MEDIA_PATH, ) from autocut.transcribe import Transcribe class TestTranscribe(unittest.TestCase): @classmethod def setUpClass(cls): logging.info("检查测试文件是否正常存在") scan_file = os.listdir(TEST_MEDIA_PATH) logging.info( "应存在文件列表：" + str(TEST_MEDIA_FILE) + str(TEST_MEDIA_FILE_LANG) + str(TEST_MEDIA_FILE_SIMPLE) + " 扫描到文件列表：" + str(scan_file) ) for file in TEST_MEDIA_FILE: assert file in scan_file for file in TEST_MEDIA_FILE_LANG: assert file in scan_file for file in TEST_MEDIA_FILE_SIMPLE: assert file in scan_file @classmethod def tearDownClass(cls): for file in os.listdir(TEST_MEDIA_PATH): if file.endswith("md") or file.endswith("srt"): os.remove(TEST_MEDIA_PATH + file) def tearDown(self): for file in TEST_MEDIA_FILE_SIMPLE: if os.path.exists(TEST_MEDIA_PATH + file.split(".")[0] + ".md"): os.remove(TEST_MEDIA_PATH + file.split(".")[0] + ".md") if os.path.exists(TEST_MEDIA_PATH + file.split(".")[0] + ".srt"): os.remove(TEST_MEDIA_PATH + file.split(".")[0] + ".srt") @parameterized.expand([param(file) for file in TEST_MEDIA_FILE]) def test_default_transcribe(self, file_name): logging.info("检查默认参数生成字幕") args = TestArgs() args.inputs = [TEST_MEDIA_PATH + file_name] transcribe = Transcribe(args) transcribe.run() self.assertTrue( os.path.exists(TEST_MEDIA_PATH + file_name.split(".")[0] + ".md") ) @parameterized.expand([param(file) for file in TEST_MEDIA_FILE]) def test_jump_done_transcribe(self, file_name): logging.info("检查默认参数跳过生成字幕") args = TestArgs() args.inputs = [TEST_MEDIA_PATH + file_name] transcribe = Transcribe(args) transcribe.run() self.assertTrue( os.path.exists(TEST_MEDIA_PATH + file_name.split(".")[0] + ".md") ) @parameterized.expand([param(file) for file in TEST_MEDIA_FILE_LANG]) def test_en_transcribe(self, file_name): logging.info("检查--lang='en'参数生成字幕") args = TestArgs() args.lang = "en" args.inputs = [TEST_MEDIA_PATH + file_name] transcribe = Transcribe(args) transcribe.run() self.assertTrue( os.path.exists(TEST_MEDIA_PATH + file_name.split(".")[0] + ".md") ) @parameterized.expand([param(file) for file in TEST_MEDIA_FILE_LANG]) def test_force_transcribe(self, file_name): logging.info("检查--force参数生成字幕") args = TestArgs() args.force = True args.inputs = [TEST_MEDIA_PATH + file_name] md0_lens = len( "".join( MD( TEST_MEDIA_PATH + file_name.split(".")[0] + ".md", args.encoding ).lines ) ) transcribe = Transcribe(args) transcribe.run() md1_lens = len( "".join( MD( TEST_MEDIA_PATH + file_name.split(".")[0] + ".md", args.encoding ).lines ) ) self.assertLessEqual(md1_lens, md0_lens) @parameterized.expand([param(file) for file in TEST_MEDIA_FILE_SIMPLE]) def test_encoding_transcribe(self, file_name): logging.info("检查--encoding参数生成字幕") args = TestArgs() args.encoding = "gbk" args.inputs = [TEST_MEDIA_PATH + file_name] transcribe = Transcribe(args) transcribe.run() with open( os.path.join(TEST_MEDIA_PATH + file_name.split(".")[0] + ".md"), encoding="gbk", ): self.assertTrue(True) @parameterized.expand([param(file) for file in TEST_MEDIA_FILE_SIMPLE]) def test_vad_transcribe(self, file_name): logging.info("检查--vad参数生成字幕") args = TestArgs() args.force = True args.vad = True args.inputs = [TEST_MEDIA_PATH + file_name] transcribe = Transcribe(args) transcribe.run() self.assertTrue( os.path.exists(TEST_MEDIA_PATH + file_name.split(".")[0] + ".md") )