Repository: yihong0618/bilingual_book_maker Branch: main Commit: d66c685c24b4 Files: 60 Total size: 264.8 KB Directory structure: gitextract_6fkzntyz/ ├── .dockerignore ├── .github/ │ └── workflows/ │ ├── docs.yaml │ ├── make_test_ebook.yaml │ └── release.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README-CN.md ├── README.md ├── book_maker/ │ ├── __init__.py │ ├── __main__.py │ ├── cli.py │ ├── config.py │ ├── loader/ │ │ ├── __init__.py │ │ ├── base_loader.py │ │ ├── epub_loader.py │ │ ├── helper.py │ │ ├── md_loader.py │ │ ├── pdf_loader.py │ │ ├── srt_loader.py │ │ └── txt_loader.py │ ├── obok.py │ ├── translator/ │ │ ├── __init__.py │ │ ├── base_translator.py │ │ ├── caiyun_translator.py │ │ ├── chatgptapi_translator.py │ │ ├── claude_translator.py │ │ ├── custom_api_translator.py │ │ ├── deepl_free_translator.py │ │ ├── deepl_translator.py │ │ ├── gemini_translator.py │ │ ├── google_translator.py │ │ ├── groq_translator.py │ │ ├── litellm_translator.py │ │ ├── qwen_translator.py │ │ ├── tencent_transmart_translator.py │ │ └── xai_translator.py │ └── utils.py ├── disclaimer.md ├── docs/ │ ├── book_source.md │ ├── cmd.md │ ├── disclaimer.md │ ├── env_settings.md │ ├── index.md │ ├── installation.md │ ├── model_lang.md │ ├── prompt.md │ └── quickstart.md ├── make_book.py ├── mkdocs.yml ├── prompt_md.json ├── prompt_md.prompt.md ├── prompt_template_sample.json ├── pyproject.toml ├── tests/ │ ├── test_epub_metadata.py │ ├── test_integration.py │ ├── test_pdf_cli.py │ └── test_pdf_loader.py └── typos.toml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ Dockerfile* docker-compose* LICENSE test_books README* .dockerignore .git .github .gitignore .vscode ================================================ FILE: .github/workflows/docs.yaml ================================================ name: Publish docs on: push: branches: - main jobs: deploy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: '3.10' - run: pip install mkdocs mkdocs-material - run: mkdocs gh-deploy --force ================================================ FILE: .github/workflows/make_test_ebook.yaml ================================================ name: CI on: push: branches: [ main ] pull_request: branches: [ main ] workflow_dispatch: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} BBM_CAIYUN_API_KEY: ${{ secrets.BBM_CAIYUN_API_KEY }} jobs: typos-check: name: Spell Check with Typos runs-on: ubuntu-latest steps: - name: Checkout Actions Repository uses: actions/checkout@v3 - name: Check spelling with custom config file uses: crate-ci/typos@v1.16.6 with: config: ./typos.toml testing: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: install python 3.10 uses: actions/setup-python@v4 with: python-version: '3.10' cache: 'pip' # caching pip dependencies - name: Check formatting (black) run: | pip install black black . --check - name: install python requirements run: pip install -r requirements.txt - name: Test install run: | pip install . - name: make normal ebook test using google translate and cli run: | bbook_maker --book_name "test_books/Liber_Esther.epub" --test --test_num 10 --model google --translate-tags div,p bbook_maker --book_name "test_books/Liber_Esther.epub" --test --test_num 20 --model google - name: make txt book test using google translate run: | python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --test_num 20 --model google - name: make txt book test with batch_size run: | python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --batch_size 30 --test_num 20 --model google - name: make caiyun translator test if: env.BBM_CAIYUN_API_KEY != null run: | python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --batch_size 30 --test_num 100 --model caiyun - name: make openai key ebook test if: env.BBM_DEEPL_API_KEY != null run: | python3 make_book.py --book_name "test_books/lemo.epub" --test --test_num 5 --language zh-hans python3 make_book.py --book_name "test_books/animal_farm.epub" --test --test_num 5 --language ja --model gpt3 --prompt prompt_template_sample.txt python3 make_book.py --book_name "test_books/animal_farm.epub" --test --test_num 5 --language ja --prompt prompt_template_sample.json python3 make_book.py --book_name test_books/Lex_Fridman_episode_322.srt --test --test_num 20 - name: Rename and Upload ePub if: env.OPENAI_API_KEY != null uses: actions/upload-artifact@v4 with: name: epub_output path: "test_books/lemo_bilingual.epub" ================================================ FILE: .github/workflows/release.yaml ================================================ name: Release and Build Docker Image permissions: contents: write on: push: tags: - "*" jobs: release-pypi: name: Build and Release PyPI runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - uses: actions/setup-python@v4 with: python-version: "3.10" - uses: actions/setup-node@v3 with: node-version: 16 - name: Build artifacts run: | pip install build python -m build - uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST .idea/ .DS_Store test_books/ # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ /test_books/*.epub log/ .chatgpt_cache.json # for user do not want to push *.srt *.txt *.bin *.epub # For markdown files in user directories .cursorrules books/ prompts/ .pdm-python ================================================ FILE: Dockerfile ================================================ FROM python:3.10-slim RUN apt-get update WORKDIR /app COPY requirements.txt . RUN pip install -r /app/requirements.txt COPY . . ENTRYPOINT ["python3", "make_book.py"] ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2023 yihong Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ SHELL := /bin/bash fmt: @echo "Running formatter ..." venv/bin/black . .PHONY:tests tests: @echo "Running tests ..." venv/bin/pytest tests/test_integration.py serve-docs: mkdocs serve ================================================ FILE: README-CN.md ================================================ # bilingual_book_maker bilingual_book_maker 是一个 AI 翻译工具，使用 ChatGPT 帮助用户制作多语言版本的 epub/txt/srt 文件和图书。该工具仅适用于翻译进入公共版权领域的 epub/txt 图书，不适用于有版权的书籍。请在使用之前阅读项目的 **[免责声明](./disclaimer.md)**。 ![image](https://user-images.githubusercontent.com/15976103/222317531-a05317c5-4eee-49de-95cd-04063d9539d9.png) ## 准备 1. ChatGPT or OpenAI token [^token] 2. epub/txt books 3. 能正常联网的环境或 proxy 4. python3.8+ ## 快速开始 本地放了一个 `test_books/animal_farm.epub` 给大家测试 ```shell pip install -r requirements.txt python3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test 或 pip install -U bbook_maker bbook --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test ``` ## 翻译服务 - 使用 `--openai_key` 指定 OpenAI API key，如果有多个可以用英文逗号分隔(xxx,xxx,xxx)，可以减少接口调用次数限制带来的错误。 或者，指定环境变量 `BBM_OPENAI_API_KEY` 来略过这个选项。 - 默认用了 [GPT-3.5-turbo](https://openai.com/blog/introducing-chatgpt-and-whisper-apis) 模型，也就是 ChatGPT 正在使用的模型。 * DeepL 使用 DeepL 封装的 api 进行翻译，需要付费。[DeepL Translator](https://rapidapi.com/splintPRO/api/dpl-translator) 来获得 token ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model deepl --deepl_key ${deepl_key} ``` * DeepL free 使用 DeepL free ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model deeplfree ``` * Claude 使用 [Claude](https://console.anthropic.com/docs) 模型进行翻译 ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model claude --claude_key ${claude_key} ``` * 谷歌翻译 ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model google ``` * 彩云小译 ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model caiyun --caiyun_key ${caiyun_key} ``` * Gemini ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model gemini --gemini_key ${gemini_key} ``` * Qwen 使用 [Qwen](https://www.aliyun.com/product/dashscope) 模型进行翻译，支持 qwen-mt-turbo 和 qwen-mt-plus 模型。 使用 `--source_lang` 指定源语言，留空为自动检测。 ```shell python3 make_book.py --book_name test_books/animal_farm.epub --qwen_key ${qwen_key} --model qwen-mt-turbo --language "Simplified Chinese" python3 make_book.py --book_name test_books/animal_farm.epub --qwen_key ${qwen_key} --model qwen-mt-plus --language "Japanese" --source_lang "English" ``` * 腾讯交互翻译 ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model tencentransmart ``` * [xAI](https://x.ai) ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model xai --xai_key ${xai_key} ``` * [Ollama](https://github.com/ollama/ollama) 使用 [Ollama](https://github.com/ollama/ollama) 自托管模型进行翻译。 如果 ollama server 不运行在本地，使用 `--api_base http://x.x.x.x:port/v1` 指向 ollama server 地址 ```shell python3 make_book.py --book_name test_books/animal_farm.epub --ollama_model ${ollama_model_name} ``` * [Groq](https://console.groq.com/keys) GroqCloud 当前支持的模型可以查看[Supported Models](https://console.groq.com/docs/models) ```shell python3 make_book.py --book_name test_books/animal_farm.epub --groq_key [your_key] --model groq --model_list llama3-8b-8192 ``` ## 使用说明 - 翻译完会生成一本 `{book_name}_bilingual.epub` 的双语书 - 如果出现了错误或使用 `CTRL+C` 中断命令，不想接下来继续翻译了，会生成一本 `{book_name}_bilingual_temp.epub` 的书，直接改成你想要的名字就可以了 ## 参数说明 - `--test`: 如果大家没付费可以加上这个先看看效果（有 limit 稍微有些慢） - `--language`: 指定目标语言 - 例如： `--language "Simplified Chinese"`，预设值为 `"Simplified Chinese"`. - 请阅读 helper message 来查找可用的目标语言： `python make_book.py --help` - `--proxy` 方便中国大陆的用户在本地测试时使用代理，传入类似 `http://127.0.0.1:7890` 的字符串 - `--resume` 手动中断后，加入命令可以从之前中断的位置继续执行。 ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model google --resume ``` - `--translate-tags` 指定需要翻译的标签，使用逗号分隔多个标签。epub 由 html 文件组成，默认情况下，只翻译 `

` 中的内容。例如: `--translate-tags h1,h2,h3,p,div` - `--book_from` 选项指定电子阅读器类型（现在只有 kobo 可用），并使用 `--device_path` 指定挂载点。 - `--api_base ${url}` 如果你遇到了墙需要用 Cloudflare Workers 替换 api_base 请使用 `--api_base ${url}` 来替换。 **请注意，此处你输入的 api 应该是'`https://xxxx/v1`'的字样，域名需要用引号包裹** - `--allow_navigable_strings` 如果你想要翻译电子书中的无标签字符串，可以使用 `--allow_navigable_strings` 参数，会将可遍历字符串加入翻译队列，**注意，在条件允许情况下，请寻找更规范的电子书** - `--prompt` 如果你想调整 prompt，你可以使用 `--prompt` 参数。有效的占位符包括 `{text}` 和 `{language}`。你可以用以下方式配置 prompt: - 如果您不需要设置 `system` 角色，可以这样：`--prompt "Translate {text} to {language}"` 或者 `--prompt prompt_template_sample.txt`（示例文本文件可以在 [./prompt_template_sample.txt](./prompt_template_sample.txt) 找到）。 - 如果您需要设置 `system` 角色，可以使用以下方式配置：`--prompt '{"user":"Translate {text} to {language}", "system": "You are a professional translator."}'`，或者 `--prompt prompt_template_sample.json`（示例 JSON 文件可以在 [./prompt_template_sample.json](./prompt_template_sample.json) 找到）。 - 你也可以用环境以下环境变量来配置 `system` 和 `user` 角色 prompt：`BBM_CHATGPTAPI_USER_MSG_TEMPLATE` 和 `BBM_CHATGPTAPI_SYS_MSG`。 该参数可以是提示模板字符串，也可以是模板 `.txt` 文件的路径。 - `--batch_size` 指定批量翻译的行数(默认行数为 10，目前只对 txt 生效) - `--accumulated_num`: 达到累计token数开始进行翻译。gpt3.5将total_token限制为4090。 例如，如果您使用`--accumulation_num 1600`，则可能会输出2200个令牌，另外200个令牌用于系统指令（system_message）和用户指令（user_message），1600+2200+200 = 4000，所以token接近极限。你必须选择一个自己合适的值，我们无法在发送之前判断是否达到限制 - `--use_context`: prompts the model to create a three-paragraph summary. If it's the beginning of the translation, it will summarize the entire passage sent (the size depending on `--accumulated_num`). For subsequent passages, it will amend the summary to include details from the most recent passage, creating a running one-paragraph context payload of the important details of the entire translated work. This improves consistency of flow and tone throughout the translation. This option is available for all ChatGPT-compatible models and Gemini models. 模型提示词将创建三段摘要。如果是翻译的开始，它将总结发送的整个段落（大小取决于`--accumulated_num`）。 对于后续的段落，它将修改摘要，以包括最近段落的细节，创建一个完整的段落上下文负载，包含整个翻译作品的重要细节。 这提高了整个翻译过程中的流畅性和语气的一致性。 此选项适用于所有ChatGPT兼容型号和Gemini型号。 - `--context_paragraph_limit`: 使用`--use_context`选项时，使用`--context_paragraph_limit`设置上下文段落数限制。 - `--temperature`: 使用 `--temperature` 设置 `chatgptapi`/`gpt4`/`claude`模型的temperature值. 如 `--temperature 0.7`. - `--block_size`: 使用`--block_size`将多个段落合并到一个块中。这可能会提高准确性并加快处理速度，但可能会干扰原始格式。必须与`--single_translate`一起使用。 例如：`--block_size 5 --single_translate`。 - `--single_translate`: 使用`--single_translate`只输出翻译后的图书，不创建双语版本。 - `--translation_style`: 如: `--translation_style "color: #808080; font-style: italic;"` - `--retranslate "$translated_filepath" "file_name_in_epub" "start_str" "end_str"(optional)`: - 重新翻译，从 start_str 到 end_str 的标记: ```shell python3 "make_book.py" --book_name "test_books/animal_farm.epub" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' 'This kind of thing is not a good symptom. Obviously' ``` - 重新翻译, 从start_str 的标记开始: ```shell python3 "make_book.py" --book_name "test_books/animal_farm.epub" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' ``` ### 示范用例 **如果使用 `pip install bbook_maker` 以下命令都可以改成 `bbook args`** ```shell # 如果你想快速测一下 python3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test # 或翻译完整本书 python3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --language zh-hans # Or translate the whole book using Gemini python3 make_book.py --book_name test_books/animal_farm.epub --gemini_key ${gemini_key} --model gemini # 指定环境变量来略过 --openai_key export OPENAI_API_KEY=${your_api_key} # Use the DeepL model with Japanese python3 make_book.py --book_name test_books/animal_farm.epub --model deepl --deepl_key ${deepl_key} --language ja # Use the Claude model with Japanese python3 make_book.py --book_name test_books/animal_farm.epub --model claude --claude_key ${claude_key} --language ja # Use the CustomAPI model with Japanese python3 make_book.py --book_name test_books/animal_farm.epub --model customapi --custom_api ${custom_api} --language ja # Translate contents in

and

python3 make_book.py --book_name test_books/animal_farm.epub --translate-tags div,p # 修改prompt python3 make_book.py --book_name test_books/animal_farm.epub --prompt prompt_template_sample.txt # 或者 python3 make_book.py --book_name test_books/animal_farm.epub --prompt "Please translate \`{text}\` to {language}" # 翻译 kobo e-reader 中，來自 Rakuten Kobo 的书籍 python3 make_book.py --book_from kobo --device_path /tmp/kobo # 翻译 txt 文件 python3 make_book.py --book_name test_books/the_little_prince.txt --test # 聚合多行翻译 txt 文件 python3 make_book.py --book_name test_books/the_little_prince.txt --test --batch_size 20 # 使用彩云小译翻译(彩云api目前只支持: 简体中文 <-> 英文， 简体中文 <-> 日语) # 彩云提供了测试token（3975l6lr5pcbvidl6jl2） # 你可以参考这个教程申请自己的token (https://bobtranslate.com/service/translate/caiyun.html) python3 make_book.py --model caiyun --caiyun_key 3975l6lr5pcbvidl6jl2 --book_name test_books/animal_farm.epub # 可以在环境变量中设置BBM_CAIYUN_API_KEY，略过--openai_key export BBM_CAIYUN_API_KEY=${your_api_key} ``` 更加小白的示例 ```shell python3 make_book.py --book_name 'animal_farm.epub' --openai_key sk-XXXXX --api_base 'https://xxxxx/v1' # 有可能你不需要 python3 而是python python make_book.py --book_name 'animal_farm.epub' --openai_key sk-XXXXX --api_base 'https://xxxxx/v1' ``` [演示视频](https://www.bilibili.com/video/BV1XX4y1d75D/?t=0h07m08s) [演示视频 2](https://www.bilibili.com/video/BV1T8411c7iU/) 使用 Azure OpenAI service ```shell python3 make_book.py --book_name 'animal_farm.epub' --openai_key XXXXX --api_base 'https://example-endpoint.openai.azure.com' --deployment_id 'deployment-name' # Or python3 is not in your PATH python make_book.py --book_name 'animal_farm.epub' --openai_key XXXXX --api_base 'https://example-endpoint.openai.azure.com' --deployment_id 'deployment-name' ``` ## 注意 1. Free trail 的 API token 有所限制，如果想要更快的速度，可以考虑付费方案 2. 欢迎提交 PR # 感谢 - @[yetone](https://github.com/yetone) # 贡献 - 任何 issue PR 都欢迎 - Issue 中有些 TODO 没做的都可以选 - 提交代码前请先执行 `black make_book.py` [^black] # 其它推荐项目 - 书译 BookTranslator -> [Book Translator](https://www.booktranslator.app) ## 赞赏 谢谢就够了 ![image](https://user-images.githubusercontent.com/15976103/222407199-1ed8930c-13a8-402b-9993-aaac8ee84744.png) [^token]: https://platform.openai.com/account/api-keys [^black]: https://github.com/psf/black ================================================ FILE: README.md ================================================ **[中文](./README-CN.md) | English** [![litellm](https://img.shields.io/badge/%20%F0%9F%9A%85%20liteLLM-OpenAI%7CAzure%7CAnthropic%7CPalm%7CCohere%7CReplicate%7CHugging%20Face-blue?color=green)](https://github.com/BerriAI/litellm) # bilingual_book_maker The bilingual_book_maker is an AI translation tool that uses ChatGPT to assist users in creating multi-language versions of epub/txt/srt/pdf files and books. This tool is exclusively designed for translating epub and other public domain works and is not intended for copyrighted works. Before using this tool, please review the project's **[disclaimer](./disclaimer.md)**. ![image](https://user-images.githubusercontent.com/15976103/222317531-a05317c5-4eee-49de-95cd-04063d9539d9.png) ## Supported Models gpt-5-mini, gpt-4, gpt-3.5-turbo, claude-2, palm, llama-2, azure-openai, command-nightly, gemini, qwen-mt-turbo, qwen-mt-plus For using Non-OpenAI models, use class `liteLLM()` - liteLLM supports all models above. Find more info here for using liteLLM: https://github.com/BerriAI/litellm/blob/main/setup.py ## Preparation 1. ChatGPT or OpenAI token [^token] 2. epub/txt/pdf books 3. Environment with internet access or proxy 4. Python 3.8+ ## Quick Start A sample book, `test_books/animal_farm.epub`, is provided for testing purposes. ```shell pip install -r requirements.txt python3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test OR pip install -U bbook_maker bbook --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test ``` ## Translate Service - Use `--openai_key` option to specify OpenAI API key. If you have multiple keys, separate them by commas (xxx,xxx,xxx) to reduce errors caused by API call limits. Or, just set environment variable `BBM_OPENAI_API_KEY` instead. - A sample book, `test_books/animal_farm.epub`, is provided for testing purposes. - The default underlying model is [GPT-3.5-turbo](https://openai.com/blog/introducing-chatgpt-and-whisper-apis), which is used by ChatGPT currently. Use `--model gpt4` to change the underlying model to `GPT4`. You can also use `GPT4omini`. - Important to note that `gpt-4` is significantly more expensive than `gpt-4-turbo`, but to avoid bumping into rate limits, we automatically balance queries across `gpt-4-1106-preview`, `gpt-4`, `gpt-4-32k`, `gpt-4-0613`,`gpt-4-32k-0613`. - If you want to use a specific model alias with OpenAI (eg `gpt-4-1106-preview` or `gpt-3.5-turbo-0125`), you can use `--model openai --model_list gpt-4-1106-preview,gpt-3.5-turbo-0125`. `--model_list` takes a comma-separated list of model aliases. - If using chatgptapi, you can add `--use_context` to add a context paragraph to each passage sent to the model for translation (see below). * DeepL Support DeepL model [DeepL Translator](https://rapidapi.com/splintPRO/api/dpl-translator) need pay to get the token ``` python3 make_book.py --book_name test_books/animal_farm.epub --model deepl --deepl_key ${deepl_key} ``` * DeepL free ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model deeplfree ``` * [Claude](https://console.anthropic.com/docs) Use [Claude](https://console.anthropic.com/docs) model to translate ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model claude --claude_key ${claude_key} ``` * Google Translate ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model google ``` * Caiyun Translate ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model caiyun --caiyun_key ${caiyun_key} ``` * Gemini Support Google [Gemini](https://aistudio.google.com/app/apikey) model, use `--model gemini` for Gemini Flash or `--model geminipro` for Gemini Pro. If you want to use a specific model alias with Gemini (eg `gemini-1.5-flash-002` or `gemini-1.5-flash-8b-exp-0924`), you can use `--model gemini --model_list gemini-1.5-flash-002,gemini-1.5-flash-8b-exp-0924`. `--model_list` takes a comma-separated list of model aliases. ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model gemini --gemini_key ${gemini_key} ``` * Qwen Support Alibaba Cloud [Qwen-MT](https://bailian.console.aliyun.com/) specialized translation model. Supports 92 languages with features like terminology intervention and translation memory. Use `--model qwen-mt-turbo` for faster/cheaper translation, or `--model qwen-mt-plus` for higher quality. Use `source_lang` to specify the source language explicitly, or leave it empty for auto-detection. ```shell python3 make_book.py --book_name test_books/animal_farm.epub --qwen_key ${qwen_key} --model qwen-mt-turbo --language "Simplified Chinese" python3 make_book.py --book_name test_books/animal_farm.epub --qwen_key ${qwen_key} --model qwen-mt-plus --language "Japanese" --source_lang "English" ``` * [Tencent TranSmart](https://transmart.qq.com) ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model tencentransmart ``` * [xAI](https://x.ai) ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model xai --xai_key ${xai_key} ``` * [Ollama](https://github.com/ollama/ollama) Support [Ollama](https://github.com/ollama/ollama) self-host models, If ollama server is not running on localhost, use `--api_base http://x.x.x.x:port/v1` to point to the ollama server address ```shell python3 make_book.py --book_name test_books/animal_farm.epub --ollama_model ${ollama_model_name} ``` * [groq](https://console.groq.com/keys) GroqCloud currently supports models: you can find from [Supported Models](https://console.groq.com/docs/models) ```shell python3 make_book.py --book_name test_books/animal_farm.epub --groq_key [your_key] --model groq --model_list llama3-8b-8192 ``` ## Use - Once the translation is complete, a bilingual book named `${book_name}_bilingual.epub` would be generated for EPUB inputs; for TXT/MD/SRT inputs a bilingual text (or subtitle) file named `${book_name}_bilingual.txt` (or `_bilingual.srt`) will be generated. For **PDF inputs** the tool will produce a bilingual `.txt` fallback and will also attempt to create `${book_name}_bilingual.epub` — if EPUB creation fails, the TXT fallback remains so you do not need to retranslate. - If there are any errors or you wish to interrupt the translation by pressing `CTRL+C`, a temporary bilingual file (for example `{book_name}_bilingual_temp.epub` or `{book_name}_bilingual_temp.txt`) would be generated. You can simply rename it to any desired name. ## Params - `--test`: Use `--test` option to preview the result if you haven't paid for the service. Note that there is a limit and it may take some time. - `--language`: Set the target language like `--language "Simplified Chinese"`. Default target language is `"Simplified Chinese"`. Read available languages by helper message: `python make_book.py --help` - `--proxy`: Use `--proxy` option to specify proxy server for internet access. Enter a string such as `http://127.0.0.1:7890`. - `--resume`: Use `--resume` option to manually resume the process after an interruption. ```shell python3 make_book.py --book_name test_books/animal_farm.epub --model google --resume ``` - `--translate-tags`: epub is made of html files. By default, we only translate contents in `

`. Use `--translate-tags` to specify tags need for translation. Use comma to separate multiple tags. For example: `--translate-tags h1,h2,h3,p,div` - `--book_from`: Use `--book_from` option to specify e-reader type (Now only `kobo` is available), and use `--device_path` to specify the mounting point. - `--api_base`: If you want to change api_base like using Cloudflare Workers, use `--api_base ` to support it. **Note: the api url should be '`https://xxxx/v1`'. Quotation marks are required.** - `--allow_navigable_strings`: If you want to translate strings in an e-book that aren't labeled with any tags, you can use the `--allow_navigable_strings` parameter. This will add the strings to the translation queue. **Note that it's best to look for e-books that are more standardized if possible.** - `--prompt`: To tweak the prompt, use the `--prompt` parameter. Valid placeholders for the `user` role template include `{text}` and `{language}`. It supports a few ways to configure the prompt: - If you don't need to set the `system` role content, you can simply set it up like this: `--prompt "Translate {text} to {language}."` or `--prompt prompt_template_sample.txt` (example of a text file can be found at [./prompt_template_sample.txt](./prompt_template_sample.txt)). - If you need to set the `system` role content, you can use the following format: `--prompt '{"user":"Translate {text} to {language}", "system": "You are a professional translator."}'` or `--prompt prompt_template_sample.json` (example of a JSON file can be found at [./prompt_template_sample.json](./prompt_template_sample.json)). - You can now use [PromptDown](https://github.com/btfranklin/promptdown) format (`.md` files) for more structured prompts: `--prompt prompt_md.prompt.md`. PromptDown supports both traditional system messages and developer messages (used by newer AI models). Example: ```markdown # Translation Prompt ## Developer Message You are a professional translator who specializes in accurate translations. ## Conversation | Role | Content | | ---- | -------------------------------------------------------------- | | User | Please translate the following text into {language}:\n\n{text} | ``` - You can also set the `user` and `system` role prompt by setting environment variables: `BBM_CHATGPTAPI_USER_MSG_TEMPLATE` and `BBM_CHATGPTAPI_SYS_MSG`. - `--batch_size`: Use the `--batch_size` parameter to specify the number of lines for batch translation (default is 10, currently only effective for txt files). - `--accumulated_num`: Wait for how many tokens have been accumulated before starting the translation. gpt3.5 limits the total_token to 4090. For example, if you use `--accumulated_num 1600`, maybe openai will output 2200 tokens and maybe 200 tokens for other messages in the system messages user messages, 1600+2200+200=4000, So you are close to reaching the limit. You have to choose your own value, there is no way to know if the limit is reached before sending - `--use_context`: prompts the model to create a three-paragraph summary. If it's the beginning of the translation, it will summarize the entire passage sent (the size depending on `--accumulated_num`). For subsequent passages, it will amend the summary to include details from the most recent passage, creating a running one-paragraph context payload of the important details of the entire translated work. This improves consistency of flow and tone throughout the translation. This option is available for all ChatGPT-compatible models and Gemini models. - `--context_paragraph_limit`: Use `--context_paragraph_limit` to set a limit on the number of context paragraphs when using the `--use_context` option. - `--parallel-workers`: Use `--parallel-workers` to enable parallel EPUB chapter processing. Values greater than `1` spin up multiple workers (recommended: `2-4`) and automatically fall back to sequential mode for single-chapter books. - `--temperature`: Use `--temperature` to set the temperature parameter for `chatgptapi`/`gpt4`/`claude` models. For example: `--temperature 0.7`. - `--block_size`: Use `--block_size` to merge multiple paragraphs into one block. This may increase accuracy and speed up the process but can disturb the original format. Must be used with `--single_translate`. For example: `--block_size 5 --single_translate`. - `--single_translate`: Use `--single_translate` to output only the translated book without creating a bilingual version. - `--translation_style`: example: `--translation_style "color: #808080; font-style: italic;"` - `--retranslate "$translated_filepath" "file_name_in_epub" "start_str" "end_str"(optional)`: Retranslate from start_str to end_str's tag: ```shell python3 "make_book.py" --book_name "test_books/animal_farm.epub" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' 'This kind of thing is not a good symptom. Obviously' ``` Retranslate start_str's tag: ```shell python3 "make_book.py" --book_name "test_books/animal_farm.epub" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' ``` ### Examples **Note if use `pip install bbook_maker` all commands can change to `bbook_maker args`** ```shell # Test quickly python3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test --language zh-hans # Test quickly for src python3 make_book.py --book_name test_books/Lex_Fridman_episode_322.srt --openai_key ${openai_key} --test # Or translate the whole book python3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --language zh-hans # Or translate the whole book using Gemini flash python3 make_book.py --book_name test_books/animal_farm.epub --gemini_key ${gemini_key} --model gemini # Translate an EPUB with parallel chapter processing python3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --parallel-workers 4 # Use a specific list of Gemini model aliases python3 make_book.py --book_name test_books/animal_farm.epub --gemini_key ${gemini_key} --model gemini --model_list gemini-1.5-flash-002,gemini-1.5-flash-8b-exp-0924 # Set env OPENAI_API_KEY to ignore option --openai_key export OPENAI_API_KEY=${your_api_key} # Use the GPT-4 model with context to Japanese python3 make_book.py --book_name test_books/animal_farm.epub --model gpt4 --use_context --language ja # Use a specific OpenAI model alias python3 make_book.py --book_name test_books/animal_farm.epub --model openai --model_list gpt-4-1106-preview --openai_key ${openai_key} **Note** you can use other `openai like` model in this way python3 make_book.py --book_name test_books/animal_farm.epub --model openai --model_list yi-34b-chat-0205 --openai_key ${openai_key} --api_base "https://api.lingyiwanwu.com/v1" # Use a specific list of OpenAI model aliases python3 make_book.py --book_name test_books/animal_farm.epub --model openai --model_list gpt-4-1106-preview,gpt-4-0125-preview,gpt-3.5-turbo-0125 --openai_key ${openai_key} # Use the DeepL model with Japanese python3 make_book.py --book_name test_books/animal_farm.epub --model deepl --deepl_key ${deepl_key} --language ja # Use the Claude model with Japanese python3 make_book.py --book_name test_books/animal_farm.epub --model claude --claude_key ${claude_key} --language ja # Use the CustomAPI model with Japanese python3 make_book.py --book_name test_books/animal_farm.epub --model customapi --custom_api ${custom_api} --language ja # Translate contents in

and

python3 make_book.py --book_name test_books/animal_farm.epub --translate-tags div,p # Tweaking the prompt python3 make_book.py --book_name test_books/animal_farm.epub --prompt prompt_template_sample.txt # or python3 make_book.py --book_name test_books/animal_farm.epub --prompt prompt_template_sample.json # or python3 make_book.py --book_name test_books/animal_farm.epub --prompt "Please translate \`{text}\` to {language}" # Translate books download from Rakuten Kobo on kobo e-reader python3 make_book.py --book_from kobo --device_path /tmp/kobo # translate txt file python3 make_book.py --book_name test_books/the_little_prince.txt --test --language zh-hans # aggregated translation txt file python3 make_book.py --book_name test_books/the_little_prince.txt --test --batch_size 20 # Using Caiyun model to translate # (the api currently only support: simplified chinese <-> english, simplified chinese <-> japanese) # the official Caiyun has provided a test token (3975l6lr5pcbvidl6jl2) # you can apply your own token by following this tutorial(https://bobtranslate.com/service/translate/caiyun.html) python3 make_book.py --model caiyun --caiyun_key 3975l6lr5pcbvidl6jl2 --book_name test_books/animal_farm.epub # Set env BBM_CAIYUN_API_KEY to ignore option --openai_key export BBM_CAIYUN_API_KEY=${your_api_key} ``` More understandable example ```shell python3 make_book.py --book_name 'animal_farm.epub' --openai_key sk-XXXXX --api_base 'https://xxxxx/v1' # Or python3 is not in your PATH python make_book.py --book_name 'animal_farm.epub' --openai_key sk-XXXXX --api_base 'https://xxxxx/v1' ``` Microsoft Azure Endpoints ```shell python3 make_book.py --book_name 'animal_farm.epub' --openai_key XXXXX --api_base 'https://example-endpoint.openai.azure.com' --deployment_id 'deployment-name' # Or python3 is not in your PATH python make_book.py --book_name 'animal_farm.epub' --openai_key XXXXX --api_base 'https://example-endpoint.openai.azure.com' --deployment_id 'deployment-name' ``` ## Docker You can use [Docker](https://www.docker.com/) if you don't want to deal with setting up the environment. ```shell # Build image docker build --tag bilingual_book_maker . # Run container # "$folder_path" represents the folder where your book file locates. Also, it is where the processed file will be stored. # Windows PowerShell $folder_path=your_folder_path # $folder_path="C:\Users\user\mybook\" $book_name=your_book_name # $book_name="animal_farm.epub" $openai_key=your_api_key # $openai_key="sk-xxx" $language=your_language # see utils.py docker run --rm --name bilingual_book_maker --mount type=bind,source=$folder_path,target='/app/test_books' bilingual_book_maker --book_name "/app/test_books/$book_name" --openai_key $openai_key --language $language # Linux export folder_path=${your_folder_path} export book_name=${your_book_name} export openai_key=${your_api_key} export language=${your_language} docker run --rm --name bilingual_book_maker --mount type=bind,source=${folder_path},target='/app/test_books' bilingual_book_maker --book_name "/app/test_books/${book_name}" --openai_key ${openai_key} --language "${language}" ``` For example: ```shell # Linux docker run --rm --name bilingual_book_maker --mount type=bind,source=/home/user/my_books,target='/app/test_books' bilingual_book_maker --book_name /app/test_books/animal_farm.epub --openai_key sk-XXX --test --test_num 1 --language zh-hant ``` ## Notes 1. API token from free trial has limit. If you want to speed up the process, consider paying for the service or use multiple OpenAI tokens 2. PR is welcome # Thanks - @[yetone](https://github.com/yetone) # Contribution - Any issues or PRs are welcome. - TODOs in the issue can also be selected. - Please run `black make_book.py`[^black] before submitting the code. # Others better - 书译 BookTranslator -> [Book Translator](https://www.booktranslator.app) ## Appreciation Thank you, that's enough. ![image](https://user-images.githubusercontent.com/15976103/222407199-1ed8930c-13a8-402b-9993-aaac8ee84744.png) [^token]: https://platform.openai.com/account/api-keys [^black]: https://github.com/psf/black ================================================ FILE: book_maker/__init__.py ================================================ ================================================ FILE: book_maker/__main__.py ================================================ from cli import main if __name__ == "__main__": main() ================================================ FILE: book_maker/cli.py ================================================ import argparse import json import os from os import environ as env from book_maker.loader import BOOK_LOADER_DICT from book_maker.translator import MODEL_DICT from book_maker.utils import LANGUAGES, TO_LANGUAGE_CODE def parse_prompt_arg(prompt_arg): prompt = None if prompt_arg is None: return prompt # Check if it's a path to a markdown file (PromptDown format) if prompt_arg.endswith(".md") and os.path.exists(prompt_arg): try: from promptdown import StructuredPrompt structured_prompt = StructuredPrompt.from_promptdown_file(prompt_arg) # Initialize our prompt structure prompt = {} # Handle developer_message or system_message # Developer message takes precedence if both are present if ( hasattr(structured_prompt, "developer_message") and structured_prompt.developer_message ): prompt["system"] = structured_prompt.developer_message elif ( hasattr(structured_prompt, "system_message") and structured_prompt.system_message ): prompt["system"] = structured_prompt.system_message # Extract user message from conversation if ( hasattr(structured_prompt, "conversation") and structured_prompt.conversation ): for message in structured_prompt.conversation: if message.role.lower() == "user": prompt["user"] = message.content break # Ensure we found a user message if "user" not in prompt or not prompt["user"]: raise ValueError( "PromptDown file must contain at least one user message" ) print(f"Successfully loaded PromptDown file: {prompt_arg}") # Validate required placeholders if any(c not in prompt["user"] for c in ["{text}"]): raise ValueError( "User message in PromptDown must contain `{text}` placeholder" ) return prompt except Exception as e: print(f"Error parsing PromptDown file: {e}") # Fall through to other parsing methods # Existing parsing logic for JSON strings and other formats if not any(prompt_arg.endswith(ext) for ext in [".json", ".txt", ".md"]): try: # user can define prompt by passing a json string # eg: --prompt '{"system": "You are a professional translator who translates computer technology books", "user": "Translate \`{text}\` to {language}"}' prompt = json.loads(prompt_arg) except json.JSONDecodeError: # if not a json string, treat it as a template string prompt = {"user": prompt_arg} elif os.path.exists(prompt_arg): if prompt_arg.endswith(".txt"): # if it's a txt file, treat it as a template string with open(prompt_arg, encoding="utf-8") as f: prompt = {"user": f.read()} elif prompt_arg.endswith(".json"): # if it's a json file, treat it as a json object # eg: --prompt prompt_template_sample.json with open(prompt_arg, encoding="utf-8") as f: prompt = json.load(f) else: raise FileNotFoundError(f"{prompt_arg} not found") # if prompt is None or any(c not in prompt["user"] for c in ["{text}", "{language}"]): if prompt is None or any(c not in prompt["user"] for c in ["{text}"]): raise ValueError("prompt must contain `{text}`") if "user" not in prompt: raise ValueError("prompt must contain the key of `user`") if (prompt.keys() - {"user", "system"}) != set(): raise ValueError("prompt can only contain the keys of `user` and `system`") print("prompt config:", prompt) return prompt def main(): translate_model_list = list(MODEL_DICT.keys()) parser = argparse.ArgumentParser() parser.add_argument( "--book_name", dest="book_name", type=str, help="path of the epub file to be translated", ) parser.add_argument( "--book_from", dest="book_from", type=str, choices=["kobo"], # support kindle later metavar="E-READER", help="e-reader type, available: {%(choices)s}", ) parser.add_argument( "--device_path", dest="device_path", type=str, help="Path of e-reader device", ) ########## KEYS ########## parser.add_argument( "--openai_key", dest="openai_key", type=str, default="", help="OpenAI api key,if you have more than one key, please use comma" " to split them to go beyond the rate limits", ) parser.add_argument( "--caiyun_key", dest="caiyun_key", type=str, help="you can apply caiyun key from here (https://dashboard.caiyunapp.com/user/sign_in/)", ) parser.add_argument( "--deepl_key", dest="deepl_key", type=str, help="you can apply deepl key from here (https://rapidapi.com/splintPRO/api/dpl-translator", ) parser.add_argument( "--claude_key", dest="claude_key", type=str, help="you can find claude key from here (https://console.anthropic.com/account/keys)", ) parser.add_argument( "--custom_api", dest="custom_api", type=str, help="you should build your own translation api", ) # for Google Gemini parser.add_argument( "--gemini_key", dest="gemini_key", type=str, help="You can get Gemini Key from https://makersuite.google.com/app/apikey", ) # for Groq parser.add_argument( "--groq_key", dest="groq_key", type=str, help="You can get Groq Key from https://console.groq.com/keys", ) # for xAI parser.add_argument( "--xai_key", dest="xai_key", type=str, help="You can get xAI Key from https://console.x.ai/", ) # for Qwen parser.add_argument( "--qwen_key", dest="qwen_key", type=str, help="You can get Qwen Key from https://bailian.console.aliyun.com/?tab=model#/api-key", ) parser.add_argument( "--test", dest="test", action="store_true", help="only the first 10 paragraphs will be translated, for testing", ) parser.add_argument( "--test_num", dest="test_num", type=int, default=10, help="how many paragraphs will be translated for testing", ) parser.add_argument( "-m", "--model", dest="model", type=str, default="chatgptapi", choices=translate_model_list, # support DeepL later metavar="MODEL", help="model to use, available: {%(choices)s}", ) parser.add_argument( "--ollama_model", dest="ollama_model", type=str, default="", metavar="MODEL", help="use ollama", ) parser.add_argument( "--language", type=str, choices=sorted(LANGUAGES.keys()) + sorted([k.title() for k in TO_LANGUAGE_CODE]), default="zh-hans", metavar="LANGUAGE", help="language to translate to, available: {%(choices)s}", ) parser.add_argument( "--resume", dest="resume", action="store_true", help="if program stop unexpected you can use this to resume", ) parser.add_argument( "-p", "--proxy", dest="proxy", type=str, default="", help="use proxy like http://127.0.0.1:7890", ) parser.add_argument( "--deployment_id", dest="deployment_id", type=str, help="the deployment name you chose when you deployed the model", ) # args to change api_base parser.add_argument( "--api_base", metavar="API_BASE_URL", dest="api_base", type=str, help="specify base url other than the OpenAI's official API address", ) parser.add_argument( "--exclude_filelist", dest="exclude_filelist", type=str, default="", help="if you have more than one file to exclude, please use comma to split them, example: --exclude_filelist 'nav.xhtml,cover.xhtml'", ) parser.add_argument( "--only_filelist", dest="only_filelist", type=str, default="", help="if you only have a few files with translations, please use comma to split them, example: --only_filelist 'nav.xhtml,cover.xhtml'", ) parser.add_argument( "--translate-tags", dest="translate_tags", type=str, default="p", help="example --translate-tags p,blockquote", ) parser.add_argument( "--exclude_translate-tags", dest="exclude_translate_tags", type=str, default="sup", help="example --exclude_translate-tags table,sup", ) parser.add_argument( "--allow_navigable_strings", dest="allow_navigable_strings", action="store_true", default=False, help="allow NavigableStrings to be translated", ) parser.add_argument( "--prompt", dest="prompt_arg", type=str, metavar="PROMPT_ARG", help="used for customizing the prompt. It can be the prompt template string, or a path to the template file. The valid placeholders are `{text}` and `{language}`.", ) parser.add_argument( "--accumulated_num", dest="accumulated_num", type=int, default=1, help="""Wait for how many tokens have been accumulated before starting the translation. gpt3.5 limits the total_token to 4090. For example, if you use --accumulated_num 1600, maybe openai will output 2200 tokens and maybe 200 tokens for other messages in the system messages user messages, 1600+2200+200=4000, So you are close to reaching the limit. You have to choose your own value, there is no way to know if the limit is reached before sending """, ) parser.add_argument( "--translation_style", dest="translation_style", type=str, help="""ex: --translation_style "color: #808080; font-style: italic;" """, ) parser.add_argument( "--batch_size", dest="batch_size", type=int, help="how many lines will be translated by aggregated translation(This options currently only applies to txt files)", ) parser.add_argument( "--retranslate", dest="retranslate", nargs=4, type=str, help="""--retranslate "$translated_filepath" "file_name_in_epub" "start_str" "end_str"(optional) Retranslate from start_str to end_str's tag: python3 "make_book.py" --book_name "test_books/animal_farm.epub" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' 'This kind of thing is not a good symptom. Obviously' Retranslate start_str's tag: python3 "make_book.py" --book_name "test_books/animal_farm.epub" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' """, ) parser.add_argument( "--single_translate", action="store_true", help="output translated book, no bilingual", ) parser.add_argument( "--use_context", dest="context_flag", action="store_true", help="adds an additional paragraph for global, updating historical context of the story to the model's input, improving the narrative consistency for the AI model (this uses ~200 more tokens each time)", ) parser.add_argument( "--context_paragraph_limit", dest="context_paragraph_limit", type=int, default=0, help="if use --use_context, set context paragraph limit", ) parser.add_argument( "--temperature", type=float, default=1.0, help="temperature parameter for `chatgptapi`/`gpt4`/`gpt4omini`/`gpt4o`/`gpt5mini`/`claude`/`gemini`", ) parser.add_argument( "--source_lang", type=str, default="auto", help="source language for translation models like `qwen` (default: auto-detect)", ) parser.add_argument( "--block_size", type=int, default=-1, help="merge multiple paragraphs into one block, may increase accuracy and speed up the process, but disturb the original format, must be used with `--single_translate`", ) parser.add_argument( "--model_list", type=str, dest="model_list", help="Rather than using our preset lists of models, specify exactly the models you want as a comma separated list `gpt-4-32k,gpt-3.5-turbo-0125` (Currently only supports: `openai`)", ) parser.add_argument( "--batch", dest="batch_flag", action="store_true", help="Enable batch translation using ChatGPT's batch API for improved efficiency", ) parser.add_argument( "--batch-use", dest="batch_use_flag", action="store_true", help="Use pre-generated batch translations to create files. Run with --batch first before using this option", ) parser.add_argument( "--interval", type=float, default=0.01, help="Request interval in seconds (e.g., 0.1 for 100ms). Currently only supported for Gemini models. Default: 0.01", ) parser.add_argument( "--parallel-workers", dest="parallel_workers", type=int, default=1, help="Number of parallel workers for EPUB chapter processing. Use 2-4 for better performance. Default: 1", ) options = parser.parse_args() if not options.book_name: print("Error: please provide the path of your book using --book_name ") exit(1) if not os.path.isfile(options.book_name): print(f"Error: the book {options.book_name!r} does not exist.") exit(1) PROXY = options.proxy if PROXY != "": os.environ["http_proxy"] = PROXY os.environ["https_proxy"] = PROXY translate_model = MODEL_DICT.get(options.model) assert translate_model is not None, "unsupported model" API_KEY = "" if options.model in [ "openai", "chatgptapi", "gpt4", "gpt4omini", "gpt4o", "gpt5mini", "o1preview", "o1", "o1mini", "o3mini", ]: if OPENAI_API_KEY := ( options.openai_key or env.get( "OPENAI_API_KEY", ) # XXX: for backward compatibility, deprecate soon or env.get( "BBM_OPENAI_API_KEY", ) # suggest adding `BBM_` prefix for all the bilingual_book_maker ENVs. ): API_KEY = OPENAI_API_KEY # patch elif options.ollama_model: # any string is ok, can't be empty API_KEY = "ollama" else: raise Exception( "OpenAI API key not provided, please google how to obtain it", ) elif options.model == "caiyun": API_KEY = options.caiyun_key or env.get("BBM_CAIYUN_API_KEY") if not API_KEY: raise Exception("Please provide caiyun key") elif options.model == "deepl": API_KEY = options.deepl_key or env.get("BBM_DEEPL_API_KEY") if not API_KEY: raise Exception("Please provide deepl key") elif options.model.startswith("claude"): API_KEY = options.claude_key or env.get("BBM_CLAUDE_API_KEY") if not API_KEY: raise Exception("Please provide claude key") elif options.model == "customapi": API_KEY = options.custom_api or env.get("BBM_CUSTOM_API") if not API_KEY: raise Exception("Please provide custom translate api") elif options.model in ["gemini", "geminipro"]: API_KEY = options.gemini_key or env.get("BBM_GOOGLE_GEMINI_KEY") elif options.model == "groq": API_KEY = options.groq_key or env.get("BBM_GROQ_API_KEY") elif options.model == "xai": API_KEY = options.xai_key or env.get("BBM_XAI_API_KEY") elif options.model.startswith("qwen-"): API_KEY = options.qwen_key or env.get("BBM_QWEN_API_KEY") else: API_KEY = "" if options.book_from == "kobo": from book_maker import obok device_path = options.device_path if device_path is None: raise Exception( "Device path is not given, please specify the path by --device_path ", ) options.book_name = obok.cli_main(device_path) book_type = options.book_name.split(".")[-1] support_type_list = list(BOOK_LOADER_DICT.keys()) if book_type not in support_type_list: raise Exception( f"now only support files of these formats: {','.join(support_type_list)}", ) if options.block_size > 0 and not options.single_translate: raise Exception( "block_size must be used with `--single_translate` because it disturbs the original format", ) book_loader = BOOK_LOADER_DICT.get(book_type) assert book_loader is not None, "unsupported loader" language = options.language if options.language in LANGUAGES: # use the value for prompt language = LANGUAGES.get(language, language) # change api_base for issue #42 model_api_base = options.api_base if options.ollama_model and not model_api_base: # ollama default api_base model_api_base = "http://localhost:11434/v1" e = book_loader( options.book_name, translate_model, API_KEY, options.resume, language=language, model_api_base=model_api_base, is_test=options.test, test_num=options.test_num, prompt_config=parse_prompt_arg(options.prompt_arg), single_translate=options.single_translate, context_flag=options.context_flag, context_paragraph_limit=options.context_paragraph_limit, temperature=options.temperature, source_lang=options.source_lang, parallel_workers=options.parallel_workers, ) # other options if options.allow_navigable_strings: e.allow_navigable_strings = True if options.translate_tags: e.translate_tags = options.translate_tags if options.exclude_translate_tags: e.exclude_translate_tags = options.exclude_translate_tags if options.exclude_filelist: e.exclude_filelist = options.exclude_filelist if options.only_filelist: e.only_filelist = options.only_filelist if options.accumulated_num > 1: e.accumulated_num = options.accumulated_num if options.translation_style: e.translation_style = options.translation_style if options.batch_size: e.batch_size = options.batch_size if options.retranslate: e.retranslate = options.retranslate if options.deployment_id: # only work for ChatGPT api for now # later maybe support others assert options.model in [ "chatgptapi", "gpt4", "gpt4omini", "gpt4o", "gpt5mini", "o1", "o1preview", "o1mini", "o3mini", ], "only support chatgptapi for deployment_id" if not options.api_base: raise ValueError("`api_base` must be provided when using `deployment_id`") e.translate_model.set_deployment_id(options.deployment_id) if options.model in ("openai", "groq"): # Currently only supports `openai` when you also have --model_list set if options.model_list: e.translate_model.set_model_list(options.model_list.split(",")) else: raise ValueError( "When using `openai` model, you must also provide `--model_list`. For default model sets use `--model chatgptapi` or `--model gpt4` or `--model gpt4omini` or `--model gpt5mini`", ) # TODO refactor, quick fix for gpt4 model if options.model == "chatgptapi": if options.ollama_model: e.translate_model.set_gpt35_models(ollama_model=options.ollama_model) else: e.translate_model.set_gpt35_models() if options.model == "gpt4": e.translate_model.set_gpt4_models() if options.model == "gpt4omini": e.translate_model.set_gpt4omini_models() if options.model == "gpt4o": e.translate_model.set_gpt4o_models() if options.model == "gpt5mini": e.translate_model.set_gpt5mini_models() if options.model == "o1preview": e.translate_model.set_o1preview_models() if options.model == "o1": e.translate_model.set_o1_models() if options.model == "o1mini": e.translate_model.set_o1mini_models() if options.model == "o3mini": e.translate_model.set_o3mini_models() if options.model.startswith("claude-"): e.translate_model.set_claude_model(options.model) if options.model.startswith("qwen-"): e.translate_model.set_qwen_model(options.model) if options.block_size > 0: e.block_size = options.block_size if options.batch_flag: e.batch_flag = options.batch_flag if options.batch_use_flag: e.batch_use_flag = options.batch_use_flag if options.model in ("gemini", "geminipro"): e.translate_model.set_interval(options.interval) if options.model == "gemini": if options.model_list: e.translate_model.set_model_list(options.model_list.split(",")) else: e.translate_model.set_geminiflash_models() if options.model == "geminipro": e.translate_model.set_geminipro_models() e.make_bilingual_book() if __name__ == "__main__": main() ================================================ FILE: book_maker/config.py ================================================ config = { "translator": { "chatgptapi": { "context_paragraph_limit": 3, "batch_context_update_interval": 50, } }, } ================================================ FILE: book_maker/loader/__init__.py ================================================ from book_maker.loader.epub_loader import EPUBBookLoader from book_maker.loader.txt_loader import TXTBookLoader from book_maker.loader.srt_loader import SRTBookLoader from book_maker.loader.md_loader import MarkdownBookLoader from book_maker.loader.pdf_loader import PDFBookLoader BOOK_LOADER_DICT = { "epub": EPUBBookLoader, "txt": TXTBookLoader, "srt": SRTBookLoader, "md": MarkdownBookLoader, "pdf": PDFBookLoader, # TODO add more here } ================================================ FILE: book_maker/loader/base_loader.py ================================================ from abc import ABC, abstractmethod class BaseBookLoader(ABC): @staticmethod def _is_special_text(text): return text.isdigit() or text.isspace() @abstractmethod def _make_new_book(self, book): pass @abstractmethod def make_bilingual_book(self): pass @abstractmethod def load_state(self): pass @abstractmethod def _save_temp_book(self): pass @abstractmethod def _save_progress(self): pass ================================================ FILE: book_maker/loader/epub_loader.py ================================================ import os import pickle import string import sys import time from concurrent.futures import ThreadPoolExecutor, as_completed from copy import copy from pathlib import Path import traceback from threading import Lock from bs4 import BeautifulSoup as bs from bs4 import Tag from bs4.element import NavigableString from ebooklib import ITEM_DOCUMENT, epub from rich import print from tqdm import tqdm from book_maker.utils import num_tokens_from_text, prompt_config_to_kwargs from .base_loader import BaseBookLoader from .helper import EPUBBookLoaderHelper, is_text_link, not_trans class EPUBBookLoader(BaseBookLoader): def __init__( self, epub_name, model, key, resume, language, model_api_base=None, is_test=False, test_num=5, prompt_config=None, single_translate=False, context_flag=False, context_paragraph_limit=0, temperature=1.0, source_lang="auto", parallel_workers=1, ): self.epub_name = epub_name self.new_epub = epub.EpubBook() self.translate_model = model( key, language, api_base=model_api_base, context_flag=context_flag, context_paragraph_limit=context_paragraph_limit, temperature=temperature, source_lang=source_lang, **prompt_config_to_kwargs(prompt_config), ) self.is_test = is_test self.test_num = test_num self.translate_tags = "p" self.exclude_translate_tags = "sup" self.allow_navigable_strings = False self.accumulated_num = 1 self.translation_style = "" self.context_flag = context_flag self.helper = EPUBBookLoaderHelper( self.translate_model, self.accumulated_num, self.translation_style, self.context_flag, ) self.retranslate = None self.exclude_filelist = "" self.only_filelist = "" self.single_translate = single_translate self.block_size = -1 self.batch_use_flag = False self.batch_flag = False self.parallel_workers = 1 self.enable_parallel = False self._progress_lock = Lock() self._translation_index = 0 self.set_parallel_workers(parallel_workers) # monkey patch for # 173 def _write_items_patch(obj): for item in obj.book.get_items(): if isinstance(item, epub.EpubNcx): obj.out.writestr( "%s/%s" % (obj.book.FOLDER_NAME, item.file_name), obj._get_ncx() ) elif isinstance(item, epub.EpubNav): obj.out.writestr( "%s/%s" % (obj.book.FOLDER_NAME, item.file_name), obj._get_nav(item), ) elif item.manifest: obj.out.writestr( "%s/%s" % (obj.book.FOLDER_NAME, item.file_name), item.content ) else: obj.out.writestr("%s" % item.file_name, item.content) def _check_deprecated(obj): pass epub.EpubWriter._write_items = _write_items_patch epub.EpubReader._check_deprecated = _check_deprecated try: self.origin_book = epub.read_epub(self.epub_name) except Exception: # tricky monkey patch for #71 if you don't know why please check the issue and ignore this # when upstream change will TODO fix this def _load_spine(obj): spine = obj.container.find("{%s}%s" % (epub.NAMESPACES["OPF"], "spine")) obj.book.spine = [ (t.get("idref"), t.get("linear", "yes")) for t in spine ] obj.book.set_direction(spine.get("page-progression-direction", None)) epub.EpubReader._load_spine = _load_spine self.origin_book = epub.read_epub(self.epub_name) self.p_to_save = [] self.resume = resume self.bin_path = f"{Path(epub_name).parent}/.{Path(epub_name).stem}.temp.bin" if self.resume: self.load_state() @staticmethod def _is_special_text(text): return ( text.isdigit() or text.isspace() or is_text_link(text) or all(char in string.punctuation for char in text) ) def _make_new_book(self, book): new_book = epub.EpubBook() allowed_ns = set(epub.NAMESPACES.keys()) | set(epub.NAMESPACES.values()) for namespace, metas in book.metadata.items(): # Only keep namespaces recognized by ebooklib if namespace not in allowed_ns: continue if isinstance(metas, dict): entries = ( (name, value, others) for name, values in metas.items() for value, others in ( (item if isinstance(item, tuple) else (item, None)) for item in values ) ) else: entries = metas for entry in entries: if not entry: continue if isinstance(entry, tuple): if len(entry) == 3: name, value, others = entry elif len(entry) == 2: name, value = entry others = None else: continue else: # Unexpected metadata format; skip gracefully continue # `others` can be {} or None if others: new_book.add_metadata(namespace, name, value, others) else: new_book.add_metadata(namespace, name, value) new_book.spine = book.spine new_book.toc = self._fix_toc_uids(book.toc) return new_book def _fix_toc_uids(self, toc, counter=None): """Fix TOC items that have uid=None to prevent TypeError when writing NCX.""" if counter is None: counter = [0] # Use list to allow mutation in nested calls fixed_toc = [] for item in toc: if isinstance(item, tuple): # Section with sub-items: (Section, [sub-items]) section, sub_items = item if hasattr(section, "uid") and section.uid is None: section.uid = f"navpoint-{counter[0]}" counter[0] += 1 fixed_sub_items = self._fix_toc_uids(sub_items, counter) fixed_toc.append((section, fixed_sub_items)) elif hasattr(item, "uid"): # Link or EpubHtml item if item.uid is None: item.uid = f"navpoint-{counter[0]}" counter[0] += 1 fixed_toc.append(item) else: fixed_toc.append(item) return fixed_toc def _extract_paragraph(self, p): for p_exclude in self.exclude_translate_tags.split(","): # for issue #280 if type(p) is NavigableString: continue for pt in p.find_all(p_exclude): pt.extract() return p def _process_paragraph(self, p, new_p, index, p_to_save_len, thread_safe=False): if self.resume and index < p_to_save_len: p.string = self.p_to_save[index] new_p.string = self.p_to_save[ index ] # Fix: also update new_p to cached translation else: t_text = "" if self.batch_flag: self.translate_model.add_to_batch_translate_queue(index, new_p.text) elif self.batch_use_flag: t_text = self.translate_model.batch_translate(index) else: t_text = self.translate_model.translate(new_p.text) if t_text is None: raise RuntimeError( "`t_text` is None: your translation model is not working as expected. Please check your translation model configuration." ) if type(p) is NavigableString: new_p = t_text self.p_to_save.append(new_p) else: new_p.string = t_text self.p_to_save.append(new_p.text) self.helper.insert_trans( p, new_p.string, self.translation_style, self.single_translate ) index += 1 if thread_safe: with self._progress_lock: if index % 20 == 0: self._save_progress() else: if index % 20 == 0: self._save_progress() return index def _process_combined_paragraph( self, p_block, index, p_to_save_len, thread_safe=False ): text = [] for p in p_block: if self.resume and index < p_to_save_len: p.string = self.p_to_save[index] else: p_text = p.text.rstrip() text.append(p_text) if self.is_test and index >= self.test_num: break index += 1 if len(text) > 0: translated_text = self.translate_model.translate("\n".join(text)) translated_text = translated_text.split("\n") text_len = len(translated_text) for i in range(text_len): t = translated_text[i] if i >= len(p_block): p = p_block[-1] else: p = p_block[i] if type(p) is NavigableString: p = t else: p.string = t self.helper.insert_trans( p, p.string, self.translation_style, self.single_translate ) if thread_safe: with self._progress_lock: self._save_progress() else: self._save_progress() return index def translate_paragraphs_acc(self, p_list, send_num): count = 0 wait_p_list = [] for i in range(len(p_list)): p = p_list[i] print(f"translating {i}/{len(p_list)}") temp_p = copy(p) for p_exclude in self.exclude_translate_tags.split(","): # for issue #280 if type(p) is NavigableString: continue for pt in temp_p.find_all(p_exclude): pt.extract() if any( [not p.text, self._is_special_text(temp_p.text), not_trans(temp_p.text)] ): if i == len(p_list) - 1: self.helper.deal_old(wait_p_list, self.single_translate) continue length = num_tokens_from_text(temp_p.text) if length > send_num: self.helper.deal_new(p, wait_p_list, self.single_translate) continue if i == len(p_list) - 1: if count + length < send_num: wait_p_list.append(p) self.helper.deal_old(wait_p_list, self.single_translate) else: self.helper.deal_new(p, wait_p_list, self.single_translate) break if count + length < send_num: count += length wait_p_list.append(p) else: self.helper.deal_old(wait_p_list, self.single_translate) wait_p_list.append(p) count = length def get_item(self, book, name): for item in book.get_items(): if item.file_name == name: return item def find_items_containing_string(self, book, search_string): matching_items = [] for item in book.get_items_of_type(ITEM_DOCUMENT): content = item.get_content() soup = bs(content, "html.parser") if search_string in soup.get_text(): matching_items.append(item) return matching_items def retranslate_book(self, index, p_to_save_len, pbar, trans_taglist, retranslate): complete_book_name = retranslate[0] fixname = retranslate[1] fixstart = retranslate[2] fixend = retranslate[3] if fixend == "": fixend = fixstart name_fix = complete_book_name complete_book = epub.read_epub(complete_book_name) if fixname == "": fixname = self.find_items_containing_string(complete_book, fixstart)[ 0 ].file_name print(f"auto find fixname: {fixname}") new_book = self._make_new_book(complete_book) complete_item = self.get_item(complete_book, fixname) if complete_item is None: return ori_item = self.get_item(self.origin_book, fixname) if ori_item is None: return content_complete = complete_item.content content_ori = ori_item.content soup_complete = bs(content_complete, "html.parser") soup_ori = bs(content_ori, "html.parser") p_list_complete = soup_complete.findAll(trans_taglist) p_list_ori = soup_ori.findAll(trans_taglist) target = None tagl = [] # extract from range find_end = False find_start = False for tag in p_list_complete: if find_end: tagl.append(tag) break if fixend in tag.text: find_end = True if fixstart in tag.text: find_start = True if find_start: if not target: target = tag.previous_sibling tagl.append(tag) for t in tagl: t.extract() flag = False extract_p_list_ori = [] for p in p_list_ori: if fixstart in p.text: flag = True if flag: extract_p_list_ori.append(p) if fixend in p.text: break for t in extract_p_list_ori: if target: target.insert_after(t) target = t for item in complete_book.get_items(): if item.file_name != fixname: new_book.add_item(item) if soup_complete: complete_item.content = soup_complete.encode() index = self.process_item( complete_item, index, p_to_save_len, pbar, new_book, trans_taglist, fixstart, fixend, ) epub.write_epub(f"{name_fix}", new_book, {}) def has_nest_child(self, element, trans_taglist): if isinstance(element, Tag): for child in element.children: if child.name in trans_taglist: return True if self.has_nest_child(child, trans_taglist): return True return False def filter_nest_list(self, p_list, trans_taglist): filtered_list = [p for p in p_list if not self.has_nest_child(p, trans_taglist)] return filtered_list def process_item( self, item, index, p_to_save_len, pbar, new_book, trans_taglist, fixstart=None, fixend=None, ): if self.only_filelist != "" and item.file_name not in self.only_filelist.split( "," ): return index elif self.only_filelist == "" and item.file_name in self.exclude_filelist.split( "," ): new_book.add_item(item) return index if not os.path.exists("log"): os.makedirs("log") content = item.content soup = bs(content, "html.parser") p_list = soup.findAll(trans_taglist) p_list = self.filter_nest_list(p_list, trans_taglist) if self.retranslate: new_p_list = [] if fixstart is None or fixend is None: return start_append = False for p in p_list: text = p.get_text() if fixstart in text or fixend in text or start_append: start_append = True new_p_list.append(p) if fixend in text: p_list = new_p_list break if self.allow_navigable_strings: p_list.extend(soup.findAll(text=True)) send_num = self.accumulated_num if send_num > 1: with open("log/buglog.txt", "a") as f: print(f"------------- {item.file_name} -------------", file=f) print("------------------------------------------------------") print(f"dealing {item.file_name} ...") self.translate_paragraphs_acc(p_list, send_num) else: is_test_done = self.is_test and index > self.test_num p_block = [] block_len = 0 for p in p_list: if is_test_done: break if not p.text or self._is_special_text(p.text): pbar.update(1) continue new_p = self._extract_paragraph(copy(p)) if self.single_translate and self.block_size > 0: p_len = num_tokens_from_text(new_p.text) block_len += p_len if block_len > self.block_size: index = self._process_combined_paragraph( p_block, index, p_to_save_len, thread_safe=False ) p_block = [p] block_len = p_len print() else: p_block.append(p) else: index = self._process_paragraph( p, new_p, index, p_to_save_len, thread_safe=False ) print() # pbar.update(delta) not pbar.update(index)? pbar.update(1) if self.is_test and index >= self.test_num: break if self.single_translate and self.block_size > 0 and len(p_block) > 0: index = self._process_combined_paragraph( p_block, index, p_to_save_len, thread_safe=False ) if soup: item.content = soup.encode(encoding="utf-8") new_book.add_item(item) return index def set_parallel_workers(self, workers): """Set number of parallel workers for chapter processing. Args: workers (int): Number of parallel workers. Will be automatically optimized based on actual chapter count during processing. """ self.parallel_workers = max(1, workers) self.enable_parallel = workers > 1 if workers > 8: print( f"⚠️ Warning: {workers} workers is quite high. Consider using 2-8 workers for optimal performance." ) def _get_next_translation_index(self): """Thread-safe method to get next translation index.""" with self._progress_lock: index = self._translation_index self._translation_index += 1 return index def _process_chapter_parallel(self, chapter_data): """Process a single chapter in parallel mode with proper accumulated_num handling.""" item, trans_taglist, p_to_save_len = chapter_data chapter_result = { "item": item, "processed_content": None, "success": False, "error": None, } try: # Create a chapter-specific translator instance to avoid context conflicts # This ensures each chapter has its own independent context thread_translator = self._create_chapter_translator() content = item.content soup = bs(content, "html.parser") p_list = soup.findAll(trans_taglist) p_list = self.filter_nest_list(p_list, trans_taglist) if self.allow_navigable_strings: p_list.extend(soup.findAll(text=True)) # Initialize chapter-specific context lists chapter_context_list = [] chapter_translated_list = [] # Apply accumulated_num logic for this chapter independently send_num = self.accumulated_num if send_num > 1: # Use accumulated translation logic for this chapter self._translate_paragraphs_acc_parallel( p_list, send_num, thread_translator, chapter_context_list, chapter_translated_list, ) else: # Process paragraphs individually for this chapter for p in p_list: if not p.text or self._is_special_text(p.text): continue new_p = self._extract_paragraph(copy(p)) index = self._get_next_translation_index() if self.resume and index < p_to_save_len: t_text = self.p_to_save[index] else: # Use chapter-specific context for translation t_text = self._translate_with_chapter_context( thread_translator, new_p.text, chapter_context_list, chapter_translated_list, ) t_text = "" if t_text is None else t_text with self._progress_lock: self.p_to_save.append(t_text) if isinstance(p, NavigableString): translated_node = NavigableString(t_text) p.insert_after(translated_node) if self.single_translate: p.extract() else: self.helper.insert_trans( p, t_text, self.translation_style, self.single_translate ) with self._progress_lock: if index % 20 == 0: self._save_progress() if soup: chapter_result["processed_content"] = soup.encode(encoding="utf-8") chapter_result["success"] = True except Exception as e: chapter_result["error"] = str(e) print(f"Error processing chapter {item.file_name}: {e}") return chapter_result def _create_chapter_translator(self): """Create a translator instance for a specific chapter with independent context.""" # Return the main translator - we'll handle context at the chapter level return self.translate_model def _translate_with_chapter_context( self, translator, text, chapter_context_list, chapter_translated_list ): """Translate text with chapter-specific context management.""" if not translator.context_flag: return translator.translate(text) # Temporarily replace global context with chapter context original_context = getattr(translator, "context_list", []) original_translated = getattr(translator, "context_translated_list", []) try: # Use chapter-specific context translator.context_list = chapter_context_list.copy() translator.context_translated_list = chapter_translated_list.copy() # Perform translation result = translator.translate(text) # Update chapter context chapter_context_list[:] = translator.context_list chapter_translated_list[:] = translator.context_translated_list return result finally: # Restore original context translator.context_list = original_context translator.context_translated_list = original_translated def _translate_paragraphs_acc_parallel( self, p_list, send_num, translator, chapter_context_list, chapter_translated_list, ): """Apply accumulated_num logic for a single chapter in parallel mode with independent context.""" from book_maker.utils import num_tokens_from_text from .helper import not_trans count = 0 wait_p_list = [] # Create chapter-specific helper instance with context-aware translation class ChapterHelper: def __init__( self, parent_loader, translator, context_list, translated_list ): self.parent_loader = parent_loader self.translator = translator self.context_list = context_list self.translated_list = translated_list def translate_with_context(self, text): return self.parent_loader._translate_with_chapter_context( self.translator, text, self.context_list, self.translated_list ) def deal_old(self, wait_p_list, single_translate): if not wait_p_list: return # Use the same translate_list logic as sequential processing # Create a temporary translator with chapter context original_context = getattr(self.translator, "context_list", []) original_translated = getattr( self.translator, "context_translated_list", [] ) try: # Set chapter context to the translator self.translator.context_list = self.context_list.copy() self.translator.context_translated_list = ( self.translated_list.copy() ) # Call translate_list for consistent batch translation logic result_txt_list = self.translator.translate_list(wait_p_list) # Update chapter context from translator self.context_list[:] = self.translator.context_list self.translated_list[:] = self.translator.context_translated_list # Apply translations using the same logic as helper.deal_old for i in range(len(wait_p_list)): if i < len(result_txt_list): p = wait_p_list[i] from .helper import shorter_result_link self.parent_loader.helper.insert_trans( p, shorter_result_link(result_txt_list[i]), self.parent_loader.translation_style, single_translate, ) finally: # Restore original context self.translator.context_list = original_context self.translator.context_translated_list = original_translated wait_p_list.clear() def deal_new(self, p, wait_p_list, single_translate): self.deal_old(wait_p_list, single_translate) translation = self.translate_with_context(p.text) self.parent_loader.helper.insert_trans( p, translation, self.parent_loader.translation_style, single_translate, ) chapter_helper = ChapterHelper( self, translator, chapter_context_list, chapter_translated_list ) for i in range(len(p_list)): p = p_list[i] temp_p = copy(p) for p_exclude in self.exclude_translate_tags.split(","): if type(p) == NavigableString: continue for pt in temp_p.find_all(p_exclude): pt.extract() if any( [not p.text, self._is_special_text(temp_p.text), not_trans(temp_p.text)] ): if i == len(p_list) - 1: chapter_helper.deal_old(wait_p_list, self.single_translate) continue length = num_tokens_from_text(temp_p.text) if length > send_num: chapter_helper.deal_new(p, wait_p_list, self.single_translate) continue if i == len(p_list) - 1: if count + length < send_num: wait_p_list.append(p) chapter_helper.deal_old(wait_p_list, self.single_translate) else: chapter_helper.deal_new(p, wait_p_list, self.single_translate) break if count + length < send_num: count += length wait_p_list.append(p) else: chapter_helper.deal_old(wait_p_list, self.single_translate) wait_p_list.append(p) count = length def batch_init_then_wait(self): name, _ = os.path.splitext(self.epub_name) if self.batch_flag or self.batch_use_flag: self.translate_model.batch_init(name) if self.batch_use_flag: start_time = time.time() while not self.translate_model.is_completed_batch(): print("Batch translation is not completed yet") time.sleep(2) if time.time() - start_time > 300: # 5 minutes raise Exception("Batch translation timed out after 5 minutes") def make_bilingual_book(self): self.helper = EPUBBookLoaderHelper( self.translate_model, self.accumulated_num, self.translation_style, self.context_flag, ) self.batch_init_then_wait() new_book = self._make_new_book(self.origin_book) all_items = list(self.origin_book.get_items()) trans_taglist = self.translate_tags.split(",") all_p_length = sum( ( 0 if ( (i.get_type() != ITEM_DOCUMENT) or (i.file_name in self.exclude_filelist.split(",")) or ( self.only_filelist and i.file_name not in self.only_filelist.split(",") ) ) else len(bs(i.content, "html.parser").findAll(trans_taglist)) ) for i in all_items ) all_p_length += self.allow_navigable_strings * sum( ( 0 if ( (i.get_type() != ITEM_DOCUMENT) or (i.file_name in self.exclude_filelist.split(",")) or ( self.only_filelist and i.file_name not in self.only_filelist.split(",") ) ) else len(bs(i.content, "html.parser").findAll(text=True)) ) for i in all_items ) pbar = tqdm(total=self.test_num) if self.is_test else tqdm(total=all_p_length) print() index = 0 p_to_save_len = len(self.p_to_save) try: if self.retranslate: self.retranslate_book( index, p_to_save_len, pbar, trans_taglist, self.retranslate ) exit(0) # Add the things that don't need to be translated first, so that you can see the img after the interruption for item in self.origin_book.get_items(): if item.get_type() != ITEM_DOCUMENT: new_book.add_item(item) document_items = list(self.origin_book.get_items_of_type(ITEM_DOCUMENT)) if self.enable_parallel and len(document_items) > 1: # Optimize worker count: no point having more workers than chapters effective_workers = min(self.parallel_workers, len(document_items)) # Parallel processing with proper accumulated_num handling print(f"🚀 Parallel processing: {len(document_items)} chapters") if effective_workers < self.parallel_workers: print( f"📊 Optimized workers: {effective_workers} (reduced from {self.parallel_workers})" ) else: print(f"📊 Using {effective_workers} workers") if self.accumulated_num > 1: print( f"📝 Each chapter applies accumulated_num={self.accumulated_num} independently" ) if self.context_flag: print( f"🔗 Context enabled: each chapter maintains independent context (limit={self.translate_model.context_paragraph_limit})" ) else: print(f"🚫 Context disabled for this translation") # Create a simpler progress bar for parallel processing pbar.close() # Close the original progress bar chapter_pbar = tqdm( total=len(document_items), desc="Chapters", unit="ch" ) chapter_data_list = [ (item, trans_taglist, p_to_save_len) for item in document_items ] with ThreadPoolExecutor(max_workers=effective_workers) as executor: future_to_item = { executor.submit( self._process_chapter_parallel, chapter_data ): chapter_data[0] for chapter_data in chapter_data_list } for future in as_completed(future_to_item): item = future_to_item[future] try: result = future.result() if result["success"] and result["processed_content"]: item.content = result["processed_content"] new_book.add_item(item) chapter_pbar.update(1) chapter_pbar.set_postfix_str( f"Latest: {item.file_name[:20]}..." ) except Exception as e: print(f"❌ Error processing {item.file_name}: {e}") new_book.add_item(item) chapter_pbar.update(1) chapter_pbar.close() print(f"✅ Completed all {len(document_items)} chapters") else: # Sequential processing (original behavior or single chapter) if len(document_items) == 1 and self.enable_parallel: print(f"📄 Single chapter detected - using sequential processing") for item in document_items: index = self.process_item( item, index, p_to_save_len, pbar, new_book, trans_taglist ) if self.accumulated_num > 1: name, _ = os.path.splitext(self.epub_name) epub.write_epub(f"{name}_bilingual.epub", new_book, {}) name, _ = os.path.splitext(self.epub_name) if self.batch_flag: self.translate_model.batch() else: epub.write_epub(f"{name}_bilingual.epub", new_book, {}) if self.accumulated_num == 1: pbar.close() except KeyboardInterrupt as e: print(e) if self.accumulated_num == 1: print("you can resume it next time") self._save_progress() self._save_temp_book() sys.exit(0) except Exception: traceback.print_exc() sys.exit(0) def load_state(self): try: with open(self.bin_path, "rb") as f: self.p_to_save = pickle.load(f) except Exception: raise Exception("can not load resume file") def _save_temp_book(self): # TODO refactor this logic origin_book_temp = epub.read_epub(self.epub_name) new_temp_book = self._make_new_book(origin_book_temp) p_to_save_len = len(self.p_to_save) trans_taglist = self.translate_tags.split(",") index = 0 try: for item in origin_book_temp.get_items(): if item.get_type() == ITEM_DOCUMENT: content = item.content soup = bs(content, "html.parser") p_list = soup.findAll(trans_taglist) if self.allow_navigable_strings: p_list.extend(soup.findAll(text=True)) for p in p_list: if not p.text or self._is_special_text(p.text): continue # TODO banch of p to translate then combine # PR welcome here if index < p_to_save_len: new_p = copy(p) if type(p) is NavigableString: new_p = self.p_to_save[index] else: new_p.string = self.p_to_save[index] self.helper.insert_trans( p, new_p.string, self.translation_style, self.single_translate, ) index += 1 else: break # for save temp book if soup: item.content = soup.encode() new_temp_book.add_item(item) name, _ = os.path.splitext(self.epub_name) epub.write_epub(f"{name}_bilingual_temp.epub", new_temp_book, {}) except Exception as e: # TODO handle it print(e) def _save_progress(self): try: with open(self.bin_path, "wb") as f: pickle.dump(self.p_to_save, f) except Exception: raise Exception("can not save resume file") ================================================ FILE: book_maker/loader/helper.py ================================================ import re import backoff import logging from copy import copy logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) class EPUBBookLoaderHelper: def __init__( self, translate_model, accumulated_num, translation_style, context_flag ): self.translate_model = translate_model self.accumulated_num = accumulated_num self.translation_style = translation_style self.context_flag = context_flag def insert_trans(self, p, text, translation_style="", single_translate=False): if text is None: text = "" if ( p.string is not None and p.string.replace(" ", "").strip() == text.replace(" ", "").strip() ): return new_p = copy(p) new_p.string = text if translation_style != "": new_p["style"] = translation_style p.insert_after(new_p) if single_translate: p.extract() @backoff.on_exception( backoff.expo, Exception, on_backoff=lambda details: logger.warning(f"retry backoff: {details}"), on_giveup=lambda details: logger.warning(f"retry abort: {details}"), jitter=None, ) def translate_with_backoff(self, text, context_flag=False): return self.translate_model.translate(text, context_flag) def deal_new(self, p, wait_p_list, single_translate=False): self.deal_old(wait_p_list, single_translate, self.context_flag) self.insert_trans( p, shorter_result_link(self.translate_with_backoff(p.text, self.context_flag)), self.translation_style, single_translate, ) def deal_old(self, wait_p_list, single_translate=False, context_flag=False): if not wait_p_list: return result_txt_list = self.translate_model.translate_list(wait_p_list) for i in range(len(wait_p_list)): if i < len(result_txt_list): p = wait_p_list[i] self.insert_trans( p, shorter_result_link(result_txt_list[i]), self.translation_style, single_translate, ) wait_p_list.clear() url_pattern = r"(http[s]?://|www\.)+(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+" def is_text_link(text): return bool(re.compile(url_pattern).match(text.strip())) def is_text_tail_link(text, num=80): text = text.strip() pattern = r".*" + url_pattern + r"$" return bool(re.compile(pattern).match(text)) and len(text) < num def shorter_result_link(text, num=20): match = re.search(url_pattern, text) if not match or len(match.group()) < num: return text return re.compile(url_pattern).sub("...", text) def is_text_source(text): return text.strip().startswith("Source: ") def is_text_list(text, num=80): text = text.strip() return re.match(r"^Listing\s*\d+", text) and len(text) < num def is_text_figure(text, num=80): text = text.strip() return re.match(r"^Figure\s*\d+", text) and len(text) < num def is_text_digit_and_space(s): for c in s: if not c.isdigit() and not c.isspace(): return False return True def is_text_isbn(s): pattern = r"^[Ee]?ISBN\s*\d[\d\s]*$" return bool(re.match(pattern, s)) def not_trans(s): return any( [ is_text_link(s), is_text_tail_link(s), is_text_source(s), is_text_list(s), is_text_figure(s), is_text_digit_and_space(s), is_text_isbn(s), ] ) ================================================ FILE: book_maker/loader/md_loader.py ================================================ import sys from pathlib import Path from book_maker.utils import prompt_config_to_kwargs from .base_loader import BaseBookLoader class MarkdownBookLoader(BaseBookLoader): def __init__( self, md_name, model, key, resume, language, model_api_base=None, is_test=False, test_num=5, prompt_config=None, single_translate=False, context_flag=False, context_paragraph_limit=0, temperature=1.0, source_lang="auto", ) -> None: self.md_name = md_name self.translate_model = model( key, language, api_base=model_api_base, temperature=temperature, source_lang=source_lang, **prompt_config_to_kwargs(prompt_config), ) self.is_test = is_test self.p_to_save = [] self.bilingual_result = [] self.bilingual_temp_result = [] self.test_num = test_num self.batch_size = 10 self.single_translate = single_translate self.md_paragraphs = [] try: with open(f"{md_name}", encoding="utf-8") as f: self.origin_book = f.read().splitlines() except Exception as e: raise Exception("can not load file") from e self.resume = resume self.bin_path = f"{Path(md_name).parent}/.{Path(md_name).stem}.temp.bin" if self.resume: self.load_state() self.process_markdown_content() def process_markdown_content(self): """将原始内容处理成 markdown 段落""" current_paragraph = [] for line in self.origin_book: # 如果是空行且当前段落不为空，保存当前段落 if not line.strip() and current_paragraph: self.md_paragraphs.append("\n".join(current_paragraph)) current_paragraph = [] # 如果是标题行，单独作为一个段落 elif line.strip().startswith("#"): if current_paragraph: self.md_paragraphs.append("\n".join(current_paragraph)) current_paragraph = [] self.md_paragraphs.append(line) # 其他情况，添加到当前段落 else: current_paragraph.append(line) # 处理最后一个段落 if current_paragraph: self.md_paragraphs.append("\n".join(current_paragraph)) @staticmethod def _is_special_text(text): return text.isdigit() or text.isspace() or len(text) == 0 def _make_new_book(self, book): pass def make_bilingual_book(self): index = 0 p_to_save_len = len(self.p_to_save) try: sliced_list = [ self.md_paragraphs[i : i + self.batch_size] for i in range(0, len(self.md_paragraphs), self.batch_size) ] for paragraphs in sliced_list: batch_text = "\n\n".join(paragraphs) if self._is_special_text(batch_text): continue if not self.resume or index >= p_to_save_len: try: max_retries = 3 retry_count = 0 while retry_count < max_retries: try: temp = self.translate_model.translate(batch_text) break except AttributeError as ae: print(f"翻译出错: {ae}") retry_count += 1 if retry_count == max_retries: raise Exception("翻译模型初始化失败") from ae except Exception as e: print(f"翻译过程中出错: {e}") raise Exception("翻译过程中出现错误") from e self.p_to_save.append(temp) if not self.single_translate: self.bilingual_result.append(batch_text) self.bilingual_result.append(temp) index += self.batch_size if self.is_test and index > self.test_num: break self.save_file( f"{Path(self.md_name).parent}/{Path(self.md_name).stem}_bilingual.md", self.bilingual_result, ) except (KeyboardInterrupt, Exception) as e: print(f"发生错误: {e}") print("程序将保存进度，您可以稍后继续") self._save_progress() self._save_temp_book() sys.exit(1) # 使用非零退出码表示错误 def _save_temp_book(self): index = 0 sliced_list = [ self.origin_book[i : i + self.batch_size] for i in range(0, len(self.origin_book), self.batch_size) ] for i in range(len(sliced_list)): batch_text = "".join(sliced_list[i]) self.bilingual_temp_result.append(batch_text) if self._is_special_text(self.origin_book[i]): continue if index < len(self.p_to_save): self.bilingual_temp_result.append(self.p_to_save[index]) index += 1 self.save_file( f"{Path(self.md_name).parent}/{Path(self.md_name).stem}_bilingual_temp.txt", self.bilingual_temp_result, ) def _save_progress(self): try: with open(self.bin_path, "w", encoding="utf-8") as f: f.write("\n".join(self.p_to_save)) except Exception as e: raise Exception("can not save resume file") from e def load_state(self): try: with open(self.bin_path, encoding="utf-8") as f: self.p_to_save = f.read().splitlines() except Exception as e: raise Exception("can not load resume file") from e def save_file(self, book_path, content): try: with open(book_path, "w", encoding="utf-8") as f: f.write("\n".join(content)) except Exception as e: raise Exception("can not save file") from e ================================================ FILE: book_maker/loader/pdf_loader.py ================================================ import sys from pathlib import Path from book_maker.utils import prompt_config_to_kwargs from .base_loader import BaseBookLoader import fitz from ebooklib import epub class PDFBookLoader(BaseBookLoader): def __init__( self, pdf_name, model, key, resume, language, model_api_base=None, is_test=False, test_num=5, prompt_config=None, single_translate=False, context_flag=False, context_paragraph_limit=0, temperature=1.0, source_lang="auto", parallel_workers=1, ) -> None: if fitz is None: raise Exception("PyMuPDF (fitz) is required to use PDF loader") self.pdf_name = pdf_name self.translate_model = model( key, language, api_base=model_api_base, temperature=temperature, source_lang=source_lang, **prompt_config_to_kwargs(prompt_config), ) self.is_test = is_test self.p_to_save = [] self.bilingual_result = [] self.bilingual_temp_result = [] self.test_num = test_num self.batch_size = 10 self.single_translate = single_translate self.parallel_workers = max(1, parallel_workers) try: doc = fitz.open(self.pdf_name) lines = [] for page in doc: text = page.get_text("text") if not text: continue lines.extend(text.splitlines()) self.origin_book = lines except Exception as e: raise Exception("can not load file") from e self.resume = resume self.bin_path = f"{Path(pdf_name).parent}/.{Path(pdf_name).stem}.temp.bin" if self.resume: self.load_state() def _make_new_book(self, book): pass def _try_create_epub(self): """Try to create an EPUB file from translated content. The EPUB is created from the `self.bilingual_result` list which alternates original and translated strings. If EPUB creation fails for any reason, this function will log the error and leave the TXT fallback intact. """ if epub is None: # ebooklib not installed; skip EPUB generation return False if not self.bilingual_result: return False try: book = epub.EpubBook() title = Path(self.pdf_name).stem # Minimal metadata try: book.set_identifier(title) book.set_title(title) book.set_language( self.translate_model.language if hasattr(self.translate_model, "language") else "en" ) except Exception: # be tolerant about metadata API differences pass chapters = [] # build chapters from bilingual_result (pairs) for i in range(0, len(self.bilingual_result), 2): orig = self.bilingual_result[i] trans = ( self.bilingual_result[i + 1] if i + 1 < len(self.bilingual_result) else "" ) # basic html content: original then translated content = "" if orig: content += ( '

' + "

" + orig.replace("\n", "
") + "

" ) if trans: content += ( '
' + "

" + trans.replace("\n", "
") + "

" ) chap = epub.EpubHtml( title=f"part_{i//2}", file_name=f"index_split_{i//2:03d}.xhtml", lang=( book.get_metadata("DC", "language")[0][0] if book.get_metadata("DC", "language") else None ), ) chap.content = content book.add_item(chap) chapters.append(chap) # table of contents and spine book.toc = tuple(chapters) book.spine = ["nav"] + chapters # add navigation files book.add_item(epub.EpubNcx()) book.add_item(epub.EpubNav()) out_path = f"{Path(self.pdf_name).parent}/{Path(self.pdf_name).stem}_bilingual.epub" epub.write_epub(out_path, book) return True except Exception as e: print(f"create epub failed: {e}") return False def make_bilingual_book(self): index = 0 p_to_save_len = len(self.p_to_save) try: sliced_list = [ self.origin_book[i : i + self.batch_size] for i in range(0, len(self.origin_book), self.batch_size) ] for i in sliced_list: # fix the format thanks https://github.com/tudoujunha batch_text = "\n".join(i) if not batch_text.strip(): continue if not self.resume or index >= p_to_save_len: try: temp = self.translate_model.translate(batch_text) except Exception as e: print(e) raise Exception("Something is wrong when translate") from e self.p_to_save.append(temp) if not self.single_translate: self.bilingual_result.append(batch_text) self.bilingual_result.append(temp) index += self.batch_size if self.is_test and index > self.test_num: break txt_out = ( f"{Path(self.pdf_name).parent}/{Path(self.pdf_name).stem}_bilingual.txt" ) self.save_file(txt_out, self.bilingual_result) # try to create an EPUB alongside the TXT fallback; if EPUB fails we still keep the TXT file epub_ok = self._try_create_epub() if epub_ok: print(f"created epub: {Path(self.pdf_name).stem}_bilingual.epub") else: print( "epub creation skipped or failed; bilingual text saved to txt fallback" ) except (KeyboardInterrupt, Exception) as e: print(e) print("you can resume it next time") self._save_progress() self._save_temp_book() sys.exit(0) def _save_temp_book(self): index = 0 sliced_list = [ self.origin_book[i : i + self.batch_size] for i in range(0, len(self.origin_book), self.batch_size) ] for i in range(len(sliced_list)): batch_text = "".join(sliced_list[i]) self.bilingual_temp_result.append(batch_text) if index < len(self.p_to_save): self.bilingual_temp_result.append(self.p_to_save[index]) index += 1 self.save_file( f"{Path(self.pdf_name).parent}/{Path(self.pdf_name).stem}_bilingual_temp.txt", self.bilingual_temp_result, ) def _save_progress(self): try: with open(self.bin_path, "w", encoding="utf-8") as f: f.write("\n".join(self.p_to_save)) except Exception as e: raise Exception("can not save resume file") from e def load_state(self): try: with open(self.bin_path, encoding="utf-8") as f: self.p_to_save = f.read().splitlines() except Exception as e: raise Exception("can not load resume file") from e def save_file(self, book_path, content): try: with open(book_path, "w", encoding="utf-8") as f: f.write("\n".join(content)) except Exception as e: raise Exception("can not save file") from e ================================================ FILE: book_maker/loader/srt_loader.py ================================================ """ inspired by: https://github.com/jesselau76/srt-gpt-translator, MIT License """ import re import sys from pathlib import Path from book_maker.utils import prompt_config_to_kwargs from .base_loader import BaseBookLoader class SRTBookLoader(BaseBookLoader): def __init__( self, srt_name, model, key, resume, language, model_api_base=None, is_test=False, test_num=5, prompt_config=None, single_translate=False, context_flag=False, context_paragraph_limit=0, temperature=1.0, source_lang="auto", ) -> None: self.srt_name = srt_name self.translate_model = model( key, language, api_base=model_api_base, temperature=temperature, source_lang=source_lang, **prompt_config_to_kwargs( { "system": "You are a srt subtitle file translator.", "user": "Translate the following subtitle text into {language}, but keep the subtitle number and timeline and newlines unchanged: \n{text}", } ), ) self.is_test = is_test self.p_to_save = [] self.bilingual_result = [] self.bilingual_temp_result = [] self.test_num = test_num self.accumulated_num = 1 self.blocks = [] self.single_translate = single_translate self.resume = resume self.bin_path = f"{Path(srt_name).parent}/.{Path(srt_name).stem}.temp.bin" if self.resume: self.load_state() def _make_new_book(self, book): pass def _parse_srt(self, srt_text): blocks = re.split("\n\s*\n", srt_text) final_blocks = [] new_block = {} for i in range(0, len(blocks)): block = blocks[i] if block.strip() == "": continue lines = block.strip().splitlines() new_block["number"] = lines[0].strip() timestamp = lines[1].strip() new_block["time"] = timestamp text = "\n".join(lines[2:]).strip() new_block["text"] = text final_blocks.append(new_block) new_block = {} return final_blocks def _get_block_text(self, block): return f"{block['number']}\n{block['time']}\n{block['text']}" def _get_block_except_text(self, block): return f"{block['number']}\n{block['time']}" def _concat_blocks(self, sliced_text: str, text: str): return f"{sliced_text}\n\n{text}" if sliced_text else text def _get_block_translate(self, block): return f"{block['number']}\n{block['text']}" def _get_block_from(self, text): text = text.strip() if not text: return {} block = text.splitlines() if len(block) < 2: return {"number": block[0], "text": ""} return {"number": block[0], "text": "\n".join(block[1:])} def _get_blocks_from(self, translate: str): if not translate: return [] blocks = [] blocks_text = translate.strip().split("\n\n") for text in blocks_text: blocks.append(self._get_block_from(text)) return blocks def _check_blocks(self, translate_blocks, origin_blocks): """ Check if the translated blocks match the original text, with only a simple check of the beginning numbers. """ if len(translate_blocks) != len(origin_blocks): return False for t in zip(translate_blocks, origin_blocks): i = 0 try: i = int(t[0].get("number", 0)) except ValueError: m = re.search(r"\s*\d+", t[0].get("number")) if m: i = int(m.group()) j = int(t[1].get("number", -1)) if i != j: print(f"check failed: {i}!={j}") return False return True def _get_sliced_list(self): sliced_list = [] sliced_text = "" begin_index = 0 for i, block in enumerate(self.blocks): text = self._get_block_translate(block) if not text: continue if len(sliced_text + text) < self.accumulated_num: sliced_text = self._concat_blocks(sliced_text, text) else: if sliced_text: sliced_list.append((begin_index, i, sliced_text)) sliced_text = text begin_index = i sliced_list.append((begin_index, len(self.blocks), sliced_text)) return sliced_list def make_bilingual_book(self): if self.accumulated_num > 512: print(f"{self.accumulated_num} is too large, shrink it to 512.") self.accumulated_num = 512 try: with open(f"{self.srt_name}", encoding="utf-8") as f: self.blocks = self._parse_srt(f.read()) except Exception as e: raise Exception("can not load file") from e index = 0 p_to_save_len = len(self.p_to_save) try: sliced_list = self._get_sliced_list() for sliced in sliced_list: begin, end, text = sliced if not self.resume or index + (end - begin) > p_to_save_len: if index < p_to_save_len: self.p_to_save = self.p_to_save[:index] try: temp = self.translate_model.translate(text) except Exception as e: print(e) raise Exception("Something is wrong when translate") from e translated_blocks = self._get_blocks_from(temp) if self.accumulated_num > 1: if not self._check_blocks( translated_blocks, self.blocks[begin:end] ): translated_blocks = [] # try to translate one by one, so don't accumulate too much print( f"retry it one by one: {self.blocks[begin]['number']} - {self.blocks[end - 1]['number']}" ) for block in self.blocks[begin:end]: try: temp = self.translate_model.translate( self._get_block_translate(block) ) except Exception as e: print(e) raise Exception( "Something is wrong when translate" ) from e translated_blocks.append(self._get_block_from(temp)) if not self._check_blocks( translated_blocks, self.blocks[begin:end] ): raise Exception( "retry failed, adjust the srt manually." ) for i, block in enumerate(translated_blocks): text = block.get("text", "") self.p_to_save.append(text) if self.single_translate: self.bilingual_result.append( f"{self._get_block_except_text(self.blocks[begin + i])}\n{text}" ) else: self.bilingual_result.append( f"{self._get_block_text(self.blocks[begin + i])}\n{text}" ) else: for i, block in enumerate(self.blocks[begin:end]): text = self.p_to_save[begin + i] if self.single_translate: self.bilingual_result.append( f"{self._get_block_except_text(self.blocks[begin + i])}\n{text}" ) else: self.bilingual_result.append( f"{self._get_block_text(self.blocks[begin + i])}\n{text}" ) index += end - begin if self.is_test and index > self.test_num: break self.save_file( f"{Path(self.srt_name).parent}/{Path(self.srt_name).stem}_bilingual.srt", self.bilingual_result, ) except (KeyboardInterrupt, Exception) as e: print(e) print("you can resume it next time") self._save_progress() self._save_temp_book() sys.exit(0) def _save_temp_book(self): for i, block in enumerate(self.blocks): if i < len(self.p_to_save): text = self.p_to_save[i] self.bilingual_temp_result.append( f"{self._get_block_text(block)}\n{text}" ) else: self.bilingual_temp_result.append(f"{self._get_block_text(block)}\n") self.save_file( f"{Path(self.srt_name).parent}/{Path(self.srt_name).stem}_bilingual_temp.srt", self.bilingual_temp_result, ) def _save_progress(self): try: with open(self.bin_path, "w", encoding="utf-8") as f: f.write("===".join(self.p_to_save)) except Exception as e: raise Exception("can not save resume file") from e def load_state(self): try: with open(self.bin_path, encoding="utf-8") as f: text = f.read() if text: self.p_to_save = text.split("===") else: self.p_to_save = [] except Exception as e: raise Exception("can not load resume file") from e def save_file(self, book_path, content): try: with open(book_path, "w", encoding="utf-8") as f: f.write("\n\n".join(content)) except Exception as e: raise Exception("can not save file") from e ================================================ FILE: book_maker/loader/txt_loader.py ================================================ import sys from pathlib import Path from book_maker.utils import prompt_config_to_kwargs from .base_loader import BaseBookLoader class TXTBookLoader(BaseBookLoader): def __init__( self, txt_name, model, key, resume, language, model_api_base=None, is_test=False, test_num=5, prompt_config=None, single_translate=False, context_flag=False, context_paragraph_limit=0, temperature=1.0, source_lang="auto", parallel_workers=1, ) -> None: self.txt_name = txt_name self.translate_model = model( key, language, api_base=model_api_base, temperature=temperature, source_lang=source_lang, **prompt_config_to_kwargs(prompt_config), ) self.is_test = is_test self.p_to_save = [] self.bilingual_result = [] self.bilingual_temp_result = [] self.test_num = test_num self.batch_size = 10 self.single_translate = single_translate self.parallel_workers = max(1, parallel_workers) try: with open(f"{txt_name}", encoding="utf-8") as f: self.origin_book = f.read().splitlines() except Exception as e: raise Exception("can not load file") from e self.resume = resume self.bin_path = f"{Path(txt_name).parent}/.{Path(txt_name).stem}.temp.bin" if self.resume: self.load_state() @staticmethod def _is_special_text(text): return text.isdigit() or text.isspace() or len(text) == 0 def _make_new_book(self, book): pass def make_bilingual_book(self): index = 0 p_to_save_len = len(self.p_to_save) try: sliced_list = [ self.origin_book[i : i + self.batch_size] for i in range(0, len(self.origin_book), self.batch_size) ] for i in sliced_list: # fix the format thanks https://github.com/tudoujunha batch_text = "\n".join(i) if self._is_special_text(batch_text): continue if not self.resume or index >= p_to_save_len: try: temp = self.translate_model.translate(batch_text) except Exception as e: print(e) raise Exception("Something is wrong when translate") from e self.p_to_save.append(temp) if not self.single_translate: self.bilingual_result.append(batch_text) self.bilingual_result.append(temp) index += self.batch_size if self.is_test and index > self.test_num: break self.save_file( f"{Path(self.txt_name).parent}/{Path(self.txt_name).stem}_bilingual.txt", self.bilingual_result, ) except (KeyboardInterrupt, Exception) as e: print(e) print("you can resume it next time") self._save_progress() self._save_temp_book() sys.exit(0) def _save_temp_book(self): index = 0 sliced_list = [ self.origin_book[i : i + self.batch_size] for i in range(0, len(self.origin_book), self.batch_size) ] for i in range(len(sliced_list)): batch_text = "".join(sliced_list[i]) self.bilingual_temp_result.append(batch_text) if self._is_special_text(self.origin_book[i]): continue if index < len(self.p_to_save): self.bilingual_temp_result.append(self.p_to_save[index]) index += 1 self.save_file( f"{Path(self.txt_name).parent}/{Path(self.txt_name).stem}_bilingual_temp.txt", self.bilingual_temp_result, ) def _save_progress(self): try: with open(self.bin_path, "w", encoding="utf-8") as f: f.write("\n".join(self.p_to_save)) except Exception as e: raise Exception("can not save resume file") from e def load_state(self): try: with open(self.bin_path, encoding="utf-8") as f: self.p_to_save = f.read().splitlines() except Exception as e: raise Exception("can not load resume file") from e def save_file(self, book_path, content): try: with open(book_path, "w", encoding="utf-8") as f: f.write("\n".join(content)) except Exception as e: raise Exception("can not save file") from e ================================================ FILE: book_maker/obok.py ================================================ # The original code comes from: # https://github.com/apprenticeharper/DeDRM_tools # Version 4.1.2 March 2023 # Update library for crypto for current Windows # Version 4.1.1 March 2023 # Make obok.py works as file selector # Version 4.1.0 February 2021 # Add detection for Kobo directory location on Linux # Version 4.0.0 September 2020 # Python 3.0 # # Version 3.2.5 December 2016 # Improve detection of good text decryption. # # Version 3.2.4 December 2016 # Remove incorrect support for Kobo Desktop under Wine # # Version 3.2.3 October 2016 # Fix for windows network user and more xml fixes # # Version 3.2.2 October 2016 # Change to the way the new database version is handled. # # Version 3.2.1 September 2016 # Update for v4.0 of Windows Desktop app. # # Version 3.2.0 January 2016 # Update for latest version of Windows Desktop app. # Support Kobo devices in the command line version. # # Version 3.1.9 November 2015 # Handle Kobo Desktop under wine on Linux # # Version 3.1.8 November 2015 # Handle the case of Kobo Arc or Vox device (i.e. don't crash). # # Version 3.1.7 October 2015 # Handle the case of no device or database more gracefully. # # Version 3.1.6 September 2015 # Enable support for Kobo devices # More character encoding fixes (unicode strings) # # Version 3.1.5 September 2015 # Removed requirement that a purchase has been made. # Also add in character encoding fixes # # Version 3.1.4 September 2015 # Updated for version 3.17 of the Windows Desktop app. # # Version 3.1.3 August 2015 # Add translations for Portuguese and Arabic # # Version 3.1.2 January 2015 # Add coding, version number and version announcement # # Version 3.05 October 2014 # Identifies DRM-free books in the dialog # # Version 3.04 September 2014 # Handles DRM-free books as well (sometimes Kobo Library doesn't # show download link for DRM-free books) # # Version 3.03 August 2014 # If PyCrypto is unavailable try to use libcrypto for AES_ECB. # # Version 3.02 August 2014 # Relax checking of application/xhtml+xml and image/jpeg content. # # Version 3.01 June 2014 # Check image/jpeg as well as application/xhtml+xml content. Fix typo # in Windows ipconfig parsing. # # Version 3.0 June 2014 # Made portable for Mac and Windows, and the only module dependency # not part of python core is PyCrypto. Major code cleanup/rewrite. # No longer tries the first MAC address; tries them all if it detects # the decryption failed. # # Updated September 2013 by Anon # Version 2.02 # Incorporated minor fixes posted at Apprentice Alf's. # # Updates July 2012 by Michael Newton # PWSD ID is no longer a MAC address, but should always # be stored in the registry. Script now works with OS X # and checks plist for values instead of registry. Must # have biplist installed for OS X support. # # Original comments left below; note the "AUTOPSY" is inaccurate. See # KoboLibrary.userkeys and KoboFile.decrypt() # ########################################################## # KOBO DRM CRACK BY # # PHYSISTICATED # ########################################################## # This app was made for Python 2.7 on Windows 32-bit # # This app needs pycrypto - get from here: # http://www.voidspace.org.uk/python/modules.shtml # # Usage: obok.py # Choose the book you want to decrypt # # Shouts to my krew - you know who you are - and one in # particular who gave me a lot of help with this - thank # you so much! # # Kopimi /K\ # Keep sharing, keep copying, but remember that nothing is # for free - make sure you compensate your favorite # authors - and cut out the middle man whenever possible # ;) ;) ;) # # DRM AUTOPSY # The Kobo DRM was incredibly easy to crack, but it took # me months to get around to making this. Here's the # basics of how it works: # 1: Get MAC address of first NIC in ipconfig (sometimes # stored in registry as pwsdid) # 2: Get user ID (stored in tons of places, this gets it # from HKEY_CURRENT_USER\Software\Kobo\Kobo Desktop # Edition\Browser\cookies) # 3: Concatenate and SHA256, take the second half - this # is your master key # 4: Open %LOCALAPPDATA%\Kobo Desktop Editions\Kobo.sqlite # and dump content_keys # 5: Unbase64 the keys, then decode these with the master # key - these are your page keys # 6: Unzip EPUB of your choice, decrypt each page with its # page key, then zip back up again # # WHY USE THIS WHEN INEPT WORKS FINE? (adobe DRM stripper) # Inept works very well, but authors on Kobo can choose # what DRM they want to use - and some have chosen not to # let people download them with Adobe Digital Editions - # they would rather lock you into a single platform. # # With Obok, you can sync Kobo Desktop, decrypt all your # ebooks, and then use them on whatever device you want # - you bought them, you own them, you can do what you # like with them. # # Obok is Kobo backwards, but it is also means "next to" # in Polish. # When you buy a real book, it is right next to you. You # can read it at home, at work, on a train, you can lend # it to a friend, you can scribble on it, and add your own # explanations/translations. # # Obok gives you this power over your ebooks - no longer # are you restricted to one device. This allows you to # embed foreign fonts into your books, as older Kobo's # can't display them properly. You can read your books # on your phones, in different PC readers, and different # ereader devices. You can share them with your friends # too, if you like - you can do that with a real book # after all. # """Manage all Kobo books, either encrypted or DRM-free.""" __version__ = "4.1.2" __about__ = f"Obok v{__version__}\nCopyright © 2012-2020 Physisticated et al." import base64 import binascii import contextlib import hashlib import os import re import shutil import sqlite3 import subprocess import sys import tempfile import xml.etree.ElementTree as ET import zipfile can_parse_xml = True try: from xml.etree import ElementTree as ET # print "using xml.etree for xml parsing" except ImportError: can_parse_xml = False # print "Cannot find xml.etree, disabling extraction of serial numbers" # List of all known hash keys KOBO_HASH_KEYS = ["88b3a2e13", "XzUhGYdFp", "NoCanLook", "QJhwzAtXL"] class ENCRYPTIONError(Exception): pass def _load_crypto_libcrypto(): from ctypes import ( CDLL, POINTER, Structure, c_char_p, c_int, c_long, create_string_buffer, ) from ctypes.util import find_library if sys.platform.startswith("win"): libcrypto = find_library("libcrypto") else: libcrypto = find_library("crypto") if libcrypto is None: raise ENCRYPTIONError("libcrypto not found") libcrypto = CDLL(libcrypto) AES_MAXNR = 14 POINTER(c_char_p) POINTER(c_int) class AES_KEY(Structure): _fields_ = [("rd_key", c_long * (4 * (AES_MAXNR + 1))), ("rounds", c_int)] AES_KEY_p = POINTER(AES_KEY) def F(restype, name, argtypes): func = getattr(libcrypto, name) func.restype = restype func.argtypes = argtypes return func AES_set_decrypt_key = F(c_int, "AES_set_decrypt_key", [c_char_p, c_int, AES_KEY_p]) AES_ecb_encrypt = F(None, "AES_ecb_encrypt", [c_char_p, c_char_p, AES_KEY_p, c_int]) class AES: def __init__(self, userkey) -> None: self._blocksize = len(userkey) if self._blocksize not in [16, 24, 32]: raise ENCRYPTIONError(_("AES improper key used")) key = self._key = AES_KEY() rv = AES_set_decrypt_key(userkey, len(userkey) * 8, key) if rv < 0: raise ENCRYPTIONError(_("Failed to initialize AES key")) def decrypt(self, data): clear = b"" for i in range(0, len(data), 16): out = create_string_buffer(16) rv = AES_ecb_encrypt(data[i : i + 16], out, self._key, 0) if rv == 0: raise ENCRYPTIONError(_("AES decryption failed")) clear += out.raw return clear return AES def _load_crypto_pycrypto(): from Crypto.Cipher import AES as _AES class AES: def __init__(self, key) -> None: self._aes = _AES.new(key, _AES.MODE_ECB) def decrypt(self, data): return self._aes.decrypt(data) return AES def _load_crypto(): AES = None cryptolist = (_load_crypto_pycrypto, _load_crypto_libcrypto) for loader in cryptolist: with contextlib.suppress(ImportError, ENCRYPTIONError): AES = loader() break return AES AES = _load_crypto() # Wrap a stream so that output gets flushed immediately # and also make sure that any unicode strings get # encoded using "replace" before writing them. class SafeUnbuffered: def __init__(self, stream) -> None: self.stream = stream self.encoding = stream.encoding if self.encoding is None: self.encoding = "utf-8" def write(self, data): if isinstance(data, str): data = data.encode(self.encoding, "replace") self.stream.buffer.write(data) self.stream.buffer.flush() def __getattr__(self, attr): return getattr(self.stream, attr) class KoboLibrary: """The Kobo library. This class represents all the information available from the data written by the Kobo Desktop Edition application, including the list of books, their titles, and the user's encryption key(s).""" def __init__(self, serials=None, device_path=None, desktopkobodir="") -> None: if serials is None: serials = [] print(__about__) self.kobodir = "" kobodb = "" # Order of checks # 1. first check if a device_path has been passed in, and whether # we can find the sqlite db in the respective place # 2. if 1., and we got some serials passed in (from saved # settings in calibre), just use it # 3. if 1. worked, but we didn't get serials, try to parse them # from the device, if this didn't work, unset everything # 4. if by now we don't have kobodir set, give up on device and # try to use the Desktop app. # step 1. check whether this looks like a real device if device_path: # we got a device path self.kobodir = os.path.join(device_path, ".kobo") # devices use KoboReader.sqlite kobodb = os.path.join(self.kobodir, "KoboReader.sqlite") if not os.path.isfile(kobodb): # device path seems to be wrong, unset it device_path = "" self.kobodir = "" kobodb = "" # step 3. we found a device but didn't get serials, try to get them # # we got a device path but no saved serial # try to get the serial from the device # get serial from device_path/.adobe-digital-editions/device.xml if self.kobodir and len(serials) == 0 and can_parse_xml: # print "get_device_settings - device_path = {0}".format(device_path) devicexml = os.path.join( device_path, ".adobe-digital-editions", "device.xml", ) # print "trying to load {0}".format(devicexml) if os.path.exists(devicexml): # print "trying to parse {0}".format(devicexml) xmltree = ET.parse(devicexml) for node in xmltree.iter(): if "deviceSerial" in node.tag: serial = node.text # print "found serial {0}".format(serial) serials.append(serial) break else: # print "cannot get serials from device." device_path = "" self.kobodir = "" kobodb = "" if self.kobodir == "": # step 4. we haven't found a device with serials, so try desktop apps if desktopkobodir != "": self.kobodir = desktopkobodir if self.kobodir == "": if sys.platform.startswith("win"): import winreg if ( sys.getwindowsversion().major > 5 and "LOCALAPPDATA" in os.environ ): # Python 2.x does not return unicode env. Use Python 3.x self.kobodir = winreg.ExpandEnvironmentStrings("%LOCALAPPDATA%") if self.kobodir == "" and "USERPROFILE" in os.environ: # Python 2.x does not return unicode env. Use Python 3.x self.kobodir = os.path.join( winreg.ExpandEnvironmentStrings("%USERPROFILE%"), "Local Settings", "Application Data", ) self.kobodir = os.path.join( self.kobodir, "Kobo", "Kobo Desktop Edition", ) elif sys.platform.startswith("darwin"): self.kobodir = os.path.join( os.environ["HOME"], "Library", "Application Support", "Kobo", "Kobo Desktop Edition", ) elif sys.platform.startswith("linux"): # sets ~/.config/calibre as the location to store the kobodir location info file and creates this directory if necessary kobodir_cache_dir = os.path.join( os.environ["HOME"], ".config", "calibre", ) if not os.path.isdir(kobodir_cache_dir): os.mkdir(kobodir_cache_dir) # appends the name of the file we're storing the kobodir location info to the above path kobodir_cache_file = f"{str(kobodir_cache_dir)}/kobo_location" """if the above file does not exist, recursively searches from the root of the filesystem until kobodir is found and stores the location of kobodir in that file so this loop can be skipped in the future""" original_stdout = sys.stdout if not os.path.isfile(kobodir_cache_file): for root, _dirs, files in os.walk("/"): for file in files: if file == "Kobo.sqlite": kobo_linux_path = str(root) with open( kobodir_cache_file, "w", encoding="utf-8", ) as f: sys.stdout = f print(kobo_linux_path, end="") sys.stdout = original_stdout f = open(kobodir_cache_file, encoding="utf-8") self.kobodir = f.read() # desktop versions use Kobo.sqlite kobodb = os.path.join(self.kobodir, "Kobo.sqlite") # check for existence of file if not os.path.isfile(kobodb): # give up here, we haven't found anything useful self.kobodir = "" kobodb = "" if self.kobodir != "": self.bookdir = os.path.join(self.kobodir, "kepub") # make a copy of the database in a temporary file # so we can ensure it's not using WAL logging which sqlite3 can't do. self.newdb = tempfile.NamedTemporaryFile(mode="wb", delete=False) print(self.newdb.name) with open(kobodb, "rb") as olddb: self.newdb.write(olddb.read(18)) self.newdb.write(b"\x01\x01") olddb.read(2) self.newdb.write(olddb.read()) self.newdb.close() self.__sqlite = sqlite3.connect(self.newdb.name) self.__cursor = self.__sqlite.cursor() self._userkeys = [] self._books = [] self._volumeID = [] self._serials = serials def close(self): """Closes the database used by the library.""" self.__cursor.close() self.__sqlite.close() # delete the temporary copy of the database os.remove(self.newdb.name) @property def userkeys(self): """The list of potential userkeys being used by this library. Only one of these will be valid. """ if len(self._userkeys) != 0: return self._userkeys for macaddr in self.__getmacaddrs(): self._userkeys.extend(self.__getuserkeys(macaddr)) return self._userkeys @property def books(self): """The list of KoboBook objects in the library.""" if len(self._books) != 0: return self._books """Drm-ed kepub""" for row in self.__cursor.execute( "SELECT DISTINCT volumeid, Title, Attribution, Series FROM content_keys, content WHERE contentid = volumeid", ): self._books.append( KoboBook( row[0], row[1], self.__bookfile(row[0]), "kepub", self.__cursor, author=row[2], series=row[3], ), ) self._volumeID.append(row[0]) """Drm-free""" for f in os.listdir(self.bookdir): if f not in self._volumeID: row = self.__cursor.execute( "SELECT Title, Attribution, Series FROM content WHERE ContentID = '" + f + "'", ).fetchone() if row is not None: fTitle = row[0] self._books.append( KoboBook( f, fTitle, self.__bookfile(f), "drm-free", self.__cursor, author=row[1], series=row[2], ), ) self._volumeID.append(f) """Sort""" self._books.sort(key=lambda x: x.title) return self._books def __bookfile(self, volumeid): """The filename needed to open a given book.""" return os.path.join(self.kobodir, "kepub", volumeid) def __getmacaddrs(self): """The list of all MAC addresses on this machine.""" macaddrs = [] if sys.platform.startswith("win"): c = re.compile( "\\s?(" + "[0-9a-f]{2}[:\\-]" * 5 + "[0-9a-f]{2})(\\s|$)", re.IGNORECASE, ) output = subprocess.Popen( "wmic nic where PhysicalAdapter=True get MACAddress", shell=True, stdout=subprocess.PIPE, text=True, ).stdout for line in output: if m := c.search(line): macaddrs.append(re.sub("-", ":", m[1]).upper()) elif sys.platform.startswith("darwin"): c = re.compile( "\\s(" + "[0-9a-f]{2}:" * 5 + "[0-9a-f]{2})(\\s|$)", re.IGNORECASE, ) output = subprocess.check_output( "/sbin/ifconfig -a", shell=True, encoding="utf-8", ) matches = c.findall(output) macaddrs.extend(m[0].upper() for m in matches) else: # probably linux # let's try ip c = re.compile( "\\s(" + "[0-9a-f]{2}:" * 5 + "[0-9a-f]{2})(\\s|$)", re.IGNORECASE, ) for line in os.popen("ip -br link"): if m := c.search(line): macaddrs.append(m[1].upper()) # let's try ipconfig under wine c = re.compile( "\\s(" + "[0-9a-f]{2}-" * 5 + "[0-9a-f]{2})(\\s|$)", re.IGNORECASE, ) for line in os.popen("ipconfig /all"): if m := c.search(line): macaddrs.append(re.sub("-", ":", m[1]).upper()) # extend the list of macaddrs in any case with the serials # cannot hurt ;-) macaddrs.extend(self._serials) return macaddrs def __getuserids(self): userids = [] cursor = self.__cursor.execute("SELECT UserID FROM user") row = cursor.fetchone() while row is not None: with contextlib.suppress(Exception): userid = row[0] userids.append(userid) row = cursor.fetchone() return userids def __getuserkeys(self, macaddr): userids = self.__getuserids() userkeys = [] for hash in KOBO_HASH_KEYS: deviceid = hashlib.sha256((hash + macaddr).encode("ascii")).hexdigest() for userid in userids: userkey = hashlib.sha256( (deviceid + userid).encode("ascii"), ).hexdigest() userkeys.append(binascii.a2b_hex(userkey[32:])) return userkeys class KoboBook: """A Kobo book. A Kobo book contains a number of unencrypted and encrypted files. This class provides a list of the encrypted files. Each book has the following instance variables: volumeid - a UUID which uniquely refers to the book in this library. title - the human-readable book title. filename - the complete path and filename of the book. type - either kepub or drm-free""" def __init__( self, volumeid, title, filename, type, cursor, author=None, series=None, ) -> None: self.volumeid = volumeid self.title = title self.author = author self.series = series self.series_index = None self.filename = filename self.type = type self.__cursor = cursor self._encryptedfiles = {} @property def encryptedfiles(self): """A dictionary of KoboFiles inside the book. The dictionary keys are the relative pathnames, which are the same as the pathnames inside the book 'zip' file.""" if self.type == "drm-free": return self._encryptedfiles if len(self._encryptedfiles) != 0: return self._encryptedfiles # Read the list of encrypted files from the DB for row in self.__cursor.execute( "SELECT elementid,elementkey FROM content_keys,content WHERE volumeid = ? AND volumeid = contentid", (self.volumeid,), ): self._encryptedfiles[row[0]] = KoboFile( row[0], None, base64.b64decode(row[1]), ) # Read the list of files from the kepub OPF manifest so that # we can get their proper MIME type. # NOTE: this requires that the OPF file is unencrypted! zin = zipfile.ZipFile(self.filename, "r") xmlns = { "ocf": "urn:oasis:names:tc:opendocument:xmlns:container", "opf": "http://www.idpf.org/2007/opf", } ocf = ET.fromstring(zin.read("META-INF/container.xml")) opffile = ocf.find(".//ocf:rootfile", xmlns).attrib["full-path"] basedir = re.sub("[^/]+$", "", opffile) opf = ET.fromstring(zin.read(opffile)) zin.close() c = re.compile("/") for item in opf.findall(".//opf:item", xmlns): # Convert relative URIs href = item.attrib["href"] if not c.match(href): href = "".join((basedir, href)) # Update books we've found from the DB. if href in self._encryptedfiles: mimetype = item.attrib["media-type"] self._encryptedfiles[href].mimetype = mimetype return self._encryptedfiles @property def has_drm(self): return self.type != "drm-free" class KoboFile: """An encrypted file in a KoboBook. Each file has the following instance variables: filename - the relative pathname inside the book zip file. mimetype - the file's MIME type, e.g. 'image/jpeg' key - the encrypted page key.""" def __init__(self, filename, mimetype, key) -> None: self.filename = filename self.mimetype = mimetype self.key = key def decrypt(self, userkey, contents): """ Decrypt the contents using the provided user key and the file page key. The caller must determine if the decrypted data is correct.""" # The userkey decrypts the page key (self.key) keyenc = AES(userkey) decryptedkey = keyenc.decrypt(self.key) # The decrypted page key decrypts the content pageenc = AES(decryptedkey) return self.__removeaespadding(pageenc.decrypt(contents)) def check(self, contents): """ If the contents uses some known MIME types, check if it conforms to the type. Throw a ValueError exception if not. If the contents uses an uncheckable MIME type, don't check it and don't throw an exception. Returns True if the content was checked, False if it was not checked.""" if self.mimetype == "application/xhtml+xml": # assume utf-8 with no BOM textoffset = 0 stride = 1 print(f"Checking text:{contents[:10]}:") # check for byte order mark if contents[:3] == b"\xef\xbb\xbf": # seems to be utf-8 with BOM print("Could be utf-8 with BOM") textoffset = 3 elif contents[:2] == b"\xfe\xff": # seems to be utf-16BE print("Could be utf-16BE") textoffset = 3 stride = 2 elif contents[:2] == b"\xff\xfe": # seems to be utf-16LE print("Could be utf-16LE") textoffset = 2 stride = 2 else: print("Perhaps utf-8 without BOM") # now check that the first few characters are in the ASCII range for i in range(textoffset, textoffset + 5 * stride, stride): if contents[i] < 32 or contents[i] > 127: # Non-ascii, so decryption probably failed print(f"Bad character at {i}, value {contents[i]}") raise ValueError print("Seems to be good text") return True if self.mimetype == "image/jpeg": if contents[:3] == b"\xff\xd8\xff": return True print(f"Bad JPEG: {contents[:3].hex()}") raise ValueError return False def __removeaespadding(self, contents): """ Remove the trailing padding, using what appears to be the CMS algorithm from RFC 5652 6.3""" lastchar = binascii.b2a_hex(contents[-1:]) strlen = int(lastchar, 16) padding = strlen if strlen == 1: return contents[:-1] if strlen < 16: for _ in range(strlen): testchar = binascii.b2a_hex(contents[-strlen : -(strlen - 1)]) if testchar != lastchar: padding = 0 if padding > 0: contents = contents[:-padding] return contents def decrypt_book(book, lib): print(f"Converting {book.title}") zin = zipfile.ZipFile(book.filename, "r") # make filename out of Unicode alphanumeric and whitespace equivalents from title outname = "{}.epub".format(re.sub("[^\\s\\w]", "_", book.title, 0, re.UNICODE)) if book.type == "drm-free": print("DRM-free book, conversion is not needed") shutil.copyfile(book.filename, outname) print(f"Book saved as {os.path.join(os.getcwd(), outname)}") return os.path.join(os.getcwd(), outname) for userkey in lib.userkeys: print(f"Trying key: {userkey.hex()}") try: zout = zipfile.ZipFile(outname, "w", zipfile.ZIP_DEFLATED) for filename in zin.namelist(): contents = zin.read(filename) if filename in book.encryptedfiles: file = book.encryptedfiles[filename] contents = file.decrypt(userkey, contents) # Parse failures mean the key is probably wrong. file.check(contents) zout.writestr(filename, contents) zout.close() print("Decryption succeeded.") print(f"Book saved as {os.path.join(os.getcwd(), outname)}") break except ValueError: print("Decryption failed.") zout.close() os.remove(outname) zin.close() return os.path.join(os.getcwd(), outname) def cli_main(devicedir): serials = [] lib = KoboLibrary(serials, devicedir) for i, book in enumerate(lib.books): print(f"{i + 1}: {book.title}") choice = input("Convert book number... ") try: num = int(choice) books = [lib.books[num - 1]] except (ValueError, IndexError): print("Invalid choice. Exiting...") sys.exit() results = [decrypt_book(book, lib) for book in books] lib.close() return results[0] if __name__ == "__main__": sys.stdout = SafeUnbuffered(sys.stdout) sys.stderr = SafeUnbuffered(sys.stderr) sys.exit(cli_main()) ================================================ FILE: book_maker/translator/__init__.py ================================================ from book_maker.translator.caiyun_translator import Caiyun from book_maker.translator.chatgptapi_translator import ChatGPTAPI from book_maker.translator.deepl_translator import DeepL from book_maker.translator.deepl_free_translator import DeepLFree from book_maker.translator.google_translator import Google from book_maker.translator.claude_translator import Claude from book_maker.translator.gemini_translator import Gemini from book_maker.translator.groq_translator import GroqClient from book_maker.translator.tencent_transmart_translator import TencentTranSmart from book_maker.translator.custom_api_translator import CustomAPI from book_maker.translator.xai_translator import XAIClient from book_maker.translator.qwen_translator import QwenTranslator MODEL_DICT = { "openai": ChatGPTAPI, "chatgptapi": ChatGPTAPI, "gpt4": ChatGPTAPI, "gpt4omini": ChatGPTAPI, "gpt4o": ChatGPTAPI, "gpt5mini": ChatGPTAPI, "o1preview": ChatGPTAPI, "o1": ChatGPTAPI, "o1mini": ChatGPTAPI, "o3mini": ChatGPTAPI, "google": Google, "caiyun": Caiyun, "deepl": DeepL, "deeplfree": DeepLFree, "claude": Claude, "claude-sonnet-4-6": Claude, "claude-opus-4-6": Claude, "claude-opus-4-5-20251101": Claude, "claude-haiku-4-5-20251001": Claude, "claude-sonnet-4-5-20250929": Claude, "claude-opus-4-1-20250805": Claude, "claude-opus-4-20250514": Claude, "claude-sonnet-4-20250514": Claude, "gemini": Gemini, "geminipro": Gemini, "groq": GroqClient, "tencentransmart": TencentTranSmart, "customapi": CustomAPI, "xai": XAIClient, "qwen": QwenTranslator, "qwen-mt-turbo": QwenTranslator, "qwen-mt-plus": QwenTranslator, # add more here } ================================================ FILE: book_maker/translator/base_translator.py ================================================ import itertools from abc import ABC, abstractmethod class Base(ABC): def __init__(self, key, language) -> None: self.keys = itertools.cycle(key.split(",")) self.language = language @abstractmethod def rotate_key(self): pass @abstractmethod def translate(self, text): pass def set_deployment_id(self, deployment_id): pass ================================================ FILE: book_maker/translator/caiyun_translator.py ================================================ import json import re import time import requests from rich import print from .base_translator import Base class Caiyun(Base): """ caiyun translator """ def __init__(self, key, language, **kwargs) -> None: super().__init__(key, language) self.api_url = "https://api.interpreter.caiyunai.com/v1/translator" self.headers = { "content-type": "application/json", "x-authorization": f"token {key}", } # caiyun api only supports: zh2en, zh2ja, en2zh, ja2zh self.translate_type = "auto2zh" if self.language == "english": self.translate_type = "auto2en" elif self.language == "japanese": self.translate_type = "auto2ja" def rotate_key(self): pass def translate(self, text): print(text) # for caiyun translate src issue #279 text_list = text.splitlines() num = None if len(text_list) > 1: if text_list[0].isdigit(): num = text_list[0] payload = { "source": text, "trans_type": self.translate_type, "request_id": "demo", "detect": True, } response = requests.request( "POST", self.api_url, data=json.dumps(payload), headers=self.headers, ) try: t_text = response.json()["target"] except Exception as e: print(str(e), response.text, "will sleep 60s for the time limit") if "limit" in response.json()["message"]: print("will sleep 60s for the time limit") time.sleep(60) response = requests.request( "POST", self.api_url, data=json.dumps(payload), headers=self.headers, ) t_text = response.json()["target"] print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") # for issue #279 if num: t_text = str(num) + "\n" + t_text return t_text ================================================ FILE: book_maker/translator/chatgptapi_translator.py ================================================ import re import time import os import shutil from copy import copy from os import environ from itertools import cycle import json from threading import Lock from openai import AzureOpenAI, OpenAI, RateLimitError from rich import print from .base_translator import Base from ..config import config CHATGPT_CONFIG = config["translator"]["chatgptapi"] PROMPT_ENV_MAP = { "user": "BBM_CHATGPTAPI_USER_MSG_TEMPLATE", "system": "BBM_CHATGPTAPI_SYS_MSG", } GPT35_MODEL_LIST = [ "gpt-3.5-turbo", "gpt-3.5-turbo-1106", "gpt-3.5-turbo-16k", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", "gpt-3.5-turbo-0301", "gpt-3.5-turbo-0125", ] GPT4_MODEL_LIST = [ "gpt-4-1106-preview", "gpt-4", "gpt-4-32k", "gpt-4o-2024-05-13", "gpt-4-0613", "gpt-4-32k-0613", ] GPT4oMINI_MODEL_LIST = [ "gpt-4o-mini", "gpt-4o-mini-2024-07-18", ] GPT4o_MODEL_LIST = [ "gpt-4o", "gpt-4o-2024-05-13", "gpt-4o-2024-08-06", "chatgpt-4o-latest", ] GPT5MINI_MODEL_LIST = [ "gpt-5-mini", ] O1PREVIEW_MODEL_LIST = [ "o1-preview", "o1-preview-2024-09-12", ] O1_MODEL_LIST = [ "o1", "o1-2024-12-17", ] O1MINI_MODEL_LIST = [ "o1-mini", "o1-mini-2024-09-12", ] O3MINI_MODEL_LIST = [ "o3-mini", ] class ChatGPTAPI(Base): DEFAULT_PROMPT = "Please help me to translate,`{text}` to {language}, please return only translated content not include the origin text" def __init__( self, key, language, api_base=None, prompt_template=None, prompt_sys_msg=None, temperature=1.0, context_flag=False, context_paragraph_limit=0, **kwargs, ) -> None: super().__init__(key, language) self.key_len = len(key.split(",")) self.openai_client = OpenAI(api_key=next(self.keys), base_url=api_base) self.api_base = api_base self.prompt_template = ( prompt_template or environ.get(PROMPT_ENV_MAP["user"]) or self.DEFAULT_PROMPT ) self.prompt_sys_msg = ( prompt_sys_msg or environ.get( "OPENAI_API_SYS_MSG", ) # XXX: for backward compatibility, deprecate soon or environ.get(PROMPT_ENV_MAP["system"]) or "" ) self.system_content = environ.get("OPENAI_API_SYS_MSG") or "" self.deployment_id = None self.temperature = temperature self.model_list = None self.context_flag = context_flag self.context_list = [] self.context_translated_list = [] if context_paragraph_limit > 0: # not set by user, use default self.context_paragraph_limit = context_paragraph_limit else: # set by user, use user's value self.context_paragraph_limit = CHATGPT_CONFIG["context_paragraph_limit"] self.batch_text_list = [] self.batch_info_cache = None self.result_content_cache = {} self._api_lock = Lock() def rotate_key(self): with self._api_lock: self.openai_client.api_key = next(self.keys) def rotate_model(self): with self._api_lock: if self.model_list: self.model = next(self.model_list) def create_messages(self, text, intermediate_messages=None): content = self.prompt_template.format( text=text, language=self.language, crlf="\n" ) sys_content = self.system_content or self.prompt_sys_msg.format(crlf="\n") messages = [ {"role": "system", "content": sys_content}, ] if intermediate_messages: messages.extend(intermediate_messages) messages.append({"role": "user", "content": content}) return messages def create_context_messages(self): messages = [] if self.context_flag: messages.append({"role": "user", "content": "\n".join(self.context_list)}) messages.append( { "role": "assistant", "content": "\n".join(self.context_translated_list), } ) return messages def create_chat_completion(self, text): messages = self.create_messages(text, self.create_context_messages()) completion = self.openai_client.chat.completions.create( model=self.model, messages=messages, temperature=self.temperature, ) return completion def get_translation(self, text): self.rotate_key() self.rotate_model() # rotate all the model to avoid the limit completion = self.create_chat_completion(text) # TODO work well or exception finish by length limit # Check if content is not None before encoding if completion.choices[0].message.content is not None: t_text = completion.choices[0].message.content.encode("utf8").decode() or "" else: t_text = "" if self.context_flag: self.save_context(text, t_text) return t_text def save_context(self, text, t_text): if self.context_paragraph_limit > 0: self.context_list.append(text) self.context_translated_list.append(t_text) # Remove the oldest context if len(self.context_list) > self.context_paragraph_limit: self.context_list.pop(0) self.context_translated_list.pop(0) def translate(self, text, needprint=True): start_time = time.time() # todo: Determine whether to print according to the cli option if needprint: print(re.sub("\n{3,}", "\n\n", text)) attempt_count = 0 max_attempts = 3 t_text = "" while attempt_count < max_attempts: try: t_text = self.get_translation(text) break except RateLimitError as e: # todo: better sleep time? why sleep alawys about key_len # 1. openai server error or own network interruption, sleep for a fixed time # 2. an apikey has no money or reach limit, don`t sleep, just replace it with another apikey # 3. all apikey reach limit, then use current sleep sleep_time = int(60 / self.key_len) print(e, f"will sleep {sleep_time} seconds") time.sleep(sleep_time) attempt_count += 1 if attempt_count == max_attempts: print(f"Get {attempt_count} consecutive exceptions") raise except Exception as e: print(str(e)) return # todo: Determine whether to print according to the cli option if needprint: print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") time.time() - start_time # print(f"translation time: {elapsed_time:.1f}s") return t_text def translate_and_split_lines(self, text): result_str = self.translate(text, False) lines = result_str.splitlines() lines = [line.strip() for line in lines if line.strip() != ""] return lines def log_retry(self, state, retry_count, elapsed_time, log_path="log/buglog.txt"): if retry_count == 0: return print(f"retry {state}") with open(log_path, "a", encoding="utf-8") as f: print( f"retry {state}, count = {retry_count}, time = {elapsed_time:.1f}s", file=f, ) def log_translation_mismatch( self, plist_len, result_list, new_str, sep, log_path="log/buglog.txt", ): if len(result_list) == plist_len: return newlist = new_str.split(sep) with open(log_path, "a", encoding="utf-8") as f: print(f"problem size: {plist_len - len(result_list)}", file=f) for i in range(len(newlist)): print(newlist[i], file=f) print(file=f) if i < len(result_list): print("............................................", file=f) print(result_list[i], file=f) print(file=f) print("=============================", file=f) print( f"bug: {plist_len} paragraphs of text translated into {len(result_list)} paragraphs", ) print("continue") def join_lines(self, text): lines = text.splitlines() new_lines = [] temp_line = [] # join for line in lines: if line.strip(): temp_line.append(line.strip()) else: if temp_line: new_lines.append(" ".join(temp_line)) temp_line = [] new_lines.append(line) if temp_line: new_lines.append(" ".join(temp_line)) text = "\n".join(new_lines) # try to fix #372 if not text: return "" # del ^M text = text.replace("^M", "\r") lines = text.splitlines() filtered_lines = [line for line in lines if line.strip() != "\r"] new_text = "\n".join(filtered_lines) return new_text def translate_list(self, plist): plist_len = len(plist) # Create a list of original texts and add clear numbering markers to each paragraph formatted_text = "" for i, p in enumerate(plist, 1): temp_p = copy(p) for sup in temp_p.find_all("sup"): sup.extract() para_text = temp_p.get_text().strip() # Using special delimiters and clear numbering formatted_text += f"PARAGRAPH {i}:\n{para_text}\n\n" print(f"plist len = {plist_len}") original_prompt_template = self.prompt_template structured_prompt = ( f"Translate the following {plist_len} paragraphs to {{language}}. " f"CRUCIAL INSTRUCTION: Format your response using EXACTLY this structure:\n\n" f"TRANSLATION OF PARAGRAPH 1:\n[Your translation of paragraph 1 here]\n\n" f"TRANSLATION OF PARAGRAPH 2:\n[Your translation of paragraph 2 here]\n\n" f"... and so on for all {plist_len} paragraphs.\n\n" f"You MUST provide EXACTLY {plist_len} translated paragraphs. " f"Do not merge, split, or rearrange paragraphs. " f"Translate each paragraph independently but consistently. " f"Keep all numbers and special formatting in your translation. " f"Each original paragraph must correspond to exactly one translated paragraph." ) self.prompt_template = structured_prompt + " ```{text}```" translated_text = self.translate(formatted_text, False) # Extract translations from structured output translated_paragraphs = [] for i in range(1, plist_len + 1): pattern = ( r"TRANSLATION OF PARAGRAPH " + str(i) + r":(.*?)(?=TRANSLATION OF PARAGRAPH \d+:|\Z)" ) matches = re.findall(pattern, translated_text, re.DOTALL) if matches: translated_paragraph = matches[0].strip() translated_paragraphs.append(translated_paragraph) else: print(f"Warning: Could not find translation for paragraph {i}") loose_pattern = ( r"(?:TRANSLATION|PARAGRAPH|PARA).*?" + str(i) + r".*?:(.*?)(?=(?:TRANSLATION|PARAGRAPH|PARA).*?\d+.*?:|\Z)" ) loose_matches = re.findall(loose_pattern, translated_text, re.DOTALL) if loose_matches: translated_paragraphs.append(loose_matches[0].strip()) else: translated_paragraphs.append("") self.prompt_template = original_prompt_template # If the number of extracted paragraphs is incorrect, try the alternative extraction method. if len(translated_paragraphs) != plist_len: print( f"Warning: Extracted {len(translated_paragraphs)}/{plist_len} paragraphs. Using fallback extraction." ) all_para_pattern = r"(?:TRANSLATION|PARAGRAPH|PARA).*?(\d+).*?:(.*?)(?=(?:TRANSLATION|PARAGRAPH|PARA).*?\d+.*?:|\Z)" all_matches = re.findall(all_para_pattern, translated_text, re.DOTALL) if all_matches: # Create a dictionary to map translation content based on paragraph numbers para_dict = {} for num_str, content in all_matches: try: num = int(num_str) if 1 <= num <= plist_len: para_dict[num] = content.strip() except ValueError: continue # Rebuild the translation list in the original order new_translated_paragraphs = [] for i in range(1, plist_len + 1): if i in para_dict: new_translated_paragraphs.append(para_dict[i]) else: new_translated_paragraphs.append("") if len(new_translated_paragraphs) == plist_len: translated_paragraphs = new_translated_paragraphs if len(translated_paragraphs) < plist_len: translated_paragraphs.extend( [""] * (plist_len - len(translated_paragraphs)) ) elif len(translated_paragraphs) > plist_len: translated_paragraphs = translated_paragraphs[:plist_len] return translated_paragraphs def extract_paragraphs(self, text, paragraph_count): """Extract paragraphs from translated text, ensuring paragraph count is preserved.""" # First try to extract by paragraph numbers (1), (2), etc. result_list = [] for i in range(1, paragraph_count + 1): pattern = rf"\({i}\)\s*(.*?)(?=\s*\({i + 1}\)|\Z)" match = re.search(pattern, text, re.DOTALL) if match: result_list.append(match.group(1).strip()) # If exact pattern matching failed, try another approach if len(result_list) != paragraph_count: pattern = r"\((\d+)\)\s*(.*?)(?=\s*\(\d+\)|\Z)" matches = re.findall(pattern, text, re.DOTALL) if matches: # Sort by paragraph number matches.sort(key=lambda x: int(x[0])) result_list = [match[1].strip() for match in matches] # Fallback to original line-splitting approach if len(result_list) != paragraph_count: lines = text.splitlines() result_list = [line.strip() for line in lines if line.strip() != ""] return result_list def set_deployment_id(self, deployment_id): self.deployment_id = deployment_id self.openai_client = AzureOpenAI( api_key=next(self.keys), azure_endpoint=self.api_base, api_version="2023-07-01-preview", azure_deployment=self.deployment_id, ) def set_gpt35_models(self, ollama_model=""): if ollama_model: self.model_list = cycle([ollama_model]) return # gpt3 all models for save the limit if self.deployment_id: self.model_list = cycle(["gpt-35-turbo"]) else: my_model_list = [ i["id"] for i in self.openai_client.models.list().model_dump()["data"] ] model_list = list(set(my_model_list) & set(GPT35_MODEL_LIST)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) def set_gpt4_models(self): # for issue #375 azure can not use model list if self.deployment_id: self.model_list = cycle(["gpt-4"]) else: my_model_list = [ i["id"] for i in self.openai_client.models.list().model_dump()["data"] ] model_list = list(set(my_model_list) & set(GPT4_MODEL_LIST)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) def set_gpt4omini_models(self): # for issue #375 azure can not use model list if self.deployment_id: self.model_list = cycle(["gpt-4o-mini"]) else: my_model_list = [ i["id"] for i in self.openai_client.models.list().model_dump()["data"] ] model_list = list(set(my_model_list) & set(GPT4oMINI_MODEL_LIST)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) def set_gpt4o_models(self): # for issue #375 azure can not use model list if self.deployment_id: self.model_list = cycle(["gpt-4o"]) else: my_model_list = [ i["id"] for i in self.openai_client.models.list().model_dump()["data"] ] model_list = list(set(my_model_list) & set(GPT4o_MODEL_LIST)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) def set_gpt5mini_models(self): # for issue #375 azure can not use model list if self.deployment_id: self.model_list = cycle(["gpt-5-mini"]) else: my_model_list = [ i["id"] for i in self.openai_client.models.list().model_dump()["data"] ] model_list = list(set(my_model_list) & set(GPT5MINI_MODEL_LIST)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) def set_o1preview_models(self): # for issue #375 azure can not use model list if self.deployment_id: self.model_list = cycle(["o1-preview"]) else: my_model_list = [ i["id"] for i in self.openai_client.models.list().model_dump()["data"] ] model_list = list(set(my_model_list) & set(O1PREVIEW_MODEL_LIST)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) def set_o1_models(self): # for issue #375 azure can not use model list if self.deployment_id: self.model_list = cycle(["o1"]) else: my_model_list = [ i["id"] for i in self.openai_client.models.list().model_dump()["data"] ] model_list = list(set(my_model_list) & set(O1_MODEL_LIST)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) def set_o1mini_models(self): # for issue #375 azure can not use model list if self.deployment_id: self.model_list = cycle(["o1-mini"]) else: my_model_list = [ i["id"] for i in self.openai_client.models.list().model_dump()["data"] ] model_list = list(set(my_model_list) & set(O1MINI_MODEL_LIST)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) def set_o3mini_models(self): # for issue #375 azure can not use model list if self.deployment_id: self.model_list = cycle(["o3-mini"]) else: my_model_list = [ i["id"] for i in self.openai_client.models.list().model_dump()["data"] ] model_list = list(set(my_model_list) & set(O3MINI_MODEL_LIST)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) def set_model_list(self, model_list): model_list = list(set(model_list)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) def batch_init(self, book_name): self.book_name = self.sanitize_book_name(book_name) def add_to_batch_translate_queue(self, book_index, text): self.batch_text_list.append({"book_index": book_index, "text": text}) def sanitize_book_name(self, book_name): # Replace any characters that are not alphanumeric, underscore, hyphen, or dot with an underscore sanitized_book_name = re.sub(r"[^\w\-_\.]", "_", book_name) # Remove leading and trailing underscores and dots sanitized_book_name = sanitized_book_name.strip("._") return sanitized_book_name def batch_metadata_file_path(self): return os.path.join(os.getcwd(), "batch_files", f"{self.book_name}_info.json") def batch_dir(self): return os.path.join(os.getcwd(), "batch_files", self.book_name) def custom_id(self, book_index): return f"{self.book_name}-{book_index}" def is_completed_batch(self): batch_metadata_file_path = self.batch_metadata_file_path() if not os.path.exists(batch_metadata_file_path): print("Batch result file does not exist") raise Exception("Batch result file does not exist") with open(batch_metadata_file_path, "r", encoding="utf-8") as f: batch_info = json.load(f) for batch_file in batch_info["batch_files"]: batch_status = self.check_batch_status(batch_file["batch_id"]) if batch_status.status != "completed": return False return True def batch_translate(self, book_index): if self.batch_info_cache is None: batch_metadata_file_path = self.batch_metadata_file_path() with open(batch_metadata_file_path, "r", encoding="utf-8") as f: self.batch_info_cache = json.load(f) batch_info = self.batch_info_cache target_batch = None for batch in batch_info["batch_files"]: if batch["start_index"] <= book_index < batch["end_index"]: target_batch = batch break if not target_batch: raise ValueError(f"No batch found for book_index {book_index}") if target_batch["batch_id"] in self.result_content_cache: result_content = self.result_content_cache[target_batch["batch_id"]] else: batch_status = self.check_batch_status(target_batch["batch_id"]) if batch_status.output_file_id is None: raise ValueError(f"Batch {target_batch['batch_id']} is not completed") result_content = self.get_batch_result(batch_status.output_file_id) self.result_content_cache[target_batch["batch_id"]] = result_content result_lines = result_content.text.split("\n") custom_id = self.custom_id(book_index) for line in result_lines: if line.strip(): result = json.loads(line) if result["custom_id"] == custom_id: return result["response"]["body"]["choices"][0]["message"][ "content" ] raise ValueError(f"No result found for custom_id {custom_id}") def create_batch_context_messages(self, index): messages = [] if self.context_flag: if index % CHATGPT_CONFIG[ "batch_context_update_interval" ] == 0 or not hasattr(self, "cached_context_messages"): context_messages = [] for i in range(index - 1, -1, -1): item = self.batch_text_list[i] if len(item["text"].split()) >= 100: context_messages.append(item["text"]) if len(context_messages) == self.context_paragraph_limit: break if len(context_messages) == self.context_paragraph_limit: print("Creating cached context messages") self.cached_context_messages = [ {"role": "user", "content": "\n".join(context_messages)}, { "role": "assistant", "content": self.get_translation( "\n".join(context_messages) ), }, ] if hasattr(self, "cached_context_messages"): messages.extend(self.cached_context_messages) return messages def make_batch_request(self, book_index, text): messages = self.create_messages( text, self.create_batch_context_messages(book_index) ) return { "custom_id": self.custom_id(book_index), "method": "POST", "url": "/v1/chat/completions", "body": { # model shuould not be rotate "model": self.batch_model, "messages": messages, "temperature": self.temperature, }, } def create_batch_files(self, dest_file_path): file_paths = [] # max request 50,000 and max size 100MB lines_per_file = 40000 current_file = 0 for i in range(0, len(self.batch_text_list), lines_per_file): current_file += 1 file_path = os.path.join(dest_file_path, f"{current_file}.jsonl") start_index = i end_index = i + lines_per_file # TODO: Split the file if it exceeds 100MB with open(file_path, "w", encoding="utf-8") as f: for text in self.batch_text_list[i : i + lines_per_file]: batch_req = self.make_batch_request( text["book_index"], text["text"] ) json.dump(batch_req, f, ensure_ascii=False) f.write("\n") file_paths.append( { "file_path": file_path, "start_index": start_index, "end_index": end_index, } ) return file_paths def batch(self): self.rotate_model() self.batch_model = self.model # current working directory batch_dir = self.batch_dir() batch_metadata_file_path = self.batch_metadata_file_path() # cleanup batch dir and result file if os.path.exists(batch_dir): shutil.rmtree(batch_dir) if os.path.exists(batch_metadata_file_path): os.remove(batch_metadata_file_path) os.makedirs(batch_dir, exist_ok=True) # batch execute batch_files = self.create_batch_files(batch_dir) batch_info = [] for batch_file in batch_files: file_id = self.upload_batch_file(batch_file["file_path"]) batch = self.batch_execute(file_id) batch_info.append( self.create_batch_info( file_id, batch, batch_file["start_index"], batch_file["end_index"] ) ) # save batch info batch_info_json = { "book_id": self.book_name, "batch_date": time.strftime("%Y-%m-%d %H:%M:%S"), "batch_files": batch_info, } with open(batch_metadata_file_path, "w", encoding="utf-8") as f: json.dump(batch_info_json, f, ensure_ascii=False, indent=2) def create_batch_info(self, file_id, batch, start_index, end_index): return { "input_file_id": file_id, "batch_id": batch.id, "start_index": start_index, "end_index": end_index, "prefix": self.book_name, } def upload_batch_file(self, file_path): batch_input_file = self.openai_client.files.create( file=open(file_path, "rb"), purpose="batch" ) return batch_input_file.id def batch_execute(self, file_id): current_time = time.strftime("%Y-%m-%d %H:%M:%S") res = self.openai_client.batches.create( input_file_id=file_id, endpoint="/v1/chat/completions", completion_window="24h", metadata={ "description": f"Batch job for {self.book_name} at {current_time}" }, ) if res.errors: print(res.errors) raise Exception(f"Batch execution failed: {res.errors}") return res def check_batch_status(self, batch_id): return self.openai_client.batches.retrieve(batch_id) def get_batch_result(self, output_file_id): return self.openai_client.files.content(output_file_id) ================================================ FILE: book_maker/translator/claude_translator.py ================================================ import re from rich import print from anthropic import Anthropic from .base_translator import Base class Claude(Base): def __init__( self, key, language, api_base=None, prompt_template=None, prompt_sys_msg=None, temperature=1.0, context_flag=False, context_paragraph_limit=5, **kwargs, ) -> None: super().__init__(key, language) self.api_url = api_base or "https://api.anthropic.com" self.client = Anthropic(base_url=api_base, api_key=key, timeout=20) self.model = "claude-haiku-4-5-20251001" # default it for now self.language = language self.prompt_template = ( prompt_template or "Help me translate the text within triple backticks into {language} and provide only the translated result.\n```{text}```" ) self.prompt_sys_msg = prompt_sys_msg or "" self.temperature = temperature self.context_flag = context_flag self.context_list = [] self.context_translated_list = [] self.context_paragraph_limit = context_paragraph_limit def rotate_key(self): pass def set_claude_model(self, model_name): self.model = model_name def create_messages(self, text, intermediate_messages=None): """Create messages for the current translation request""" current_msg = { "role": "user", "content": self.prompt_template.format( text=text, language=self.language, ), } messages = [] if intermediate_messages: messages.extend(intermediate_messages) messages.append(current_msg) return messages def create_context_messages(self): """Create a message pair containing all context paragraphs""" if not self.context_flag or not self.context_list: return [] # Create a single message pair for all previous context return [ { "role": "user", "content": self.prompt_template.format( text="\n\n".join(self.context_list), language=self.language, ), }, {"role": "assistant", "content": "\n\n".join(self.context_translated_list)}, ] def save_context(self, text, t_text): """Save the current translation pair to context""" if not self.context_flag: return self.context_list.append(text) self.context_translated_list.append(t_text) # Keep only the most recent paragraphs within the limit if len(self.context_list) > self.context_paragraph_limit: self.context_list.pop(0) self.context_translated_list.pop(0) def translate(self, text): print(text) self.rotate_key() # Create messages with context messages = self.create_messages(text, self.create_context_messages()) r = self.client.messages.create( max_tokens=4096, messages=messages, system=self.prompt_sys_msg, temperature=self.temperature, model=self.model, ) t_text = r.content[0].text if self.context_flag: self.save_context(text, t_text) print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") return t_text ================================================ FILE: book_maker/translator/custom_api_translator.py ================================================ from .base_translator import Base import re import json import requests import time from rich import print class CustomAPI(Base): """ Custom API translator """ def __init__(self, custom_api, language, **kwargs) -> None: super().__init__(custom_api, language) self.language = language self.custom_api = custom_api def rotate_key(self): pass def translate(self, text): print(text) custom_api = self.custom_api data = {"text": text, "source_lang": "auto", "target_lang": self.language} post_data = json.dumps(data) r = requests.post(url=custom_api, data=post_data, timeout=10).text t_text = json.loads(r)["data"] print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") time.sleep(5) return t_text ================================================ FILE: book_maker/translator/deepl_free_translator.py ================================================ import time import random import re from book_maker.utils import LANGUAGES, TO_LANGUAGE_CODE from .base_translator import Base from rich import print from PyDeepLX import PyDeepLX class DeepLFree(Base): """ DeepL free translator """ def __init__(self, key, language, **kwargs) -> None: super().__init__(key, language) l = language if language in LANGUAGES else TO_LANGUAGE_CODE.get(language) if l not in [ "bg", "zh", "cs", "da", "nl", "en-US", "en-GB", "et", "fi", "fr", "de", "el", "hu", "id", "it", "ja", "lv", "lt", "pl", "pt-PT", "pt-BR", "ro", "ru", "sk", "sl", "es", "sv", "tr", "uk", "ko", "nb", ]: raise Exception(f"DeepL do not support {l}") self.language = l self.time_random = [0.3, 0.5, 1, 1.3, 1.5, 2] def rotate_key(self): pass def translate(self, text): print(text) t_text = str(PyDeepLX.translate(text, "EN", self.language)) # spider rule time.sleep(random.choice(self.time_random)) print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") return t_text ================================================ FILE: book_maker/translator/deepl_translator.py ================================================ import json import time import requests import re from book_maker.utils import LANGUAGES, TO_LANGUAGE_CODE from .base_translator import Base from rich import print class DeepL(Base): """ DeepL translator """ def __init__(self, key, language, **kwargs) -> None: super().__init__(key, language) self.api_url = "https://dpl-translator.p.rapidapi.com/translate" self.headers = { "content-type": "application/json", "X-RapidAPI-Key": "", "X-RapidAPI-Host": "dpl-translator.p.rapidapi.com", } l = language if language in LANGUAGES else TO_LANGUAGE_CODE.get(language) if l not in [ "bg", "zh", "cs", "da", "nl", "en-US", "en-GB", "et", "fi", "fr", "de", "el", "hu", "id", "it", "ja", "lv", "lt", "pl", "pt-PT", "pt-BR", "ro", "ru", "sk", "sl", "es", "sv", "tr", "uk", "ko", "nb", ]: raise Exception(f"DeepL do not support {l}") self.language = l def rotate_key(self): self.headers["X-RapidAPI-Key"] = f"{next(self.keys)}" def translate(self, text): self.rotate_key() print(text) payload = {"text": text, "source": "EN", "target": self.language} try: response = requests.request( "POST", self.api_url, data=json.dumps(payload), headers=self.headers, ) except Exception as e: print(e) time.sleep(30) response = requests.request( "POST", self.api_url, data=json.dumps(payload), headers=self.headers, ) t_text = response.json().get("text", "") print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") return t_text ================================================ FILE: book_maker/translator/gemini_translator.py ================================================ import re import time from os import environ from itertools import cycle import google.generativeai as genai from google.generativeai.types.generation_types import ( StopCandidateException, BlockedPromptException, ) from rich import print from .base_translator import Base generation_config = { "temperature": 1.0, "top_p": 1, "top_k": 1, "max_output_tokens": 8192, } safety_settings = { "HATE": "BLOCK_NONE", "HARASSMENT": "BLOCK_NONE", "SEXUAL": "BLOCK_NONE", "DANGEROUS": "BLOCK_NONE", } PROMPT_ENV_MAP = { "user": "BBM_GEMINIAPI_USER_MSG_TEMPLATE", "system": "BBM_GEMINIAPI_SYS_MSG", } GEMINIPRO_MODEL_LIST = [ "gemini-1.5-pro", "gemini-1.5-pro-latest", "gemini-1.5-pro-001", "gemini-1.5-pro-002", ] GEMINIFLASH_MODEL_LIST = [ "gemini-1.5-flash", "gemini-1.5-flash-latest", "gemini-1.5-flash-001", "gemini-1.5-flash-002", "gemini-2.0-flash-exp", "gemini-2.5-flash-preview-04-17", ] class Gemini(Base): """ Google gemini translator """ DEFAULT_PROMPT = "Please help me to translate,`{text}` to {language}, please return only translated content not include the origin text" def __init__( self, key, language, prompt_template=None, prompt_sys_msg=None, context_flag=False, temperature=1.0, **kwargs, ) -> None: super().__init__(key, language) self.context_flag = context_flag self.prompt = ( prompt_template or environ.get(PROMPT_ENV_MAP["user"]) or self.DEFAULT_PROMPT ) self.prompt_sys_msg = ( prompt_sys_msg or environ.get(PROMPT_ENV_MAP["system"]) or None # Allow None, but not empty string ) self.interval = 3 genai.configure(api_key=next(self.keys)) generation_config["temperature"] = temperature def create_convo(self): model = genai.GenerativeModel( model_name=self.model, generation_config=generation_config, safety_settings=safety_settings, system_instruction=self.prompt_sys_msg, ) self.convo = model.start_chat() # print(model) # Uncomment to debug and inspect the model details. def rotate_model(self): self.model = next(self.model_list) self.create_convo() print(f"Using model {self.model}") def rotate_key(self): genai.configure(api_key=next(self.keys)) self.create_convo() def translate(self, text): delay = 1 exponential_base = 2 attempt_count = 0 max_attempts = 7 t_text = "" print(text) # same for caiyun translate src issue #279 gemini for #374 text_list = text.splitlines() num = None if len(text_list) > 1: if text_list[0].isdigit(): num = text_list[0] while attempt_count < max_attempts: try: self.convo.send_message( self.prompt.format(text=text, language=self.language) ) t_text = self.convo.last.text.strip() # 检查是否包含特定标签,如果有则只返回标签内的内容 tag_pattern = ( r"(.*?)" ) tag_match = re.search(tag_pattern, t_text, re.DOTALL) if tag_match: print( "[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]" ) t_text = tag_match.group(1).strip() # print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") break except StopCandidateException as e: print( f"Translation failed due to StopCandidateException: {e} Attempting to switch model..." ) self.rotate_model() except BlockedPromptException as e: print( f"Translation failed due to BlockedPromptException: {e} Attempting to switch model..." ) self.rotate_model() except Exception as e: print( f"Translation failed due to {type(e).__name__}: {e} Will sleep {delay} seconds" ) time.sleep(delay) delay *= exponential_base self.rotate_key() if attempt_count >= 1: self.rotate_model() attempt_count += 1 if attempt_count == max_attempts: print(f"Translation failed after {max_attempts} attempts.") return if self.context_flag: if len(self.convo.history) > 10: self.convo.history = self.convo.history[2:] else: self.convo.history = [] print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") # for rate limit(RPM) time.sleep(self.interval) if num: t_text = str(num) + "\n" + t_text return t_text def set_interval(self, interval): self.interval = interval def set_geminipro_models(self): self.set_models(GEMINIPRO_MODEL_LIST) def set_geminiflash_models(self): self.set_models(GEMINIFLASH_MODEL_LIST) def set_models(self, allowed_models): available_models = [ re.sub(r"^models/", "", i.name) for i in genai.list_models() ] model_list = sorted( list(set(available_models) & set(allowed_models)), key=allowed_models.index, ) print(f"Using model list {model_list}") self.model_list = cycle(model_list) self.rotate_model() def set_model_list(self, model_list): # keep the order of input model_list = sorted(list(set(model_list)), key=model_list.index) print(f"Using model list {model_list}") self.model_list = cycle(model_list) self.rotate_model() ================================================ FILE: book_maker/translator/google_translator.py ================================================ import re import requests from rich import print from book_maker.utils import TO_LANGUAGE_CODE from .base_translator import Base class Google(Base): """ google translate """ def __init__(self, key, language, **kwargs) -> None: super().__init__(key, language) # Convert language name to code if needed, otherwise use as-is if language.lower() in TO_LANGUAGE_CODE: language_code = TO_LANGUAGE_CODE[language.lower()] else: language_code = language self.api_url = f"https://translate.google.com/translate_a/single?client=it&dt=qca&dt=t&dt=rmt&dt=bd&dt=rms&dt=sos&dt=md&dt=gt&dt=ld&dt=ss&dt=ex&otf=2&dj=1&hl=en&ie=UTF-8&oe=UTF-8&sl=auto&tl={language_code}" self.headers = { "Content-Type": "application/x-www-form-urlencoded", "User-Agent": "GoogleTranslate/6.29.59279 (iPhone; iOS 15.4; en; iPhone14,2)", } # TODO support more models here self.session = requests.session() self.language = language def rotate_key(self): pass def translate(self, text): print(text) """r = self.session.post( self.api_url, headers=self.headers, data=f"q={requests.utils.quote(text)}", ) if not r.ok: return text t_text = "".join( [sentence.get("trans", "") for sentence in r.json()["sentences"]], )""" t_text = self._retry_translate(text) print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") return t_text def _retry_translate(self, text, timeout=3): time = 0 while time <= timeout: time += 1 r = self.session.post( self.api_url, headers=self.headers, data=f"q={requests.utils.quote(text)}", timeout=3, ) if r.ok: t_text = "".join( [sentence.get("trans", "") for sentence in r.json()["sentences"]], ) return t_text return text ================================================ FILE: book_maker/translator/groq_translator.py ================================================ from groq import Groq from .chatgptapi_translator import ChatGPTAPI from os import linesep from itertools import cycle GROQ_MODEL_LIST = [ "llama3-8b-8192", "llama3-70b-8192", "mixtral-8x7b-32768", "gemma-7b-it", ] class GroqClient(ChatGPTAPI): def rotate_model(self): if not self.model_list: model_list = list(set(GROQ_MODEL_LIST)) print(f"Using model list {model_list}") self.model_list = cycle(model_list) self.model = next(self.model_list) def create_chat_completion(self, text): self.groq_client = Groq(api_key=next(self.keys)) content = f"{self.prompt_template.format(text=text, language=self.language, crlf=linesep)}" sys_content = self.system_content or self.prompt_sys_msg.format(crlf="\n") messages = [ {"role": "system", "content": sys_content}, {"role": "user", "content": content}, ] if self.deployment_id: return self.groq_client.chat.completions.create( engine=self.deployment_id, messages=messages, temperature=self.temperature, azure=True, ) return self.groq_client.chat.completions.create( model=self.model, messages=messages, temperature=self.temperature, ) ================================================ FILE: book_maker/translator/litellm_translator.py ================================================ from os import linesep from litellm import completion from book_maker.translator.chatgptapi_translator import ChatGPTAPI PROMPT_ENV_MAP = { "user": "BBM_CHATGPTAPI_USER_MSG_TEMPLATE", "system": "BBM_CHATGPTAPI_SYS_MSG", } class liteLLM(ChatGPTAPI): def create_chat_completion(self, text): # content = self.prompt_template.format( # text=text, language=self.language, crlf="\n" # ) content = f"{self.context if self.context_flag else ''} {self.prompt_template.format(text=text, language=self.language, crlf=linesep)}" sys_content = self.system_content or self.prompt_sys_msg.format(crlf="\n") context_sys_str = "For each passage given, you may be provided a summary of the story up until this point (wrapped in tags '' and '') for context within the query, to provide background context of the story up until this point. If it's provided, use the context summary to aid you in translation with deeper comprehension, and write a new summary above the returned translation, wrapped in '' HTML-like tags, including important details (if relevant) from the new passage, retaining the most important key details from the existing summary, and dropping out less important details. If the summary is blank, assume it is the start of the story and write a summary from scratch. Do not make the summary longer than a paragraph, and smaller details can be replaced based on the relative importance of new details. The summary should be formatted in straightforward, inornate text, briefly summarising the entire story (from the start, including information before the given passage, leading up to the given passage) to act as an instructional payload for a Large-Language AI Model to fully understand the context of the passage." sys_content = f"{self.system_content or self.prompt_sys_msg.format(crlf=linesep)} {context_sys_str if self.context_flag else ''} " messages = [ {"role": "system", "content": sys_content}, {"role": "user", "content": content}, ] if self.deployment_id: return completion( engine=self.deployment_id, messages=messages, temperature=self.temperature, azure=True, ) return completion( model="gpt-3.5-turbo", messages=messages, temperature=self.temperature, ) ================================================ FILE: book_maker/translator/qwen_translator.py ================================================ import re import time from rich import print from openai import OpenAI from .base_translator import Base class QwenTranslator(Base): """ Qwen-MT translator using Alibaba Cloud's DashScope API Specialized translation model supporting 92 languages with advanced features Official documentation: https://help.aliyun.com/document_detail/2860790.html Todo: support more languages, terminology, and domain hints """ # Language mapping from bilingual_book_maker format to Qwen language codes LANGUAGE_MAP = { # Common languages "english": "English", "chinese": "Chinese", "simplified chinese": "Chinese", "traditional chinese": "Traditional Chinese", "japanese": "Japanese", "korean": "Korean", "spanish": "Spanish", "french": "French", "german": "German", "portuguese": "Portuguese", "italian": "Italian", "russian": "Russian", "arabic": "Arabic", "hindi": "Hindi", "thai": "Thai", "vietnamese": "Vietnamese", "indonesian": "Indonesian", "malay": "Malay", "dutch": "Dutch", "turkish": "Turkish", "polish": "Polish", "czech": "Czech", "hungarian": "Hungarian", "romanian": "Romanian", "greek": "Greek", "hebrew": "Hebrew", "finnish": "Finnish", "danish": "Danish", "swedish": "Swedish", "norwegian": "Norwegian Bokmål", "ukrainian": "Ukrainian", "bulgarian": "Bulgarian", "serbian": "Serbian", "croatian": "Croatian", "slovenian": "Slovenian", "slovak": "Slovak", "lithuanian": "Lithuanian", "latvian": "Latvian", "estonian": "Estonian", # Add more mappings as needed } def __init__( self, key, language, model="qwen-mt-turbo", source_lang="auto", api_base=None, prompt_template=None, # Not used for translation models prompt_sys_msg=None, # Not used for translation models temperature=None, # Not used for translation models context_flag=False, context_paragraph_limit=5, terminology=None, domain_hint=None, **kwargs, ) -> None: super().__init__(key, language) # API configuration self.api_base = api_base or "https://dashscope.aliyuncs.com/compatible-mode/v1" self.client = OpenAI( api_key=next(self.keys), base_url=self.api_base, timeout=60 ) # Model configuration self.model = self.set_qwen_model(model) self.source_lang = source_lang self.target_lang = self._map_language(language) # Advanced features self.terminology = self.set_terminology(terminology) self.domain_hint = self.set_domain_hint(domain_hint) # Context/Translation memory support self.context_flag = context_flag self.context_list = [] self.context_translated_list = [] self.context_paragraph_limit = context_paragraph_limit print("[bold blue]Qwen Translator initialized:[/bold blue]") print(f" Model: {self.model}") print(f" Source Language: {self.source_lang}") print(f" Target Language: {self.target_lang}") if self.domain_hint: print(f" Domain Hint: {self.domain_hint}") def rotate_key(self): """Rotate API key for load balancing""" try: self.client.api_key = next(self.keys) except StopIteration: pass def _map_language(self, language): """Map language name to Qwen language format""" language_lower = language.lower().strip() # Direct mapping if language_lower in self.LANGUAGE_MAP: return self.LANGUAGE_MAP[language_lower] # Try partial matching for common variations for key, value in self.LANGUAGE_MAP.items(): if language_lower in key or key in language_lower: return value # Fallback to original language name with proper capitalization return language.title() def _create_translation_options(self): """Create translation options for the API request""" options = {"source_lang": self.source_lang, "target_lang": self.target_lang} # Add terminology if provided if self.terminology and len(self.terminology) > 0: options["terms"] = self.terminology # Add domain hint if provided (must be in English) if self.domain_hint and len(self.domain_hint) > 0: options["domains"] = self.domain_hint # Add translation memory if context is enabled if self.context_flag and self.context_list: tm_list = [] for src, tgt in zip(self.context_list, self.context_translated_list): tm_list.append({"source": src, "target": tgt}) if tm_list: options["tm_list"] = tm_list return options def save_context(self, text, t_text): """Save the current translation pair to context for translation memory""" if not self.context_flag: return self.context_list.append(text) self.context_translated_list.append(t_text) # Keep only the most recent paragraphs within the limit if len(self.context_list) > self.context_paragraph_limit: self.context_list.pop(0) self.context_translated_list.pop(0) def translate(self, text, needprint=True): """Main translation method""" start_time = time.time() if needprint: print(re.sub(r"\n{3,}", "\n\n", text)) attempt_count = 0 max_attempts = 3 t_text = "" while attempt_count < max_attempts: try: self.rotate_key() # Prepare messages messages = [{"role": "user", "content": text}] # Create translation options translation_options = self._create_translation_options() # Make API request completion = self.client.chat.completions.create( model=self.model, messages=messages, extra_body={"translation_options": translation_options}, ) # Extract translated text if completion.choices[0].message.content: t_text = completion.choices[0].message.content.strip() else: t_text = "" # Save to context for translation memory if self.context_flag and t_text: self.save_context(text, t_text) break except Exception as e: attempt_count += 1 print( f"[red]Translation attempt {attempt_count} failed: {str(e)}[/red]" ) if attempt_count >= max_attempts: print( f"[red]Translation failed after {max_attempts} attempts[/red]" ) t_text = text # Fallback to original text else: time.sleep(1) # Wait before retry if needprint: print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") end_time = time.time() print(f"[dim]Translation time: {end_time - start_time:.2f}s[/dim]") return t_text def set_terminology(self, terminology): """Set custom terminology for translation Args: terminology: List of dict with 'source' and 'target' keys e.g., [{"source": "API", "target": "应用程序接口"}] """ self.terminology = terminology or [] print(f"[blue]Terminology updated: {len(self.terminology)} terms[/blue]") def set_domain_hint(self, domain_hint): """Set domain hint for specialized translation Args: domain_hint: String describing the domain in English e.g., "Technical documentation for software development" """ self.domain_hint = domain_hint or "" print(f"[blue]Domain hint set: {self.domain_hint}[/blue]") def set_qwen_model(self, model_name): """Set Qwen model type Args: model_name: Either "qwen-mt-turbo" or "qwen-mt-plus" """ if model_name in ["qwen-mt-turbo", "qwen-mt-plus"]: self.model = model_name print(f"[blue]Qwen model set to: {self.model}[/blue]") else: self.model = "qwen-mt-turbo" print( f"[red]Invalid Qwen model: {model_name}. Using default: {self.model}[/red]" ) ================================================ FILE: book_maker/translator/tencent_transmart_translator.py ================================================ import re import time import uuid import requests from rich import print from .base_translator import Base class TencentTranSmart(Base): """ Tencent TranSmart translator """ def __init__(self, key, language, **kwargs) -> None: super().__init__(key, language) self.api_url = "https://transmart.qq.com/api/imt" self.header = { "authority": "transmart.qq.com", "content-type": "application/json", "origin": "https://transmart.qq.com", "referer": "https://transmart.qq.com/zh-CN/index", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36", } self.uuid = str(uuid.uuid4()) self.session = requests.Session() self.translate_type = "zh" if self.language == "english": self.translate_type = "en" def rotate_key(self): pass def translate(self, text): print(text) source_language, text_list = self.text_analysis(text) client_key = self.get_client_key() api_form_data = { "header": { "fn": "auto_translation", "client_key": client_key, }, "type": "plain", "model_category": "normal", "source": { "lang": source_language, "text_list": [""] + text_list + [""], }, "target": {"lang": self.translate_type}, } response = self.session.post( self.api_url, json=api_form_data, headers=self.header, timeout=3 ) t_text = "".join(response.json()["auto_translation"]) print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") return t_text def text_analysis(self, text): client_key = self.get_client_key() self.header.update({"Cookie": "TSMT_CLIENT_KEY={}".format(client_key)}) analysis_request_data = { "header": { "fn": "text_analysis", "session": "", "client_key": client_key, "user": "", }, "text": text, "type": "plain", "normalize": {"merge_broken_line": "false"}, } r = self.session.post( self.api_url, json=analysis_request_data, headers=self.header ) if not r.ok: return text response_json_data = r.json() text_list = [item["tgt_str"] for item in response_json_data["sentence_list"]] language = response_json_data["language"] return language, text_list def get_client_key(self): return "browser-chrome-121.0.0-Windows_10-{}-{}".format( self.uuid, int(time.time() * 1e3) ) ================================================ FILE: book_maker/translator/xai_translator.py ================================================ from openai import OpenAI from .chatgptapi_translator import ChatGPTAPI XAI_MODEL_LIST = [ "grok-beta", ] class XAIClient(ChatGPTAPI): def __init__(self, key, language, api_base=None, **kwargs) -> None: super().__init__(key, language) self.model_list = XAI_MODEL_LIST self.api_url = str(api_base) if api_base else "https://api.x.ai/v1" self.openai_client = OpenAI(api_key=key, base_url=self.api_url) def rotate_model(self): self.model = self.model_list[0] ================================================ FILE: book_maker/utils.py ================================================ import tiktoken # Borrowed from : https://github.com/openai/whisper LANGUAGES = { "en": "english", "zh-hans": "simplified chinese", "zh": "simplified chinese", "zh-hant": "traditional chinese", "zh-yue": "cantonese", "de": "german", "es": "spanish", "ru": "russian", "ko": "korean", "fr": "french", "ja": "japanese", "pt": "portuguese", "tr": "turkish", "pl": "polish", "ca": "catalan", "nl": "dutch", "ar": "arabic", "sv": "swedish", "it": "italian", "id": "indonesian", "hi": "hindi", "fi": "finnish", "vi": "vietnamese", "he": "hebrew", "uk": "ukrainian", "el": "greek", "ms": "malay", "cs": "czech", "ro": "romanian", "da": "danish", "hu": "hungarian", "ta": "tamil", "no": "norwegian", "th": "thai", "ur": "urdu", "hr": "croatian", "bg": "bulgarian", "lt": "lithuanian", "la": "latin", "mi": "maori", "ml": "malayalam", "cy": "welsh", "sk": "slovak", "te": "telugu", "fa": "persian", "lv": "latvian", "bn": "bengali", "sr": "serbian", "az": "azerbaijani", "sl": "slovenian", "kn": "kannada", "et": "estonian", "mk": "macedonian", "br": "breton", "eu": "basque", "is": "icelandic", "hy": "armenian", "ne": "nepali", "mn": "mongolian", "bs": "bosnian", "kk": "kazakh", "sq": "albanian", "sw": "swahili", "gl": "galician", "mr": "marathi", "pa": "punjabi", "si": "sinhala", "km": "khmer", "sn": "shona", "yo": "yoruba", "so": "somali", "af": "afrikaans", "oc": "occitan", "ka": "georgian", "be": "belarusian", "tg": "tajik", "sd": "sindhi", "gu": "gujarati", "am": "amharic", "yi": "yiddish", "lo": "lao", "uz": "uzbek", "fo": "faroese", "ht": "haitian creole", "ps": "pashto", "tk": "turkmen", "nn": "nynorsk", "mt": "maltese", "sa": "sanskrit", "lb": "luxembourgish", "my": "myanmar", "bo": "tibetan", "tl": "tagalog", "mg": "malagasy", "as": "assamese", "tt": "tatar", "haw": "hawaiian", "ln": "lingala", "ha": "hausa", "ba": "bashkir", "jw": "javanese", "su": "sundanese", } # language code lookup by name, with a few language aliases TO_LANGUAGE_CODE = { **{language: code for code, language in LANGUAGES.items()}, "burmese": "my", "valencian": "ca", "flemish": "nl", "haitian": "ht", "letzeburgesch": "lb", "pushto": "ps", "panjabi": "pa", "moldavian": "ro", "moldovan": "ro", "sinhalese": "si", "castilian": "es", } def prompt_config_to_kwargs(prompt_config): prompt_config = prompt_config or {} return dict( prompt_template=prompt_config.get("user", None), prompt_sys_msg=prompt_config.get("system", None), ) # ref: https://platform.openai.com/docs/guides/chat/introduction def num_tokens_from_text(text, model="gpt-3.5-turbo-0301"): messages = ( { "role": "user", "content": text, }, ) """Returns the number of tokens used by a list of messages.""" try: encoding = tiktoken.encoding_for_model(model) except KeyError: encoding = tiktoken.get_encoding("cl100k_base") if model == "gpt-3.5-turbo-0301": # note: future models may deviate from this num_tokens = 0 for message in messages: num_tokens += ( 4 # every message follows {role/name}\n{content}\n ) for key, value in message.items(): num_tokens += len(encoding.encode(value)) if key == "name": # if there's a name, the role is omitted num_tokens += -1 # role is always required and always 1 token num_tokens += 2 # every reply is primed with assistant return num_tokens else: raise NotImplementedError( f"""num_tokens_from_messages() is not presently implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""" ) ================================================ FILE: disclaimer.md ================================================ Disclaimer: 1. The purpose of this project, bilingual_book_maker, is to assist users in creating multilingual versions of epub files and books. It is only applicable to books that have entered the public domain and is not intended for use with copyrighted material. We strongly advise users to read the copyright information carefully before using this project and to comply with relevant laws and regulations in order to protect their own and others' rights. 2. In no event shall the authors or developers be liable for any loss or damage caused by the use of this project. Users assume all risks associated with the use of this project. Users must confirm that they have obtained permission from the original copyright holder or used open source EPUB files before using this project to avoid potential copyright risks. If you have any concerns or suggestions about the use of this project, please contact us through the issues section. 免责声明： 1. 该项目设计目的是为了帮助用户制作多语言版本的epub文件和图书，仅适用于进入公共版权领域书籍，不适用于有版权的书籍。我们强烈建议用户在使用该项目时仔细阅读其版权信息并遵守相关法律和规定，以保护自己和他人的权益。 2. 在任何情况下，作者和开发者不对因使用该项目而导致的任何损失或损害承担任何责任。使用该项目的风险由用户自行承担。用户必须在使用该项目之前，确认其已获得了原著作权人的许可或使用了公开可用的开源EPUB文件，以避免可能存在的版权风险。 如果您对该项目的使用有任何疑虑或建议，请通过 issues 与我们联系。 ================================================ FILE: docs/book_source.md ================================================ # Translate from Different Sources ## txt/srt Txt files and srt files are plain text files. This program can translate plain text. python3 make_book.py --book_name test_books/the_little_prince.txt --test --language zh-hans ## epub epub is made of html files. By default, we only translate contents in `

`. Use `--translate-tags` to specify tags need for translation. Use comma to separate multiple tags. For example: `--translate-tags h1,h2,h3,p,div` bbook_maker --book_name test_books/animal_farm.epub --openai_key ${openai_key} --translate-tags div,p If you want to translate strings in an e-book that aren't labeled with any tags, you can use the `--allow_navigable_strings` parameter. This will add the strings to the translation queue.
**Note that it's best to look for e-books that are more standardized if possible.** ## e-reader Use `--book_from` option to specify e-reader type (Now only `kobo` is available), and use `--device_path` to specify the mounting point. # Translate books download from Rakuten Kobo on kobo e-reader bbook_maker --book_from kobo --device_path /tmp/kobo ================================================ FILE: docs/cmd.md ================================================ # Command Line Options ## Test translate `--test`
Use this option to preview the result if you haven't paid for the service or just want to test. Note that there is a limit and it may take some time. ```sh bbook_maker --book_name test_books/Lex_Fridman_episode_322.srt --openai_key ${openai_key} --test ``` ```sh bbook_maker --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test --language zh-hans ``` `--test_num `
Use this option to set how many paragraph you want to translate for testing. Default is 10. ## Resume `--resume`
Use this option to manually resume the process after an interruption. ## Retranslate (epub only) `--retranslate `
If a file in epub is not translated well, it supports to re-translate part of epub separately. This option take 4 arguments: `translated_filepath`, `file_name_in_epub`, `start_str`, `end_str`. `end_str` is optional. - Retranslate from start_str to end_str's tag: bbook_maker --book_name "test_books/animal_farm.epub" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' 'This kind of thing is not a good symptom. Obviously' - Retranslate start_str's tag: bbook_maker --book_name "test_books/animal_farm.epub" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' - Retranslate start_str's tag, auto find filename: bbook_maker --book_name "test_books/animal_farm.epub" --retranslate 'test_books/animal_farm_bilingual.epub' '' 'in spite of the present book shortage which' **Warning:** **It deletes from the tag at start_str of the finished book to the next tag at end_str, and then re-translates.** **Therefore, please make sure that the next tag of end_str is the translated content. (If end_str is not provided, the next label of start_str is guaranteed to be the translated content.) There can be missing translations between the two strings, but if end_str is not translated, there will be problems.** ## Customize output style (epub only) `--translation_style `
Support changing the output style of epub files. bbook_maker --book_name test_books/animal_farm.epub --translation_style "color: #4a4a4a; font-style: normal; background-color: #f7f7f7; padding: 5px; margin: 10px 0; border-radius: 5px;" ![output_style](https://user-images.githubusercontent.com/89069008/226104545-7c029bb1-5325-46d4-a1eb-ec4e7bbaee97.png) ## Proxy `--proxy `
Use this option to specify proxy server for internet access. Enter a string such as `http://127.0.0.1:7890` . ## API base `--api_base `
If you want to change api_base like using Cloudflare Workers, use this option to support it.
bbook_maker --book_name 'animal_farm.epub' --openai_key sk-XXXXX --api_base 'https://xxxxx/v1' **Note: the api url should be '`https://xxxx/v1`'. Quotation marks are required.** ## Microsoft Azure Endpoints `--api_base ` `--deployment_id `
You can use the api endpoint provided from Microsoft. bbook_maker --book_name 'animal_farm.epub' --openai_key XXXXX --api_base 'https://example-endpoint.openai.azure.com' --deployment_id 'deployment-name' **Note : Current only support chatgptapi model for deployment_id. And `api_base` must be provided when using `deployment_id`. You can check [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=web-portal) for more information about `deployment_id`.** ## Batch size (txt only) `--batch_size`
Use this parameter to specify the number of lines for batch translation. Default is 10. (Currently only effective for txt files). ```sh python3 make_book.py --book_name test_books/the_little_prince.txt --test --batch_size 20 ``` ## Accumulated Num `--accumulated_num `
Wait for how many tokens have been accumulated before starting the translation. gpt3.5 limits the total_token to 4090. For example, if you use --accumulated_num 1600, maybe openai will output 2200 tokens and maybe 200 tokens for other messages in the system messages user messages. 1600+2200+200=4000, so you are close to the limit. You have to choose your own value, there is no way to tell if the limit is reached before sending request. ================================================ FILE: docs/disclaimer.md ================================================ Disclaimer: 1. The purpose of this project, bilingual_book_maker, is to assist users in creating multilingual versions of epub files and books. It is only applicable to books that have entered the public domain and is not intended for use with copyrighted material. We strongly advise users to read the copyright information carefully before using this project and to comply with relevant laws and regulations in order to protect their own and others' rights. 2. In no event shall the authors or developers be liable for any loss or damage caused by the use of this project. Users assume all risks associated with the use of this project. Users must confirm that they have obtained permission from the original copyright holder or used open source EPUB files before using this project to avoid potential copyright risks. If you have any concerns or suggestions about the use of this project, please contact us through the issues section. 免责声明： 1. 该项目设计目的是为了帮助用户制作多语言版本的epub文件和图书，仅适用于进入公共版权领域书籍，不适用于有版权的书籍。我们强烈建议用户在使用该项目时仔细阅读其版权信息并遵守相关法律和规定，以保护自己和他人的权益。 2. 在任何情况下，作者和开发者不对因使用该项目而导致的任何损失或损害承担任何责任。使用该项目的风险由用户自行承担。用户必须在使用该项目之前，确认其已获得了原著作权人的许可或使用了公开可用的开源EPUB文件，以避免可能存在的版权风险。 如果您对该项目的使用有任何疑虑或建议，请通过 issues 与我们联系。 ================================================ FILE: docs/env_settings.md ================================================ # Environment Settings You can also write information into env to skip some options. ## Model keys ``` # Set env BBM_OPENAI_API_KEY to ignore option --openai_key export BBM_OPENAI_API_KEY=${your_api_key} # Set env BBM_CAIYUN_API_KEY to ignore option --caiyun_key export BBM_CAIYUN_API_KEY=${your_api_key} ``` ================================================ FILE: docs/index.md ================================================ # bilingual book maker The `bilingual_book_maker` is an AI translation tool that uses ChatGPT to assist users in creating multi-language versions of epub/txt files and books. This tool is exclusively designed for translating epub books that have entered the public domain and is not intended for copyrighted works. Before using this tool, please review the project's **[disclaimer](disclaimer.md)**. ================================================ FILE: docs/installation.md ================================================ # Installation ## pip bilingual_book_maker has been published as a [Python package](https://pypi.org/project/bbook-maker/) and can be install by `pip`. (Recommend in a virtual environment.) ```sh pip install -U bbook_maker ``` ## git You can also install from github if you want to use the latest version. ```sh git clone git@github.com:yihong0618/bilingual_book_maker.git pip install . ``` ================================================ FILE: docs/model_lang.md ================================================ # Model and Languages ## Models `-m, --model `
Currently `bbook_maker` supports these models: `chatgptapi` , `gpt3` , `google` , `caiyun` , `deepl` , `deeplfree` , `gpt4` , `gpt4omini` , `gpt5mini` , `o1-preview` , `o1` , `o1-mini` , `o3-mini` , `claude` , `customapi`. Default model is `chatgptapi` . ### OPENAI models There are several models you can choose from. * gpt3 bbook_maker --book_name test_books/animal_farm.epub --model gpt3 --openai_key ${openai_key} * chatgpiapi `chatgptapi` is [GPT-3.5-turbo](https://openai.com/blog/introducing-chatgpt-and-whisper-apis), which is used by ChatGPT currently. bbook_maker --book_name test_books/animal_farm.epub --model chatgptapi --openai_key ${openai_key} * gpt4 bbook_maker --book_name test_books/animal_farm.epub --model gpt4 --openai_key ${openai_key} If using `gpt4` , you can add `--use_context` to add a context paragraph to each passage sent to the model for translation. bbook_maker --book_name test_books/animal_farm.epub --model gpt4 --openai_key ${openai_key} --use_context The option `--use_context` prompts the GPT4 model to create a one-paragraph summary. If it is the beginning of the translation, it will summarize the entire passage sent (the size depending on `--accumulated_num` ). If it has any proceeding passage, it will amend the summary to include details from the most recent passage, creating a running one-paragraph context payload of the important details of the entire translated work, which improves consistency of flow and tone of each translation. * gpt5mini `gpt5mini` uses the `gpt-5-mini` model. bbook_maker --book_name test_books/animal_farm.epub --model gpt5mini --openai_key ${openai_key} **Note 1: Use `--openai_key` option to specify OpenAI API key. If you have multiple keys, separate them by commas (xxx, xxx, xxx) to reduce errors caused by API call limits.** **Note 2: You can just set the environment variable `BBM_OPENAI_API_KEY` instead the openai_key. See [Environment setting](settings.md).** ### CAIYUN Using Caiyun model to translate. The api currently only support: 1. Simplified Chinese <-> English 2. Simplified Chinese <-> Japanese The official Caiyun has provided a test token (3975l6lr5pcbvidl6jl2). You can apply your own token by following this [tutorial].(https://bobtranslate.com/service/translate/caiyun.html) bbook_maker --model caiyun --caiyun_key 3975l6lr5pcbvidl6jl2 --book_name test_books/animal_farm.epub ### DEEPL There are two models you can choose from. * deepl: [DeepL Translator](https://rapidapi.com/splintPRO/api/dpl-translator).
Need to pay to get the token. Use `--model deepl --deepl_key ${deepl_key}` bbook_maker --book_name test_books/animal_farm.epub --model deepl --deepl_key ${deepl_key} * deeplfree: DeepL free model bbook_maker --book_name test_books/animal_farm.epub --model deeplfree ### Claude Support [Claude](https://console.anthropic.com/docs) model. Use `--model claude --claude_key ${claude_key}` . bbook_maker --book_name test_books/animal_farm.epub --model claude --claude_key ${claude_key} ### Custom API Support CustomAPI model. Use `--model customapi --custom_api ${custom_api}` . bbook_maker --book_name test_books/animal_farm.epub --model customapi --custom_api ${custom_api} ### Google Support google model. Use `--model google` ## Languages `--language `
Set target languages. All models except for `caiyun` supports lots of languages. You can use `bbook_maker --help` to check available languages. Default target language is `"Simplified Chinese"` . ```sh bbook_maker --book_name test_books/animal_farm.epub --model chatgptapi --openai_key ${openai_key} --language ja ``` ```sh bbook_maker --book_name test_books/animal_farm.epub --model chatgptapi --openai_key ${openai_key} --language "Simplified Chinese" ``` ================================================ FILE: docs/prompt.md ================================================ # Tweak the prompt To tweak the prompt, use the `--prompt` parameter. Valid placeholders for the `user` role template include `{text}` and `{language}`. It supports a few ways to configure the prompt: - If you don't need to set the `system` role content, you can simply set it up like this: `--prompt "Translate {text} to {language}."` or `--prompt prompt_template_sample.txt` # prompt_template_sample.txt Translate the given text to {language}. Be faithful or accurate in translation. Make the translation readable or intelligible. Be elegant or natural in translation. If the text cannot be translated, return the original text as is. Do not translate person's name. Do not add any additional text in the translation. The text to be translated is: {text} - If you need to set the `system` role content, you can use the following format: `--prompt '{"user":"Translate {text} to {language}", "system": "You are a professional translator."}'` or `--prompt prompt_template_sample.json` # prompt_template_sample.json { "system": "You are a professional translator.", "user": "Translate the given text to {language}. Be faithful or accurate in translation. Make the translation readable or intelligible. Be elegant or natural in translation. If the text cannot be translated, return the original text as is. Do not translate person's name. Do not add any additional text in the translation. The text to be translated is:\n{text}" } You can also set the `user` and `system` role prompt by setting environment variables: `BBM_CHATGPTAPI_USER_MSG_TEMPLATE` and `BBM_CHATGPTAPI_SYS_MSG`. - You can now use PromptDown format (`.md` files) for more structured prompts: `--prompt prompt_md.prompt.md` # Translation Prompt ## System Message You are a professional translator who specializes in accurate translations. ## Conversation | Role | Content | |-------|------------------------------------------| | User | Please translate the following text into {language}:\n\n{text} | # OR using Developer Message (for newer AI models) # Translation Prompt ## Developer Message You are a professional translator who specializes in accurate translations. ## Conversation | Role | Content | |-------|------------------------------------------| | User | Please translate the following text into {language}:\n\n{text} | ## Examples ```sh python3 make_book.py --book_name test_books/animal_farm.epub --prompt prompt_template_sample.txt # or python3 make_book.py --book_name test_books/animal_farm.epub --prompt prompt_template_sample.json # or python3 make_book.py --book_name test_books/animal_farm.epub --prompt "Please translate \`{text}\` to {language}" ``` ================================================ FILE: docs/quickstart.md ================================================ # QuickStart After successfully install the package, you can see `bbook-maker` is in the output of `pip list`. ## Preparation 1. ChatGPT or OpenAI [token](https://platform.openai.com/account/api-keys) 2. epub/txt books 3. Environment with internet access or proxy 4. Python 3.8+ ## Use You can use by command `bbook_maker`. A sample book, `test_books/animal_farm.epub`, is provided for testing purposes. ```sh bbook_maker --book_name ${path of a book} --openai_key ${openai_key} # Example bbook_maker --book_name test_books/animal_farm.epub --openai_key ${openai_key} ``` Or, you can use the [script](https://github.com/yihong0618/bilingual_book_maker/blob/main/make_book.py) provided by repository. ```sh python3 make_book.py --book_name ${path of a book} --openai_key ${openai_key} # Example python3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} ``` Once the translation is complete, a bilingual book named `${book_name}_bilingual.epub` would be generated. **Note: If there are any errors or you wish to interrupt the translation by pressing `CTRL+C`. A book named `${book_name}_bilingual_temp.epub` would be generated. You can simply rename it to any desired name.** ================================================ FILE: make_book.py ================================================ from book_maker.cli import main if __name__ == "__main__": main() ================================================ FILE: mkdocs.yml ================================================ site_name: bilingual book maker theme: name: material features: - navigation.tabs - navigation.tabs.sticky - content.code.copy nav: - Home : index.md - Getting started: - Installation: installation.md - QuickStart: quickstart.md - Usage: - Model and languages: model_lang.md - Command line options: cmd.md - Translate from different source: book_source.md - Environment setting: env_settings.md - Tweak the prompt: prompt.md - Disclaimer: disclaimer.md ================================================ FILE: prompt_md.json ================================================ { "system": "You are a highly skilled translator responsible for translating the content of books in Markdown format from English into Chinese.", "user": "## Strategies\nYou will follow a three-step translation process:\n### 1. Translate the input content from English into Chinese, respect the intention of the original text, keep the original Markdown format unchanged, and do not delete or omit any content, nor add additional explanations or remarks.\n### 2. Read the original text and the translation carefully, and then put forward constructive criticism and helpful suggestions to improve the translation. The final style and tone of the translation should conform to the Chinese language style.\nYou must strictly follow the rules below.\n- Never change the Markdown markup structure. Don't add or remove links. Do not change any URL.\n- Never touch or change the contents of code blocks even if they appear to have a bug.\n- Always preserve the original line breaks. Do not add or remove blank lines.\n- Never touch any permalink at the end of each heading.\n- Never touch HTML-like tags such as ``.\nWhen writing suggestions, pay attention to whether there are ways to improve the translation in terms of:\n- Accuracy (by correcting errors such as additions, mistranslations, omissions or untranslated text).\n- Fluency (by applying the rules of Chinese grammar, spelling and punctuation, and ensuring there is no unnecessary repetition).\n- Conciseness and abbreviation (please appropriately simplify and abbreviate the translation result while keeping the original meaning unchanged to avoid the translation being too lengthy).\n### 3. Based on the results of steps 1 and 2, refine and polish the translation, and do not add additional explanations or remarks.\n## Output\nFor each step of the translation process, output the results within the appropriate XML tags:\n\n[Insert your initial translation here.]\n\n\n[Insert your reflection on the translation and put forward specific here, useful and constructive suggestions to improve the translation. Each suggestion should target a specific part of the translation.]\n\n\n[Insert your refined and polished translation here.]\n\n## Input\nThe following is the content of the book that needs to be translated within the tag:\n{text}" } ================================================ FILE: prompt_md.prompt.md ================================================ # Translation Prompt ## Developer Message You are a professional translator who specializes in accurate, natural-sounding translations that preserve the original meaning, tone, and style of the text. ## Conversation | Role | Content | |-------|---------------------------------------------------------------------------| | User | Please translate the following text into {language}:\n\n{text} | ================================================ FILE: prompt_template_sample.json ================================================ { "system": "You are a highly skilled academic translator. Please complete the translation task according to the following instructions and provide only the final polished translation.", "user": "## Strategies\nYou will follow a three-step translation process:\n### Step.1 Initial Direct Translation: Translate the content from English to Chinese sentence by sentence, respecting the original intent without deleting, omitting, or adding any extra explanations or notes.\n ### Step.2 Reflection and Revision: Carefully review both the input content and the initial direct translation from Step 1. Check if the translation conveys the original meaning, if the grammatical structure is correct, if word choices are appropriate, and if there are any ambiguities or polysemous words. The final style and tone should conform to Chinese language conventions. \nYou must strictly follow the rules below.\n- Don't add or remove links. Do not change any URL.\n- Do not translate the reference list.\n- Never touch,change or translate the mathematical formulas.\n- Never touch,change or translate the contents of code blocks even if they appear to have a bug.\n- Always preserve the original line breaks. Do not add or remove blank lines.\nProvide constructive criticism and helpful suggestions to improve: \n- translation accuracy (correct additions, mistranslations, omissions, or untranslated text errors),\n- fluency (apply Chinese grammar, spelling, and punctuation rules, and ensure no unnecessary repetition), \n- conciseness (streamline the translation results while maintaining the original meaning, avoiding wordiness).\n ### Step.3 Polish and Optimize: Based on the results from Steps 1 and 2, refine and polish the translation, ensuring the final translation adheres to Chinese style without additional explanations or notes. The content to be translated is wrapped in the following tags:\n\n{text}. \n\nPlease write and output only the final polished translation here: " } ================================================ FILE: pyproject.toml ================================================ [project] name = "bbook-maker" description = "The bilingual_book_maker is an AI translation tool that uses ChatGPT to assist users in creating multi-language versions of epub/txt files and books." readme = "README.md" license = {text = "MIT"} dynamic = ["version"] requires-python = ">=3.10" authors = [ { name = "yihong0618", email = "zouzou0208@gmail.com" }, ] classifiers = [ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", ] dependencies = [ "anthropic", "backoff", "bs4", "ebooklib", "google-generativeai", "langdetect", "litellm", "openai>=1.1.1", "PyDeepLX", "requests", "rich", "tiktoken", "tqdm", "groq>=0.5.0", "promptdown>=0.9.0", "PyMuPDF", ] [project.scripts] bbook_maker = "book_maker.cli:main" promptdown = "promptdown_cli:main" [project.urls] Homepage = "https://github.com/yihong0618/bilingual_book_maker" [tool.pdm] plugins = ["pdm-autoexport"] [[tool.pdm.autoexport]] filename = "requirements.txt" without-hashes = true [build-system] requires = ["pdm-backend>=2.0.0"] build-backend = "pdm.backend" [tool.pdm.version] source = "scm" ================================================ FILE: tests/test_epub_metadata.py ================================================ import pytest from ebooklib import epub from book_maker.loader.epub_loader import EPUBBookLoader def test_epub_loader_handles_custom_metadata(tmp_path): source_book = epub.EpubBook() source_book.add_metadata("DC", "title", "Metadata Copy Test", {"id": "title-id"}) source_book.add_metadata("DC", "creator", "Tester", {"role": "aut"}) # Simulate a namespace that ebooklib does not recognise; the legacy approach # copied this verbatim and ebooklib failed while writing the book back. source_book.metadata["custom"] = [ ("foo-tag", "bar-value", {"attr": "value"}), ] legacy_book = epub.EpubBook() legacy_book.metadata = source_book.metadata with pytest.raises(AttributeError): epub.write_epub(str(tmp_path / "legacy.epub"), legacy_book) loader = EPUBBookLoader.__new__(EPUBBookLoader) rebuilt_book = loader._make_new_book(source_book) output_path = tmp_path / "rebuilt.epub" epub.write_epub(str(output_path), rebuilt_book) assert output_path.exists() dc_namespace = epub.NAMESPACES["DC"] titles = rebuilt_book.metadata[dc_namespace]["title"] creators = rebuilt_book.metadata[dc_namespace]["creator"] assert ("Metadata Copy Test", {"id": "title-id"}) in titles assert ("Tester", {"role": "aut"}) in creators assert "custom" not in rebuilt_book.metadata ================================================ FILE: tests/test_integration.py ================================================ import os import shutil import subprocess import sys from pathlib import Path import pytest @pytest.fixture() def test_book_dir() -> str: """Return test book dir""" # TODO: Can move this to conftest.py if there will be more unittests return str(Path(__file__).parent.parent / "test_books") def test_google_translate_epub(test_book_dir, tmpdir): """Test google translate epub""" shutil.copyfile( os.path.join(test_book_dir, "Liber_Esther.epub"), os.path.join(tmpdir, "Liber_Esther.epub"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "Liber_Esther.epub"), "--test", "--test_num", "20", "--model", "google", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "Liber_Esther_bilingual.epub")) assert os.path.getsize(os.path.join(tmpdir, "Liber_Esther_bilingual.epub")) != 0 def test_deepl_free_translate_epub(test_book_dir, tmpdir): """Test deepl free translate epub""" shutil.copyfile( os.path.join(test_book_dir, "Liber_Esther.epub"), os.path.join(tmpdir, "Liber_Esther.epub"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "Liber_Esther.epub"), "--test", "--test_num", "20", "--model", "deeplfree", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "Liber_Esther_bilingual.epub")) assert os.path.getsize(os.path.join(tmpdir, "Liber_Esther_bilingual.epub")) != 0 def test_google_translate_epub_cli(): pass def test_google_translate_txt(test_book_dir, tmpdir): """Test google translate txt""" shutil.copyfile( os.path.join(test_book_dir, "the_little_prince.txt"), os.path.join(tmpdir, "the_little_prince.txt"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "the_little_prince.txt"), "--test", "--test_num", "20", "--model", "google", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) assert os.path.getsize(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) != 0 def test_google_translate_txt_batch_size(test_book_dir, tmpdir): """Test google translate txt with batch_size""" shutil.copyfile( os.path.join(test_book_dir, "the_little_prince.txt"), os.path.join(tmpdir, "the_little_prince.txt"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "the_little_prince.txt"), "--test", "--batch_size", "30", "--test_num", "20", "--model", "google", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) assert os.path.getsize(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) != 0 @pytest.mark.skipif( not os.environ.get("BBM_CAIYUN_API_KEY"), reason="No BBM_CAIYUN_API_KEY in environment variable.", ) def test_caiyun_translate_txt(test_book_dir, tmpdir): """Test caiyun translate txt""" shutil.copyfile( os.path.join(test_book_dir, "the_little_prince.txt"), os.path.join(tmpdir, "the_little_prince.txt"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "the_little_prince.txt"), "--test", "--batch_size", "10", "--test_num", "100", "--model", "caiyun", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) assert os.path.getsize(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) != 0 @pytest.mark.skipif( not os.environ.get("BBM_DEEPL_API_KEY"), reason="No BBM_DEEPL_API_KEY in environment variable.", ) def test_deepl_translate_txt(test_book_dir, tmpdir): shutil.copyfile( os.path.join(test_book_dir, "the_little_prince.txt"), os.path.join(tmpdir, "the_little_prince.txt"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "the_little_prince.txt"), "--test", "--batch_size", "30", "--test_num", "20", "--model", "deepl", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) assert os.path.getsize(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) != 0 @pytest.mark.skipif( not os.environ.get("BBM_DEEPL_API_KEY"), reason="No BBM_DEEPL_API_KEY in environment variable.", ) def test_deepl_translate_srt(test_book_dir, tmpdir): shutil.copyfile( os.path.join(test_book_dir, "Lex_Fridman_episode_322.srt"), os.path.join(tmpdir, "Lex_Fridman_episode_322.srt"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "Lex_Fridman_episode_322.srt"), "--test", "--batch_size", "30", "--test_num", "2", "--model", "deepl", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "Lex_Fridman_episode_322_bilingual.srt")) assert ( os.path.getsize(os.path.join(tmpdir, "Lex_Fridman_episode_322_bilingual.srt")) != 0 ) @pytest.mark.skipif( not os.environ.get("OPENAI_API_KEY"), reason="No OPENAI_API_KEY in environment variable.", ) def test_openai_translate_epub_zh_hans(test_book_dir, tmpdir): shutil.copyfile( os.path.join(test_book_dir, "lemo.epub"), os.path.join(tmpdir, "lemo.epub"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "lemo.epub"), "--test", "--test_num", "5", "--language", "zh-hans", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "lemo_bilingual.epub")) assert os.path.getsize(os.path.join(tmpdir, "lemo_bilingual.epub")) != 0 @pytest.mark.skipif( not os.environ.get("OPENAI_API_KEY"), reason="No OPENAI_API_KEY in environment variable.", ) def test_openai_translate_epub_ja_prompt_txt(test_book_dir, tmpdir): shutil.copyfile( os.path.join(test_book_dir, "animal_farm.epub"), os.path.join(tmpdir, "animal_farm.epub"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "animal_farm.epub"), "--test", "--test_num", "5", "--language", "ja", "--model", "gpt3", "--prompt", "prompt_template_sample.txt", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "animal_farm_bilingual.epub")) assert os.path.getsize(os.path.join(tmpdir, "animal_farm_bilingual.epub")) != 0 @pytest.mark.skipif( not os.environ.get("OPENAI_API_KEY"), reason="No OPENAI_API_KEY in environment variable.", ) def test_openai_translate_epub_ja_prompt_json(test_book_dir, tmpdir): shutil.copyfile( os.path.join(test_book_dir, "animal_farm.epub"), os.path.join(tmpdir, "animal_farm.epub"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "animal_farm.epub"), "--test", "--test_num", "5", "--language", "ja", "--prompt", "prompt_template_sample.json", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "animal_farm_bilingual.epub")) assert os.path.getsize(os.path.join(tmpdir, "animal_farm_bilingual.epub")) != 0 @pytest.mark.skipif( not os.environ.get("OPENAI_API_KEY"), reason="No OPENAI_API_KEY in environment variable.", ) def test_openai_translate_srt(test_book_dir, tmpdir): shutil.copyfile( os.path.join(test_book_dir, "Lex_Fridman_episode_322.srt"), os.path.join(tmpdir, "Lex_Fridman_episode_322.srt"), ) subprocess.run( [ sys.executable, "make_book.py", "--book_name", os.path.join(tmpdir, "Lex_Fridman_episode_322.srt"), "--test", "--test_num", "20", ], env=os.environ.copy(), ) assert os.path.isfile(os.path.join(tmpdir, "Lex_Fridman_episode_322_bilingual.srt")) assert ( os.path.getsize(os.path.join(tmpdir, "Lex_Fridman_episode_322_bilingual.srt")) != 0 ) ================================================ FILE: tests/test_pdf_cli.py ================================================ import subprocess import sys from pathlib import Path import pytest fitz = pytest.importorskip("fitz") def test_pdf_cli_creates_txt_and_optional_epub(tmp_path): pdf_path = tmp_path / "cli_test.pdf" doc = fitz.open() page = doc.new_page() page.insert_text((72, 72), "CLI test\nPDF content") doc.save(str(pdf_path)) # run CLI subprocess.run( [ sys.executable, "make_book.py", "--book_name", str(pdf_path), "--test", "--test_num", "5", "--model", "google", ], check=True, ) txt_out = tmp_path / "cli_test_bilingual.txt" assert txt_out.exists() assert txt_out.stat().st_size > 0 # if ebooklib is installed, an epub should be created try: import ebooklib except Exception: ebooklib = None if ebooklib is not None: epub_out = tmp_path / "cli_test_bilingual.epub" assert epub_out.exists() assert epub_out.stat().st_size > 0 ================================================ FILE: tests/test_pdf_loader.py ================================================ import os from pathlib import Path import pytest fitz = pytest.importorskip("fitz") from book_maker.loader.pdf_loader import PDFBookLoader class DummyModel: def __init__( self, key, language, api_base=None, temperature=1.0, source_lang="auto", **kwargs, ): pass def translate(self, text): return f"{text}" def translate_list(self, texts): return [f"{t}" for t in texts] def test_pdf_loader_extracts_and_translates(tmp_path): pdf_path = tmp_path / "test.pdf" doc = fitz.open() page = doc.new_page() page.insert_text((72, 72), "Hello world\nThis is a PDF test") doc.save(str(pdf_path)) loader = PDFBookLoader( str(pdf_path), DummyModel, key="", resume=False, language="en", is_test=True, test_num=5, ) assert len(loader.origin_book) > 0 loader.make_bilingual_book() out_file = tmp_path / "test_bilingual.txt" assert out_file.exists() assert out_file.stat().st_size > 0 # basic content check content = out_file.read_text(encoding="utf-8") assert "" in content # if ebooklib is installed, an EPUB should also be produced try: import ebooklib except Exception: ebooklib = None if ebooklib is not None: epub_file = tmp_path / "test_bilingual.epub" assert epub_file.exists() assert epub_file.stat().st_size > 0 ================================================ FILE: typos.toml ================================================ # See https://github.com/crate-ci/typos/blob/master/docs/reference.md to configure typos [default.extend-words] sur = "sur" banch = "banch" # TODO: not sure if this is a typo or not fo = "fo" ba = "ba" [files] extend-exclude = ["LICENSE"]