[
{
"path": ".dockerignore",
"content": "Dockerfile*\ndocker-compose*\nLICENSE\ntest_books\nREADME*\n.dockerignore\n.git\n.github\n.gitignore\n.vscode"
},
{
"path": ".github/workflows/docs.yaml",
"content": "name: Publish docs\non:\n push:\n branches:\n - main\n\njobs:\n deploy:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v2\n - uses: actions/setup-python@v2\n with:\n python-version: '3.10'\n - run: pip install mkdocs mkdocs-material\n - run: mkdocs gh-deploy --force\n"
},
{
"path": ".github/workflows/make_test_ebook.yaml",
"content": "name: CI\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n workflow_dispatch:\n \nenv:\n ACTIONS_ALLOW_UNSECURE_COMMANDS: true\n OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n BBM_CAIYUN_API_KEY: ${{ secrets.BBM_CAIYUN_API_KEY }}\n\njobs:\n typos-check:\n name: Spell Check with Typos\n runs-on: ubuntu-latest\n steps:\n - name: Checkout Actions Repository\n uses: actions/checkout@v3\n - name: Check spelling with custom config file\n uses: crate-ci/typos@v1.16.6\n with:\n config: ./typos.toml\n testing:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - name: install python 3.10\n uses: actions/setup-python@v4\n with:\n python-version: '3.10'\n cache: 'pip' # caching pip dependencies\n - name: Check formatting (black)\n run: |\n pip install black\n black . --check\n - name: install python requirements\n run: pip install -r requirements.txt\n\n - name: Test install\n run: |\n pip install .\n\n - name: make normal ebook test using google translate and cli\n run: |\n bbook_maker --book_name \"test_books/Liber_Esther.epub\" --test --test_num 10 --model google --translate-tags div,p\n bbook_maker --book_name \"test_books/Liber_Esther.epub\" --test --test_num 20 --model google\n\n - name: make txt book test using google translate\n run: |\n python3 make_book.py --book_name \"test_books/the_little_prince.txt\" --test --test_num 20 --model google\n\n - name: make txt book test with batch_size\n run: |\n python3 make_book.py --book_name \"test_books/the_little_prince.txt\" --test --batch_size 30 --test_num 20 --model google\n \n - name: make caiyun translator test\n if: env.BBM_CAIYUN_API_KEY != null\n run: |\n python3 make_book.py --book_name \"test_books/the_little_prince.txt\" --test --batch_size 30 --test_num 100 --model caiyun\n\n - name: make openai key ebook test\n if: env.BBM_DEEPL_API_KEY != null\n run: |\n python3 make_book.py --book_name \"test_books/lemo.epub\" --test --test_num 5 --language zh-hans\n python3 make_book.py --book_name \"test_books/animal_farm.epub\" --test --test_num 5 --language ja --model gpt3 --prompt prompt_template_sample.txt\n python3 make_book.py --book_name \"test_books/animal_farm.epub\" --test --test_num 5 --language ja --prompt prompt_template_sample.json\n python3 make_book.py --book_name test_books/Lex_Fridman_episode_322.srt --test --test_num 20\n \n - name: Rename and Upload ePub\n if: env.OPENAI_API_KEY != null\n uses: actions/upload-artifact@v4\n with:\n name: epub_output\n path: \"test_books/lemo_bilingual.epub\"\n"
},
{
"path": ".github/workflows/release.yaml",
"content": "name: Release and Build Docker Image\n\npermissions:\n contents: write\n\non:\n push:\n tags:\n - \"*\"\n\njobs:\n release-pypi:\n name: Build and Release PyPI\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v3\n with:\n fetch-depth: 0\n\n - uses: actions/setup-python@v4\n with:\n python-version: \"3.10\"\n\n - uses: actions/setup-node@v3\n with:\n node-version: 16\n\n - name: Build artifacts\n run: |\n pip install build\n python -m build\n\n - uses: pypa/gh-action-pypi-publish@release/v1\n with:\n password: ${{ secrets.PYPI_API_TOKEN }}\n\n"
},
{
"path": ".gitignore",
"content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n.idea/\n.DS_Store\ntest_books/\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n/test_books/*.epub\nlog/\n.chatgpt_cache.json\n# for user do not want to push\n*.srt\n*.txt\n*.bin\n*.epub\n\n# For markdown files in user directories\n.cursorrules\nbooks/\nprompts/\n.pdm-python\n"
},
{
"path": "Dockerfile",
"content": "FROM python:3.10-slim\n\nRUN apt-get update\n\nWORKDIR /app\n\nCOPY requirements.txt .\n\nRUN pip install -r /app/requirements.txt\n\nCOPY . .\n\nENTRYPOINT [\"python3\", \"make_book.py\"]\n"
},
{
"path": "LICENSE",
"content": "MIT License\n\nCopyright (c) 2023 yihong\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
},
{
"path": "Makefile",
"content": "SHELL := /bin/bash\n\nfmt:\n\t@echo \"Running formatter ...\"\n\tvenv/bin/black .\n\n.PHONY:tests\ntests:\n\t@echo \"Running tests ...\"\n\tvenv/bin/pytest tests/test_integration.py\n\nserve-docs:\n\tmkdocs serve\n"
},
{
"path": "README-CN.md",
"content": "# bilingual_book_maker\n\nbilingual_book_maker 是一个 AI 翻译工具，使用 ChatGPT 帮助用户制作多语言版本的 epub/txt/srt 文件和图书。该工具仅适用于翻译进入公共版权领域的 epub/txt 图书，不适用于有版权的书籍。请在使用之前阅读项目的 **[免责声明](./disclaimer.md)**。\n\n\n\n## 准备\n\n1. ChatGPT or OpenAI token [^token]\n2. epub/txt books\n3. 能正常联网的环境或 proxy\n4. python3.8+\n\n## 快速开始\n\n本地放了一个 `test_books/animal_farm.epub` 给大家测试\n\n```shell\npip install -r requirements.txt\npython3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test\n或\npip install -U bbook_maker\nbbook --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test\n```\n\n## 翻译服务\n\n- 使用 `--openai_key` 指定 OpenAI API key，如果有多个可以用英文逗号分隔(xxx,xxx,xxx)，可以减少接口调用次数限制带来的错误。\n 或者，指定环境变量 `BBM_OPENAI_API_KEY` 来略过这个选项。\n- 默认用了 [GPT-3.5-turbo](https://openai.com/blog/introducing-chatgpt-and-whisper-apis) 模型，也就是 ChatGPT 正在使用的模型。\n\n* DeepL\n\n 使用 DeepL 封装的 api 进行翻译，需要付费。[DeepL Translator](https://rapidapi.com/splintPRO/api/dpl-translator) 来获得 token\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model deepl --deepl_key ${deepl_key}\n ```\n\n* DeepL free\n\n 使用 DeepL free\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model deeplfree\n ```\n\n* Claude\n\n 使用 [Claude](https://console.anthropic.com/docs) 模型进行翻译\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model claude --claude_key ${claude_key}\n ```\n\n* 谷歌翻译\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model google\n ```\n\n* 彩云小译\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model caiyun --caiyun_key ${caiyun_key}\n ```\n\n* Gemini\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model gemini --gemini_key ${gemini_key}\n ```\n\n* Qwen\n\n 使用 [Qwen](https://www.aliyun.com/product/dashscope) 模型进行翻译，支持 qwen-mt-turbo 和 qwen-mt-plus 模型。\n\n 使用 `--source_lang` 指定源语言，留空为自动检测。\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --qwen_key ${qwen_key} --model qwen-mt-turbo --language \"Simplified Chinese\"\n python3 make_book.py --book_name test_books/animal_farm.epub --qwen_key ${qwen_key} --model qwen-mt-plus --language \"Japanese\" --source_lang \"English\"\n ```\n\n* 腾讯交互翻译\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model tencentransmart\n ```\n\n* [xAI](https://x.ai)\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model xai --xai_key ${xai_key}\n ```\n\n* [Ollama](https://github.com/ollama/ollama)\n\n 使用 [Ollama](https://github.com/ollama/ollama) 自托管模型进行翻译。\n 如果 ollama server 不运行在本地，使用 `--api_base http://x.x.x.x:port/v1` 指向 ollama server 地址\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --ollama_model ${ollama_model_name}\n ```\n\n* [Groq](https://console.groq.com/keys)\n\n GroqCloud 当前支持的模型可以查看[Supported Models](https://console.groq.com/docs/models)\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --groq_key [your_key] --model groq --model_list llama3-8b-8192\n ```\n\n## 使用说明\n\n- 翻译完会生成一本 `{book_name}_bilingual.epub` 的双语书\n- 如果出现了错误或使用 `CTRL+C` 中断命令，不想接下来继续翻译了，会生成一本 `{book_name}_bilingual_temp.epub` 的书，直接改成你想要的名字就可以了\n\n## 参数说明\n\n- `--test`:\n\n 如果大家没付费可以加上这个先看看效果（有 limit 稍微有些慢）\n\n- `--language`: 指定目标语言\n\n - 例如： `--language \"Simplified Chinese\"`，预设值为 `\"Simplified Chinese\"`.\n - 请阅读 helper message 来查找可用的目标语言： `python make_book.py --help`\n\n- `--proxy`\n\n 方便中国大陆的用户在本地测试时使用代理，传入类似 `http://127.0.0.1:7890` 的字符串\n\n- `--resume`\n\n 手动中断后，加入命令可以从之前中断的位置继续执行。\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model google --resume\n ```\n\n- `--translate-tags`\n\n 指定需要翻译的标签，使用逗号分隔多个标签。epub 由 html 文件组成，默认情况下，只翻译 `
` 中的内容。例如: `--translate-tags h1,h2,h3,p,div`\n\n- `--book_from`\n\n 选项指定电子阅读器类型（现在只有 kobo 可用），并使用 `--device_path` 指定挂载点。\n\n- `--api_base ${url}`\n\n 如果你遇到了墙需要用 Cloudflare Workers 替换 api_base 请使用 `--api_base ${url}` 来替换。\n **请注意，此处你输入的 api 应该是'`https://xxxx/v1`'的字样，域名需要用引号包裹**\n\n- `--allow_navigable_strings`\n\n 如果你想要翻译电子书中的无标签字符串，可以使用 `--allow_navigable_strings` 参数，会将可遍历字符串加入翻译队列，**注意，在条件允许情况下，请寻找更规范的电子书**\n\n- `--prompt`\n\n 如果你想调整 prompt，你可以使用 `--prompt` 参数。有效的占位符包括 `{text}` 和 `{language}`。你可以用以下方式配置 prompt:\n\n - 如果您不需要设置 `system` 角色，可以这样：`--prompt \"Translate {text} to {language}\"` 或者 `--prompt prompt_template_sample.txt`（示例文本文件可以在 [./prompt_template_sample.txt](./prompt_template_sample.txt) 找到）。\n\n - 如果您需要设置 `system` 角色，可以使用以下方式配置：`--prompt '{\"user\":\"Translate {text} to {language}\", \"system\": \"You are a professional translator.\"}'`，或者 `--prompt prompt_template_sample.json`（示例 JSON 文件可以在 [./prompt_template_sample.json](./prompt_template_sample.json) 找到）。\n\n - 你也可以用环境以下环境变量来配置 `system` 和 `user` 角色 prompt：`BBM_CHATGPTAPI_USER_MSG_TEMPLATE` 和 `BBM_CHATGPTAPI_SYS_MSG`。\n 该参数可以是提示模板字符串，也可以是模板 `.txt` 文件的路径。\n\n- `--batch_size`\n\n 指定批量翻译的行数(默认行数为 10，目前只对 txt 生效)\n\n- `--accumulated_num`:\n\n 达到累计token数开始进行翻译。gpt3.5将total_token限制为4090。\n 例如，如果您使用`--accumulation_num 1600`，则可能会输出2200个令牌，另外200个令牌用于系统指令（system_message）和用户指令（user_message），1600+2200+200 = 4000，所以token接近极限。你必须选择一个自己合适的值，我们无法在发送之前判断是否达到限制\n\n- `--use_context`:\n\n prompts the model to create a three-paragraph summary. If it's the beginning of the translation, it will summarize the entire passage sent (the size depending on `--accumulated_num`).\n For subsequent passages, it will amend the summary to include details from the most recent passage, creating a running one-paragraph context payload of the important details of the entire translated work. This improves consistency of flow and tone throughout the translation. This option is available for all ChatGPT-compatible models and Gemini models.\n\n 模型提示词将创建三段摘要。如果是翻译的开始，它将总结发送的整个段落（大小取决于`--accumulated_num`）。\n 对于后续的段落，它将修改摘要，以包括最近段落的细节，创建一个完整的段落上下文负载，包含整个翻译作品的重要细节。 这提高了整个翻译过程中的流畅性和语气的一致性。 此选项适用于所有ChatGPT兼容型号和Gemini型号。\n\n - `--context_paragraph_limit`:\n\n 使用`--use_context`选项时，使用`--context_paragraph_limit`设置上下文段落数限制。\n\n- `--temperature`:\n\n 使用 `--temperature` 设置 `chatgptapi`/`gpt4`/`claude`模型的temperature值.\n 如 `--temperature 0.7`.\n\n- `--block_size`:\n\n 使用`--block_size`将多个段落合并到一个块中。这可能会提高准确性并加快处理速度，但可能会干扰原始格式。必须与`--single_translate`一起使用。\n 例如：`--block_size 5 --single_translate`。\n\n- `--single_translate`:\n\n 使用`--single_translate`只输出翻译后的图书，不创建双语版本。\n\n- `--translation_style`:\n\n 如: `--translation_style \"color: #808080; font-style: italic;\"`\n\n- `--retranslate \"$translated_filepath\" \"file_name_in_epub\" \"start_str\" \"end_str\"(optional)`:\n\n - 重新翻译，从 start_str 到 end_str 的标记:\n\n ```shell\n python3 \"make_book.py\" --book_name \"test_books/animal_farm.epub\" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' 'This kind of thing is not a good symptom. Obviously'\n ```\n\n - 重新翻译, 从start_str 的标记开始:\n\n ```shell\n python3 \"make_book.py\" --book_name \"test_books/animal_farm.epub\" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which'\n ```\n\n### 示范用例\n\n**如果使用 `pip install bbook_maker` 以下命令都可以改成 `bbook args`**\n\n```shell\n# 如果你想快速测一下\npython3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test\n\n# 或翻译完整本书\npython3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --language zh-hans\n\n# Or translate the whole book using Gemini\npython3 make_book.py --book_name test_books/animal_farm.epub --gemini_key ${gemini_key} --model gemini\n\n# 指定环境变量来略过 --openai_key\nexport OPENAI_API_KEY=${your_api_key}\n\n# Use the DeepL model with Japanese\npython3 make_book.py --book_name test_books/animal_farm.epub --model deepl --deepl_key ${deepl_key} --language ja\n\n# Use the Claude model with Japanese\npython3 make_book.py --book_name test_books/animal_farm.epub --model claude --claude_key ${claude_key} --language ja\n\n# Use the CustomAPI model with Japanese\npython3 make_book.py --book_name test_books/animal_farm.epub --model customapi --custom_api ${custom_api} --language ja\n\n# Translate contents in
and
\npython3 make_book.py --book_name test_books/animal_farm.epub --translate-tags div,p\n\n# 修改prompt\npython3 make_book.py --book_name test_books/animal_farm.epub --prompt prompt_template_sample.txt\n# 或者\npython3 make_book.py --book_name test_books/animal_farm.epub --prompt \"Please translate \\`{text}\\` to {language}\"\n# 翻译 kobo e-reader 中，來自 Rakuten Kobo 的书籍\npython3 make_book.py --book_from kobo --device_path /tmp/kobo\n\n# 翻译 txt 文件\npython3 make_book.py --book_name test_books/the_little_prince.txt --test\n# 聚合多行翻译 txt 文件\npython3 make_book.py --book_name test_books/the_little_prince.txt --test --batch_size 20\n\n\n# 使用彩云小译翻译(彩云api目前只支持: 简体中文 <-> 英文， 简体中文 <-> 日语)\n# 彩云提供了测试token（3975l6lr5pcbvidl6jl2）\n# 你可以参考这个教程申请自己的token (https://bobtranslate.com/service/translate/caiyun.html)\npython3 make_book.py --model caiyun --caiyun_key 3975l6lr5pcbvidl6jl2 --book_name test_books/animal_farm.epub\n# 可以在环境变量中设置BBM_CAIYUN_API_KEY，略过--openai_key\nexport BBM_CAIYUN_API_KEY=${your_api_key}\n```\n\n更加小白的示例\n\n```shell\npython3 make_book.py --book_name 'animal_farm.epub' --openai_key sk-XXXXX --api_base 'https://xxxxx/v1'\n\n# 有可能你不需要 python3 而是python\npython make_book.py --book_name 'animal_farm.epub' --openai_key sk-XXXXX --api_base 'https://xxxxx/v1'\n```\n\n[演示视频](https://www.bilibili.com/video/BV1XX4y1d75D/?t=0h07m08s)\n[演示视频 2](https://www.bilibili.com/video/BV1T8411c7iU/)\n\n使用 Azure OpenAI service\n\n```shell\npython3 make_book.py --book_name 'animal_farm.epub' --openai_key XXXXX --api_base 'https://example-endpoint.openai.azure.com' --deployment_id 'deployment-name'\n\n# Or python3 is not in your PATH\npython make_book.py --book_name 'animal_farm.epub' --openai_key XXXXX --api_base 'https://example-endpoint.openai.azure.com' --deployment_id 'deployment-name'\n```\n\n## 注意\n\n1. Free trail 的 API token 有所限制，如果想要更快的速度，可以考虑付费方案\n2. 欢迎提交 PR\n\n# 感谢\n\n- @[yetone](https://github.com/yetone)\n\n# 贡献\n\n- 任何 issue PR 都欢迎\n- Issue 中有些 TODO 没做的都可以选\n- 提交代码前请先执行 `black make_book.py` [^black]\n\n# 其它推荐项目\n\n- 书译 BookTranslator -> [Book Translator](https://www.booktranslator.app)\n\n## 赞赏\n\n谢谢就够了\n\n\n\n[^token]: https://platform.openai.com/account/api-keys\n[^black]: https://github.com/psf/black\n"
},
{
"path": "README.md",
"content": "**[中文](./README-CN.md) | English**\n[](https://github.com/BerriAI/litellm)\n\n# bilingual_book_maker\n\nThe bilingual_book_maker is an AI translation tool that uses ChatGPT to assist users in creating multi-language versions of epub/txt/srt/pdf files and books. This tool is exclusively designed for translating epub and other public domain works and is not intended for copyrighted works. Before using this tool, please review the project's **[disclaimer](./disclaimer.md)**.\n\n\n\n## Supported Models\n\ngpt-5-mini, gpt-4, gpt-3.5-turbo, claude-2, palm, llama-2, azure-openai, command-nightly, gemini, qwen-mt-turbo, qwen-mt-plus\nFor using Non-OpenAI models, use class `liteLLM()` - liteLLM supports all models above.\nFind more info here for using liteLLM: https://github.com/BerriAI/litellm/blob/main/setup.py\n\n## Preparation\n\n1. ChatGPT or OpenAI token [^token]\n2. epub/txt/pdf books\n3. Environment with internet access or proxy\n4. Python 3.8+\n\n## Quick Start\n\nA sample book, `test_books/animal_farm.epub`, is provided for testing purposes.\n\n```shell\npip install -r requirements.txt\npython3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test\nOR\npip install -U bbook_maker\nbbook --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test\n```\n\n## Translate Service\n\n- Use `--openai_key` option to specify OpenAI API key. If you have multiple keys, separate them by commas (xxx,xxx,xxx) to reduce errors caused by API call limits.\n Or, just set environment variable `BBM_OPENAI_API_KEY` instead.\n- A sample book, `test_books/animal_farm.epub`, is provided for testing purposes.\n- The default underlying model is [GPT-3.5-turbo](https://openai.com/blog/introducing-chatgpt-and-whisper-apis), which is used by ChatGPT currently. Use `--model gpt4` to change the underlying model to `GPT4`. You can also use `GPT4omini`.\n- Important to note that `gpt-4` is significantly more expensive than `gpt-4-turbo`, but to avoid bumping into rate limits, we automatically balance queries across `gpt-4-1106-preview`, `gpt-4`, `gpt-4-32k`, `gpt-4-0613`,`gpt-4-32k-0613`.\n- If you want to use a specific model alias with OpenAI (eg `gpt-4-1106-preview` or `gpt-3.5-turbo-0125`), you can use `--model openai --model_list gpt-4-1106-preview,gpt-3.5-turbo-0125`. `--model_list` takes a comma-separated list of model aliases.\n- If using chatgptapi, you can add `--use_context` to add a context paragraph to each passage sent to the model for translation (see below).\n\n* DeepL\n Support DeepL model [DeepL Translator](https://rapidapi.com/splintPRO/api/dpl-translator) need pay to get the token\n\n ```\n python3 make_book.py --book_name test_books/animal_farm.epub --model deepl --deepl_key ${deepl_key}\n ```\n\n* DeepL free\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model deeplfree\n ```\n\n* [Claude](https://console.anthropic.com/docs)\n\n Use [Claude](https://console.anthropic.com/docs) model to translate\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model claude --claude_key ${claude_key}\n ```\n\n* Google Translate\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model google\n ```\n\n* Caiyun Translate\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model caiyun --caiyun_key ${caiyun_key}\n ```\n\n* Gemini\n\n Support Google [Gemini](https://aistudio.google.com/app/apikey) model, use `--model gemini` for Gemini Flash or `--model geminipro` for Gemini Pro.\n If you want to use a specific model alias with Gemini (eg `gemini-1.5-flash-002` or `gemini-1.5-flash-8b-exp-0924`), you can use `--model gemini --model_list gemini-1.5-flash-002,gemini-1.5-flash-8b-exp-0924`. `--model_list` takes a comma-separated list of model aliases.\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model gemini --gemini_key ${gemini_key}\n ```\n\n* Qwen\n\n Support Alibaba Cloud [Qwen-MT](https://bailian.console.aliyun.com/) specialized translation model. Supports 92 languages with features like terminology intervention and translation memory.\n Use `--model qwen-mt-turbo` for faster/cheaper translation, or `--model qwen-mt-plus` for higher quality.\n\n Use `source_lang` to specify the source language explicitly, or leave it empty for auto-detection.\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --qwen_key ${qwen_key} --model qwen-mt-turbo --language \"Simplified Chinese\"\n python3 make_book.py --book_name test_books/animal_farm.epub --qwen_key ${qwen_key} --model qwen-mt-plus --language \"Japanese\" --source_lang \"English\"\n ```\n\n* [Tencent TranSmart](https://transmart.qq.com)\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model tencentransmart\n ```\n\n* [xAI](https://x.ai)\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model xai --xai_key ${xai_key}\n ```\n\n* [Ollama](https://github.com/ollama/ollama)\n\n Support [Ollama](https://github.com/ollama/ollama) self-host models,\n If ollama server is not running on localhost, use `--api_base http://x.x.x.x:port/v1` to point to the ollama server address\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --ollama_model ${ollama_model_name}\n ```\n\n* [groq](https://console.groq.com/keys)\n\n GroqCloud currently supports models: you can find from [Supported Models](https://console.groq.com/docs/models)\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --groq_key [your_key] --model groq --model_list llama3-8b-8192\n ```\n\n## Use\n\n- Once the translation is complete, a bilingual book named `${book_name}_bilingual.epub` would be generated for EPUB inputs; for TXT/MD/SRT inputs a bilingual text (or subtitle) file named `${book_name}_bilingual.txt` (or `_bilingual.srt`) will be generated. For **PDF inputs** the tool will produce a bilingual `.txt` fallback and will also attempt to create `${book_name}_bilingual.epub` — if EPUB creation fails, the TXT fallback remains so you do not need to retranslate.\n- If there are any errors or you wish to interrupt the translation by pressing `CTRL+C`, a temporary bilingual file (for example `{book_name}_bilingual_temp.epub` or `{book_name}_bilingual_temp.txt`) would be generated. You can simply rename it to any desired name.\n\n## Params\n\n- `--test`:\n\n Use `--test` option to preview the result if you haven't paid for the service. Note that there is a limit and it may take some time.\n\n- `--language`:\n\n Set the target language like `--language \"Simplified Chinese\"`. Default target language is `\"Simplified Chinese\"`.\n Read available languages by helper message: `python make_book.py --help`\n\n- `--proxy`:\n\n Use `--proxy` option to specify proxy server for internet access. Enter a string such as `http://127.0.0.1:7890`.\n\n- `--resume`:\n\n Use `--resume` option to manually resume the process after an interruption.\n\n ```shell\n python3 make_book.py --book_name test_books/animal_farm.epub --model google --resume\n ```\n\n- `--translate-tags`:\n\n epub is made of html files. By default, we only translate contents in `
`.\n Use `--translate-tags` to specify tags need for translation. Use comma to separate multiple tags.\n For example: `--translate-tags h1,h2,h3,p,div`\n\n- `--book_from`:\n\n Use `--book_from` option to specify e-reader type (Now only `kobo` is available), and use `--device_path` to specify the mounting point.\n\n- `--api_base`:\n\n If you want to change api_base like using Cloudflare Workers, use `--api_base ` to support it.\n **Note: the api url should be '`https://xxxx/v1`'. Quotation marks are required.**\n\n- `--allow_navigable_strings`:\n\n If you want to translate strings in an e-book that aren't labeled with any tags, you can use the `--allow_navigable_strings` parameter. This will add the strings to the translation queue. **Note that it's best to look for e-books that are more standardized if possible.**\n\n- `--prompt`:\n\n To tweak the prompt, use the `--prompt` parameter. Valid placeholders for the `user` role template include `{text}` and `{language}`. It supports a few ways to configure the prompt:\n\n - If you don't need to set the `system` role content, you can simply set it up like this: `--prompt \"Translate {text} to {language}.\"` or `--prompt prompt_template_sample.txt` (example of a text file can be found at [./prompt_template_sample.txt](./prompt_template_sample.txt)).\n\n - If you need to set the `system` role content, you can use the following format: `--prompt '{\"user\":\"Translate {text} to {language}\", \"system\": \"You are a professional translator.\"}'` or `--prompt prompt_template_sample.json` (example of a JSON file can be found at [./prompt_template_sample.json](./prompt_template_sample.json)).\n \n - You can now use [PromptDown](https://github.com/btfranklin/promptdown) format (`.md` files) for more structured prompts: `--prompt prompt_md.prompt.md`. PromptDown supports both traditional system messages and developer messages (used by newer AI models). Example:\n \n ```markdown\n # Translation Prompt\n \n ## Developer Message\n You are a professional translator who specializes in accurate translations.\n \n ## Conversation\n \n | Role | Content |\n | ---- | -------------------------------------------------------------- |\n | User | Please translate the following text into {language}:\\n\\n{text} |\n ```\n\n - You can also set the `user` and `system` role prompt by setting environment variables: `BBM_CHATGPTAPI_USER_MSG_TEMPLATE` and `BBM_CHATGPTAPI_SYS_MSG`.\n\n- `--batch_size`:\n\n Use the `--batch_size` parameter to specify the number of lines for batch translation (default is 10, currently only effective for txt files).\n\n- `--accumulated_num`:\n\n Wait for how many tokens have been accumulated before starting the translation. gpt3.5 limits the total_token to 4090. For example, if you use `--accumulated_num 1600`, maybe openai will output 2200 tokens and maybe 200 tokens for other messages in the system messages user messages, 1600+2200+200=4000, So you are close to reaching the limit. You have to choose your own\n value, there is no way to know if the limit is reached before sending\n\n- `--use_context`:\n\n prompts the model to create a three-paragraph summary. If it's the beginning of the translation, it will summarize the entire passage sent (the size depending on `--accumulated_num`).\n For subsequent passages, it will amend the summary to include details from the most recent passage, creating a running one-paragraph context payload of the important details of the entire translated work. This improves consistency of flow and tone throughout the translation. This option is available for all ChatGPT-compatible models and Gemini models.\n\n- `--context_paragraph_limit`:\n\n Use `--context_paragraph_limit` to set a limit on the number of context paragraphs when using the `--use_context` option.\n\n- `--parallel-workers`:\n\n Use `--parallel-workers` to enable parallel EPUB chapter processing. Values greater than `1` spin up multiple workers (recommended: `2-4`) and automatically fall back to sequential mode for single-chapter books.\n\n- `--temperature`:\n\n Use `--temperature` to set the temperature parameter for `chatgptapi`/`gpt4`/`claude` models.\n For example: `--temperature 0.7`.\n\n- `--block_size`:\n\n Use `--block_size` to merge multiple paragraphs into one block. This may increase accuracy and speed up the process but can disturb the original format. Must be used with `--single_translate`.\n For example: `--block_size 5 --single_translate`.\n\n- `--single_translate`:\n\n Use `--single_translate` to output only the translated book without creating a bilingual version.\n\n- `--translation_style`:\n\n example: `--translation_style \"color: #808080; font-style: italic;\"`\n\n- `--retranslate \"$translated_filepath\" \"file_name_in_epub\" \"start_str\" \"end_str\"(optional)`:\n\n Retranslate from start_str to end_str's tag:\n\n ```shell\n python3 \"make_book.py\" --book_name \"test_books/animal_farm.epub\" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' 'This kind of thing is not a good symptom. Obviously'\n ```\n\n Retranslate start_str's tag:\n\n ```shell\n python3 \"make_book.py\" --book_name \"test_books/animal_farm.epub\" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which'\n ```\n\n### Examples\n\n**Note if use `pip install bbook_maker` all commands can change to `bbook_maker args`**\n\n```shell\n# Test quickly\npython3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test --language zh-hans\n\n# Test quickly for src\npython3 make_book.py --book_name test_books/Lex_Fridman_episode_322.srt --openai_key ${openai_key} --test\n\n# Or translate the whole book\npython3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --language zh-hans\n\n# Or translate the whole book using Gemini flash\npython3 make_book.py --book_name test_books/animal_farm.epub --gemini_key ${gemini_key} --model gemini\n\n# Translate an EPUB with parallel chapter processing\npython3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key} --parallel-workers 4\n\n# Use a specific list of Gemini model aliases\npython3 make_book.py --book_name test_books/animal_farm.epub --gemini_key ${gemini_key} --model gemini --model_list gemini-1.5-flash-002,gemini-1.5-flash-8b-exp-0924\n\n# Set env OPENAI_API_KEY to ignore option --openai_key\nexport OPENAI_API_KEY=${your_api_key}\n\n# Use the GPT-4 model with context to Japanese\npython3 make_book.py --book_name test_books/animal_farm.epub --model gpt4 --use_context --language ja\n\n# Use a specific OpenAI model alias\npython3 make_book.py --book_name test_books/animal_farm.epub --model openai --model_list gpt-4-1106-preview --openai_key ${openai_key}\n\n**Note** you can use other `openai like` model in this way\npython3 make_book.py --book_name test_books/animal_farm.epub --model openai --model_list yi-34b-chat-0205 --openai_key ${openai_key} --api_base \"https://api.lingyiwanwu.com/v1\"\n\n# Use a specific list of OpenAI model aliases\npython3 make_book.py --book_name test_books/animal_farm.epub --model openai --model_list gpt-4-1106-preview,gpt-4-0125-preview,gpt-3.5-turbo-0125 --openai_key ${openai_key}\n\n# Use the DeepL model with Japanese\npython3 make_book.py --book_name test_books/animal_farm.epub --model deepl --deepl_key ${deepl_key} --language ja\n\n# Use the Claude model with Japanese\npython3 make_book.py --book_name test_books/animal_farm.epub --model claude --claude_key ${claude_key} --language ja\n\n# Use the CustomAPI model with Japanese\npython3 make_book.py --book_name test_books/animal_farm.epub --model customapi --custom_api ${custom_api} --language ja\n\n# Translate contents in
and
\npython3 make_book.py --book_name test_books/animal_farm.epub --translate-tags div,p\n\n# Tweaking the prompt\npython3 make_book.py --book_name test_books/animal_farm.epub --prompt prompt_template_sample.txt\n# or\npython3 make_book.py --book_name test_books/animal_farm.epub --prompt prompt_template_sample.json\n# or\npython3 make_book.py --book_name test_books/animal_farm.epub --prompt \"Please translate \\`{text}\\` to {language}\"\n\n# Translate books download from Rakuten Kobo on kobo e-reader\npython3 make_book.py --book_from kobo --device_path /tmp/kobo\n\n# translate txt file\npython3 make_book.py --book_name test_books/the_little_prince.txt --test --language zh-hans\n# aggregated translation txt file\npython3 make_book.py --book_name test_books/the_little_prince.txt --test --batch_size 20\n\n# Using Caiyun model to translate\n# (the api currently only support: simplified chinese <-> english, simplified chinese <-> japanese)\n# the official Caiyun has provided a test token (3975l6lr5pcbvidl6jl2)\n# you can apply your own token by following this tutorial(https://bobtranslate.com/service/translate/caiyun.html)\npython3 make_book.py --model caiyun --caiyun_key 3975l6lr5pcbvidl6jl2 --book_name test_books/animal_farm.epub\n\n\n# Set env BBM_CAIYUN_API_KEY to ignore option --openai_key\nexport BBM_CAIYUN_API_KEY=${your_api_key}\n\n```\n\nMore understandable example\n\n```shell\npython3 make_book.py --book_name 'animal_farm.epub' --openai_key sk-XXXXX --api_base 'https://xxxxx/v1'\n\n# Or python3 is not in your PATH\npython make_book.py --book_name 'animal_farm.epub' --openai_key sk-XXXXX --api_base 'https://xxxxx/v1'\n```\n\nMicrosoft Azure Endpoints\n\n```shell\npython3 make_book.py --book_name 'animal_farm.epub' --openai_key XXXXX --api_base 'https://example-endpoint.openai.azure.com' --deployment_id 'deployment-name'\n\n# Or python3 is not in your PATH\npython make_book.py --book_name 'animal_farm.epub' --openai_key XXXXX --api_base 'https://example-endpoint.openai.azure.com' --deployment_id 'deployment-name'\n```\n\n## Docker\n\nYou can use [Docker](https://www.docker.com/) if you don't want to deal with setting up the environment.\n\n```shell\n# Build image\ndocker build --tag bilingual_book_maker .\n\n# Run container\n# \"$folder_path\" represents the folder where your book file locates. Also, it is where the processed file will be stored.\n\n# Windows PowerShell\n$folder_path=your_folder_path # $folder_path=\"C:\\Users\\user\\mybook\\\"\n$book_name=your_book_name # $book_name=\"animal_farm.epub\"\n$openai_key=your_api_key # $openai_key=\"sk-xxx\"\n$language=your_language # see utils.py\n\ndocker run --rm --name bilingual_book_maker --mount type=bind,source=$folder_path,target='/app/test_books' bilingual_book_maker --book_name \"/app/test_books/$book_name\" --openai_key $openai_key --language $language\n\n# Linux\nexport folder_path=${your_folder_path}\nexport book_name=${your_book_name}\nexport openai_key=${your_api_key}\nexport language=${your_language}\n\ndocker run --rm --name bilingual_book_maker --mount type=bind,source=${folder_path},target='/app/test_books' bilingual_book_maker --book_name \"/app/test_books/${book_name}\" --openai_key ${openai_key} --language \"${language}\"\n```\n\nFor example:\n\n```shell\n# Linux\ndocker run --rm --name bilingual_book_maker --mount type=bind,source=/home/user/my_books,target='/app/test_books' bilingual_book_maker --book_name /app/test_books/animal_farm.epub --openai_key sk-XXX --test --test_num 1 --language zh-hant\n```\n\n## Notes\n\n1. API token from free trial has limit. If you want to speed up the process, consider paying for the service or use multiple OpenAI tokens\n2. PR is welcome\n\n# Thanks\n\n- @[yetone](https://github.com/yetone)\n\n# Contribution\n\n- Any issues or PRs are welcome.\n- TODOs in the issue can also be selected.\n- Please run `black make_book.py`[^black] before submitting the code.\n\n# Others better\n\n- 书译 BookTranslator -> [Book Translator](https://www.booktranslator.app)\n\n## Appreciation\n\nThank you, that's enough.\n\n\n\n[^token]: https://platform.openai.com/account/api-keys\n[^black]: https://github.com/psf/black\n"
},
{
"path": "book_maker/__init__.py",
"content": ""
},
{
"path": "book_maker/__main__.py",
"content": "from cli import main\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "book_maker/cli.py",
"content": "import argparse\nimport json\nimport os\nfrom os import environ as env\n\nfrom book_maker.loader import BOOK_LOADER_DICT\nfrom book_maker.translator import MODEL_DICT\nfrom book_maker.utils import LANGUAGES, TO_LANGUAGE_CODE\n\n\ndef parse_prompt_arg(prompt_arg):\n prompt = None\n if prompt_arg is None:\n return prompt\n\n # Check if it's a path to a markdown file (PromptDown format)\n if prompt_arg.endswith(\".md\") and os.path.exists(prompt_arg):\n try:\n from promptdown import StructuredPrompt\n\n structured_prompt = StructuredPrompt.from_promptdown_file(prompt_arg)\n\n # Initialize our prompt structure\n prompt = {}\n\n # Handle developer_message or system_message\n # Developer message takes precedence if both are present\n if (\n hasattr(structured_prompt, \"developer_message\")\n and structured_prompt.developer_message\n ):\n prompt[\"system\"] = structured_prompt.developer_message\n elif (\n hasattr(structured_prompt, \"system_message\")\n and structured_prompt.system_message\n ):\n prompt[\"system\"] = structured_prompt.system_message\n\n # Extract user message from conversation\n if (\n hasattr(structured_prompt, \"conversation\")\n and structured_prompt.conversation\n ):\n for message in structured_prompt.conversation:\n if message.role.lower() == \"user\":\n prompt[\"user\"] = message.content\n break\n\n # Ensure we found a user message\n if \"user\" not in prompt or not prompt[\"user\"]:\n raise ValueError(\n \"PromptDown file must contain at least one user message\"\n )\n\n print(f\"Successfully loaded PromptDown file: {prompt_arg}\")\n\n # Validate required placeholders\n if any(c not in prompt[\"user\"] for c in [\"{text}\"]):\n raise ValueError(\n \"User message in PromptDown must contain `{text}` placeholder\"\n )\n\n return prompt\n except Exception as e:\n print(f\"Error parsing PromptDown file: {e}\")\n # Fall through to other parsing methods\n\n # Existing parsing logic for JSON strings and other formats\n if not any(prompt_arg.endswith(ext) for ext in [\".json\", \".txt\", \".md\"]):\n try:\n # user can define prompt by passing a json string\n # eg: --prompt '{\"system\": \"You are a professional translator who translates computer technology books\", \"user\": \"Translate \\`{text}\\` to {language}\"}'\n prompt = json.loads(prompt_arg)\n except json.JSONDecodeError:\n # if not a json string, treat it as a template string\n prompt = {\"user\": prompt_arg}\n\n elif os.path.exists(prompt_arg):\n if prompt_arg.endswith(\".txt\"):\n # if it's a txt file, treat it as a template string\n with open(prompt_arg, encoding=\"utf-8\") as f:\n prompt = {\"user\": f.read()}\n elif prompt_arg.endswith(\".json\"):\n # if it's a json file, treat it as a json object\n # eg: --prompt prompt_template_sample.json\n with open(prompt_arg, encoding=\"utf-8\") as f:\n prompt = json.load(f)\n else:\n raise FileNotFoundError(f\"{prompt_arg} not found\")\n\n # if prompt is None or any(c not in prompt[\"user\"] for c in [\"{text}\", \"{language}\"]):\n if prompt is None or any(c not in prompt[\"user\"] for c in [\"{text}\"]):\n raise ValueError(\"prompt must contain `{text}`\")\n\n if \"user\" not in prompt:\n raise ValueError(\"prompt must contain the key of `user`\")\n\n if (prompt.keys() - {\"user\", \"system\"}) != set():\n raise ValueError(\"prompt can only contain the keys of `user` and `system`\")\n\n print(\"prompt config:\", prompt)\n return prompt\n\n\ndef main():\n translate_model_list = list(MODEL_DICT.keys())\n parser = argparse.ArgumentParser()\n parser.add_argument(\n \"--book_name\",\n dest=\"book_name\",\n type=str,\n help=\"path of the epub file to be translated\",\n )\n parser.add_argument(\n \"--book_from\",\n dest=\"book_from\",\n type=str,\n choices=[\"kobo\"], # support kindle later\n metavar=\"E-READER\",\n help=\"e-reader type, available: {%(choices)s}\",\n )\n parser.add_argument(\n \"--device_path\",\n dest=\"device_path\",\n type=str,\n help=\"Path of e-reader device\",\n )\n ########## KEYS ##########\n parser.add_argument(\n \"--openai_key\",\n dest=\"openai_key\",\n type=str,\n default=\"\",\n help=\"OpenAI api key,if you have more than one key, please use comma\"\n \" to split them to go beyond the rate limits\",\n )\n parser.add_argument(\n \"--caiyun_key\",\n dest=\"caiyun_key\",\n type=str,\n help=\"you can apply caiyun key from here (https://dashboard.caiyunapp.com/user/sign_in/)\",\n )\n parser.add_argument(\n \"--deepl_key\",\n dest=\"deepl_key\",\n type=str,\n help=\"you can apply deepl key from here (https://rapidapi.com/splintPRO/api/dpl-translator\",\n )\n parser.add_argument(\n \"--claude_key\",\n dest=\"claude_key\",\n type=str,\n help=\"you can find claude key from here (https://console.anthropic.com/account/keys)\",\n )\n\n parser.add_argument(\n \"--custom_api\",\n dest=\"custom_api\",\n type=str,\n help=\"you should build your own translation api\",\n )\n\n # for Google Gemini\n parser.add_argument(\n \"--gemini_key\",\n dest=\"gemini_key\",\n type=str,\n help=\"You can get Gemini Key from https://makersuite.google.com/app/apikey\",\n )\n\n # for Groq\n parser.add_argument(\n \"--groq_key\",\n dest=\"groq_key\",\n type=str,\n help=\"You can get Groq Key from https://console.groq.com/keys\",\n )\n\n # for xAI\n parser.add_argument(\n \"--xai_key\",\n dest=\"xai_key\",\n type=str,\n help=\"You can get xAI Key from https://console.x.ai/\",\n )\n\n # for Qwen\n parser.add_argument(\n \"--qwen_key\",\n dest=\"qwen_key\",\n type=str,\n help=\"You can get Qwen Key from https://bailian.console.aliyun.com/?tab=model#/api-key\",\n )\n\n parser.add_argument(\n \"--test\",\n dest=\"test\",\n action=\"store_true\",\n help=\"only the first 10 paragraphs will be translated, for testing\",\n )\n parser.add_argument(\n \"--test_num\",\n dest=\"test_num\",\n type=int,\n default=10,\n help=\"how many paragraphs will be translated for testing\",\n )\n parser.add_argument(\n \"-m\",\n \"--model\",\n dest=\"model\",\n type=str,\n default=\"chatgptapi\",\n choices=translate_model_list, # support DeepL later\n metavar=\"MODEL\",\n help=\"model to use, available: {%(choices)s}\",\n )\n parser.add_argument(\n \"--ollama_model\",\n dest=\"ollama_model\",\n type=str,\n default=\"\",\n metavar=\"MODEL\",\n help=\"use ollama\",\n )\n parser.add_argument(\n \"--language\",\n type=str,\n choices=sorted(LANGUAGES.keys())\n + sorted([k.title() for k in TO_LANGUAGE_CODE]),\n default=\"zh-hans\",\n metavar=\"LANGUAGE\",\n help=\"language to translate to, available: {%(choices)s}\",\n )\n parser.add_argument(\n \"--resume\",\n dest=\"resume\",\n action=\"store_true\",\n help=\"if program stop unexpected you can use this to resume\",\n )\n parser.add_argument(\n \"-p\",\n \"--proxy\",\n dest=\"proxy\",\n type=str,\n default=\"\",\n help=\"use proxy like http://127.0.0.1:7890\",\n )\n parser.add_argument(\n \"--deployment_id\",\n dest=\"deployment_id\",\n type=str,\n help=\"the deployment name you chose when you deployed the model\",\n )\n # args to change api_base\n parser.add_argument(\n \"--api_base\",\n metavar=\"API_BASE_URL\",\n dest=\"api_base\",\n type=str,\n help=\"specify base url other than the OpenAI's official API address\",\n )\n parser.add_argument(\n \"--exclude_filelist\",\n dest=\"exclude_filelist\",\n type=str,\n default=\"\",\n help=\"if you have more than one file to exclude, please use comma to split them, example: --exclude_filelist 'nav.xhtml,cover.xhtml'\",\n )\n parser.add_argument(\n \"--only_filelist\",\n dest=\"only_filelist\",\n type=str,\n default=\"\",\n help=\"if you only have a few files with translations, please use comma to split them, example: --only_filelist 'nav.xhtml,cover.xhtml'\",\n )\n parser.add_argument(\n \"--translate-tags\",\n dest=\"translate_tags\",\n type=str,\n default=\"p\",\n help=\"example --translate-tags p,blockquote\",\n )\n parser.add_argument(\n \"--exclude_translate-tags\",\n dest=\"exclude_translate_tags\",\n type=str,\n default=\"sup\",\n help=\"example --exclude_translate-tags table,sup\",\n )\n parser.add_argument(\n \"--allow_navigable_strings\",\n dest=\"allow_navigable_strings\",\n action=\"store_true\",\n default=False,\n help=\"allow NavigableStrings to be translated\",\n )\n parser.add_argument(\n \"--prompt\",\n dest=\"prompt_arg\",\n type=str,\n metavar=\"PROMPT_ARG\",\n help=\"used for customizing the prompt. It can be the prompt template string, or a path to the template file. The valid placeholders are `{text}` and `{language}`.\",\n )\n parser.add_argument(\n \"--accumulated_num\",\n dest=\"accumulated_num\",\n type=int,\n default=1,\n help=\"\"\"Wait for how many tokens have been accumulated before starting the translation.\ngpt3.5 limits the total_token to 4090.\nFor example, if you use --accumulated_num 1600, maybe openai will output 2200 tokens\nand maybe 200 tokens for other messages in the system messages user messages, 1600+2200+200=4000,\nSo you are close to reaching the limit. You have to choose your own value, there is no way to know if the limit is reached before sending\n\"\"\",\n )\n parser.add_argument(\n \"--translation_style\",\n dest=\"translation_style\",\n type=str,\n help=\"\"\"ex: --translation_style \"color: #808080; font-style: italic;\" \"\"\",\n )\n parser.add_argument(\n \"--batch_size\",\n dest=\"batch_size\",\n type=int,\n help=\"how many lines will be translated by aggregated translation(This options currently only applies to txt files)\",\n )\n parser.add_argument(\n \"--retranslate\",\n dest=\"retranslate\",\n nargs=4,\n type=str,\n help=\"\"\"--retranslate \"$translated_filepath\" \"file_name_in_epub\" \"start_str\" \"end_str\"(optional)\n Retranslate from start_str to end_str's tag:\n python3 \"make_book.py\" --book_name \"test_books/animal_farm.epub\" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' 'This kind of thing is not a good symptom. Obviously'\n Retranslate start_str's tag:\n python3 \"make_book.py\" --book_name \"test_books/animal_farm.epub\" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which'\n\"\"\",\n )\n parser.add_argument(\n \"--single_translate\",\n action=\"store_true\",\n help=\"output translated book, no bilingual\",\n )\n parser.add_argument(\n \"--use_context\",\n dest=\"context_flag\",\n action=\"store_true\",\n help=\"adds an additional paragraph for global, updating historical context of the story to the model's input, improving the narrative consistency for the AI model (this uses ~200 more tokens each time)\",\n )\n parser.add_argument(\n \"--context_paragraph_limit\",\n dest=\"context_paragraph_limit\",\n type=int,\n default=0,\n help=\"if use --use_context, set context paragraph limit\",\n )\n parser.add_argument(\n \"--temperature\",\n type=float,\n default=1.0,\n help=\"temperature parameter for `chatgptapi`/`gpt4`/`gpt4omini`/`gpt4o`/`gpt5mini`/`claude`/`gemini`\",\n )\n parser.add_argument(\n \"--source_lang\",\n type=str,\n default=\"auto\",\n help=\"source language for translation models like `qwen` (default: auto-detect)\",\n )\n parser.add_argument(\n \"--block_size\",\n type=int,\n default=-1,\n help=\"merge multiple paragraphs into one block, may increase accuracy and speed up the process, but disturb the original format, must be used with `--single_translate`\",\n )\n parser.add_argument(\n \"--model_list\",\n type=str,\n dest=\"model_list\",\n help=\"Rather than using our preset lists of models, specify exactly the models you want as a comma separated list `gpt-4-32k,gpt-3.5-turbo-0125` (Currently only supports: `openai`)\",\n )\n parser.add_argument(\n \"--batch\",\n dest=\"batch_flag\",\n action=\"store_true\",\n help=\"Enable batch translation using ChatGPT's batch API for improved efficiency\",\n )\n parser.add_argument(\n \"--batch-use\",\n dest=\"batch_use_flag\",\n action=\"store_true\",\n help=\"Use pre-generated batch translations to create files. Run with --batch first before using this option\",\n )\n parser.add_argument(\n \"--interval\",\n type=float,\n default=0.01,\n help=\"Request interval in seconds (e.g., 0.1 for 100ms). Currently only supported for Gemini models. Default: 0.01\",\n )\n parser.add_argument(\n \"--parallel-workers\",\n dest=\"parallel_workers\",\n type=int,\n default=1,\n help=\"Number of parallel workers for EPUB chapter processing. Use 2-4 for better performance. Default: 1\",\n )\n\n options = parser.parse_args()\n\n if not options.book_name:\n print(\"Error: please provide the path of your book using --book_name \")\n exit(1)\n if not os.path.isfile(options.book_name):\n print(f\"Error: the book {options.book_name!r} does not exist.\")\n exit(1)\n\n PROXY = options.proxy\n if PROXY != \"\":\n os.environ[\"http_proxy\"] = PROXY\n os.environ[\"https_proxy\"] = PROXY\n\n translate_model = MODEL_DICT.get(options.model)\n assert translate_model is not None, \"unsupported model\"\n API_KEY = \"\"\n if options.model in [\n \"openai\",\n \"chatgptapi\",\n \"gpt4\",\n \"gpt4omini\",\n \"gpt4o\",\n \"gpt5mini\",\n \"o1preview\",\n \"o1\",\n \"o1mini\",\n \"o3mini\",\n ]:\n if OPENAI_API_KEY := (\n options.openai_key\n or env.get(\n \"OPENAI_API_KEY\",\n ) # XXX: for backward compatibility, deprecate soon\n or env.get(\n \"BBM_OPENAI_API_KEY\",\n ) # suggest adding `BBM_` prefix for all the bilingual_book_maker ENVs.\n ):\n API_KEY = OPENAI_API_KEY\n # patch\n elif options.ollama_model:\n # any string is ok, can't be empty\n API_KEY = \"ollama\"\n else:\n raise Exception(\n \"OpenAI API key not provided, please google how to obtain it\",\n )\n elif options.model == \"caiyun\":\n API_KEY = options.caiyun_key or env.get(\"BBM_CAIYUN_API_KEY\")\n if not API_KEY:\n raise Exception(\"Please provide caiyun key\")\n elif options.model == \"deepl\":\n API_KEY = options.deepl_key or env.get(\"BBM_DEEPL_API_KEY\")\n if not API_KEY:\n raise Exception(\"Please provide deepl key\")\n elif options.model.startswith(\"claude\"):\n API_KEY = options.claude_key or env.get(\"BBM_CLAUDE_API_KEY\")\n if not API_KEY:\n raise Exception(\"Please provide claude key\")\n elif options.model == \"customapi\":\n API_KEY = options.custom_api or env.get(\"BBM_CUSTOM_API\")\n if not API_KEY:\n raise Exception(\"Please provide custom translate api\")\n elif options.model in [\"gemini\", \"geminipro\"]:\n API_KEY = options.gemini_key or env.get(\"BBM_GOOGLE_GEMINI_KEY\")\n elif options.model == \"groq\":\n API_KEY = options.groq_key or env.get(\"BBM_GROQ_API_KEY\")\n elif options.model == \"xai\":\n API_KEY = options.xai_key or env.get(\"BBM_XAI_API_KEY\")\n elif options.model.startswith(\"qwen-\"):\n API_KEY = options.qwen_key or env.get(\"BBM_QWEN_API_KEY\")\n else:\n API_KEY = \"\"\n\n if options.book_from == \"kobo\":\n from book_maker import obok\n\n device_path = options.device_path\n if device_path is None:\n raise Exception(\n \"Device path is not given, please specify the path by --device_path \",\n )\n options.book_name = obok.cli_main(device_path)\n\n book_type = options.book_name.split(\".\")[-1]\n support_type_list = list(BOOK_LOADER_DICT.keys())\n if book_type not in support_type_list:\n raise Exception(\n f\"now only support files of these formats: {','.join(support_type_list)}\",\n )\n\n if options.block_size > 0 and not options.single_translate:\n raise Exception(\n \"block_size must be used with `--single_translate` because it disturbs the original format\",\n )\n\n book_loader = BOOK_LOADER_DICT.get(book_type)\n assert book_loader is not None, \"unsupported loader\"\n language = options.language\n if options.language in LANGUAGES:\n # use the value for prompt\n language = LANGUAGES.get(language, language)\n\n # change api_base for issue #42\n model_api_base = options.api_base\n\n if options.ollama_model and not model_api_base:\n # ollama default api_base\n model_api_base = \"http://localhost:11434/v1\"\n\n e = book_loader(\n options.book_name,\n translate_model,\n API_KEY,\n options.resume,\n language=language,\n model_api_base=model_api_base,\n is_test=options.test,\n test_num=options.test_num,\n prompt_config=parse_prompt_arg(options.prompt_arg),\n single_translate=options.single_translate,\n context_flag=options.context_flag,\n context_paragraph_limit=options.context_paragraph_limit,\n temperature=options.temperature,\n source_lang=options.source_lang,\n parallel_workers=options.parallel_workers,\n )\n # other options\n if options.allow_navigable_strings:\n e.allow_navigable_strings = True\n if options.translate_tags:\n e.translate_tags = options.translate_tags\n if options.exclude_translate_tags:\n e.exclude_translate_tags = options.exclude_translate_tags\n if options.exclude_filelist:\n e.exclude_filelist = options.exclude_filelist\n if options.only_filelist:\n e.only_filelist = options.only_filelist\n if options.accumulated_num > 1:\n e.accumulated_num = options.accumulated_num\n if options.translation_style:\n e.translation_style = options.translation_style\n if options.batch_size:\n e.batch_size = options.batch_size\n if options.retranslate:\n e.retranslate = options.retranslate\n if options.deployment_id:\n # only work for ChatGPT api for now\n # later maybe support others\n assert options.model in [\n \"chatgptapi\",\n \"gpt4\",\n \"gpt4omini\",\n \"gpt4o\",\n \"gpt5mini\",\n \"o1\",\n \"o1preview\",\n \"o1mini\",\n \"o3mini\",\n ], \"only support chatgptapi for deployment_id\"\n if not options.api_base:\n raise ValueError(\"`api_base` must be provided when using `deployment_id`\")\n e.translate_model.set_deployment_id(options.deployment_id)\n if options.model in (\"openai\", \"groq\"):\n # Currently only supports `openai` when you also have --model_list set\n if options.model_list:\n e.translate_model.set_model_list(options.model_list.split(\",\"))\n else:\n raise ValueError(\n \"When using `openai` model, you must also provide `--model_list`. For default model sets use `--model chatgptapi` or `--model gpt4` or `--model gpt4omini` or `--model gpt5mini`\",\n )\n # TODO refactor, quick fix for gpt4 model\n if options.model == \"chatgptapi\":\n if options.ollama_model:\n e.translate_model.set_gpt35_models(ollama_model=options.ollama_model)\n else:\n e.translate_model.set_gpt35_models()\n if options.model == \"gpt4\":\n e.translate_model.set_gpt4_models()\n if options.model == \"gpt4omini\":\n e.translate_model.set_gpt4omini_models()\n if options.model == \"gpt4o\":\n e.translate_model.set_gpt4o_models()\n if options.model == \"gpt5mini\":\n e.translate_model.set_gpt5mini_models()\n if options.model == \"o1preview\":\n e.translate_model.set_o1preview_models()\n if options.model == \"o1\":\n e.translate_model.set_o1_models()\n if options.model == \"o1mini\":\n e.translate_model.set_o1mini_models()\n if options.model == \"o3mini\":\n e.translate_model.set_o3mini_models()\n if options.model.startswith(\"claude-\"):\n e.translate_model.set_claude_model(options.model)\n if options.model.startswith(\"qwen-\"):\n e.translate_model.set_qwen_model(options.model)\n if options.block_size > 0:\n e.block_size = options.block_size\n if options.batch_flag:\n e.batch_flag = options.batch_flag\n if options.batch_use_flag:\n e.batch_use_flag = options.batch_use_flag\n\n if options.model in (\"gemini\", \"geminipro\"):\n e.translate_model.set_interval(options.interval)\n if options.model == \"gemini\":\n if options.model_list:\n e.translate_model.set_model_list(options.model_list.split(\",\"))\n else:\n e.translate_model.set_geminiflash_models()\n if options.model == \"geminipro\":\n e.translate_model.set_geminipro_models()\n\n e.make_bilingual_book()\n\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "book_maker/config.py",
"content": "config = {\n \"translator\": {\n \"chatgptapi\": {\n \"context_paragraph_limit\": 3,\n \"batch_context_update_interval\": 50,\n }\n },\n}\n"
},
{
"path": "book_maker/loader/__init__.py",
"content": "from book_maker.loader.epub_loader import EPUBBookLoader\nfrom book_maker.loader.txt_loader import TXTBookLoader\nfrom book_maker.loader.srt_loader import SRTBookLoader\nfrom book_maker.loader.md_loader import MarkdownBookLoader\nfrom book_maker.loader.pdf_loader import PDFBookLoader\n\nBOOK_LOADER_DICT = {\n \"epub\": EPUBBookLoader,\n \"txt\": TXTBookLoader,\n \"srt\": SRTBookLoader,\n \"md\": MarkdownBookLoader,\n \"pdf\": PDFBookLoader,\n # TODO add more here\n}\n"
},
{
"path": "book_maker/loader/base_loader.py",
"content": "from abc import ABC, abstractmethod\n\n\nclass BaseBookLoader(ABC):\n @staticmethod\n def _is_special_text(text):\n return text.isdigit() or text.isspace()\n\n @abstractmethod\n def _make_new_book(self, book):\n pass\n\n @abstractmethod\n def make_bilingual_book(self):\n pass\n\n @abstractmethod\n def load_state(self):\n pass\n\n @abstractmethod\n def _save_temp_book(self):\n pass\n\n @abstractmethod\n def _save_progress(self):\n pass\n"
},
{
"path": "book_maker/loader/epub_loader.py",
"content": "import os\nimport pickle\nimport string\nimport sys\nimport time\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom copy import copy\nfrom pathlib import Path\nimport traceback\nfrom threading import Lock\n\nfrom bs4 import BeautifulSoup as bs\nfrom bs4 import Tag\nfrom bs4.element import NavigableString\nfrom ebooklib import ITEM_DOCUMENT, epub\nfrom rich import print\nfrom tqdm import tqdm\n\nfrom book_maker.utils import num_tokens_from_text, prompt_config_to_kwargs\n\nfrom .base_loader import BaseBookLoader\nfrom .helper import EPUBBookLoaderHelper, is_text_link, not_trans\n\n\nclass EPUBBookLoader(BaseBookLoader):\n def __init__(\n self,\n epub_name,\n model,\n key,\n resume,\n language,\n model_api_base=None,\n is_test=False,\n test_num=5,\n prompt_config=None,\n single_translate=False,\n context_flag=False,\n context_paragraph_limit=0,\n temperature=1.0,\n source_lang=\"auto\",\n parallel_workers=1,\n ):\n self.epub_name = epub_name\n self.new_epub = epub.EpubBook()\n self.translate_model = model(\n key,\n language,\n api_base=model_api_base,\n context_flag=context_flag,\n context_paragraph_limit=context_paragraph_limit,\n temperature=temperature,\n source_lang=source_lang,\n **prompt_config_to_kwargs(prompt_config),\n )\n self.is_test = is_test\n self.test_num = test_num\n self.translate_tags = \"p\"\n self.exclude_translate_tags = \"sup\"\n self.allow_navigable_strings = False\n self.accumulated_num = 1\n self.translation_style = \"\"\n self.context_flag = context_flag\n self.helper = EPUBBookLoaderHelper(\n self.translate_model,\n self.accumulated_num,\n self.translation_style,\n self.context_flag,\n )\n self.retranslate = None\n self.exclude_filelist = \"\"\n self.only_filelist = \"\"\n self.single_translate = single_translate\n self.block_size = -1\n self.batch_use_flag = False\n self.batch_flag = False\n self.parallel_workers = 1\n self.enable_parallel = False\n self._progress_lock = Lock()\n self._translation_index = 0\n self.set_parallel_workers(parallel_workers)\n\n # monkey patch for # 173\n def _write_items_patch(obj):\n for item in obj.book.get_items():\n if isinstance(item, epub.EpubNcx):\n obj.out.writestr(\n \"%s/%s\" % (obj.book.FOLDER_NAME, item.file_name), obj._get_ncx()\n )\n elif isinstance(item, epub.EpubNav):\n obj.out.writestr(\n \"%s/%s\" % (obj.book.FOLDER_NAME, item.file_name),\n obj._get_nav(item),\n )\n elif item.manifest:\n obj.out.writestr(\n \"%s/%s\" % (obj.book.FOLDER_NAME, item.file_name), item.content\n )\n else:\n obj.out.writestr(\"%s\" % item.file_name, item.content)\n\n def _check_deprecated(obj):\n pass\n\n epub.EpubWriter._write_items = _write_items_patch\n epub.EpubReader._check_deprecated = _check_deprecated\n\n try:\n self.origin_book = epub.read_epub(self.epub_name)\n except Exception:\n # tricky monkey patch for #71 if you don't know why please check the issue and ignore this\n # when upstream change will TODO fix this\n def _load_spine(obj):\n spine = obj.container.find(\"{%s}%s\" % (epub.NAMESPACES[\"OPF\"], \"spine\"))\n\n obj.book.spine = [\n (t.get(\"idref\"), t.get(\"linear\", \"yes\")) for t in spine\n ]\n obj.book.set_direction(spine.get(\"page-progression-direction\", None))\n\n epub.EpubReader._load_spine = _load_spine\n self.origin_book = epub.read_epub(self.epub_name)\n\n self.p_to_save = []\n self.resume = resume\n self.bin_path = f\"{Path(epub_name).parent}/.{Path(epub_name).stem}.temp.bin\"\n if self.resume:\n self.load_state()\n\n @staticmethod\n def _is_special_text(text):\n return (\n text.isdigit()\n or text.isspace()\n or is_text_link(text)\n or all(char in string.punctuation for char in text)\n )\n\n def _make_new_book(self, book):\n new_book = epub.EpubBook()\n allowed_ns = set(epub.NAMESPACES.keys()) | set(epub.NAMESPACES.values())\n\n for namespace, metas in book.metadata.items():\n # Only keep namespaces recognized by ebooklib\n if namespace not in allowed_ns:\n continue\n\n if isinstance(metas, dict):\n entries = (\n (name, value, others)\n for name, values in metas.items()\n for value, others in (\n (item if isinstance(item, tuple) else (item, None))\n for item in values\n )\n )\n else:\n entries = metas\n\n for entry in entries:\n if not entry:\n continue\n\n if isinstance(entry, tuple):\n if len(entry) == 3:\n name, value, others = entry\n elif len(entry) == 2:\n name, value = entry\n others = None\n else:\n continue\n else:\n # Unexpected metadata format; skip gracefully\n continue\n\n # `others` can be {} or None\n if others:\n new_book.add_metadata(namespace, name, value, others)\n else:\n new_book.add_metadata(namespace, name, value)\n\n new_book.spine = book.spine\n new_book.toc = self._fix_toc_uids(book.toc)\n return new_book\n\n def _fix_toc_uids(self, toc, counter=None):\n \"\"\"Fix TOC items that have uid=None to prevent TypeError when writing NCX.\"\"\"\n if counter is None:\n counter = [0] # Use list to allow mutation in nested calls\n\n fixed_toc = []\n for item in toc:\n if isinstance(item, tuple):\n # Section with sub-items: (Section, [sub-items])\n section, sub_items = item\n if hasattr(section, \"uid\") and section.uid is None:\n section.uid = f\"navpoint-{counter[0]}\"\n counter[0] += 1\n fixed_sub_items = self._fix_toc_uids(sub_items, counter)\n fixed_toc.append((section, fixed_sub_items))\n elif hasattr(item, \"uid\"):\n # Link or EpubHtml item\n if item.uid is None:\n item.uid = f\"navpoint-{counter[0]}\"\n counter[0] += 1\n fixed_toc.append(item)\n else:\n fixed_toc.append(item)\n\n return fixed_toc\n\n def _extract_paragraph(self, p):\n for p_exclude in self.exclude_translate_tags.split(\",\"):\n # for issue #280\n if type(p) is NavigableString:\n continue\n for pt in p.find_all(p_exclude):\n pt.extract()\n return p\n\n def _process_paragraph(self, p, new_p, index, p_to_save_len, thread_safe=False):\n if self.resume and index < p_to_save_len:\n p.string = self.p_to_save[index]\n new_p.string = self.p_to_save[\n index\n ] # Fix: also update new_p to cached translation\n else:\n t_text = \"\"\n if self.batch_flag:\n self.translate_model.add_to_batch_translate_queue(index, new_p.text)\n elif self.batch_use_flag:\n t_text = self.translate_model.batch_translate(index)\n else:\n t_text = self.translate_model.translate(new_p.text)\n if t_text is None:\n raise RuntimeError(\n \"`t_text` is None: your translation model is not working as expected. Please check your translation model configuration.\"\n )\n if type(p) is NavigableString:\n new_p = t_text\n self.p_to_save.append(new_p)\n else:\n new_p.string = t_text\n self.p_to_save.append(new_p.text)\n\n self.helper.insert_trans(\n p, new_p.string, self.translation_style, self.single_translate\n )\n index += 1\n\n if thread_safe:\n with self._progress_lock:\n if index % 20 == 0:\n self._save_progress()\n else:\n if index % 20 == 0:\n self._save_progress()\n return index\n\n def _process_combined_paragraph(\n self, p_block, index, p_to_save_len, thread_safe=False\n ):\n text = []\n\n for p in p_block:\n if self.resume and index < p_to_save_len:\n p.string = self.p_to_save[index]\n else:\n p_text = p.text.rstrip()\n text.append(p_text)\n\n if self.is_test and index >= self.test_num:\n break\n\n index += 1\n\n if len(text) > 0:\n translated_text = self.translate_model.translate(\"\\n\".join(text))\n translated_text = translated_text.split(\"\\n\")\n text_len = len(translated_text)\n\n for i in range(text_len):\n t = translated_text[i]\n\n if i >= len(p_block):\n p = p_block[-1]\n else:\n p = p_block[i]\n\n if type(p) is NavigableString:\n p = t\n else:\n p.string = t\n\n self.helper.insert_trans(\n p, p.string, self.translation_style, self.single_translate\n )\n\n if thread_safe:\n with self._progress_lock:\n self._save_progress()\n else:\n self._save_progress()\n return index\n\n def translate_paragraphs_acc(self, p_list, send_num):\n count = 0\n wait_p_list = []\n for i in range(len(p_list)):\n p = p_list[i]\n print(f\"translating {i}/{len(p_list)}\")\n temp_p = copy(p)\n\n for p_exclude in self.exclude_translate_tags.split(\",\"):\n # for issue #280\n if type(p) is NavigableString:\n continue\n for pt in temp_p.find_all(p_exclude):\n pt.extract()\n\n if any(\n [not p.text, self._is_special_text(temp_p.text), not_trans(temp_p.text)]\n ):\n if i == len(p_list) - 1:\n self.helper.deal_old(wait_p_list, self.single_translate)\n continue\n length = num_tokens_from_text(temp_p.text)\n if length > send_num:\n self.helper.deal_new(p, wait_p_list, self.single_translate)\n continue\n if i == len(p_list) - 1:\n if count + length < send_num:\n wait_p_list.append(p)\n self.helper.deal_old(wait_p_list, self.single_translate)\n else:\n self.helper.deal_new(p, wait_p_list, self.single_translate)\n break\n if count + length < send_num:\n count += length\n wait_p_list.append(p)\n else:\n self.helper.deal_old(wait_p_list, self.single_translate)\n wait_p_list.append(p)\n count = length\n\n def get_item(self, book, name):\n for item in book.get_items():\n if item.file_name == name:\n return item\n\n def find_items_containing_string(self, book, search_string):\n matching_items = []\n\n for item in book.get_items_of_type(ITEM_DOCUMENT):\n content = item.get_content()\n soup = bs(content, \"html.parser\")\n if search_string in soup.get_text():\n matching_items.append(item)\n\n return matching_items\n\n def retranslate_book(self, index, p_to_save_len, pbar, trans_taglist, retranslate):\n complete_book_name = retranslate[0]\n fixname = retranslate[1]\n fixstart = retranslate[2]\n fixend = retranslate[3]\n\n if fixend == \"\":\n fixend = fixstart\n\n name_fix = complete_book_name\n\n complete_book = epub.read_epub(complete_book_name)\n\n if fixname == \"\":\n fixname = self.find_items_containing_string(complete_book, fixstart)[\n 0\n ].file_name\n print(f\"auto find fixname: {fixname}\")\n\n new_book = self._make_new_book(complete_book)\n\n complete_item = self.get_item(complete_book, fixname)\n if complete_item is None:\n return\n\n ori_item = self.get_item(self.origin_book, fixname)\n if ori_item is None:\n return\n\n content_complete = complete_item.content\n content_ori = ori_item.content\n soup_complete = bs(content_complete, \"html.parser\")\n soup_ori = bs(content_ori, \"html.parser\")\n\n p_list_complete = soup_complete.findAll(trans_taglist)\n p_list_ori = soup_ori.findAll(trans_taglist)\n\n target = None\n tagl = []\n\n # extract from range\n find_end = False\n find_start = False\n for tag in p_list_complete:\n if find_end:\n tagl.append(tag)\n break\n\n if fixend in tag.text:\n find_end = True\n if fixstart in tag.text:\n find_start = True\n\n if find_start:\n if not target:\n target = tag.previous_sibling\n tagl.append(tag)\n\n for t in tagl:\n t.extract()\n\n flag = False\n extract_p_list_ori = []\n for p in p_list_ori:\n if fixstart in p.text:\n flag = True\n if flag:\n extract_p_list_ori.append(p)\n if fixend in p.text:\n break\n\n for t in extract_p_list_ori:\n if target:\n target.insert_after(t)\n target = t\n\n for item in complete_book.get_items():\n if item.file_name != fixname:\n new_book.add_item(item)\n if soup_complete:\n complete_item.content = soup_complete.encode()\n\n index = self.process_item(\n complete_item,\n index,\n p_to_save_len,\n pbar,\n new_book,\n trans_taglist,\n fixstart,\n fixend,\n )\n epub.write_epub(f\"{name_fix}\", new_book, {})\n\n def has_nest_child(self, element, trans_taglist):\n if isinstance(element, Tag):\n for child in element.children:\n if child.name in trans_taglist:\n return True\n if self.has_nest_child(child, trans_taglist):\n return True\n return False\n\n def filter_nest_list(self, p_list, trans_taglist):\n filtered_list = [p for p in p_list if not self.has_nest_child(p, trans_taglist)]\n return filtered_list\n\n def process_item(\n self,\n item,\n index,\n p_to_save_len,\n pbar,\n new_book,\n trans_taglist,\n fixstart=None,\n fixend=None,\n ):\n if self.only_filelist != \"\" and item.file_name not in self.only_filelist.split(\n \",\"\n ):\n return index\n elif self.only_filelist == \"\" and item.file_name in self.exclude_filelist.split(\n \",\"\n ):\n new_book.add_item(item)\n return index\n\n if not os.path.exists(\"log\"):\n os.makedirs(\"log\")\n\n content = item.content\n soup = bs(content, \"html.parser\")\n p_list = soup.findAll(trans_taglist)\n\n p_list = self.filter_nest_list(p_list, trans_taglist)\n\n if self.retranslate:\n new_p_list = []\n\n if fixstart is None or fixend is None:\n return\n\n start_append = False\n for p in p_list:\n text = p.get_text()\n if fixstart in text or fixend in text or start_append:\n start_append = True\n new_p_list.append(p)\n if fixend in text:\n p_list = new_p_list\n break\n\n if self.allow_navigable_strings:\n p_list.extend(soup.findAll(text=True))\n\n send_num = self.accumulated_num\n if send_num > 1:\n with open(\"log/buglog.txt\", \"a\") as f:\n print(f\"------------- {item.file_name} -------------\", file=f)\n\n print(\"------------------------------------------------------\")\n print(f\"dealing {item.file_name} ...\")\n self.translate_paragraphs_acc(p_list, send_num)\n else:\n is_test_done = self.is_test and index > self.test_num\n p_block = []\n block_len = 0\n for p in p_list:\n if is_test_done:\n break\n if not p.text or self._is_special_text(p.text):\n pbar.update(1)\n continue\n\n new_p = self._extract_paragraph(copy(p))\n if self.single_translate and self.block_size > 0:\n p_len = num_tokens_from_text(new_p.text)\n block_len += p_len\n if block_len > self.block_size:\n index = self._process_combined_paragraph(\n p_block, index, p_to_save_len, thread_safe=False\n )\n p_block = [p]\n block_len = p_len\n print()\n else:\n p_block.append(p)\n else:\n index = self._process_paragraph(\n p, new_p, index, p_to_save_len, thread_safe=False\n )\n print()\n\n # pbar.update(delta) not pbar.update(index)?\n pbar.update(1)\n\n if self.is_test and index >= self.test_num:\n break\n if self.single_translate and self.block_size > 0 and len(p_block) > 0:\n index = self._process_combined_paragraph(\n p_block, index, p_to_save_len, thread_safe=False\n )\n\n if soup:\n item.content = soup.encode(encoding=\"utf-8\")\n new_book.add_item(item)\n\n return index\n\n def set_parallel_workers(self, workers):\n \"\"\"Set number of parallel workers for chapter processing.\n\n Args:\n workers (int): Number of parallel workers. Will be automatically\n optimized based on actual chapter count during processing.\n \"\"\"\n self.parallel_workers = max(1, workers)\n self.enable_parallel = workers > 1\n\n if workers > 8:\n print(\n f\"⚠️ Warning: {workers} workers is quite high. Consider using 2-8 workers for optimal performance.\"\n )\n\n def _get_next_translation_index(self):\n \"\"\"Thread-safe method to get next translation index.\"\"\"\n with self._progress_lock:\n index = self._translation_index\n self._translation_index += 1\n return index\n\n def _process_chapter_parallel(self, chapter_data):\n \"\"\"Process a single chapter in parallel mode with proper accumulated_num handling.\"\"\"\n item, trans_taglist, p_to_save_len = chapter_data\n chapter_result = {\n \"item\": item,\n \"processed_content\": None,\n \"success\": False,\n \"error\": None,\n }\n\n try:\n # Create a chapter-specific translator instance to avoid context conflicts\n # This ensures each chapter has its own independent context\n thread_translator = self._create_chapter_translator()\n\n content = item.content\n soup = bs(content, \"html.parser\")\n p_list = soup.findAll(trans_taglist)\n p_list = self.filter_nest_list(p_list, trans_taglist)\n\n if self.allow_navigable_strings:\n p_list.extend(soup.findAll(text=True))\n\n # Initialize chapter-specific context lists\n chapter_context_list = []\n chapter_translated_list = []\n\n # Apply accumulated_num logic for this chapter independently\n send_num = self.accumulated_num\n if send_num > 1:\n # Use accumulated translation logic for this chapter\n self._translate_paragraphs_acc_parallel(\n p_list,\n send_num,\n thread_translator,\n chapter_context_list,\n chapter_translated_list,\n )\n else:\n # Process paragraphs individually for this chapter\n for p in p_list:\n if not p.text or self._is_special_text(p.text):\n continue\n\n new_p = self._extract_paragraph(copy(p))\n index = self._get_next_translation_index()\n\n if self.resume and index < p_to_save_len:\n t_text = self.p_to_save[index]\n else:\n # Use chapter-specific context for translation\n t_text = self._translate_with_chapter_context(\n thread_translator,\n new_p.text,\n chapter_context_list,\n chapter_translated_list,\n )\n t_text = \"\" if t_text is None else t_text\n with self._progress_lock:\n self.p_to_save.append(t_text)\n\n if isinstance(p, NavigableString):\n translated_node = NavigableString(t_text)\n p.insert_after(translated_node)\n if self.single_translate:\n p.extract()\n else:\n self.helper.insert_trans(\n p, t_text, self.translation_style, self.single_translate\n )\n\n with self._progress_lock:\n if index % 20 == 0:\n self._save_progress()\n\n if soup:\n chapter_result[\"processed_content\"] = soup.encode(encoding=\"utf-8\")\n chapter_result[\"success\"] = True\n\n except Exception as e:\n chapter_result[\"error\"] = str(e)\n print(f\"Error processing chapter {item.file_name}: {e}\")\n\n return chapter_result\n\n def _create_chapter_translator(self):\n \"\"\"Create a translator instance for a specific chapter with independent context.\"\"\"\n # Return the main translator - we'll handle context at the chapter level\n return self.translate_model\n\n def _translate_with_chapter_context(\n self, translator, text, chapter_context_list, chapter_translated_list\n ):\n \"\"\"Translate text with chapter-specific context management.\"\"\"\n if not translator.context_flag:\n return translator.translate(text)\n\n # Temporarily replace global context with chapter context\n original_context = getattr(translator, \"context_list\", [])\n original_translated = getattr(translator, \"context_translated_list\", [])\n\n try:\n # Use chapter-specific context\n translator.context_list = chapter_context_list.copy()\n translator.context_translated_list = chapter_translated_list.copy()\n\n # Perform translation\n result = translator.translate(text)\n\n # Update chapter context\n chapter_context_list[:] = translator.context_list\n chapter_translated_list[:] = translator.context_translated_list\n\n return result\n\n finally:\n # Restore original context\n translator.context_list = original_context\n translator.context_translated_list = original_translated\n\n def _translate_paragraphs_acc_parallel(\n self,\n p_list,\n send_num,\n translator,\n chapter_context_list,\n chapter_translated_list,\n ):\n \"\"\"Apply accumulated_num logic for a single chapter in parallel mode with independent context.\"\"\"\n from book_maker.utils import num_tokens_from_text\n from .helper import not_trans\n\n count = 0\n wait_p_list = []\n\n # Create chapter-specific helper instance with context-aware translation\n class ChapterHelper:\n def __init__(\n self, parent_loader, translator, context_list, translated_list\n ):\n self.parent_loader = parent_loader\n self.translator = translator\n self.context_list = context_list\n self.translated_list = translated_list\n\n def translate_with_context(self, text):\n return self.parent_loader._translate_with_chapter_context(\n self.translator, text, self.context_list, self.translated_list\n )\n\n def deal_old(self, wait_p_list, single_translate):\n if not wait_p_list:\n return\n\n # Use the same translate_list logic as sequential processing\n # Create a temporary translator with chapter context\n original_context = getattr(self.translator, \"context_list\", [])\n original_translated = getattr(\n self.translator, \"context_translated_list\", []\n )\n\n try:\n # Set chapter context to the translator\n self.translator.context_list = self.context_list.copy()\n self.translator.context_translated_list = (\n self.translated_list.copy()\n )\n\n # Call translate_list for consistent batch translation logic\n result_txt_list = self.translator.translate_list(wait_p_list)\n\n # Update chapter context from translator\n self.context_list[:] = self.translator.context_list\n self.translated_list[:] = self.translator.context_translated_list\n\n # Apply translations using the same logic as helper.deal_old\n for i in range(len(wait_p_list)):\n if i < len(result_txt_list):\n p = wait_p_list[i]\n from .helper import shorter_result_link\n\n self.parent_loader.helper.insert_trans(\n p,\n shorter_result_link(result_txt_list[i]),\n self.parent_loader.translation_style,\n single_translate,\n )\n\n finally:\n # Restore original context\n self.translator.context_list = original_context\n self.translator.context_translated_list = original_translated\n\n wait_p_list.clear()\n\n def deal_new(self, p, wait_p_list, single_translate):\n self.deal_old(wait_p_list, single_translate)\n translation = self.translate_with_context(p.text)\n self.parent_loader.helper.insert_trans(\n p,\n translation,\n self.parent_loader.translation_style,\n single_translate,\n )\n\n chapter_helper = ChapterHelper(\n self, translator, chapter_context_list, chapter_translated_list\n )\n\n for i in range(len(p_list)):\n p = p_list[i]\n temp_p = copy(p)\n\n for p_exclude in self.exclude_translate_tags.split(\",\"):\n if type(p) == NavigableString:\n continue\n for pt in temp_p.find_all(p_exclude):\n pt.extract()\n\n if any(\n [not p.text, self._is_special_text(temp_p.text), not_trans(temp_p.text)]\n ):\n if i == len(p_list) - 1:\n chapter_helper.deal_old(wait_p_list, self.single_translate)\n continue\n\n length = num_tokens_from_text(temp_p.text)\n if length > send_num:\n chapter_helper.deal_new(p, wait_p_list, self.single_translate)\n continue\n\n if i == len(p_list) - 1:\n if count + length < send_num:\n wait_p_list.append(p)\n chapter_helper.deal_old(wait_p_list, self.single_translate)\n else:\n chapter_helper.deal_new(p, wait_p_list, self.single_translate)\n break\n\n if count + length < send_num:\n count += length\n wait_p_list.append(p)\n else:\n chapter_helper.deal_old(wait_p_list, self.single_translate)\n wait_p_list.append(p)\n count = length\n\n def batch_init_then_wait(self):\n name, _ = os.path.splitext(self.epub_name)\n if self.batch_flag or self.batch_use_flag:\n self.translate_model.batch_init(name)\n if self.batch_use_flag:\n start_time = time.time()\n while not self.translate_model.is_completed_batch():\n print(\"Batch translation is not completed yet\")\n time.sleep(2)\n if time.time() - start_time > 300: # 5 minutes\n raise Exception(\"Batch translation timed out after 5 minutes\")\n\n def make_bilingual_book(self):\n self.helper = EPUBBookLoaderHelper(\n self.translate_model,\n self.accumulated_num,\n self.translation_style,\n self.context_flag,\n )\n self.batch_init_then_wait()\n new_book = self._make_new_book(self.origin_book)\n all_items = list(self.origin_book.get_items())\n trans_taglist = self.translate_tags.split(\",\")\n all_p_length = sum(\n (\n 0\n if (\n (i.get_type() != ITEM_DOCUMENT)\n or (i.file_name in self.exclude_filelist.split(\",\"))\n or (\n self.only_filelist\n and i.file_name not in self.only_filelist.split(\",\")\n )\n )\n else len(bs(i.content, \"html.parser\").findAll(trans_taglist))\n )\n for i in all_items\n )\n all_p_length += self.allow_navigable_strings * sum(\n (\n 0\n if (\n (i.get_type() != ITEM_DOCUMENT)\n or (i.file_name in self.exclude_filelist.split(\",\"))\n or (\n self.only_filelist\n and i.file_name not in self.only_filelist.split(\",\")\n )\n )\n else len(bs(i.content, \"html.parser\").findAll(text=True))\n )\n for i in all_items\n )\n pbar = tqdm(total=self.test_num) if self.is_test else tqdm(total=all_p_length)\n print()\n index = 0\n p_to_save_len = len(self.p_to_save)\n try:\n if self.retranslate:\n self.retranslate_book(\n index, p_to_save_len, pbar, trans_taglist, self.retranslate\n )\n exit(0)\n # Add the things that don't need to be translated first, so that you can see the img after the interruption\n for item in self.origin_book.get_items():\n if item.get_type() != ITEM_DOCUMENT:\n new_book.add_item(item)\n\n document_items = list(self.origin_book.get_items_of_type(ITEM_DOCUMENT))\n\n if self.enable_parallel and len(document_items) > 1:\n # Optimize worker count: no point having more workers than chapters\n effective_workers = min(self.parallel_workers, len(document_items))\n\n # Parallel processing with proper accumulated_num handling\n print(f\"🚀 Parallel processing: {len(document_items)} chapters\")\n if effective_workers < self.parallel_workers:\n print(\n f\"📊 Optimized workers: {effective_workers} (reduced from {self.parallel_workers})\"\n )\n else:\n print(f\"📊 Using {effective_workers} workers\")\n\n if self.accumulated_num > 1:\n print(\n f\"📝 Each chapter applies accumulated_num={self.accumulated_num} independently\"\n )\n\n if self.context_flag:\n print(\n f\"🔗 Context enabled: each chapter maintains independent context (limit={self.translate_model.context_paragraph_limit})\"\n )\n else:\n print(f\"🚫 Context disabled for this translation\")\n\n # Create a simpler progress bar for parallel processing\n pbar.close() # Close the original progress bar\n chapter_pbar = tqdm(\n total=len(document_items), desc=\"Chapters\", unit=\"ch\"\n )\n\n chapter_data_list = [\n (item, trans_taglist, p_to_save_len) for item in document_items\n ]\n\n with ThreadPoolExecutor(max_workers=effective_workers) as executor:\n future_to_item = {\n executor.submit(\n self._process_chapter_parallel, chapter_data\n ): chapter_data[0]\n for chapter_data in chapter_data_list\n }\n\n for future in as_completed(future_to_item):\n item = future_to_item[future]\n try:\n result = future.result()\n if result[\"success\"] and result[\"processed_content\"]:\n item.content = result[\"processed_content\"]\n new_book.add_item(item)\n chapter_pbar.update(1)\n chapter_pbar.set_postfix_str(\n f\"Latest: {item.file_name[:20]}...\"\n )\n\n except Exception as e:\n print(f\"❌ Error processing {item.file_name}: {e}\")\n new_book.add_item(item)\n chapter_pbar.update(1)\n\n chapter_pbar.close()\n print(f\"✅ Completed all {len(document_items)} chapters\")\n else:\n # Sequential processing (original behavior or single chapter)\n if len(document_items) == 1 and self.enable_parallel:\n print(f\"📄 Single chapter detected - using sequential processing\")\n\n for item in document_items:\n index = self.process_item(\n item, index, p_to_save_len, pbar, new_book, trans_taglist\n )\n\n if self.accumulated_num > 1:\n name, _ = os.path.splitext(self.epub_name)\n epub.write_epub(f\"{name}_bilingual.epub\", new_book, {})\n name, _ = os.path.splitext(self.epub_name)\n if self.batch_flag:\n self.translate_model.batch()\n else:\n epub.write_epub(f\"{name}_bilingual.epub\", new_book, {})\n if self.accumulated_num == 1:\n pbar.close()\n except KeyboardInterrupt as e:\n print(e)\n if self.accumulated_num == 1:\n print(\"you can resume it next time\")\n self._save_progress()\n self._save_temp_book()\n sys.exit(0)\n except Exception:\n traceback.print_exc()\n sys.exit(0)\n\n def load_state(self):\n try:\n with open(self.bin_path, \"rb\") as f:\n self.p_to_save = pickle.load(f)\n except Exception:\n raise Exception(\"can not load resume file\")\n\n def _save_temp_book(self):\n # TODO refactor this logic\n origin_book_temp = epub.read_epub(self.epub_name)\n new_temp_book = self._make_new_book(origin_book_temp)\n p_to_save_len = len(self.p_to_save)\n trans_taglist = self.translate_tags.split(\",\")\n index = 0\n try:\n for item in origin_book_temp.get_items():\n if item.get_type() == ITEM_DOCUMENT:\n content = item.content\n soup = bs(content, \"html.parser\")\n p_list = soup.findAll(trans_taglist)\n if self.allow_navigable_strings:\n p_list.extend(soup.findAll(text=True))\n for p in p_list:\n if not p.text or self._is_special_text(p.text):\n continue\n # TODO banch of p to translate then combine\n # PR welcome here\n if index < p_to_save_len:\n new_p = copy(p)\n if type(p) is NavigableString:\n new_p = self.p_to_save[index]\n else:\n new_p.string = self.p_to_save[index]\n self.helper.insert_trans(\n p,\n new_p.string,\n self.translation_style,\n self.single_translate,\n )\n index += 1\n else:\n break\n # for save temp book\n if soup:\n item.content = soup.encode()\n new_temp_book.add_item(item)\n name, _ = os.path.splitext(self.epub_name)\n epub.write_epub(f\"{name}_bilingual_temp.epub\", new_temp_book, {})\n except Exception as e:\n # TODO handle it\n print(e)\n\n def _save_progress(self):\n try:\n with open(self.bin_path, \"wb\") as f:\n pickle.dump(self.p_to_save, f)\n except Exception:\n raise Exception(\"can not save resume file\")\n"
},
{
"path": "book_maker/loader/helper.py",
"content": "import re\nimport backoff\nimport logging\nfrom copy import copy\n\nlogging.basicConfig(level=logging.WARNING)\nlogger = logging.getLogger(__name__)\n\n\nclass EPUBBookLoaderHelper:\n def __init__(\n self, translate_model, accumulated_num, translation_style, context_flag\n ):\n self.translate_model = translate_model\n self.accumulated_num = accumulated_num\n self.translation_style = translation_style\n self.context_flag = context_flag\n\n def insert_trans(self, p, text, translation_style=\"\", single_translate=False):\n if text is None:\n text = \"\"\n if (\n p.string is not None\n and p.string.replace(\" \", \"\").strip() == text.replace(\" \", \"\").strip()\n ):\n return\n new_p = copy(p)\n new_p.string = text\n if translation_style != \"\":\n new_p[\"style\"] = translation_style\n p.insert_after(new_p)\n if single_translate:\n p.extract()\n\n @backoff.on_exception(\n backoff.expo,\n Exception,\n on_backoff=lambda details: logger.warning(f\"retry backoff: {details}\"),\n on_giveup=lambda details: logger.warning(f\"retry abort: {details}\"),\n jitter=None,\n )\n def translate_with_backoff(self, text, context_flag=False):\n return self.translate_model.translate(text, context_flag)\n\n def deal_new(self, p, wait_p_list, single_translate=False):\n self.deal_old(wait_p_list, single_translate, self.context_flag)\n self.insert_trans(\n p,\n shorter_result_link(self.translate_with_backoff(p.text, self.context_flag)),\n self.translation_style,\n single_translate,\n )\n\n def deal_old(self, wait_p_list, single_translate=False, context_flag=False):\n if not wait_p_list:\n return\n\n result_txt_list = self.translate_model.translate_list(wait_p_list)\n\n for i in range(len(wait_p_list)):\n if i < len(result_txt_list):\n p = wait_p_list[i]\n self.insert_trans(\n p,\n shorter_result_link(result_txt_list[i]),\n self.translation_style,\n single_translate,\n )\n\n wait_p_list.clear()\n\n\nurl_pattern = r\"(http[s]?://|www\\.)+(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+\"\n\n\ndef is_text_link(text):\n return bool(re.compile(url_pattern).match(text.strip()))\n\n\ndef is_text_tail_link(text, num=80):\n text = text.strip()\n pattern = r\".*\" + url_pattern + r\"$\"\n return bool(re.compile(pattern).match(text)) and len(text) < num\n\n\ndef shorter_result_link(text, num=20):\n match = re.search(url_pattern, text)\n\n if not match or len(match.group()) < num:\n return text\n\n return re.compile(url_pattern).sub(\"...\", text)\n\n\ndef is_text_source(text):\n return text.strip().startswith(\"Source: \")\n\n\ndef is_text_list(text, num=80):\n text = text.strip()\n return re.match(r\"^Listing\\s*\\d+\", text) and len(text) < num\n\n\ndef is_text_figure(text, num=80):\n text = text.strip()\n return re.match(r\"^Figure\\s*\\d+\", text) and len(text) < num\n\n\ndef is_text_digit_and_space(s):\n for c in s:\n if not c.isdigit() and not c.isspace():\n return False\n return True\n\n\ndef is_text_isbn(s):\n pattern = r\"^[Ee]?ISBN\\s*\\d[\\d\\s]*$\"\n return bool(re.match(pattern, s))\n\n\ndef not_trans(s):\n return any(\n [\n is_text_link(s),\n is_text_tail_link(s),\n is_text_source(s),\n is_text_list(s),\n is_text_figure(s),\n is_text_digit_and_space(s),\n is_text_isbn(s),\n ]\n )\n"
},
{
"path": "book_maker/loader/md_loader.py",
"content": "import sys\nfrom pathlib import Path\n\nfrom book_maker.utils import prompt_config_to_kwargs\n\nfrom .base_loader import BaseBookLoader\n\n\nclass MarkdownBookLoader(BaseBookLoader):\n def __init__(\n self,\n md_name,\n model,\n key,\n resume,\n language,\n model_api_base=None,\n is_test=False,\n test_num=5,\n prompt_config=None,\n single_translate=False,\n context_flag=False,\n context_paragraph_limit=0,\n temperature=1.0,\n source_lang=\"auto\",\n ) -> None:\n self.md_name = md_name\n self.translate_model = model(\n key,\n language,\n api_base=model_api_base,\n temperature=temperature,\n source_lang=source_lang,\n **prompt_config_to_kwargs(prompt_config),\n )\n self.is_test = is_test\n self.p_to_save = []\n self.bilingual_result = []\n self.bilingual_temp_result = []\n self.test_num = test_num\n self.batch_size = 10\n self.single_translate = single_translate\n self.md_paragraphs = []\n\n try:\n with open(f\"{md_name}\", encoding=\"utf-8\") as f:\n self.origin_book = f.read().splitlines()\n\n except Exception as e:\n raise Exception(\"can not load file\") from e\n\n self.resume = resume\n self.bin_path = f\"{Path(md_name).parent}/.{Path(md_name).stem}.temp.bin\"\n if self.resume:\n self.load_state()\n\n self.process_markdown_content()\n\n def process_markdown_content(self):\n \"\"\"将原始内容处理成 markdown 段落\"\"\"\n current_paragraph = []\n for line in self.origin_book:\n # 如果是空行且当前段落不为空，保存当前段落\n if not line.strip() and current_paragraph:\n self.md_paragraphs.append(\"\\n\".join(current_paragraph))\n current_paragraph = []\n # 如果是标题行，单独作为一个段落\n elif line.strip().startswith(\"#\"):\n if current_paragraph:\n self.md_paragraphs.append(\"\\n\".join(current_paragraph))\n current_paragraph = []\n self.md_paragraphs.append(line)\n # 其他情况，添加到当前段落\n else:\n current_paragraph.append(line)\n\n # 处理最后一个段落\n if current_paragraph:\n self.md_paragraphs.append(\"\\n\".join(current_paragraph))\n\n @staticmethod\n def _is_special_text(text):\n return text.isdigit() or text.isspace() or len(text) == 0\n\n def _make_new_book(self, book):\n pass\n\n def make_bilingual_book(self):\n index = 0\n p_to_save_len = len(self.p_to_save)\n\n try:\n sliced_list = [\n self.md_paragraphs[i : i + self.batch_size]\n for i in range(0, len(self.md_paragraphs), self.batch_size)\n ]\n for paragraphs in sliced_list:\n batch_text = \"\\n\\n\".join(paragraphs)\n if self._is_special_text(batch_text):\n continue\n if not self.resume or index >= p_to_save_len:\n try:\n max_retries = 3\n retry_count = 0\n while retry_count < max_retries:\n try:\n temp = self.translate_model.translate(batch_text)\n break\n except AttributeError as ae:\n print(f\"翻译出错: {ae}\")\n retry_count += 1\n if retry_count == max_retries:\n raise Exception(\"翻译模型初始化失败\") from ae\n except Exception as e:\n print(f\"翻译过程中出错: {e}\")\n raise Exception(\"翻译过程中出现错误\") from e\n\n self.p_to_save.append(temp)\n if not self.single_translate:\n self.bilingual_result.append(batch_text)\n self.bilingual_result.append(temp)\n index += self.batch_size\n if self.is_test and index > self.test_num:\n break\n\n self.save_file(\n f\"{Path(self.md_name).parent}/{Path(self.md_name).stem}_bilingual.md\",\n self.bilingual_result,\n )\n\n except (KeyboardInterrupt, Exception) as e:\n print(f\"发生错误: {e}\")\n print(\"程序将保存进度，您可以稍后继续\")\n self._save_progress()\n self._save_temp_book()\n sys.exit(1) # 使用非零退出码表示错误\n\n def _save_temp_book(self):\n index = 0\n sliced_list = [\n self.origin_book[i : i + self.batch_size]\n for i in range(0, len(self.origin_book), self.batch_size)\n ]\n\n for i in range(len(sliced_list)):\n batch_text = \"\".join(sliced_list[i])\n self.bilingual_temp_result.append(batch_text)\n if self._is_special_text(self.origin_book[i]):\n continue\n if index < len(self.p_to_save):\n self.bilingual_temp_result.append(self.p_to_save[index])\n index += 1\n\n self.save_file(\n f\"{Path(self.md_name).parent}/{Path(self.md_name).stem}_bilingual_temp.txt\",\n self.bilingual_temp_result,\n )\n\n def _save_progress(self):\n try:\n with open(self.bin_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\"\\n\".join(self.p_to_save))\n except Exception as e:\n raise Exception(\"can not save resume file\") from e\n\n def load_state(self):\n try:\n with open(self.bin_path, encoding=\"utf-8\") as f:\n self.p_to_save = f.read().splitlines()\n except Exception as e:\n raise Exception(\"can not load resume file\") from e\n\n def save_file(self, book_path, content):\n try:\n with open(book_path, \"w\", encoding=\"utf-8\") as f:\n f.write(\"\\n\".join(content))\n except Exception as e:\n raise Exception(\"can not save file\") from e\n"
},
{
"path": "book_maker/loader/pdf_loader.py",
"content": "import sys\nfrom pathlib import Path\n\nfrom book_maker.utils import prompt_config_to_kwargs\n\nfrom .base_loader import BaseBookLoader\n\nimport fitz\n\nfrom ebooklib import epub\n\n\nclass PDFBookLoader(BaseBookLoader):\n def __init__(\n self,\n pdf_name,\n model,\n key,\n resume,\n language,\n model_api_base=None,\n is_test=False,\n test_num=5,\n prompt_config=None,\n single_translate=False,\n context_flag=False,\n context_paragraph_limit=0,\n temperature=1.0,\n source_lang=\"auto\",\n parallel_workers=1,\n ) -> None:\n if fitz is None:\n raise Exception(\"PyMuPDF (fitz) is required to use PDF loader\")\n\n self.pdf_name = pdf_name\n self.translate_model = model(\n key,\n language,\n api_base=model_api_base,\n temperature=temperature,\n source_lang=source_lang,\n **prompt_config_to_kwargs(prompt_config),\n )\n self.is_test = is_test\n self.p_to_save = []\n self.bilingual_result = []\n self.bilingual_temp_result = []\n self.test_num = test_num\n self.batch_size = 10\n self.single_translate = single_translate\n self.parallel_workers = max(1, parallel_workers)\n\n try:\n doc = fitz.open(self.pdf_name)\n lines = []\n for page in doc:\n text = page.get_text(\"text\")\n if not text:\n continue\n lines.extend(text.splitlines())\n self.origin_book = lines\n except Exception as e:\n raise Exception(\"can not load file\") from e\n\n self.resume = resume\n self.bin_path = f\"{Path(pdf_name).parent}/.{Path(pdf_name).stem}.temp.bin\"\n if self.resume:\n self.load_state()\n\n def _make_new_book(self, book):\n pass\n\n def _try_create_epub(self):\n \"\"\"Try to create an EPUB file from translated content.\n\n The EPUB is created from the `self.bilingual_result` list which alternates\n original and translated strings. If EPUB creation fails for any reason,\n this function will log the error and leave the TXT fallback intact.\n \"\"\"\n if epub is None:\n # ebooklib not installed; skip EPUB generation\n return False\n\n if not self.bilingual_result:\n return False\n\n try:\n book = epub.EpubBook()\n title = Path(self.pdf_name).stem\n # Minimal metadata\n try:\n book.set_identifier(title)\n book.set_title(title)\n book.set_language(\n self.translate_model.language\n if hasattr(self.translate_model, \"language\")\n else \"en\"\n )\n except Exception:\n # be tolerant about metadata API differences\n pass\n\n chapters = []\n # build chapters from bilingual_result (pairs)\n for i in range(0, len(self.bilingual_result), 2):\n orig = self.bilingual_result[i]\n trans = (\n self.bilingual_result[i + 1]\n if i + 1 < len(self.bilingual_result)\n else \"\"\n )\n # basic html content: original then translated\n content = \"\"\n if orig:\n content += (\n '
`. Use `--translate-tags` to specify tags need for translation. Use comma to separate multiple tags. For example: `--translate-tags h1,h2,h3,p,div`\n\n bbook_maker --book_name test_books/animal_farm.epub --openai_key ${openai_key} --translate-tags div,p\n\nIf you want to translate strings in an e-book that aren't labeled with any tags, you can use the `--allow_navigable_strings` parameter. This will add the strings to the translation queue. \n**Note that it's best to look for e-books that are more standardized if possible.**\n\n## e-reader\nUse `--book_from` option to specify e-reader type (Now only `kobo` is available), and use `--device_path` to specify the mounting point.\n\n # Translate books download from Rakuten Kobo on kobo e-reader\n bbook_maker --book_from kobo --device_path /tmp/kobo\n"
},
{
"path": "docs/cmd.md",
"content": "# Command Line Options\n\n## Test translate\n`--test` \n\nUse this option to preview the result if you haven't paid for the service or just want to test. Note that there is a limit and it may take some time.\n\n```sh\nbbook_maker --book_name test_books/Lex_Fridman_episode_322.srt --openai_key ${openai_key} --test\n```\n\n```sh\nbbook_maker --book_name test_books/animal_farm.epub --openai_key ${openai_key} --test --language zh-hans\n```\n\n`--test_num ` \n\nUse this option to set how many paragraph you want to translate for testing. Default is 10.\n\n## Resume\n`--resume` \n\nUse this option to manually resume the process after an interruption.\n\n## Retranslate (epub only)\n`--retranslate ` \n\nIf a file in epub is not translated well, it supports to re-translate part of epub separately.\n\nThis option take 4 arguments: `translated_filepath`, `file_name_in_epub`, `start_str`, `end_str`. `end_str` is optional.\n\n- Retranslate from start_str to end_str's tag:\n\n bbook_maker --book_name \"test_books/animal_farm.epub\" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which' 'This kind of thing is not a good symptom. Obviously'\n\n- Retranslate start_str's tag:\n \n bbook_maker --book_name \"test_books/animal_farm.epub\" --retranslate 'test_books/animal_farm_bilingual.epub' 'index_split_002.html' 'in spite of the present book shortage which'\n\n- Retranslate start_str's tag, auto find filename:\n \n bbook_maker --book_name \"test_books/animal_farm.epub\" --retranslate 'test_books/animal_farm_bilingual.epub' '' 'in spite of the present book shortage which'\n\n**Warning:**\n\n**It deletes from the tag at start_str of the finished book to the next tag at end_str, and then re-translates.**\n\n**Therefore, please make sure that the next tag of end_str is the translated content. (If end_str is not provided, the next label of start_str is guaranteed to be the translated content.) There can be missing translations between the two strings, but if end_str is not translated, there will be problems.**\n\n\n\n\n## Customize output style (epub only)\n`--translation_style ` \n\nSupport changing the output style of epub files.\n\n bbook_maker --book_name test_books/animal_farm.epub --translation_style \"color: #4a4a4a; font-style: normal; background-color: #f7f7f7; padding: 5px; margin: 10px 0; border-radius: 5px;\"\n\n\n## Proxy\n`--proxy ` \n\nUse this option to specify proxy server for internet access. Enter a string such as `http://127.0.0.1:7890` .\n\n## API base\n`--api_base ` \n\nIf you want to change api_base like using Cloudflare Workers, use this option to support it. \n\n bbook_maker --book_name 'animal_farm.epub' --openai_key sk-XXXXX --api_base 'https://xxxxx/v1'\n**Note: the api url should be '`https://xxxx/v1`'. Quotation marks are required.**\n\n## Microsoft Azure Endpoints\n`--api_base ` `--deployment_id ` \n\nYou can use the api endpoint provided from Microsoft.\n\n\n bbook_maker --book_name 'animal_farm.epub' --openai_key XXXXX --api_base 'https://example-endpoint.openai.azure.com' --deployment_id 'deployment-name'\n\n**Note : Current only support chatgptapi model for deployment_id. And `api_base` must be provided when using `deployment_id`. You can check [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource?pivots=web-portal) for more information about `deployment_id`.**\n\n## Batch size (txt only)\n`--batch_size` \n\nUse this parameter to specify the number of lines for batch translation. Default is 10. (Currently only effective for txt files).\n```sh\npython3 make_book.py --book_name test_books/the_little_prince.txt --test --batch_size 20\n```\n\n## Accumulated Num\n`--accumulated_num ` \n\nWait for how many tokens have been accumulated before starting the translation. gpt3.5 limits the total_token to 4090. \n\nFor example, if you use --accumulated_num 1600, maybe openai will\noutput 2200 tokens and maybe 200 tokens for other messages in the system messages user messages. 1600+2200+200=4000, so you are close to the limit. \n\nYou have to choose your own\nvalue, there is no way to tell if the limit is reached before sending request.\n"
},
{
"path": "docs/disclaimer.md",
"content": "Disclaimer:\n\n1. The purpose of this project, bilingual_book_maker, is to assist users in creating multilingual versions of epub files and books. It is only applicable to books that have entered the public domain and is not intended for use with copyrighted material. We strongly advise users to read the copyright information carefully before using this project and to comply with relevant laws and regulations in order to protect their own and others' rights.\n2. In no event shall the authors or developers be liable for any loss or damage caused by the use of this project. Users assume all risks associated with the use of this project. Users must confirm that they have obtained permission from the original copyright holder or used open source EPUB files before using this project to avoid potential copyright risks.\n\nIf you have any concerns or suggestions about the use of this project, please contact us through the issues section.\n\n\n免责声明：\n\n1. 该项目设计目的是为了帮助用户制作多语言版本的epub文件和图书，仅适用于进入公共版权领域书籍，不适用于有版权的书籍。我们强烈建议用户在使用该项目时仔细阅读其版权信息并遵守相关法律和规定，以保护自己和他人的权益。\n2. 在任何情况下，作者和开发者不对因使用该项目而导致的任何损失或损害承担任何责任。使用该项目的风险由用户自行承担。用户必须在使用该项目之前，确认其已获得了原著作权人的许可或使用了公开可用的开源EPUB文件，以避免可能存在的版权风险。\n\n如果您对该项目的使用有任何疑虑或建议，请通过 issues 与我们联系。\n"
},
{
"path": "docs/env_settings.md",
"content": "# Environment Settings\nYou can also write information into env to skip some options.\n\n## Model keys\n```\n# Set env BBM_OPENAI_API_KEY to ignore option --openai_key\nexport BBM_OPENAI_API_KEY=${your_api_key}\n\n# Set env BBM_CAIYUN_API_KEY to ignore option --caiyun_key\nexport BBM_CAIYUN_API_KEY=${your_api_key}\n```"
},
{
"path": "docs/index.md",
"content": "# bilingual book maker\n\nThe `bilingual_book_maker` is an AI translation tool that uses ChatGPT to assist users in creating multi-language versions of epub/txt files and books.\n\nThis tool is exclusively designed for translating epub books that have entered the public domain and is not intended for copyrighted works. Before using this tool, please review the project's **[disclaimer](disclaimer.md)**."
},
{
"path": "docs/installation.md",
"content": "# Installation\n## pip\nbilingual_book_maker has been published as a [Python package](https://pypi.org/project/bbook-maker/) and can be install by `pip`. (Recommend in a virtual environment.)\n```sh\npip install -U bbook_maker\n```\n\n## git\nYou can also install from github if you want to use the latest version.\n```sh\ngit clone git@github.com:yihong0618/bilingual_book_maker.git\npip install .\n```"
},
{
"path": "docs/model_lang.md",
"content": "# Model and Languages\n## Models\n`-m, --model ` \n\nCurrently `bbook_maker` supports these models: `chatgptapi` , `gpt3` , `google` , `caiyun` , `deepl` , `deeplfree` , `gpt4` , `gpt4omini` , `gpt5mini` , `o1-preview` , `o1` , `o1-mini` , `o3-mini` , `claude` , `customapi`.\nDefault model is `chatgptapi` . \n\n### OPENAI models\n\nThere are several models you can choose from.\n\n* gpt3\n\n \n\n bbook_maker --book_name test_books/animal_farm.epub --model gpt3 --openai_key ${openai_key}\n\n \n\n* chatgpiapi\n\n\n `chatgptapi` is [GPT-3.5-turbo](https://openai.com/blog/introducing-chatgpt-and-whisper-apis), which is used by ChatGPT currently.\n\n bbook_maker --book_name test_books/animal_farm.epub --model chatgptapi --openai_key ${openai_key}\n\n* gpt4\n\n \n\n bbook_maker --book_name test_books/animal_farm.epub --model gpt4 --openai_key ${openai_key}\n\n If using `gpt4` , you can add `--use_context` to add a context paragraph to each passage sent to the model for translation.\n\n \n\n \n bbook_maker --book_name test_books/animal_farm.epub --model gpt4 --openai_key ${openai_key} --use_context\n\n The option `--use_context` prompts the GPT4 model to create a one-paragraph summary. \n\n \n\n If it is the beginning of the translation, it will summarize the entire passage sent (the size depending on `--accumulated_num` ).\n\n \n\n If it has any proceeding passage, it will amend the summary to include details from the most recent passage, creating a running one-paragraph context payload of the important details of the entire translated work, which improves consistency of flow and tone of each translation.\n\n* gpt5mini\n\n `gpt5mini` uses the `gpt-5-mini` model.\n\n bbook_maker --book_name test_books/animal_farm.epub --model gpt5mini --openai_key ${openai_key}\n\n**Note 1: Use `--openai_key` option to specify OpenAI API key. If you have multiple keys, separate them by commas (xxx, xxx, xxx) to reduce errors caused by API call limits.**\n\n**Note 2: You can just set the environment variable `BBM_OPENAI_API_KEY` instead the openai_key. See [Environment setting](settings.md).**\n\n### CAIYUN \n\nUsing Caiyun model to translate. The api currently only support: \n\n \n\n1. Simplified Chinese <-> English\n2. Simplified Chinese <-> Japanese\n\nThe official Caiyun has provided a test token (3975l6lr5pcbvidl6jl2). You can apply your own token by following this [tutorial].(https://bobtranslate.com/service/translate/caiyun.html)\n\n \n bbook_maker --model caiyun --caiyun_key 3975l6lr5pcbvidl6jl2 --book_name test_books/animal_farm.epub\n\n### DEEPL\n\nThere are two models you can choose from.\n\n \n\n* deepl: [DeepL Translator](https://rapidapi.com/splintPRO/api/dpl-translator). \n\n \n\n Need to pay to get the token. Use `--model deepl --deepl_key ${deepl_key}`\n\n \n\n bbook_maker --book_name test_books/animal_farm.epub --model deepl --deepl_key ${deepl_key}\n\n \n\n* deeplfree: DeepL free model\n\n \n\n bbook_maker --book_name test_books/animal_farm.epub --model deeplfree\n\n### Claude\n\nSupport [Claude](https://console.anthropic.com/docs) model. Use `--model claude --claude_key ${claude_key}` .\n\n bbook_maker --book_name test_books/animal_farm.epub --model claude --claude_key ${claude_key}\n \n\n### Custom API\nSupport CustomAPI model. Use `--model customapi --custom_api ${custom_api}` .\n\n bbook_maker --book_name test_books/animal_farm.epub --model customapi --custom_api ${custom_api} \n\n### Google\n\nSupport google model. Use `--model google`\n\n## Languages\n`--language ` \n\nSet target languages. All models except for `caiyun` supports lots of languages. You can use `bbook_maker --help` to check available languages. Default target language is `\"Simplified Chinese\"` .\n\n```sh\nbbook_maker --book_name test_books/animal_farm.epub --model chatgptapi --openai_key ${openai_key} --language ja\n```\n\n```sh\nbbook_maker --book_name test_books/animal_farm.epub --model chatgptapi --openai_key ${openai_key} --language \"Simplified Chinese\"\n```"
},
{
"path": "docs/prompt.md",
"content": "# Tweak the prompt\n\nTo tweak the prompt, use the `--prompt` parameter. Valid placeholders for the `user` role template include `{text}` and `{language}`. It supports a few ways to configure the prompt:\n\n- If you don't need to set the `system` role content, you can simply set it up like this: `--prompt \"Translate {text} to {language}.\"` or `--prompt prompt_template_sample.txt`\n\n # prompt_template_sample.txt\n Translate the given text to {language}. Be faithful or accurate in translation. Make the translation readable or intelligible. Be elegant or natural in translation. If the text cannot be translated, return the original text as is. Do not translate person's name. Do not add any additional text in the translation. The text to be translated is: \n {text}\n \n\n- If you need to set the `system` role content, you can use the following format: `--prompt '{\"user\":\"Translate {text} to {language}\", \"system\": \"You are a professional translator.\"}'` or `--prompt prompt_template_sample.json`\n\n # prompt_template_sample.json\n {\n \"system\": \"You are a professional translator.\", \n \"user\": \"Translate the given text to {language}. Be faithful or accurate in translation. Make the translation readable or intelligible. Be elegant or natural in translation. If the text cannot be translated, return the original text as is. Do not translate person's name. Do not add any additional text in the translation. The text to be translated is:\\n{text}\"\n }\n\nYou can also set the `user` and `system` role prompt by setting environment variables: `BBM_CHATGPTAPI_USER_MSG_TEMPLATE` and `BBM_CHATGPTAPI_SYS_MSG`.\n\n- You can now use PromptDown format (`.md` files) for more structured prompts: `--prompt prompt_md.prompt.md`\n\n # Translation Prompt\n \n ## System Message\n You are a professional translator who specializes in accurate translations.\n \n ## Conversation\n \n | Role | Content |\n |-------|------------------------------------------|\n | User | Please translate the following text into {language}:\\n\\n{text} |\n \n # OR using Developer Message (for newer AI models)\n \n # Translation Prompt\n \n ## Developer Message\n You are a professional translator who specializes in accurate translations.\n \n ## Conversation\n \n | Role | Content |\n |-------|------------------------------------------|\n | User | Please translate the following text into {language}:\\n\\n{text} |\n\n## Examples\n```sh\npython3 make_book.py --book_name test_books/animal_farm.epub --prompt prompt_template_sample.txt\n# or\npython3 make_book.py --book_name test_books/animal_farm.epub --prompt prompt_template_sample.json\n# or\npython3 make_book.py --book_name test_books/animal_farm.epub --prompt \"Please translate \\`{text}\\` to {language}\"\n```\n"
},
{
"path": "docs/quickstart.md",
"content": "# QuickStart\nAfter successfully install the package, you can see `bbook-maker` is in the output of `pip list`.\n\n## Preparation\n1. ChatGPT or OpenAI [token](https://platform.openai.com/account/api-keys)\n2. epub/txt books\n3. Environment with internet access or proxy\n4. Python 3.8+\n\n## Use\nYou can use by command `bbook_maker`. A sample book, `test_books/animal_farm.epub`, is provided for testing purposes.\n```sh\nbbook_maker --book_name ${path of a book} --openai_key ${openai_key}\n\n# Example\nbbook_maker --book_name test_books/animal_farm.epub --openai_key ${openai_key}\n```\nOr, you can use the [script](https://github.com/yihong0618/bilingual_book_maker/blob/main/make_book.py) provided by repository.\n```sh\npython3 make_book.py --book_name ${path of a book} --openai_key ${openai_key}\n\n# Example\npython3 make_book.py --book_name test_books/animal_farm.epub --openai_key ${openai_key}\n```\n\nOnce the translation is complete, a bilingual book named `${book_name}_bilingual.epub` would be generated.\n\n\n**Note: If there are any errors or you wish to interrupt the translation by pressing `CTRL+C`. A book named `${book_name}_bilingual_temp.epub` would be generated. You can simply rename it to any desired name.**\n"
},
{
"path": "make_book.py",
"content": "from book_maker.cli import main\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "mkdocs.yml",
"content": "site_name: bilingual book maker\ntheme:\n name: material\n features:\n - navigation.tabs\n - navigation.tabs.sticky\n - content.code.copy\n\nnav: \n - Home : index.md\n - Getting started: \n - Installation: installation.md\n - QuickStart: quickstart.md\n - Usage:\n - Model and languages: model_lang.md\n - Command line options: cmd.md\n - Translate from different source: book_source.md\n - Environment setting: env_settings.md\n - Tweak the prompt: prompt.md\n - Disclaimer: disclaimer.md\n "
},
{
"path": "prompt_md.json",
"content": "{\n \"system\": \"You are a highly skilled translator responsible for translating the content of books in Markdown format from English into Chinese.\",\n \"user\": \"## Strategies\\nYou will follow a three-step translation process:\\n### 1. Translate the input content from English into Chinese, respect the intention of the original text, keep the original Markdown format unchanged, and do not delete or omit any content, nor add additional explanations or remarks.\\n### 2. Read the original text and the translation carefully, and then put forward constructive criticism and helpful suggestions to improve the translation. The final style and tone of the translation should conform to the Chinese language style.\\nYou must strictly follow the rules below.\\n- Never change the Markdown markup structure. Don't add or remove links. Do not change any URL.\\n- Never touch or change the contents of code blocks even if they appear to have a bug.\\n- Always preserve the original line breaks. Do not add or remove blank lines.\\n- Never touch any permalink at the end of each heading.\\n- Never touch HTML-like tags such as ``.\\nWhen writing suggestions, pay attention to whether there are ways to improve the translation in terms of:\\n- Accuracy (by correcting errors such as additions, mistranslations, omissions or untranslated text).\\n- Fluency (by applying the rules of Chinese grammar, spelling and punctuation, and ensuring there is no unnecessary repetition).\\n- Conciseness and abbreviation (please appropriately simplify and abbreviate the translation result while keeping the original meaning unchanged to avoid the translation being too lengthy).\\n### 3. Based on the results of steps 1 and 2, refine and polish the translation, and do not add additional explanations or remarks.\\n## Output\\nFor each step of the translation process, output the results within the appropriate XML tags:\\n\\n[Insert your initial translation here.]\\n\\n\\n[Insert your reflection on the translation and put forward specific here, useful and constructive suggestions to improve the translation. Each suggestion should target a specific part of the translation.]\\n\\n\\n[Insert your refined and polished translation here.]\\n\\n## Input\\nThe following is the content of the book that needs to be translated within the tag:\\n{text}\"\n}"
},
{
"path": "prompt_md.prompt.md",
"content": "# Translation Prompt\n\n## Developer Message\n\nYou are a professional translator who specializes in accurate, natural-sounding translations that preserve the original meaning, tone, and style of the text.\n\n## Conversation\n\n| Role | Content |\n|-------|---------------------------------------------------------------------------|\n| User | Please translate the following text into {language}:\\n\\n{text} |\n"
},
{
"path": "prompt_template_sample.json",
"content": "{\n \"system\": \"You are a highly skilled academic translator. Please complete the translation task according to the following instructions and provide only the final polished translation.\",\n \"user\": \"## Strategies\\nYou will follow a three-step translation process:\\n### Step.1 Initial Direct Translation: Translate the content from English to Chinese sentence by sentence, respecting the original intent without deleting, omitting, or adding any extra explanations or notes.\\n ### Step.2 Reflection and Revision: Carefully review both the input content and the initial direct translation from Step 1. Check if the translation conveys the original meaning, if the grammatical structure is correct, if word choices are appropriate, and if there are any ambiguities or polysemous words. The final style and tone should conform to Chinese language conventions. \\nYou must strictly follow the rules below.\\n- Don't add or remove links. Do not change any URL.\\n- Do not translate the reference list.\\n- Never touch,change or translate the mathematical formulas.\\n- Never touch,change or translate the contents of code blocks even if they appear to have a bug.\\n- Always preserve the original line breaks. Do not add or remove blank lines.\\nProvide constructive criticism and helpful suggestions to improve: \\n- translation accuracy (correct additions, mistranslations, omissions, or untranslated text errors),\\n- fluency (apply Chinese grammar, spelling, and punctuation rules, and ensure no unnecessary repetition), \\n- conciseness (streamline the translation results while maintaining the original meaning, avoiding wordiness).\\n ### Step.3 Polish and Optimize: Based on the results from Steps 1 and 2, refine and polish the translation, ensuring the final translation adheres to Chinese style without additional explanations or notes. The content to be translated is wrapped in the following tags:\\n\\n{text}. \\n\\nPlease write and output only the final polished translation here: \"\n}\n"
},
{
"path": "pyproject.toml",
"content": "[project]\nname = \"bbook-maker\"\ndescription = \"The bilingual_book_maker is an AI translation tool that uses ChatGPT to assist users in creating multi-language versions of epub/txt files and books.\"\nreadme = \"README.md\"\nlicense = {text = \"MIT\"}\ndynamic = [\"version\"]\nrequires-python = \">=3.10\"\nauthors = [\n { name = \"yihong0618\", email = \"zouzou0208@gmail.com\" },\n]\nclassifiers = [\n \"License :: OSI Approved :: MIT License\",\n \"Operating System :: OS Independent\",\n \"Programming Language :: Python :: 3\",\n]\ndependencies = [\n \"anthropic\",\n \"backoff\",\n \"bs4\",\n \"ebooklib\",\n \"google-generativeai\",\n \"langdetect\",\n \"litellm\",\n \"openai>=1.1.1\",\n \"PyDeepLX\",\n \"requests\",\n \"rich\",\n \"tiktoken\",\n \"tqdm\",\n \"groq>=0.5.0\",\n \"promptdown>=0.9.0\",\n \"PyMuPDF\",\n]\n\n[project.scripts]\nbbook_maker = \"book_maker.cli:main\"\npromptdown = \"promptdown_cli:main\"\n\n[project.urls]\nHomepage = \"https://github.com/yihong0618/bilingual_book_maker\"\n\n[tool.pdm]\nplugins = [\"pdm-autoexport\"]\n[[tool.pdm.autoexport]]\nfilename = \"requirements.txt\"\nwithout-hashes = true\n[build-system]\nrequires = [\"pdm-backend>=2.0.0\"]\nbuild-backend = \"pdm.backend\"\n[tool.pdm.version]\nsource = \"scm\"\n"
},
{
"path": "tests/test_epub_metadata.py",
"content": "import pytest\nfrom ebooklib import epub\n\nfrom book_maker.loader.epub_loader import EPUBBookLoader\n\n\ndef test_epub_loader_handles_custom_metadata(tmp_path):\n source_book = epub.EpubBook()\n source_book.add_metadata(\"DC\", \"title\", \"Metadata Copy Test\", {\"id\": \"title-id\"})\n source_book.add_metadata(\"DC\", \"creator\", \"Tester\", {\"role\": \"aut\"})\n\n # Simulate a namespace that ebooklib does not recognise; the legacy approach\n # copied this verbatim and ebooklib failed while writing the book back.\n source_book.metadata[\"custom\"] = [\n (\"foo-tag\", \"bar-value\", {\"attr\": \"value\"}),\n ]\n\n legacy_book = epub.EpubBook()\n legacy_book.metadata = source_book.metadata\n with pytest.raises(AttributeError):\n epub.write_epub(str(tmp_path / \"legacy.epub\"), legacy_book)\n\n loader = EPUBBookLoader.__new__(EPUBBookLoader)\n rebuilt_book = loader._make_new_book(source_book)\n\n output_path = tmp_path / \"rebuilt.epub\"\n epub.write_epub(str(output_path), rebuilt_book)\n assert output_path.exists()\n\n dc_namespace = epub.NAMESPACES[\"DC\"]\n titles = rebuilt_book.metadata[dc_namespace][\"title\"]\n creators = rebuilt_book.metadata[dc_namespace][\"creator\"]\n\n assert (\"Metadata Copy Test\", {\"id\": \"title-id\"}) in titles\n assert (\"Tester\", {\"role\": \"aut\"}) in creators\n assert \"custom\" not in rebuilt_book.metadata\n"
},
{
"path": "tests/test_integration.py",
"content": "import os\nimport shutil\nimport subprocess\nimport sys\nfrom pathlib import Path\n\nimport pytest\n\n\n@pytest.fixture()\ndef test_book_dir() -> str:\n \"\"\"Return test book dir\"\"\"\n # TODO: Can move this to conftest.py if there will be more unittests\n return str(Path(__file__).parent.parent / \"test_books\")\n\n\ndef test_google_translate_epub(test_book_dir, tmpdir):\n \"\"\"Test google translate epub\"\"\"\n shutil.copyfile(\n os.path.join(test_book_dir, \"Liber_Esther.epub\"),\n os.path.join(tmpdir, \"Liber_Esther.epub\"),\n )\n\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"Liber_Esther.epub\"),\n \"--test\",\n \"--test_num\",\n \"20\",\n \"--model\",\n \"google\",\n ],\n env=os.environ.copy(),\n )\n\n assert os.path.isfile(os.path.join(tmpdir, \"Liber_Esther_bilingual.epub\"))\n assert os.path.getsize(os.path.join(tmpdir, \"Liber_Esther_bilingual.epub\")) != 0\n\n\ndef test_deepl_free_translate_epub(test_book_dir, tmpdir):\n \"\"\"Test deepl free translate epub\"\"\"\n shutil.copyfile(\n os.path.join(test_book_dir, \"Liber_Esther.epub\"),\n os.path.join(tmpdir, \"Liber_Esther.epub\"),\n )\n\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"Liber_Esther.epub\"),\n \"--test\",\n \"--test_num\",\n \"20\",\n \"--model\",\n \"deeplfree\",\n ],\n env=os.environ.copy(),\n )\n\n assert os.path.isfile(os.path.join(tmpdir, \"Liber_Esther_bilingual.epub\"))\n assert os.path.getsize(os.path.join(tmpdir, \"Liber_Esther_bilingual.epub\")) != 0\n\n\ndef test_google_translate_epub_cli():\n pass\n\n\ndef test_google_translate_txt(test_book_dir, tmpdir):\n \"\"\"Test google translate txt\"\"\"\n shutil.copyfile(\n os.path.join(test_book_dir, \"the_little_prince.txt\"),\n os.path.join(tmpdir, \"the_little_prince.txt\"),\n )\n\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"the_little_prince.txt\"),\n \"--test\",\n \"--test_num\",\n \"20\",\n \"--model\",\n \"google\",\n ],\n env=os.environ.copy(),\n )\n assert os.path.isfile(os.path.join(tmpdir, \"the_little_prince_bilingual.txt\"))\n assert os.path.getsize(os.path.join(tmpdir, \"the_little_prince_bilingual.txt\")) != 0\n\n\ndef test_google_translate_txt_batch_size(test_book_dir, tmpdir):\n \"\"\"Test google translate txt with batch_size\"\"\"\n shutil.copyfile(\n os.path.join(test_book_dir, \"the_little_prince.txt\"),\n os.path.join(tmpdir, \"the_little_prince.txt\"),\n )\n\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"the_little_prince.txt\"),\n \"--test\",\n \"--batch_size\",\n \"30\",\n \"--test_num\",\n \"20\",\n \"--model\",\n \"google\",\n ],\n env=os.environ.copy(),\n )\n\n assert os.path.isfile(os.path.join(tmpdir, \"the_little_prince_bilingual.txt\"))\n assert os.path.getsize(os.path.join(tmpdir, \"the_little_prince_bilingual.txt\")) != 0\n\n\n@pytest.mark.skipif(\n not os.environ.get(\"BBM_CAIYUN_API_KEY\"),\n reason=\"No BBM_CAIYUN_API_KEY in environment variable.\",\n)\ndef test_caiyun_translate_txt(test_book_dir, tmpdir):\n \"\"\"Test caiyun translate txt\"\"\"\n shutil.copyfile(\n os.path.join(test_book_dir, \"the_little_prince.txt\"),\n os.path.join(tmpdir, \"the_little_prince.txt\"),\n )\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"the_little_prince.txt\"),\n \"--test\",\n \"--batch_size\",\n \"10\",\n \"--test_num\",\n \"100\",\n \"--model\",\n \"caiyun\",\n ],\n env=os.environ.copy(),\n )\n\n assert os.path.isfile(os.path.join(tmpdir, \"the_little_prince_bilingual.txt\"))\n assert os.path.getsize(os.path.join(tmpdir, \"the_little_prince_bilingual.txt\")) != 0\n\n\n@pytest.mark.skipif(\n not os.environ.get(\"BBM_DEEPL_API_KEY\"),\n reason=\"No BBM_DEEPL_API_KEY in environment variable.\",\n)\ndef test_deepl_translate_txt(test_book_dir, tmpdir):\n shutil.copyfile(\n os.path.join(test_book_dir, \"the_little_prince.txt\"),\n os.path.join(tmpdir, \"the_little_prince.txt\"),\n )\n\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"the_little_prince.txt\"),\n \"--test\",\n \"--batch_size\",\n \"30\",\n \"--test_num\",\n \"20\",\n \"--model\",\n \"deepl\",\n ],\n env=os.environ.copy(),\n )\n\n assert os.path.isfile(os.path.join(tmpdir, \"the_little_prince_bilingual.txt\"))\n assert os.path.getsize(os.path.join(tmpdir, \"the_little_prince_bilingual.txt\")) != 0\n\n\n@pytest.mark.skipif(\n not os.environ.get(\"BBM_DEEPL_API_KEY\"),\n reason=\"No BBM_DEEPL_API_KEY in environment variable.\",\n)\ndef test_deepl_translate_srt(test_book_dir, tmpdir):\n shutil.copyfile(\n os.path.join(test_book_dir, \"Lex_Fridman_episode_322.srt\"),\n os.path.join(tmpdir, \"Lex_Fridman_episode_322.srt\"),\n )\n\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"Lex_Fridman_episode_322.srt\"),\n \"--test\",\n \"--batch_size\",\n \"30\",\n \"--test_num\",\n \"2\",\n \"--model\",\n \"deepl\",\n ],\n env=os.environ.copy(),\n )\n\n assert os.path.isfile(os.path.join(tmpdir, \"Lex_Fridman_episode_322_bilingual.srt\"))\n assert (\n os.path.getsize(os.path.join(tmpdir, \"Lex_Fridman_episode_322_bilingual.srt\"))\n != 0\n )\n\n\n@pytest.mark.skipif(\n not os.environ.get(\"OPENAI_API_KEY\"),\n reason=\"No OPENAI_API_KEY in environment variable.\",\n)\ndef test_openai_translate_epub_zh_hans(test_book_dir, tmpdir):\n shutil.copyfile(\n os.path.join(test_book_dir, \"lemo.epub\"),\n os.path.join(tmpdir, \"lemo.epub\"),\n )\n\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"lemo.epub\"),\n \"--test\",\n \"--test_num\",\n \"5\",\n \"--language\",\n \"zh-hans\",\n ],\n env=os.environ.copy(),\n )\n assert os.path.isfile(os.path.join(tmpdir, \"lemo_bilingual.epub\"))\n assert os.path.getsize(os.path.join(tmpdir, \"lemo_bilingual.epub\")) != 0\n\n\n@pytest.mark.skipif(\n not os.environ.get(\"OPENAI_API_KEY\"),\n reason=\"No OPENAI_API_KEY in environment variable.\",\n)\ndef test_openai_translate_epub_ja_prompt_txt(test_book_dir, tmpdir):\n shutil.copyfile(\n os.path.join(test_book_dir, \"animal_farm.epub\"),\n os.path.join(tmpdir, \"animal_farm.epub\"),\n )\n\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"animal_farm.epub\"),\n \"--test\",\n \"--test_num\",\n \"5\",\n \"--language\",\n \"ja\",\n \"--model\",\n \"gpt3\",\n \"--prompt\",\n \"prompt_template_sample.txt\",\n ],\n env=os.environ.copy(),\n )\n assert os.path.isfile(os.path.join(tmpdir, \"animal_farm_bilingual.epub\"))\n assert os.path.getsize(os.path.join(tmpdir, \"animal_farm_bilingual.epub\")) != 0\n\n\n@pytest.mark.skipif(\n not os.environ.get(\"OPENAI_API_KEY\"),\n reason=\"No OPENAI_API_KEY in environment variable.\",\n)\ndef test_openai_translate_epub_ja_prompt_json(test_book_dir, tmpdir):\n shutil.copyfile(\n os.path.join(test_book_dir, \"animal_farm.epub\"),\n os.path.join(tmpdir, \"animal_farm.epub\"),\n )\n\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"animal_farm.epub\"),\n \"--test\",\n \"--test_num\",\n \"5\",\n \"--language\",\n \"ja\",\n \"--prompt\",\n \"prompt_template_sample.json\",\n ],\n env=os.environ.copy(),\n )\n assert os.path.isfile(os.path.join(tmpdir, \"animal_farm_bilingual.epub\"))\n assert os.path.getsize(os.path.join(tmpdir, \"animal_farm_bilingual.epub\")) != 0\n\n\n@pytest.mark.skipif(\n not os.environ.get(\"OPENAI_API_KEY\"),\n reason=\"No OPENAI_API_KEY in environment variable.\",\n)\ndef test_openai_translate_srt(test_book_dir, tmpdir):\n shutil.copyfile(\n os.path.join(test_book_dir, \"Lex_Fridman_episode_322.srt\"),\n os.path.join(tmpdir, \"Lex_Fridman_episode_322.srt\"),\n )\n\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n os.path.join(tmpdir, \"Lex_Fridman_episode_322.srt\"),\n \"--test\",\n \"--test_num\",\n \"20\",\n ],\n env=os.environ.copy(),\n )\n assert os.path.isfile(os.path.join(tmpdir, \"Lex_Fridman_episode_322_bilingual.srt\"))\n assert (\n os.path.getsize(os.path.join(tmpdir, \"Lex_Fridman_episode_322_bilingual.srt\"))\n != 0\n )\n"
},
{
"path": "tests/test_pdf_cli.py",
"content": "import subprocess\nimport sys\nfrom pathlib import Path\n\nimport pytest\n\nfitz = pytest.importorskip(\"fitz\")\n\n\ndef test_pdf_cli_creates_txt_and_optional_epub(tmp_path):\n pdf_path = tmp_path / \"cli_test.pdf\"\n doc = fitz.open()\n page = doc.new_page()\n page.insert_text((72, 72), \"CLI test\\nPDF content\")\n doc.save(str(pdf_path))\n\n # run CLI\n subprocess.run(\n [\n sys.executable,\n \"make_book.py\",\n \"--book_name\",\n str(pdf_path),\n \"--test\",\n \"--test_num\",\n \"5\",\n \"--model\",\n \"google\",\n ],\n check=True,\n )\n\n txt_out = tmp_path / \"cli_test_bilingual.txt\"\n assert txt_out.exists()\n assert txt_out.stat().st_size > 0\n\n # if ebooklib is installed, an epub should be created\n try:\n import ebooklib\n except Exception:\n ebooklib = None\n\n if ebooklib is not None:\n epub_out = tmp_path / \"cli_test_bilingual.epub\"\n assert epub_out.exists()\n assert epub_out.stat().st_size > 0\n"
},
{
"path": "tests/test_pdf_loader.py",
"content": "import os\nfrom pathlib import Path\n\nimport pytest\n\nfitz = pytest.importorskip(\"fitz\")\n\nfrom book_maker.loader.pdf_loader import PDFBookLoader\n\n\nclass DummyModel:\n def __init__(\n self,\n key,\n language,\n api_base=None,\n temperature=1.0,\n source_lang=\"auto\",\n **kwargs,\n ):\n pass\n\n def translate(self, text):\n return f\"{text}\"\n\n def translate_list(self, texts):\n return [f\"{t}\" for t in texts]\n\n\ndef test_pdf_loader_extracts_and_translates(tmp_path):\n pdf_path = tmp_path / \"test.pdf\"\n doc = fitz.open()\n page = doc.new_page()\n page.insert_text((72, 72), \"Hello world\\nThis is a PDF test\")\n doc.save(str(pdf_path))\n\n loader = PDFBookLoader(\n str(pdf_path),\n DummyModel,\n key=\"\",\n resume=False,\n language=\"en\",\n is_test=True,\n test_num=5,\n )\n\n assert len(loader.origin_book) > 0\n\n loader.make_bilingual_book()\n\n out_file = tmp_path / \"test_bilingual.txt\"\n assert out_file.exists()\n assert out_file.stat().st_size > 0\n # basic content check\n content = out_file.read_text(encoding=\"utf-8\")\n assert \"\" in content\n\n # if ebooklib is installed, an EPUB should also be produced\n try:\n import ebooklib\n except Exception:\n ebooklib = None\n\n if ebooklib is not None:\n epub_file = tmp_path / \"test_bilingual.epub\"\n assert epub_file.exists()\n assert epub_file.stat().st_size > 0\n"
},
{
"path": "typos.toml",
"content": "# See https://github.com/crate-ci/typos/blob/master/docs/reference.md to configure typos\n[default.extend-words]\nsur = \"sur\"\nbanch = \"banch\" # TODO: not sure if this is a typo or not\nfo = \"fo\"\nba = \"ba\"\n[files]\nextend-exclude = [\"LICENSE\"]\n"
}
]